summaryrefslogtreecommitdiffstats
path: root/third_party/rust/rkv
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 19:33:14 +0000
commit36d22d82aa202bb199967e9512281e9a53db42c9 (patch)
tree105e8c98ddea1c1e4784a60a5a6410fa416be2de /third_party/rust/rkv
parentInitial commit. (diff)
downloadfirefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz
firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--third_party/rust/rkv/.cargo-checksum.json1
-rw-r--r--third_party/rust/rkv/CODE_OF_CONDUCT.md15
-rw-r--r--third_party/rust/rkv/Cargo.lock369
-rw-r--r--third_party/rust/rkv/Cargo.toml135
-rw-r--r--third_party/rust/rkv/LICENSE202
-rw-r--r--third_party/rust/rkv/README.md77
-rw-r--r--third_party/rust/rkv/examples/README.md11
-rw-r--r--third_party/rust/rkv/examples/iterator.rs76
-rw-r--r--third_party/rust/rkv/examples/simple-store.rs252
-rwxr-xr-xthird_party/rust/rkv/run-all-examples.sh14
-rw-r--r--third_party/rust/rkv/src/backend.rs40
-rw-r--r--third_party/rust/rkv/src/backend/common.rs44
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb.rs34
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs998
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs79
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs69
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/database.rs16
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/environment.rs299
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/error.rs57
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/flags.rs123
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/info.rs35
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/iter.rs41
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/stat.rs39
-rw-r--r--third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs107
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe.rs30
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/cursor.rs98
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/database.rs41
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/environment.rs332
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/error.rs79
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/flags.rs124
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/info.rs35
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/iter.rs24
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/snapshot.rs141
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/stat.rs39
-rw-r--r--third_party/rust/rkv/src/backend/impl_safe/transaction.rs208
-rw-r--r--third_party/rust/rkv/src/backend/traits.rs202
-rw-r--r--third_party/rust/rkv/src/bin/dump.rs47
-rw-r--r--third_party/rust/rkv/src/bin/rand.rs106
-rw-r--r--third_party/rust/rkv/src/env.rs320
-rw-r--r--third_party/rust/rkv/src/error.rs150
-rw-r--r--third_party/rust/rkv/src/helpers.rs43
-rw-r--r--third_party/rust/rkv/src/lib.rs236
-rw-r--r--third_party/rust/rkv/src/manager.rs247
-rw-r--r--third_party/rust/rkv/src/migrator.rs179
-rw-r--r--third_party/rust/rkv/src/readwrite.rs154
-rw-r--r--third_party/rust/rkv/src/store.rs52
-rw-r--r--third_party/rust/rkv/src/store/integer.rs637
-rw-r--r--third_party/rust/rkv/src/store/integermulti.rs709
-rw-r--r--third_party/rust/rkv/src/store/keys.rs46
-rw-r--r--third_party/rust/rkv/src/store/keys/encodables.rs27
-rw-r--r--third_party/rust/rkv/src/store/keys/primitives.rs15
-rw-r--r--third_party/rust/rkv/src/store/multi.rs133
-rw-r--r--third_party/rust/rkv/src/store/single.rs132
-rw-r--r--third_party/rust/rkv/src/value.rs252
-rw-r--r--third_party/rust/rkv/tests/env-all.rs328
-rw-r--r--third_party/rust/rkv/tests/env-lmdb.rs1647
-rw-r--r--third_party/rust/rkv/tests/env-migration.rs607
-rw-r--r--third_party/rust/rkv/tests/env-safe.rs1370
-rw-r--r--third_party/rust/rkv/tests/integer-store.rs88
-rw-r--r--third_party/rust/rkv/tests/manager.rs380
-rw-r--r--third_party/rust/rkv/tests/multi-integer-store.rs122
-rw-r--r--third_party/rust/rkv/tests/test_txn.rs131
62 files changed, 12644 insertions, 0 deletions
diff --git a/third_party/rust/rkv/.cargo-checksum.json b/third_party/rust/rkv/.cargo-checksum.json
new file mode 100644
index 0000000000..ce8553651a
--- /dev/null
+++ b/third_party/rust/rkv/.cargo-checksum.json
@@ -0,0 +1 @@
+{"files":{"CODE_OF_CONDUCT.md":"902d5357af363426631d907e641e220b3ec89039164743f8442b3f120479b7cf","Cargo.lock":"511877a29c2732398795b0d85f7060414e4e1fadb66d41b75278f2184dfd321a","Cargo.toml":"fd7520f047ef50fddcb4b1fac66357809873f83f7893bb2c79dd9c0bf9e52b75","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"ce68102d518acabb52f18dfb516d58419d9b6cabf68b4f5b3ad490b41ea2a4da","examples/README.md":"143767fc145bf167ce269a65138cb3f7086cb715b8bc4f73626da82966e646f4","examples/iterator.rs":"a8bbee46a4d2bada4b5c0bf8e9e84541900ddcd99c974ea4ab95d51c12d3d670","examples/simple-store.rs":"cec2b0b1303c72f7b2a2d16e3b2bfd292c696388e418a8b280801321bdd17aa0","run-all-examples.sh":"7f9d11d01017f77e1c9d26e3e82dfca8c6930deaec85e864458e33a7fa267de0","src/backend.rs":"f33cf8b3c7627dd562d2713997360217919d6b7333cd5a6ddab7e979b2b6b5b6","src/backend/common.rs":"3dd1b7fbe0c62b0731904358b11f82054c918044de6271c502fb0d38b813b67d","src/backend/impl_lmdb.rs":"0a5a73e11881bb610f59b8c221f724bfa2bf963e3ae5356428a9d4c528b581d0","src/backend/impl_lmdb/arch_migrator.rs":"c41c06ed82e667eb39f66c35eb5c6aef7e1bd205eaa25bd49e444ce37aea5141","src/backend/impl_lmdb/arch_migrator_error.rs":"09c8b75df05a3b9271810f1d5cb87b95ab0a38ed8622a7a856efde6fb12652be","src/backend/impl_lmdb/cursor.rs":"77a7611f8638c5358f74f79cccde18581d3f6078f72c3a8419abde144791bfdf","src/backend/impl_lmdb/database.rs":"c52ab76a4389c525a998eef0302e709d57a22f2627a967b2246e98ae15f4a999","src/backend/impl_lmdb/environment.rs":"eedd9d00e8a2a2d303dd3b4dbf171b9a40d027db9b066ae1400e97aa1348bfae","src/backend/impl_lmdb/error.rs":"9a8dbe3e036408398cd31d6f49dd1f0ac1ac0d14cbf4e884f1f986f3bdb2ce1b","src/backend/impl_lmdb/flags.rs":"8b54eaa3469aefe874441c5cf072c315c32cddf02896c7afd4bf3b1d36d4b5c0","src/backend/impl_lmdb/info.rs":"e44d9100c0acc179263f41c70d2d139faa1b19efe6948c951958986c5fc90dbf","src/backend/impl_lmdb/iter.rs":"792399008f0903fd18bce1a79592639706487636f3084060315667babe567769","src/backend/impl_lmdb/stat.rs":"ec3100fee914cfe4805a7c588344c707c027bad7b37b43b7455baa0772cb64f9","src/backend/impl_lmdb/transaction.rs":"4ee1b2dc57762bdf0c4fe46860a58d4804918023574424d584466dd8943af630","src/backend/impl_safe.rs":"c033006743b50fb06aff3da64acc2f31739e12beb9674533548162ce5dd8fd01","src/backend/impl_safe/cursor.rs":"570cdf7c59bae2d55716f23c13bdb4ee5c606feef2eff004eeed4c90cad377ca","src/backend/impl_safe/database.rs":"d34d159f5cc48ef944f6e55cca0cc75fd5b03ff1caee016e53f0f619b7e55d3e","src/backend/impl_safe/environment.rs":"d21c365c76b4083ec4c0b776884706d6a2c19a0e010bb0281c6bb894b781ed9b","src/backend/impl_safe/error.rs":"8edceffeb838cba60aa5710ffd4395d07a733e9248a0e046c5f9012dde1c7990","src/backend/impl_safe/flags.rs":"78f1cb5f67ae5dd28d3b1334dd93766052238b0ad4c26b7623b20cdb68fed094","src/backend/impl_safe/info.rs":"c9dc67d989394dd7a25e99681e4a25f19b4ca3992eb18b719fb89742fae635b2","src/backend/impl_safe/iter.rs":"b98b54b51b474cb1e31f90136b64871baff6c31261d60bd4f79faa329768f2e8","src/backend/impl_safe/snapshot.rs":"1d815f26778b410fd8255930177f2e389b05ee99a3f407a20c56b071c871ff12","src/backend/impl_safe/stat.rs":"77ea9937c2ff839cba4ed5597b4804550915d5c50fce0fc86133bf23cff49d95","src/backend/impl_safe/transaction.rs":"73d147f6b7eaf8188f1ebd500566c9e6468d0c63cad2b87bfa3e060acbd4e3af","src/backend/traits.rs":"e0b004f75aaee32130fa17e54b4eaec596c3a27e7e9d3106b8894ce5f6928f8b","src/bin/dump.rs":"85ce7883462e10e956337b9daa885fb004f3fa26e3463a418e7f8cd4b3d0c1c1","src/bin/rand.rs":"8a4d2abfaa44f6fac84f38c5d6b54df01a7d200c1c1122588c273796aea20078","src/env.rs":"7070316199fcb49edbaecb6ee735bc4e73105c20b035758b76ce9b1714b141ad","src/error.rs":"6734a4de74cf57b8d5fdfe51fb064d3e5e8d378b3af380f6c6fe1b87d48a21e6","src/helpers.rs":"2e25f7f654ccc4fe39577165311334ee4c79ef3d339805716a79b1a641280fb6","src/lib.rs":"cbfc78c0ab2700abf773b49c270eef8d101f0d3f62cdf923b7e688cea743a6fd","src/manager.rs":"5bc03929d1909bea175f426e9935efaba81aefda2b89f0c0ee5c156c4f6169b6","src/migrator.rs":"38b90fa62489812545a8069f0bea510e6c4eb2923609862af28406a34eae2f2c","src/readwrite.rs":"2060edc24ac52d49f39bca90f77524850bb274273e955d6a9199f366d3ec7ee2","src/store.rs":"74900d03e1b3aab2a0215c5c76f89f71c737613ce3cc51f91f4c43ebdd560072","src/store/integer.rs":"9050293f27e5133e0046645e5c9909feaad65ea732b361ec55521b4dd2860e67","src/store/integermulti.rs":"100f61f8c9ba8003ffda5c500cc9bf1f90b2aa4940b87e9abf547801b6919a1a","src/store/keys.rs":"584bf897df7a0a10dd2841cf31cb693124748757902374d215b4d7caf79baae5","src/store/keys/encodables.rs":"d8b5dd6f49cab4a44b6f638c473ad372f589669e7ef9bd6935aa960840b95063","src/store/keys/primitives.rs":"f714e54dd6507e048cf867deecd15a5a452298b8255575aa9eb1c67317fff5dd","src/store/multi.rs":"a37fad0619c714b4c012ac62fc89404bf85a07174e9200d9a801ebb8b22ab92f","src/store/single.rs":"28f4b115114f7faa6fcf654304204e2236c24e83c83859c555a014edcbd1c617","src/value.rs":"85588982933909600d995929a323abe70bb0d2cb2850f1ad229b829e9d5234b4","tests/env-all.rs":"e0176053eeff63601b17b740a5884e8f063e1ff87069ea7e0b109f66e2eefc41","tests/env-lmdb.rs":"6d60fe2f1c19d436ce6f755cde508a75e023c0e91938ab53173ebe0645c16a58","tests/env-migration.rs":"b284429434a922f30bbaf9e4073f97a94ae77b553efda627d5c8d0bc67a92bbd","tests/env-safe.rs":"47ecf6b074e6a1097699804206cb9a1307974d045fb2494c7398110bfe036636","tests/integer-store.rs":"3b0d4ed83ea170abbe748c5e19382e33cfd9298ca65de0afe7cdf01b2f8f8aef","tests/manager.rs":"07290f6d4f5eb864aa5bbab04d22e2dc2f5e2affaae49af74dcfd320c7d9d106","tests/multi-integer-store.rs":"dca20148ad0942349e15cd23bf726a54bd2d49ca61af82d52709e46d9009bfb4","tests/test_txn.rs":"3ccaa25da0ed8f8788be64576d2a503193b21792c3495cbe6874a68e23096351"},"package":"9f0ea3af1393b22f8fe25615b6fa5d13072b7b622e66acffc8b12b2baa0342b1"} \ No newline at end of file
diff --git a/third_party/rust/rkv/CODE_OF_CONDUCT.md b/third_party/rust/rkv/CODE_OF_CONDUCT.md
new file mode 100644
index 0000000000..498baa3fb0
--- /dev/null
+++ b/third_party/rust/rkv/CODE_OF_CONDUCT.md
@@ -0,0 +1,15 @@
+# Community Participation Guidelines
+
+This repository is governed by Mozilla's code of conduct and etiquette guidelines.
+For more details, please read the
+[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/).
+
+## How to Report
+For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page.
+
+<!--
+## Project Specific Etiquette
+
+In some cases, there will be additional project etiquette i.e.: (https://bugzilla.mozilla.org/page.cgi?id=etiquette.html).
+Please update for your project.
+-->
diff --git a/third_party/rust/rkv/Cargo.lock b/third_party/rust/rkv/Cargo.lock
new file mode 100644
index 0000000000..75e8ee49f0
--- /dev/null
+++ b/third_party/rust/rkv/Cargo.lock
@@ -0,0 +1,369 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "arrayref"
+version = "0.3.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bincode"
+version = "1.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "byteorder"
+version = "1.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
+
+[[package]]
+name = "cc"
+version = "1.0.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76a284da2e6fe2092f2353e51713435363112dfd60030e22add80be333fb928f"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "fastrand"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499"
+dependencies = [
+ "instant",
+]
+
+[[package]]
+name = "form_urlencoded"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "id-arena"
+version = "2.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005"
+
+[[package]]
+name = "idna"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89"
+
+[[package]]
+name = "lmdb-rkv"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "447a296f7aca299cfbb50f4e4f3d49451549af655fb7215d7f8c0c3d64bad42b"
+dependencies = [
+ "bitflags",
+ "byteorder",
+ "libc",
+ "lmdb-rkv-sys",
+]
+
+[[package]]
+name = "lmdb-rkv-sys"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
+[[package]]
+name = "log"
+version = "0.4.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578ede34cf02f8924ab9447f50c28075b4d3e5b269972345e7e0372b38c6cdcd"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "ordered-float"
+version = "3.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d84eb1409416d254e4a9c8fa56cc24701755025b458f0fcd8e59e1f5f40c23bf"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "paste"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1de2e551fb905ac83f73f7aedf2f0cb4a0da7e35efa24a202a936269f1f18e1"
+
+[[package]]
+name = "percent-encoding"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "rkv"
+version = "0.18.4"
+dependencies = [
+ "arrayref",
+ "bincode",
+ "bitflags",
+ "byteorder",
+ "id-arena",
+ "lazy_static",
+ "lmdb-rkv",
+ "log",
+ "ordered-float",
+ "paste",
+ "serde",
+ "serde_derive",
+ "tempfile",
+ "thiserror",
+ "url",
+ "uuid",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.147"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d193d69bae983fc11a79df82342761dfbf28a99fc8d203dca4c3c1b590948965"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.147"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f1d362ca8fc9c3e3a7484440752472d68a6caa98f1ab81d99b5dfe517cec852"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cdb1ef4eaeeaddc8fbd371e5017057064af0911902ef36b39801f67cc6d79e4"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "libc",
+ "redox_syscall",
+ "remove_dir_all",
+ "winapi",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "url"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+]
+
+[[package]]
+name = "uuid"
+version = "1.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "422ee0de9031b5b948b97a8fc04e3aa35230001a722ddd27943e0be31564ce4c"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
diff --git a/third_party/rust/rkv/Cargo.toml b/third_party/rust/rkv/Cargo.toml
new file mode 100644
index 0000000000..8d925c46ef
--- /dev/null
+++ b/third_party/rust/rkv/Cargo.toml
@@ -0,0 +1,135 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2018"
+rust-version = "1.58"
+name = "rkv"
+version = "0.18.4"
+authors = [
+ "Richard Newman <rnewman@twinql.com>",
+ "Nan Jiang <najiang@mozilla.com>",
+ "Myk Melez <myk@mykzilla.org>",
+ "Victor Porof <vporof@mozilla.com>",
+]
+exclude = ["/tests/envs/*"]
+description = "A simple, humane, typed key-value storage solution"
+homepage = "https://github.com/mozilla/rkv"
+documentation = "https://docs.rs/rkv"
+readme = "README.md"
+keywords = [
+ "lmdb",
+ "database",
+ "storage",
+]
+categories = ["database"]
+license = "Apache-2.0"
+repository = "https://github.com/mozilla/rkv"
+
+[[bin]]
+name = "rand"
+path = "src/bin/rand.rs"
+required-features = ["lmdb"]
+
+[[bin]]
+name = "dump"
+path = "src/bin/dump.rs"
+required-features = ["lmdb"]
+
+[[test]]
+name = "env-all"
+required-features = ["lmdb"]
+
+[[test]]
+name = "env-lmdb"
+required-features = ["lmdb"]
+
+[[test]]
+name = "env-migration"
+required-features = ["lmdb"]
+
+[dependencies.arrayref]
+version = "0.3"
+
+[dependencies.bincode]
+version = "1.0"
+
+[dependencies.bitflags]
+version = "1.2"
+
+[dependencies.byteorder]
+version = "1"
+
+[dependencies.id-arena]
+version = "2.2"
+
+[dependencies.lazy_static]
+version = "1.1"
+
+[dependencies.lmdb-rkv]
+version = "0.14"
+optional = true
+
+[dependencies.log]
+version = "0.4.4"
+
+[dependencies.ordered-float]
+version = "3.0.0"
+
+[dependencies.paste]
+version = "1.0.6"
+
+[dependencies.serde]
+version = "1.0.144"
+features = [
+ "derive",
+ "rc",
+]
+
+[dependencies.serde_derive]
+version = "1.0"
+
+[dependencies.thiserror]
+version = "1.0"
+
+[dependencies.url]
+version = "2.0"
+
+[dependencies.uuid]
+version = "1.0"
+
+[dev-dependencies.byteorder]
+version = "1"
+
+[dev-dependencies.tempfile]
+version = "3"
+
+[features]
+db-dup-sort = []
+db-int-key = []
+default = [
+ "db-dup-sort",
+ "db-int-key",
+]
+lmdb = ["lmdb-rkv"]
+no-canonicalize-path = []
+with-asan = [
+ "lmdb",
+ "lmdb-rkv/with-asan",
+]
+with-fuzzer = [
+ "lmdb",
+ "lmdb-rkv/with-fuzzer",
+]
+with-fuzzer-no-link = [
+ "lmdb",
+ "lmdb-rkv/with-fuzzer-no-link",
+]
diff --git a/third_party/rust/rkv/LICENSE b/third_party/rust/rkv/LICENSE
new file mode 100644
index 0000000000..d645695673
--- /dev/null
+++ b/third_party/rust/rkv/LICENSE
@@ -0,0 +1,202 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
diff --git a/third_party/rust/rkv/README.md b/third_party/rust/rkv/README.md
new file mode 100644
index 0000000000..bc50d1f0a6
--- /dev/null
+++ b/third_party/rust/rkv/README.md
@@ -0,0 +1,77 @@
+# rkv
+
+[![CI Build Status](https://github.com/mozilla/rkv/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/mozilla/rkv/actions/workflows/ci.yml)
+[![Documentation](https://docs.rs/rkv/badge.svg)](https://docs.rs/rkv/)
+[![Crate](https://img.shields.io/crates/v/rkv.svg)](https://crates.io/crates/rkv)
+
+The [rkv Rust crate](https://crates.io/crates/rkv) is a simple, humane, typed key-value storage solution. It supports multiple backend engines with varying guarantees, such as [LMDB](http://www.lmdb.tech/doc/) for performance, or "SafeMode" for reliability.
+
+## ⚠️ Warning ⚠️
+
+To use rkv in production/release environments at Mozilla, you may do so with the "SafeMode" backend, for example:
+
+```rust
+use rkv::{Manager, Rkv};
+use rkv::backend::{SafeMode, SafeModeEnvironment};
+
+let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
+let shared_rkv = manager.get_or_create(path, Rkv::new::<SafeMode>).unwrap();
+
+...
+```
+
+The "SafeMode" backend performs well, with two caveats: the entire database is stored in memory, and write transactions are synchronously written to disk (only on commit).
+
+In the future, it will be advisable to switch to a different backend with better performance guarantees. We're working on either fixing some LMDB crashes, or offering more choices of backend engines (e.g. SQLite).
+
+## Use
+
+Comprehensive information about using rkv is available in its [online documentation](https://docs.rs/rkv/), which can also be generated for local consumption:
+
+```sh
+cargo doc --open
+```
+
+## Build
+
+Build this project as you would build other Rust crates:
+
+```sh
+cargo build
+```
+
+### Features
+
+There are several features that you can opt-in and out of when using rkv:
+
+By default, `db-dup-sort` and `db-int-key` features offer high level database APIs which allow multiple values per key, and optimizations around integer-based keys respectively. Opt out of these default features when specifying the rkv dependency in your Cargo.toml file to disable them; doing so avoids a certain amount of overhead required to support them.
+
+To aid fuzzing efforts, `with-asan`, `with-fuzzer`, and `with-fuzzer-no-link` configure the build scripts responsible with compiling the underlying backing engines (e.g. LMDB) to build with these LLMV features enabled. Please refer to the official LLVM/Clang documentation on them for more informatiuon. These features are also disabled by default.
+
+## Test
+
+Test this project as you would test other Rust crates:
+
+```sh
+cargo test
+```
+
+The project includes unit and doc tests embedded in the `src/` files, integration tests in the `tests/` subdirectory, and usage examples in the `examples/` subdirectory. To ensure your changes don't break examples, also run them via the run-all-examples.sh shell script:
+
+```sh
+./run-all-examples.sh
+```
+
+Note: the test fixtures in the `tests/envs/` subdirectory aren't included in the package published to crates.io, so you must clone this repository in order to run the tests that depend on those fixtures or use the `rand` and `dump` executables to recreate them.
+
+## Contribute
+
+Of the various open source archetypes described in [A Framework for Purposeful Open Source](https://medium.com/mozilla-open-innovation/whats-your-open-source-strategy-here-are-10-answers-383221b3f9d3), the rkv project most closely resembles the Specialty Library, and we welcome contributions. Please report problems or ask questions using this repo's GitHub [issue tracker](https://github.com/mozilla/rkv/issues) and submit [pull requests](https://github.com/mozilla/rkv/pulls) for code and documentation changes.
+
+rkv relies on the latest [rustfmt](https://github.com/rust-lang-nursery/rustfmt) for code formatting, so please make sure your pull request passes the rustfmt before submitting it for review. See rustfmt's [quick start](https://github.com/rust-lang-nursery/rustfmt#quick-start) for installation details.
+
+We follow Mozilla's [Community Participation Guidelines](https://www.mozilla.org/en-US/about/governance/policies/participation/) while contributing to this project.
+
+## License
+
+The rkv source code is licensed under the Apache License, Version 2.0, as described in the [LICENSE](https://github.com/mozilla/rkv/blob/master/LICENSE) file.
diff --git a/third_party/rust/rkv/examples/README.md b/third_party/rust/rkv/examples/README.md
new file mode 100644
index 0000000000..0e5a3e6d67
--- /dev/null
+++ b/third_party/rust/rkv/examples/README.md
@@ -0,0 +1,11 @@
+## Examples of how to use rkv
+
+All examples can be executed with:
+
+```
+cargo run --example $name
+```
+
+* [`simple-store`](simple-store.rs) - a simple key/value store that showcases the basic usage of rkv.
+
+* [`iterator`](iterator.rs) - a demo that showcases the basic usage of iterators in rkv.
diff --git a/third_party/rust/rkv/examples/iterator.rs b/third_party/rust/rkv/examples/iterator.rs
new file mode 100644
index 0000000000..445dc37c28
--- /dev/null
+++ b/third_party/rust/rkv/examples/iterator.rs
@@ -0,0 +1,76 @@
+// Any copyright is dedicated to the Public Domain.
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+//! A demo that showcases the basic usage of iterators in rkv.
+//!
+//! You can test this out by running:
+//!
+//! cargo run --example iterator
+
+use std::{fs, str};
+
+use tempfile::Builder;
+
+use rkv::{
+ backend::{SafeMode, SafeModeDatabase, SafeModeEnvironment},
+ Manager, Rkv, SingleStore, StoreError, StoreOptions, Value,
+};
+
+fn main() {
+ let root = Builder::new().prefix("iterator").tempdir().unwrap();
+ fs::create_dir_all(root.path()).unwrap();
+ let p = root.path();
+
+ let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
+ let created_arc = manager.get_or_create(p, Rkv::new::<SafeMode>).unwrap();
+ let k = created_arc.read().unwrap();
+ let store = k.open_single("store", StoreOptions::create()).unwrap();
+
+ populate_store(&k, store).unwrap();
+
+ let reader = k.read().unwrap();
+
+ println!("Iterating from the beginning...");
+ // Reader::iter_start() iterates from the first item in the store, and
+ // returns the (key, value) tuples in order.
+ let mut iter = store.iter_start(&reader).unwrap();
+ while let Some(Ok((country, city))) = iter.next() {
+ let country = str::from_utf8(country).unwrap();
+ println!("{country}, {city:?}");
+ }
+
+ println!();
+ println!("Iterating from the given key...");
+ // Reader::iter_from() iterates from the first key equal to or greater
+ // than the given key.
+ let mut iter = store.iter_from(&reader, "Japan").unwrap();
+ while let Some(Ok((country, city))) = iter.next() {
+ println!("{}, {city:?}", str::from_utf8(country).unwrap());
+ }
+
+ println!();
+ println!("Iterating from the given prefix...");
+ let mut iter = store.iter_from(&reader, "Un").unwrap();
+ while let Some(Ok((country, city))) = iter.next() {
+ println!("{}, {city:?}", str::from_utf8(country).unwrap());
+ }
+}
+
+fn populate_store(
+ k: &Rkv<SafeModeEnvironment>,
+ store: SingleStore<SafeModeDatabase>,
+) -> Result<(), StoreError> {
+ let mut writer = k.write()?;
+ for (country, city) in vec![
+ ("Canada", Value::Str("Ottawa")),
+ ("United States of America", Value::Str("Washington")),
+ ("Germany", Value::Str("Berlin")),
+ ("France", Value::Str("Paris")),
+ ("Italy", Value::Str("Rome")),
+ ("United Kingdom", Value::Str("London")),
+ ("Japan", Value::Str("Tokyo")),
+ ] {
+ store.put(&mut writer, country, &city)?;
+ }
+ writer.commit()
+}
diff --git a/third_party/rust/rkv/examples/simple-store.rs b/third_party/rust/rkv/examples/simple-store.rs
new file mode 100644
index 0000000000..c097a2f573
--- /dev/null
+++ b/third_party/rust/rkv/examples/simple-store.rs
@@ -0,0 +1,252 @@
+// Any copyright is dedicated to the Public Domain.
+// http://creativecommons.org/publicdomain/zero/1.0/
+
+//! A simple rkv demo that showcases the basic usage (put/get/delete) of rkv.
+//!
+//! You can test this out by running:
+//!
+//! cargo run --example simple-store
+
+use std::fs;
+
+use tempfile::Builder;
+
+use rkv::{
+ backend::{SafeMode, SafeModeDatabase, SafeModeEnvironment, SafeModeRwTransaction},
+ Manager, Rkv, StoreOptions, Value,
+};
+
+type MultiStore = rkv::MultiStore<SafeModeDatabase>;
+type Writer<'w> = rkv::Writer<SafeModeRwTransaction<'w>>;
+
+fn getput<'w, 's>(store: MultiStore, writer: &'w mut Writer, ids: &'s mut Vec<String>) {
+ let keys = vec!["str1", "str2", "str3"];
+ // we convert the writer into a cursor so that we can safely read
+ for k in keys.iter() {
+ // this is a multi-valued database, so get returns an iterator
+ let mut iter = store.get(writer, k).unwrap();
+ while let Some(Ok((_key, val))) = iter.next() {
+ if let Value::Str(s) = val {
+ ids.push(s.to_owned());
+ } else {
+ panic!("didn't get a string back!");
+ }
+ }
+ }
+ for id in ids {
+ store.put(writer, &id, &Value::Blob(b"weeeeeee")).unwrap();
+ }
+}
+
+fn delete(store: MultiStore, writer: &mut Writer) {
+ let keys = vec!["str1", "str2", "str3"];
+ let vals = vec!["string uno", "string quatro", "string siete"];
+ // we convert the writer into a cursor so that we can safely read
+ for i in 0..keys.len() {
+ store
+ .delete(writer, &keys[i], &Value::Str(vals[i]))
+ .unwrap();
+ }
+}
+
+fn main() {
+ let root = Builder::new().prefix("simple-db").tempdir().unwrap();
+ fs::create_dir_all(root.path()).unwrap();
+ let p = root.path();
+
+ // The manager enforces that each process opens the same lmdb environment at most once
+ let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
+ let created_arc = manager.get_or_create(p, Rkv::new::<SafeMode>).unwrap();
+ let k = created_arc.read().unwrap();
+
+ // Creates a store called "store"
+ let store = k.open_single("store", StoreOptions::create()).unwrap();
+ let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
+
+ println!("Inserting data...");
+ {
+ // Use a writer to mutate the store
+ let mut writer = k.write().unwrap();
+ store.put(&mut writer, "int", &Value::I64(1234)).unwrap();
+ store
+ .put(&mut writer, "uint", &Value::U64(1234_u64))
+ .unwrap();
+ store
+ .put(&mut writer, "float", &Value::F64(1234.0.into()))
+ .unwrap();
+ store
+ .put(&mut writer, "instant", &Value::Instant(1_528_318_073_700))
+ .unwrap();
+ store
+ .put(&mut writer, "boolean", &Value::Bool(true))
+ .unwrap();
+ store
+ .put(&mut writer, "string", &Value::Str("héllo, yöu"))
+ .unwrap();
+ store
+ .put(
+ &mut writer,
+ "json",
+ &Value::Json(r#"{"foo":"bar", "number": 1}"#),
+ )
+ .unwrap();
+ store
+ .put(&mut writer, "blob", &Value::Blob(b"blob"))
+ .unwrap();
+ writer.commit().unwrap();
+ }
+
+ println!("Testing getput");
+ {
+ let mut ids = Vec::new();
+ let mut writer = k.write().unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("string uno"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("string dos"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("string tres"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("string quatro"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("string cinco"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("string seis"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("string siete"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("string ocho"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("string nueve"))
+ .unwrap();
+ getput(multistore, &mut writer, &mut ids);
+ writer.commit().unwrap();
+ let mut writer = k.write().unwrap();
+ delete(multistore, &mut writer);
+ writer.commit().unwrap();
+ }
+
+ println!("Looking up keys...");
+ {
+ // Use a reader to query the store
+ let reader = k.read().unwrap();
+ println!("Get int {:?}", store.get(&reader, "int").unwrap());
+ println!("Get uint {:?}", store.get(&reader, "uint").unwrap());
+ println!("Get float {:?}", store.get(&reader, "float").unwrap());
+ println!("Get instant {:?}", store.get(&reader, "instant").unwrap());
+ println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap());
+ println!("Get string {:?}", store.get(&reader, "string").unwrap());
+ println!("Get json {:?}", store.get(&reader, "json").unwrap());
+ println!("Get blob {:?}", store.get(&reader, "blob").unwrap());
+ println!(
+ "Get non-existent {:?}",
+ store.get(&reader, "non-existent").unwrap()
+ );
+ }
+
+ println!("Looking up keys via Writer.get()...");
+ {
+ let mut writer = k.write().unwrap();
+ store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+ store.put(&mut writer, "bar", &Value::Str("baz")).unwrap();
+ store.delete(&mut writer, "foo").unwrap();
+ println!(
+ "It should be None! ({:?})",
+ store.get(&writer, "foo").unwrap()
+ );
+ println!("Get bar ({:?})", store.get(&writer, "bar").unwrap());
+ writer.commit().unwrap();
+ let reader = k.read().expect("reader");
+ println!(
+ "It should be None! ({:?})",
+ store.get(&reader, "foo").unwrap()
+ );
+ println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
+ }
+
+ println!("Aborting transaction...");
+ {
+ // Aborting a write transaction rollbacks the change(s)
+ let mut writer = k.write().unwrap();
+ store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+ writer.abort();
+
+ let reader = k.read().expect("reader");
+ println!(
+ "It should be None! ({:?})",
+ store.get(&reader, "foo").unwrap()
+ );
+ // Explicitly aborting a transaction is not required unless an early
+ // abort is desired, since both read and write transactions will
+ // implicitly be aborted once they go out of scope.
+ }
+
+ println!("Deleting keys...");
+ {
+ // Deleting a key/value also requires a write transaction
+ let mut writer = k.write().unwrap();
+ store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+ store.delete(&mut writer, "foo").unwrap();
+ println!(
+ "It should be None! ({:?})",
+ store.get(&writer, "foo").unwrap()
+ );
+ writer.commit().unwrap();
+
+ // Committing a transaction consumes the writer, preventing you
+ // from reusing it by failing and reporting a compile-time error.
+ // This line would report error[E0382]: use of moved value: `writer`.
+ // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap();
+ }
+
+ println!("Clearing store...");
+ {
+ // Clearing a store deletes all the entries in that store
+ let mut writer = k.write().unwrap();
+ store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+ store.put(&mut writer, "bar", &Value::Str("baz")).unwrap();
+ store.clear(&mut writer).unwrap();
+ writer.commit().unwrap();
+
+ let reader = k.read().expect("reader");
+ println!(
+ "It should be None! ({:?})",
+ store.get(&reader, "foo").unwrap()
+ );
+ println!(
+ "It should be None! ({:?})",
+ store.get(&reader, "bar").unwrap()
+ );
+ }
+
+ println!("Write and read on multiple stores...");
+ {
+ let another_store = k
+ .open_single("another_store", StoreOptions::create())
+ .unwrap();
+ let mut writer = k.write().unwrap();
+ store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+ another_store
+ .put(&mut writer, "foo", &Value::Str("baz"))
+ .unwrap();
+ writer.commit().unwrap();
+
+ let reader = k.read().unwrap();
+ println!(
+ "Get from store value: {:?}",
+ store.get(&reader, "foo").unwrap()
+ );
+ println!(
+ "Get from another store value: {:?}",
+ another_store.get(&reader, "foo").unwrap()
+ );
+ }
+}
diff --git a/third_party/rust/rkv/run-all-examples.sh b/third_party/rust/rkv/run-all-examples.sh
new file mode 100755
index 0000000000..fa7eb959d6
--- /dev/null
+++ b/third_party/rust/rkv/run-all-examples.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+cargo build --examples
+
+for file in examples/*; do
+ filename=$(basename ${file})
+ extension=${filename##*.}
+ example_name=${filename%.*}
+ if [[ "${extension}" = "rs" ]]; then
+ cargo run --example ${example_name}
+ fi
+done
diff --git a/third_party/rust/rkv/src/backend.rs b/third_party/rust/rkv/src/backend.rs
new file mode 100644
index 0000000000..63fa08a40c
--- /dev/null
+++ b/third_party/rust/rkv/src/backend.rs
@@ -0,0 +1,40 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod common;
+#[cfg(feature = "lmdb")]
+mod impl_lmdb;
+mod impl_safe;
+mod traits;
+
+pub use common::*;
+pub use traits::*;
+
+#[cfg(feature = "lmdb")]
+pub use impl_lmdb::{
+ ArchMigrateError as LmdbArchMigrateError, ArchMigrateResult as LmdbArchMigrateResult,
+ ArchMigrator as LmdbArchMigrator, DatabaseFlagsImpl as LmdbDatabaseFlags,
+ DatabaseImpl as LmdbDatabase, EnvironmentBuilderImpl as Lmdb,
+ EnvironmentFlagsImpl as LmdbEnvironmentFlags, EnvironmentImpl as LmdbEnvironment,
+ ErrorImpl as LmdbError, InfoImpl as LmdbInfo, IterImpl as LmdbIter,
+ RoCursorImpl as LmdbRoCursor, RoTransactionImpl as LmdbRoTransaction,
+ RwCursorImpl as LmdbRwCursor, RwTransactionImpl as LmdbRwTransaction, StatImpl as LmdbStat,
+ WriteFlagsImpl as LmdbWriteFlags,
+};
+
+pub use impl_safe::{
+ DatabaseFlagsImpl as SafeModeDatabaseFlags, DatabaseImpl as SafeModeDatabase,
+ EnvironmentBuilderImpl as SafeMode, EnvironmentFlagsImpl as SafeModeEnvironmentFlags,
+ EnvironmentImpl as SafeModeEnvironment, ErrorImpl as SafeModeError, InfoImpl as SafeModeInfo,
+ IterImpl as SafeModeIter, RoCursorImpl as SafeModeRoCursor,
+ RoTransactionImpl as SafeModeRoTransaction, RwCursorImpl as SafeModeRwCursor,
+ RwTransactionImpl as SafeModeRwTransaction, StatImpl as SafeModeStat,
+ WriteFlagsImpl as SafeModeWriteFlags,
+};
diff --git a/third_party/rust/rkv/src/backend/common.rs b/third_party/rust/rkv/src/backend/common.rs
new file mode 100644
index 0000000000..bea3839d03
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/common.rs
@@ -0,0 +1,44 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![allow(non_camel_case_types)]
+
+pub enum EnvironmentFlags {
+ FIXED_MAP,
+ NO_SUB_DIR,
+ WRITE_MAP,
+ READ_ONLY,
+ NO_META_SYNC,
+ NO_SYNC,
+ MAP_ASYNC,
+ NO_TLS,
+ NO_LOCK,
+ NO_READAHEAD,
+ NO_MEM_INIT,
+}
+
+pub enum DatabaseFlags {
+ REVERSE_KEY,
+ #[cfg(feature = "db-dup-sort")]
+ DUP_SORT,
+ #[cfg(feature = "db-dup-sort")]
+ DUP_FIXED,
+ #[cfg(feature = "db-int-key")]
+ INTEGER_KEY,
+ INTEGER_DUP,
+ REVERSE_DUP,
+}
+
+pub enum WriteFlags {
+ NO_OVERWRITE,
+ NO_DUP_DATA,
+ CURRENT,
+ APPEND,
+ APPEND_DUP,
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb.rs b/third_party/rust/rkv/src/backend/impl_lmdb.rs
new file mode 100644
index 0000000000..5364214598
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb.rs
@@ -0,0 +1,34 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod arch_migrator;
+mod arch_migrator_error;
+mod cursor;
+mod database;
+mod environment;
+mod error;
+mod flags;
+mod info;
+mod iter;
+mod stat;
+mod transaction;
+
+pub use arch_migrator::{
+ MigrateError as ArchMigrateError, MigrateResult as ArchMigrateResult, Migrator as ArchMigrator,
+};
+pub use cursor::{RoCursorImpl, RwCursorImpl};
+pub use database::DatabaseImpl;
+pub use environment::{EnvironmentBuilderImpl, EnvironmentImpl};
+pub use error::ErrorImpl;
+pub use flags::{DatabaseFlagsImpl, EnvironmentFlagsImpl, WriteFlagsImpl};
+pub use info::InfoImpl;
+pub use iter::IterImpl;
+pub use stat::StatImpl;
+pub use transaction::{RoTransactionImpl, RwTransactionImpl};
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs
new file mode 100644
index 0000000000..6a91db1a3b
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs
@@ -0,0 +1,998 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![allow(dead_code)] // TODO: Get rid of unused struct members
+#![allow(clippy::upper_case_acronyms)] // TODO: Consider renaming things like `BRANCH`
+
+//! A utility for migrating data from one LMDB environment to another. Notably, this tool
+//! can migrate data from an enviroment created with a different bit-depth than the
+//! current rkv consumer, which enables the consumer to retrieve data from an environment
+//! that can't be read directly using the rkv APIs.
+//!
+//! The utility supports both 32-bit and 64-bit LMDB source environments, and it
+//! automatically migrates data in both the default database and any named (sub)
+//! databases. It also migrates the source environment's "map size" and "max DBs"
+//! configuration options to the destination environment.
+//!
+//! The destination environment must be at the rkv consumer's bit depth and should be
+//! empty of data. It can be an empty directory, in which case the utility will create a
+//! new LMDB environment within the directory.
+//!
+//! The tool currently has these limitations:
+//!
+//! 1. It doesn't support migration from environments created with
+//! `EnvironmentFlags::NO_SUB_DIR`. To migrate such an environment, create a
+//! temporary directory, copy the environment's data file to a file called data.mdb in
+//! the temporary directory, then migrate the temporary directory as the source
+//! environment.
+//! 2. It doesn't support migration from databases created with DatabaseFlags::DUP_SORT`
+//! (with or without `DatabaseFlags::DUP_FIXED`).
+//! 3. It doesn't account for existing data in the destination environment, which means
+//! that it can overwrite data (causing data loss) or fail to migrate data if the
+//! destination environment contains existing data.
+//!
+//! ## Basic Usage
+//!
+//! Call `Migrator::new()` with the path to the source environment to create a `Migrator`
+//! instance; then call the instance's `migrate()` method with the path to the destination
+//! environment to migrate data from the source to the destination environment. For
+//! example, this snippet migrates data from the tests/envs/ref_env_32 environment to a
+//! new environment in a temporary directory:
+//!
+//! ```
+//! use rkv::migrator::LmdbArchMigrator as Migrator;
+//! use std::path::Path;
+//! use tempfile::tempdir;
+//! let mut migrator = Migrator::new(Path::new("tests/envs/ref_env_32")).unwrap();
+//! migrator.migrate(&tempdir().unwrap().path()).unwrap();
+//! ```
+//!
+//! Both `Migrator::new()` and `migrate()` return a `MigrateResult` that is either an
+//! `Ok()` result or an `Err<MigrateError>`, where `MigrateError` is an enum whose
+//! variants identify specific kinds of migration failures.
+
+use std::{
+ collections::{BTreeMap, HashMap},
+ convert::TryFrom,
+ fs::File,
+ io::{Cursor, Read, Seek, SeekFrom, Write},
+ path::{Path, PathBuf},
+ rc::Rc,
+ str,
+};
+
+use bitflags::bitflags;
+use byteorder::{LittleEndian, ReadBytesExt};
+use lmdb::{DatabaseFlags, Environment, Transaction, WriteFlags};
+
+pub use super::arch_migrator_error::MigrateError;
+
+const PAGESIZE: u16 = 4096;
+
+// The magic number is 0xBEEFC0DE, which is 0xDEC0EFBE in little-endian. It appears at
+// offset 12 on 32-bit systems and 16 on 64-bit systems. We don't support big-endian
+// migration, but presumably we could do so by detecting the order of the bytes.
+const MAGIC: [u8; 4] = [0xDE, 0xC0, 0xEF, 0xBE];
+
+pub type MigrateResult<T> = Result<T, MigrateError>;
+
+bitflags! {
+ #[derive(Default)]
+ struct PageFlags: u16 {
+ const BRANCH = 0x01;
+ const LEAF = 0x02;
+ const OVERFLOW = 0x04;
+ const META = 0x08;
+ const DIRTY = 0x10;
+ const LEAF2 = 0x20;
+ const SUBP = 0x40;
+ const LOOSE = 0x4000;
+ const KEEP = 0x8000;
+ }
+}
+
+bitflags! {
+ #[derive(Default)]
+ struct NodeFlags: u16 {
+ const BIGDATA = 0x01;
+ const SUBDATA = 0x02;
+ const DUPDATA = 0x04;
+ }
+}
+
+// The bit depth of the executable that created an LMDB environment. The Migrator
+// determines this automatically based on the location of the magic number in data.mdb.
+#[derive(Clone, Copy, PartialEq)]
+enum Bits {
+ U32,
+ U64,
+}
+
+impl Bits {
+ // The size of usize for the bit-depth represented by the enum variant.
+ fn size(self) -> usize {
+ match self {
+ Bits::U32 => 4,
+ Bits::U64 => 8,
+ }
+ }
+}
+
+// The equivalent of PAGEHDRSZ in LMDB, except that this one varies by bits.
+fn page_header_size(bits: Bits) -> u64 {
+ match bits {
+ Bits::U32 => 12,
+ Bits::U64 => 16,
+ }
+}
+
+// The equivalent of P_INVALID in LMDB, except that this one varies by bits.
+fn validate_page_num(page_num: u64, bits: Bits) -> MigrateResult<()> {
+ let invalid_page_num = match bits {
+ Bits::U32 => u64::from(!0u32),
+ Bits::U64 => !0u64,
+ };
+
+ if page_num == invalid_page_num {
+ return Err(MigrateError::InvalidPageNum);
+ }
+
+ Ok(())
+}
+
+#[derive(Clone, Debug, Default)]
+struct Database {
+ md_pad: u32,
+ md_flags: DatabaseFlags,
+ md_depth: u16,
+ md_branch_pages: u64,
+ md_leaf_pages: u64,
+ md_overflow_pages: u64,
+ md_entries: u64,
+ md_root: u64,
+}
+
+impl Database {
+ fn new(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<Database> {
+ Ok(Database {
+ md_pad: cursor.read_u32::<LittleEndian>()?,
+ md_flags: DatabaseFlags::from_bits(cursor.read_u16::<LittleEndian>()?.into())
+ .ok_or(MigrateError::InvalidDatabaseBits)?,
+ md_depth: cursor.read_u16::<LittleEndian>()?,
+ md_branch_pages: cursor.read_uint::<LittleEndian>(bits.size())?,
+ md_leaf_pages: cursor.read_uint::<LittleEndian>(bits.size())?,
+ md_overflow_pages: cursor.read_uint::<LittleEndian>(bits.size())?,
+ md_entries: cursor.read_uint::<LittleEndian>(bits.size())?,
+ md_root: cursor.read_uint::<LittleEndian>(bits.size())?,
+ })
+ }
+}
+
+#[derive(Debug, Default)]
+struct Databases {
+ free: Database,
+ main: Database,
+}
+
+#[derive(Debug, Default)]
+struct MetaData {
+ mm_magic: u32,
+ mm_version: u32,
+ mm_address: u64,
+ mm_mapsize: u64,
+ mm_dbs: Databases,
+ mm_last_pg: u64,
+ mm_txnid: u64,
+}
+
+#[derive(Debug)]
+enum LeafNode {
+ Regular {
+ mn_lo: u16,
+ mn_hi: u16,
+ mn_flags: NodeFlags,
+ mn_ksize: u16,
+ mv_size: u32,
+ key: Vec<u8>,
+ value: Vec<u8>,
+ },
+ BigData {
+ mn_lo: u16,
+ mn_hi: u16,
+ mn_flags: NodeFlags,
+ mn_ksize: u16,
+ mv_size: u32,
+ key: Vec<u8>,
+ overflow_pgno: u64,
+ },
+ SubData {
+ mn_lo: u16,
+ mn_hi: u16,
+ mn_flags: NodeFlags,
+ mn_ksize: u16,
+ mv_size: u32,
+ key: Vec<u8>,
+ value: Vec<u8>,
+ db: Database,
+ },
+}
+
+#[derive(Debug, Default)]
+struct BranchNode {
+ mp_pgno: u64,
+ mn_ksize: u16,
+ mn_data: Vec<u8>,
+}
+
+#[derive(Debug)]
+enum PageHeader {
+ Regular {
+ mp_pgno: u64,
+ mp_flags: PageFlags,
+ pb_lower: u16,
+ pb_upper: u16,
+ },
+ Overflow {
+ mp_pgno: u64,
+ mp_flags: PageFlags,
+ pb_pages: u32,
+ },
+}
+
+#[derive(Debug)]
+enum Page {
+ META(MetaData),
+ LEAF(Vec<LeafNode>),
+ BRANCH(Vec<BranchNode>),
+}
+
+impl Page {
+ fn new(buf: Vec<u8>, bits: Bits) -> MigrateResult<Page> {
+ let mut cursor = std::io::Cursor::new(&buf[..]);
+
+ match Self::parse_page_header(&mut cursor, bits)? {
+ PageHeader::Regular {
+ mp_flags, pb_lower, ..
+ } => {
+ if mp_flags.contains(PageFlags::LEAF2) || mp_flags.contains(PageFlags::SUBP) {
+ // We don't yet support DUPFIXED and DUPSORT databases.
+ return Err(MigrateError::UnsupportedPageHeaderVariant);
+ }
+
+ if mp_flags.contains(PageFlags::META) {
+ let meta_data = Self::parse_meta_data(&mut cursor, bits)?;
+ Ok(Page::META(meta_data))
+ } else if mp_flags.contains(PageFlags::LEAF) {
+ let nodes = Self::parse_leaf_nodes(&mut cursor, pb_lower, bits)?;
+ Ok(Page::LEAF(nodes))
+ } else if mp_flags.contains(PageFlags::BRANCH) {
+ let nodes = Self::parse_branch_nodes(&mut cursor, pb_lower, bits)?;
+ Ok(Page::BRANCH(nodes))
+ } else {
+ Err(MigrateError::UnexpectedPageHeaderVariant)
+ }
+ }
+ PageHeader::Overflow { .. } => {
+ // There isn't anything to do, nor should we try to instantiate
+ // a page of this type, as we only access them when reading
+ // a value that is too large to fit into a leaf node.
+ Err(MigrateError::UnexpectedPageHeaderVariant)
+ }
+ }
+ }
+
+ fn parse_page_header(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<PageHeader> {
+ let mp_pgno = cursor.read_uint::<LittleEndian>(bits.size())?;
+ let _mp_pad = cursor.read_u16::<LittleEndian>()?;
+ let mp_flags = PageFlags::from_bits(cursor.read_u16::<LittleEndian>()?)
+ .ok_or(MigrateError::InvalidPageBits)?;
+
+ if mp_flags.contains(PageFlags::OVERFLOW) {
+ let pb_pages = cursor.read_u32::<LittleEndian>()?;
+ Ok(PageHeader::Overflow {
+ mp_pgno,
+ mp_flags,
+ pb_pages,
+ })
+ } else {
+ let pb_lower = cursor.read_u16::<LittleEndian>()?;
+ let pb_upper = cursor.read_u16::<LittleEndian>()?;
+ Ok(PageHeader::Regular {
+ mp_pgno,
+ mp_flags,
+ pb_lower,
+ pb_upper,
+ })
+ }
+ }
+
+ fn parse_meta_data(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<MetaData> {
+ cursor.seek(SeekFrom::Start(page_header_size(bits)))?;
+
+ Ok(MetaData {
+ mm_magic: cursor.read_u32::<LittleEndian>()?,
+ mm_version: cursor.read_u32::<LittleEndian>()?,
+ mm_address: cursor.read_uint::<LittleEndian>(bits.size())?,
+ mm_mapsize: cursor.read_uint::<LittleEndian>(bits.size())?,
+ mm_dbs: Databases {
+ free: Database::new(cursor, bits)?,
+ main: Database::new(cursor, bits)?,
+ },
+ mm_last_pg: cursor.read_uint::<LittleEndian>(bits.size())?,
+ mm_txnid: cursor.read_uint::<LittleEndian>(bits.size())?,
+ })
+ }
+
+ fn parse_leaf_nodes(
+ cursor: &mut Cursor<&[u8]>,
+ pb_lower: u16,
+ bits: Bits,
+ ) -> MigrateResult<Vec<LeafNode>> {
+ cursor.set_position(page_header_size(bits));
+ let num_keys = Self::num_keys(pb_lower, bits);
+ let mp_ptrs = Self::parse_mp_ptrs(cursor, num_keys)?;
+
+ let mut leaf_nodes = Vec::with_capacity(num_keys as usize);
+
+ for mp_ptr in mp_ptrs {
+ cursor.set_position(u64::from(mp_ptr));
+ leaf_nodes.push(Self::parse_leaf_node(cursor, bits)?);
+ }
+
+ Ok(leaf_nodes)
+ }
+
+ fn parse_leaf_node(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<LeafNode> {
+ // The order of the mn_lo and mn_hi fields is endian-dependent and would be
+ // reversed in an LMDB environment created on a big-endian system.
+ let mn_lo = cursor.read_u16::<LittleEndian>()?;
+ let mn_hi = cursor.read_u16::<LittleEndian>()?;
+
+ let mn_flags = NodeFlags::from_bits(cursor.read_u16::<LittleEndian>()?)
+ .ok_or(MigrateError::InvalidNodeBits)?;
+ let mn_ksize = cursor.read_u16::<LittleEndian>()?;
+
+ let start = usize::try_from(cursor.position())?;
+ let end = usize::try_from(cursor.position() + u64::from(mn_ksize))?;
+ let key = cursor.get_ref()[start..end].to_vec();
+ cursor.set_position(end as u64);
+
+ let mv_size = Self::leaf_node_size(mn_lo, mn_hi);
+ if mn_flags.contains(NodeFlags::BIGDATA) {
+ let overflow_pgno = cursor.read_uint::<LittleEndian>(bits.size())?;
+ Ok(LeafNode::BigData {
+ mn_lo,
+ mn_hi,
+ mn_flags,
+ mn_ksize,
+ mv_size,
+ key,
+ overflow_pgno,
+ })
+ } else if mn_flags.contains(NodeFlags::SUBDATA) {
+ let start = usize::try_from(cursor.position())?;
+ let end = usize::try_from(cursor.position() + u64::from(mv_size))?;
+ let value = cursor.get_ref()[start..end].to_vec();
+ let mut cursor = std::io::Cursor::new(&value[..]);
+ let db = Database::new(&mut cursor, bits)?;
+ validate_page_num(db.md_root, bits)?;
+ Ok(LeafNode::SubData {
+ mn_lo,
+ mn_hi,
+ mn_flags,
+ mn_ksize,
+ mv_size,
+ key,
+ value,
+ db,
+ })
+ } else {
+ let start = usize::try_from(cursor.position())?;
+ let end = usize::try_from(cursor.position() + u64::from(mv_size))?;
+ let value = cursor.get_ref()[start..end].to_vec();
+ Ok(LeafNode::Regular {
+ mn_lo,
+ mn_hi,
+ mn_flags,
+ mn_ksize,
+ mv_size,
+ key,
+ value,
+ })
+ }
+ }
+
+ fn leaf_node_size(mn_lo: u16, mn_hi: u16) -> u32 {
+ u32::from(mn_lo) + ((u32::from(mn_hi)) << 16)
+ }
+
+ fn parse_branch_nodes(
+ cursor: &mut Cursor<&[u8]>,
+ pb_lower: u16,
+ bits: Bits,
+ ) -> MigrateResult<Vec<BranchNode>> {
+ let num_keys = Self::num_keys(pb_lower, bits);
+ let mp_ptrs = Self::parse_mp_ptrs(cursor, num_keys)?;
+
+ let mut branch_nodes = Vec::with_capacity(num_keys as usize);
+
+ for mp_ptr in mp_ptrs {
+ cursor.set_position(u64::from(mp_ptr));
+ branch_nodes.push(Self::parse_branch_node(cursor, bits)?)
+ }
+
+ Ok(branch_nodes)
+ }
+
+ fn parse_branch_node(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<BranchNode> {
+ // The order of the mn_lo and mn_hi fields is endian-dependent and would be
+ // reversed in an LMDB environment created on a big-endian system.
+ let mn_lo = cursor.read_u16::<LittleEndian>()?;
+ let mn_hi = cursor.read_u16::<LittleEndian>()?;
+
+ let mn_flags = cursor.read_u16::<LittleEndian>()?;
+
+ // Branch nodes overload the mn_lo, mn_hi, and mn_flags fields to store the page
+ // number, so we derive the number from those fields.
+ let mp_pgno = Self::branch_node_page_num(mn_lo, mn_hi, mn_flags, bits);
+
+ let mn_ksize = cursor.read_u16::<LittleEndian>()?;
+
+ let position = cursor.position();
+ let start = usize::try_from(position)?;
+ let end = usize::try_from(position + u64::from(mn_ksize))?;
+ let mn_data = cursor.get_ref()[start..end].to_vec();
+ cursor.set_position(end as u64);
+
+ Ok(BranchNode {
+ mp_pgno,
+ mn_ksize,
+ mn_data,
+ })
+ }
+
+ fn branch_node_page_num(mn_lo: u16, mn_hi: u16, mn_flags: u16, bits: Bits) -> u64 {
+ let mut page_num = u64::from(u32::from(mn_lo) + (u32::from(mn_hi) << 16));
+ if bits == Bits::U64 {
+ page_num += u64::from(mn_flags) << 32;
+ }
+ page_num
+ }
+
+ fn parse_mp_ptrs(cursor: &mut Cursor<&[u8]>, num_keys: u64) -> MigrateResult<Vec<u16>> {
+ let mut mp_ptrs = Vec::with_capacity(num_keys as usize);
+ for _ in 0..num_keys {
+ mp_ptrs.push(cursor.read_u16::<LittleEndian>()?);
+ }
+ Ok(mp_ptrs)
+ }
+
+ fn num_keys(pb_lower: u16, bits: Bits) -> u64 {
+ (u64::from(pb_lower) - page_header_size(bits)) >> 1
+ }
+}
+
+pub struct Migrator {
+ file: File,
+ bits: Bits,
+}
+
+impl Migrator {
+ /// Create a new Migrator for the LMDB environment at the given path. This tries to
+ /// open the data.mdb file in the environment and determine the bit depth of the
+ /// executable that created it, so it can fail and return an Err if the file can't be
+ /// opened or the depth determined.
+ pub fn new(path: &Path) -> MigrateResult<Migrator> {
+ let mut path = PathBuf::from(path);
+ path.push("data.mdb");
+ let mut file = File::open(&path)?;
+
+ file.seek(SeekFrom::Start(page_header_size(Bits::U32)))?;
+ let mut buf = [0; 4];
+ file.read_exact(&mut buf)?;
+
+ let bits = if buf == MAGIC {
+ Bits::U32
+ } else {
+ file.seek(SeekFrom::Start(page_header_size(Bits::U64)))?;
+ file.read_exact(&mut buf)?;
+ if buf == MAGIC {
+ Bits::U64
+ } else {
+ return Err(MigrateError::IndeterminateBitDepth);
+ }
+ };
+
+ Ok(Migrator { file, bits })
+ }
+
+ /// Dump the data in one of the databases in the LMDB environment. If the `database`
+ /// paremeter is None, then we dump the data in the main database. If it's the name
+ /// of a subdatabase, then we dump the data in that subdatabase.
+ ///
+ /// Note that the output isn't identical to that of the `mdb_dump` utility, since
+ /// `mdb_dump` includes subdatabase key/value pairs when dumping the main database,
+ /// and those values are architecture-dependent, since they contain pointer-sized
+ /// data.
+ ///
+ /// If we wanted to support identical output, we could parameterize inclusion of
+ /// subdatabase pairs in get_pairs() and include them when dumping data, while
+ /// continuing to exclude them when migrating data.
+ pub fn dump<T: Write>(&mut self, database: Option<&str>, mut out: T) -> MigrateResult<()> {
+ let meta_data = self.get_meta_data()?;
+ let root_page_num = meta_data.mm_dbs.main.md_root;
+ let root_page = Rc::new(self.get_page(root_page_num)?);
+
+ let pairs;
+ if let Some(database) = database {
+ let subdbs = self.get_subdbs(root_page)?;
+ let database = subdbs
+ .get(database.as_bytes())
+ .ok_or_else(|| MigrateError::DatabaseNotFound(database.to_string()))?;
+ let root_page_num = database.md_root;
+ let root_page = Rc::new(self.get_page(root_page_num)?);
+ pairs = self.get_pairs(root_page)?;
+ } else {
+ pairs = self.get_pairs(root_page)?;
+ }
+
+ out.write_all(b"VERSION=3\n")?;
+ out.write_all(b"format=bytevalue\n")?;
+ if let Some(database) = database {
+ writeln!(out, "database={database}")?;
+ }
+ out.write_all(b"type=btree\n")?;
+ writeln!(out, "mapsize={}", meta_data.mm_mapsize)?;
+ out.write_all(b"maxreaders=126\n")?;
+ out.write_all(b"db_pagesize=4096\n")?;
+ out.write_all(b"HEADER=END\n")?;
+
+ for (key, value) in pairs {
+ out.write_all(b" ")?;
+ for byte in key {
+ write!(out, "{byte:02x}")?;
+ }
+ out.write_all(b"\n")?;
+ out.write_all(b" ")?;
+ for byte in value {
+ write!(out, "{byte:02x}")?;
+ }
+ out.write_all(b"\n")?;
+ }
+
+ out.write_all(b"DATA=END\n")?;
+
+ Ok(())
+ }
+
+ /// Migrate all data in all of databases in the existing LMDB environment to a new
+ /// environment. This includes all key/value pairs in the main database that aren't
+ /// metadata about subdatabases and all key/value pairs in all subdatabases.
+ ///
+ /// We also set the map size and maximum databases of the new environment to their
+ /// values for the existing environment. But we don't set other metadata, and we
+ /// don't check that the new environment is empty before migrating data.
+ ///
+ /// Thus it's possible for this to overwrite existing data or fail to migrate data if
+ /// the new environment isn't empty. It's the consumer's responsibility to ensure
+ /// that data can be safely migrated to the new environment. In general, this means
+ /// that environment should be empty.
+ pub fn migrate(&mut self, dest: &Path) -> MigrateResult<()> {
+ let meta_data = self.get_meta_data()?;
+ let root_page_num = meta_data.mm_dbs.main.md_root;
+ validate_page_num(root_page_num, self.bits)?;
+ let root_page = Rc::new(self.get_page(root_page_num)?);
+ let subdbs = self.get_subdbs(Rc::clone(&root_page))?;
+
+ let env = Environment::new()
+ .set_map_size(meta_data.mm_mapsize as usize)
+ .set_max_dbs(subdbs.len() as u32)
+ .open(dest)?;
+
+ // Create the databases before we open a read-write transaction, since database
+ // creation requires its own read-write transaction, which would hang while
+ // awaiting completion of an existing one.
+ env.create_db(None, meta_data.mm_dbs.main.md_flags)?;
+ for (subdb_name, subdb_info) in &subdbs {
+ env.create_db(Some(str::from_utf8(subdb_name)?), subdb_info.md_flags)?;
+ }
+
+ // Now open the read-write transaction that we'll use to migrate all the data.
+ let mut txn = env.begin_rw_txn()?;
+
+ // Migrate the main database.
+ let pairs = self.get_pairs(root_page)?;
+ let db = env.open_db(None)?;
+ for (key, value) in pairs {
+ // If we knew that the target database was empty, we could specify
+ // WriteFlags::APPEND to speed up the migration.
+ txn.put(db, &key, &value, WriteFlags::empty())?;
+ }
+
+ // Migrate subdatabases.
+ for (subdb_name, subdb_info) in &subdbs {
+ let root_page = Rc::new(self.get_page(subdb_info.md_root)?);
+ let pairs = self.get_pairs(root_page)?;
+ let db = env.open_db(Some(str::from_utf8(subdb_name)?))?;
+ for (key, value) in pairs {
+ // If we knew that the target database was empty, we could specify
+ // WriteFlags::APPEND to speed up the migration.
+ txn.put(db, &key, &value, WriteFlags::empty())?;
+ }
+ }
+
+ txn.commit()?;
+
+ Ok(())
+ }
+
+ fn get_subdbs(&mut self, root_page: Rc<Page>) -> MigrateResult<HashMap<Vec<u8>, Database>> {
+ let mut subdbs = HashMap::new();
+ let mut pages = vec![root_page];
+
+ while let Some(page) = pages.pop() {
+ match &*page {
+ Page::BRANCH(nodes) => {
+ for branch in nodes {
+ pages.push(Rc::new(self.get_page(branch.mp_pgno)?));
+ }
+ }
+ Page::LEAF(nodes) => {
+ for leaf in nodes {
+ if let LeafNode::SubData { key, db, .. } = leaf {
+ subdbs.insert(key.to_vec(), db.clone());
+ };
+ }
+ }
+ _ => {
+ return Err(MigrateError::UnexpectedPageVariant);
+ }
+ }
+ }
+
+ Ok(subdbs)
+ }
+
+ fn get_pairs(&mut self, root_page: Rc<Page>) -> MigrateResult<BTreeMap<Vec<u8>, Vec<u8>>> {
+ let mut pairs = BTreeMap::new();
+ let mut pages = vec![root_page];
+
+ while let Some(page) = pages.pop() {
+ match &*page {
+ Page::BRANCH(nodes) => {
+ for branch in nodes {
+ pages.push(Rc::new(self.get_page(branch.mp_pgno)?));
+ }
+ }
+ Page::LEAF(nodes) => {
+ for leaf in nodes {
+ match leaf {
+ LeafNode::Regular { key, value, .. } => {
+ pairs.insert(key.to_vec(), value.to_vec());
+ }
+ LeafNode::BigData {
+ mv_size,
+ key,
+ overflow_pgno,
+ ..
+ } => {
+ // Perhaps we could reduce memory consumption during a
+ // migration by waiting to read big data until it's time
+ // to write it to the new database.
+ let value = self.read_data(
+ *overflow_pgno * u64::from(PAGESIZE)
+ + page_header_size(self.bits),
+ *mv_size as usize,
+ )?;
+ pairs.insert(key.to_vec(), value);
+ }
+ LeafNode::SubData { .. } => {
+ // We don't include subdatabase leaves in pairs, since
+ // there's no architecture-neutral representation of them,
+ // and in any case they're meta-data that should get
+ // recreated when we migrate the subdatabases themselves.
+ //
+ // If we wanted to create identical dumps to those
+ // produced by `mdb_dump`, however, we could allow
+ // consumers to specify that they'd like to include these
+ // records.
+ }
+ };
+ }
+ }
+ _ => {
+ return Err(MigrateError::UnexpectedPageVariant);
+ }
+ }
+ }
+
+ Ok(pairs)
+ }
+
+ fn read_data(&mut self, offset: u64, size: usize) -> MigrateResult<Vec<u8>> {
+ self.file.seek(SeekFrom::Start(offset))?;
+ let mut buf: Vec<u8> = vec![0; size];
+ self.file.read_exact(&mut buf[0..size])?;
+ Ok(buf.to_vec())
+ }
+
+ fn get_page(&mut self, page_no: u64) -> MigrateResult<Page> {
+ Page::new(
+ self.read_data(page_no * u64::from(PAGESIZE), usize::from(PAGESIZE))?,
+ self.bits,
+ )
+ }
+
+ fn get_meta_data(&mut self) -> MigrateResult<MetaData> {
+ let (page0, page1) = (self.get_page(0)?, self.get_page(1)?);
+
+ match (page0, page1) {
+ (Page::META(meta0), Page::META(meta1)) => {
+ let meta = if meta1.mm_txnid > meta0.mm_txnid {
+ meta1
+ } else {
+ meta0
+ };
+ if meta.mm_magic != 0xBE_EF_C0_DE {
+ return Err(MigrateError::InvalidMagicNum);
+ }
+ if meta.mm_version != 1 && meta.mm_version != 999 {
+ return Err(MigrateError::InvalidDataVersion);
+ }
+ Ok(meta)
+ }
+ _ => Err(MigrateError::UnexpectedPageVariant),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ use std::{env, fs, mem::size_of};
+
+ use lmdb::{Environment, Error as LmdbError};
+ use tempfile::{tempdir, tempfile};
+
+ fn compare_files(ref_file: &mut File, new_file: &mut File) -> MigrateResult<()> {
+ ref_file.seek(SeekFrom::Start(0))?;
+ new_file.seek(SeekFrom::Start(0))?;
+
+ let ref_buf = &mut [0; 1024];
+ let new_buf = &mut [0; 1024];
+
+ loop {
+ match ref_file.read(ref_buf) {
+ Err(err) => panic!("{}", err),
+ Ok(ref_len) => match new_file.read(new_buf) {
+ Err(err) => panic!("{}", err),
+ Ok(new_len) => {
+ assert_eq!(ref_len, new_len);
+ if ref_len == 0 {
+ break;
+ };
+ assert_eq!(ref_buf[0..ref_len], new_buf[0..new_len]);
+ }
+ },
+ }
+ }
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_dump_32() -> MigrateResult<()> {
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_32"].iter().collect();
+
+ // Dump data from the test env to a new dump file.
+ let mut migrator = Migrator::new(&test_env_path)?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(None, &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump.txt"].iter().collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_dump_32_subdb() -> MigrateResult<()> {
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_32"].iter().collect();
+
+ // Dump data from the test env to a new dump file.
+ let mut migrator = Migrator::new(&test_env_path)?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(Some("subdb"), &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"]
+ .iter()
+ .collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_dump_64() -> MigrateResult<()> {
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_64"].iter().collect();
+
+ // Dump data from the test env to a new dump file.
+ let mut migrator = Migrator::new(&test_env_path)?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(None, &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump.txt"].iter().collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_dump_64_subdb() -> MigrateResult<()> {
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_64"].iter().collect();
+
+ // Dump data from the test env to a new dump file.
+ let mut migrator = Migrator::new(&test_env_path)?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(Some("subdb"), &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"]
+ .iter()
+ .collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_migrate_64() -> MigrateResult<()> {
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_64"].iter().collect();
+
+ // Migrate data from the old env to a new one.
+ let new_env = tempdir()?;
+ let mut migrator = Migrator::new(&test_env_path)?;
+ migrator.migrate(new_env.path())?;
+
+ // Dump data from the new env to a new dump file.
+ let mut migrator = Migrator::new(new_env.path())?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(Some("subdb"), &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"]
+ .iter()
+ .collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_migrate_32() -> MigrateResult<()> {
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_32"].iter().collect();
+
+ // Migrate data from the old env to a new one.
+ let new_env = tempdir()?;
+ let mut migrator = Migrator::new(&test_env_path)?;
+ migrator.migrate(new_env.path())?;
+
+ // Dump data from the new env to a new dump file.
+ let mut migrator = Migrator::new(new_env.path())?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(Some("subdb"), &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"]
+ .iter()
+ .collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ Ok(())
+ }
+
+ #[test]
+ fn test_migrate_and_replace() -> MigrateResult<()> {
+ let test_env_name = match size_of::<usize>() {
+ 4 => "ref_env_64",
+ 8 => "ref_env_32",
+ _ => panic!("only 32- and 64-bit depths are supported"),
+ };
+
+ let cwd = env::current_dir()?;
+ let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?;
+ let test_env_path: PathBuf = [cwd, "tests", "envs", test_env_name].iter().collect();
+
+ let old_env = tempdir()?;
+ fs::copy(
+ test_env_path.join("data.mdb"),
+ old_env.path().join("data.mdb"),
+ )?;
+ fs::copy(
+ test_env_path.join("lock.mdb"),
+ old_env.path().join("lock.mdb"),
+ )?;
+
+ // Confirm that it isn't possible to open the old environment with LMDB.
+ assert_eq!(
+ match Environment::new().open(old_env.path()) {
+ Err(err) => err,
+ _ => panic!("opening the environment should have failed"),
+ },
+ LmdbError::Invalid
+ );
+
+ // Migrate data from the old env to a new one.
+ let new_env = tempdir()?;
+ let mut migrator = Migrator::new(old_env.path())?;
+ migrator.migrate(new_env.path())?;
+
+ // Dump data from the new env to a new dump file.
+ let mut migrator = Migrator::new(new_env.path())?;
+ let mut new_dump_file = tempfile()?;
+ migrator.dump(Some("subdb"), &new_dump_file)?;
+
+ // Open the reference dump file.
+ let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"]
+ .iter()
+ .collect();
+ let mut ref_dump_file = File::open(ref_dump_file_path)?;
+
+ // Compare the new dump file to the reference dump file.
+ compare_files(&mut ref_dump_file, &mut new_dump_file)?;
+
+ // Overwrite the old env's files with the new env's files and confirm that it's now
+ // possible to open the old env with LMDB.
+ fs::copy(
+ new_env.path().join("data.mdb"),
+ old_env.path().join("data.mdb"),
+ )?;
+ fs::copy(
+ new_env.path().join("lock.mdb"),
+ old_env.path().join("lock.mdb"),
+ )?;
+ assert!(Environment::new().open(old_env.path()).is_ok());
+
+ Ok(())
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs
new file mode 100644
index 0000000000..e23bb49770
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs
@@ -0,0 +1,79 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{io, num, str};
+
+use thiserror::Error;
+
+#[derive(Debug, Error)]
+pub enum MigrateError {
+ #[error("database not found: {0:?}")]
+ DatabaseNotFound(String),
+
+ #[error("{0}")]
+ FromString(String),
+
+ #[error("couldn't determine bit depth")]
+ IndeterminateBitDepth,
+
+ #[error("I/O error: {0:?}")]
+ IoError(#[from] io::Error),
+
+ #[error("invalid DatabaseFlags bits")]
+ InvalidDatabaseBits,
+
+ #[error("invalid data version")]
+ InvalidDataVersion,
+
+ #[error("invalid magic number")]
+ InvalidMagicNum,
+
+ #[error("invalid NodeFlags bits")]
+ InvalidNodeBits,
+
+ #[error("invalid PageFlags bits")]
+ InvalidPageBits,
+
+ #[error("invalid page number")]
+ InvalidPageNum,
+
+ #[error("lmdb backend error: {0}")]
+ LmdbError(#[from] lmdb::Error),
+
+ #[error("string conversion error")]
+ StringConversionError,
+
+ #[error("TryFromInt error: {0:?}")]
+ TryFromIntError(#[from] num::TryFromIntError),
+
+ #[error("unexpected Page variant")]
+ UnexpectedPageVariant,
+
+ #[error("unexpected PageHeader variant")]
+ UnexpectedPageHeaderVariant,
+
+ #[error("unsupported PageHeader variant")]
+ UnsupportedPageHeaderVariant,
+
+ #[error("UTF8 error: {0:?}")]
+ Utf8Error(#[from] str::Utf8Error),
+}
+
+impl From<&str> for MigrateError {
+ fn from(e: &str) -> MigrateError {
+ MigrateError::FromString(e.to_string())
+ }
+}
+
+impl From<String> for MigrateError {
+ fn from(e: String) -> MigrateError {
+ MigrateError::FromString(e)
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs b/third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs
new file mode 100644
index 0000000000..760abce451
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs
@@ -0,0 +1,69 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use lmdb::Cursor;
+
+use super::IterImpl;
+use crate::backend::traits::BackendRoCursor;
+
+#[derive(Debug)]
+pub struct RoCursorImpl<'c>(pub(crate) lmdb::RoCursor<'c>);
+
+impl<'c> BackendRoCursor<'c> for RoCursorImpl<'c> {
+ type Iter = IterImpl<'c, lmdb::RoCursor<'c>>;
+
+ fn into_iter(self) -> Self::Iter {
+ // We call RoCursor.iter() instead of RoCursor.iter_start() because
+ // the latter panics when there are no items in the store, whereas the
+ // former returns an iterator that yields no items. And since we create
+ // the Cursor and don't change its position, we can be sure that a call
+ // to Cursor.iter() will start at the beginning.
+ IterImpl::new(self.0, lmdb::RoCursor::iter)
+ }
+
+ fn into_iter_from<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ IterImpl::new(self.0, |cursor| cursor.iter_from(key))
+ }
+
+ fn into_iter_dup_of<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ IterImpl::new(self.0, |cursor| cursor.iter_dup_of(key))
+ }
+}
+
+#[derive(Debug)]
+pub struct RwCursorImpl<'c>(pub(crate) lmdb::RwCursor<'c>);
+
+impl<'c> BackendRoCursor<'c> for RwCursorImpl<'c> {
+ type Iter = IterImpl<'c, lmdb::RwCursor<'c>>;
+
+ fn into_iter(self) -> Self::Iter {
+ IterImpl::new(self.0, lmdb::RwCursor::iter)
+ }
+
+ fn into_iter_from<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ IterImpl::new(self.0, |cursor| cursor.iter_from(key))
+ }
+
+ fn into_iter_dup_of<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ IterImpl::new(self.0, |cursor| cursor.iter_dup_of(key))
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/database.rs b/third_party/rust/rkv/src/backend/impl_lmdb/database.rs
new file mode 100644
index 0000000000..8edee5c2c3
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/database.rs
@@ -0,0 +1,16 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendDatabase;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub struct DatabaseImpl(pub(crate) lmdb::Database);
+
+impl BackendDatabase for DatabaseImpl {}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/environment.rs b/third_party/rust/rkv/src/backend/impl_lmdb/environment.rs
new file mode 100644
index 0000000000..e528fad057
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/environment.rs
@@ -0,0 +1,299 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ fs,
+ path::{Path, PathBuf},
+};
+
+use lmdb::Error as LmdbError;
+
+use super::{
+ DatabaseFlagsImpl, DatabaseImpl, EnvironmentFlagsImpl, ErrorImpl, InfoImpl, RoTransactionImpl,
+ RwTransactionImpl, StatImpl,
+};
+use crate::backend::traits::{
+ BackendEnvironment, BackendEnvironmentBuilder, BackendInfo, BackendIter, BackendRoCursor,
+ BackendRoCursorTransaction, BackendStat,
+};
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub struct EnvironmentBuilderImpl {
+ builder: lmdb::EnvironmentBuilder,
+ env_path_type: EnvironmentPathType,
+ env_lock_type: EnvironmentLockType,
+ env_db_type: EnvironmentDefaultDbType,
+ make_dir_if_needed: bool,
+}
+
+impl<'b> BackendEnvironmentBuilder<'b> for EnvironmentBuilderImpl {
+ type Environment = EnvironmentImpl;
+ type Error = ErrorImpl;
+ type Flags = EnvironmentFlagsImpl;
+
+ fn new() -> EnvironmentBuilderImpl {
+ EnvironmentBuilderImpl {
+ builder: lmdb::Environment::new(),
+ env_path_type: EnvironmentPathType::SubDir,
+ env_lock_type: EnvironmentLockType::Lockfile,
+ env_db_type: EnvironmentDefaultDbType::SingleDatabase,
+ make_dir_if_needed: false,
+ }
+ }
+
+ fn set_flags<T>(&mut self, flags: T) -> &mut Self
+ where
+ T: Into<Self::Flags>,
+ {
+ let flags = flags.into();
+ if flags.0 == lmdb::EnvironmentFlags::NO_SUB_DIR {
+ self.env_path_type = EnvironmentPathType::NoSubDir;
+ }
+ if flags.0 == lmdb::EnvironmentFlags::NO_LOCK {
+ self.env_lock_type = EnvironmentLockType::NoLockfile;
+ }
+ self.builder.set_flags(flags.0);
+ self
+ }
+
+ fn set_max_readers(&mut self, max_readers: u32) -> &mut Self {
+ self.builder.set_max_readers(max_readers);
+ self
+ }
+
+ fn set_max_dbs(&mut self, max_dbs: u32) -> &mut Self {
+ if max_dbs > 0 {
+ self.env_db_type = EnvironmentDefaultDbType::MultipleNamedDatabases
+ }
+ self.builder.set_max_dbs(max_dbs);
+ self
+ }
+
+ fn set_map_size(&mut self, size: usize) -> &mut Self {
+ self.builder.set_map_size(size);
+ self
+ }
+
+ fn set_make_dir_if_needed(&mut self, make_dir_if_needed: bool) -> &mut Self {
+ self.make_dir_if_needed = make_dir_if_needed;
+ self
+ }
+
+ fn set_discard_if_corrupted(&mut self, _discard_if_corrupted: bool) -> &mut Self {
+ // Unfortunately, when opening a database, LMDB doesn't handle all the ways it could have
+ // been corrupted. Prefer using the `SafeMode` backend if this is important.
+ unimplemented!();
+ }
+
+ fn open(&self, path: &Path) -> Result<Self::Environment, Self::Error> {
+ match self.env_path_type {
+ EnvironmentPathType::NoSubDir => {
+ if !path.is_file() {
+ return Err(ErrorImpl::UnsuitableEnvironmentPath(path.into()));
+ }
+ }
+ EnvironmentPathType::SubDir => {
+ if !path.is_dir() {
+ if !self.make_dir_if_needed {
+ return Err(ErrorImpl::UnsuitableEnvironmentPath(path.into()));
+ }
+ fs::create_dir_all(path)?;
+ }
+ }
+ }
+
+ self.builder
+ .open(path)
+ .map_err(ErrorImpl::LmdbError)
+ .and_then(|lmdbenv| {
+ EnvironmentImpl::new(
+ path,
+ self.env_path_type,
+ self.env_lock_type,
+ self.env_db_type,
+ lmdbenv,
+ )
+ })
+ }
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum EnvironmentPathType {
+ SubDir,
+ NoSubDir,
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum EnvironmentLockType {
+ Lockfile,
+ NoLockfile,
+}
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub enum EnvironmentDefaultDbType {
+ SingleDatabase,
+ MultipleNamedDatabases,
+}
+
+#[derive(Debug)]
+pub struct EnvironmentImpl {
+ path: PathBuf,
+ env_path_type: EnvironmentPathType,
+ env_lock_type: EnvironmentLockType,
+ env_db_type: EnvironmentDefaultDbType,
+ lmdbenv: lmdb::Environment,
+}
+
+impl EnvironmentImpl {
+ pub(crate) fn new(
+ path: &Path,
+ env_path_type: EnvironmentPathType,
+ env_lock_type: EnvironmentLockType,
+ env_db_type: EnvironmentDefaultDbType,
+ lmdbenv: lmdb::Environment,
+ ) -> Result<EnvironmentImpl, ErrorImpl> {
+ Ok(EnvironmentImpl {
+ path: path.to_path_buf(),
+ env_path_type,
+ env_lock_type,
+ env_db_type,
+ lmdbenv,
+ })
+ }
+}
+
+impl<'e> BackendEnvironment<'e> for EnvironmentImpl {
+ type Database = DatabaseImpl;
+ type Error = ErrorImpl;
+ type Flags = DatabaseFlagsImpl;
+ type Info = InfoImpl;
+ type RoTransaction = RoTransactionImpl<'e>;
+ type RwTransaction = RwTransactionImpl<'e>;
+ type Stat = StatImpl;
+
+ fn get_dbs(&self) -> Result<Vec<Option<String>>, Self::Error> {
+ if self.env_db_type == EnvironmentDefaultDbType::SingleDatabase {
+ return Ok(vec![None]);
+ }
+ let db = self
+ .lmdbenv
+ .open_db(None)
+ .map(DatabaseImpl)
+ .map_err(ErrorImpl::LmdbError)?;
+ let reader = self.begin_ro_txn()?;
+ let cursor = reader.open_ro_cursor(&db)?;
+ let mut iter = cursor.into_iter();
+ let mut store = vec![];
+ while let Some(result) = iter.next() {
+ let (key, _) = result?;
+ let name = String::from_utf8(key.to_owned())
+ .map_err(|_| ErrorImpl::LmdbError(lmdb::Error::Corrupted))?;
+ store.push(Some(name));
+ }
+ Ok(store)
+ }
+
+ fn open_db(&self, name: Option<&str>) -> Result<Self::Database, Self::Error> {
+ self.lmdbenv
+ .open_db(name)
+ .map(DatabaseImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn create_db(
+ &self,
+ name: Option<&str>,
+ flags: Self::Flags,
+ ) -> Result<Self::Database, Self::Error> {
+ self.lmdbenv
+ .create_db(name, flags.0)
+ .map(DatabaseImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn begin_ro_txn(&'e self) -> Result<Self::RoTransaction, Self::Error> {
+ self.lmdbenv
+ .begin_ro_txn()
+ .map(RoTransactionImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn begin_rw_txn(&'e self) -> Result<Self::RwTransaction, Self::Error> {
+ self.lmdbenv
+ .begin_rw_txn()
+ .map(RwTransactionImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn sync(&self, force: bool) -> Result<(), Self::Error> {
+ self.lmdbenv.sync(force).map_err(ErrorImpl::LmdbError)
+ }
+
+ fn stat(&self) -> Result<Self::Stat, Self::Error> {
+ self.lmdbenv
+ .stat()
+ .map(StatImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn info(&self) -> Result<Self::Info, Self::Error> {
+ self.lmdbenv
+ .info()
+ .map(InfoImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn freelist(&self) -> Result<usize, Self::Error> {
+ self.lmdbenv.freelist().map_err(ErrorImpl::LmdbError)
+ }
+
+ fn load_ratio(&self) -> Result<Option<f32>, Self::Error> {
+ let stat = self.stat()?;
+ let info = self.info()?;
+ let freelist = self.freelist()?;
+
+ let last_pgno = info.last_pgno() + 1; // pgno is 0 based.
+ let total_pgs = info.map_size() / stat.page_size();
+ if freelist > last_pgno {
+ return Err(ErrorImpl::LmdbError(LmdbError::Corrupted));
+ }
+ let used_pgs = last_pgno - freelist;
+ Ok(Some(used_pgs as f32 / total_pgs as f32))
+ }
+
+ fn set_map_size(&self, size: usize) -> Result<(), Self::Error> {
+ self.lmdbenv
+ .set_map_size(size)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ fn get_files_on_disk(&self) -> Vec<PathBuf> {
+ let mut store = vec![];
+
+ if self.env_path_type == EnvironmentPathType::NoSubDir {
+ // The option NO_SUB_DIR could change the default directory layout; therefore this should
+ // probably return the path used to create environment, along with the custom lockfile
+ // when available.
+ unimplemented!();
+ }
+
+ let mut db_filename = self.path.clone();
+ db_filename.push("data.mdb");
+ store.push(db_filename);
+
+ if self.env_lock_type == EnvironmentLockType::Lockfile {
+ let mut lock_filename = self.path.clone();
+ lock_filename.push("lock.mdb");
+ store.push(lock_filename);
+ }
+
+ store
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/error.rs b/third_party/rust/rkv/src/backend/impl_lmdb/error.rs
new file mode 100644
index 0000000000..a2fd8a7859
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/error.rs
@@ -0,0 +1,57 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{fmt, io, path::PathBuf};
+
+use crate::{backend::traits::BackendError, error::StoreError};
+
+#[derive(Debug)]
+pub enum ErrorImpl {
+ LmdbError(lmdb::Error),
+ UnsuitableEnvironmentPath(PathBuf),
+ IoError(io::Error),
+}
+
+impl BackendError for ErrorImpl {}
+
+impl fmt::Display for ErrorImpl {
+ fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ ErrorImpl::LmdbError(e) => e.fmt(fmt),
+ ErrorImpl::UnsuitableEnvironmentPath(_) => write!(fmt, "UnsuitableEnvironmentPath"),
+ ErrorImpl::IoError(e) => e.fmt(fmt),
+ }
+ }
+}
+
+impl Into<StoreError> for ErrorImpl {
+ fn into(self) -> StoreError {
+ match self {
+ ErrorImpl::LmdbError(lmdb::Error::Corrupted) => StoreError::DatabaseCorrupted,
+ ErrorImpl::LmdbError(lmdb::Error::NotFound) => StoreError::KeyValuePairNotFound,
+ ErrorImpl::LmdbError(lmdb::Error::BadValSize) => StoreError::KeyValuePairBadSize,
+ ErrorImpl::LmdbError(lmdb::Error::Invalid) => StoreError::FileInvalid,
+ ErrorImpl::LmdbError(lmdb::Error::MapFull) => StoreError::MapFull,
+ ErrorImpl::LmdbError(lmdb::Error::DbsFull) => StoreError::DbsFull,
+ ErrorImpl::LmdbError(lmdb::Error::ReadersFull) => StoreError::ReadersFull,
+ ErrorImpl::LmdbError(error) => StoreError::LmdbError(error),
+ ErrorImpl::UnsuitableEnvironmentPath(path) => {
+ StoreError::UnsuitableEnvironmentPath(path)
+ }
+ ErrorImpl::IoError(error) => StoreError::IoError(error),
+ }
+ }
+}
+
+impl From<io::Error> for ErrorImpl {
+ fn from(e: io::Error) -> ErrorImpl {
+ ErrorImpl::IoError(e)
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/flags.rs b/third_party/rust/rkv/src/backend/impl_lmdb/flags.rs
new file mode 100644
index 0000000000..7a16bb7c46
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/flags.rs
@@ -0,0 +1,123 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::{
+ common::{DatabaseFlags, EnvironmentFlags, WriteFlags},
+ traits::{BackendDatabaseFlags, BackendEnvironmentFlags, BackendFlags, BackendWriteFlags},
+};
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone, Default)]
+pub struct EnvironmentFlagsImpl(pub(crate) lmdb::EnvironmentFlags);
+
+impl BackendFlags for EnvironmentFlagsImpl {
+ fn empty() -> EnvironmentFlagsImpl {
+ EnvironmentFlagsImpl(lmdb::EnvironmentFlags::empty())
+ }
+}
+
+impl BackendEnvironmentFlags for EnvironmentFlagsImpl {
+ fn set(&mut self, flag: EnvironmentFlags, value: bool) {
+ self.0.set(flag.into(), value)
+ }
+}
+
+impl Into<EnvironmentFlagsImpl> for EnvironmentFlags {
+ fn into(self) -> EnvironmentFlagsImpl {
+ EnvironmentFlagsImpl(self.into())
+ }
+}
+
+impl Into<lmdb::EnvironmentFlags> for EnvironmentFlags {
+ fn into(self) -> lmdb::EnvironmentFlags {
+ match self {
+ EnvironmentFlags::FIXED_MAP => lmdb::EnvironmentFlags::FIXED_MAP,
+ EnvironmentFlags::NO_SUB_DIR => lmdb::EnvironmentFlags::NO_SUB_DIR,
+ EnvironmentFlags::WRITE_MAP => lmdb::EnvironmentFlags::WRITE_MAP,
+ EnvironmentFlags::READ_ONLY => lmdb::EnvironmentFlags::READ_ONLY,
+ EnvironmentFlags::NO_META_SYNC => lmdb::EnvironmentFlags::NO_META_SYNC,
+ EnvironmentFlags::NO_SYNC => lmdb::EnvironmentFlags::NO_SYNC,
+ EnvironmentFlags::MAP_ASYNC => lmdb::EnvironmentFlags::MAP_ASYNC,
+ EnvironmentFlags::NO_TLS => lmdb::EnvironmentFlags::NO_TLS,
+ EnvironmentFlags::NO_LOCK => lmdb::EnvironmentFlags::NO_LOCK,
+ EnvironmentFlags::NO_READAHEAD => lmdb::EnvironmentFlags::NO_READAHEAD,
+ EnvironmentFlags::NO_MEM_INIT => lmdb::EnvironmentFlags::NO_MEM_INIT,
+ }
+ }
+}
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone, Default)]
+pub struct DatabaseFlagsImpl(pub(crate) lmdb::DatabaseFlags);
+
+impl BackendFlags for DatabaseFlagsImpl {
+ fn empty() -> DatabaseFlagsImpl {
+ DatabaseFlagsImpl(lmdb::DatabaseFlags::empty())
+ }
+}
+
+impl BackendDatabaseFlags for DatabaseFlagsImpl {
+ fn set(&mut self, flag: DatabaseFlags, value: bool) {
+ self.0.set(flag.into(), value)
+ }
+}
+
+impl Into<DatabaseFlagsImpl> for DatabaseFlags {
+ fn into(self) -> DatabaseFlagsImpl {
+ DatabaseFlagsImpl(self.into())
+ }
+}
+
+impl Into<lmdb::DatabaseFlags> for DatabaseFlags {
+ fn into(self) -> lmdb::DatabaseFlags {
+ match self {
+ DatabaseFlags::REVERSE_KEY => lmdb::DatabaseFlags::REVERSE_KEY,
+ #[cfg(feature = "db-dup-sort")]
+ DatabaseFlags::DUP_SORT => lmdb::DatabaseFlags::DUP_SORT,
+ #[cfg(feature = "db-dup-sort")]
+ DatabaseFlags::DUP_FIXED => lmdb::DatabaseFlags::DUP_FIXED,
+ #[cfg(feature = "db-int-key")]
+ DatabaseFlags::INTEGER_KEY => lmdb::DatabaseFlags::INTEGER_KEY,
+ DatabaseFlags::INTEGER_DUP => lmdb::DatabaseFlags::INTEGER_DUP,
+ DatabaseFlags::REVERSE_DUP => lmdb::DatabaseFlags::REVERSE_DUP,
+ }
+ }
+}
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone, Default)]
+pub struct WriteFlagsImpl(pub(crate) lmdb::WriteFlags);
+
+impl BackendFlags for WriteFlagsImpl {
+ fn empty() -> WriteFlagsImpl {
+ WriteFlagsImpl(lmdb::WriteFlags::empty())
+ }
+}
+
+impl BackendWriteFlags for WriteFlagsImpl {
+ fn set(&mut self, flag: WriteFlags, value: bool) {
+ self.0.set(flag.into(), value)
+ }
+}
+
+impl Into<WriteFlagsImpl> for WriteFlags {
+ fn into(self) -> WriteFlagsImpl {
+ WriteFlagsImpl(self.into())
+ }
+}
+
+impl Into<lmdb::WriteFlags> for WriteFlags {
+ fn into(self) -> lmdb::WriteFlags {
+ match self {
+ WriteFlags::NO_OVERWRITE => lmdb::WriteFlags::NO_OVERWRITE,
+ WriteFlags::NO_DUP_DATA => lmdb::WriteFlags::NO_DUP_DATA,
+ WriteFlags::CURRENT => lmdb::WriteFlags::CURRENT,
+ WriteFlags::APPEND => lmdb::WriteFlags::APPEND,
+ WriteFlags::APPEND_DUP => lmdb::WriteFlags::APPEND_DUP,
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/info.rs b/third_party/rust/rkv/src/backend/impl_lmdb/info.rs
new file mode 100644
index 0000000000..6188065c07
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/info.rs
@@ -0,0 +1,35 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendInfo;
+
+pub struct InfoImpl(pub(crate) lmdb::Info);
+
+impl BackendInfo for InfoImpl {
+ fn map_size(&self) -> usize {
+ self.0.map_size()
+ }
+
+ fn last_pgno(&self) -> usize {
+ self.0.last_pgno()
+ }
+
+ fn last_txnid(&self) -> usize {
+ self.0.last_txnid()
+ }
+
+ fn max_readers(&self) -> usize {
+ self.0.max_readers() as usize
+ }
+
+ fn num_readers(&self) -> usize {
+ self.0.num_readers() as usize
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/iter.rs b/third_party/rust/rkv/src/backend/impl_lmdb/iter.rs
new file mode 100644
index 0000000000..519d361d4f
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/iter.rs
@@ -0,0 +1,41 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use super::ErrorImpl;
+use crate::backend::traits::BackendIter;
+
+pub struct IterImpl<'i, C> {
+ // LMDB semantics dictate that a cursor must be valid for the entire lifetime
+ // of an iterator. In other words, cursors must not be dropped while an
+ // iterator built from it is alive. Unfortunately, the LMDB crate API does
+ // not express this through the type system, so we must enforce it somehow.
+ #[allow(dead_code)]
+ cursor: C,
+ iter: lmdb::Iter<'i>,
+}
+
+impl<'i, C> IterImpl<'i, C> {
+ pub(crate) fn new(
+ mut cursor: C,
+ to_iter: impl FnOnce(&mut C) -> lmdb::Iter<'i>,
+ ) -> IterImpl<'i, C> {
+ let iter = to_iter(&mut cursor);
+ IterImpl { cursor, iter }
+ }
+}
+
+impl<'i, C> BackendIter<'i> for IterImpl<'i, C> {
+ type Error = ErrorImpl;
+
+ #[allow(clippy::type_complexity)]
+ fn next(&mut self) -> Option<Result<(&'i [u8], &'i [u8]), Self::Error>> {
+ self.iter.next().map(|e| e.map_err(ErrorImpl::LmdbError))
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/stat.rs b/third_party/rust/rkv/src/backend/impl_lmdb/stat.rs
new file mode 100644
index 0000000000..b0de8c5051
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/stat.rs
@@ -0,0 +1,39 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendStat;
+
+pub struct StatImpl(pub(crate) lmdb::Stat);
+
+impl BackendStat for StatImpl {
+ fn page_size(&self) -> usize {
+ self.0.page_size() as usize
+ }
+
+ fn depth(&self) -> usize {
+ self.0.depth() as usize
+ }
+
+ fn branch_pages(&self) -> usize {
+ self.0.branch_pages()
+ }
+
+ fn leaf_pages(&self) -> usize {
+ self.0.leaf_pages()
+ }
+
+ fn overflow_pages(&self) -> usize {
+ self.0.overflow_pages()
+ }
+
+ fn entries(&self) -> usize {
+ self.0.entries()
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs b/third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs
new file mode 100644
index 0000000000..21752763fd
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs
@@ -0,0 +1,107 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use lmdb::Transaction;
+
+use super::{DatabaseImpl, ErrorImpl, RoCursorImpl, WriteFlagsImpl};
+use crate::backend::traits::{
+ BackendRoCursorTransaction, BackendRoTransaction, BackendRwCursorTransaction,
+ BackendRwTransaction,
+};
+
+#[derive(Debug)]
+pub struct RoTransactionImpl<'t>(pub(crate) lmdb::RoTransaction<'t>);
+
+impl<'t> BackendRoTransaction for RoTransactionImpl<'t> {
+ type Database = DatabaseImpl;
+ type Error = ErrorImpl;
+
+ fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> {
+ self.0.get(db.0, &key).map_err(ErrorImpl::LmdbError)
+ }
+
+ fn abort(self) {
+ self.0.abort()
+ }
+}
+
+impl<'t> BackendRoCursorTransaction<'t> for RoTransactionImpl<'t> {
+ type RoCursor = RoCursorImpl<'t>;
+
+ fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> {
+ self.0
+ .open_ro_cursor(db.0)
+ .map(RoCursorImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+}
+
+#[derive(Debug)]
+pub struct RwTransactionImpl<'t>(pub(crate) lmdb::RwTransaction<'t>);
+
+impl<'t> BackendRwTransaction for RwTransactionImpl<'t> {
+ type Database = DatabaseImpl;
+ type Error = ErrorImpl;
+ type Flags = WriteFlagsImpl;
+
+ fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> {
+ self.0.get(db.0, &key).map_err(ErrorImpl::LmdbError)
+ }
+
+ fn put(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: &[u8],
+ flags: Self::Flags,
+ ) -> Result<(), Self::Error> {
+ self.0
+ .put(db.0, &key, &value, flags.0)
+ .map_err(ErrorImpl::LmdbError)
+ }
+
+ #[cfg(not(feature = "db-dup-sort"))]
+ fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error> {
+ self.0.del(db.0, &key, None).map_err(ErrorImpl::LmdbError)
+ }
+
+ #[cfg(feature = "db-dup-sort")]
+ fn del(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: Option<&[u8]>,
+ ) -> Result<(), Self::Error> {
+ self.0.del(db.0, &key, value).map_err(ErrorImpl::LmdbError)
+ }
+
+ fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error> {
+ self.0.clear_db(db.0).map_err(ErrorImpl::LmdbError)
+ }
+
+ fn commit(self) -> Result<(), Self::Error> {
+ self.0.commit().map_err(ErrorImpl::LmdbError)
+ }
+
+ fn abort(self) {
+ self.0.abort()
+ }
+}
+
+impl<'t> BackendRwCursorTransaction<'t> for RwTransactionImpl<'t> {
+ type RoCursor = RoCursorImpl<'t>;
+
+ fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> {
+ self.0
+ .open_ro_cursor(db.0)
+ .map(RoCursorImpl)
+ .map_err(ErrorImpl::LmdbError)
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe.rs b/third_party/rust/rkv/src/backend/impl_safe.rs
new file mode 100644
index 0000000000..66ca012973
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe.rs
@@ -0,0 +1,30 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod cursor;
+mod database;
+mod environment;
+mod error;
+mod flags;
+mod info;
+mod iter;
+mod snapshot;
+mod stat;
+mod transaction;
+
+pub use cursor::{RoCursorImpl, RwCursorImpl};
+pub use database::DatabaseImpl;
+pub use environment::{EnvironmentBuilderImpl, EnvironmentImpl};
+pub use error::ErrorImpl;
+pub use flags::{DatabaseFlagsImpl, EnvironmentFlagsImpl, WriteFlagsImpl};
+pub use info::InfoImpl;
+pub use iter::IterImpl;
+pub use stat::StatImpl;
+pub use transaction::{RoTransactionImpl, RwTransactionImpl};
diff --git a/third_party/rust/rkv/src/backend/impl_safe/cursor.rs b/third_party/rust/rkv/src/backend/impl_safe/cursor.rs
new file mode 100644
index 0000000000..c3bfefad94
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/cursor.rs
@@ -0,0 +1,98 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use super::{snapshot::Snapshot, IterImpl};
+use crate::backend::traits::BackendRoCursor;
+
+#[derive(Debug)]
+pub struct RoCursorImpl<'c>(pub(crate) &'c Snapshot);
+
+#[cfg(not(feature = "db-dup-sort"))]
+impl<'c> BackendRoCursor<'c> for RoCursorImpl<'c> {
+ type Iter = IterImpl<'c>;
+
+ fn into_iter(self) -> Self::Iter {
+ IterImpl(Box::new(self.0.iter()))
+ }
+
+ fn into_iter_from<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ IterImpl(Box::new(
+ self.0.iter().skip_while(move |&(k, _)| k < key.as_ref()),
+ ))
+ }
+
+ fn into_iter_dup_of<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ IterImpl(Box::new(
+ self.0.iter().filter(move |&(k, _)| k == key.as_ref()),
+ ))
+ }
+}
+
+#[cfg(feature = "db-dup-sort")]
+impl<'c> BackendRoCursor<'c> for RoCursorImpl<'c> {
+ type Iter = IterImpl<'c>;
+
+ fn into_iter(self) -> Self::Iter {
+ let flattened = self
+ .0
+ .iter()
+ .flat_map(|(key, values)| values.map(move |value| (key, value)));
+ IterImpl(Box::new(flattened))
+ }
+
+ fn into_iter_from<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ let skipped = self.0.iter().skip_while(move |&(k, _)| k < key.as_ref());
+ let flattened = skipped.flat_map(|(key, values)| values.map(move |value| (key, value)));
+ IterImpl(Box::new(flattened))
+ }
+
+ fn into_iter_dup_of<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ let filtered = self.0.iter().filter(move |&(k, _)| k == key.as_ref());
+ let flattened = filtered.flat_map(|(key, values)| values.map(move |value| (key, value)));
+ IterImpl(Box::new(flattened))
+ }
+}
+
+#[derive(Debug)]
+pub struct RwCursorImpl<'c>(&'c mut Snapshot);
+
+impl<'c> BackendRoCursor<'c> for RwCursorImpl<'c> {
+ type Iter = IterImpl<'c>;
+
+ fn into_iter(self) -> Self::Iter {
+ unimplemented!()
+ }
+
+ fn into_iter_from<K>(self, _key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ unimplemented!()
+ }
+
+ fn into_iter_dup_of<K>(self, _key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c,
+ {
+ unimplemented!()
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/database.rs b/third_party/rust/rkv/src/backend/impl_safe/database.rs
new file mode 100644
index 0000000000..f8eb4aef76
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/database.rs
@@ -0,0 +1,41 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use id_arena::Id;
+use serde_derive::{Deserialize, Serialize};
+
+use super::{snapshot::Snapshot, DatabaseFlagsImpl};
+use crate::backend::traits::BackendDatabase;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)]
+pub struct DatabaseImpl(pub(crate) Id<Database>);
+
+impl BackendDatabase for DatabaseImpl {}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct Database {
+ snapshot: Snapshot,
+}
+
+impl Database {
+ pub(crate) fn new(flags: Option<DatabaseFlagsImpl>, snapshot: Option<Snapshot>) -> Database {
+ Database {
+ snapshot: snapshot.unwrap_or_else(|| Snapshot::new(flags)),
+ }
+ }
+
+ pub(crate) fn snapshot(&self) -> Snapshot {
+ self.snapshot.clone()
+ }
+
+ pub(crate) fn replace(&mut self, snapshot: Snapshot) -> Snapshot {
+ std::mem::replace(&mut self.snapshot, snapshot)
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/environment.rs b/third_party/rust/rkv/src/backend/impl_safe/environment.rs
new file mode 100644
index 0000000000..17cfb87f1e
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/environment.rs
@@ -0,0 +1,332 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ borrow::Cow,
+ collections::HashMap,
+ fs,
+ ops::DerefMut,
+ path::{Path, PathBuf},
+ sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
+};
+
+use id_arena::Arena;
+use log::warn;
+
+use super::{
+ database::Database, DatabaseFlagsImpl, DatabaseImpl, EnvironmentFlagsImpl, ErrorImpl, InfoImpl,
+ RoTransactionImpl, RwTransactionImpl, StatImpl,
+};
+use crate::backend::traits::{BackendEnvironment, BackendEnvironmentBuilder};
+
+const DEFAULT_DB_FILENAME: &str = "data.safe.bin";
+
+type DatabaseArena = Arena<Database>;
+type DatabaseNameMap = HashMap<Option<String>, DatabaseImpl>;
+
+#[derive(Debug, PartialEq, Eq, Copy, Clone)]
+pub struct EnvironmentBuilderImpl {
+ flags: EnvironmentFlagsImpl,
+ max_readers: Option<usize>,
+ max_dbs: Option<usize>,
+ map_size: Option<usize>,
+ make_dir_if_needed: bool,
+ discard_if_corrupted: bool,
+}
+
+impl<'b> BackendEnvironmentBuilder<'b> for EnvironmentBuilderImpl {
+ type Environment = EnvironmentImpl;
+ type Error = ErrorImpl;
+ type Flags = EnvironmentFlagsImpl;
+
+ fn new() -> EnvironmentBuilderImpl {
+ EnvironmentBuilderImpl {
+ flags: EnvironmentFlagsImpl::empty(),
+ max_readers: None,
+ max_dbs: None,
+ map_size: None,
+ make_dir_if_needed: false,
+ discard_if_corrupted: false,
+ }
+ }
+
+ fn set_flags<T>(&mut self, flags: T) -> &mut Self
+ where
+ T: Into<Self::Flags>,
+ {
+ self.flags = flags.into();
+ self
+ }
+
+ fn set_max_readers(&mut self, max_readers: u32) -> &mut Self {
+ self.max_readers = Some(max_readers as usize);
+ self
+ }
+
+ fn set_max_dbs(&mut self, max_dbs: u32) -> &mut Self {
+ self.max_dbs = Some(max_dbs as usize);
+ self
+ }
+
+ fn set_map_size(&mut self, map_size: usize) -> &mut Self {
+ self.map_size = Some(map_size);
+ self
+ }
+
+ fn set_make_dir_if_needed(&mut self, make_dir_if_needed: bool) -> &mut Self {
+ self.make_dir_if_needed = make_dir_if_needed;
+ self
+ }
+
+ fn set_discard_if_corrupted(&mut self, discard_if_corrupted: bool) -> &mut Self {
+ self.discard_if_corrupted = discard_if_corrupted;
+ self
+ }
+
+ fn open(&self, path: &Path) -> Result<Self::Environment, Self::Error> {
+ // Technically NO_SUB_DIR should change these checks here, but they're both currently
+ // unimplemented with this storage backend.
+ if !path.is_dir() {
+ if !self.make_dir_if_needed {
+ return Err(ErrorImpl::UnsuitableEnvironmentPath(path.into()));
+ }
+ fs::create_dir_all(path)?;
+ }
+ let mut env = EnvironmentImpl::new(
+ path,
+ self.flags,
+ self.max_readers,
+ self.max_dbs,
+ self.map_size,
+ )?;
+ env.read_from_disk(self.discard_if_corrupted)?;
+ Ok(env)
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct EnvironmentDbs {
+ pub(crate) arena: DatabaseArena,
+ pub(crate) name_map: DatabaseNameMap,
+}
+
+#[derive(Debug)]
+pub(crate) struct EnvironmentDbsRefMut<'a> {
+ pub(crate) arena: &'a mut DatabaseArena,
+ pub(crate) name_map: &'a mut DatabaseNameMap,
+}
+
+impl<'a> From<&'a mut EnvironmentDbs> for EnvironmentDbsRefMut<'a> {
+ fn from(dbs: &mut EnvironmentDbs) -> EnvironmentDbsRefMut {
+ EnvironmentDbsRefMut {
+ arena: &mut dbs.arena,
+ name_map: &mut dbs.name_map,
+ }
+ }
+}
+
+#[derive(Debug)]
+pub struct EnvironmentImpl {
+ path: PathBuf,
+ max_dbs: usize,
+ dbs: RwLock<EnvironmentDbs>,
+ ro_txns: Arc<()>,
+ rw_txns: Arc<()>,
+}
+
+impl EnvironmentImpl {
+ fn serialize(&self) -> Result<Vec<u8>, ErrorImpl> {
+ let dbs = self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)?;
+ let data: HashMap<_, _> = dbs
+ .name_map
+ .iter()
+ .map(|(name, id)| (name, &dbs.arena[id.0]))
+ .collect();
+ Ok(bincode::serialize(&data)?)
+ }
+
+ fn deserialize(
+ bytes: &[u8],
+ discard_if_corrupted: bool,
+ ) -> Result<(DatabaseArena, DatabaseNameMap), ErrorImpl> {
+ let mut arena = DatabaseArena::new();
+ let mut name_map = HashMap::new();
+ let data: HashMap<_, _> = match bincode::deserialize(bytes) {
+ Err(_) if discard_if_corrupted => Ok(HashMap::new()),
+ result => result,
+ }?;
+ for (name, db) in data {
+ name_map.insert(name, DatabaseImpl(arena.alloc(db)));
+ }
+ Ok((arena, name_map))
+ }
+}
+
+impl EnvironmentImpl {
+ pub(crate) fn new(
+ path: &Path,
+ flags: EnvironmentFlagsImpl,
+ max_readers: Option<usize>,
+ max_dbs: Option<usize>,
+ map_size: Option<usize>,
+ ) -> Result<EnvironmentImpl, ErrorImpl> {
+ if !flags.is_empty() {
+ warn!("Ignoring `flags={:?}`", flags);
+ }
+ if let Some(max_readers) = max_readers {
+ warn!("Ignoring `max_readers={}`", max_readers);
+ }
+ if let Some(map_size) = map_size {
+ warn!("Ignoring `map_size={}`", map_size);
+ }
+
+ Ok(EnvironmentImpl {
+ path: path.to_path_buf(),
+ max_dbs: max_dbs.unwrap_or(std::usize::MAX),
+ dbs: RwLock::new(EnvironmentDbs {
+ arena: DatabaseArena::new(),
+ name_map: HashMap::new(),
+ }),
+ ro_txns: Arc::new(()),
+ rw_txns: Arc::new(()),
+ })
+ }
+
+ pub(crate) fn read_from_disk(&mut self, discard_if_corrupted: bool) -> Result<(), ErrorImpl> {
+ let mut path = Cow::from(&self.path);
+ if fs::metadata(&path)?.is_dir() {
+ path.to_mut().push(DEFAULT_DB_FILENAME);
+ };
+ if fs::metadata(&path).is_err() {
+ return Ok(());
+ };
+ let (arena, name_map) = Self::deserialize(&fs::read(&path)?, discard_if_corrupted)?;
+ self.dbs = RwLock::new(EnvironmentDbs { arena, name_map });
+ Ok(())
+ }
+
+ pub(crate) fn write_to_disk(&self) -> Result<(), ErrorImpl> {
+ let mut path = Cow::from(&self.path);
+ if fs::metadata(&path)?.is_dir() {
+ path.to_mut().push(DEFAULT_DB_FILENAME);
+ };
+
+ // Write to a temp file first.
+ let tmp_path = path.with_extension("tmp");
+ fs::write(&tmp_path, self.serialize()?)?;
+
+ // Atomically move that file to the database file.
+ fs::rename(tmp_path, path)?;
+ Ok(())
+ }
+
+ pub(crate) fn dbs(&self) -> Result<RwLockReadGuard<EnvironmentDbs>, ErrorImpl> {
+ self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)
+ }
+
+ pub(crate) fn dbs_mut(&self) -> Result<RwLockWriteGuard<EnvironmentDbs>, ErrorImpl> {
+ self.dbs.write().map_err(|_| ErrorImpl::EnvPoisonError)
+ }
+}
+
+impl<'e> BackendEnvironment<'e> for EnvironmentImpl {
+ type Database = DatabaseImpl;
+ type Error = ErrorImpl;
+ type Flags = DatabaseFlagsImpl;
+ type Info = InfoImpl;
+ type RoTransaction = RoTransactionImpl<'e>;
+ type RwTransaction = RwTransactionImpl<'e>;
+ type Stat = StatImpl;
+
+ fn get_dbs(&self) -> Result<Vec<Option<String>>, Self::Error> {
+ let dbs = self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)?;
+ Ok(dbs.name_map.keys().map(|key| key.to_owned()).collect())
+ }
+
+ fn open_db(&self, name: Option<&str>) -> Result<Self::Database, Self::Error> {
+ if Arc::strong_count(&self.ro_txns) > 1 {
+ return Err(ErrorImpl::DbsIllegalOpen);
+ }
+ // TOOD: don't reallocate `name`.
+ let key = name.map(String::from);
+ let dbs = self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)?;
+ let db = dbs.name_map.get(&key).ok_or(ErrorImpl::DbNotFoundError)?;
+ Ok(*db)
+ }
+
+ fn create_db(
+ &self,
+ name: Option<&str>,
+ flags: Self::Flags,
+ ) -> Result<Self::Database, Self::Error> {
+ if Arc::strong_count(&self.ro_txns) > 1 {
+ return Err(ErrorImpl::DbsIllegalOpen);
+ }
+ // TOOD: don't reallocate `name`.
+ let key = name.map(String::from);
+ let mut dbs = self.dbs.write().map_err(|_| ErrorImpl::EnvPoisonError)?;
+ if dbs.name_map.keys().filter_map(|k| k.as_ref()).count() >= self.max_dbs && name.is_some() {
+ return Err(ErrorImpl::DbsFull);
+ }
+ let parts = EnvironmentDbsRefMut::from(dbs.deref_mut());
+ let arena = parts.arena;
+ let name_map = parts.name_map;
+ let id = name_map
+ .entry(key)
+ .or_insert_with(|| DatabaseImpl(arena.alloc(Database::new(Some(flags), None))));
+ Ok(*id)
+ }
+
+ fn begin_ro_txn(&'e self) -> Result<Self::RoTransaction, Self::Error> {
+ RoTransactionImpl::new(self, self.ro_txns.clone())
+ }
+
+ fn begin_rw_txn(&'e self) -> Result<Self::RwTransaction, Self::Error> {
+ RwTransactionImpl::new(self, self.rw_txns.clone())
+ }
+
+ fn sync(&self, force: bool) -> Result<(), Self::Error> {
+ warn!("Ignoring `force={}`", force);
+ self.write_to_disk()
+ }
+
+ fn stat(&self) -> Result<Self::Stat, Self::Error> {
+ Ok(StatImpl)
+ }
+
+ fn info(&self) -> Result<Self::Info, Self::Error> {
+ Ok(InfoImpl)
+ }
+
+ fn freelist(&self) -> Result<usize, Self::Error> {
+ unimplemented!()
+ }
+
+ fn load_ratio(&self) -> Result<Option<f32>, Self::Error> {
+ warn!("`load_ratio()` is irrelevant for this storage backend.");
+ Ok(None)
+ }
+
+ fn set_map_size(&self, size: usize) -> Result<(), Self::Error> {
+ warn!(
+ "`set_map_size({})` is ignored by this storage backend.",
+ size
+ );
+ Ok(())
+ }
+
+ fn get_files_on_disk(&self) -> Vec<PathBuf> {
+ // Technically NO_SUB_DIR and NO_LOCK should change this output, but
+ // they're both currently unimplemented with this storage backend.
+ let mut db_filename = self.path.clone();
+ db_filename.push(DEFAULT_DB_FILENAME);
+ vec![db_filename]
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/error.rs b/third_party/rust/rkv/src/backend/impl_safe/error.rs
new file mode 100644
index 0000000000..f086cdc954
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/error.rs
@@ -0,0 +1,79 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{fmt, io, path::PathBuf};
+
+use bincode::Error as BincodeError;
+
+use crate::{backend::traits::BackendError, error::StoreError};
+
+#[derive(Debug)]
+pub enum ErrorImpl {
+ KeyValuePairNotFound,
+ EnvPoisonError,
+ DbsFull,
+ DbsIllegalOpen,
+ DbNotFoundError,
+ DbIsForeignError,
+ UnsuitableEnvironmentPath(PathBuf),
+ IoError(io::Error),
+ BincodeError(BincodeError),
+}
+
+impl BackendError for ErrorImpl {}
+
+impl fmt::Display for ErrorImpl {
+ fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
+ match self {
+ ErrorImpl::KeyValuePairNotFound => write!(fmt, "KeyValuePairNotFound (safe mode)"),
+ ErrorImpl::EnvPoisonError => write!(fmt, "EnvPoisonError (safe mode)"),
+ ErrorImpl::DbsFull => write!(fmt, "DbsFull (safe mode)"),
+ ErrorImpl::DbsIllegalOpen => write!(fmt, "DbIllegalOpen (safe mode)"),
+ ErrorImpl::DbNotFoundError => write!(fmt, "DbNotFoundError (safe mode)"),
+ ErrorImpl::DbIsForeignError => write!(fmt, "DbIsForeignError (safe mode)"),
+ ErrorImpl::UnsuitableEnvironmentPath(_) => {
+ write!(fmt, "UnsuitableEnvironmentPath (safe mode)")
+ }
+ ErrorImpl::IoError(e) => e.fmt(fmt),
+ ErrorImpl::BincodeError(e) => e.fmt(fmt),
+ }
+ }
+}
+
+impl Into<StoreError> for ErrorImpl {
+ fn into(self) -> StoreError {
+ // The `StoreError::KeyValuePairBadSize` error is unused, because this
+ // backend supports keys and values of arbitrary sizes.
+ // The `StoreError::MapFull` and `StoreError::ReadersFull` are
+ // unimplemented yet, but they should be in the future.
+ match self {
+ ErrorImpl::KeyValuePairNotFound => StoreError::KeyValuePairNotFound,
+ ErrorImpl::BincodeError(_) => StoreError::FileInvalid,
+ ErrorImpl::DbsFull => StoreError::DbsFull,
+ ErrorImpl::UnsuitableEnvironmentPath(path) => {
+ StoreError::UnsuitableEnvironmentPath(path)
+ }
+ ErrorImpl::IoError(error) => StoreError::IoError(error),
+ _ => StoreError::SafeModeError(self),
+ }
+ }
+}
+
+impl From<io::Error> for ErrorImpl {
+ fn from(e: io::Error) -> ErrorImpl {
+ ErrorImpl::IoError(e)
+ }
+}
+
+impl From<BincodeError> for ErrorImpl {
+ fn from(e: BincodeError) -> ErrorImpl {
+ ErrorImpl::BincodeError(e)
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/flags.rs b/third_party/rust/rkv/src/backend/impl_safe/flags.rs
new file mode 100644
index 0000000000..a0d623614e
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/flags.rs
@@ -0,0 +1,124 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use bitflags::bitflags;
+use serde_derive::{Deserialize, Serialize};
+
+use crate::backend::{
+ common::{DatabaseFlags, EnvironmentFlags, WriteFlags},
+ traits::{BackendDatabaseFlags, BackendEnvironmentFlags, BackendFlags, BackendWriteFlags},
+};
+
+bitflags! {
+ #[derive(Default, Serialize, Deserialize)]
+ pub struct EnvironmentFlagsImpl: u32 {
+ const NIL = 0b0000_0000;
+ }
+}
+
+impl BackendFlags for EnvironmentFlagsImpl {
+ fn empty() -> EnvironmentFlagsImpl {
+ EnvironmentFlagsImpl::empty()
+ }
+}
+
+impl BackendEnvironmentFlags for EnvironmentFlagsImpl {
+ fn set(&mut self, flag: EnvironmentFlags, value: bool) {
+ self.set(flag.into(), value)
+ }
+}
+
+impl Into<EnvironmentFlagsImpl> for EnvironmentFlags {
+ fn into(self) -> EnvironmentFlagsImpl {
+ match self {
+ EnvironmentFlags::FIXED_MAP => unimplemented!(),
+ EnvironmentFlags::NO_SUB_DIR => unimplemented!(),
+ EnvironmentFlags::WRITE_MAP => unimplemented!(),
+ EnvironmentFlags::READ_ONLY => unimplemented!(),
+ EnvironmentFlags::NO_META_SYNC => unimplemented!(),
+ EnvironmentFlags::NO_SYNC => unimplemented!(),
+ EnvironmentFlags::MAP_ASYNC => unimplemented!(),
+ EnvironmentFlags::NO_TLS => unimplemented!(),
+ EnvironmentFlags::NO_LOCK => unimplemented!(),
+ EnvironmentFlags::NO_READAHEAD => unimplemented!(),
+ EnvironmentFlags::NO_MEM_INIT => unimplemented!(),
+ }
+ }
+}
+
+bitflags! {
+ #[derive(Default, Serialize, Deserialize)]
+ pub struct DatabaseFlagsImpl: u32 {
+ const NIL = 0b0000_0000;
+ #[cfg(feature = "db-dup-sort")]
+ const DUP_SORT = 0b0000_0001;
+ #[cfg(feature = "db-int-key")]
+ const INTEGER_KEY = 0b0000_0010;
+ }
+}
+
+impl BackendFlags for DatabaseFlagsImpl {
+ fn empty() -> DatabaseFlagsImpl {
+ DatabaseFlagsImpl::empty()
+ }
+}
+
+impl BackendDatabaseFlags for DatabaseFlagsImpl {
+ fn set(&mut self, flag: DatabaseFlags, value: bool) {
+ self.set(flag.into(), value)
+ }
+}
+
+impl Into<DatabaseFlagsImpl> for DatabaseFlags {
+ fn into(self) -> DatabaseFlagsImpl {
+ match self {
+ DatabaseFlags::REVERSE_KEY => unimplemented!(),
+ #[cfg(feature = "db-dup-sort")]
+ DatabaseFlags::DUP_SORT => DatabaseFlagsImpl::DUP_SORT,
+ #[cfg(feature = "db-dup-sort")]
+ DatabaseFlags::DUP_FIXED => unimplemented!(),
+ #[cfg(feature = "db-int-key")]
+ DatabaseFlags::INTEGER_KEY => DatabaseFlagsImpl::INTEGER_KEY,
+ DatabaseFlags::INTEGER_DUP => unimplemented!(),
+ DatabaseFlags::REVERSE_DUP => unimplemented!(),
+ }
+ }
+}
+
+bitflags! {
+ #[derive(Default, Serialize, Deserialize)]
+ pub struct WriteFlagsImpl: u32 {
+ const NIL = 0b0000_0000;
+ }
+}
+
+impl BackendFlags for WriteFlagsImpl {
+ fn empty() -> WriteFlagsImpl {
+ WriteFlagsImpl::empty()
+ }
+}
+
+impl BackendWriteFlags for WriteFlagsImpl {
+ fn set(&mut self, flag: WriteFlags, value: bool) {
+ self.set(flag.into(), value)
+ }
+}
+
+impl Into<WriteFlagsImpl> for WriteFlags {
+ fn into(self) -> WriteFlagsImpl {
+ match self {
+ WriteFlags::NO_OVERWRITE => unimplemented!(),
+ WriteFlags::NO_DUP_DATA => unimplemented!(),
+ WriteFlags::CURRENT => unimplemented!(),
+ WriteFlags::APPEND => unimplemented!(),
+ WriteFlags::APPEND_DUP => unimplemented!(),
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/info.rs b/third_party/rust/rkv/src/backend/impl_safe/info.rs
new file mode 100644
index 0000000000..18f0f51da3
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/info.rs
@@ -0,0 +1,35 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendInfo;
+
+pub struct InfoImpl;
+
+impl BackendInfo for InfoImpl {
+ fn map_size(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn last_pgno(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn last_txnid(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn max_readers(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn num_readers(&self) -> usize {
+ unimplemented!()
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/iter.rs b/third_party/rust/rkv/src/backend/impl_safe/iter.rs
new file mode 100644
index 0000000000..a784c00a5b
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/iter.rs
@@ -0,0 +1,24 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use super::ErrorImpl;
+use crate::backend::traits::BackendIter;
+
+// FIXME: Use generics instead.
+pub struct IterImpl<'i>(pub(crate) Box<dyn Iterator<Item = (&'i [u8], &'i [u8])> + 'i>);
+
+impl<'i> BackendIter<'i> for IterImpl<'i> {
+ type Error = ErrorImpl;
+
+ #[allow(clippy::type_complexity)]
+ fn next(&mut self) -> Option<Result<(&'i [u8], &'i [u8]), Self::Error>> {
+ self.0.next().map(Ok)
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/snapshot.rs b/third_party/rust/rkv/src/backend/impl_safe/snapshot.rs
new file mode 100644
index 0000000000..95f5690d34
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/snapshot.rs
@@ -0,0 +1,141 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ collections::{BTreeMap, BTreeSet},
+ sync::Arc,
+};
+
+use serde_derive::{Deserialize, Serialize};
+
+use super::DatabaseFlagsImpl;
+
+type Key = Box<[u8]>;
+type Value = Box<[u8]>;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct Snapshot {
+ flags: DatabaseFlagsImpl,
+ #[cfg(not(feature = "db-dup-sort"))]
+ map: Arc<BTreeMap<Key, Value>>,
+ #[cfg(feature = "db-dup-sort")]
+ map: Arc<BTreeMap<Key, BTreeSet<Value>>>,
+}
+
+impl Snapshot {
+ pub(crate) fn new(flags: Option<DatabaseFlagsImpl>) -> Snapshot {
+ Snapshot {
+ flags: flags.unwrap_or_default(),
+ map: Default::default(),
+ }
+ }
+
+ pub(crate) fn flags(&self) -> &DatabaseFlagsImpl {
+ &self.flags
+ }
+
+ pub(crate) fn clear(&mut self) {
+ self.map = Default::default();
+ }
+}
+
+#[cfg(not(feature = "db-dup-sort"))]
+impl Snapshot {
+ pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> {
+ self.map.get(key).map(|value| value.as_ref())
+ }
+
+ pub(crate) fn put(&mut self, key: &[u8], value: &[u8]) {
+ let map = Arc::make_mut(&mut self.map);
+ map.insert(Box::from(key), Box::from(value));
+ }
+
+ pub(crate) fn del(&mut self, key: &[u8]) -> Option<()> {
+ let map = Arc::make_mut(&mut self.map);
+ map.remove(key).map(|_| ())
+ }
+
+ pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], &[u8])> {
+ self.map
+ .iter()
+ .map(|(key, value)| (key.as_ref(), value.as_ref()))
+ }
+}
+
+#[cfg(feature = "db-dup-sort")]
+impl Snapshot {
+ pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> {
+ self.map
+ .get(key)
+ .and_then(|v| v.iter().next())
+ .map(|v| v.as_ref())
+ }
+
+ pub(crate) fn put(&mut self, key: &[u8], value: &[u8]) {
+ let map = Arc::make_mut(&mut self.map);
+ match map.get_mut(key) {
+ None => {
+ let mut values = BTreeSet::new();
+ values.insert(Box::from(value));
+ map.insert(Box::from(key), values);
+ }
+ Some(values) => {
+ values.clear();
+ values.insert(Box::from(value));
+ }
+ }
+ }
+
+ pub(crate) fn del(&mut self, key: &[u8]) -> Option<()> {
+ let map = Arc::make_mut(&mut self.map);
+ match map.get_mut(key) {
+ None => None,
+ Some(values) => {
+ let was_empty = values.is_empty();
+ values.clear();
+ Some(()).filter(|_| !was_empty)
+ }
+ }
+ }
+
+ pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], impl Iterator<Item = &[u8]>)> {
+ self.map
+ .iter()
+ .map(|(key, values)| (key.as_ref(), values.iter().map(|value| value.as_ref())))
+ }
+}
+
+#[cfg(feature = "db-dup-sort")]
+impl Snapshot {
+ pub(crate) fn put_dup(&mut self, key: &[u8], value: &[u8]) {
+ let map = Arc::make_mut(&mut self.map);
+ match map.get_mut(key) {
+ None => {
+ let mut values = BTreeSet::new();
+ values.insert(Box::from(value));
+ map.insert(Box::from(key), values);
+ }
+ Some(values) => {
+ values.insert(Box::from(value));
+ }
+ }
+ }
+
+ pub(crate) fn del_exact(&mut self, key: &[u8], value: &[u8]) -> Option<()> {
+ let map = Arc::make_mut(&mut self.map);
+ match map.get_mut(key) {
+ None => None,
+ Some(values) => {
+ let was_removed = values.remove(value);
+ Some(()).filter(|_| was_removed)
+ }
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/stat.rs b/third_party/rust/rkv/src/backend/impl_safe/stat.rs
new file mode 100644
index 0000000000..c117b56833
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/stat.rs
@@ -0,0 +1,39 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::backend::traits::BackendStat;
+
+pub struct StatImpl;
+
+impl BackendStat for StatImpl {
+ fn page_size(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn depth(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn branch_pages(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn leaf_pages(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn overflow_pages(&self) -> usize {
+ unimplemented!()
+ }
+
+ fn entries(&self) -> usize {
+ unimplemented!()
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/impl_safe/transaction.rs b/third_party/rust/rkv/src/backend/impl_safe/transaction.rs
new file mode 100644
index 0000000000..d37352a42d
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/impl_safe/transaction.rs
@@ -0,0 +1,208 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![allow(dead_code)] // TODO: Get rid of unused struct members
+
+use std::{collections::HashMap, sync::Arc};
+
+use super::{
+ snapshot::Snapshot, DatabaseImpl, EnvironmentImpl, ErrorImpl, RoCursorImpl, WriteFlagsImpl,
+};
+use crate::backend::traits::{
+ BackendRoCursorTransaction, BackendRoTransaction, BackendRwCursorTransaction,
+ BackendRwTransaction,
+};
+
+#[derive(Debug)]
+pub struct RoTransactionImpl<'t> {
+ env: &'t EnvironmentImpl,
+ snapshots: HashMap<DatabaseImpl, Snapshot>,
+ idx: Arc<()>,
+}
+
+impl<'t> RoTransactionImpl<'t> {
+ pub(crate) fn new(
+ env: &'t EnvironmentImpl,
+ idx: Arc<()>,
+ ) -> Result<RoTransactionImpl<'t>, ErrorImpl> {
+ let snapshots = env
+ .dbs()?
+ .arena
+ .iter()
+ .map(|(id, db)| (DatabaseImpl(id), db.snapshot()))
+ .collect();
+ Ok(RoTransactionImpl {
+ env,
+ snapshots,
+ idx,
+ })
+ }
+}
+
+impl<'t> BackendRoTransaction for RoTransactionImpl<'t> {
+ type Database = DatabaseImpl;
+ type Error = ErrorImpl;
+
+ fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> {
+ let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?;
+ snapshot.get(key).ok_or(ErrorImpl::KeyValuePairNotFound)
+ }
+
+ fn abort(self) {
+ // noop
+ }
+}
+
+impl<'t> BackendRoCursorTransaction<'t> for RoTransactionImpl<'t> {
+ type RoCursor = RoCursorImpl<'t>;
+
+ fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> {
+ let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?;
+ Ok(RoCursorImpl(snapshot))
+ }
+}
+
+#[derive(Debug)]
+pub struct RwTransactionImpl<'t> {
+ env: &'t EnvironmentImpl,
+ snapshots: HashMap<DatabaseImpl, Snapshot>,
+ idx: Arc<()>,
+}
+
+impl<'t> RwTransactionImpl<'t> {
+ pub(crate) fn new(
+ env: &'t EnvironmentImpl,
+ idx: Arc<()>,
+ ) -> Result<RwTransactionImpl<'t>, ErrorImpl> {
+ let snapshots = env
+ .dbs()?
+ .arena
+ .iter()
+ .map(|(id, db)| (DatabaseImpl(id), db.snapshot()))
+ .collect();
+ Ok(RwTransactionImpl {
+ env,
+ snapshots,
+ idx,
+ })
+ }
+}
+
+impl<'t> BackendRwTransaction for RwTransactionImpl<'t> {
+ type Database = DatabaseImpl;
+ type Error = ErrorImpl;
+ type Flags = WriteFlagsImpl;
+
+ fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> {
+ let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?;
+ snapshot.get(key).ok_or(ErrorImpl::KeyValuePairNotFound)
+ }
+
+ #[cfg(not(feature = "db-dup-sort"))]
+ fn put(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: &[u8],
+ _flags: Self::Flags,
+ ) -> Result<(), Self::Error> {
+ let snapshot = self
+ .snapshots
+ .get_mut(db)
+ .ok_or_else(|| ErrorImpl::DbIsForeignError)?;
+ snapshot.put(key, value);
+ Ok(())
+ }
+
+ #[cfg(feature = "db-dup-sort")]
+ fn put(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: &[u8],
+ _flags: Self::Flags,
+ ) -> Result<(), Self::Error> {
+ use super::DatabaseFlagsImpl;
+ let snapshot = self
+ .snapshots
+ .get_mut(db)
+ .ok_or(ErrorImpl::DbIsForeignError)?;
+ if snapshot.flags().contains(DatabaseFlagsImpl::DUP_SORT) {
+ snapshot.put_dup(key, value);
+ } else {
+ snapshot.put(key, value);
+ }
+ Ok(())
+ }
+
+ #[cfg(not(feature = "db-dup-sort"))]
+ fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error> {
+ let snapshot = self
+ .snapshots
+ .get_mut(db)
+ .ok_or_else(|| ErrorImpl::DbIsForeignError)?;
+ let deleted = snapshot.del(key);
+ Ok(deleted.ok_or_else(|| ErrorImpl::KeyValuePairNotFound)?)
+ }
+
+ #[cfg(feature = "db-dup-sort")]
+ fn del(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: Option<&[u8]>,
+ ) -> Result<(), Self::Error> {
+ use super::DatabaseFlagsImpl;
+ let snapshot = self
+ .snapshots
+ .get_mut(db)
+ .ok_or(ErrorImpl::DbIsForeignError)?;
+ let deleted = match (value, snapshot.flags()) {
+ (Some(value), flags) if flags.contains(DatabaseFlagsImpl::DUP_SORT) => {
+ snapshot.del_exact(key, value)
+ }
+ _ => snapshot.del(key),
+ };
+ deleted.ok_or(ErrorImpl::KeyValuePairNotFound)
+ }
+
+ fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error> {
+ let snapshot = self
+ .snapshots
+ .get_mut(db)
+ .ok_or(ErrorImpl::DbIsForeignError)?;
+ snapshot.clear();
+ Ok(())
+ }
+
+ fn commit(self) -> Result<(), Self::Error> {
+ let mut dbs = self.env.dbs_mut()?;
+
+ for (id, snapshot) in self.snapshots {
+ let db = dbs.arena.get_mut(id.0).ok_or(ErrorImpl::DbIsForeignError)?;
+ db.replace(snapshot);
+ }
+
+ drop(dbs);
+ self.env.write_to_disk()
+ }
+
+ fn abort(self) {
+ // noop
+ }
+}
+
+impl<'t> BackendRwCursorTransaction<'t> for RwTransactionImpl<'t> {
+ type RoCursor = RoCursorImpl<'t>;
+
+ fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> {
+ let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?;
+ Ok(RoCursorImpl(snapshot))
+ }
+}
diff --git a/third_party/rust/rkv/src/backend/traits.rs b/third_party/rust/rkv/src/backend/traits.rs
new file mode 100644
index 0000000000..1dda15a85c
--- /dev/null
+++ b/third_party/rust/rkv/src/backend/traits.rs
@@ -0,0 +1,202 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ fmt::{Debug, Display},
+ path::{Path, PathBuf},
+};
+
+use crate::{
+ backend::common::{DatabaseFlags, EnvironmentFlags, WriteFlags},
+ error::StoreError,
+};
+
+pub trait BackendError: Debug + Display + Into<StoreError> {}
+
+pub trait BackendDatabase: Debug + Eq + PartialEq + Copy + Clone {}
+
+pub trait BackendFlags: Debug + Eq + PartialEq + Copy + Clone + Default {
+ fn empty() -> Self;
+}
+
+pub trait BackendEnvironmentFlags: BackendFlags {
+ fn set(&mut self, flag: EnvironmentFlags, value: bool);
+}
+
+pub trait BackendDatabaseFlags: BackendFlags {
+ fn set(&mut self, flag: DatabaseFlags, value: bool);
+}
+
+pub trait BackendWriteFlags: BackendFlags {
+ fn set(&mut self, flag: WriteFlags, value: bool);
+}
+
+pub trait BackendStat {
+ fn page_size(&self) -> usize;
+
+ fn depth(&self) -> usize;
+
+ fn branch_pages(&self) -> usize;
+
+ fn leaf_pages(&self) -> usize;
+
+ fn overflow_pages(&self) -> usize;
+
+ fn entries(&self) -> usize;
+}
+
+pub trait BackendInfo {
+ fn map_size(&self) -> usize;
+
+ fn last_pgno(&self) -> usize;
+
+ fn last_txnid(&self) -> usize;
+
+ fn max_readers(&self) -> usize;
+
+ fn num_readers(&self) -> usize;
+}
+
+pub trait BackendEnvironmentBuilder<'b>: Debug + Eq + PartialEq + Copy + Clone {
+ type Error: BackendError;
+ type Environment: BackendEnvironment<'b>;
+ type Flags: BackendEnvironmentFlags;
+
+ fn new() -> Self;
+
+ fn set_flags<T>(&mut self, flags: T) -> &mut Self
+ where
+ T: Into<Self::Flags>;
+
+ fn set_max_dbs(&mut self, max_dbs: u32) -> &mut Self;
+
+ fn set_max_readers(&mut self, max_readers: u32) -> &mut Self;
+
+ fn set_map_size(&mut self, size: usize) -> &mut Self;
+
+ fn set_make_dir_if_needed(&mut self, make_dir_if_needed: bool) -> &mut Self;
+
+ fn set_discard_if_corrupted(&mut self, discard_if_corrupted: bool) -> &mut Self;
+
+ fn open(&self, path: &Path) -> Result<Self::Environment, Self::Error>;
+}
+
+pub trait BackendEnvironment<'e>: Debug {
+ type Error: BackendError;
+ type Database: BackendDatabase;
+ type Flags: BackendDatabaseFlags;
+ type Stat: BackendStat;
+ type Info: BackendInfo;
+ type RoTransaction: BackendRoCursorTransaction<'e, Database = Self::Database>;
+ type RwTransaction: BackendRwCursorTransaction<'e, Database = Self::Database>;
+
+ fn get_dbs(&self) -> Result<Vec<Option<String>>, Self::Error>;
+
+ fn open_db(&self, name: Option<&str>) -> Result<Self::Database, Self::Error>;
+
+ fn create_db(
+ &self,
+ name: Option<&str>,
+ flags: Self::Flags,
+ ) -> Result<Self::Database, Self::Error>;
+
+ fn begin_ro_txn(&'e self) -> Result<Self::RoTransaction, Self::Error>;
+
+ fn begin_rw_txn(&'e self) -> Result<Self::RwTransaction, Self::Error>;
+
+ fn sync(&self, force: bool) -> Result<(), Self::Error>;
+
+ fn stat(&self) -> Result<Self::Stat, Self::Error>;
+
+ fn info(&self) -> Result<Self::Info, Self::Error>;
+
+ fn freelist(&self) -> Result<usize, Self::Error>;
+
+ fn load_ratio(&self) -> Result<Option<f32>, Self::Error>;
+
+ fn set_map_size(&self, size: usize) -> Result<(), Self::Error>;
+
+ fn get_files_on_disk(&self) -> Vec<PathBuf>;
+}
+
+pub trait BackendRoTransaction: Debug {
+ type Error: BackendError;
+ type Database: BackendDatabase;
+
+ fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error>;
+
+ fn abort(self);
+}
+
+pub trait BackendRwTransaction: Debug {
+ type Error: BackendError;
+ type Database: BackendDatabase;
+ type Flags: BackendWriteFlags;
+
+ fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error>;
+
+ fn put(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: &[u8],
+ flags: Self::Flags,
+ ) -> Result<(), Self::Error>;
+
+ #[cfg(not(feature = "db-dup-sort"))]
+ fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error>;
+
+ #[cfg(feature = "db-dup-sort")]
+ fn del(
+ &mut self,
+ db: &Self::Database,
+ key: &[u8],
+ value: Option<&[u8]>,
+ ) -> Result<(), Self::Error>;
+
+ fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error>;
+
+ fn commit(self) -> Result<(), Self::Error>;
+
+ fn abort(self);
+}
+
+pub trait BackendRoCursorTransaction<'t>: BackendRoTransaction {
+ type RoCursor: BackendRoCursor<'t>;
+
+ fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error>;
+}
+
+pub trait BackendRwCursorTransaction<'t>: BackendRwTransaction {
+ type RoCursor: BackendRoCursor<'t>;
+
+ fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error>;
+}
+
+pub trait BackendRoCursor<'c>: Debug {
+ type Iter: BackendIter<'c>;
+
+ fn into_iter(self) -> Self::Iter;
+
+ fn into_iter_from<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c;
+
+ fn into_iter_dup_of<K>(self, key: K) -> Self::Iter
+ where
+ K: AsRef<[u8]> + 'c;
+}
+
+pub trait BackendIter<'i> {
+ type Error: BackendError;
+
+ #[allow(clippy::type_complexity)]
+ fn next(&mut self) -> Option<Result<(&'i [u8], &'i [u8]), Self::Error>>;
+}
diff --git a/third_party/rust/rkv/src/bin/dump.rs b/third_party/rust/rkv/src/bin/dump.rs
new file mode 100644
index 0000000000..c0e12b986f
--- /dev/null
+++ b/third_party/rust/rkv/src/bin/dump.rs
@@ -0,0 +1,47 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{env::args, io, path::Path};
+
+use rkv::migrator::{LmdbArchMigrateError, LmdbArchMigrator};
+
+fn main() -> Result<(), LmdbArchMigrateError> {
+ let mut cli_args = args();
+ let mut db_name = None;
+ let mut env_path = None;
+
+ // The first arg is the name of the program, which we can ignore.
+ cli_args.next();
+
+ while let Some(arg) = cli_args.next() {
+ if &arg[0..1] == "-" {
+ match &arg[1..] {
+ "s" => {
+ db_name = match cli_args.next() {
+ None => return Err("-s must be followed by database name".into()),
+ Some(str) => Some(str),
+ };
+ }
+ str => return Err(format!("arg -{str} not recognized").into()),
+ }
+ } else {
+ if env_path.is_some() {
+ return Err("must provide only one path to the LMDB environment".into());
+ }
+ env_path = Some(arg);
+ }
+ }
+
+ let env_path = env_path.ok_or("must provide a path to the LMDB environment")?;
+ let mut migrator = LmdbArchMigrator::new(Path::new(&env_path))?;
+ migrator.dump(db_name.as_deref(), io::stdout()).unwrap();
+
+ Ok(())
+}
diff --git a/third_party/rust/rkv/src/bin/rand.rs b/third_party/rust/rkv/src/bin/rand.rs
new file mode 100644
index 0000000000..56d578c22e
--- /dev/null
+++ b/third_party/rust/rkv/src/bin/rand.rs
@@ -0,0 +1,106 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+//! A command-line utility to create an LMDB environment containing random data.
+//! It requires one flag, `-s path/to/environment`, which specifies the location
+//! where the tool should create the environment. Optionally, you may specify
+//! the number of key/value pairs to create via the `-n <number>` flag
+//! (for which the default value is 50).
+
+use std::{env::args, fs, fs::File, io::Read, path::Path};
+
+use rkv::{
+ backend::{BackendEnvironmentBuilder, Lmdb},
+ Rkv, StoreOptions, Value,
+};
+
+fn main() {
+ let mut args = args();
+ let mut database = None;
+ let mut path = None;
+ let mut num_pairs = 50;
+
+ // The first arg is the name of the program, which we can ignore.
+ args.next();
+
+ while let Some(arg) = args.next() {
+ if &arg[0..1] == "-" {
+ match &arg[1..] {
+ "s" => {
+ database = match args.next() {
+ None => panic!("-s must be followed by database arg"),
+ Some(str) => Some(str),
+ };
+ }
+ "n" => {
+ num_pairs = match args.next() {
+ None => panic!("-s must be followed by number of pairs"),
+ Some(str) => str.parse().expect("number"),
+ };
+ }
+ str => panic!("arg -{} not recognized", str),
+ }
+ } else {
+ if path.is_some() {
+ panic!("must provide only one path to the LMDB environment");
+ }
+ path = Some(arg);
+ }
+ }
+
+ if path.is_none() {
+ panic!("must provide a path to the LMDB environment");
+ }
+
+ let path = path.unwrap();
+ fs::create_dir_all(&path).expect("dir created");
+
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(2);
+ // Allocate enough map to accommodate the largest random collection.
+ // We currently do this by allocating twice the maximum possible size
+ // of the pairs (assuming maximum key and value sizes).
+ builder.set_map_size((511 + 65535) * num_pairs * 2);
+ let rkv = Rkv::from_builder(Path::new(&path), builder).expect("Rkv");
+ let store = rkv
+ .open_single(database.as_deref(), StoreOptions::create())
+ .expect("opened");
+ let mut writer = rkv.write().expect("writer");
+
+ // Generate random values for the number of keys and key/value lengths.
+ // On Linux, "Just use /dev/urandom!" <https://www.2uo.de/myths-about-urandom/>.
+ // On macOS it doesn't matter (/dev/random and /dev/urandom are identical).
+ let mut random = File::open("/dev/urandom").unwrap();
+ let mut nums = [0u8; 4];
+ random.read_exact(&mut nums).unwrap();
+
+ // Generate 0–255 pairs.
+ for _ in 0..num_pairs {
+ // Generate key and value lengths. The key must be 1–511 bytes long.
+ // The value length can be 0 and is essentially unbounded; we generate
+ // value lengths of 0–0xffff (65535).
+ // NB: the modulus method for generating a random number within a range
+ // introduces distribution skew, but we don't need it to be perfect.
+ let key_len = ((u16::from(nums[0]) + (u16::from(nums[1]) << 8)) % 511 + 1) as usize;
+ let value_len = (u16::from(nums[2]) + (u16::from(nums[3]) << 8)) as usize;
+
+ let mut key: Vec<u8> = vec![0; key_len];
+ random.read_exact(&mut key[0..key_len]).unwrap();
+
+ let mut value: Vec<u8> = vec![0; value_len];
+ random.read_exact(&mut value[0..value_len]).unwrap();
+
+ store
+ .put(&mut writer, key, &Value::Blob(&value))
+ .expect("wrote");
+ }
+
+ writer.commit().expect("committed");
+}
diff --git a/third_party/rust/rkv/src/env.rs b/third_party/rust/rkv/src/env.rs
new file mode 100644
index 0000000000..32e4fe5035
--- /dev/null
+++ b/third_party/rust/rkv/src/env.rs
@@ -0,0 +1,320 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ fs,
+ os::raw::c_uint,
+ path::{Path, PathBuf},
+};
+
+#[cfg(any(feature = "db-dup-sort", feature = "db-int-key"))]
+use crate::backend::{BackendDatabaseFlags, DatabaseFlags};
+use crate::{
+ backend::{
+ BackendEnvironment, BackendEnvironmentBuilder, BackendRoCursorTransaction,
+ BackendRwCursorTransaction, SafeModeError,
+ },
+ error::{CloseError, StoreError},
+ readwrite::{Reader, Writer},
+ store::{single::SingleStore, CloseOptions, Options as StoreOptions},
+};
+
+#[cfg(feature = "db-dup-sort")]
+use crate::store::multi::MultiStore;
+
+#[cfg(feature = "db-int-key")]
+use crate::store::integer::IntegerStore;
+#[cfg(feature = "db-int-key")]
+use crate::store::keys::PrimitiveInt;
+
+#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+use crate::store::integermulti::MultiIntegerStore;
+
+pub static DEFAULT_MAX_DBS: c_uint = 5;
+
+/// Wrapper around an `Environment` (e.g. such as an `LMDB` or `SafeMode` environment).
+#[derive(Debug)]
+pub struct Rkv<E> {
+ _path: PathBuf,
+ env: E,
+}
+
+/// Static methods.
+impl<'e, E> Rkv<E>
+where
+ E: BackendEnvironment<'e>,
+{
+ pub fn environment_builder<B>() -> B
+ where
+ B: BackendEnvironmentBuilder<'e, Environment = E>,
+ {
+ B::new()
+ }
+
+ /// Return a new Rkv environment that supports up to `DEFAULT_MAX_DBS` open databases.
+ #[allow(clippy::new_ret_no_self)]
+ pub fn new<B>(path: &Path) -> Result<Rkv<E>, StoreError>
+ where
+ B: BackendEnvironmentBuilder<'e, Environment = E>,
+ {
+ Rkv::with_capacity::<B>(path, DEFAULT_MAX_DBS)
+ }
+
+ /// Return a new Rkv environment that supports the specified number of open databases.
+ pub fn with_capacity<B>(path: &Path, max_dbs: c_uint) -> Result<Rkv<E>, StoreError>
+ where
+ B: BackendEnvironmentBuilder<'e, Environment = E>,
+ {
+ let mut builder = B::new();
+ builder.set_max_dbs(max_dbs);
+
+ // Future: set flags, maximum size, etc. here if necessary.
+ Rkv::from_builder(path, builder)
+ }
+
+ /// Return a new Rkv environment from the provided builder.
+ pub fn from_builder<B>(path: &Path, builder: B) -> Result<Rkv<E>, StoreError>
+ where
+ B: BackendEnvironmentBuilder<'e, Environment = E>,
+ {
+ Ok(Rkv {
+ _path: path.into(),
+ env: builder.open(path).map_err(|e| e.into())?,
+ })
+ }
+}
+
+/// Store creation methods.
+impl<'e, E> Rkv<E>
+where
+ E: BackendEnvironment<'e>,
+{
+ /// Return all created databases.
+ pub fn get_dbs(&self) -> Result<Vec<Option<String>>, StoreError> {
+ self.env.get_dbs().map_err(|e| e.into())
+ }
+
+ /// Create or Open an existing database in (&[u8] -> Single Value) mode.
+ /// Note: that create=true cannot be called concurrently with other operations so if
+ /// you are sure that the database exists, call this with create=false.
+ pub fn open_single<'s, T>(
+ &self,
+ name: T,
+ opts: StoreOptions<E::Flags>,
+ ) -> Result<SingleStore<E::Database>, StoreError>
+ where
+ T: Into<Option<&'s str>>,
+ {
+ self.open(name, opts).map(SingleStore::new)
+ }
+
+ /// Create or Open an existing database in (Integer -> Single Value) mode.
+ /// Note: that create=true cannot be called concurrently with other operations so if
+ /// you are sure that the database exists, call this with create=false.
+ #[cfg(feature = "db-int-key")]
+ pub fn open_integer<'s, T, K>(
+ &self,
+ name: T,
+ mut opts: StoreOptions<E::Flags>,
+ ) -> Result<IntegerStore<E::Database, K>, StoreError>
+ where
+ K: PrimitiveInt,
+ T: Into<Option<&'s str>>,
+ {
+ opts.flags.set(DatabaseFlags::INTEGER_KEY, true);
+ self.open(name, opts).map(IntegerStore::new)
+ }
+
+ /// Create or Open an existing database in (&[u8] -> Multiple Values) mode.
+ /// Note: that create=true cannot be called concurrently with other operations so if
+ /// you are sure that the database exists, call this with create=false.
+ #[cfg(feature = "db-dup-sort")]
+ pub fn open_multi<'s, T>(
+ &self,
+ name: T,
+ mut opts: StoreOptions<E::Flags>,
+ ) -> Result<MultiStore<E::Database>, StoreError>
+ where
+ T: Into<Option<&'s str>>,
+ {
+ opts.flags.set(DatabaseFlags::DUP_SORT, true);
+ self.open(name, opts).map(MultiStore::new)
+ }
+
+ /// Create or Open an existing database in (Integer -> Multiple Values) mode.
+ /// Note: that create=true cannot be called concurrently with other operations so if
+ /// you are sure that the database exists, call this with create=false.
+ #[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+ pub fn open_multi_integer<'s, T, K>(
+ &self,
+ name: T,
+ mut opts: StoreOptions<E::Flags>,
+ ) -> Result<MultiIntegerStore<E::Database, K>, StoreError>
+ where
+ K: PrimitiveInt,
+ T: Into<Option<&'s str>>,
+ {
+ opts.flags.set(DatabaseFlags::INTEGER_KEY, true);
+ opts.flags.set(DatabaseFlags::DUP_SORT, true);
+ self.open(name, opts).map(MultiIntegerStore::new)
+ }
+
+ fn open<'s, T>(&self, name: T, opts: StoreOptions<E::Flags>) -> Result<E::Database, StoreError>
+ where
+ T: Into<Option<&'s str>>,
+ {
+ if opts.create {
+ self.env
+ .create_db(name.into(), opts.flags)
+ .map_err(|e| match e.into() {
+ #[cfg(feature = "lmdb")]
+ StoreError::LmdbError(lmdb::Error::BadRslot) => {
+ StoreError::open_during_transaction()
+ }
+ StoreError::SafeModeError(SafeModeError::DbsIllegalOpen) => {
+ StoreError::open_during_transaction()
+ }
+ e => e,
+ })
+ } else {
+ self.env.open_db(name.into()).map_err(|e| match e.into() {
+ #[cfg(feature = "lmdb")]
+ StoreError::LmdbError(lmdb::Error::BadRslot) => {
+ StoreError::open_during_transaction()
+ }
+ StoreError::SafeModeError(SafeModeError::DbsIllegalOpen) => {
+ StoreError::open_during_transaction()
+ }
+ e => e,
+ })
+ }
+ }
+}
+
+/// Read and write accessors.
+impl<'e, E> Rkv<E>
+where
+ E: BackendEnvironment<'e>,
+{
+ /// Create a read transaction. There can be multiple concurrent readers for an
+ /// environment, up to the maximum specified by LMDB (default 126), and you can open
+ /// readers while a write transaction is active.
+ pub fn read<T>(&'e self) -> Result<Reader<T>, StoreError>
+ where
+ E: BackendEnvironment<'e, RoTransaction = T>,
+ T: BackendRoCursorTransaction<'e, Database = E::Database>,
+ {
+ Ok(Reader::new(self.env.begin_ro_txn().map_err(|e| e.into())?))
+ }
+
+ /// Create a write transaction. There can be only one write transaction active at any
+ /// given time, so trying to create a second one will block until the first is
+ /// committed or aborted.
+ pub fn write<T>(&'e self) -> Result<Writer<T>, StoreError>
+ where
+ E: BackendEnvironment<'e, RwTransaction = T>,
+ T: BackendRwCursorTransaction<'e, Database = E::Database>,
+ {
+ Ok(Writer::new(self.env.begin_rw_txn().map_err(|e| e.into())?))
+ }
+}
+
+/// Other environment methods.
+impl<'e, E> Rkv<E>
+where
+ E: BackendEnvironment<'e>,
+{
+ /// Flush the data buffers to disk. This call is only useful, when the environment was
+ /// open with either `NO_SYNC`, `NO_META_SYNC` or `MAP_ASYNC` (see below). The call is
+ /// not valid if the environment was opened with `READ_ONLY`.
+ ///
+ /// Data is always written to disk when `transaction.commit()` is called, but the
+ /// operating system may keep it buffered. LMDB always flushes the OS buffers upon
+ /// commit as well, unless the environment was opened with `NO_SYNC` or in part
+ /// `NO_META_SYNC`.
+ ///
+ /// `force`: if true, force a synchronous flush. Otherwise if the environment has the
+ /// `NO_SYNC` flag set the flushes will be omitted, and with `MAP_ASYNC` they will
+ /// be asynchronous.
+ pub fn sync(&self, force: bool) -> Result<(), StoreError> {
+ self.env.sync(force).map_err(|e| e.into())
+ }
+
+ /// Retrieve statistics about this environment.
+ ///
+ /// It includes:
+ /// * Page size in bytes
+ /// * B-tree depth
+ /// * Number of internal (non-leaf) pages
+ /// * Number of leaf pages
+ /// * Number of overflow pages
+ /// * Number of data entries
+ pub fn stat(&self) -> Result<E::Stat, StoreError> {
+ self.env.stat().map_err(|e| e.into())
+ }
+
+ /// Retrieve information about this environment.
+ ///
+ /// It includes:
+ /// * Map size in bytes
+ /// * The last used page number
+ /// * The last transaction ID
+ /// * Max number of readers allowed
+ /// * Number of readers in use
+ pub fn info(&self) -> Result<E::Info, StoreError> {
+ self.env.info().map_err(|e| e.into())
+ }
+
+ /// Retrieve the load ratio (# of used pages / total pages) about this environment.
+ ///
+ /// With the formular: (last_page_no - freelist_pages) / total_pages.
+ /// A value of `None` means that the backend doesn't ever need to be resized.
+ pub fn load_ratio(&self) -> Result<Option<f32>, StoreError> {
+ self.env.load_ratio().map_err(|e| e.into())
+ }
+
+ /// Sets the size of the memory map to use for the environment.
+ ///
+ /// This can be used to resize the map when the environment is already open. You can
+ /// also use `Rkv::environment_builder()` to set the map size during the `Rkv`
+ /// initialization.
+ ///
+ /// Note:
+ ///
+ /// * No active transactions allowed when performing resizing in this process. It's up
+ /// to the consumer to enforce that.
+ ///
+ /// * The size should be a multiple of the OS page size. Any attempt to set a size
+ /// smaller than the space already consumed by the environment will be silently
+ /// changed to the current size of the used space.
+ ///
+ /// * In the multi-process case, once a process resizes the map, other processes need
+ /// to either re-open the environment, or call set_map_size with size 0 to update
+ /// the environment. Otherwise, new transaction creation will fail with
+ /// `LmdbError::MapResized`.
+ pub fn set_map_size(&self, size: usize) -> Result<(), StoreError> {
+ self.env.set_map_size(size).map_err(Into::into)
+ }
+
+ /// Closes this environment and optionally deletes all its files from disk. Doesn't
+ /// delete the folder used when opening the environment.
+ pub fn close(self, options: CloseOptions) -> Result<(), CloseError> {
+ let files = self.env.get_files_on_disk();
+ drop(self);
+
+ if options.delete {
+ for file in files {
+ fs::remove_file(file)?;
+ }
+ }
+
+ Ok(())
+ }
+}
diff --git a/third_party/rust/rkv/src/error.rs b/third_party/rust/rkv/src/error.rs
new file mode 100644
index 0000000000..f50dda73e4
--- /dev/null
+++ b/third_party/rust/rkv/src/error.rs
@@ -0,0 +1,150 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{io, path::PathBuf, sync, thread, thread::ThreadId};
+
+use thiserror::Error;
+
+pub use crate::backend::SafeModeError;
+use crate::value::Type;
+
+#[derive(Debug, Error)]
+pub enum DataError {
+ #[error("unknown type tag: {0}")]
+ UnknownType(u8),
+
+ #[error("unexpected type tag: expected {expected}, got {actual}")]
+ UnexpectedType { expected: Type, actual: Type },
+
+ #[error("empty data; expected tag")]
+ Empty,
+
+ #[error("invalid value for type {value_type}: {err}")]
+ DecodingError {
+ value_type: Type,
+ err: Box<bincode::ErrorKind>,
+ },
+
+ #[error("couldn't encode value: {0}")]
+ EncodingError(#[from] Box<bincode::ErrorKind>),
+
+ #[error("invalid uuid bytes")]
+ InvalidUuid,
+}
+
+#[derive(Debug, Error)]
+pub enum StoreError {
+ #[error("manager poisoned")]
+ ManagerPoisonError,
+
+ #[error("database corrupted")]
+ DatabaseCorrupted,
+
+ #[error("key/value pair not found")]
+ KeyValuePairNotFound,
+
+ #[error("unsupported size of key/DB name/data")]
+ KeyValuePairBadSize,
+
+ #[error("file is not a valid database")]
+ FileInvalid,
+
+ #[error("environment mapsize reached")]
+ MapFull,
+
+ #[error("environment maxdbs reached")]
+ DbsFull,
+
+ #[error("environment maxreaders reached")]
+ ReadersFull,
+
+ #[error("I/O error: {0:?}")]
+ IoError(#[from] io::Error),
+
+ #[error("environment path does not exist or not the right type: {0:?}")]
+ UnsuitableEnvironmentPath(PathBuf),
+
+ #[error("data error: {0:?}")]
+ DataError(#[from] DataError),
+
+ #[cfg(feature = "lmdb")]
+ #[error("lmdb backend error: {0}")]
+ LmdbError(lmdb::Error),
+
+ #[error("safe mode backend error: {0}")]
+ SafeModeError(SafeModeError),
+
+ #[error("read transaction already exists in thread {0:?}")]
+ ReadTransactionAlreadyExists(ThreadId),
+
+ #[error("attempted to open DB during transaction in thread {0:?}")]
+ OpenAttemptedDuringTransaction(ThreadId),
+}
+
+impl StoreError {
+ pub fn open_during_transaction() -> StoreError {
+ StoreError::OpenAttemptedDuringTransaction(thread::current().id())
+ }
+
+ pub fn read_transaction_already_exists() -> StoreError {
+ StoreError::ReadTransactionAlreadyExists(thread::current().id())
+ }
+}
+
+impl<T> From<sync::PoisonError<T>> for StoreError {
+ fn from(_: sync::PoisonError<T>) -> StoreError {
+ StoreError::ManagerPoisonError
+ }
+}
+
+#[derive(Debug, Error)]
+pub enum CloseError {
+ #[error("manager poisoned")]
+ ManagerPoisonError,
+
+ #[error("close attempted while manager has an environment still open")]
+ EnvironmentStillOpen,
+
+ #[error("close attempted while an environment not known to the manager is still open")]
+ UnknownEnvironmentStillOpen,
+
+ #[error("I/O error: {0:?}")]
+ IoError(#[from] io::Error),
+}
+
+impl<T> From<sync::PoisonError<T>> for CloseError {
+ fn from(_: sync::PoisonError<T>) -> CloseError {
+ CloseError::ManagerPoisonError
+ }
+}
+
+#[derive(Debug, Error)]
+pub enum MigrateError {
+ #[error("store error: {0}")]
+ StoreError(#[from] StoreError),
+
+ #[error("close error: {0}")]
+ CloseError(#[from] CloseError),
+
+ #[error("manager poisoned")]
+ ManagerPoisonError,
+
+ #[error("source is empty")]
+ SourceEmpty,
+
+ #[error("destination is not empty")]
+ DestinationNotEmpty,
+}
+
+impl<T> From<sync::PoisonError<T>> for MigrateError {
+ fn from(_: sync::PoisonError<T>) -> MigrateError {
+ MigrateError::ManagerPoisonError
+ }
+}
diff --git a/third_party/rust/rkv/src/helpers.rs b/third_party/rust/rkv/src/helpers.rs
new file mode 100644
index 0000000000..89dcdf8dd7
--- /dev/null
+++ b/third_party/rust/rkv/src/helpers.rs
@@ -0,0 +1,43 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ io,
+ path::{Path, PathBuf},
+};
+
+use url::Url;
+
+use crate::{error::StoreError, value::Value};
+
+pub(crate) fn read_transform(value: Result<&[u8], StoreError>) -> Result<Value, StoreError> {
+ match value {
+ Ok(bytes) => Value::from_tagged_slice(bytes).map_err(StoreError::DataError),
+ Err(e) => Err(e),
+ }
+}
+
+// Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869.
+// Otherwise, `Env::from_builder()` will panic with error_no(123).
+pub(crate) fn canonicalize_path<'p, P>(path: P) -> io::Result<PathBuf>
+where
+ P: Into<&'p Path>,
+{
+ let canonical = path.into().canonicalize()?;
+
+ Ok(if cfg!(target_os = "windows") {
+ let map_err = |_| io::Error::new(io::ErrorKind::Other, "path canonicalization error");
+ Url::from_file_path(&canonical)
+ .and_then(|url| url.to_file_path())
+ .map_err(map_err)?
+ } else {
+ canonical
+ })
+}
diff --git a/third_party/rust/rkv/src/lib.rs b/third_party/rust/rkv/src/lib.rs
new file mode 100644
index 0000000000..0c8951e191
--- /dev/null
+++ b/third_party/rust/rkv/src/lib.rs
@@ -0,0 +1,236 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![allow(clippy::from_over_into)] // TODO: `Into` implementations in [safe/lmdb]/flags.rs
+
+//! A simple, humane, typed key-value storage solution. It supports multiple backend
+//! engines with varying guarantees, such as [LMDB](http://www.lmdb.tech/doc/) for
+//! performance, or "SafeMode" for reliability.
+//!
+//! It aims to achieve the following:
+//!
+//! - Avoid sharp edges (e.g., obscure error codes for common situations).
+//! - Correctly restrict access to one handle per process via a
+//! [Manager](struct.Manager.html).
+//! - Use Rust's type system to make single-typed key stores safe and ergonomic.
+//! - Encode and decode values via [bincode](https://docs.rs/bincode/)/[serde](https://docs.rs/serde/)
+//! and type tags, achieving platform-independent storage and input/output flexibility.
+//!
+//! It exposes these primary abstractions:
+//!
+//! - [Manager](struct.Manager.html): a singleton that controls access to environments
+//! - [Rkv](struct.Rkv.html): an environment contains a set of key/value databases
+//! - [SingleStore](store/single/struct.SingleStore.html): a database contains a set of
+//! key/value pairs
+//!
+//! Keys can be anything that implements `AsRef<[u8]>` or integers
+//! (when accessing an [IntegerStore](store/integer/struct.IntegerStore.html)).
+//!
+//! Values can be any of the types defined by the [Value](value/enum.Value.html) enum,
+//! including:
+//!
+//! - booleans (`Value::Bool`)
+//! - integers (`Value::I64`, `Value::U64`)
+//! - floats (`Value::F64`)
+//! - strings (`Value::Str`)
+//! - blobs (`Value::Blob`)
+//!
+//! See [Value](value/enum.Value.html) for the complete list of supported types.
+//!
+//! ## Basic Usage
+//! ```
+//! use rkv::{Manager, Rkv, SingleStore, Value, StoreOptions};
+//! use rkv::backend::{SafeMode, SafeModeEnvironment};
+//! use std::fs;
+//! use tempfile::Builder;
+//!
+//! // First determine the path to the environment, which is represented on disk as a
+//! // directory containing two files:
+//! //
+//! // * a data file containing the key/value stores
+//! // * a lock file containing metadata about current transactions
+//! //
+//! // In this example, we use the `tempfile` crate to create the directory.
+//! //
+//! let root = Builder::new().prefix("simple-db").tempdir().unwrap();
+//! fs::create_dir_all(root.path()).unwrap();
+//! let path = root.path();
+//!
+//! // The `Manager` enforces that each process opens the same environment at most once by
+//! // caching a handle to each environment that it opens. Use it to retrieve the handle
+//! // to an opened environment—or create one if it hasn't already been opened:
+//! let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
+//! let created_arc = manager.get_or_create(path, Rkv::new::<SafeMode>).unwrap();
+//! let env = created_arc.read().unwrap();
+//!
+//! // Then you can use the environment handle to get a handle to a datastore:
+//! let store = env.open_single("mydb", StoreOptions::create()).unwrap();
+//!
+//! {
+//! // Use a write transaction to mutate the store via a `Writer`. There can be only
+//! // one writer for a given environment, so opening a second one will block until
+//! // the first completes.
+//! let mut writer = env.write().unwrap();
+//!
+//! // Keys are `AsRef<[u8]>`, while values are `Value` enum instances. Use the `Blob`
+//! // variant to store arbitrary collections of bytes. Putting data returns a
+//! // `Result<(), StoreError>`, where StoreError is an enum identifying the reason
+//! // for a failure.
+//! store.put(&mut writer, "int", &Value::I64(1234)).unwrap();
+//! store.put(&mut writer, "uint", &Value::U64(1234_u64)).unwrap();
+//! store.put(&mut writer, "float", &Value::F64(1234.0.into())).unwrap();
+//! store.put(&mut writer, "instant", &Value::Instant(1528318073700)).unwrap();
+//! store.put(&mut writer, "boolean", &Value::Bool(true)).unwrap();
+//! store.put(&mut writer, "string", &Value::Str("Héllo, wörld!")).unwrap();
+//! store.put(&mut writer, "json", &Value::Json(r#"{"foo":"bar", "number": 1}"#)).unwrap();
+//! store.put(&mut writer, "blob", &Value::Blob(b"blob")).unwrap();
+//!
+//! // You must commit a write transaction before the writer goes out of scope, or the
+//! // transaction will abort and the data won't persist.
+//! writer.commit().unwrap();
+//! }
+//!
+//! {
+//! // Use a read transaction to query the store via a `Reader`. There can be multiple
+//! // concurrent readers for a store, and readers never block on a writer nor other
+//! // readers.
+//! let reader = env.read().expect("reader");
+//!
+//! // Keys are `AsRef<u8>`, and the return value is `Result<Option<Value>, StoreError>`.
+//! println!("Get int {:?}", store.get(&reader, "int").unwrap());
+//! println!("Get uint {:?}", store.get(&reader, "uint").unwrap());
+//! println!("Get float {:?}", store.get(&reader, "float").unwrap());
+//! println!("Get instant {:?}", store.get(&reader, "instant").unwrap());
+//! println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap());
+//! println!("Get string {:?}", store.get(&reader, "string").unwrap());
+//! println!("Get json {:?}", store.get(&reader, "json").unwrap());
+//! println!("Get blob {:?}", store.get(&reader, "blob").unwrap());
+//!
+//! // Retrieving a non-existent value returns `Ok(None)`.
+//! println!("Get non-existent value {:?}", store.get(&reader, "non-existent").unwrap());
+//!
+//! // A read transaction will automatically close once the reader goes out of scope,
+//! // so isn't necessary to close it explicitly, although you can do so by calling
+//! // `Reader.abort()`.
+//! }
+//!
+//! {
+//! // Aborting a write transaction rolls back the change(s).
+//! let mut writer = env.write().unwrap();
+//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+//! writer.abort();
+//! let reader = env.read().expect("reader");
+//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
+//! }
+//!
+//! {
+//! // Explicitly aborting a transaction is not required unless an early abort is
+//! // desired, since both read and write transactions will implicitly be aborted once
+//! // they go out of scope.
+//! {
+//! let mut writer = env.write().unwrap();
+//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+//! }
+//! let reader = env.read().expect("reader");
+//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
+//! }
+//!
+//! {
+//! // Deleting a key/value pair also requires a write transaction.
+//! let mut writer = env.write().unwrap();
+//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+//! store.put(&mut writer, "bar", &Value::Str("baz")).unwrap();
+//! store.delete(&mut writer, "foo").unwrap();
+//!
+//! // A write transaction also supports reading, and the version of the store that it
+//! // reads includes the changes it has made regardless of the commit state of that
+//! // transaction.
+
+//! // In the code above, "foo" and "bar" were put into the store, then "foo" was
+//! // deleted so only "bar" will return a result when the database is queried via the
+//! // writer.
+//! println!("It should be None! ({:?})", store.get(&writer, "foo").unwrap());
+//! println!("Get bar ({:?})", store.get(&writer, "bar").unwrap());
+//!
+//! // But a reader won't see that change until the write transaction is committed.
+//! {
+//! let reader = env.read().expect("reader");
+//! println!("Get foo {:?}", store.get(&reader, "foo").unwrap());
+//! println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
+//! }
+//! writer.commit().unwrap();
+//! {
+//! let reader = env.read().expect("reader");
+//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
+//! println!("Get bar {:?}", store.get(&reader, "bar").unwrap());
+//! }
+//!
+//! // Committing a transaction consumes the writer, preventing you from reusing it by
+//! // failing at compile time with an error. This line would report "error[E0382]:
+//! // borrow of moved value: `writer`".
+//! // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap();
+//! }
+//!
+//! {
+//! // Clearing all the entries in the store with a write transaction.
+//! {
+//! let mut writer = env.write().unwrap();
+//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap();
+//! store.put(&mut writer, "bar", &Value::Str("baz")).unwrap();
+//! writer.commit().unwrap();
+//! }
+//!
+//! {
+//! let mut writer = env.write().unwrap();
+//! store.clear(&mut writer).unwrap();
+//! writer.commit().unwrap();
+//! }
+//!
+//! {
+//! let reader = env.read().expect("reader");
+//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap());
+//! println!("It should be None! ({:?})", store.get(&reader, "bar").unwrap());
+//! }
+//!
+//! }
+//!
+//! ```
+
+mod env;
+mod error;
+mod helpers;
+mod manager;
+mod readwrite;
+
+pub mod backend;
+#[cfg(feature = "lmdb")]
+pub mod migrator;
+pub mod store;
+pub mod value;
+
+pub use backend::{DatabaseFlags, EnvironmentFlags, WriteFlags};
+pub use env::Rkv;
+pub use error::{DataError, MigrateError, StoreError};
+pub use manager::Manager;
+#[cfg(feature = "lmdb")]
+pub use migrator::Migrator;
+pub use readwrite::{Readable, Reader, Writer};
+pub use store::{keys::EncodableKey, single::SingleStore, CloseOptions, Options as StoreOptions};
+pub use value::{OwnedValue, Value};
+
+#[cfg(feature = "db-dup-sort")]
+pub use store::multi::MultiStore;
+
+#[cfg(feature = "db-int-key")]
+pub use store::integer::IntegerStore;
+#[cfg(feature = "db-int-key")]
+pub use store::keys::PrimitiveInt;
+
+#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+pub use store::integermulti::MultiIntegerStore;
diff --git a/third_party/rust/rkv/src/manager.rs b/third_party/rust/rkv/src/manager.rs
new file mode 100644
index 0000000000..045564497c
--- /dev/null
+++ b/third_party/rust/rkv/src/manager.rs
@@ -0,0 +1,247 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{
+ collections::{btree_map::Entry, BTreeMap},
+ os::raw::c_uint,
+ path::{Path, PathBuf},
+ result,
+ sync::{Arc, RwLock},
+};
+
+use lazy_static::lazy_static;
+
+#[cfg(feature = "lmdb")]
+use crate::backend::LmdbEnvironment;
+use crate::{
+ backend::{BackendEnvironment, BackendEnvironmentBuilder, SafeModeEnvironment},
+ error::{CloseError, StoreError},
+ helpers::canonicalize_path,
+ store::CloseOptions,
+ Rkv,
+};
+
+type Result<T> = result::Result<T, StoreError>;
+type CloseResult<T> = result::Result<T, CloseError>;
+type SharedRkv<E> = Arc<RwLock<Rkv<E>>>;
+
+#[cfg(feature = "lmdb")]
+lazy_static! {
+ static ref MANAGER_LMDB: RwLock<Manager<LmdbEnvironment>> = RwLock::new(Manager::new());
+}
+
+lazy_static! {
+ static ref MANAGER_SAFE_MODE: RwLock<Manager<SafeModeEnvironment>> =
+ RwLock::new(Manager::new());
+}
+
+/// A process is only permitted to have one open handle to each Rkv environment. This
+/// manager exists to enforce that constraint: don't open environments directly.
+///
+/// By default, path canonicalization is enabled for identifying RKV instances. This
+/// is true by default, because it helps enforce the constraints guaranteed by
+/// this manager. However, path canonicalization might crash in some fringe
+/// circumstances, so the `no-canonicalize-path` feature offers the possibility of
+/// disabling it. See: https://bugzilla.mozilla.org/show_bug.cgi?id=1531887
+///
+/// When path canonicalization is disabled, you *must* ensure an RKV environment is
+/// always created or retrieved with the same path.
+pub struct Manager<E> {
+ environments: BTreeMap<PathBuf, SharedRkv<E>>,
+}
+
+impl<'e, E> Manager<E>
+where
+ E: BackendEnvironment<'e>,
+{
+ fn new() -> Manager<E> {
+ Manager {
+ environments: Default::default(),
+ }
+ }
+
+ /// Return the open env at `path`, returning `None` if it has not already been opened.
+ pub fn get<'p, P>(&self, path: P) -> Result<Option<SharedRkv<E>>>
+ where
+ P: Into<&'p Path>,
+ {
+ let canonical = if cfg!(feature = "no-canonicalize-path") {
+ path.into().to_path_buf()
+ } else {
+ canonicalize_path(path)?
+ };
+ Ok(self.environments.get(&canonical).cloned())
+ }
+
+ /// Return the open env at `path`, or create it by calling `f`.
+ pub fn get_or_create<'p, F, P>(&mut self, path: P, f: F) -> Result<SharedRkv<E>>
+ where
+ F: FnOnce(&Path) -> Result<Rkv<E>>,
+ P: Into<&'p Path>,
+ {
+ let canonical = if cfg!(feature = "no-canonicalize-path") {
+ path.into().to_path_buf()
+ } else {
+ canonicalize_path(path)?
+ };
+ Ok(match self.environments.entry(canonical) {
+ Entry::Occupied(e) => e.get().clone(),
+ Entry::Vacant(e) => {
+ let k = Arc::new(RwLock::new(f(e.key().as_path())?));
+ e.insert(k).clone()
+ }
+ })
+ }
+
+ /// Return the open env at `path` with `capacity`, or create it by calling `f`.
+ pub fn get_or_create_with_capacity<'p, F, P>(
+ &mut self,
+ path: P,
+ capacity: c_uint,
+ f: F,
+ ) -> Result<SharedRkv<E>>
+ where
+ F: FnOnce(&Path, c_uint) -> Result<Rkv<E>>,
+ P: Into<&'p Path>,
+ {
+ let canonical = if cfg!(feature = "no-canonicalize-path") {
+ path.into().to_path_buf()
+ } else {
+ canonicalize_path(path)?
+ };
+ Ok(match self.environments.entry(canonical) {
+ Entry::Occupied(e) => e.get().clone(),
+ Entry::Vacant(e) => {
+ let k = Arc::new(RwLock::new(f(e.key().as_path(), capacity)?));
+ e.insert(k).clone()
+ }
+ })
+ }
+
+ /// Return a new Rkv environment from the builder, or create it by calling `f`.
+ pub fn get_or_create_from_builder<'p, F, P, B>(
+ &mut self,
+ path: P,
+ builder: B,
+ f: F,
+ ) -> Result<SharedRkv<E>>
+ where
+ F: FnOnce(&Path, B) -> Result<Rkv<E>>,
+ P: Into<&'p Path>,
+ B: BackendEnvironmentBuilder<'e, Environment = E>,
+ {
+ let canonical = if cfg!(feature = "no-canonicalize-path") {
+ path.into().to_path_buf()
+ } else {
+ canonicalize_path(path)?
+ };
+ Ok(match self.environments.entry(canonical) {
+ Entry::Occupied(e) => e.get().clone(),
+ Entry::Vacant(e) => {
+ let k = Arc::new(RwLock::new(f(e.key().as_path(), builder)?));
+ e.insert(k).clone()
+ }
+ })
+ }
+
+ /// Tries to close the specified environment.
+ /// Returns an error when other users of this environment still exist.
+ pub fn try_close<'p, P>(&mut self, path: P, options: CloseOptions) -> CloseResult<()>
+ where
+ P: Into<&'p Path>,
+ {
+ let canonical = if cfg!(feature = "no-canonicalize-path") {
+ path.into().to_path_buf()
+ } else {
+ canonicalize_path(path)?
+ };
+ match self.environments.entry(canonical) {
+ Entry::Vacant(_) => Ok(()),
+ Entry::Occupied(e) if Arc::strong_count(e.get()) > 1 => {
+ Err(CloseError::EnvironmentStillOpen)
+ }
+ Entry::Occupied(e) => {
+ let env = Arc::try_unwrap(e.remove())
+ .map_err(|_| CloseError::UnknownEnvironmentStillOpen)?;
+ env.into_inner()?.close(options)?;
+ Ok(())
+ }
+ }
+ }
+}
+
+#[cfg(feature = "lmdb")]
+impl Manager<LmdbEnvironment> {
+ pub fn singleton() -> &'static RwLock<Manager<LmdbEnvironment>> {
+ &MANAGER_LMDB
+ }
+}
+
+impl Manager<SafeModeEnvironment> {
+ pub fn singleton() -> &'static RwLock<Manager<SafeModeEnvironment>> {
+ &MANAGER_SAFE_MODE
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::*;
+
+ use std::fs;
+
+ use tempfile::Builder;
+
+ #[cfg(feature = "lmdb")]
+ use backend::Lmdb;
+
+ /// Test that one can mutate managed Rkv instances in surprising ways.
+ #[cfg(feature = "lmdb")]
+ #[test]
+ fn test_mutate_managed_rkv() {
+ let mut manager = Manager::<LmdbEnvironment>::new();
+
+ let root1 = Builder::new()
+ .prefix("test_mutate_managed_rkv_1")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root1.path()).expect("dir created");
+ let path1 = root1.path();
+ let arc = manager
+ .get_or_create(path1, Rkv::new::<Lmdb>)
+ .expect("created");
+
+ // Arc<RwLock<>> has interior mutability, so we can replace arc's Rkv instance with a new
+ // instance that has a different path.
+ let root2 = Builder::new()
+ .prefix("test_mutate_managed_rkv_2")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root2.path()).expect("dir created");
+ let path2 = root2.path();
+ {
+ let mut rkv = arc.write().expect("guard");
+ let rkv2 = Rkv::new::<Lmdb>(path2).expect("Rkv");
+ *rkv = rkv2;
+ }
+
+ // Arc now has a different internal Rkv with path2, but it's still mapped to path1 in
+ // manager, so its pointer is equal to a new Arc for path1.
+ let path1_arc = manager.get(path1).expect("success").expect("existed");
+ assert!(Arc::ptr_eq(&path1_arc, &arc));
+
+ // Meanwhile, a new Arc for path2 has a different pointer, even though its Rkv's path is
+ // the same as arc's current path.
+ let path2_arc = manager
+ .get_or_create(path2, Rkv::new::<Lmdb>)
+ .expect("success");
+ assert!(!Arc::ptr_eq(&path2_arc, &arc));
+ }
+}
diff --git a/third_party/rust/rkv/src/migrator.rs b/third_party/rust/rkv/src/migrator.rs
new file mode 100644
index 0000000000..6dc8d60371
--- /dev/null
+++ b/third_party/rust/rkv/src/migrator.rs
@@ -0,0 +1,179 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+//! A simple utility for migrating data from one RVK environment to another. Notably, this
+//! tool can migrate data from an enviroment created with a different backend than the
+//! current RKV consumer (e.g from Lmdb to SafeMode).
+//!
+//! The utility doesn't support migrating between 32-bit and 64-bit LMDB environments yet,
+//! see `arch_migrator` if this is needed. However, this utility is ultimately intended to
+//! handle all possible migrations.
+//!
+//! The destination environment should be empty of data, otherwise an error is returned.
+//!
+//! There are 3 versions of the migration methods:
+//! * `migrate_<src>_to_<dst>`, where `<src>` and `<dst>` are the source and destination
+//! environment types. You're responsive with opening both these environments, handling
+//! all errors, and performing any cleanup if necessary.
+//! * `open_and_migrate_<src>_to_<dst>`, which is similar the the above, but automatically
+//! attempts to open the source environment and delete all of its supporting files if
+//! there's no other environment open at that path. You're still responsible with
+//! handling all errors.
+//! * `easy_migrate_<src>_to_<dst>` which is similar to the above, but ignores the
+//! migration and doesn't delete any files if the source environment is invalid
+//! (corrupted), unavailable (path not accessible or incompatible with configuration),
+//! or empty (database has no records).
+//!
+//! The tool currently has these limitations:
+//!
+//! 1. It doesn't support migration from environments created with
+//! `EnvironmentFlags::NO_SUB_DIR`. To migrate such an environment, create a temporary
+//! directory, copy the environment's data files in the temporary directory, then
+//! migrate the temporary directory as the source environment.
+//! 2. It doesn't support migration from databases created with DatabaseFlags::DUP_SORT`
+//! (with or without `DatabaseFlags::DUP_FIXED`) nor with `DatabaseFlags::INTEGER_KEY`.
+//! This effectively means that migration is limited to `SingleStore`s.
+//! 3. It doesn't allow for existing data in the destination environment, which means that
+//! it cannot overwrite nor append data.
+
+use crate::{
+ backend::{LmdbEnvironment, SafeModeEnvironment},
+ error::MigrateError,
+ Rkv, StoreOptions,
+};
+
+pub use crate::backend::{LmdbArchMigrateError, LmdbArchMigrateResult, LmdbArchMigrator};
+
+// FIXME: should parametrize this instead.
+
+macro_rules! fn_migrator {
+ ($name:tt, $src_env:ty, $dst_env:ty) => {
+ /// Migrate all data in all of databases from the source environment to the destination
+ /// environment. This includes all key/value pairs in the main database that aren't
+ /// metadata about subdatabases and all key/value pairs in all subdatabases.
+ ///
+ /// Other backend-specific metadata such as map size or maximum databases left intact on
+ /// the given environments.
+ ///
+ /// The destination environment should be empty of data, otherwise an error is returned.
+ pub fn $name<S, D>(src_env: S, dst_env: D) -> Result<(), MigrateError>
+ where
+ S: std::ops::Deref<Target = Rkv<$src_env>>,
+ D: std::ops::Deref<Target = Rkv<$dst_env>>,
+ {
+ let src_dbs = src_env.get_dbs().unwrap();
+ if src_dbs.is_empty() {
+ return Err(MigrateError::SourceEmpty);
+ }
+ let dst_dbs = dst_env.get_dbs().unwrap();
+ if !dst_dbs.is_empty() {
+ return Err(MigrateError::DestinationNotEmpty);
+ }
+ for name in src_dbs {
+ let src_store = src_env.open_single(name.as_deref(), StoreOptions::default())?;
+ let dst_store = dst_env.open_single(name.as_deref(), StoreOptions::create())?;
+ let reader = src_env.read()?;
+ let mut writer = dst_env.write()?;
+ let mut iter = src_store.iter_start(&reader)?;
+ while let Some(Ok((key, value))) = iter.next() {
+ dst_store.put(&mut writer, key, &value).expect("wrote");
+ }
+ writer.commit()?;
+ }
+ Ok(())
+ }
+ };
+
+ (open $migrate:tt, $name:tt, $builder:tt, $src_env:ty, $dst_env:ty) => {
+ /// Same as the the `migrate_x_to_y` migration method above, but automatically attempts
+ /// to open the source environment. Finally, deletes all of its supporting files if
+ /// there's no other environment open at that path and the migration succeeded.
+ pub fn $name<F, D>(path: &std::path::Path, build: F, dst_env: D) -> Result<(), MigrateError>
+ where
+ F: FnOnce(crate::backend::$builder) -> crate::backend::$builder,
+ D: std::ops::Deref<Target = Rkv<$dst_env>>,
+ {
+ use crate::{backend::*, CloseOptions};
+
+ let mut manager = crate::Manager::<$src_env>::singleton().write()?;
+ let mut builder = Rkv::<$src_env>::environment_builder::<$builder>();
+ builder.set_max_dbs(crate::env::DEFAULT_MAX_DBS);
+ builder = build(builder);
+
+ let src_env =
+ manager.get_or_create_from_builder(path, builder, Rkv::from_builder::<$builder>)?;
+ Migrator::$migrate(src_env.read()?, dst_env)?;
+
+ drop(src_env);
+ manager.try_close(path, CloseOptions::delete_files_on_disk())?;
+
+ Ok(())
+ }
+ };
+
+ (easy $migrate:tt, $name:tt, $src_env:ty, $dst_env:ty) => {
+ /// Same as the `open_and_migrate_x_to_y` migration method above, but ignores the
+ /// migration and doesn't delete any files if the following conditions apply:
+ /// - Source environment is invalid/corrupted, unavailable, or empty.
+ /// - Destination environment is not empty.
+ /// Use this instead of the other migration methods if:
+ /// - You're not concerned by throwing away old data and starting fresh with a new store.
+ /// - You'll never want to overwrite data in the new store from the old store.
+ pub fn $name<D>(path: &std::path::Path, dst_env: D) -> Result<(), MigrateError>
+ where
+ D: std::ops::Deref<Target = Rkv<$dst_env>>,
+ {
+ match Migrator::$migrate(path, |builder| builder, dst_env) {
+ // Source environment is an invalid file or corrupted database.
+ Err(crate::MigrateError::StoreError(crate::StoreError::FileInvalid)) => Ok(()),
+ Err(crate::MigrateError::StoreError(crate::StoreError::DatabaseCorrupted)) => {
+ Ok(())
+ }
+ // Path not accessible.
+ Err(crate::MigrateError::StoreError(crate::StoreError::IoError(_))) => Ok(()),
+ // Path accessible but incompatible for configuration.
+ Err(crate::MigrateError::StoreError(
+ crate::StoreError::UnsuitableEnvironmentPath(_),
+ )) => Ok(()),
+ // Couldn't close source environment and delete files on disk (e.g. other stores still open).
+ Err(crate::MigrateError::CloseError(_)) => Ok(()),
+ // Nothing to migrate.
+ Err(crate::MigrateError::SourceEmpty) => Ok(()),
+ // Migrating would overwrite.
+ Err(crate::MigrateError::DestinationNotEmpty) => Ok(()),
+ result => result,
+ }?;
+
+ Ok(())
+ }
+ };
+}
+
+macro_rules! fns_migrator {
+ ($src:tt, $dst:tt) => {
+ paste::item! {
+ fns_migrator!([<migrate_ $src _to_ $dst>], $src, $dst);
+ fns_migrator!([<migrate_ $dst _to_ $src>], $dst, $src);
+ }
+ };
+ ($name:tt, $src:tt, $dst:tt) => {
+ paste::item! {
+ fn_migrator!($name, [<$src:camel Environment>], [<$dst:camel Environment>]);
+ fn_migrator!(open $name, [<open_and_ $name>], [<$src:camel>], [<$src:camel Environment>], [<$dst:camel Environment>]);
+ fn_migrator!(easy [<open_and_ $name>], [<easy_ $name>], [<$src:camel Environment>], [<$dst:camel Environment>]);
+ }
+ };
+}
+
+pub struct Migrator;
+
+impl Migrator {
+ fns_migrator!(lmdb, safe_mode);
+}
diff --git a/third_party/rust/rkv/src/readwrite.rs b/third_party/rust/rkv/src/readwrite.rs
new file mode 100644
index 0000000000..38b4746210
--- /dev/null
+++ b/third_party/rust/rkv/src/readwrite.rs
@@ -0,0 +1,154 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::{
+ backend::{
+ BackendDatabase, BackendRoCursor, BackendRoCursorTransaction, BackendRoTransaction,
+ BackendRwCursorTransaction, BackendRwTransaction,
+ },
+ error::StoreError,
+ helpers::read_transform,
+ value::Value,
+};
+
+pub struct Reader<T>(T);
+pub struct Writer<T>(T);
+
+pub trait Readable<'r> {
+ type Database: BackendDatabase;
+ type RoCursor: BackendRoCursor<'r>;
+
+ fn get<K>(&'r self, db: &Self::Database, k: &K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ K: AsRef<[u8]>;
+
+ fn open_ro_cursor(&'r self, db: &Self::Database) -> Result<Self::RoCursor, StoreError>;
+}
+
+impl<'r, T> Readable<'r> for Reader<T>
+where
+ T: BackendRoCursorTransaction<'r>,
+{
+ type Database = T::Database;
+ type RoCursor = T::RoCursor;
+
+ fn get<K>(&'r self, db: &T::Database, k: &K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ K: AsRef<[u8]>,
+ {
+ let bytes = self.0.get(db, k.as_ref()).map_err(|e| e.into());
+ match read_transform(bytes).map(Some) {
+ Err(StoreError::KeyValuePairNotFound) => Ok(None),
+ result => result,
+ }
+ }
+
+ fn open_ro_cursor(&'r self, db: &T::Database) -> Result<T::RoCursor, StoreError> {
+ self.0.open_ro_cursor(db).map_err(|e| e.into())
+ }
+}
+
+impl<T> Reader<T> {
+ pub(crate) fn new(txn: T) -> Reader<T> {
+ Reader(txn)
+ }
+}
+
+impl<T> Reader<T>
+where
+ T: BackendRoTransaction,
+{
+ pub fn abort(self) {
+ self.0.abort();
+ }
+}
+
+impl<'r, T> Readable<'r> for Writer<T>
+where
+ T: BackendRwCursorTransaction<'r>,
+{
+ type Database = T::Database;
+ type RoCursor = T::RoCursor;
+
+ fn get<K>(&'r self, db: &T::Database, k: &K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ K: AsRef<[u8]>,
+ {
+ let bytes = self.0.get(db, k.as_ref()).map_err(|e| e.into());
+ match read_transform(bytes).map(Some) {
+ Err(StoreError::KeyValuePairNotFound) => Ok(None),
+ result => result,
+ }
+ }
+
+ fn open_ro_cursor(&'r self, db: &T::Database) -> Result<T::RoCursor, StoreError> {
+ self.0.open_ro_cursor(db).map_err(|e| e.into())
+ }
+}
+
+impl<T> Writer<T> {
+ pub(crate) fn new(txn: T) -> Writer<T> {
+ Writer(txn)
+ }
+}
+
+impl<T> Writer<T>
+where
+ T: BackendRwTransaction,
+{
+ pub fn commit(self) -> Result<(), StoreError> {
+ self.0.commit().map_err(|e| e.into())
+ }
+
+ pub fn abort(self) {
+ self.0.abort();
+ }
+
+ pub(crate) fn put<K>(
+ &mut self,
+ db: &T::Database,
+ k: &K,
+ v: &Value,
+ flags: T::Flags,
+ ) -> Result<(), StoreError>
+ where
+ K: AsRef<[u8]>,
+ {
+ // TODO: don't allocate twice.
+ self.0
+ .put(db, k.as_ref(), &v.to_bytes()?, flags)
+ .map_err(|e| e.into())
+ }
+
+ #[cfg(not(feature = "db-dup-sort"))]
+ pub(crate) fn delete<K>(&mut self, db: &T::Database, k: &K) -> Result<(), StoreError>
+ where
+ K: AsRef<[u8]>,
+ {
+ self.0.del(db, k.as_ref()).map_err(|e| e.into())
+ }
+
+ #[cfg(feature = "db-dup-sort")]
+ pub(crate) fn delete<K>(
+ &mut self,
+ db: &T::Database,
+ k: &K,
+ v: Option<&[u8]>,
+ ) -> Result<(), StoreError>
+ where
+ K: AsRef<[u8]>,
+ {
+ self.0.del(db, k.as_ref(), v).map_err(|e| e.into())
+ }
+
+ pub(crate) fn clear(&mut self, db: &T::Database) -> Result<(), StoreError> {
+ self.0.clear_db(db).map_err(|e| e.into())
+ }
+}
diff --git a/third_party/rust/rkv/src/store.rs b/third_party/rust/rkv/src/store.rs
new file mode 100644
index 0000000000..c8ac2c583f
--- /dev/null
+++ b/third_party/rust/rkv/src/store.rs
@@ -0,0 +1,52 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+pub mod keys;
+pub mod single;
+
+#[cfg(feature = "db-dup-sort")]
+pub mod multi;
+
+#[cfg(feature = "db-int-key")]
+pub mod integer;
+
+#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+pub mod integermulti;
+
+use crate::backend::BackendDatabaseFlags;
+
+#[derive(Default, Debug, Copy, Clone)]
+pub struct Options<F> {
+ pub create: bool,
+ pub flags: F,
+}
+
+impl<F> Options<F>
+where
+ F: BackendDatabaseFlags,
+{
+ pub fn create() -> Options<F> {
+ Options {
+ create: true,
+ flags: F::empty(),
+ }
+ }
+}
+
+#[derive(Default, Debug, Copy, Clone)]
+pub struct CloseOptions {
+ pub delete: bool,
+}
+
+impl CloseOptions {
+ pub fn delete_files_on_disk() -> CloseOptions {
+ CloseOptions { delete: true }
+ }
+}
diff --git a/third_party/rust/rkv/src/store/integer.rs b/third_party/rust/rkv/src/store/integer.rs
new file mode 100644
index 0000000000..f150f09c5f
--- /dev/null
+++ b/third_party/rust/rkv/src/store/integer.rs
@@ -0,0 +1,637 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::marker::PhantomData;
+
+use crate::{
+ backend::{BackendDatabase, BackendRwTransaction},
+ error::StoreError,
+ readwrite::{Readable, Writer},
+ store::{
+ keys::{Key, PrimitiveInt},
+ single::SingleStore,
+ },
+ value::Value,
+};
+
+type EmptyResult = Result<(), StoreError>;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub struct IntegerStore<D, K> {
+ inner: SingleStore<D>,
+ phantom: PhantomData<K>,
+}
+
+impl<D, K> IntegerStore<D, K>
+where
+ D: BackendDatabase,
+ K: PrimitiveInt,
+{
+ pub(crate) fn new(db: D) -> IntegerStore<D, K> {
+ IntegerStore {
+ inner: SingleStore::new(db),
+ phantom: PhantomData,
+ }
+ }
+
+ pub fn get<'r, R>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ R: Readable<'r, Database = D>,
+ {
+ self.inner.get(reader, Key::new(&k)?)
+ }
+
+ pub fn put<T>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.put(writer, Key::new(&k)?, v)
+ }
+
+ pub fn delete<T>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.delete(writer, Key::new(&k)?)
+ }
+
+ pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.clear(writer)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::*;
+
+ use std::fs;
+
+ use tempfile::Builder;
+
+ #[test]
+ fn test_integer_keys() {
+ let root = Builder::new()
+ .prefix("test_integer_keys")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ macro_rules! test_integer_keys {
+ ($type:ty, $key:expr) => {{
+ let mut writer = k.write().expect("writer");
+
+ s.put(&mut writer, $key, &Value::Str("hello!"))
+ .expect("write");
+ assert_eq!(
+ s.get(&writer, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ s.get(&reader, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ }};
+ }
+
+ test_integer_keys!(u32, std::u32::MIN);
+ test_integer_keys!(u32, std::u32::MAX);
+ }
+
+ #[test]
+ fn test_clear() {
+ let root = Builder::new()
+ .prefix("test_integer_clear")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 3, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!")));
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), None);
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_dup() {
+ let root = Builder::new()
+ .prefix("test_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("foo!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("bar!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), None);
+ assert_eq!(s.get(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), None);
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_del() {
+ let root = Builder::new()
+ .prefix("test_integer_del")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("foo!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("bar!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), None);
+ assert_eq!(s.get(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1).expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 2).expect_err("not deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 3).expect_err("not deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_persist() {
+ let root = Builder::new()
+ .prefix("test_integer_persist")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 3, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!")));
+ writer.commit().expect("committed");
+ }
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 2).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 3).expect("read"), Some(Value::Str("hello!")));
+ }
+ }
+
+ #[test]
+ fn test_intertwine_read_write() {
+ let root = Builder::new()
+ .prefix("test_integer_intertwine_read_write")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), None);
+ assert_eq!(s.get(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ let reader = k.read().expect("reader");
+ let mut writer = k.write().expect("writer");
+
+ {
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+
+ {
+ s.put(&mut writer, 1, &Value::Str("goodbye!"))
+ .expect("write");
+ s.put(&mut writer, 2, &Value::Str("goodbye!"))
+ .expect("write");
+ s.put(&mut writer, 3, &Value::Str("goodbye!"))
+ .expect("write");
+ assert_eq!(
+ s.get(&writer, 1).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&writer, 2).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&writer, 3).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ writer.commit().expect("committed");
+ }
+
+ {
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(
+ s.get(&writer, 2).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&writer, 3).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ writer.commit().expect("committed");
+ }
+
+ {
+ let reader = k.write().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(
+ s.get(&reader, 2).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&reader, 3).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ reader.commit().expect("committed");
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests_safe {
+ use super::*;
+ use crate::*;
+
+ use std::fs;
+
+ use tempfile::Builder;
+
+ #[test]
+ fn test_integer_keys() {
+ let root = Builder::new()
+ .prefix("test_integer_keys")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ macro_rules! test_integer_keys {
+ ($type:ty, $key:expr) => {{
+ let mut writer = k.write().expect("writer");
+
+ s.put(&mut writer, $key, &Value::Str("hello!"))
+ .expect("write");
+ assert_eq!(
+ s.get(&writer, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ s.get(&reader, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ }};
+ }
+
+ test_integer_keys!(u32, std::u32::MIN);
+ test_integer_keys!(u32, std::u32::MAX);
+ }
+
+ #[test]
+ fn test_clear() {
+ let root = Builder::new()
+ .prefix("test_integer_clear")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 3, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!")));
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), None);
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_dup() {
+ let root = Builder::new()
+ .prefix("test_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("foo!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("bar!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), None);
+ assert_eq!(s.get(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), None);
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_del() {
+ let root = Builder::new()
+ .prefix("test_integer_del")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("foo!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("bar!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), None);
+ assert_eq!(s.get(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1).expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 2).expect_err("not deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 3).expect_err("not deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_persist() {
+ let root = Builder::new()
+ .prefix("test_integer_persist")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 3, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!")));
+ writer.commit().expect("committed");
+ }
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 2).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 3).expect("read"), Some(Value::Str("hello!")));
+ }
+ }
+
+ #[test]
+ fn test_intertwine_read_write() {
+ let root = Builder::new()
+ .prefix("test_integer_intertwine_read_write")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&writer, 2).expect("read"), None);
+ assert_eq!(s.get(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ let reader = k.read().expect("reader");
+ let mut writer = k.write().expect("writer");
+
+ {
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+
+ {
+ s.put(&mut writer, 1, &Value::Str("goodbye!"))
+ .expect("write");
+ s.put(&mut writer, 2, &Value::Str("goodbye!"))
+ .expect("write");
+ s.put(&mut writer, 3, &Value::Str("goodbye!"))
+ .expect("write");
+ assert_eq!(
+ s.get(&writer, 1).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&writer, 2).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&writer, 3).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ writer.commit().expect("committed");
+ }
+
+ {
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(s.get(&reader, 2).expect("read"), None);
+ assert_eq!(s.get(&reader, 3).expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(
+ s.get(&writer, 2).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&writer, 3).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ writer.commit().expect("committed");
+ }
+
+ {
+ let reader = k.write().expect("reader");
+ assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!")));
+ assert_eq!(
+ s.get(&reader, 2).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ assert_eq!(
+ s.get(&reader, 3).expect("read"),
+ Some(Value::Str("goodbye!"))
+ );
+ reader.commit().expect("committed");
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/store/integermulti.rs b/third_party/rust/rkv/src/store/integermulti.rs
new file mode 100644
index 0000000000..71dd0aae04
--- /dev/null
+++ b/third_party/rust/rkv/src/store/integermulti.rs
@@ -0,0 +1,709 @@
+// Copyright 2018 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::marker::PhantomData;
+
+use crate::{
+ backend::{BackendDatabase, BackendIter, BackendRoCursor, BackendRwTransaction},
+ error::StoreError,
+ readwrite::{Readable, Writer},
+ store::{
+ keys::{Key, PrimitiveInt},
+ multi::{Iter, MultiStore},
+ },
+ value::Value,
+};
+
+type EmptyResult = Result<(), StoreError>;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub struct MultiIntegerStore<D, K> {
+ inner: MultiStore<D>,
+ phantom: PhantomData<K>,
+}
+
+impl<D, K> MultiIntegerStore<D, K>
+where
+ D: BackendDatabase,
+ K: PrimitiveInt,
+{
+ pub(crate) fn new(db: D) -> MultiIntegerStore<D, K> {
+ MultiIntegerStore {
+ inner: MultiStore::new(db),
+ phantom: PhantomData,
+ }
+ }
+
+ pub fn get<'r, R, I, C>(&self, reader: &'r R, k: K) -> Result<Iter<'r, I>, StoreError>
+ where
+ R: Readable<'r, Database = D, RoCursor = C>,
+ I: BackendIter<'r>,
+ C: BackendRoCursor<'r, Iter = I>,
+ K: 'r,
+ {
+ self.inner.get(reader, Key::new(&k)?)
+ }
+
+ pub fn get_first<'r, R>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ R: Readable<'r, Database = D>,
+ {
+ self.inner.get_first(reader, Key::new(&k)?)
+ }
+
+ pub fn put<T>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.put(writer, Key::new(&k)?, v)
+ }
+
+ pub fn put_with_flags<T>(
+ &self,
+ writer: &mut Writer<T>,
+ k: K,
+ v: &Value,
+ flags: T::Flags,
+ ) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.put_with_flags(writer, Key::new(&k)?, v, flags)
+ }
+
+ pub fn delete_all<T>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.delete_all(writer, Key::new(&k)?)
+ }
+
+ pub fn delete<T>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.delete(writer, Key::new(&k)?, v)
+ }
+
+ pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ self.inner.clear(writer)
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use crate::*;
+
+ use std::fs;
+
+ use tempfile::Builder;
+
+ #[test]
+ fn test_integer_keys() {
+ let root = Builder::new()
+ .prefix("test_integer_keys")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ macro_rules! test_integer_keys {
+ ($type:ty, $key:expr) => {{
+ let mut writer = k.write().expect("writer");
+
+ s.put(&mut writer, $key, &Value::Str("hello!"))
+ .expect("write");
+ assert_eq!(
+ s.get_first(&writer, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ s.get_first(&reader, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ }};
+ }
+
+ test_integer_keys!(u32, std::u32::MIN);
+ test_integer_keys!(u32, std::u32::MAX);
+ }
+
+ #[test]
+ fn test_clear() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_clear")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ assert_eq!(
+ s.get_first(&writer, 1).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ assert_eq!(
+ s.get_first(&writer, 2).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ assert_eq!(s.get_first(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get_first(&reader, 1).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 2).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_dup() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ assert_eq!(
+ s.get_first(&writer, 1).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ assert_eq!(s.get_first(&writer, 2).expect("read"), None);
+ assert_eq!(s.get_first(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get_first(&reader, 1).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 2).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_dup_2() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+
+ let mut iter = s.get(&writer, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ }
+
+ #[test]
+ fn test_del() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ {
+ let mut iter = s.get(&writer, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello!"))
+ .expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello!"))
+ .expect_err("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert!(iter.next().is_none());
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello1!"))
+ .expect_err("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert!(iter.next().is_none());
+ }
+ }
+
+ #[test]
+ fn test_persist() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_persist")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ {
+ let mut iter = s.get(&writer, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ writer.commit().expect("committed");
+ }
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests_safe {
+ use super::*;
+ use crate::*;
+
+ use std::fs;
+
+ use tempfile::Builder;
+
+ #[test]
+ fn test_integer_keys() {
+ let root = Builder::new()
+ .prefix("test_integer_keys")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ macro_rules! test_integer_keys {
+ ($type:ty, $key:expr) => {{
+ let mut writer = k.write().expect("writer");
+
+ s.put(&mut writer, $key, &Value::Str("hello!"))
+ .expect("write");
+ assert_eq!(
+ s.get_first(&writer, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ s.get_first(&reader, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ }};
+ }
+
+ test_integer_keys!(u32, std::u32::MIN);
+ test_integer_keys!(u32, std::u32::MAX);
+ }
+
+ #[test]
+ fn test_clear() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_clear")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ assert_eq!(
+ s.get_first(&writer, 1).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ assert_eq!(
+ s.get_first(&writer, 2).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ assert_eq!(s.get_first(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get_first(&reader, 1).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 2).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_dup() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ assert_eq!(
+ s.get_first(&writer, 1).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ assert_eq!(s.get_first(&writer, 2).expect("read"), None);
+ assert_eq!(s.get_first(&writer, 3).expect("read"), None);
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get_first(&reader, 1).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 2).expect("read"), None);
+ assert_eq!(s.get_first(&reader, 3).expect("read"), None);
+ }
+ }
+
+ #[test]
+ fn test_dup_2() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+
+ let mut iter = s.get(&writer, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ }
+
+ #[test]
+ fn test_del() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_dup")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ {
+ let mut iter = s.get(&writer, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello!"))
+ .expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello!"))
+ .expect_err("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert!(iter.next().is_none());
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ s.delete(&mut writer, 1, &Value::Str("hello1!"))
+ .expect_err("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert!(iter.next().is_none());
+ }
+ }
+
+ #[test]
+ fn test_persist() {
+ let root = Builder::new()
+ .prefix("test_multi_integer_persist")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, 1, &Value::Str("hello!")).expect("write");
+ s.put(&mut writer, 1, &Value::Str("hello1!"))
+ .expect("write");
+ s.put(&mut writer, 2, &Value::Str("hello!")).expect("write");
+ {
+ let mut iter = s.get(&writer, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ writer.commit().expect("committed");
+ }
+
+ {
+ let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ let reader = k.read().expect("reader");
+ let mut iter = s.get(&reader, 1).expect("read");
+ assert_eq!(
+ iter.next().expect("first").expect("ok").1,
+ Value::Str("hello!")
+ );
+ assert_eq!(
+ iter.next().expect("second").expect("ok").1,
+ Value::Str("hello1!")
+ );
+ assert!(iter.next().is_none());
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/store/keys.rs b/third_party/rust/rkv/src/store/keys.rs
new file mode 100644
index 0000000000..26a13db47a
--- /dev/null
+++ b/third_party/rust/rkv/src/store/keys.rs
@@ -0,0 +1,46 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+mod encodables;
+mod primitives;
+
+use std::marker::PhantomData;
+
+use crate::error::DataError;
+
+pub use encodables::*;
+pub use primitives::*;
+
+pub(crate) struct Key<K> {
+ bytes: Vec<u8>,
+ phantom: PhantomData<K>,
+}
+
+impl<K> AsRef<[u8]> for Key<K>
+where
+ K: EncodableKey,
+{
+ fn as_ref(&self) -> &[u8] {
+ self.bytes.as_ref()
+ }
+}
+
+impl<K> Key<K>
+where
+ K: EncodableKey,
+{
+ #[allow(clippy::new_ret_no_self)]
+ pub fn new(k: &K) -> Result<Key<K>, DataError> {
+ Ok(Key {
+ bytes: k.to_bytes()?,
+ phantom: PhantomData,
+ })
+ }
+}
diff --git a/third_party/rust/rkv/src/store/keys/encodables.rs b/third_party/rust/rkv/src/store/keys/encodables.rs
new file mode 100644
index 0000000000..85e3eacdf5
--- /dev/null
+++ b/third_party/rust/rkv/src/store/keys/encodables.rs
@@ -0,0 +1,27 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use bincode::serialize;
+use serde::Serialize;
+
+use crate::error::DataError;
+
+pub trait EncodableKey {
+ fn to_bytes(&self) -> Result<Vec<u8>, DataError>;
+}
+
+impl<T> EncodableKey for T
+where
+ T: Serialize,
+{
+ fn to_bytes(&self) -> Result<Vec<u8>, DataError> {
+ serialize(self).map_err(|e| e.into())
+ }
+}
diff --git a/third_party/rust/rkv/src/store/keys/primitives.rs b/third_party/rust/rkv/src/store/keys/primitives.rs
new file mode 100644
index 0000000000..26282b7cbb
--- /dev/null
+++ b/third_party/rust/rkv/src/store/keys/primitives.rs
@@ -0,0 +1,15 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use crate::store::keys::EncodableKey;
+
+pub trait PrimitiveInt: EncodableKey {}
+
+impl PrimitiveInt for u32 {}
diff --git a/third_party/rust/rkv/src/store/multi.rs b/third_party/rust/rkv/src/store/multi.rs
new file mode 100644
index 0000000000..c7f816b1ec
--- /dev/null
+++ b/third_party/rust/rkv/src/store/multi.rs
@@ -0,0 +1,133 @@
+// Copyright 2018 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::marker::PhantomData;
+
+use crate::{
+ backend::{BackendDatabase, BackendFlags, BackendIter, BackendRoCursor, BackendRwTransaction},
+ error::StoreError,
+ helpers::read_transform,
+ readwrite::{Readable, Writer},
+ value::Value,
+};
+
+type EmptyResult = Result<(), StoreError>;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub struct MultiStore<D> {
+ db: D,
+}
+
+pub struct Iter<'i, I> {
+ iter: I,
+ phantom: PhantomData<&'i ()>,
+}
+
+impl<D> MultiStore<D>
+where
+ D: BackendDatabase,
+{
+ pub(crate) fn new(db: D) -> MultiStore<D> {
+ MultiStore { db }
+ }
+
+ /// Provides a cursor to all of the values for the duplicate entries that match this
+ /// key
+ pub fn get<'r, R, I, C, K>(&self, reader: &'r R, k: K) -> Result<Iter<'r, I>, StoreError>
+ where
+ R: Readable<'r, Database = D, RoCursor = C>,
+ I: BackendIter<'r>,
+ C: BackendRoCursor<'r, Iter = I>,
+ K: AsRef<[u8]> + 'r,
+ {
+ let cursor = reader.open_ro_cursor(&self.db)?;
+ let iter = cursor.into_iter_dup_of(k);
+
+ Ok(Iter {
+ iter,
+ phantom: PhantomData,
+ })
+ }
+
+ /// Provides the first value that matches this key
+ pub fn get_first<'r, R, K>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ R: Readable<'r, Database = D>,
+ K: AsRef<[u8]>,
+ {
+ reader.get(&self.db, &k)
+ }
+
+ /// Insert a value at the specified key.
+ /// This put will allow duplicate entries. If you wish to have duplicate entries
+ /// rejected, use the `put_with_flags` function and specify NO_DUP_DATA
+ pub fn put<T, K>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.put(&self.db, &k, v, T::Flags::empty())
+ }
+
+ pub fn put_with_flags<T, K>(
+ &self,
+ writer: &mut Writer<T>,
+ k: K,
+ v: &Value,
+ flags: T::Flags,
+ ) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.put(&self.db, &k, v, flags)
+ }
+
+ pub fn delete_all<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.delete(&self.db, &k, None)
+ }
+
+ pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.delete(&self.db, &k, Some(&v.to_bytes()?))
+ }
+
+ pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ {
+ writer.clear(&self.db)
+ }
+}
+
+impl<'i, I> Iterator for Iter<'i, I>
+where
+ I: BackendIter<'i>,
+{
+ type Item = Result<(&'i [u8], Value<'i>), StoreError>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ match self.iter.next() {
+ None => None,
+ Some(Ok((key, bytes))) => match read_transform(Ok(bytes)) {
+ Ok(val) => Some(Ok((key, val))),
+ Err(err) => Some(Err(err)),
+ },
+ Some(Err(err)) => Some(Err(err.into())),
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/store/single.rs b/third_party/rust/rkv/src/store/single.rs
new file mode 100644
index 0000000000..c456e6aaa6
--- /dev/null
+++ b/third_party/rust/rkv/src/store/single.rs
@@ -0,0 +1,132 @@
+// Copyright 2018 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::marker::PhantomData;
+
+use crate::{
+ backend::{BackendDatabase, BackendFlags, BackendIter, BackendRoCursor, BackendRwTransaction},
+ error::StoreError,
+ helpers::read_transform,
+ readwrite::{Readable, Writer},
+ value::Value,
+};
+
+type EmptyResult = Result<(), StoreError>;
+
+#[derive(Debug, Eq, PartialEq, Copy, Clone)]
+pub struct SingleStore<D> {
+ db: D,
+}
+
+pub struct Iter<'i, I> {
+ iter: I,
+ phantom: PhantomData<&'i ()>,
+}
+
+impl<D> SingleStore<D>
+where
+ D: BackendDatabase,
+{
+ pub(crate) fn new(db: D) -> SingleStore<D> {
+ SingleStore { db }
+ }
+
+ pub fn get<'r, R, K>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError>
+ where
+ R: Readable<'r, Database = D>,
+ K: AsRef<[u8]>,
+ {
+ reader.get(&self.db, &k)
+ }
+
+ // TODO: flags
+ pub fn put<T, K>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.put(&self.db, &k, v, T::Flags::empty())
+ }
+
+ #[cfg(not(feature = "db-dup-sort"))]
+ pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.delete(&self.db, &k)
+ }
+
+ #[cfg(feature = "db-dup-sort")]
+ pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult
+ where
+ T: BackendRwTransaction<Database = D>,
+ K: AsRef<[u8]>,
+ {
+ writer.delete(&self.db, &k, None)
+ }
+
+ pub fn iter_start<'r, R, I, C>(&self, reader: &'r R) -> Result<Iter<'r, I>, StoreError>
+ where
+ R: Readable<'r, Database = D, RoCursor = C>,
+ I: BackendIter<'r>,
+ C: BackendRoCursor<'r, Iter = I>,
+ {
+ let cursor = reader.open_ro_cursor(&self.db)?;
+ let iter = cursor.into_iter();
+
+ Ok(Iter {
+ iter,
+ phantom: PhantomData,
+ })
+ }
+
+ pub fn iter_from<'r, R, I, C, K>(&self, reader: &'r R, k: K) -> Result<Iter<'r, I>, StoreError>
+ where
+ R: Readable<'r, Database = D, RoCursor = C>,
+ I: BackendIter<'r>,
+ C: BackendRoCursor<'r, Iter = I>,
+ K: AsRef<[u8]> + 'r,
+ {
+ let cursor = reader.open_ro_cursor(&self.db)?;
+ let iter = cursor.into_iter_from(k);
+
+ Ok(Iter {
+ iter,
+ phantom: PhantomData,
+ })
+ }
+
+ pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult
+ where
+ D: BackendDatabase,
+ T: BackendRwTransaction<Database = D>,
+ {
+ writer.clear(&self.db)
+ }
+}
+
+impl<'i, I> Iterator for Iter<'i, I>
+where
+ I: BackendIter<'i>,
+{
+ type Item = Result<(&'i [u8], Value<'i>), StoreError>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ match self.iter.next() {
+ None => None,
+ Some(Ok((key, bytes))) => match read_transform(Ok(bytes)) {
+ Ok(val) => Some(Ok((key, val))),
+ Err(err) => Some(Err(err)),
+ },
+ Some(Err(err)) => Some(Err(err.into())),
+ }
+ }
+}
diff --git a/third_party/rust/rkv/src/value.rs b/third_party/rust/rkv/src/value.rs
new file mode 100644
index 0000000000..762607acd3
--- /dev/null
+++ b/third_party/rust/rkv/src/value.rs
@@ -0,0 +1,252 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::fmt;
+
+use arrayref::array_ref;
+use bincode::{deserialize, serialize, serialized_size};
+use ordered_float::OrderedFloat;
+use uuid::{Bytes, Uuid};
+
+use crate::error::DataError;
+
+/// We define a set of types, associated with simple integers, to annotate values stored
+/// in LMDB. This is to avoid an accidental 'cast' from a value of one type to another.
+/// For this reason we don't simply use `deserialize` from the `bincode` crate.
+#[repr(u8)]
+#[derive(Debug, PartialEq, Eq)]
+pub enum Type {
+ Bool = 1,
+ U64 = 2,
+ I64 = 3,
+ F64 = 4,
+ Instant = 5, // Millisecond-precision timestamp.
+ Uuid = 6,
+ Str = 7,
+ Json = 8,
+ Blob = 9,
+}
+
+/// We use manual tagging, because <https://github.com/serde-rs/serde/issues/610>.
+impl Type {
+ pub fn from_tag(tag: u8) -> Result<Type, DataError> {
+ #![allow(clippy::unnecessary_lazy_evaluations)]
+ Type::from_primitive(tag).ok_or_else(|| DataError::UnknownType(tag))
+ }
+
+ #[allow(clippy::wrong_self_convention)]
+ pub fn to_tag(self) -> u8 {
+ self as u8
+ }
+
+ fn from_primitive(p: u8) -> Option<Type> {
+ match p {
+ 1 => Some(Type::Bool),
+ 2 => Some(Type::U64),
+ 3 => Some(Type::I64),
+ 4 => Some(Type::F64),
+ 5 => Some(Type::Instant),
+ 6 => Some(Type::Uuid),
+ 7 => Some(Type::Str),
+ 8 => Some(Type::Json),
+ 9 => Some(Type::Blob),
+ _ => None,
+ }
+ }
+}
+
+impl fmt::Display for Type {
+ fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> {
+ f.write_str(match *self {
+ Type::Bool => "bool",
+ Type::U64 => "u64",
+ Type::I64 => "i64",
+ Type::F64 => "f64",
+ Type::Instant => "instant",
+ Type::Uuid => "uuid",
+ Type::Str => "str",
+ Type::Json => "json",
+ Type::Blob => "blob",
+ })
+ }
+}
+
+#[derive(Debug, Eq, PartialEq)]
+pub enum Value<'v> {
+ Bool(bool),
+ U64(u64),
+ I64(i64),
+ F64(OrderedFloat<f64>),
+ Instant(i64), // Millisecond-precision timestamp.
+ Uuid(&'v Bytes),
+ Str(&'v str),
+ Json(&'v str),
+ Blob(&'v [u8]),
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum OwnedValue {
+ Bool(bool),
+ U64(u64),
+ I64(i64),
+ F64(f64),
+ Instant(i64), // Millisecond-precision timestamp.
+ Uuid(Uuid),
+ Str(String),
+ Json(String), // TODO
+ Blob(Vec<u8>),
+}
+
+fn uuid(bytes: &[u8]) -> Result<Value, DataError> {
+ if bytes.len() == 16 {
+ Ok(Value::Uuid(array_ref![bytes, 0, 16]))
+ } else {
+ Err(DataError::InvalidUuid)
+ }
+}
+
+impl<'v> Value<'v> {
+ pub fn from_tagged_slice(slice: &'v [u8]) -> Result<Value<'v>, DataError> {
+ let (tag, data) = slice.split_first().ok_or(DataError::Empty)?;
+ let t = Type::from_tag(*tag)?;
+ Value::from_type_and_data(t, data)
+ }
+
+ fn from_type_and_data(t: Type, data: &'v [u8]) -> Result<Value<'v>, DataError> {
+ if t == Type::Uuid {
+ return deserialize(data)
+ .map_err(|e| DataError::DecodingError {
+ value_type: t,
+ err: e,
+ })
+ .map(uuid)?;
+ }
+
+ match t {
+ Type::Bool => deserialize(data).map(Value::Bool),
+ Type::U64 => deserialize(data).map(Value::U64),
+ Type::I64 => deserialize(data).map(Value::I64),
+ Type::F64 => deserialize(data).map(OrderedFloat).map(Value::F64),
+ Type::Instant => deserialize(data).map(Value::Instant),
+ Type::Str => deserialize(data).map(Value::Str),
+ Type::Json => deserialize(data).map(Value::Json),
+ Type::Blob => deserialize(data).map(Value::Blob),
+ Type::Uuid => {
+ // Processed above to avoid verbose duplication of error transforms.
+ unreachable!()
+ }
+ }
+ .map_err(|e| DataError::DecodingError {
+ value_type: t,
+ err: e,
+ })
+ }
+
+ pub fn to_bytes(&self) -> Result<Vec<u8>, DataError> {
+ match self {
+ Value::Bool(v) => serialize(&(Type::Bool.to_tag(), *v)),
+ Value::U64(v) => serialize(&(Type::U64.to_tag(), *v)),
+ Value::I64(v) => serialize(&(Type::I64.to_tag(), *v)),
+ Value::F64(v) => serialize(&(Type::F64.to_tag(), v.0)),
+ Value::Instant(v) => serialize(&(Type::Instant.to_tag(), *v)),
+ Value::Str(v) => serialize(&(Type::Str.to_tag(), v)),
+ Value::Json(v) => serialize(&(Type::Json.to_tag(), v)),
+ Value::Blob(v) => serialize(&(Type::Blob.to_tag(), v)),
+ Value::Uuid(v) => serialize(&(Type::Uuid.to_tag(), v)),
+ }
+ .map_err(DataError::EncodingError)
+ }
+
+ pub fn serialized_size(&self) -> Result<u64, DataError> {
+ match self {
+ Value::Bool(v) => serialized_size(&(Type::Bool.to_tag(), *v)),
+ Value::U64(v) => serialized_size(&(Type::U64.to_tag(), *v)),
+ Value::I64(v) => serialized_size(&(Type::I64.to_tag(), *v)),
+ Value::F64(v) => serialized_size(&(Type::F64.to_tag(), v.0)),
+ Value::Instant(v) => serialized_size(&(Type::Instant.to_tag(), *v)),
+ Value::Str(v) => serialized_size(&(Type::Str.to_tag(), v)),
+ Value::Json(v) => serialized_size(&(Type::Json.to_tag(), v)),
+ Value::Blob(v) => serialized_size(&(Type::Blob.to_tag(), v)),
+ Value::Uuid(v) => serialized_size(&(Type::Uuid.to_tag(), v)),
+ }
+ .map_err(DataError::EncodingError)
+ }
+}
+
+impl<'v> From<&'v Value<'v>> for OwnedValue {
+ fn from(value: &Value) -> OwnedValue {
+ match value {
+ Value::Bool(v) => OwnedValue::Bool(*v),
+ Value::U64(v) => OwnedValue::U64(*v),
+ Value::I64(v) => OwnedValue::I64(*v),
+ Value::F64(v) => OwnedValue::F64(**v),
+ Value::Instant(v) => OwnedValue::Instant(*v),
+ Value::Uuid(v) => OwnedValue::Uuid(Uuid::from_bytes(**v)),
+ Value::Str(v) => OwnedValue::Str((*v).to_string()),
+ Value::Json(v) => OwnedValue::Json((*v).to_string()),
+ Value::Blob(v) => OwnedValue::Blob(v.to_vec()),
+ }
+ }
+}
+
+impl<'v> From<&'v OwnedValue> for Value<'v> {
+ fn from(value: &OwnedValue) -> Value {
+ match value {
+ OwnedValue::Bool(v) => Value::Bool(*v),
+ OwnedValue::U64(v) => Value::U64(*v),
+ OwnedValue::I64(v) => Value::I64(*v),
+ OwnedValue::F64(v) => Value::F64(OrderedFloat::from(*v)),
+ OwnedValue::Instant(v) => Value::Instant(*v),
+ OwnedValue::Uuid(v) => Value::Uuid(v.as_bytes()),
+ OwnedValue::Str(v) => Value::Str(v),
+ OwnedValue::Json(v) => Value::Json(v),
+ OwnedValue::Blob(v) => Value::Blob(v),
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_value_serialized_size() {
+ // | Value enum | tag: 1 byte | value_payload |
+ // |----------------------------------------------------------|
+ // | I64 | 1 | 8 |
+ // | U64 | 1 | 8 |
+ // | Bool | 1 | 1 |
+ // | Instant | 1 | 8 |
+ // | F64 | 1 | 8 |
+ // | Uuid | 1 | 16 |
+ // | Str/Blob/Json | 1 |(8: len + sizeof(payload))|
+ assert_eq!(Value::I64(-1000).serialized_size().unwrap(), 9);
+ assert_eq!(Value::U64(1000u64).serialized_size().unwrap(), 9);
+ assert_eq!(Value::Bool(true).serialized_size().unwrap(), 2);
+ assert_eq!(
+ Value::Instant(1_558_020_865_224).serialized_size().unwrap(),
+ 9
+ );
+ assert_eq!(
+ Value::F64(OrderedFloat(10000.1)).serialized_size().unwrap(),
+ 9
+ );
+ assert_eq!(Value::Str("hello!").serialized_size().unwrap(), 15);
+ assert_eq!(Value::Str("¡Hola").serialized_size().unwrap(), 15);
+ assert_eq!(Value::Blob(b"hello!").serialized_size().unwrap(), 15);
+ assert_eq!(
+ uuid(b"\x9f\xe2\xc4\xe9\x3f\x65\x4f\xdb\xb2\x4c\x02\xb1\x52\x59\x71\x6c")
+ .unwrap()
+ .serialized_size()
+ .unwrap(),
+ 17
+ );
+ }
+}
diff --git a/third_party/rust/rkv/tests/env-all.rs b/third_party/rust/rkv/tests/env-all.rs
new file mode 100644
index 0000000000..a8d93c786e
--- /dev/null
+++ b/third_party/rust/rkv/tests/env-all.rs
@@ -0,0 +1,328 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::fs;
+
+use tempfile::Builder;
+
+use rkv::{
+ backend::{Lmdb, SafeMode},
+ Rkv, StoreOptions, Value,
+};
+
+#[test]
+fn test_open_safe_same_dir_as_lmdb() {
+ let root = Builder::new()
+ .prefix("test_open_safe_same_dir_as_lmdb")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Create database of type A and save to disk.
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ writer.commit().expect("committed");
+ k.sync(true).expect("synced");
+ }
+ // Verify that database of type A was written to disk.
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Create database of type B and verify that it is empty.
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let _ = k
+ .open_single("sk", StoreOptions::default())
+ .expect_err("not opened");
+ }
+ // Verify that database of type A wasn't changed.
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Create database of type B and save to disk (type A exists at the same path).
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo1", &Value::I64(5678))
+ .expect("wrote");
+ sk.put(&mut writer, "bar1", &Value::Bool(false))
+ .expect("wrote");
+ sk.put(&mut writer, "baz1", &Value::Str("héllo~ yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo1").expect("read"),
+ Some(Value::I64(5678))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar1").expect("read"),
+ Some(Value::Bool(false))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz1").expect("read"),
+ Some(Value::Str("héllo~ yöu"))
+ );
+ writer.commit().expect("committed");
+ k.sync(true).expect("synced");
+ }
+ // Verify that database of type B was written to disk.
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo1").expect("read"),
+ Some(Value::I64(5678))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar1").expect("read"),
+ Some(Value::Bool(false))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz1").expect("read"),
+ Some(Value::Str("héllo~ yöu"))
+ );
+ }
+ // Verify that database of type A still wasn't changed.
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+}
+
+#[test]
+fn test_open_lmdb_same_dir_as_safe() {
+ let root = Builder::new()
+ .prefix("test_open_lmdb_same_dir_as_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Create database of type A and save to disk.
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ writer.commit().expect("committed");
+ k.sync(true).expect("synced");
+ }
+ // Verify that database of type A was written to disk.
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Create database of type B and verify that it is empty.
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let _ = k
+ .open_single("sk", StoreOptions::default())
+ .expect_err("not opened");
+ }
+ // Verify that database of type A wasn't changed.
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Create database of type B and save to disk (type A exists at the same path).
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo1", &Value::I64(5678))
+ .expect("wrote");
+ sk.put(&mut writer, "bar1", &Value::Bool(false))
+ .expect("wrote");
+ sk.put(&mut writer, "baz1", &Value::Str("héllo~ yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo1").expect("read"),
+ Some(Value::I64(5678))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar1").expect("read"),
+ Some(Value::Bool(false))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz1").expect("read"),
+ Some(Value::Str("héllo~ yöu"))
+ );
+ writer.commit().expect("committed");
+ k.sync(true).expect("synced");
+ }
+ // Verify that database of type B was written to disk.
+ {
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo1").expect("read"),
+ Some(Value::I64(5678))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar1").expect("read"),
+ Some(Value::Bool(false))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz1").expect("read"),
+ Some(Value::Str("héllo~ yöu"))
+ );
+ }
+ // Verify that database of type A still wasn't changed.
+ {
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+}
diff --git a/third_party/rust/rkv/tests/env-lmdb.rs b/third_party/rust/rkv/tests/env-lmdb.rs
new file mode 100644
index 0000000000..e3aca394ab
--- /dev/null
+++ b/third_party/rust/rkv/tests/env-lmdb.rs
@@ -0,0 +1,1647 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable
+// deprecates `clippy::cyclomatic_complexity`.
+#![allow(clippy::complexity)]
+
+use std::{
+ fs,
+ path::Path,
+ str,
+ sync::{Arc, RwLock},
+ thread,
+};
+
+use byteorder::{ByteOrder, LittleEndian};
+use tempfile::Builder;
+
+use rkv::{
+ backend::{
+ BackendEnvironmentBuilder, BackendInfo, BackendStat, Lmdb, LmdbDatabase, LmdbEnvironment,
+ LmdbRwTransaction,
+ },
+ EnvironmentFlags, Rkv, SingleStore, StoreError, StoreOptions, Value, Writer,
+};
+
+fn check_rkv(k: &Rkv<LmdbEnvironment>) {
+ let _ = k
+ .open_single(None, StoreOptions::create())
+ .expect("created default");
+
+ let s = k.open_single("s", StoreOptions::create()).expect("opened");
+ let reader = k.read().expect("reader");
+
+ let result = s.get(&reader, "foo");
+ assert_eq!(None, result.expect("success but no value"));
+}
+
+// The default size is 1MB.
+const DEFAULT_SIZE: usize = 1024 * 1024;
+
+/// We can't open a directory that doesn't exist.
+#[test]
+fn test_open_fails() {
+ let root = Builder::new()
+ .prefix("test_open_fails")
+ .tempdir()
+ .expect("tempdir");
+ assert!(root.path().exists());
+
+ let nope = root.path().join("nope/");
+ assert!(!nope.exists());
+
+ let pb = nope.to_path_buf();
+ match Rkv::new::<Lmdb>(nope.as_path()).err() {
+ Some(StoreError::UnsuitableEnvironmentPath(p)) => {
+ assert_eq!(pb, p);
+ }
+ _ => panic!("expected error"),
+ };
+}
+
+#[test]
+fn test_open() {
+ let root = Builder::new()
+ .prefix("test_open")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ check_rkv(&k);
+}
+
+#[test]
+fn test_open_from_builder() {
+ let root = Builder::new()
+ .prefix("test_open_from_builder")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(2);
+
+ let k = Rkv::from_builder(root.path(), builder).expect("rkv");
+ check_rkv(&k);
+}
+
+#[test]
+fn test_open_from_builder_with_no_subdir_1() {
+ let root = Builder::new()
+ .prefix("test_open_from_builder")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ {
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(2);
+
+ let k = Rkv::from_builder(root.path(), builder).expect("rkv");
+ check_rkv(&k);
+ }
+ {
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_flags(EnvironmentFlags::NO_SUB_DIR);
+ builder.set_max_dbs(2);
+
+ let mut datamdb = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+
+ let k = Rkv::from_builder(&datamdb, builder).expect("rkv");
+ check_rkv(&k);
+ }
+}
+
+#[test]
+#[should_panic(expected = "rkv: UnsuitableEnvironmentPath")]
+fn test_open_from_builder_with_no_subdir_2() {
+ let root = Builder::new()
+ .prefix("test_open_from_builder")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ {
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(2);
+
+ let k = Rkv::from_builder(root.path(), builder).expect("rkv");
+ check_rkv(&k);
+ }
+ {
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_flags(EnvironmentFlags::NO_SUB_DIR);
+ builder.set_max_dbs(2);
+
+ let mut datamdb = root.path().to_path_buf();
+ datamdb.push("bogus.mdb");
+
+ let k = Rkv::from_builder(&datamdb, builder).expect("rkv");
+ check_rkv(&k);
+ }
+}
+
+#[test]
+fn test_open_from_builder_with_dir_1() {
+ let root = Builder::new()
+ .prefix("test_open_from_builder")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(2);
+ builder.set_make_dir_if_needed(true);
+
+ let k = Rkv::from_builder(root.path(), builder).expect("rkv");
+ check_rkv(&k);
+}
+
+#[test]
+#[should_panic(expected = "rkv: UnsuitableEnvironmentPath(\"bogus\")")]
+fn test_open_from_builder_with_dir_2() {
+ let root = Path::new("bogus");
+ println!("Root path: {root:?}");
+ assert!(!root.is_dir());
+
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(2);
+
+ let k = Rkv::from_builder(root, builder).expect("rkv");
+ check_rkv(&k);
+}
+
+#[test]
+#[should_panic(expected = "opened: DbsFull")]
+fn test_create_with_capacity_1() {
+ let root = Builder::new()
+ .prefix("test_create_with_capacity")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ // This errors with "opened: DbsFull" because we specified a capacity of one (database),
+ // and check_rkv already opened one (plus the default database, which doesn't count
+ // against the limit).
+ let _zzz = k
+ .open_single("zzz", StoreOptions::create())
+ .expect("opened");
+}
+
+#[test]
+fn test_create_with_capacity_2() {
+ let root = Builder::new()
+ .prefix("test_create_with_capacity")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ // This doesn't error with "opened: DbsFull" with because even though we specified a
+ // capacity of one (database), and check_rkv already opened one, the default database
+ // doesn't count against the limit.
+ let _zzz = k.open_single(None, StoreOptions::create()).expect("opened");
+}
+
+#[test]
+#[should_panic(expected = "opened: DbsFull")]
+fn test_open_with_capacity_1() {
+ let root = Builder::new()
+ .prefix("test_open_with_capacity")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ let _zzz = k
+ .open_single("zzz", StoreOptions::default())
+ .expect("opened");
+}
+
+#[test]
+fn test_open_with_capacity_2() {
+ let root = Builder::new()
+ .prefix("test_open_with_capacity")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ let _zzz = k
+ .open_single(None, StoreOptions::default())
+ .expect("opened");
+}
+
+#[test]
+fn test_list_dbs_1() {
+ let root = Builder::new()
+ .prefix("test_list_dbs")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ let dbs = k.get_dbs().unwrap();
+ assert_eq!(dbs, vec![Some("s".to_owned())]);
+}
+
+#[test]
+fn test_list_dbs_2() {
+ let root = Builder::new()
+ .prefix("test_list_dbs")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 2).expect("rkv");
+ check_rkv(&k);
+
+ let _ = k
+ .open_single("zzz", StoreOptions::create())
+ .expect("opened");
+
+ let dbs = k.get_dbs().unwrap();
+ assert_eq!(dbs, vec![Some("s".to_owned()), Some("zzz".to_owned())]);
+}
+
+#[test]
+fn test_list_dbs_3() {
+ let root = Builder::new()
+ .prefix("test_list_dbs")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<Lmdb>(root.path(), 0).expect("rkv");
+
+ let _ = k.open_single(None, StoreOptions::create()).expect("opened");
+
+ let dbs = k.get_dbs().unwrap();
+ assert_eq!(dbs, vec![None]);
+}
+
+fn get_larger_than_default_map_size_value() -> usize {
+ // The LMDB C library and lmdb Rust crate docs for setting the map size
+ // <http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5>
+ // <https://docs.rs/lmdb/0.8.0/lmdb/struct.EnvironmentBuilder.html#method.set_map_size>
+ // both say that the default map size is 10,485,760 bytes, i.e. 10MiB.
+ //
+ // But the DEFAULT_MAPSIZE define in the LMDB code
+ // https://github.com/LMDB/lmdb/blob/26c7df88e44e31623d0802a564f24781acdefde3/libraries/liblmdb/mdb.c#L729
+ // sets the default map size to 1,048,576 bytes, i.e. 1MiB.
+ //
+ DEFAULT_SIZE + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */
+}
+
+#[test]
+#[should_panic(expected = "wrote: MapFull")]
+fn test_exceed_map_size() {
+ let root = Builder::new()
+ .prefix("test_exceed_map_size")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("test", StoreOptions::create())
+ .expect("opened");
+
+ // Writing a large enough value should cause LMDB to fail on MapFull.
+ // We write a string that is larger than the default map size.
+ let val = "x".repeat(get_larger_than_default_map_size_value());
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str(&val))
+ .expect("wrote");
+}
+
+#[test]
+#[should_panic(expected = "wrote: KeyValuePairBadSize")]
+fn test_exceed_key_size_limit() {
+ let root = Builder::new()
+ .prefix("test_exceed_key_size_limit")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k
+ .open_single("test", StoreOptions::create())
+ .expect("opened");
+
+ let key = "k".repeat(512);
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, key, &Value::Str("val")).expect("wrote");
+}
+
+#[test]
+fn test_increase_map_size() {
+ let root = Builder::new()
+ .prefix("test_open_with_map_size")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ // Set the map size to the size of the value we'll store in it + 100KiB,
+ // which ensures that there's enough space for the value and metadata.
+ builder.set_map_size(
+ get_larger_than_default_map_size_value() + 100 * 1024, /* 100KiB */
+ );
+ builder.set_max_dbs(2);
+ let k = Rkv::from_builder(root.path(), builder).unwrap();
+ let sk = k
+ .open_single("test", StoreOptions::create())
+ .expect("opened");
+ let val = "x".repeat(get_larger_than_default_map_size_value());
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str(&val))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::Str(&val))
+ );
+}
+
+#[test]
+fn test_round_trip_and_transactions() {
+ let root = Builder::new()
+ .prefix("test_round_trip_and_transactions")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&writer, "noo").expect("read"),
+ Some(Value::F64(1234.0.into()))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+
+ // Isolation. Reads won't return values.
+ let r = &k.read().unwrap();
+ assert_eq!(sk.get(r, "foo").expect("read"), None);
+ assert_eq!(sk.get(r, "bar").expect("read"), None);
+ assert_eq!(sk.get(r, "baz").expect("read"), None);
+ }
+
+ // Dropped: tx rollback. Reads will still return nothing.
+
+ {
+ let r = &k.read().unwrap();
+ assert_eq!(sk.get(r, "foo").expect("read"), None);
+ assert_eq!(sk.get(r, "bar").expect("read"), None);
+ assert_eq!(sk.get(r, "baz").expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+
+ writer.commit().expect("committed");
+ }
+
+ // Committed. Reads will succeed.
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
+ assert_eq!(
+ sk.get(&r, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.delete(&mut writer, "foo").expect("deleted");
+ sk.delete(&mut writer, "bar").expect("deleted");
+ sk.delete(&mut writer, "baz").expect("deleted");
+ assert_eq!(sk.get(&writer, "foo").expect("read"), None);
+ assert_eq!(sk.get(&writer, "bar").expect("read"), None);
+ assert_eq!(sk.get(&writer, "baz").expect("read"), None);
+
+ // Isolation. Reads still return values.
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
+ assert_eq!(
+ sk.get(&r, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+
+ // Dropped: tx rollback. Reads will still return values.
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
+ assert_eq!(
+ sk.get(&r, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.delete(&mut writer, "foo").expect("deleted");
+ sk.delete(&mut writer, "bar").expect("deleted");
+ sk.delete(&mut writer, "baz").expect("deleted");
+ assert_eq!(sk.get(&writer, "foo").expect("read"), None);
+ assert_eq!(sk.get(&writer, "bar").expect("read"), None);
+ assert_eq!(sk.get(&writer, "baz").expect("read"), None);
+
+ writer.commit().expect("committed");
+ }
+
+ // Committed. Reads will succeed but return None to indicate a missing value.
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), None);
+ assert_eq!(sk.get(&r, "bar").expect("read"), None);
+ assert_eq!(sk.get(&r, "baz").expect("read"), None);
+ }
+}
+
+#[test]
+fn test_single_store_clear() {
+ let root = Builder::new()
+ .prefix("test_single_store_clear")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let r = k.read().unwrap();
+ let iter = sk.iter_start(&r).expect("iter");
+ assert_eq!(iter.count(), 0);
+ }
+}
+
+#[test]
+#[should_panic(expected = "KeyValuePairNotFound")]
+fn test_single_store_delete_nonexistent() {
+ let root = Builder::new()
+ .prefix("test_single_store_delete_nonexistent")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.delete(&mut writer, "bogus").unwrap();
+}
+
+#[test]
+#[cfg(feature = "db-dup-sort")]
+fn test_multi_put_get_del() {
+ let root = Builder::new()
+ .prefix("test_multi_put_get_del")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
+
+ let mut writer = k.write().unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 bar"))
+ .unwrap();
+ writer.commit().unwrap();
+
+ let writer = k.write().unwrap();
+ {
+ let mut iter = multistore.get(&writer, "str1").unwrap();
+ let (id, val) = iter.next().unwrap().unwrap();
+ assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 bar")));
+ let (id, val) = iter.next().unwrap().unwrap();
+ assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 foo")));
+ }
+ writer.commit().unwrap();
+
+ let mut writer = k.write().unwrap();
+ multistore
+ .delete(&mut writer, "str1", &Value::Str("str1 foo"))
+ .unwrap();
+ assert_eq!(
+ multistore.get_first(&writer, "str1").unwrap(),
+ Some(Value::Str("str1 bar"))
+ );
+ multistore
+ .delete(&mut writer, "str2", &Value::Str("str2 bar"))
+ .unwrap();
+ assert_eq!(
+ multistore.get_first(&writer, "str2").unwrap(),
+ Some(Value::Str("str2 foo"))
+ );
+ multistore.delete_all(&mut writer, "str3").unwrap();
+ assert_eq!(multistore.get_first(&writer, "str3").unwrap(), None);
+ writer.commit().unwrap();
+}
+
+#[test]
+#[cfg(feature = "db-dup-sort")]
+fn test_multiple_store_clear() {
+ let root = Builder::new()
+ .prefix("test_multiple_store_clear")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let multistore = k
+ .open_multi("multistore", StoreOptions::create())
+ .expect("opened");
+
+ {
+ let mut writer = k.write().expect("writer");
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 bar"))
+ .unwrap();
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ multistore.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(multistore.get_first(&r, "str1").expect("read"), None);
+ assert_eq!(multistore.get_first(&r, "str2").expect("read"), None);
+ assert_eq!(multistore.get_first(&r, "str3").expect("read"), None);
+ }
+}
+
+#[test]
+fn test_open_store_for_read() {
+ let root = Builder::new()
+ .prefix("test_open_store_for_read")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+
+ // First create the store, and start a write transaction on it.
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("write");
+
+ // Open the same store for read, note that the write transaction is still in progress,
+ // it should not block the reader though.
+ let sk_readonly = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+ writer.commit().expect("commit");
+
+ // Now the write transaction is committed, any followed reads should see its change.
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk_readonly.get(&reader, "foo").expect("read"),
+ Some(Value::Str("bar"))
+ );
+}
+
+#[test]
+#[should_panic(expected = "open a missing store")]
+fn test_open_a_missing_store() {
+ let root = Builder::new()
+ .prefix("test_open_a_missing_store")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let _sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("open a missing store");
+}
+
+#[test]
+#[should_panic(expected = "new failed: FileInvalid")]
+fn test_open_a_broken_store() {
+ let root = Builder::new()
+ .prefix("test_open_a_missing_store")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let dbfile = root.path().join("data.mdb");
+ fs::write(dbfile, "bogus").expect("dbfile created");
+
+ let _ = Rkv::new::<Lmdb>(root.path()).expect("new failed");
+}
+
+#[test]
+fn test_open_fail_with_badrslot() {
+ let root = Builder::new()
+ .prefix("test_open_fail_with_badrslot")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+
+ // First create the store
+ let _sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Open a reader on this store
+ let _reader = k.read().expect("reader");
+
+ // Open the same store for read while the reader is in progress will panic
+ let store = k.open_single("sk", StoreOptions::default());
+ match store {
+ Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (),
+ _ => panic!("should panic"),
+ }
+}
+
+#[test]
+fn test_read_before_write_num() {
+ let root = Builder::new()
+ .prefix("test_read_before_write_num")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Test reading a number, modifying it, and then writing it back.
+ // We have to be done with the Value::I64 before calling Writer::put,
+ // as the Value::I64 borrows an immutable reference to the Writer.
+ // So we extract and copy its primitive value.
+
+ fn get_existing_foo(
+ store: SingleStore<LmdbDatabase>,
+ writer: &Writer<LmdbRwTransaction>,
+ ) -> Option<i64> {
+ match store.get(writer, "foo").expect("read") {
+ Some(Value::I64(val)) => Some(val),
+ _ => None,
+ }
+ }
+
+ let mut writer = k.write().expect("writer");
+ let mut existing = get_existing_foo(sk, &writer).unwrap_or(99);
+ existing += 1;
+ sk.put(&mut writer, "foo", &Value::I64(existing))
+ .expect("success");
+
+ let updated = get_existing_foo(sk, &writer).unwrap_or(99);
+ assert_eq!(updated, 100);
+ writer.commit().expect("commit");
+}
+
+#[test]
+fn test_read_before_write_str() {
+ let root = Builder::new()
+ .prefix("test_read_before_write_str")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Test reading a string, modifying it, and then writing it back.
+ // We have to be done with the Value::Str before calling Writer::put,
+ // as the Value::Str (and its underlying &str) borrows an immutable
+ // reference to the Writer. So we copy it to a String.
+
+ fn get_existing_foo(
+ store: SingleStore<LmdbDatabase>,
+ writer: &Writer<LmdbRwTransaction>,
+ ) -> Option<String> {
+ match store.get(writer, "foo").expect("read") {
+ Some(Value::Str(val)) => Some(val.to_string()),
+ _ => None,
+ }
+ }
+
+ let mut writer = k.write().expect("writer");
+ let mut existing = get_existing_foo(sk, &writer).unwrap_or_default();
+ existing.push('…');
+ sk.put(&mut writer, "foo", &Value::Str(&existing))
+ .expect("write");
+
+ let updated = get_existing_foo(sk, &writer).unwrap_or_default();
+ assert_eq!(updated, "…");
+ writer.commit().expect("commit");
+}
+
+#[test]
+fn test_concurrent_read_transactions_prohibited() {
+ let root = Builder::new()
+ .prefix("test_concurrent_reads_prohibited")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let _first = k.read().expect("reader");
+ let second = k.read();
+
+ match second {
+ Err(StoreError::ReadTransactionAlreadyExists(t)) => {
+ println!("Thread was {t:?}");
+ }
+ Err(e) => {
+ println!("Got error {e:?}");
+ }
+ _ => {
+ panic!("Expected error.");
+ }
+ }
+}
+
+#[test]
+fn test_isolation() {
+ let root = Builder::new()
+ .prefix("test_isolation")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let s = k.open_single("s", StoreOptions::create()).expect("opened");
+
+ // Add one field.
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let reader = k.read().unwrap();
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+ }
+
+ // Establish a long-lived reader that outlasts a writer.
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+
+ // Start a write transaction.
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote");
+
+ // The reader and writer are isolated.
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999)));
+
+ // If we commit the writer, we still have isolation.
+ writer.commit().expect("committed");
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+
+ // A new reader sees the committed value. Note that LMDB doesn't allow two
+ // read transactions to exist in the same thread, so we abort the previous one.
+ reader.abort();
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999)));
+}
+
+#[test]
+fn test_blob() {
+ let root = Builder::new()
+ .prefix("test_round_trip_blob")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ assert_eq!(sk.get(&writer, "foo").expect("read"), None);
+ sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4]))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::Blob(&[1, 2, 3, 4]))
+ );
+
+ fn u16_to_u8(src: &[u16]) -> Vec<u8> {
+ let mut dst = vec![0; 2 * src.len()];
+ LittleEndian::write_u16_into(src, &mut dst);
+ dst
+ }
+
+ fn u8_to_u16(src: &[u8]) -> Vec<u16> {
+ let mut dst = vec![0; src.len() / 2];
+ LittleEndian::read_u16_into(src, &mut dst);
+ dst
+ }
+
+ // When storing UTF-16 strings as blobs, we'll need to convert
+ // their [u16] backing storage to [u8]. Test that converting, writing,
+ // reading, and converting back works as expected.
+ let u16_array = [1000, 10000, 54321, 65535];
+ assert_eq!(sk.get(&writer, "bar").expect("read"), None);
+ sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array)))
+ .expect("wrote");
+ let u8_array = match sk.get(&writer, "bar").expect("read") {
+ Some(Value::Blob(val)) => val,
+ _ => &[],
+ };
+ assert_eq!(u8_to_u16(u8_array), u16_array);
+}
+
+#[test]
+fn test_sync() {
+ let root = Builder::new()
+ .prefix("test_sync")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let mut builder = Rkv::environment_builder::<Lmdb>();
+ builder.set_max_dbs(1);
+ builder.set_flags(EnvironmentFlags::NO_SYNC);
+ {
+ let k = Rkv::from_builder(root.path(), builder).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ k.sync(true).expect("synced");
+ }
+ }
+ let k = Rkv::from_builder(root.path(), builder).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+}
+
+#[test]
+#[cfg(feature = "db-int-key")]
+fn test_stat() {
+ let root = Builder::new()
+ .prefix("test_stat")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ for i in 0..5 {
+ let sk = k
+ .open_integer(&format!("sk{i}")[..], StoreOptions::create())
+ .expect("opened");
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, i, &Value::I64(i64::from(i)))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+ }
+ assert_eq!(k.stat().expect("stat").depth(), 1);
+ assert_eq!(k.stat().expect("stat").entries(), 5);
+ assert_eq!(k.stat().expect("stat").branch_pages(), 0);
+ assert_eq!(k.stat().expect("stat").leaf_pages(), 1);
+}
+
+#[test]
+fn test_info() {
+ let root = Builder::new()
+ .prefix("test_info")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ writer.commit().expect("commited");
+
+ let info = k.info().expect("info");
+
+ // The default size is 1MB.
+ assert_eq!(info.map_size(), DEFAULT_SIZE);
+ // Should greater than 0 after the write txn.
+ assert!(info.last_pgno() > 0);
+ // A txn to open_single + a txn to write.
+ assert_eq!(info.last_txnid(), 2);
+ // The default max readers is 126.
+ assert_eq!(info.max_readers(), 126);
+ assert_eq!(info.num_readers(), 0);
+
+ // A new reader should increment the reader counter.
+ let _reader = k.read().expect("reader");
+ let info = k.info().expect("info");
+
+ assert_eq!(info.num_readers(), 1);
+}
+
+#[test]
+fn test_load_ratio() {
+ let root = Builder::new()
+ .prefix("test_load_ratio")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ writer.commit().expect("commited");
+ let ratio = k.load_ratio().expect("ratio").unwrap();
+ assert!(ratio > 0.0_f32 && ratio < 1.0_f32);
+
+ // Put data to database should increase the load ratio.
+ let mut writer = k.write().expect("writer");
+ sk.put(
+ &mut writer,
+ "bar",
+ &Value::Str(&"more-than-4KB".repeat(1000)),
+ )
+ .expect("wrote");
+ writer.commit().expect("commited");
+ let new_ratio = k.load_ratio().expect("ratio").unwrap();
+ assert!(new_ratio > ratio);
+
+ // Clear the database so that all the used pages should go to freelist, hence the ratio
+ // should decrease.
+ let mut writer = k.write().expect("writer");
+ sk.clear(&mut writer).expect("clear");
+ writer.commit().expect("commited");
+ let after_clear_ratio = k.load_ratio().expect("ratio").unwrap();
+ assert!(after_clear_ratio < new_ratio);
+}
+
+#[test]
+fn test_set_map_size() {
+ let root = Builder::new()
+ .prefix("test_size_map_size")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ assert_eq!(k.info().expect("info").map_size(), DEFAULT_SIZE);
+
+ k.set_map_size(2 * DEFAULT_SIZE).expect("resized");
+
+ // Should be able to write.
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ writer.commit().expect("commited");
+
+ assert_eq!(k.info().expect("info").map_size(), 2 * DEFAULT_SIZE);
+}
+
+#[test]
+fn test_iter() {
+ let root = Builder::new()
+ .prefix("test_iter")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // An iterator over an empty store returns no values.
+ {
+ let reader = k.read().unwrap();
+ let mut iter = sk.iter_start(&reader).unwrap();
+ assert!(iter.next().is_none());
+ }
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!"))
+ .expect("wrote");
+ sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+
+ // Reader.iter() returns (key, value) tuples ordered by key.
+ let mut iter = sk.iter_start(&reader).unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "bar");
+ assert_eq!(val, Value::Bool(true));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "baz");
+ assert_eq!(val, Value::Str("héllo, yöu"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "foo");
+ assert_eq!(val, Value::I64(1234));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
+ assert_eq!(val, Value::Str("Emil.RuleZ!"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterators don't loop. Once one returns None, additional calls
+ // to its next() method will always return None.
+ assert!(iter.next().is_none());
+
+ // Reader.iter_from() begins iteration at the first key equal to
+ // or greater than the given key.
+ let mut iter = sk.iter_from(&reader, "moo").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Reader.iter_from() works as expected when the given key is a prefix
+ // of a key in the store.
+ let mut iter = sk.iter_from(&reader, "no").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+}
+
+#[test]
+fn test_iter_from_key_greater_than_existing() {
+ let root = Builder::new()
+ .prefix("test_iter_from_key_greater_than_existing")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+ let mut iter = sk.iter_from(&reader, "nuu").unwrap();
+ assert!(iter.next().is_none());
+}
+
+#[test]
+fn test_multiple_store_read_write() {
+ let root = Builder::new()
+ .prefix("test_multiple_store_read_write")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let s1 = k
+ .open_single("store_1", StoreOptions::create())
+ .expect("opened");
+ let s2 = k
+ .open_single("store_2", StoreOptions::create())
+ .expect("opened");
+ let s3 = k
+ .open_single("store_3", StoreOptions::create())
+ .expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ s1.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote");
+ s3.put(&mut writer, "foo", &Value::Bool(true))
+ .expect("wrote");
+
+ assert_eq!(
+ s1.get(&writer, "foo").expect("read"),
+ Some(Value::Str("bar"))
+ );
+ assert_eq!(s2.get(&writer, "foo").expect("read"), Some(Value::I64(123)));
+ assert_eq!(
+ s3.get(&writer, "foo").expect("read"),
+ Some(Value::Bool(true))
+ );
+
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("unbound_reader");
+ assert_eq!(
+ s1.get(&reader, "foo").expect("read"),
+ Some(Value::Str("bar"))
+ );
+ assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123)));
+ assert_eq!(
+ s3.get(&reader, "foo").expect("read"),
+ Some(Value::Bool(true))
+ );
+ reader.abort();
+
+ // test delete across multiple stores
+ let mut writer = k.write().expect("writer");
+ s1.delete(&mut writer, "foo").expect("deleted");
+ s2.delete(&mut writer, "foo").expect("deleted");
+ s3.delete(&mut writer, "foo").expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s1.get(&reader, "key").expect("value"), None);
+ assert_eq!(s2.get(&reader, "key").expect("value"), None);
+ assert_eq!(s3.get(&reader, "key").expect("value"), None);
+}
+
+#[test]
+fn test_multiple_store_iter() {
+ let root = Builder::new()
+ .prefix("test_multiple_store_iter")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let s1 = k
+ .open_single("store_1", StoreOptions::create())
+ .expect("opened");
+ let s2 = k
+ .open_single("store_2", StoreOptions::create())
+ .expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ // Write to "s1"
+ s1.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ s1.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ s1.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ s1.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!"))
+ .expect("wrote");
+ s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則"))
+ .expect("wrote");
+ // &mut writer to "s2"
+ s2.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ s2.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ s2.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ s2.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!"))
+ .expect("wrote");
+ s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+
+ // Iterate through the whole store in "s1"
+ let mut iter = s1.iter_start(&reader).unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "bar");
+ assert_eq!(val, Value::Bool(true));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "baz");
+ assert_eq!(val, Value::Str("héllo, yöu"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "foo");
+ assert_eq!(val, Value::I64(1234));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
+ assert_eq!(val, Value::Str("Emil.RuleZ!"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate through the whole store in "s2"
+ let mut iter = s2.iter_start(&reader).unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "bar");
+ assert_eq!(val, Value::Bool(true));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "baz");
+ assert_eq!(val, Value::Str("héllo, yöu"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "foo");
+ assert_eq!(val, Value::I64(1234));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
+ assert_eq!(val, Value::Str("Emil.RuleZ!"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given key in "s1"
+ let mut iter = s1.iter_from(&reader, "moo").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given key in "s2"
+ let mut iter = s2.iter_from(&reader, "moo").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given prefix in "s1"
+ let mut iter = s1.iter_from(&reader, "no").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given prefix in "s2"
+ let mut iter = s2.iter_from(&reader, "no").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+}
+
+#[test]
+fn test_store_multiple_thread() {
+ let root = Builder::new()
+ .prefix("test_multiple_thread")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let rkv_arc = Arc::new(RwLock::new(
+ Rkv::new::<Lmdb>(root.path()).expect("new succeeded"),
+ ));
+ let store = rkv_arc
+ .read()
+ .unwrap()
+ .open_single("test", StoreOptions::create())
+ .expect("opened");
+
+ let num_threads = 10;
+ let mut write_handles = Vec::with_capacity(num_threads as usize);
+ let mut read_handles = Vec::with_capacity(num_threads as usize);
+
+ // Note that this isn't intended to demonstrate a good use of threads.
+ // For this shape of data, it would be more performant to write/read
+ // all values using one transaction in a single thread. The point here
+ // is just to confirm that a store can be shared by multiple threads.
+
+ // For each KV pair, spawn a thread that writes it to the store.
+ for i in 0..num_threads {
+ let rkv_arc = rkv_arc.clone();
+ write_handles.push(thread::spawn(move || {
+ let rkv = rkv_arc.write().expect("rkv");
+ let mut writer = rkv.write().expect("writer");
+ store
+ .put(&mut writer, i.to_string(), &Value::U64(i))
+ .expect("written");
+ writer.commit().unwrap();
+ }));
+ }
+ for handle in write_handles {
+ handle.join().expect("joined");
+ }
+
+ // For each KV pair, spawn a thread that reads it from the store
+ // and returns its value.
+ for i in 0..num_threads {
+ let rkv_arc = rkv_arc.clone();
+ read_handles.push(thread::spawn(move || {
+ let rkv = rkv_arc.read().expect("rkv");
+ let reader = rkv.read().expect("reader");
+ let value = match store.get(&reader, i.to_string()) {
+ Ok(Some(Value::U64(value))) => value,
+ Ok(Some(_)) => panic!("value type unexpected"),
+ Ok(None) => panic!("value not found"),
+ Err(err) => panic!("{}", err),
+ };
+ assert_eq!(value, i);
+ value
+ }));
+ }
+
+ // Sum the values returned from the threads and confirm that they're
+ // equal to the sum of values written to the threads.
+ let thread_sum: u64 = read_handles
+ .into_iter()
+ .map(|handle| handle.join().expect("value"))
+ .sum();
+ assert_eq!(thread_sum, (0..num_threads).sum());
+}
+
+#[test]
+fn test_use_value_as_key() {
+ let root = Builder::new()
+ .prefix("test_use_value_as_key")
+ .tempdir()
+ .expect("tempdir");
+ let rkv = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let store = rkv
+ .open_single("store", StoreOptions::create())
+ .expect("opened");
+
+ {
+ let mut writer = rkv.write().expect("writer");
+ store
+ .put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ store
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ // It's possible to retrieve a value with a Reader and then use it
+ // as a key with a Writer.
+ {
+ let reader = &rkv.read().unwrap();
+ if let Some(Value::Str(key)) = store.get(reader, "foo").expect("read") {
+ let mut writer = rkv.write().expect("writer");
+ store.delete(&mut writer, key).expect("deleted");
+ writer.commit().expect("committed");
+ }
+ }
+
+ {
+ let mut writer = rkv.write().expect("writer");
+ store
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ // You can also retrieve a Value with a Writer and then use it as a key
+ // with the same Writer if you copy the value to an owned type
+ // so the Writer isn't still being borrowed by the retrieved value
+ // when you try to borrow the Writer again to modify that value.
+ {
+ let mut writer = rkv.write().expect("writer");
+ if let Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") {
+ let key = value.to_owned();
+ store.delete(&mut writer, key).expect("deleted");
+ writer.commit().expect("committed");
+ }
+ }
+
+ {
+ let name1 = rkv
+ .open_single("name1", StoreOptions::create())
+ .expect("opened");
+ let name2 = rkv
+ .open_single("name2", StoreOptions::create())
+ .expect("opened");
+ let mut writer = rkv.write().expect("writer");
+ name1
+ .put(&mut writer, "key1", &Value::Str("bar"))
+ .expect("wrote");
+ name1
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ name2
+ .put(&mut writer, "key2", &Value::Str("bar"))
+ .expect("wrote");
+ name2
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ // You can also iterate (store, key) pairs to retrieve foreign keys,
+ // then iterate those foreign keys to modify/delete them.
+ //
+ // You need to open the stores in advance, since opening a store
+ // uses a write transaction internally, so opening them while a writer
+ // is extant will hang.
+ //
+ // And you need to copy the values to an owned type so the Writer isn't
+ // still being borrowed by a retrieved value when you try to borrow
+ // the Writer again to modify another value.
+ let fields = vec![
+ (
+ rkv.open_single("name1", StoreOptions::create())
+ .expect("opened"),
+ "key1",
+ ),
+ (
+ rkv.open_single("name2", StoreOptions::create())
+ .expect("opened"),
+ "key2",
+ ),
+ ];
+ {
+ let mut foreignkeys = Vec::new();
+ let mut writer = rkv.write().expect("writer");
+ for (store, key) in fields.iter() {
+ if let Some(Value::Str(value)) = store.get(&writer, key).expect("read") {
+ foreignkeys.push((store, value.to_owned()));
+ }
+ }
+ for (store, key) in foreignkeys.iter() {
+ store.delete(&mut writer, key).expect("deleted");
+ }
+ writer.commit().expect("committed");
+ }
+}
diff --git a/third_party/rust/rkv/tests/env-migration.rs b/third_party/rust/rkv/tests/env-migration.rs
new file mode 100644
index 0000000000..7026c25de8
--- /dev/null
+++ b/third_party/rust/rkv/tests/env-migration.rs
@@ -0,0 +1,607 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{fs, path::Path};
+
+use tempfile::Builder;
+
+use rkv::{
+ backend::{Lmdb, LmdbEnvironment, SafeMode, SafeModeEnvironment},
+ Manager, Migrator, Rkv, StoreOptions, Value,
+};
+
+macro_rules! populate_store {
+ ($env:expr) => {
+ let store = $env
+ .open_single("store", StoreOptions::create())
+ .expect("opened");
+ let mut writer = $env.write().expect("writer");
+ store
+ .put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ store
+ .put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ store
+ .put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ };
+}
+
+#[test]
+fn test_open_migrator_lmdb_to_safe() {
+ let root = Builder::new()
+ .prefix("test_open_migrator_lmdb_to_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Populate source environment and persist to disk.
+ {
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+ }
+ // Check if the files were written to disk.
+ {
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ assert!(datamdb.exists());
+ assert!(lockmdb.exists());
+ }
+ // Verify that database was written to disk.
+ {
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let store = src_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = src_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Open and migrate.
+ {
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env)
+ .expect("migrated");
+ }
+ // Verify that the database was indeed migrated.
+ {
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let store = dst_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = dst_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Check if the old files were deleted from disk.
+ {
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ assert!(!datamdb.exists());
+ assert!(!lockmdb.exists());
+ }
+}
+
+#[test]
+fn test_open_migrator_safe_to_lmdb() {
+ let root = Builder::new()
+ .prefix("test_open_migrator_safe_to_lmdb")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Populate source environment and persist to disk.
+ {
+ let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+ }
+ // Check if the files were written to disk.
+ {
+ let mut safebin = root.path().to_path_buf();
+ safebin.push("data.safe.bin");
+ assert!(safebin.exists());
+ }
+ // Verify that database was written to disk.
+ {
+ let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let store = src_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = src_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Open and migrate.
+ {
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env)
+ .expect("migrated");
+ }
+ // Verify that the database was indeed migrated.
+ {
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let store = dst_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = dst_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Check if the old files were deleted from disk.
+ {
+ let mut safebin = root.path().to_path_buf();
+ safebin.push("data.safe.bin");
+ assert!(!safebin.exists());
+ }
+}
+
+#[test]
+fn test_open_migrator_round_trip() {
+ let root = Builder::new()
+ .prefix("test_open_migrator_lmdb_to_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Populate source environment and persist to disk.
+ {
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+ }
+ // Open and migrate.
+ {
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env)
+ .expect("migrated");
+ }
+ // Open and migrate back.
+ {
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env)
+ .expect("migrated");
+ }
+ // Verify that the database was indeed migrated twice.
+ {
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let store = dst_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = dst_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+ // Check if the right files are finally present on disk.
+ {
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ let mut safebin = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ safebin.push("data.safe.bin");
+ assert!(datamdb.exists());
+ assert!(lockmdb.exists());
+ assert!(!safebin.exists());
+ }
+}
+
+#[test]
+fn test_easy_migrator_no_dir_1() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_no_dir")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // This won't fail with IoError even though the path is a bogus path, because this
+ // is the "easy mode" migration which automatically handles (ignores) this error.
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::easy_migrate_lmdb_to_safe_mode(Path::new("bogus"), &dst_env).expect("migrated");
+
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ let mut safebin = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ safebin.push("data.safe.bin");
+ assert!(!datamdb.exists());
+ assert!(!lockmdb.exists());
+ assert!(!safebin.exists()); // safe mode doesn't write an empty db to disk
+}
+
+#[test]
+fn test_easy_migrator_no_dir_2() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_no_dir")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // This won't fail with IoError even though the path is a bogus path, because this
+ // is the "easy mode" migration which automatically handles (ignores) this error.
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::easy_migrate_safe_mode_to_lmdb(Path::new("bogus"), &dst_env).expect("migrated");
+
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ let mut safebin = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ safebin.push("data.safe.bin");
+ assert!(datamdb.exists()); // lmdb writes an empty db to disk
+ assert!(lockmdb.exists());
+ assert!(!safebin.exists());
+}
+
+#[test]
+fn test_easy_migrator_invalid_1() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_invalid")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let dbfile = root.path().join("data.mdb");
+ fs::write(dbfile, "bogus").expect("dbfile created");
+
+ // This won't fail with FileInvalid even though the database is a bogus file, because this
+ // is the "easy mode" migration which automatically handles (ignores) this error.
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated");
+
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ let mut safebin = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ safebin.push("data.safe.bin");
+ assert!(datamdb.exists()); // corrupted db isn't deleted
+ assert!(lockmdb.exists());
+ assert!(!safebin.exists());
+}
+
+#[test]
+fn test_easy_migrator_invalid_2() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_invalid")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let dbfile = root.path().join("data.safe.bin");
+ fs::write(dbfile, "bogus").expect("dbfile created");
+
+ // This won't fail with FileInvalid even though the database is a bogus file, because this
+ // is the "easy mode" migration which automatically handles (ignores) this error.
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated");
+
+ let mut datamdb = root.path().to_path_buf();
+ let mut lockmdb = root.path().to_path_buf();
+ let mut safebin = root.path().to_path_buf();
+ datamdb.push("data.mdb");
+ lockmdb.push("lock.mdb");
+ safebin.push("data.safe.bin");
+ assert!(datamdb.exists()); // lmdb writes an empty db to disk
+ assert!(lockmdb.exists());
+ assert!(safebin.exists()); // corrupted db isn't deleted
+}
+
+#[test]
+#[should_panic(expected = "migrated: SourceEmpty")]
+fn test_migrator_lmdb_to_safe_1() {
+ let root = Builder::new()
+ .prefix("test_migrate_lmdb_to_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated");
+}
+
+#[test]
+#[should_panic(expected = "migrated: DestinationNotEmpty")]
+fn test_migrator_lmdb_to_safe_2() {
+ let root = Builder::new()
+ .prefix("test_migrate_lmdb_to_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ populate_store!(&dst_env);
+ Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated");
+}
+
+#[test]
+fn test_migrator_lmdb_to_safe_3() {
+ let root = Builder::new()
+ .prefix("test_migrate_lmdb_to_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated");
+
+ let store = dst_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = dst_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+}
+
+#[test]
+#[should_panic(expected = "migrated: SourceEmpty")]
+fn test_migrator_safe_to_lmdb_1() {
+ let root = Builder::new()
+ .prefix("test_migrate_safe_to_lmdb")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated");
+}
+
+#[test]
+#[should_panic(expected = "migrated: DestinationNotEmpty")]
+fn test_migrator_safe_to_lmdb_2() {
+ let root = Builder::new()
+ .prefix("test_migrate_safe_to_lmdb")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ populate_store!(&dst_env);
+ Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated");
+}
+
+#[test]
+fn test_migrator_safe_to_lmdb_3() {
+ let root = Builder::new()
+ .prefix("test_migrate_safe_to_lmdb")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated");
+
+ let store = dst_env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = dst_env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+}
+
+#[test]
+fn test_easy_migrator_failed_migration_1() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_failed_migration_1")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let dbfile = root.path().join("data.mdb");
+ fs::write(&dbfile, "bogus").expect("bogus dbfile created");
+
+ // This won't fail with FileInvalid even though the database is a bogus file, because this
+ // is the "easy mode" migration which automatically handles (ignores) this error.
+ let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated");
+
+ // Populate destination environment and persist to disk.
+ populate_store!(&dst_env);
+ dst_env.sync(true).expect("synced");
+
+ // Delete bogus file and create a valid source environment in its place.
+ fs::remove_file(&dbfile).expect("bogus dbfile removed");
+ let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+
+ // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty.
+ Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated");
+}
+
+#[test]
+fn test_easy_migrator_failed_migration_2() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_failed_migration_2")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let dbfile = root.path().join("data.safe.bin");
+ fs::write(&dbfile, "bogus").expect("bogus dbfile created");
+
+ // This won't fail with FileInvalid even though the database is a bogus file, because this
+ // is the "easy mode" migration which automatically handles (ignores) this error.
+ let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded");
+ Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated");
+
+ // Populate destination environment and persist to disk.
+ populate_store!(&dst_env);
+ dst_env.sync(true).expect("synced");
+
+ // Delete bogus file and create a valid source environment in its place.
+ fs::remove_file(&dbfile).expect("bogus dbfile removed");
+ let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+
+ // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty.
+ Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated");
+}
+
+fn test_easy_migrator_from_manager_failed_migration_1() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_from_manager_failed_migration_1")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ {
+ let mut src_manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
+ let created_src_arc = src_manager
+ .get_or_create(root.path(), Rkv::new::<Lmdb>)
+ .unwrap();
+ let src_env = created_src_arc.read().unwrap();
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+ }
+ {
+ let mut dst_manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
+ let created_dst_arc_1 = dst_manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .unwrap();
+ let dst_env_1 = created_dst_arc_1.read().unwrap();
+ populate_store!(&dst_env_1);
+ dst_env_1.sync(true).expect("synced");
+ }
+
+ // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty.
+ let dst_manager = Manager::<SafeModeEnvironment>::singleton().read().unwrap();
+ let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap();
+ let dst_env_2 = created_dst_arc_2.read().unwrap();
+ Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), dst_env_2).expect("migrated");
+}
+
+fn test_easy_migrator_from_manager_failed_migration_2() {
+ let root = Builder::new()
+ .prefix("test_easy_migrator_from_manager_failed_migration_2")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ {
+ let mut src_manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap();
+ let created_src_arc = src_manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .unwrap();
+ let src_env = created_src_arc.read().unwrap();
+ populate_store!(&src_env);
+ src_env.sync(true).expect("synced");
+ }
+ {
+ let mut dst_manager = Manager::<LmdbEnvironment>::singleton().write().unwrap();
+ let created_dst_arc_1 = dst_manager
+ .get_or_create(root.path(), Rkv::new::<Lmdb>)
+ .unwrap();
+ let dst_env_1 = created_dst_arc_1.read().unwrap();
+ populate_store!(&dst_env_1);
+ dst_env_1.sync(true).expect("synced");
+ }
+
+ // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty.
+ let dst_manager = Manager::<LmdbEnvironment>::singleton().read().unwrap();
+ let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap();
+ let dst_env_2 = created_dst_arc_2.read().unwrap();
+ Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), dst_env_2).expect("migrated");
+}
+
+#[test]
+fn test_easy_migrator_from_manager_failed_migration() {
+ test_easy_migrator_from_manager_failed_migration_1();
+ test_easy_migrator_from_manager_failed_migration_2();
+}
diff --git a/third_party/rust/rkv/tests/env-safe.rs b/third_party/rust/rkv/tests/env-safe.rs
new file mode 100644
index 0000000000..9c20e39507
--- /dev/null
+++ b/third_party/rust/rkv/tests/env-safe.rs
@@ -0,0 +1,1370 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable
+// deprecates `clippy::cyclomatic_complexity`.
+#![allow(clippy::complexity)]
+
+use std::{
+ fs,
+ path::Path,
+ str,
+ sync::{Arc, RwLock},
+ thread,
+};
+
+use byteorder::{ByteOrder, LittleEndian};
+use tempfile::Builder;
+
+use rkv::{
+ backend::{
+ BackendEnvironmentBuilder, SafeMode, SafeModeDatabase, SafeModeEnvironment,
+ SafeModeRwTransaction,
+ },
+ Rkv, SingleStore, StoreError, StoreOptions, Value, Writer,
+};
+
+fn check_rkv(k: &Rkv<SafeModeEnvironment>) {
+ let _ = k
+ .open_single(None, StoreOptions::create())
+ .expect("created default");
+
+ let s = k.open_single("s", StoreOptions::create()).expect("opened");
+ let reader = k.read().expect("reader");
+
+ let result = s.get(&reader, "foo");
+ assert_eq!(None, result.expect("success but no value"));
+}
+
+/// We can't open a directory that doesn't exist.
+#[test]
+fn test_open_fails_safe() {
+ let root = Builder::new()
+ .prefix("test_open_fails_safe")
+ .tempdir()
+ .expect("tempdir");
+ assert!(root.path().exists());
+
+ let nope = root.path().join("nope/");
+ assert!(!nope.exists());
+
+ let pb = nope.to_path_buf();
+ match Rkv::new::<SafeMode>(nope.as_path()).err() {
+ Some(StoreError::UnsuitableEnvironmentPath(p)) => {
+ assert_eq!(pb, p);
+ }
+ _ => panic!("expected error"),
+ };
+}
+
+#[test]
+fn test_open_safe() {
+ let root = Builder::new()
+ .prefix("test_open_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ check_rkv(&k);
+}
+
+#[test]
+fn test_open_from_builder_safe() {
+ let root = Builder::new()
+ .prefix("test_open_from_builder_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let mut builder = Rkv::environment_builder::<SafeMode>();
+ builder.set_max_dbs(2);
+
+ let k = Rkv::from_builder(root.path(), builder).expect("rkv");
+ check_rkv(&k);
+}
+
+#[test]
+fn test_open_from_builder_with_dir_safe_1() {
+ let root = Builder::new()
+ .prefix("test_open_from_builder_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+
+ let mut builder = Rkv::environment_builder::<SafeMode>();
+ builder.set_max_dbs(2);
+ builder.set_make_dir_if_needed(true);
+
+ let k = Rkv::from_builder(root.path(), builder).expect("rkv");
+ check_rkv(&k);
+}
+
+#[test]
+#[should_panic(expected = "rkv: UnsuitableEnvironmentPath(\"bogus\")")]
+fn test_open_from_builder_with_dir_safe_2() {
+ let root = Path::new("bogus");
+ println!("Root path: {root:?}");
+ assert!(!root.is_dir());
+
+ let mut builder = Rkv::environment_builder::<SafeMode>();
+ builder.set_max_dbs(2);
+
+ let k = Rkv::from_builder(root, builder).expect("rkv");
+ check_rkv(&k);
+}
+
+#[test]
+#[should_panic(expected = "opened: DbsFull")]
+fn test_create_with_capacity_safe_1() {
+ let root = Builder::new()
+ .prefix("test_create_with_capacity_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ // This errors with "opened: DbsFull" because we specified a capacity of one (database),
+ // and check_rkv already opened one (plus the default database, which doesn't count
+ // against the limit).
+ let _zzz = k
+ .open_single("zzz", StoreOptions::create())
+ .expect("opened");
+}
+
+#[test]
+fn test_create_with_capacity_safe_2() {
+ let root = Builder::new()
+ .prefix("test_create_with_capacity_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ // This doesn't error with "opened: DbsFull" because even though we specified a capacity
+ // of one (database), and check_rkv already opened one, the default database doesn't
+ // count against the limit).
+ let _zzz = k.open_single(None, StoreOptions::create()).expect("opened");
+}
+
+#[test]
+#[should_panic(expected = "opened: SafeModeError(DbNotFoundError)")]
+fn test_open_with_capacity_safe_1() {
+ let root = Builder::new()
+ .prefix("test_open_with_capacity_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ let _zzz = k
+ .open_single("zzz", StoreOptions::default())
+ .expect("opened");
+}
+
+#[test]
+fn test_open_with_capacity_safe_2() {
+ let root = Builder::new()
+ .prefix("test_open_with_capacity_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ let _zzz = k
+ .open_single(None, StoreOptions::default())
+ .expect("opened");
+}
+
+#[test]
+fn test_list_dbs_safe_1() {
+ let root = Builder::new()
+ .prefix("test_list_dbs_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv");
+ check_rkv(&k);
+
+ let mut dbs = k.get_dbs().unwrap();
+ dbs.sort();
+ assert_eq!(dbs, vec![None, Some("s".to_owned())]);
+}
+
+#[test]
+fn test_list_dbs_safe_2() {
+ let root = Builder::new()
+ .prefix("test_list_dbs_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 2).expect("rkv");
+ check_rkv(&k);
+
+ let _ = k
+ .open_single("zzz", StoreOptions::create())
+ .expect("opened");
+
+ let mut dbs = k.get_dbs().unwrap();
+ dbs.sort();
+ assert_eq!(
+ dbs,
+ vec![None, Some("s".to_owned()), Some("zzz".to_owned())]
+ );
+}
+
+#[test]
+fn test_list_dbs_safe_3() {
+ let root = Builder::new()
+ .prefix("test_list_dbs_safe")
+ .tempdir()
+ .expect("tempdir");
+ println!("Root path: {:?}", root.path());
+ fs::create_dir_all(root.path()).expect("dir created");
+ assert!(root.path().is_dir());
+
+ let k = Rkv::with_capacity::<SafeMode>(root.path(), 0).expect("rkv");
+
+ let _ = k.open_single(None, StoreOptions::create()).expect("opened");
+
+ let mut dbs = k.get_dbs().unwrap();
+ dbs.sort();
+ assert_eq!(dbs, vec![None]);
+}
+
+#[test]
+fn test_round_trip_and_transactions_safe() {
+ let root = Builder::new()
+ .prefix("test_round_trip_and_transactions_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&writer, "noo").expect("read"),
+ Some(Value::F64(1234.0.into()))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+
+ // Isolation. Reads won't return values.
+ let r = &k.read().unwrap();
+ assert_eq!(sk.get(r, "foo").expect("read"), None);
+ assert_eq!(sk.get(r, "bar").expect("read"), None);
+ assert_eq!(sk.get(r, "baz").expect("read"), None);
+ }
+
+ // Dropped: tx rollback. Reads will still return nothing.
+
+ {
+ let r = &k.read().unwrap();
+ assert_eq!(sk.get(r, "foo").expect("read"), None);
+ assert_eq!(sk.get(r, "bar").expect("read"), None);
+ assert_eq!(sk.get(r, "baz").expect("read"), None);
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ sk.get(&writer, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ sk.get(&writer, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+
+ writer.commit().expect("committed");
+ }
+
+ // Committed. Reads will succeed.
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
+ assert_eq!(
+ sk.get(&r, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.delete(&mut writer, "foo").expect("deleted");
+ sk.delete(&mut writer, "bar").expect("deleted");
+ sk.delete(&mut writer, "baz").expect("deleted");
+ assert_eq!(sk.get(&writer, "foo").expect("read"), None);
+ assert_eq!(sk.get(&writer, "bar").expect("read"), None);
+ assert_eq!(sk.get(&writer, "baz").expect("read"), None);
+
+ // Isolation. Reads still return values.
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
+ assert_eq!(
+ sk.get(&r, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+
+ // Dropped: tx rollback. Reads will still return values.
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true)));
+ assert_eq!(
+ sk.get(&r, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.delete(&mut writer, "foo").expect("deleted");
+ sk.delete(&mut writer, "bar").expect("deleted");
+ sk.delete(&mut writer, "baz").expect("deleted");
+ assert_eq!(sk.get(&writer, "foo").expect("read"), None);
+ assert_eq!(sk.get(&writer, "bar").expect("read"), None);
+ assert_eq!(sk.get(&writer, "baz").expect("read"), None);
+
+ writer.commit().expect("committed");
+ }
+
+ // Committed. Reads will succeed but return None to indicate a missing value.
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(sk.get(&r, "foo").expect("read"), None);
+ assert_eq!(sk.get(&r, "bar").expect("read"), None);
+ assert_eq!(sk.get(&r, "baz").expect("read"), None);
+ }
+}
+
+#[test]
+fn test_single_store_clear_safe() {
+ let root = Builder::new()
+ .prefix("test_single_store_clear_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ sk.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let r = k.read().unwrap();
+ let iter = sk.iter_start(&r).expect("iter");
+ assert_eq!(iter.count(), 0);
+ }
+}
+
+#[test]
+#[should_panic(expected = "KeyValuePairNotFound")]
+fn test_single_store_delete_nonexistent_safe() {
+ let root = Builder::new()
+ .prefix("test_single_store_delete_nonexistent_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.delete(&mut writer, "bogus").unwrap();
+}
+
+#[test]
+#[cfg(feature = "db-dup-sort")]
+fn test_multi_put_get_del_safe() {
+ let root = Builder::new()
+ .prefix("test_multi_put_get_del_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap();
+
+ let mut writer = k.write().unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 bar"))
+ .unwrap();
+ writer.commit().unwrap();
+
+ let writer = k.write().unwrap();
+ {
+ let mut iter = multistore.get(&writer, "str1").unwrap();
+ let (id, val) = iter.next().unwrap().unwrap();
+ assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 bar")));
+ let (id, val) = iter.next().unwrap().unwrap();
+ assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 foo")));
+ }
+ writer.commit().unwrap();
+
+ let mut writer = k.write().unwrap();
+ multistore
+ .delete(&mut writer, "str1", &Value::Str("str1 foo"))
+ .unwrap();
+ assert_eq!(
+ multistore.get_first(&writer, "str1").unwrap(),
+ Some(Value::Str("str1 bar"))
+ );
+ multistore
+ .delete(&mut writer, "str2", &Value::Str("str2 bar"))
+ .unwrap();
+ assert_eq!(
+ multistore.get_first(&writer, "str2").unwrap(),
+ Some(Value::Str("str2 foo"))
+ );
+ multistore.delete_all(&mut writer, "str3").unwrap();
+ assert_eq!(multistore.get_first(&writer, "str3").unwrap(), None);
+ writer.commit().unwrap();
+}
+
+#[test]
+#[cfg(feature = "db-dup-sort")]
+fn test_multiple_store_clear_safe() {
+ let root = Builder::new()
+ .prefix("test_multiple_store_clear_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let multistore = k
+ .open_multi("multistore", StoreOptions::create())
+ .expect("opened");
+
+ {
+ let mut writer = k.write().expect("writer");
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str1", &Value::Str("str1 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str2", &Value::Str("str2 bar"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 foo"))
+ .unwrap();
+ multistore
+ .put(&mut writer, "str3", &Value::Str("str3 bar"))
+ .unwrap();
+ writer.commit().expect("committed");
+ }
+
+ {
+ let mut writer = k.write().expect("writer");
+ multistore.clear(&mut writer).expect("cleared");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let r = k.read().unwrap();
+ assert_eq!(multistore.get_first(&r, "str1").expect("read"), None);
+ assert_eq!(multistore.get_first(&r, "str2").expect("read"), None);
+ assert_eq!(multistore.get_first(&r, "str3").expect("read"), None);
+ }
+}
+
+#[test]
+fn test_open_store_for_read_safe() {
+ let root = Builder::new()
+ .prefix("test_open_store_for_read_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+
+ // First create the store, and start a write transaction on it.
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("write");
+
+ // Open the same store for read, note that the write transaction is still in progress,
+ // it should not block the reader though.
+ let sk_readonly = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+ writer.commit().expect("commit");
+
+ // Now the write transaction is committed, any followed reads should see its change.
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk_readonly.get(&reader, "foo").expect("read"),
+ Some(Value::Str("bar"))
+ );
+}
+
+#[test]
+#[should_panic(expected = "open a missing store")]
+fn test_open_a_missing_store_safe() {
+ let root = Builder::new()
+ .prefix("test_open_a_missing_store_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let _sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("open a missing store");
+}
+
+#[test]
+#[should_panic(expected = "new failed: FileInvalid")]
+fn test_open_a_broken_store_safe() {
+ let root = Builder::new()
+ .prefix("test_open_a_missing_store_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let dbfile = root.path().join("data.safe.bin");
+ fs::write(dbfile, "bogus").expect("dbfile created");
+
+ let _ = Rkv::new::<SafeMode>(root.path()).expect("new failed");
+}
+
+#[test]
+fn test_open_fail_with_badrslot_safe() {
+ let root = Builder::new()
+ .prefix("test_open_fail_with_badrslot_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+
+ // First create the store
+ let _sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Open a reader on this store
+ let _reader = k.read().expect("reader");
+
+ // Open the same store for read while the reader is in progress will panic
+ let store = k.open_single("sk", StoreOptions::default());
+ match store {
+ Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (),
+ _ => panic!("should panic"),
+ }
+}
+
+#[test]
+fn test_create_fail_with_badrslot_safe() {
+ let root = Builder::new()
+ .prefix("test_create_fail_with_badrslot_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+
+ // First create the store
+ let _sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Open a reader on this store
+ let _reader = k.read().expect("reader");
+
+ // Open the same store for read while the reader is in progress will panic
+ let store = k.open_single("sk", StoreOptions::create());
+ match store {
+ Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (),
+ _ => panic!("should panic"),
+ }
+}
+
+#[test]
+fn test_read_before_write_num_safe() {
+ let root = Builder::new()
+ .prefix("test_read_before_write_num_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Test reading a number, modifying it, and then writing it back.
+ // We have to be done with the Value::I64 before calling Writer::put,
+ // as the Value::I64 borrows an immutable reference to the Writer.
+ // So we extract and copy its primitive value.
+
+ fn get_existing_foo(
+ store: SingleStore<SafeModeDatabase>,
+ writer: &Writer<SafeModeRwTransaction>,
+ ) -> Option<i64> {
+ match store.get(writer, "foo").expect("read") {
+ Some(Value::I64(val)) => Some(val),
+ _ => None,
+ }
+ }
+
+ let mut writer = k.write().expect("writer");
+ let mut existing = get_existing_foo(sk, &writer).unwrap_or(99);
+ existing += 1;
+ sk.put(&mut writer, "foo", &Value::I64(existing))
+ .expect("success");
+
+ let updated = get_existing_foo(sk, &writer).unwrap_or(99);
+ assert_eq!(updated, 100);
+ writer.commit().expect("commit");
+}
+
+#[test]
+fn test_read_before_write_str_safe() {
+ let root = Builder::new()
+ .prefix("test_read_before_write_str_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // Test reading a string, modifying it, and then writing it back.
+ // We have to be done with the Value::Str before calling Writer::put,
+ // as the Value::Str (and its underlying &str) borrows an immutable
+ // reference to the Writer. So we copy it to a String.
+
+ fn get_existing_foo(
+ store: SingleStore<SafeModeDatabase>,
+ writer: &Writer<SafeModeRwTransaction>,
+ ) -> Option<String> {
+ match store.get(writer, "foo").expect("read") {
+ Some(Value::Str(val)) => Some(val.to_string()),
+ _ => None,
+ }
+ }
+
+ let mut writer = k.write().expect("writer");
+ let mut existing = get_existing_foo(sk, &writer).unwrap_or_default();
+ existing.push('…');
+ sk.put(&mut writer, "foo", &Value::Str(&existing))
+ .expect("write");
+
+ let updated = get_existing_foo(sk, &writer).unwrap_or_default();
+ assert_eq!(updated, "…");
+ writer.commit().expect("commit");
+}
+
+#[test]
+fn test_isolation_safe() {
+ let root = Builder::new()
+ .prefix("test_isolation_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_single("s", StoreOptions::create()).expect("opened");
+
+ // Add one field.
+ {
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ {
+ let reader = k.read().unwrap();
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+ }
+
+ // Establish a long-lived reader that outlasts a writer.
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+
+ // Start a write transaction.
+ let mut writer = k.write().expect("writer");
+ s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote");
+
+ // The reader and writer are isolated.
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+ assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999)));
+
+ // If we commit the writer, we still have isolation.
+ writer.commit().expect("committed");
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234)));
+
+ // A new reader sees the committed value. Note that LMDB doesn't allow two
+ // read transactions to exist in the same thread, so we abort the previous one.
+ reader.abort();
+ let reader = k.read().expect("reader");
+ assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999)));
+}
+
+#[test]
+fn test_blob_safe() {
+ let root = Builder::new()
+ .prefix("test_round_trip_blob_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ assert_eq!(sk.get(&writer, "foo").expect("read"), None);
+ sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4]))
+ .expect("wrote");
+ assert_eq!(
+ sk.get(&writer, "foo").expect("read"),
+ Some(Value::Blob(&[1, 2, 3, 4]))
+ );
+
+ fn u16_to_u8(src: &[u16]) -> Vec<u8> {
+ let mut dst = vec![0; 2 * src.len()];
+ LittleEndian::write_u16_into(src, &mut dst);
+ dst
+ }
+
+ fn u8_to_u16(src: &[u8]) -> Vec<u16> {
+ let mut dst = vec![0; src.len() / 2];
+ LittleEndian::read_u16_into(src, &mut dst);
+ dst
+ }
+
+ // When storing UTF-16 strings as blobs, we'll need to convert
+ // their [u16] backing storage to [u8]. Test that converting, writing,
+ // reading, and converting back works as expected.
+ let u16_array = [1000, 10000, 54321, 65535];
+ assert_eq!(sk.get(&writer, "bar").expect("read"), None);
+ sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array)))
+ .expect("wrote");
+ let u8_array = match sk.get(&writer, "bar").expect("read") {
+ Some(Value::Blob(val)) => val,
+ _ => &[],
+ };
+ assert_eq!(u8_to_u16(u8_array), u16_array);
+}
+
+#[test]
+fn test_sync_safe() {
+ let root = Builder::new()
+ .prefix("test_sync_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let mut builder = Rkv::environment_builder::<SafeMode>();
+ builder.set_max_dbs(1);
+ {
+ let k = Rkv::from_builder(root.path(), builder).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+ {
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ k.sync(true).expect("synced");
+ }
+ }
+ let k = Rkv::from_builder(root.path(), builder).expect("new succeeded");
+ let sk = k
+ .open_single("sk", StoreOptions::default())
+ .expect("opened");
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ sk.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+}
+
+#[test]
+fn test_iter_safe() {
+ let root = Builder::new()
+ .prefix("test_iter_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ // An iterator over an empty store returns no values.
+ {
+ let reader = k.read().unwrap();
+ let mut iter = sk.iter_start(&reader).unwrap();
+ assert!(iter.next().is_none());
+ }
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!"))
+ .expect("wrote");
+ sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+
+ // Reader.iter() returns (key, value) tuples ordered by key.
+ let mut iter = sk.iter_start(&reader).unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "bar");
+ assert_eq!(val, Value::Bool(true));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "baz");
+ assert_eq!(val, Value::Str("héllo, yöu"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "foo");
+ assert_eq!(val, Value::I64(1234));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
+ assert_eq!(val, Value::Str("Emil.RuleZ!"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterators don't loop. Once one returns None, additional calls
+ // to its next() method will always return None.
+ assert!(iter.next().is_none());
+
+ // Reader.iter_from() begins iteration at the first key equal to
+ // or greater than the given key.
+ let mut iter = sk.iter_from(&reader, "moo").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Reader.iter_from() works as expected when the given key is a prefix
+ // of a key in the store.
+ let mut iter = sk.iter_from(&reader, "no").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+}
+
+#[test]
+fn test_iter_from_key_greater_than_existing_safe() {
+ let root = Builder::new()
+ .prefix("test_iter_from_key_greater_than_existing_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let sk = k.open_single("sk", StoreOptions::create()).expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ sk.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ sk.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ sk.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ sk.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+ let mut iter = sk.iter_from(&reader, "nuu").unwrap();
+ assert!(iter.next().is_none());
+}
+
+#[test]
+fn test_multiple_store_read_write_safe() {
+ let root = Builder::new()
+ .prefix("test_multiple_store_read_write_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let s1 = k
+ .open_single("store_1", StoreOptions::create())
+ .expect("opened");
+ let s2 = k
+ .open_single("store_2", StoreOptions::create())
+ .expect("opened");
+ let s3 = k
+ .open_single("store_3", StoreOptions::create())
+ .expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ s1.put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote");
+ s3.put(&mut writer, "foo", &Value::Bool(true))
+ .expect("wrote");
+
+ assert_eq!(
+ s1.get(&writer, "foo").expect("read"),
+ Some(Value::Str("bar"))
+ );
+ assert_eq!(s2.get(&writer, "foo").expect("read"), Some(Value::I64(123)));
+ assert_eq!(
+ s3.get(&writer, "foo").expect("read"),
+ Some(Value::Bool(true))
+ );
+
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("unbound_reader");
+ assert_eq!(
+ s1.get(&reader, "foo").expect("read"),
+ Some(Value::Str("bar"))
+ );
+ assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123)));
+ assert_eq!(
+ s3.get(&reader, "foo").expect("read"),
+ Some(Value::Bool(true))
+ );
+ reader.abort();
+
+ // test delete across multiple stores
+ let mut writer = k.write().expect("writer");
+ s1.delete(&mut writer, "foo").expect("deleted");
+ s2.delete(&mut writer, "foo").expect("deleted");
+ s3.delete(&mut writer, "foo").expect("deleted");
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(s1.get(&reader, "key").expect("value"), None);
+ assert_eq!(s2.get(&reader, "key").expect("value"), None);
+ assert_eq!(s3.get(&reader, "key").expect("value"), None);
+}
+
+#[test]
+fn test_multiple_store_iter_safe() {
+ let root = Builder::new()
+ .prefix("test_multiple_store_iter_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let s1 = k
+ .open_single("store_1", StoreOptions::create())
+ .expect("opened");
+ let s2 = k
+ .open_single("store_2", StoreOptions::create())
+ .expect("opened");
+
+ let mut writer = k.write().expect("writer");
+ // Write to "s1"
+ s1.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ s1.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ s1.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ s1.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!"))
+ .expect("wrote");
+ s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則"))
+ .expect("wrote");
+ // &mut writer to "s2"
+ s2.put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ s2.put(&mut writer, "noo", &Value::F64(1234.0.into()))
+ .expect("wrote");
+ s2.put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ s2.put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!"))
+ .expect("wrote");
+ s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+
+ let reader = k.read().unwrap();
+
+ // Iterate through the whole store in "s1"
+ let mut iter = s1.iter_start(&reader).unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "bar");
+ assert_eq!(val, Value::Bool(true));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "baz");
+ assert_eq!(val, Value::Str("héllo, yöu"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "foo");
+ assert_eq!(val, Value::I64(1234));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
+ assert_eq!(val, Value::Str("Emil.RuleZ!"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate through the whole store in "s2"
+ let mut iter = s2.iter_start(&reader).unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "bar");
+ assert_eq!(val, Value::Bool(true));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "baz");
+ assert_eq!(val, Value::Str("héllo, yöu"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "foo");
+ assert_eq!(val, Value::I64(1234));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst");
+ assert_eq!(val, Value::Str("Emil.RuleZ!"));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given key in "s1"
+ let mut iter = s1.iter_from(&reader, "moo").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given key in "s2"
+ let mut iter = s2.iter_from(&reader, "moo").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given prefix in "s1"
+ let mut iter = s1.iter_from(&reader, "no").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+
+ // Iterate from a given prefix in "s2"
+ let mut iter = s2.iter_from(&reader, "no").unwrap();
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "noo");
+ assert_eq!(val, Value::F64(1234.0.into()));
+ let (key, val) = iter.next().unwrap().unwrap();
+ assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客");
+ assert_eq!(val, Value::Str("米克規則"));
+ assert!(iter.next().is_none());
+}
+
+#[test]
+fn test_store_multiple_thread_safe() {
+ let root = Builder::new()
+ .prefix("test_multiple_thread_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let rkv_arc = Arc::new(RwLock::new(
+ Rkv::new::<SafeMode>(root.path()).expect("new succeeded"),
+ ));
+ let store = rkv_arc
+ .read()
+ .unwrap()
+ .open_single("test", StoreOptions::create())
+ .expect("opened");
+
+ let num_threads = 10;
+ let mut write_handles = Vec::with_capacity(num_threads as usize);
+ let mut read_handles = Vec::with_capacity(num_threads as usize);
+
+ // Note that this isn't intended to demonstrate a good use of threads.
+ // For this shape of data, it would be more performant to write/read
+ // all values using one transaction in a single thread. The point here
+ // is just to confirm that a store can be shared by multiple threads.
+
+ // For each KV pair, spawn a thread that writes it to the store.
+ for i in 0..num_threads {
+ let rkv_arc = rkv_arc.clone();
+ write_handles.push(thread::spawn(move || {
+ let rkv = rkv_arc.write().expect("rkv");
+ let mut writer = rkv.write().expect("writer");
+ store
+ .put(&mut writer, i.to_string(), &Value::U64(i))
+ .expect("written");
+ writer.commit().unwrap();
+ }));
+ }
+ for handle in write_handles {
+ handle.join().expect("joined");
+ }
+
+ // For each KV pair, spawn a thread that reads it from the store
+ // and returns its value.
+ for i in 0..num_threads {
+ let rkv_arc = rkv_arc.clone();
+ read_handles.push(thread::spawn(move || {
+ let rkv = rkv_arc.read().expect("rkv");
+ let reader = rkv.read().expect("reader");
+ let value = match store.get(&reader, i.to_string()) {
+ Ok(Some(Value::U64(value))) => value,
+ Ok(Some(_)) => panic!("value type unexpected"),
+ Ok(None) => panic!("value not found"),
+ Err(err) => panic!("{}", err),
+ };
+ assert_eq!(value, i);
+ value
+ }));
+ }
+
+ // Sum the values returned from the threads and confirm that they're
+ // equal to the sum of values written to the threads.
+ let thread_sum: u64 = read_handles
+ .into_iter()
+ .map(|handle| handle.join().expect("value"))
+ .sum();
+ assert_eq!(thread_sum, (0..num_threads).sum());
+}
+
+#[test]
+fn test_use_value_as_key_safe() {
+ let root = Builder::new()
+ .prefix("test_use_value_as_key_safe")
+ .tempdir()
+ .expect("tempdir");
+ let rkv = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let store = rkv
+ .open_single("store", StoreOptions::create())
+ .expect("opened");
+
+ {
+ let mut writer = rkv.write().expect("writer");
+ store
+ .put(&mut writer, "foo", &Value::Str("bar"))
+ .expect("wrote");
+ store
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ // It's possible to retrieve a value with a Reader and then use it
+ // as a key with a Writer.
+ {
+ let reader = &rkv.read().unwrap();
+ if let Some(Value::Str(key)) = store.get(reader, "foo").expect("read") {
+ let mut writer = rkv.write().expect("writer");
+ store.delete(&mut writer, key).expect("deleted");
+ writer.commit().expect("committed");
+ }
+ }
+
+ {
+ let mut writer = rkv.write().expect("writer");
+ store
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ // You can also retrieve a Value with a Writer and then use it as a key
+ // with the same Writer if you copy the value to an owned type
+ // so the Writer isn't still being borrowed by the retrieved value
+ // when you try to borrow the Writer again to modify that value.
+ {
+ let mut writer = rkv.write().expect("writer");
+ if let Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") {
+ let key = value.to_owned();
+ store.delete(&mut writer, key).expect("deleted");
+ writer.commit().expect("committed");
+ }
+ }
+
+ {
+ let name1 = rkv
+ .open_single("name1", StoreOptions::create())
+ .expect("opened");
+ let name2 = rkv
+ .open_single("name2", StoreOptions::create())
+ .expect("opened");
+ let mut writer = rkv.write().expect("writer");
+ name1
+ .put(&mut writer, "key1", &Value::Str("bar"))
+ .expect("wrote");
+ name1
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ name2
+ .put(&mut writer, "key2", &Value::Str("bar"))
+ .expect("wrote");
+ name2
+ .put(&mut writer, "bar", &Value::Str("baz"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ }
+
+ // You can also iterate (store, key) pairs to retrieve foreign keys,
+ // then iterate those foreign keys to modify/delete them.
+ //
+ // You need to open the stores in advance, since opening a store
+ // uses a write transaction internally, so opening them while a writer
+ // is extant will hang.
+ //
+ // And you need to copy the values to an owned type so the Writer isn't
+ // still being borrowed by a retrieved value when you try to borrow
+ // the Writer again to modify another value.
+ let fields = vec![
+ (
+ rkv.open_single("name1", StoreOptions::create())
+ .expect("opened"),
+ "key1",
+ ),
+ (
+ rkv.open_single("name2", StoreOptions::create())
+ .expect("opened"),
+ "key2",
+ ),
+ ];
+ {
+ let mut foreignkeys = Vec::new();
+ let mut writer = rkv.write().expect("writer");
+ for (store, key) in fields.iter() {
+ if let Some(Value::Str(value)) = store.get(&writer, key).expect("read") {
+ foreignkeys.push((store, value.to_owned()));
+ }
+ }
+ for (store, key) in foreignkeys.iter() {
+ store.delete(&mut writer, key).expect("deleted");
+ }
+ writer.commit().expect("committed");
+ }
+}
diff --git a/third_party/rust/rkv/tests/integer-store.rs b/third_party/rust/rkv/tests/integer-store.rs
new file mode 100644
index 0000000000..4f27776e79
--- /dev/null
+++ b/third_party/rust/rkv/tests/integer-store.rs
@@ -0,0 +1,88 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![cfg(feature = "db-int-key")]
+#![allow(clippy::many_single_char_names)]
+
+use std::fs;
+
+use serde_derive::Serialize;
+use tempfile::Builder;
+
+use rkv::{backend::SafeMode, PrimitiveInt, Rkv, StoreOptions, Value};
+
+#[test]
+fn test_integer_keys() {
+ let root = Builder::new()
+ .prefix("test_integer_keys")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let s = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ macro_rules! test_integer_keys {
+ ($store:expr, $key:expr) => {{
+ let mut writer = k.write().expect("writer");
+
+ $store
+ .put(&mut writer, $key, &Value::Str("hello!"))
+ .expect("write");
+ assert_eq!(
+ $store.get(&writer, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ assert_eq!(
+ $store.get(&reader, $key).expect("read"),
+ Some(Value::Str("hello!"))
+ );
+ }};
+ }
+
+ // The integer module provides only the u32 integer key variant
+ // of IntegerStore, so we can use it without further ado.
+ test_integer_keys!(s, std::u32::MIN);
+ test_integer_keys!(s, std::u32::MAX);
+
+ // If you want to use another integer key variant, you need to implement
+ // a newtype, implement PrimitiveInt, and implement or derive Serialize
+ // for it. Here we do so for the i32 type.
+
+ // DANGER! Doing this enables you to open a store with multiple,
+ // different integer key types, which may result in unexpected behavior.
+ // Make sure you know what you're doing!
+
+ let t = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ #[derive(Serialize)]
+ struct I32(i32);
+ impl PrimitiveInt for I32 {}
+ test_integer_keys!(t, I32(std::i32::MIN));
+ test_integer_keys!(t, I32(std::i32::MAX));
+
+ let u = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ #[derive(Serialize)]
+ struct U16(u16);
+ impl PrimitiveInt for U16 {}
+ test_integer_keys!(u, U16(std::u16::MIN));
+ test_integer_keys!(u, U16(std::u16::MAX));
+
+ let v = k.open_integer("s", StoreOptions::create()).expect("open");
+
+ #[derive(Serialize)]
+ struct U64(u64);
+ impl PrimitiveInt for U64 {}
+ test_integer_keys!(v, U64(std::u64::MIN));
+ test_integer_keys!(v, U64(std::u64::MAX));
+}
diff --git a/third_party/rust/rkv/tests/manager.rs b/third_party/rust/rkv/tests/manager.rs
new file mode 100644
index 0000000000..fe6b6cd026
--- /dev/null
+++ b/third_party/rust/rkv/tests/manager.rs
@@ -0,0 +1,380 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+
+use std::{fs, sync::Arc};
+
+use tempfile::Builder;
+
+#[cfg(feature = "lmdb")]
+use rkv::backend::{Lmdb, LmdbEnvironment};
+use rkv::{
+ backend::{BackendEnvironmentBuilder, SafeMode, SafeModeEnvironment},
+ CloseOptions, Rkv, StoreOptions, Value,
+};
+
+/// Test that a manager can be created with simple type inference.
+#[cfg(feature = "lmdb")]
+#[test]
+#[allow(clippy::let_underscore_lock)]
+fn test_simple() {
+ type Manager = rkv::Manager<LmdbEnvironment>;
+
+ let _unused = Manager::singleton().write().unwrap();
+}
+
+/// Test that a manager can be created with simple type inference.
+#[test]
+#[allow(clippy::let_underscore_lock)]
+fn test_simple_safe() {
+ type Manager = rkv::Manager<SafeModeEnvironment>;
+
+ let _unused = Manager::singleton().write().unwrap();
+}
+
+/// Test that a shared Rkv instance can be created with simple type inference.
+#[cfg(feature = "lmdb")]
+#[test]
+fn test_simple_2() {
+ type Manager = rkv::Manager<LmdbEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_simple_2")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let mut manager = Manager::singleton().write().unwrap();
+ let _ = manager
+ .get_or_create(root.path(), Rkv::new::<Lmdb>)
+ .unwrap();
+}
+
+/// Test that a shared Rkv instance can be created with simple type inference.
+#[test]
+fn test_simple_safe_2() {
+ type Manager = rkv::Manager<SafeModeEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_simple_safe_2")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let mut manager = Manager::singleton().write().unwrap();
+ let _ = manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .unwrap();
+}
+
+/// Test that the manager will return the same Rkv instance each time for each path.
+#[cfg(feature = "lmdb")]
+#[test]
+fn test_same() {
+ type Manager = rkv::Manager<LmdbEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_same")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let p = root.path();
+ assert!(Manager::singleton()
+ .read()
+ .unwrap()
+ .get(p)
+ .expect("success")
+ .is_none());
+
+ let created_arc = Manager::singleton()
+ .write()
+ .unwrap()
+ .get_or_create(p, Rkv::new::<Lmdb>)
+ .expect("created");
+ let fetched_arc = Manager::singleton()
+ .read()
+ .unwrap()
+ .get(p)
+ .expect("success")
+ .expect("existed");
+ assert!(Arc::ptr_eq(&created_arc, &fetched_arc));
+}
+
+/// Test that the manager will return the same Rkv instance each time for each path.
+#[test]
+fn test_same_safe() {
+ type Manager = rkv::Manager<SafeModeEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_same_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let p = root.path();
+ assert!(Manager::singleton()
+ .read()
+ .unwrap()
+ .get(p)
+ .expect("success")
+ .is_none());
+
+ let created_arc = Manager::singleton()
+ .write()
+ .unwrap()
+ .get_or_create(p, Rkv::new::<SafeMode>)
+ .expect("created");
+ let fetched_arc = Manager::singleton()
+ .read()
+ .unwrap()
+ .get(p)
+ .expect("success")
+ .expect("existed");
+ assert!(Arc::ptr_eq(&created_arc, &fetched_arc));
+}
+
+/// Test that the manager will return the same Rkv instance each time for each path.
+#[cfg(feature = "lmdb")]
+#[test]
+fn test_same_with_capacity() {
+ type Manager = rkv::Manager<LmdbEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_same_with_capacity")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let mut manager = Manager::singleton().write().unwrap();
+
+ let p = root.path();
+ assert!(manager.get(p).expect("success").is_none());
+
+ let created_arc = manager
+ .get_or_create_with_capacity(p, 10, Rkv::with_capacity::<Lmdb>)
+ .expect("created");
+ let fetched_arc = manager.get(p).expect("success").expect("existed");
+ assert!(Arc::ptr_eq(&created_arc, &fetched_arc));
+}
+
+/// Test that the manager will return the same Rkv instance each time for each path.
+#[test]
+fn test_same_with_capacity_safe() {
+ type Manager = rkv::Manager<SafeModeEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_same_with_capacity_safe")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let mut manager = Manager::singleton().write().unwrap();
+
+ let p = root.path();
+ assert!(manager.get(p).expect("success").is_none());
+
+ let created_arc = manager
+ .get_or_create_with_capacity(p, 10, Rkv::with_capacity::<SafeMode>)
+ .expect("created");
+ let fetched_arc = manager.get(p).expect("success").expect("existed");
+ assert!(Arc::ptr_eq(&created_arc, &fetched_arc));
+}
+
+/// Some storage drivers are able to discard when the database is corrupted at runtime.
+/// Test how these managers can discard corrupted databases and re-open.
+#[test]
+fn test_safe_mode_corrupt_while_open_1() {
+ type Manager = rkv::Manager<SafeModeEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_safe_mode_corrupt_while_open_1")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Create environment.
+ let mut manager = Manager::singleton().write().unwrap();
+ let shared_env = manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .expect("created");
+ let env = shared_env.read().unwrap();
+
+ // Write some data.
+ let store = env
+ .open_single("store", StoreOptions::create())
+ .expect("opened");
+ let mut writer = env.write().expect("writer");
+ store
+ .put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ store
+ .put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ store
+ .put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ env.sync(true).expect("synced");
+
+ // Verify it was flushed to disk.
+ let mut safebin = root.path().to_path_buf();
+ safebin.push("data.safe.bin");
+ assert!(safebin.exists());
+
+ // Oops, corruption.
+ fs::write(&safebin, "bogus").expect("dbfile corrupted");
+
+ // Close everything.
+ drop(env);
+ drop(shared_env);
+ manager
+ .try_close(root.path(), CloseOptions::default())
+ .expect("closed without deleting");
+ assert!(manager.get(root.path()).expect("success").is_none());
+
+ // Recreating environment fails.
+ manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .expect_err("not created");
+ assert!(manager.get(root.path()).expect("success").is_none());
+
+ // But we can use a builder and pass `discard_if_corrupted` to deal with it.
+ let mut builder = Rkv::environment_builder::<SafeMode>();
+ builder.set_discard_if_corrupted(true);
+ manager
+ .get_or_create_from_builder(root.path(), builder, Rkv::from_builder::<SafeMode>)
+ .expect("created");
+ assert!(manager.get(root.path()).expect("success").is_some());
+}
+
+/// Some storage drivers are able to recover when the database is corrupted at runtime.
+/// Test how these managers can recover corrupted databases while open.
+#[test]
+fn test_safe_mode_corrupt_while_open_2() {
+ type Manager = rkv::Manager<SafeModeEnvironment>;
+
+ let root = Builder::new()
+ .prefix("test_safe_mode_corrupt_while_open_2")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ // Create environment.
+ let mut manager = Manager::singleton().write().unwrap();
+ let shared_env = manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .expect("created");
+ let env = shared_env.read().unwrap();
+
+ // Write some data.
+ let store = env
+ .open_single("store", StoreOptions::create())
+ .expect("opened");
+ let mut writer = env.write().expect("writer");
+ store
+ .put(&mut writer, "foo", &Value::I64(1234))
+ .expect("wrote");
+ store
+ .put(&mut writer, "bar", &Value::Bool(true))
+ .expect("wrote");
+ store
+ .put(&mut writer, "baz", &Value::Str("héllo, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ env.sync(true).expect("synced");
+
+ // Verify it was flushed to disk.
+ let mut safebin = root.path().to_path_buf();
+ safebin.push("data.safe.bin");
+ assert!(safebin.exists());
+
+ // Oops, corruption.
+ fs::write(&safebin, "bogus").expect("dbfile corrupted");
+
+ // Reading still works. Magic.
+ let store = env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ reader.abort();
+
+ // Writing still works, dbfile will be un-corrupted.
+ let store = env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let mut writer = env.write().expect("writer");
+ store
+ .put(&mut writer, "foo2", &Value::I64(5678))
+ .expect("wrote");
+ store
+ .put(&mut writer, "bar2", &Value::Bool(false))
+ .expect("wrote");
+ store
+ .put(&mut writer, "baz2", &Value::Str("byé, yöu"))
+ .expect("wrote");
+ writer.commit().expect("committed");
+ env.sync(true).expect("synced");
+
+ // Close everything.
+ drop(env);
+ drop(shared_env);
+ manager
+ .try_close(root.path(), CloseOptions::default())
+ .expect("closed without deleting");
+ assert!(manager.get(root.path()).expect("success").is_none());
+
+ // Recreate environment.
+ let shared_env = manager
+ .get_or_create(root.path(), Rkv::new::<SafeMode>)
+ .expect("created");
+ let env = shared_env.read().unwrap();
+
+ // Verify that the dbfile is not corrupted.
+ let store = env
+ .open_single("store", StoreOptions::default())
+ .expect("opened");
+ let reader = env.read().expect("reader");
+ assert_eq!(
+ store.get(&reader, "foo").expect("read"),
+ Some(Value::I64(1234))
+ );
+ assert_eq!(
+ store.get(&reader, "bar").expect("read"),
+ Some(Value::Bool(true))
+ );
+ assert_eq!(
+ store.get(&reader, "baz").expect("read"),
+ Some(Value::Str("héllo, yöu"))
+ );
+ assert_eq!(
+ store.get(&reader, "foo2").expect("read"),
+ Some(Value::I64(5678))
+ );
+ assert_eq!(
+ store.get(&reader, "bar2").expect("read"),
+ Some(Value::Bool(false))
+ );
+ assert_eq!(
+ store.get(&reader, "baz2").expect("read"),
+ Some(Value::Str("byé, yöu"))
+ );
+}
diff --git a/third_party/rust/rkv/tests/multi-integer-store.rs b/third_party/rust/rkv/tests/multi-integer-store.rs
new file mode 100644
index 0000000000..bce0060288
--- /dev/null
+++ b/third_party/rust/rkv/tests/multi-integer-store.rs
@@ -0,0 +1,122 @@
+// Copyright 2018 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![cfg(all(feature = "db-dup-sort", feature = "db-int-key"))]
+#![allow(clippy::many_single_char_names)]
+
+use std::fs;
+
+use serde_derive::Serialize;
+use tempfile::Builder;
+
+use rkv::{backend::SafeMode, PrimitiveInt, Rkv, StoreOptions, Value};
+
+#[test]
+fn test_multi_integer_keys() {
+ let root = Builder::new()
+ .prefix("test_integer_keys")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let s = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ macro_rules! test_integer_keys {
+ ($store:expr, $key:expr) => {{
+ let mut writer = k.write().expect("writer");
+
+ $store
+ .put(&mut writer, $key, &Value::Str("hello1"))
+ .expect("write");
+ $store
+ .put(&mut writer, $key, &Value::Str("hello2"))
+ .expect("write");
+ $store
+ .put(&mut writer, $key, &Value::Str("hello3"))
+ .expect("write");
+ let vals = $store
+ .get(&writer, $key)
+ .expect("read")
+ .map(|result| result.expect("ok"))
+ .map(|(_, v)| v)
+ .collect::<Vec<Value>>();
+ assert_eq!(
+ vals,
+ vec![
+ Value::Str("hello1"),
+ Value::Str("hello2"),
+ Value::Str("hello3")
+ ]
+ );
+ writer.commit().expect("committed");
+
+ let reader = k.read().expect("reader");
+ let vals = $store
+ .get(&reader, $key)
+ .expect("read")
+ .map(|result| result.expect("ok"))
+ .map(|(_, v)| v)
+ .collect::<Vec<Value>>();
+ assert_eq!(
+ vals,
+ vec![
+ Value::Str("hello1"),
+ Value::Str("hello2"),
+ Value::Str("hello3")
+ ]
+ );
+ }};
+ }
+
+ // The integer module provides only the u32 integer key variant
+ // of IntegerStore, so we can use it without further ado.
+ test_integer_keys!(s, std::u32::MIN);
+ test_integer_keys!(s, std::u32::MAX);
+
+ // If you want to use another integer key variant, you need to implement
+ // a newtype, implement PrimitiveInt, and implement or derive Serialize
+ // for it. Here we do so for the i32 type.
+
+ // DANGER! Doing this enables you to open a store with multiple,
+ // different integer key types, which may result in unexpected behavior.
+ // Make sure you know what you're doing!
+
+ let t = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ #[derive(Serialize)]
+ struct I32(i32);
+ impl PrimitiveInt for I32 {}
+ test_integer_keys!(t, I32(std::i32::MIN));
+ test_integer_keys!(t, I32(std::i32::MAX));
+
+ let u = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ #[derive(Serialize)]
+ struct U16(u16);
+ impl PrimitiveInt for U16 {}
+ test_integer_keys!(u, U16(std::u16::MIN));
+ test_integer_keys!(u, U16(std::u16::MAX));
+
+ let v = k
+ .open_multi_integer("s", StoreOptions::create())
+ .expect("open");
+
+ #[derive(Serialize)]
+ struct U64(u64);
+ impl PrimitiveInt for U64 {}
+ test_integer_keys!(v, U64(std::u64::MIN));
+ test_integer_keys!(v, U64(std::u64::MAX));
+}
diff --git a/third_party/rust/rkv/tests/test_txn.rs b/third_party/rust/rkv/tests/test_txn.rs
new file mode 100644
index 0000000000..1e8fdfb416
--- /dev/null
+++ b/third_party/rust/rkv/tests/test_txn.rs
@@ -0,0 +1,131 @@
+// Copyright 2018-2019 Mozilla
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+// this file except in compliance with the License. You may obtain a copy of the
+// License at http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software distributed
+// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+// CONDITIONS OF ANY KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations under the License.
+#![cfg(feature = "db-dup-sort")]
+
+use std::fs;
+
+use tempfile::Builder;
+
+use rkv::{
+ backend::{SafeMode, SafeModeDatabase, SafeModeRoCursor, SafeModeRwTransaction},
+ Readable, Rkv, StoreOptions, Value, Writer,
+};
+
+/// Consider a struct like this:
+/// struct Sample {
+/// id: u64,
+/// value: String,
+/// date: String,
+/// }
+/// We would like to index all of the fields so that we can search for the struct not only
+/// by ID but also by value and date. When we index the fields individually in their own
+/// tables, it is important that we run all operations within a single transaction to
+/// ensure coherence of the indices.
+/// This test features helper functions for reading and writing the parts of the struct.
+/// Note that the reader functions take `Readable` because they might run within a Read
+/// Transaction or a Write Transaction. The test demonstrates fetching values via both.
+
+type SingleStore = rkv::SingleStore<SafeModeDatabase>;
+type MultiStore = rkv::MultiStore<SafeModeDatabase>;
+
+#[test]
+fn read_many() {
+ let root = Builder::new()
+ .prefix("test_txns")
+ .tempdir()
+ .expect("tempdir");
+ fs::create_dir_all(root.path()).expect("dir created");
+ let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded");
+ let samplestore = k.open_single("s", StoreOptions::create()).expect("open");
+ let datestore = k.open_multi("m", StoreOptions::create()).expect("open");
+ let valuestore = k.open_multi("m", StoreOptions::create()).expect("open");
+
+ {
+ let mut writer = k.write().expect("env write lock");
+
+ for id in 0..30_u64 {
+ let value = format!("value{id}");
+ let date = format!("2019-06-{id}");
+ put_id_field(&mut writer, datestore, &date, id);
+ put_id_field(&mut writer, valuestore, &value, id);
+ put_sample(&mut writer, samplestore, id, &value);
+ }
+
+ // now we read in the same transaction
+ for id in 0..30_u64 {
+ let value = format!("value{id}");
+ let date = format!("2019-06-{id}");
+ let ids = get_ids_by_field(&writer, datestore, &date);
+ let ids2 = get_ids_by_field(&writer, valuestore, &value);
+ let samples = get_samples(&writer, samplestore, &ids);
+ let samples2 = get_samples(&writer, samplestore, &ids2);
+ println!("{samples:?}, {samples2:?}");
+ }
+ }
+
+ {
+ let reader = k.read().expect("env read lock");
+ for id in 0..30_u64 {
+ let value = format!("value{id}");
+ let date = format!("2019-06-{id}");
+ let ids = get_ids_by_field(&reader, datestore, &date);
+ let ids2 = get_ids_by_field(&reader, valuestore, &value);
+ let samples = get_samples(&reader, samplestore, &ids);
+ let samples2 = get_samples(&reader, samplestore, &ids2);
+ println!("{samples:?}, {samples2:?}");
+ }
+ }
+}
+
+fn get_ids_by_field<'t, T>(txn: &'t T, store: MultiStore, field: &'t str) -> Vec<u64>
+where
+ T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>,
+{
+ store
+ .get(txn, field)
+ .expect("get iterator")
+ .map(|id| match id.expect("field") {
+ (_, Value::U64(id)) => id,
+ _ => panic!("getting value in iter"),
+ })
+ .collect::<Vec<u64>>()
+}
+
+fn get_samples<'t, T>(txn: &'t T, samplestore: SingleStore, ids: &[u64]) -> Vec<String>
+where
+ T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>,
+{
+ ids.iter()
+ .map(|id| {
+ let bytes = id.to_be_bytes();
+ match samplestore.get(txn, &bytes).expect("fetch sample") {
+ Some(Value::Str(sample)) => String::from(sample),
+ Some(_) => panic!("wrong type"),
+ None => panic!("no sample for this id!"),
+ }
+ })
+ .collect::<Vec<String>>()
+}
+
+fn put_sample(
+ txn: &mut Writer<SafeModeRwTransaction>,
+ samplestore: SingleStore,
+ id: u64,
+ value: &str,
+) {
+ let idbytes = id.to_be_bytes();
+ samplestore
+ .put(txn, &idbytes, &Value::Str(value))
+ .expect("put id");
+}
+
+fn put_id_field(txn: &mut Writer<SafeModeRwTransaction>, store: MultiStore, field: &str, id: u64) {
+ store.put(txn, field, &Value::U64(id)).expect("put id");
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-12 02:00:24 +0000