diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
| commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
| tree | f435a8308119effd964b339f76abb83a57c29483 /third_party/rust/rkv | |
| parent | Initial commit. (diff) | |
| download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip | |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/rkv')
62 files changed, 12831 insertions, 0 deletions
diff --git a/third_party/rust/rkv/.cargo-checksum.json b/third_party/rust/rkv/.cargo-checksum.json new file mode 100644 index 0000000000..0f345e2d6c --- /dev/null +++ b/third_party/rust/rkv/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"CODE_OF_CONDUCT.md":"902d5357af363426631d907e641e220b3ec89039164743f8442b3f120479b7cf","Cargo.lock":"708d91db1e2a9fee7463557c38c48c9d5dc1a7a36388cff99b11da0d1ab48130","Cargo.toml":"2801544a53f38d2650aa1594e9c28266031c5cfcaf50fa7d5ef44d817b342570","LICENSE":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","README.md":"ce68102d518acabb52f18dfb516d58419d9b6cabf68b4f5b3ad490b41ea2a4da","examples/README.md":"143767fc145bf167ce269a65138cb3f7086cb715b8bc4f73626da82966e646f4","examples/iterator.rs":"a8bbee46a4d2bada4b5c0bf8e9e84541900ddcd99c974ea4ab95d51c12d3d670","examples/simple-store.rs":"cec2b0b1303c72f7b2a2d16e3b2bfd292c696388e418a8b280801321bdd17aa0","run-all-examples.sh":"7f9d11d01017f77e1c9d26e3e82dfca8c6930deaec85e864458e33a7fa267de0","src/backend.rs":"f33cf8b3c7627dd562d2713997360217919d6b7333cd5a6ddab7e979b2b6b5b6","src/backend/common.rs":"47f6bf8b5efa720a1c1e5f6c874232371bc053ac83b82ee7f34a951c7bd8cbdb","src/backend/impl_lmdb.rs":"0a5a73e11881bb610f59b8c221f724bfa2bf963e3ae5356428a9d4c528b581d0","src/backend/impl_lmdb/arch_migrator.rs":"b5cf21110e6824320b0bbb0cf6556ff57c06790057e5807ad269a3992f65b7e9","src/backend/impl_lmdb/arch_migrator_error.rs":"09c8b75df05a3b9271810f1d5cb87b95ab0a38ed8622a7a856efde6fb12652be","src/backend/impl_lmdb/cursor.rs":"77a7611f8638c5358f74f79cccde18581d3f6078f72c3a8419abde144791bfdf","src/backend/impl_lmdb/database.rs":"c52ab76a4389c525a998eef0302e709d57a22f2627a967b2246e98ae15f4a999","src/backend/impl_lmdb/environment.rs":"d6521922384e0504c5c0f6cc639877350a6c32f7cf342df8bf48879fbfc59911","src/backend/impl_lmdb/error.rs":"9a8dbe3e036408398cd31d6f49dd1f0ac1ac0d14cbf4e884f1f986f3bdb2ce1b","src/backend/impl_lmdb/flags.rs":"8b54eaa3469aefe874441c5cf072c315c32cddf02896c7afd4bf3b1d36d4b5c0","src/backend/impl_lmdb/info.rs":"e44d9100c0acc179263f41c70d2d139faa1b19efe6948c951958986c5fc90dbf","src/backend/impl_lmdb/iter.rs":"792399008f0903fd18bce1a79592639706487636f3084060315667babe567769","src/backend/impl_lmdb/stat.rs":"ec3100fee914cfe4805a7c588344c707c027bad7b37b43b7455baa0772cb64f9","src/backend/impl_lmdb/transaction.rs":"4ee1b2dc57762bdf0c4fe46860a58d4804918023574424d584466dd8943af630","src/backend/impl_safe.rs":"c033006743b50fb06aff3da64acc2f31739e12beb9674533548162ce5dd8fd01","src/backend/impl_safe/cursor.rs":"570cdf7c59bae2d55716f23c13bdb4ee5c606feef2eff004eeed4c90cad377ca","src/backend/impl_safe/database.rs":"d34d159f5cc48ef944f6e55cca0cc75fd5b03ff1caee016e53f0f619b7e55d3e","src/backend/impl_safe/environment.rs":"25444b26dff736eb96b2c13a90d41733a144ccbbc41af4f960a1e07706d1aaed","src/backend/impl_safe/error.rs":"8edceffeb838cba60aa5710ffd4395d07a733e9248a0e046c5f9012dde1c7990","src/backend/impl_safe/flags.rs":"812dd1b88cbd91ef5adeaf3b2c69c57c0746b0ada192a0ca880b228018ea5be4","src/backend/impl_safe/info.rs":"c9dc67d989394dd7a25e99681e4a25f19b4ca3992eb18b719fb89742fae635b2","src/backend/impl_safe/iter.rs":"b98b54b51b474cb1e31f90136b64871baff6c31261d60bd4f79faa329768f2e8","src/backend/impl_safe/snapshot.rs":"1d815f26778b410fd8255930177f2e389b05ee99a3f407a20c56b071c871ff12","src/backend/impl_safe/stat.rs":"77ea9937c2ff839cba4ed5597b4804550915d5c50fce0fc86133bf23cff49d95","src/backend/impl_safe/transaction.rs":"73d147f6b7eaf8188f1ebd500566c9e6468d0c63cad2b87bfa3e060acbd4e3af","src/backend/traits.rs":"87ff3fc10baadb5b9ba1211df989cbe49f0e587c499329542c9e13366c528044","src/bin/dump.rs":"85ce7883462e10e956337b9daa885fb004f3fa26e3463a418e7f8cd4b3d0c1c1","src/bin/rand.rs":"8a4d2abfaa44f6fac84f38c5d6b54df01a7d200c1c1122588c273796aea20078","src/env.rs":"7070316199fcb49edbaecb6ee735bc4e73105c20b035758b76ce9b1714b141ad","src/error.rs":"6734a4de74cf57b8d5fdfe51fb064d3e5e8d378b3af380f6c6fe1b87d48a21e6","src/helpers.rs":"2e25f7f654ccc4fe39577165311334ee4c79ef3d339805716a79b1a641280fb6","src/lib.rs":"cbfc78c0ab2700abf773b49c270eef8d101f0d3f62cdf923b7e688cea743a6fd","src/manager.rs":"5bc03929d1909bea175f426e9935efaba81aefda2b89f0c0ee5c156c4f6169b6","src/migrator.rs":"38b90fa62489812545a8069f0bea510e6c4eb2923609862af28406a34eae2f2c","src/readwrite.rs":"2060edc24ac52d49f39bca90f77524850bb274273e955d6a9199f366d3ec7ee2","src/store.rs":"74900d03e1b3aab2a0215c5c76f89f71c737613ce3cc51f91f4c43ebdd560072","src/store/integer.rs":"9050293f27e5133e0046645e5c9909feaad65ea732b361ec55521b4dd2860e67","src/store/integermulti.rs":"100f61f8c9ba8003ffda5c500cc9bf1f90b2aa4940b87e9abf547801b6919a1a","src/store/keys.rs":"584bf897df7a0a10dd2841cf31cb693124748757902374d215b4d7caf79baae5","src/store/keys/encodables.rs":"d8b5dd6f49cab4a44b6f638c473ad372f589669e7ef9bd6935aa960840b95063","src/store/keys/primitives.rs":"f714e54dd6507e048cf867deecd15a5a452298b8255575aa9eb1c67317fff5dd","src/store/multi.rs":"a37fad0619c714b4c012ac62fc89404bf85a07174e9200d9a801ebb8b22ab92f","src/store/single.rs":"28f4b115114f7faa6fcf654304204e2236c24e83c83859c555a014edcbd1c617","src/value.rs":"85588982933909600d995929a323abe70bb0d2cb2850f1ad229b829e9d5234b4","tests/env-all.rs":"e0176053eeff63601b17b740a5884e8f063e1ff87069ea7e0b109f66e2eefc41","tests/env-lmdb.rs":"6d60fe2f1c19d436ce6f755cde508a75e023c0e91938ab53173ebe0645c16a58","tests/env-migration.rs":"b284429434a922f30bbaf9e4073f97a94ae77b553efda627d5c8d0bc67a92bbd","tests/env-safe.rs":"47ecf6b074e6a1097699804206cb9a1307974d045fb2494c7398110bfe036636","tests/integer-store.rs":"3b0d4ed83ea170abbe748c5e19382e33cfd9298ca65de0afe7cdf01b2f8f8aef","tests/manager.rs":"d3c945cb4a114e13580bab2e55fce155d909094abce57bbfa61dc7bd12e48751","tests/multi-integer-store.rs":"dca20148ad0942349e15cd23bf726a54bd2d49ca61af82d52709e46d9009bfb4","tests/test_txn.rs":"3ccaa25da0ed8f8788be64576d2a503193b21792c3495cbe6874a68e23096351"},"package":"2c6d906922d99c677624d2042a93f89b2b7df0f6411032237d5d99a602c2487c"}
\ No newline at end of file diff --git a/third_party/rust/rkv/CODE_OF_CONDUCT.md b/third_party/rust/rkv/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..498baa3fb0 --- /dev/null +++ b/third_party/rust/rkv/CODE_OF_CONDUCT.md @@ -0,0 +1,15 @@ +# Community Participation Guidelines + +This repository is governed by Mozilla's code of conduct and etiquette guidelines. +For more details, please read the +[Mozilla Community Participation Guidelines](https://www.mozilla.org/about/governance/policies/participation/). + +## How to Report +For more information on how to report violations of the Community Participation Guidelines, please read our '[How to Report](https://www.mozilla.org/about/governance/policies/participation/reporting/)' page. + +<!-- +## Project Specific Etiquette + +In some cases, there will be additional project etiquette i.e.: (https://bugzilla.mozilla.org/page.cgi?id=etiquette.html). +Please update for your project. +--> diff --git a/third_party/rust/rkv/Cargo.lock b/third_party/rust/rkv/Cargo.lock new file mode 100644 index 0000000000..683a1c11cd --- /dev/null +++ b/third_party/rust/rkv/Cargo.lock @@ -0,0 +1,429 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "arrayref" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b4930d2cb77ce62f89ee5d5289b4ac049559b1c45539271f5ed4fdc7db34545" + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" +dependencies = [ + "serde", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "cc" +version = "1.0.83" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "errno" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" + +[[package]] +name = "linux-raw-sys" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" + +[[package]] +name = "lmdb-rkv" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447a296f7aca299cfbb50f4e4f3d49451549af655fb7215d7f8c0c3d64bad42b" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "libc", + "lmdb-rkv-sys", +] + +[[package]] +name = "lmdb-rkv-sys" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61b9ce6b3be08acefa3003c57b7565377432a89ec24476bbe72e11d101f852fe" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + +[[package]] +name = "log" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" + +[[package]] +name = "num-traits" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ordered-float" +version = "3.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1e1c390732d15f1d48471625cd92d154e66db2c56645e29a9cd26f4699f72dc" +dependencies = [ + "num-traits", +] + +[[package]] +name = "paste" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pkg-config" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" + +[[package]] +name = "proc-macro2" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rkv" +version = "0.19.0" +dependencies = [ + "arrayref", + "bincode", + "bitflags 2.4.1", + "byteorder", + "id-arena", + "lazy_static", + "lmdb-rkv", + "log", + "ordered-float", + "paste", + "serde", + "serde_derive", + "tempfile", + "thiserror", + "url", + "uuid", +] + +[[package]] +name = "rustix" +version = "0.38.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +dependencies = [ + "bitflags 2.4.1", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "serde" +version = "1.0.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "syn" +version = "2.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +dependencies = [ + "cfg-if", + "fastrand", + "redox_syscall", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "unicode-bidi" +version = "0.3.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "uuid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/third_party/rust/rkv/Cargo.toml b/third_party/rust/rkv/Cargo.toml new file mode 100644 index 0000000000..03a068c458 --- /dev/null +++ b/third_party/rust/rkv/Cargo.toml @@ -0,0 +1,136 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +rust-version = "1.66" +name = "rkv" +version = "0.19.0" +authors = [ + "Richard Newman <rnewman@twinql.com>", + "Nan Jiang <najiang@mozilla.com>", + "Myk Melez <myk@mykzilla.org>", + "Victor Porof <vporof@mozilla.com>", +] +exclude = ["/tests/envs/*"] +description = "A simple, humane, typed key-value storage solution" +homepage = "https://github.com/mozilla/rkv" +documentation = "https://docs.rs/rkv" +readme = "README.md" +keywords = [ + "lmdb", + "database", + "storage", +] +categories = ["database"] +license = "Apache-2.0" +repository = "https://github.com/mozilla/rkv" + +[[bin]] +name = "rand" +path = "src/bin/rand.rs" +required-features = ["lmdb"] + +[[bin]] +name = "dump" +path = "src/bin/dump.rs" +required-features = ["lmdb"] + +[[test]] +name = "env-all" +required-features = ["lmdb"] + +[[test]] +name = "env-lmdb" +required-features = ["lmdb"] + +[[test]] +name = "env-migration" +required-features = ["lmdb"] + +[dependencies.arrayref] +version = "0.3" + +[dependencies.bincode] +version = "1.0" + +[dependencies.bitflags] +version = "2.4.1" +features = ["serde"] + +[dependencies.byteorder] +version = "1" + +[dependencies.id-arena] +version = "2.2" + +[dependencies.lazy_static] +version = "1.1" + +[dependencies.lmdb-rkv] +version = "0.14" +optional = true + +[dependencies.log] +version = "0.4.20" + +[dependencies.ordered-float] +version = "3.0.0" + +[dependencies.paste] +version = "1.0.6" + +[dependencies.serde] +version = "1.0.144" +features = [ + "derive", + "rc", +] + +[dependencies.serde_derive] +version = "1.0" + +[dependencies.thiserror] +version = "1.0" + +[dependencies.url] +version = "2.0" + +[dependencies.uuid] +version = "1.0" + +[dev-dependencies.byteorder] +version = "1" + +[dev-dependencies.tempfile] +version = "3" + +[features] +db-dup-sort = [] +db-int-key = [] +default = [ + "db-dup-sort", + "db-int-key", +] +lmdb = ["lmdb-rkv"] +no-canonicalize-path = [] +with-asan = [ + "lmdb", + "lmdb-rkv/with-asan", +] +with-fuzzer = [ + "lmdb", + "lmdb-rkv/with-fuzzer", +] +with-fuzzer-no-link = [ + "lmdb", + "lmdb-rkv/with-fuzzer-no-link", +] diff --git a/third_party/rust/rkv/LICENSE b/third_party/rust/rkv/LICENSE new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/third_party/rust/rkv/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/rust/rkv/README.md b/third_party/rust/rkv/README.md new file mode 100644 index 0000000000..bc50d1f0a6 --- /dev/null +++ b/third_party/rust/rkv/README.md @@ -0,0 +1,77 @@ +# rkv + +[](https://github.com/mozilla/rkv/actions/workflows/ci.yml) +[](https://docs.rs/rkv/) +[](https://crates.io/crates/rkv) + +The [rkv Rust crate](https://crates.io/crates/rkv) is a simple, humane, typed key-value storage solution. It supports multiple backend engines with varying guarantees, such as [LMDB](http://www.lmdb.tech/doc/) for performance, or "SafeMode" for reliability. + +## ⚠️ Warning ⚠️ + +To use rkv in production/release environments at Mozilla, you may do so with the "SafeMode" backend, for example: + +```rust +use rkv::{Manager, Rkv}; +use rkv::backend::{SafeMode, SafeModeEnvironment}; + +let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap(); +let shared_rkv = manager.get_or_create(path, Rkv::new::<SafeMode>).unwrap(); + +... +``` + +The "SafeMode" backend performs well, with two caveats: the entire database is stored in memory, and write transactions are synchronously written to disk (only on commit). + +In the future, it will be advisable to switch to a different backend with better performance guarantees. We're working on either fixing some LMDB crashes, or offering more choices of backend engines (e.g. SQLite). + +## Use + +Comprehensive information about using rkv is available in its [online documentation](https://docs.rs/rkv/), which can also be generated for local consumption: + +```sh +cargo doc --open +``` + +## Build + +Build this project as you would build other Rust crates: + +```sh +cargo build +``` + +### Features + +There are several features that you can opt-in and out of when using rkv: + +By default, `db-dup-sort` and `db-int-key` features offer high level database APIs which allow multiple values per key, and optimizations around integer-based keys respectively. Opt out of these default features when specifying the rkv dependency in your Cargo.toml file to disable them; doing so avoids a certain amount of overhead required to support them. + +To aid fuzzing efforts, `with-asan`, `with-fuzzer`, and `with-fuzzer-no-link` configure the build scripts responsible with compiling the underlying backing engines (e.g. LMDB) to build with these LLMV features enabled. Please refer to the official LLVM/Clang documentation on them for more informatiuon. These features are also disabled by default. + +## Test + +Test this project as you would test other Rust crates: + +```sh +cargo test +``` + +The project includes unit and doc tests embedded in the `src/` files, integration tests in the `tests/` subdirectory, and usage examples in the `examples/` subdirectory. To ensure your changes don't break examples, also run them via the run-all-examples.sh shell script: + +```sh +./run-all-examples.sh +``` + +Note: the test fixtures in the `tests/envs/` subdirectory aren't included in the package published to crates.io, so you must clone this repository in order to run the tests that depend on those fixtures or use the `rand` and `dump` executables to recreate them. + +## Contribute + +Of the various open source archetypes described in [A Framework for Purposeful Open Source](https://medium.com/mozilla-open-innovation/whats-your-open-source-strategy-here-are-10-answers-383221b3f9d3), the rkv project most closely resembles the Specialty Library, and we welcome contributions. Please report problems or ask questions using this repo's GitHub [issue tracker](https://github.com/mozilla/rkv/issues) and submit [pull requests](https://github.com/mozilla/rkv/pulls) for code and documentation changes. + +rkv relies on the latest [rustfmt](https://github.com/rust-lang-nursery/rustfmt) for code formatting, so please make sure your pull request passes the rustfmt before submitting it for review. See rustfmt's [quick start](https://github.com/rust-lang-nursery/rustfmt#quick-start) for installation details. + +We follow Mozilla's [Community Participation Guidelines](https://www.mozilla.org/en-US/about/governance/policies/participation/) while contributing to this project. + +## License + +The rkv source code is licensed under the Apache License, Version 2.0, as described in the [LICENSE](https://github.com/mozilla/rkv/blob/master/LICENSE) file. diff --git a/third_party/rust/rkv/examples/README.md b/third_party/rust/rkv/examples/README.md new file mode 100644 index 0000000000..0e5a3e6d67 --- /dev/null +++ b/third_party/rust/rkv/examples/README.md @@ -0,0 +1,11 @@ +## Examples of how to use rkv + +All examples can be executed with: + +``` +cargo run --example $name +``` + +* [`simple-store`](simple-store.rs) - a simple key/value store that showcases the basic usage of rkv. + +* [`iterator`](iterator.rs) - a demo that showcases the basic usage of iterators in rkv. diff --git a/third_party/rust/rkv/examples/iterator.rs b/third_party/rust/rkv/examples/iterator.rs new file mode 100644 index 0000000000..445dc37c28 --- /dev/null +++ b/third_party/rust/rkv/examples/iterator.rs @@ -0,0 +1,76 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ + +//! A demo that showcases the basic usage of iterators in rkv. +//! +//! You can test this out by running: +//! +//! cargo run --example iterator + +use std::{fs, str}; + +use tempfile::Builder; + +use rkv::{ + backend::{SafeMode, SafeModeDatabase, SafeModeEnvironment}, + Manager, Rkv, SingleStore, StoreError, StoreOptions, Value, +}; + +fn main() { + let root = Builder::new().prefix("iterator").tempdir().unwrap(); + fs::create_dir_all(root.path()).unwrap(); + let p = root.path(); + + let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap(); + let created_arc = manager.get_or_create(p, Rkv::new::<SafeMode>).unwrap(); + let k = created_arc.read().unwrap(); + let store = k.open_single("store", StoreOptions::create()).unwrap(); + + populate_store(&k, store).unwrap(); + + let reader = k.read().unwrap(); + + println!("Iterating from the beginning..."); + // Reader::iter_start() iterates from the first item in the store, and + // returns the (key, value) tuples in order. + let mut iter = store.iter_start(&reader).unwrap(); + while let Some(Ok((country, city))) = iter.next() { + let country = str::from_utf8(country).unwrap(); + println!("{country}, {city:?}"); + } + + println!(); + println!("Iterating from the given key..."); + // Reader::iter_from() iterates from the first key equal to or greater + // than the given key. + let mut iter = store.iter_from(&reader, "Japan").unwrap(); + while let Some(Ok((country, city))) = iter.next() { + println!("{}, {city:?}", str::from_utf8(country).unwrap()); + } + + println!(); + println!("Iterating from the given prefix..."); + let mut iter = store.iter_from(&reader, "Un").unwrap(); + while let Some(Ok((country, city))) = iter.next() { + println!("{}, {city:?}", str::from_utf8(country).unwrap()); + } +} + +fn populate_store( + k: &Rkv<SafeModeEnvironment>, + store: SingleStore<SafeModeDatabase>, +) -> Result<(), StoreError> { + let mut writer = k.write()?; + for (country, city) in vec![ + ("Canada", Value::Str("Ottawa")), + ("United States of America", Value::Str("Washington")), + ("Germany", Value::Str("Berlin")), + ("France", Value::Str("Paris")), + ("Italy", Value::Str("Rome")), + ("United Kingdom", Value::Str("London")), + ("Japan", Value::Str("Tokyo")), + ] { + store.put(&mut writer, country, &city)?; + } + writer.commit() +} diff --git a/third_party/rust/rkv/examples/simple-store.rs b/third_party/rust/rkv/examples/simple-store.rs new file mode 100644 index 0000000000..c097a2f573 --- /dev/null +++ b/third_party/rust/rkv/examples/simple-store.rs @@ -0,0 +1,252 @@ +// Any copyright is dedicated to the Public Domain. +// http://creativecommons.org/publicdomain/zero/1.0/ + +//! A simple rkv demo that showcases the basic usage (put/get/delete) of rkv. +//! +//! You can test this out by running: +//! +//! cargo run --example simple-store + +use std::fs; + +use tempfile::Builder; + +use rkv::{ + backend::{SafeMode, SafeModeDatabase, SafeModeEnvironment, SafeModeRwTransaction}, + Manager, Rkv, StoreOptions, Value, +}; + +type MultiStore = rkv::MultiStore<SafeModeDatabase>; +type Writer<'w> = rkv::Writer<SafeModeRwTransaction<'w>>; + +fn getput<'w, 's>(store: MultiStore, writer: &'w mut Writer, ids: &'s mut Vec<String>) { + let keys = vec!["str1", "str2", "str3"]; + // we convert the writer into a cursor so that we can safely read + for k in keys.iter() { + // this is a multi-valued database, so get returns an iterator + let mut iter = store.get(writer, k).unwrap(); + while let Some(Ok((_key, val))) = iter.next() { + if let Value::Str(s) = val { + ids.push(s.to_owned()); + } else { + panic!("didn't get a string back!"); + } + } + } + for id in ids { + store.put(writer, &id, &Value::Blob(b"weeeeeee")).unwrap(); + } +} + +fn delete(store: MultiStore, writer: &mut Writer) { + let keys = vec!["str1", "str2", "str3"]; + let vals = vec!["string uno", "string quatro", "string siete"]; + // we convert the writer into a cursor so that we can safely read + for i in 0..keys.len() { + store + .delete(writer, &keys[i], &Value::Str(vals[i])) + .unwrap(); + } +} + +fn main() { + let root = Builder::new().prefix("simple-db").tempdir().unwrap(); + fs::create_dir_all(root.path()).unwrap(); + let p = root.path(); + + // The manager enforces that each process opens the same lmdb environment at most once + let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap(); + let created_arc = manager.get_or_create(p, Rkv::new::<SafeMode>).unwrap(); + let k = created_arc.read().unwrap(); + + // Creates a store called "store" + let store = k.open_single("store", StoreOptions::create()).unwrap(); + let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap(); + + println!("Inserting data..."); + { + // Use a writer to mutate the store + let mut writer = k.write().unwrap(); + store.put(&mut writer, "int", &Value::I64(1234)).unwrap(); + store + .put(&mut writer, "uint", &Value::U64(1234_u64)) + .unwrap(); + store + .put(&mut writer, "float", &Value::F64(1234.0.into())) + .unwrap(); + store + .put(&mut writer, "instant", &Value::Instant(1_528_318_073_700)) + .unwrap(); + store + .put(&mut writer, "boolean", &Value::Bool(true)) + .unwrap(); + store + .put(&mut writer, "string", &Value::Str("héllo, yöu")) + .unwrap(); + store + .put( + &mut writer, + "json", + &Value::Json(r#"{"foo":"bar", "number": 1}"#), + ) + .unwrap(); + store + .put(&mut writer, "blob", &Value::Blob(b"blob")) + .unwrap(); + writer.commit().unwrap(); + } + + println!("Testing getput"); + { + let mut ids = Vec::new(); + let mut writer = k.write().unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("string uno")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("string dos")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("string tres")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("string quatro")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("string cinco")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("string seis")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("string siete")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("string ocho")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("string nueve")) + .unwrap(); + getput(multistore, &mut writer, &mut ids); + writer.commit().unwrap(); + let mut writer = k.write().unwrap(); + delete(multistore, &mut writer); + writer.commit().unwrap(); + } + + println!("Looking up keys..."); + { + // Use a reader to query the store + let reader = k.read().unwrap(); + println!("Get int {:?}", store.get(&reader, "int").unwrap()); + println!("Get uint {:?}", store.get(&reader, "uint").unwrap()); + println!("Get float {:?}", store.get(&reader, "float").unwrap()); + println!("Get instant {:?}", store.get(&reader, "instant").unwrap()); + println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap()); + println!("Get string {:?}", store.get(&reader, "string").unwrap()); + println!("Get json {:?}", store.get(&reader, "json").unwrap()); + println!("Get blob {:?}", store.get(&reader, "blob").unwrap()); + println!( + "Get non-existent {:?}", + store.get(&reader, "non-existent").unwrap() + ); + } + + println!("Looking up keys via Writer.get()..."); + { + let mut writer = k.write().unwrap(); + store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); + store.put(&mut writer, "bar", &Value::Str("baz")).unwrap(); + store.delete(&mut writer, "foo").unwrap(); + println!( + "It should be None! ({:?})", + store.get(&writer, "foo").unwrap() + ); + println!("Get bar ({:?})", store.get(&writer, "bar").unwrap()); + writer.commit().unwrap(); + let reader = k.read().expect("reader"); + println!( + "It should be None! ({:?})", + store.get(&reader, "foo").unwrap() + ); + println!("Get bar {:?}", store.get(&reader, "bar").unwrap()); + } + + println!("Aborting transaction..."); + { + // Aborting a write transaction rollbacks the change(s) + let mut writer = k.write().unwrap(); + store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); + writer.abort(); + + let reader = k.read().expect("reader"); + println!( + "It should be None! ({:?})", + store.get(&reader, "foo").unwrap() + ); + // Explicitly aborting a transaction is not required unless an early + // abort is desired, since both read and write transactions will + // implicitly be aborted once they go out of scope. + } + + println!("Deleting keys..."); + { + // Deleting a key/value also requires a write transaction + let mut writer = k.write().unwrap(); + store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); + store.delete(&mut writer, "foo").unwrap(); + println!( + "It should be None! ({:?})", + store.get(&writer, "foo").unwrap() + ); + writer.commit().unwrap(); + + // Committing a transaction consumes the writer, preventing you + // from reusing it by failing and reporting a compile-time error. + // This line would report error[E0382]: use of moved value: `writer`. + // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap(); + } + + println!("Clearing store..."); + { + // Clearing a store deletes all the entries in that store + let mut writer = k.write().unwrap(); + store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); + store.put(&mut writer, "bar", &Value::Str("baz")).unwrap(); + store.clear(&mut writer).unwrap(); + writer.commit().unwrap(); + + let reader = k.read().expect("reader"); + println!( + "It should be None! ({:?})", + store.get(&reader, "foo").unwrap() + ); + println!( + "It should be None! ({:?})", + store.get(&reader, "bar").unwrap() + ); + } + + println!("Write and read on multiple stores..."); + { + let another_store = k + .open_single("another_store", StoreOptions::create()) + .unwrap(); + let mut writer = k.write().unwrap(); + store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); + another_store + .put(&mut writer, "foo", &Value::Str("baz")) + .unwrap(); + writer.commit().unwrap(); + + let reader = k.read().unwrap(); + println!( + "Get from store value: {:?}", + store.get(&reader, "foo").unwrap() + ); + println!( + "Get from another store value: {:?}", + another_store.get(&reader, "foo").unwrap() + ); + } +} diff --git a/third_party/rust/rkv/run-all-examples.sh b/third_party/rust/rkv/run-all-examples.sh new file mode 100644 index 0000000000..fa7eb959d6 --- /dev/null +++ b/third_party/rust/rkv/run-all-examples.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +set -e + +cargo build --examples + +for file in examples/*; do + filename=$(basename ${file}) + extension=${filename##*.} + example_name=${filename%.*} + if [[ "${extension}" = "rs" ]]; then + cargo run --example ${example_name} + fi +done diff --git a/third_party/rust/rkv/src/backend.rs b/third_party/rust/rkv/src/backend.rs new file mode 100644 index 0000000000..63fa08a40c --- /dev/null +++ b/third_party/rust/rkv/src/backend.rs @@ -0,0 +1,40 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +mod common; +#[cfg(feature = "lmdb")] +mod impl_lmdb; +mod impl_safe; +mod traits; + +pub use common::*; +pub use traits::*; + +#[cfg(feature = "lmdb")] +pub use impl_lmdb::{ + ArchMigrateError as LmdbArchMigrateError, ArchMigrateResult as LmdbArchMigrateResult, + ArchMigrator as LmdbArchMigrator, DatabaseFlagsImpl as LmdbDatabaseFlags, + DatabaseImpl as LmdbDatabase, EnvironmentBuilderImpl as Lmdb, + EnvironmentFlagsImpl as LmdbEnvironmentFlags, EnvironmentImpl as LmdbEnvironment, + ErrorImpl as LmdbError, InfoImpl as LmdbInfo, IterImpl as LmdbIter, + RoCursorImpl as LmdbRoCursor, RoTransactionImpl as LmdbRoTransaction, + RwCursorImpl as LmdbRwCursor, RwTransactionImpl as LmdbRwTransaction, StatImpl as LmdbStat, + WriteFlagsImpl as LmdbWriteFlags, +}; + +pub use impl_safe::{ + DatabaseFlagsImpl as SafeModeDatabaseFlags, DatabaseImpl as SafeModeDatabase, + EnvironmentBuilderImpl as SafeMode, EnvironmentFlagsImpl as SafeModeEnvironmentFlags, + EnvironmentImpl as SafeModeEnvironment, ErrorImpl as SafeModeError, InfoImpl as SafeModeInfo, + IterImpl as SafeModeIter, RoCursorImpl as SafeModeRoCursor, + RoTransactionImpl as SafeModeRoTransaction, RwCursorImpl as SafeModeRwCursor, + RwTransactionImpl as SafeModeRwTransaction, StatImpl as SafeModeStat, + WriteFlagsImpl as SafeModeWriteFlags, +}; diff --git a/third_party/rust/rkv/src/backend/common.rs b/third_party/rust/rkv/src/backend/common.rs new file mode 100644 index 0000000000..89b091b289 --- /dev/null +++ b/third_party/rust/rkv/src/backend/common.rs @@ -0,0 +1,57 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![allow(non_camel_case_types)] + +pub enum EnvironmentFlags { + FIXED_MAP, + NO_SUB_DIR, + WRITE_MAP, + READ_ONLY, + NO_META_SYNC, + NO_SYNC, + MAP_ASYNC, + NO_TLS, + NO_LOCK, + NO_READAHEAD, + NO_MEM_INIT, +} + +pub enum DatabaseFlags { + REVERSE_KEY, + #[cfg(feature = "db-dup-sort")] + DUP_SORT, + #[cfg(feature = "db-dup-sort")] + DUP_FIXED, + #[cfg(feature = "db-int-key")] + INTEGER_KEY, + INTEGER_DUP, + REVERSE_DUP, +} + +pub enum WriteFlags { + NO_OVERWRITE, + NO_DUP_DATA, + CURRENT, + APPEND, + APPEND_DUP, +} + +/// Strategy to use when corrupted data is detected while opening a database. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RecoveryStrategy { + /// Bubble up the error on detecting a corrupted data file. The default. + Error, + + /// Discard the corrupted data and start with an empty database. + Discard, + + /// Move the corrupted data file to `$file.corrupt` and start with an empty database. + Rename, +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb.rs b/third_party/rust/rkv/src/backend/impl_lmdb.rs new file mode 100644 index 0000000000..5364214598 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb.rs @@ -0,0 +1,34 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +mod arch_migrator; +mod arch_migrator_error; +mod cursor; +mod database; +mod environment; +mod error; +mod flags; +mod info; +mod iter; +mod stat; +mod transaction; + +pub use arch_migrator::{ + MigrateError as ArchMigrateError, MigrateResult as ArchMigrateResult, Migrator as ArchMigrator, +}; +pub use cursor::{RoCursorImpl, RwCursorImpl}; +pub use database::DatabaseImpl; +pub use environment::{EnvironmentBuilderImpl, EnvironmentImpl}; +pub use error::ErrorImpl; +pub use flags::{DatabaseFlagsImpl, EnvironmentFlagsImpl, WriteFlagsImpl}; +pub use info::InfoImpl; +pub use iter::IterImpl; +pub use stat::StatImpl; +pub use transaction::{RoTransactionImpl, RwTransactionImpl}; diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs new file mode 100644 index 0000000000..78142609ac --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator.rs @@ -0,0 +1,998 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![allow(dead_code)] // TODO: Get rid of unused struct members +#![allow(clippy::upper_case_acronyms)] // TODO: Consider renaming things like `BRANCH` + +//! A utility for migrating data from one LMDB environment to another. Notably, this tool +//! can migrate data from an enviroment created with a different bit-depth than the +//! current rkv consumer, which enables the consumer to retrieve data from an environment +//! that can't be read directly using the rkv APIs. +//! +//! The utility supports both 32-bit and 64-bit LMDB source environments, and it +//! automatically migrates data in both the default database and any named (sub) +//! databases. It also migrates the source environment's "map size" and "max DBs" +//! configuration options to the destination environment. +//! +//! The destination environment must be at the rkv consumer's bit depth and should be +//! empty of data. It can be an empty directory, in which case the utility will create a +//! new LMDB environment within the directory. +//! +//! The tool currently has these limitations: +//! +//! 1. It doesn't support migration from environments created with +//! `EnvironmentFlags::NO_SUB_DIR`. To migrate such an environment, create a +//! temporary directory, copy the environment's data file to a file called data.mdb in +//! the temporary directory, then migrate the temporary directory as the source +//! environment. +//! 2. It doesn't support migration from databases created with DatabaseFlags::DUP_SORT` +//! (with or without `DatabaseFlags::DUP_FIXED`). +//! 3. It doesn't account for existing data in the destination environment, which means +//! that it can overwrite data (causing data loss) or fail to migrate data if the +//! destination environment contains existing data. +//! +//! ## Basic Usage +//! +//! Call `Migrator::new()` with the path to the source environment to create a `Migrator` +//! instance; then call the instance's `migrate()` method with the path to the destination +//! environment to migrate data from the source to the destination environment. For +//! example, this snippet migrates data from the tests/envs/ref_env_32 environment to a +//! new environment in a temporary directory: +//! +//! ``` +//! use rkv::migrator::LmdbArchMigrator as Migrator; +//! use std::path::Path; +//! use tempfile::tempdir; +//! let mut migrator = Migrator::new(Path::new("tests/envs/ref_env_32")).unwrap(); +//! migrator.migrate(&tempdir().unwrap().path()).unwrap(); +//! ``` +//! +//! Both `Migrator::new()` and `migrate()` return a `MigrateResult` that is either an +//! `Ok()` result or an `Err<MigrateError>`, where `MigrateError` is an enum whose +//! variants identify specific kinds of migration failures. + +use std::{ + collections::{BTreeMap, HashMap}, + convert::TryFrom, + fs::File, + io::{Cursor, Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, + rc::Rc, + str, +}; + +use bitflags::bitflags; +use byteorder::{LittleEndian, ReadBytesExt}; +use lmdb::{DatabaseFlags, Environment, Transaction, WriteFlags}; + +pub use super::arch_migrator_error::MigrateError; + +const PAGESIZE: u16 = 4096; + +// The magic number is 0xBEEFC0DE, which is 0xDEC0EFBE in little-endian. It appears at +// offset 12 on 32-bit systems and 16 on 64-bit systems. We don't support big-endian +// migration, but presumably we could do so by detecting the order of the bytes. +const MAGIC: [u8; 4] = [0xDE, 0xC0, 0xEF, 0xBE]; + +pub type MigrateResult<T> = Result<T, MigrateError>; + +bitflags! { + #[derive(Default, PartialEq, Eq, Debug, Clone, Copy)] + struct PageFlags: u16 { + const BRANCH = 0x01; + const LEAF = 0x02; + const OVERFLOW = 0x04; + const META = 0x08; + const DIRTY = 0x10; + const LEAF2 = 0x20; + const SUBP = 0x40; + const LOOSE = 0x4000; + const KEEP = 0x8000; + } +} + +bitflags! { + #[derive(Default, PartialEq, Eq, Debug, Clone, Copy)] + struct NodeFlags: u16 { + const BIGDATA = 0x01; + const SUBDATA = 0x02; + const DUPDATA = 0x04; + } +} + +// The bit depth of the executable that created an LMDB environment. The Migrator +// determines this automatically based on the location of the magic number in data.mdb. +#[derive(Clone, Copy, PartialEq)] +enum Bits { + U32, + U64, +} + +impl Bits { + // The size of usize for the bit-depth represented by the enum variant. + fn size(self) -> usize { + match self { + Bits::U32 => 4, + Bits::U64 => 8, + } + } +} + +// The equivalent of PAGEHDRSZ in LMDB, except that this one varies by bits. +fn page_header_size(bits: Bits) -> u64 { + match bits { + Bits::U32 => 12, + Bits::U64 => 16, + } +} + +// The equivalent of P_INVALID in LMDB, except that this one varies by bits. +fn validate_page_num(page_num: u64, bits: Bits) -> MigrateResult<()> { + let invalid_page_num = match bits { + Bits::U32 => u64::from(!0u32), + Bits::U64 => !0u64, + }; + + if page_num == invalid_page_num { + return Err(MigrateError::InvalidPageNum); + } + + Ok(()) +} + +#[derive(Clone, Debug, Default)] +struct Database { + md_pad: u32, + md_flags: DatabaseFlags, + md_depth: u16, + md_branch_pages: u64, + md_leaf_pages: u64, + md_overflow_pages: u64, + md_entries: u64, + md_root: u64, +} + +impl Database { + fn new(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<Database> { + Ok(Database { + md_pad: cursor.read_u32::<LittleEndian>()?, + md_flags: DatabaseFlags::from_bits(cursor.read_u16::<LittleEndian>()?.into()) + .ok_or(MigrateError::InvalidDatabaseBits)?, + md_depth: cursor.read_u16::<LittleEndian>()?, + md_branch_pages: cursor.read_uint::<LittleEndian>(bits.size())?, + md_leaf_pages: cursor.read_uint::<LittleEndian>(bits.size())?, + md_overflow_pages: cursor.read_uint::<LittleEndian>(bits.size())?, + md_entries: cursor.read_uint::<LittleEndian>(bits.size())?, + md_root: cursor.read_uint::<LittleEndian>(bits.size())?, + }) + } +} + +#[derive(Debug, Default)] +struct Databases { + free: Database, + main: Database, +} + +#[derive(Debug, Default)] +struct MetaData { + mm_magic: u32, + mm_version: u32, + mm_address: u64, + mm_mapsize: u64, + mm_dbs: Databases, + mm_last_pg: u64, + mm_txnid: u64, +} + +#[derive(Debug)] +enum LeafNode { + Regular { + mn_lo: u16, + mn_hi: u16, + mn_flags: NodeFlags, + mn_ksize: u16, + mv_size: u32, + key: Vec<u8>, + value: Vec<u8>, + }, + BigData { + mn_lo: u16, + mn_hi: u16, + mn_flags: NodeFlags, + mn_ksize: u16, + mv_size: u32, + key: Vec<u8>, + overflow_pgno: u64, + }, + SubData { + mn_lo: u16, + mn_hi: u16, + mn_flags: NodeFlags, + mn_ksize: u16, + mv_size: u32, + key: Vec<u8>, + value: Vec<u8>, + db: Database, + }, +} + +#[derive(Debug, Default)] +struct BranchNode { + mp_pgno: u64, + mn_ksize: u16, + mn_data: Vec<u8>, +} + +#[derive(Debug)] +enum PageHeader { + Regular { + mp_pgno: u64, + mp_flags: PageFlags, + pb_lower: u16, + pb_upper: u16, + }, + Overflow { + mp_pgno: u64, + mp_flags: PageFlags, + pb_pages: u32, + }, +} + +#[derive(Debug)] +enum Page { + META(MetaData), + LEAF(Vec<LeafNode>), + BRANCH(Vec<BranchNode>), +} + +impl Page { + fn new(buf: Vec<u8>, bits: Bits) -> MigrateResult<Page> { + let mut cursor = std::io::Cursor::new(&buf[..]); + + match Self::parse_page_header(&mut cursor, bits)? { + PageHeader::Regular { + mp_flags, pb_lower, .. + } => { + if mp_flags.contains(PageFlags::LEAF2) || mp_flags.contains(PageFlags::SUBP) { + // We don't yet support DUPFIXED and DUPSORT databases. + return Err(MigrateError::UnsupportedPageHeaderVariant); + } + + if mp_flags.contains(PageFlags::META) { + let meta_data = Self::parse_meta_data(&mut cursor, bits)?; + Ok(Page::META(meta_data)) + } else if mp_flags.contains(PageFlags::LEAF) { + let nodes = Self::parse_leaf_nodes(&mut cursor, pb_lower, bits)?; + Ok(Page::LEAF(nodes)) + } else if mp_flags.contains(PageFlags::BRANCH) { + let nodes = Self::parse_branch_nodes(&mut cursor, pb_lower, bits)?; + Ok(Page::BRANCH(nodes)) + } else { + Err(MigrateError::UnexpectedPageHeaderVariant) + } + } + PageHeader::Overflow { .. } => { + // There isn't anything to do, nor should we try to instantiate + // a page of this type, as we only access them when reading + // a value that is too large to fit into a leaf node. + Err(MigrateError::UnexpectedPageHeaderVariant) + } + } + } + + fn parse_page_header(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<PageHeader> { + let mp_pgno = cursor.read_uint::<LittleEndian>(bits.size())?; + let _mp_pad = cursor.read_u16::<LittleEndian>()?; + let mp_flags = PageFlags::from_bits(cursor.read_u16::<LittleEndian>()?) + .ok_or(MigrateError::InvalidPageBits)?; + + if mp_flags.contains(PageFlags::OVERFLOW) { + let pb_pages = cursor.read_u32::<LittleEndian>()?; + Ok(PageHeader::Overflow { + mp_pgno, + mp_flags, + pb_pages, + }) + } else { + let pb_lower = cursor.read_u16::<LittleEndian>()?; + let pb_upper = cursor.read_u16::<LittleEndian>()?; + Ok(PageHeader::Regular { + mp_pgno, + mp_flags, + pb_lower, + pb_upper, + }) + } + } + + fn parse_meta_data(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<MetaData> { + cursor.seek(SeekFrom::Start(page_header_size(bits)))?; + + Ok(MetaData { + mm_magic: cursor.read_u32::<LittleEndian>()?, + mm_version: cursor.read_u32::<LittleEndian>()?, + mm_address: cursor.read_uint::<LittleEndian>(bits.size())?, + mm_mapsize: cursor.read_uint::<LittleEndian>(bits.size())?, + mm_dbs: Databases { + free: Database::new(cursor, bits)?, + main: Database::new(cursor, bits)?, + }, + mm_last_pg: cursor.read_uint::<LittleEndian>(bits.size())?, + mm_txnid: cursor.read_uint::<LittleEndian>(bits.size())?, + }) + } + + fn parse_leaf_nodes( + cursor: &mut Cursor<&[u8]>, + pb_lower: u16, + bits: Bits, + ) -> MigrateResult<Vec<LeafNode>> { + cursor.set_position(page_header_size(bits)); + let num_keys = Self::num_keys(pb_lower, bits); + let mp_ptrs = Self::parse_mp_ptrs(cursor, num_keys)?; + + let mut leaf_nodes = Vec::with_capacity(num_keys as usize); + + for mp_ptr in mp_ptrs { + cursor.set_position(u64::from(mp_ptr)); + leaf_nodes.push(Self::parse_leaf_node(cursor, bits)?); + } + + Ok(leaf_nodes) + } + + fn parse_leaf_node(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<LeafNode> { + // The order of the mn_lo and mn_hi fields is endian-dependent and would be + // reversed in an LMDB environment created on a big-endian system. + let mn_lo = cursor.read_u16::<LittleEndian>()?; + let mn_hi = cursor.read_u16::<LittleEndian>()?; + + let mn_flags = NodeFlags::from_bits(cursor.read_u16::<LittleEndian>()?) + .ok_or(MigrateError::InvalidNodeBits)?; + let mn_ksize = cursor.read_u16::<LittleEndian>()?; + + let start = usize::try_from(cursor.position())?; + let end = usize::try_from(cursor.position() + u64::from(mn_ksize))?; + let key = cursor.get_ref()[start..end].to_vec(); + cursor.set_position(end as u64); + + let mv_size = Self::leaf_node_size(mn_lo, mn_hi); + if mn_flags.contains(NodeFlags::BIGDATA) { + let overflow_pgno = cursor.read_uint::<LittleEndian>(bits.size())?; + Ok(LeafNode::BigData { + mn_lo, + mn_hi, + mn_flags, + mn_ksize, + mv_size, + key, + overflow_pgno, + }) + } else if mn_flags.contains(NodeFlags::SUBDATA) { + let start = usize::try_from(cursor.position())?; + let end = usize::try_from(cursor.position() + u64::from(mv_size))?; + let value = cursor.get_ref()[start..end].to_vec(); + let mut cursor = std::io::Cursor::new(&value[..]); + let db = Database::new(&mut cursor, bits)?; + validate_page_num(db.md_root, bits)?; + Ok(LeafNode::SubData { + mn_lo, + mn_hi, + mn_flags, + mn_ksize, + mv_size, + key, + value, + db, + }) + } else { + let start = usize::try_from(cursor.position())?; + let end = usize::try_from(cursor.position() + u64::from(mv_size))?; + let value = cursor.get_ref()[start..end].to_vec(); + Ok(LeafNode::Regular { + mn_lo, + mn_hi, + mn_flags, + mn_ksize, + mv_size, + key, + value, + }) + } + } + + fn leaf_node_size(mn_lo: u16, mn_hi: u16) -> u32 { + u32::from(mn_lo) + ((u32::from(mn_hi)) << 16) + } + + fn parse_branch_nodes( + cursor: &mut Cursor<&[u8]>, + pb_lower: u16, + bits: Bits, + ) -> MigrateResult<Vec<BranchNode>> { + let num_keys = Self::num_keys(pb_lower, bits); + let mp_ptrs = Self::parse_mp_ptrs(cursor, num_keys)?; + + let mut branch_nodes = Vec::with_capacity(num_keys as usize); + + for mp_ptr in mp_ptrs { + cursor.set_position(u64::from(mp_ptr)); + branch_nodes.push(Self::parse_branch_node(cursor, bits)?) + } + + Ok(branch_nodes) + } + + fn parse_branch_node(cursor: &mut Cursor<&[u8]>, bits: Bits) -> MigrateResult<BranchNode> { + // The order of the mn_lo and mn_hi fields is endian-dependent and would be + // reversed in an LMDB environment created on a big-endian system. + let mn_lo = cursor.read_u16::<LittleEndian>()?; + let mn_hi = cursor.read_u16::<LittleEndian>()?; + + let mn_flags = cursor.read_u16::<LittleEndian>()?; + + // Branch nodes overload the mn_lo, mn_hi, and mn_flags fields to store the page + // number, so we derive the number from those fields. + let mp_pgno = Self::branch_node_page_num(mn_lo, mn_hi, mn_flags, bits); + + let mn_ksize = cursor.read_u16::<LittleEndian>()?; + + let position = cursor.position(); + let start = usize::try_from(position)?; + let end = usize::try_from(position + u64::from(mn_ksize))?; + let mn_data = cursor.get_ref()[start..end].to_vec(); + cursor.set_position(end as u64); + + Ok(BranchNode { + mp_pgno, + mn_ksize, + mn_data, + }) + } + + fn branch_node_page_num(mn_lo: u16, mn_hi: u16, mn_flags: u16, bits: Bits) -> u64 { + let mut page_num = u64::from(u32::from(mn_lo) + (u32::from(mn_hi) << 16)); + if bits == Bits::U64 { + page_num += u64::from(mn_flags) << 32; + } + page_num + } + + fn parse_mp_ptrs(cursor: &mut Cursor<&[u8]>, num_keys: u64) -> MigrateResult<Vec<u16>> { + let mut mp_ptrs = Vec::with_capacity(num_keys as usize); + for _ in 0..num_keys { + mp_ptrs.push(cursor.read_u16::<LittleEndian>()?); + } + Ok(mp_ptrs) + } + + fn num_keys(pb_lower: u16, bits: Bits) -> u64 { + (u64::from(pb_lower) - page_header_size(bits)) >> 1 + } +} + +pub struct Migrator { + file: File, + bits: Bits, +} + +impl Migrator { + /// Create a new Migrator for the LMDB environment at the given path. This tries to + /// open the data.mdb file in the environment and determine the bit depth of the + /// executable that created it, so it can fail and return an Err if the file can't be + /// opened or the depth determined. + pub fn new(path: &Path) -> MigrateResult<Migrator> { + let mut path = PathBuf::from(path); + path.push("data.mdb"); + let mut file = File::open(&path)?; + + file.seek(SeekFrom::Start(page_header_size(Bits::U32)))?; + let mut buf = [0; 4]; + file.read_exact(&mut buf)?; + + let bits = if buf == MAGIC { + Bits::U32 + } else { + file.seek(SeekFrom::Start(page_header_size(Bits::U64)))?; + file.read_exact(&mut buf)?; + if buf == MAGIC { + Bits::U64 + } else { + return Err(MigrateError::IndeterminateBitDepth); + } + }; + + Ok(Migrator { file, bits }) + } + + /// Dump the data in one of the databases in the LMDB environment. If the `database` + /// paremeter is None, then we dump the data in the main database. If it's the name + /// of a subdatabase, then we dump the data in that subdatabase. + /// + /// Note that the output isn't identical to that of the `mdb_dump` utility, since + /// `mdb_dump` includes subdatabase key/value pairs when dumping the main database, + /// and those values are architecture-dependent, since they contain pointer-sized + /// data. + /// + /// If we wanted to support identical output, we could parameterize inclusion of + /// subdatabase pairs in get_pairs() and include them when dumping data, while + /// continuing to exclude them when migrating data. + pub fn dump<T: Write>(&mut self, database: Option<&str>, mut out: T) -> MigrateResult<()> { + let meta_data = self.get_meta_data()?; + let root_page_num = meta_data.mm_dbs.main.md_root; + let root_page = Rc::new(self.get_page(root_page_num)?); + + let pairs; + if let Some(database) = database { + let subdbs = self.get_subdbs(root_page)?; + let database = subdbs + .get(database.as_bytes()) + .ok_or_else(|| MigrateError::DatabaseNotFound(database.to_string()))?; + let root_page_num = database.md_root; + let root_page = Rc::new(self.get_page(root_page_num)?); + pairs = self.get_pairs(root_page)?; + } else { + pairs = self.get_pairs(root_page)?; + } + + out.write_all(b"VERSION=3\n")?; + out.write_all(b"format=bytevalue\n")?; + if let Some(database) = database { + writeln!(out, "database={database}")?; + } + out.write_all(b"type=btree\n")?; + writeln!(out, "mapsize={}", meta_data.mm_mapsize)?; + out.write_all(b"maxreaders=126\n")?; + out.write_all(b"db_pagesize=4096\n")?; + out.write_all(b"HEADER=END\n")?; + + for (key, value) in pairs { + out.write_all(b" ")?; + for byte in key { + write!(out, "{byte:02x}")?; + } + out.write_all(b"\n")?; + out.write_all(b" ")?; + for byte in value { + write!(out, "{byte:02x}")?; + } + out.write_all(b"\n")?; + } + + out.write_all(b"DATA=END\n")?; + + Ok(()) + } + + /// Migrate all data in all of databases in the existing LMDB environment to a new + /// environment. This includes all key/value pairs in the main database that aren't + /// metadata about subdatabases and all key/value pairs in all subdatabases. + /// + /// We also set the map size and maximum databases of the new environment to their + /// values for the existing environment. But we don't set other metadata, and we + /// don't check that the new environment is empty before migrating data. + /// + /// Thus it's possible for this to overwrite existing data or fail to migrate data if + /// the new environment isn't empty. It's the consumer's responsibility to ensure + /// that data can be safely migrated to the new environment. In general, this means + /// that environment should be empty. + pub fn migrate(&mut self, dest: &Path) -> MigrateResult<()> { + let meta_data = self.get_meta_data()?; + let root_page_num = meta_data.mm_dbs.main.md_root; + validate_page_num(root_page_num, self.bits)?; + let root_page = Rc::new(self.get_page(root_page_num)?); + let subdbs = self.get_subdbs(Rc::clone(&root_page))?; + + let env = Environment::new() + .set_map_size(meta_data.mm_mapsize as usize) + .set_max_dbs(subdbs.len() as u32) + .open(dest)?; + + // Create the databases before we open a read-write transaction, since database + // creation requires its own read-write transaction, which would hang while + // awaiting completion of an existing one. + env.create_db(None, meta_data.mm_dbs.main.md_flags)?; + for (subdb_name, subdb_info) in &subdbs { + env.create_db(Some(str::from_utf8(subdb_name)?), subdb_info.md_flags)?; + } + + // Now open the read-write transaction that we'll use to migrate all the data. + let mut txn = env.begin_rw_txn()?; + + // Migrate the main database. + let pairs = self.get_pairs(root_page)?; + let db = env.open_db(None)?; + for (key, value) in pairs { + // If we knew that the target database was empty, we could specify + // WriteFlags::APPEND to speed up the migration. + txn.put(db, &key, &value, WriteFlags::empty())?; + } + + // Migrate subdatabases. + for (subdb_name, subdb_info) in &subdbs { + let root_page = Rc::new(self.get_page(subdb_info.md_root)?); + let pairs = self.get_pairs(root_page)?; + let db = env.open_db(Some(str::from_utf8(subdb_name)?))?; + for (key, value) in pairs { + // If we knew that the target database was empty, we could specify + // WriteFlags::APPEND to speed up the migration. + txn.put(db, &key, &value, WriteFlags::empty())?; + } + } + + txn.commit()?; + + Ok(()) + } + + fn get_subdbs(&mut self, root_page: Rc<Page>) -> MigrateResult<HashMap<Vec<u8>, Database>> { + let mut subdbs = HashMap::new(); + let mut pages = vec![root_page]; + + while let Some(page) = pages.pop() { + match &*page { + Page::BRANCH(nodes) => { + for branch in nodes { + pages.push(Rc::new(self.get_page(branch.mp_pgno)?)); + } + } + Page::LEAF(nodes) => { + for leaf in nodes { + if let LeafNode::SubData { key, db, .. } = leaf { + subdbs.insert(key.to_vec(), db.clone()); + }; + } + } + _ => { + return Err(MigrateError::UnexpectedPageVariant); + } + } + } + + Ok(subdbs) + } + + fn get_pairs(&mut self, root_page: Rc<Page>) -> MigrateResult<BTreeMap<Vec<u8>, Vec<u8>>> { + let mut pairs = BTreeMap::new(); + let mut pages = vec![root_page]; + + while let Some(page) = pages.pop() { + match &*page { + Page::BRANCH(nodes) => { + for branch in nodes { + pages.push(Rc::new(self.get_page(branch.mp_pgno)?)); + } + } + Page::LEAF(nodes) => { + for leaf in nodes { + match leaf { + LeafNode::Regular { key, value, .. } => { + pairs.insert(key.to_vec(), value.to_vec()); + } + LeafNode::BigData { + mv_size, + key, + overflow_pgno, + .. + } => { + // Perhaps we could reduce memory consumption during a + // migration by waiting to read big data until it's time + // to write it to the new database. + let value = self.read_data( + *overflow_pgno * u64::from(PAGESIZE) + + page_header_size(self.bits), + *mv_size as usize, + )?; + pairs.insert(key.to_vec(), value); + } + LeafNode::SubData { .. } => { + // We don't include subdatabase leaves in pairs, since + // there's no architecture-neutral representation of them, + // and in any case they're meta-data that should get + // recreated when we migrate the subdatabases themselves. + // + // If we wanted to create identical dumps to those + // produced by `mdb_dump`, however, we could allow + // consumers to specify that they'd like to include these + // records. + } + }; + } + } + _ => { + return Err(MigrateError::UnexpectedPageVariant); + } + } + } + + Ok(pairs) + } + + fn read_data(&mut self, offset: u64, size: usize) -> MigrateResult<Vec<u8>> { + self.file.seek(SeekFrom::Start(offset))?; + let mut buf: Vec<u8> = vec![0; size]; + self.file.read_exact(&mut buf[0..size])?; + Ok(buf.to_vec()) + } + + fn get_page(&mut self, page_no: u64) -> MigrateResult<Page> { + Page::new( + self.read_data(page_no * u64::from(PAGESIZE), usize::from(PAGESIZE))?, + self.bits, + ) + } + + fn get_meta_data(&mut self) -> MigrateResult<MetaData> { + let (page0, page1) = (self.get_page(0)?, self.get_page(1)?); + + match (page0, page1) { + (Page::META(meta0), Page::META(meta1)) => { + let meta = if meta1.mm_txnid > meta0.mm_txnid { + meta1 + } else { + meta0 + }; + if meta.mm_magic != 0xBE_EF_C0_DE { + return Err(MigrateError::InvalidMagicNum); + } + if meta.mm_version != 1 && meta.mm_version != 999 { + return Err(MigrateError::InvalidDataVersion); + } + Ok(meta) + } + _ => Err(MigrateError::UnexpectedPageVariant), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use std::{env, fs, mem::size_of}; + + use lmdb::{Environment, Error as LmdbError}; + use tempfile::{tempdir, tempfile}; + + fn compare_files(ref_file: &mut File, new_file: &mut File) -> MigrateResult<()> { + ref_file.seek(SeekFrom::Start(0))?; + new_file.seek(SeekFrom::Start(0))?; + + let ref_buf = &mut [0; 1024]; + let new_buf = &mut [0; 1024]; + + loop { + match ref_file.read(ref_buf) { + Err(err) => panic!("{}", err), + Ok(ref_len) => match new_file.read(new_buf) { + Err(err) => panic!("{}", err), + Ok(new_len) => { + assert_eq!(ref_len, new_len); + if ref_len == 0 { + break; + }; + assert_eq!(ref_buf[0..ref_len], new_buf[0..new_len]); + } + }, + } + } + + Ok(()) + } + + #[test] + fn test_dump_32() -> MigrateResult<()> { + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_32"].iter().collect(); + + // Dump data from the test env to a new dump file. + let mut migrator = Migrator::new(&test_env_path)?; + let mut new_dump_file = tempfile()?; + migrator.dump(None, &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump.txt"].iter().collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + Ok(()) + } + + #[test] + fn test_dump_32_subdb() -> MigrateResult<()> { + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_32"].iter().collect(); + + // Dump data from the test env to a new dump file. + let mut migrator = Migrator::new(&test_env_path)?; + let mut new_dump_file = tempfile()?; + migrator.dump(Some("subdb"), &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"] + .iter() + .collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + Ok(()) + } + + #[test] + fn test_dump_64() -> MigrateResult<()> { + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_64"].iter().collect(); + + // Dump data from the test env to a new dump file. + let mut migrator = Migrator::new(&test_env_path)?; + let mut new_dump_file = tempfile()?; + migrator.dump(None, &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump.txt"].iter().collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + Ok(()) + } + + #[test] + fn test_dump_64_subdb() -> MigrateResult<()> { + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_64"].iter().collect(); + + // Dump data from the test env to a new dump file. + let mut migrator = Migrator::new(&test_env_path)?; + let mut new_dump_file = tempfile()?; + migrator.dump(Some("subdb"), &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"] + .iter() + .collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + Ok(()) + } + + #[test] + fn test_migrate_64() -> MigrateResult<()> { + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_64"].iter().collect(); + + // Migrate data from the old env to a new one. + let new_env = tempdir()?; + let mut migrator = Migrator::new(&test_env_path)?; + migrator.migrate(new_env.path())?; + + // Dump data from the new env to a new dump file. + let mut migrator = Migrator::new(new_env.path())?; + let mut new_dump_file = tempfile()?; + migrator.dump(Some("subdb"), &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"] + .iter() + .collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + Ok(()) + } + + #[test] + fn test_migrate_32() -> MigrateResult<()> { + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", "ref_env_32"].iter().collect(); + + // Migrate data from the old env to a new one. + let new_env = tempdir()?; + let mut migrator = Migrator::new(&test_env_path)?; + migrator.migrate(new_env.path())?; + + // Dump data from the new env to a new dump file. + let mut migrator = Migrator::new(new_env.path())?; + let mut new_dump_file = tempfile()?; + migrator.dump(Some("subdb"), &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"] + .iter() + .collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + Ok(()) + } + + #[test] + fn test_migrate_and_replace() -> MigrateResult<()> { + let test_env_name = match size_of::<usize>() { + 4 => "ref_env_64", + 8 => "ref_env_32", + _ => panic!("only 32- and 64-bit depths are supported"), + }; + + let cwd = env::current_dir()?; + let cwd = cwd.to_str().ok_or(MigrateError::StringConversionError)?; + let test_env_path: PathBuf = [cwd, "tests", "envs", test_env_name].iter().collect(); + + let old_env = tempdir()?; + fs::copy( + test_env_path.join("data.mdb"), + old_env.path().join("data.mdb"), + )?; + fs::copy( + test_env_path.join("lock.mdb"), + old_env.path().join("lock.mdb"), + )?; + + // Confirm that it isn't possible to open the old environment with LMDB. + assert_eq!( + match Environment::new().open(old_env.path()) { + Err(err) => err, + _ => panic!("opening the environment should have failed"), + }, + LmdbError::Invalid + ); + + // Migrate data from the old env to a new one. + let new_env = tempdir()?; + let mut migrator = Migrator::new(old_env.path())?; + migrator.migrate(new_env.path())?; + + // Dump data from the new env to a new dump file. + let mut migrator = Migrator::new(new_env.path())?; + let mut new_dump_file = tempfile()?; + migrator.dump(Some("subdb"), &new_dump_file)?; + + // Open the reference dump file. + let ref_dump_file_path: PathBuf = [cwd, "tests", "envs", "ref_dump_subdb.txt"] + .iter() + .collect(); + let mut ref_dump_file = File::open(ref_dump_file_path)?; + + // Compare the new dump file to the reference dump file. + compare_files(&mut ref_dump_file, &mut new_dump_file)?; + + // Overwrite the old env's files with the new env's files and confirm that it's now + // possible to open the old env with LMDB. + fs::copy( + new_env.path().join("data.mdb"), + old_env.path().join("data.mdb"), + )?; + fs::copy( + new_env.path().join("lock.mdb"), + old_env.path().join("lock.mdb"), + )?; + assert!(Environment::new().open(old_env.path()).is_ok()); + + Ok(()) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs new file mode 100644 index 0000000000..e23bb49770 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/arch_migrator_error.rs @@ -0,0 +1,79 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{io, num, str}; + +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum MigrateError { + #[error("database not found: {0:?}")] + DatabaseNotFound(String), + + #[error("{0}")] + FromString(String), + + #[error("couldn't determine bit depth")] + IndeterminateBitDepth, + + #[error("I/O error: {0:?}")] + IoError(#[from] io::Error), + + #[error("invalid DatabaseFlags bits")] + InvalidDatabaseBits, + + #[error("invalid data version")] + InvalidDataVersion, + + #[error("invalid magic number")] + InvalidMagicNum, + + #[error("invalid NodeFlags bits")] + InvalidNodeBits, + + #[error("invalid PageFlags bits")] + InvalidPageBits, + + #[error("invalid page number")] + InvalidPageNum, + + #[error("lmdb backend error: {0}")] + LmdbError(#[from] lmdb::Error), + + #[error("string conversion error")] + StringConversionError, + + #[error("TryFromInt error: {0:?}")] + TryFromIntError(#[from] num::TryFromIntError), + + #[error("unexpected Page variant")] + UnexpectedPageVariant, + + #[error("unexpected PageHeader variant")] + UnexpectedPageHeaderVariant, + + #[error("unsupported PageHeader variant")] + UnsupportedPageHeaderVariant, + + #[error("UTF8 error: {0:?}")] + Utf8Error(#[from] str::Utf8Error), +} + +impl From<&str> for MigrateError { + fn from(e: &str) -> MigrateError { + MigrateError::FromString(e.to_string()) + } +} + +impl From<String> for MigrateError { + fn from(e: String) -> MigrateError { + MigrateError::FromString(e) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs b/third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs new file mode 100644 index 0000000000..760abce451 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/cursor.rs @@ -0,0 +1,69 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use lmdb::Cursor; + +use super::IterImpl; +use crate::backend::traits::BackendRoCursor; + +#[derive(Debug)] +pub struct RoCursorImpl<'c>(pub(crate) lmdb::RoCursor<'c>); + +impl<'c> BackendRoCursor<'c> for RoCursorImpl<'c> { + type Iter = IterImpl<'c, lmdb::RoCursor<'c>>; + + fn into_iter(self) -> Self::Iter { + // We call RoCursor.iter() instead of RoCursor.iter_start() because + // the latter panics when there are no items in the store, whereas the + // former returns an iterator that yields no items. And since we create + // the Cursor and don't change its position, we can be sure that a call + // to Cursor.iter() will start at the beginning. + IterImpl::new(self.0, lmdb::RoCursor::iter) + } + + fn into_iter_from<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + IterImpl::new(self.0, |cursor| cursor.iter_from(key)) + } + + fn into_iter_dup_of<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + IterImpl::new(self.0, |cursor| cursor.iter_dup_of(key)) + } +} + +#[derive(Debug)] +pub struct RwCursorImpl<'c>(pub(crate) lmdb::RwCursor<'c>); + +impl<'c> BackendRoCursor<'c> for RwCursorImpl<'c> { + type Iter = IterImpl<'c, lmdb::RwCursor<'c>>; + + fn into_iter(self) -> Self::Iter { + IterImpl::new(self.0, lmdb::RwCursor::iter) + } + + fn into_iter_from<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + IterImpl::new(self.0, |cursor| cursor.iter_from(key)) + } + + fn into_iter_dup_of<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + IterImpl::new(self.0, |cursor| cursor.iter_dup_of(key)) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/database.rs b/third_party/rust/rkv/src/backend/impl_lmdb/database.rs new file mode 100644 index 0000000000..8edee5c2c3 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/database.rs @@ -0,0 +1,16 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::backend::traits::BackendDatabase; + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub struct DatabaseImpl(pub(crate) lmdb::Database); + +impl BackendDatabase for DatabaseImpl {} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/environment.rs b/third_party/rust/rkv/src/backend/impl_lmdb/environment.rs new file mode 100644 index 0000000000..a8a72ae8f2 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/environment.rs @@ -0,0 +1,301 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + fs, + path::{Path, PathBuf}, +}; + +use lmdb::Error as LmdbError; + +use super::{ + DatabaseFlagsImpl, DatabaseImpl, EnvironmentFlagsImpl, ErrorImpl, InfoImpl, RoTransactionImpl, + RwTransactionImpl, StatImpl, +}; +use crate::backend::common::RecoveryStrategy; +use crate::backend::traits::{ + BackendEnvironment, BackendEnvironmentBuilder, BackendInfo, BackendIter, BackendRoCursor, + BackendRoCursorTransaction, BackendStat, +}; + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub struct EnvironmentBuilderImpl { + builder: lmdb::EnvironmentBuilder, + env_path_type: EnvironmentPathType, + env_lock_type: EnvironmentLockType, + env_db_type: EnvironmentDefaultDbType, + make_dir_if_needed: bool, +} + +impl<'b> BackendEnvironmentBuilder<'b> for EnvironmentBuilderImpl { + type Environment = EnvironmentImpl; + type Error = ErrorImpl; + type Flags = EnvironmentFlagsImpl; + + fn new() -> EnvironmentBuilderImpl { + EnvironmentBuilderImpl { + builder: lmdb::Environment::new(), + env_path_type: EnvironmentPathType::SubDir, + env_lock_type: EnvironmentLockType::Lockfile, + env_db_type: EnvironmentDefaultDbType::SingleDatabase, + make_dir_if_needed: false, + } + } + + fn set_flags<T>(&mut self, flags: T) -> &mut Self + where + T: Into<Self::Flags>, + { + let flags = flags.into(); + if flags.0 == lmdb::EnvironmentFlags::NO_SUB_DIR { + self.env_path_type = EnvironmentPathType::NoSubDir; + } + if flags.0 == lmdb::EnvironmentFlags::NO_LOCK { + self.env_lock_type = EnvironmentLockType::NoLockfile; + } + self.builder.set_flags(flags.0); + self + } + + fn set_max_readers(&mut self, max_readers: u32) -> &mut Self { + self.builder.set_max_readers(max_readers); + self + } + + fn set_max_dbs(&mut self, max_dbs: u32) -> &mut Self { + if max_dbs > 0 { + self.env_db_type = EnvironmentDefaultDbType::MultipleNamedDatabases + } + self.builder.set_max_dbs(max_dbs); + self + } + + fn set_map_size(&mut self, size: usize) -> &mut Self { + self.builder.set_map_size(size); + self + } + + fn set_make_dir_if_needed(&mut self, make_dir_if_needed: bool) -> &mut Self { + self.make_dir_if_needed = make_dir_if_needed; + self + } + + /// **UNIMPLEMENTED.** Will panic at runtime. + fn set_corruption_recovery_strategy(&mut self, _strategy: RecoveryStrategy) -> &mut Self { + // Unfortunately, when opening a database, LMDB doesn't handle all the ways it could have + // been corrupted. Prefer using the `SafeMode` backend if this is important. + unimplemented!(); + } + + fn open(&self, path: &Path) -> Result<Self::Environment, Self::Error> { + match self.env_path_type { + EnvironmentPathType::NoSubDir => { + if !path.is_file() { + return Err(ErrorImpl::UnsuitableEnvironmentPath(path.into())); + } + } + EnvironmentPathType::SubDir => { + if !path.is_dir() { + if !self.make_dir_if_needed { + return Err(ErrorImpl::UnsuitableEnvironmentPath(path.into())); + } + fs::create_dir_all(path)?; + } + } + } + + self.builder + .open(path) + .map_err(ErrorImpl::LmdbError) + .and_then(|lmdbenv| { + EnvironmentImpl::new( + path, + self.env_path_type, + self.env_lock_type, + self.env_db_type, + lmdbenv, + ) + }) + } +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum EnvironmentPathType { + SubDir, + NoSubDir, +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum EnvironmentLockType { + Lockfile, + NoLockfile, +} + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub enum EnvironmentDefaultDbType { + SingleDatabase, + MultipleNamedDatabases, +} + +#[derive(Debug)] +pub struct EnvironmentImpl { + path: PathBuf, + env_path_type: EnvironmentPathType, + env_lock_type: EnvironmentLockType, + env_db_type: EnvironmentDefaultDbType, + lmdbenv: lmdb::Environment, +} + +impl EnvironmentImpl { + pub(crate) fn new( + path: &Path, + env_path_type: EnvironmentPathType, + env_lock_type: EnvironmentLockType, + env_db_type: EnvironmentDefaultDbType, + lmdbenv: lmdb::Environment, + ) -> Result<EnvironmentImpl, ErrorImpl> { + Ok(EnvironmentImpl { + path: path.to_path_buf(), + env_path_type, + env_lock_type, + env_db_type, + lmdbenv, + }) + } +} + +impl<'e> BackendEnvironment<'e> for EnvironmentImpl { + type Database = DatabaseImpl; + type Error = ErrorImpl; + type Flags = DatabaseFlagsImpl; + type Info = InfoImpl; + type RoTransaction = RoTransactionImpl<'e>; + type RwTransaction = RwTransactionImpl<'e>; + type Stat = StatImpl; + + fn get_dbs(&self) -> Result<Vec<Option<String>>, Self::Error> { + if self.env_db_type == EnvironmentDefaultDbType::SingleDatabase { + return Ok(vec![None]); + } + let db = self + .lmdbenv + .open_db(None) + .map(DatabaseImpl) + .map_err(ErrorImpl::LmdbError)?; + let reader = self.begin_ro_txn()?; + let cursor = reader.open_ro_cursor(&db)?; + let mut iter = cursor.into_iter(); + let mut store = vec![]; + while let Some(result) = iter.next() { + let (key, _) = result?; + let name = String::from_utf8(key.to_owned()) + .map_err(|_| ErrorImpl::LmdbError(lmdb::Error::Corrupted))?; + store.push(Some(name)); + } + Ok(store) + } + + fn open_db(&self, name: Option<&str>) -> Result<Self::Database, Self::Error> { + self.lmdbenv + .open_db(name) + .map(DatabaseImpl) + .map_err(ErrorImpl::LmdbError) + } + + fn create_db( + &self, + name: Option<&str>, + flags: Self::Flags, + ) -> Result<Self::Database, Self::Error> { + self.lmdbenv + .create_db(name, flags.0) + .map(DatabaseImpl) + .map_err(ErrorImpl::LmdbError) + } + + fn begin_ro_txn(&'e self) -> Result<Self::RoTransaction, Self::Error> { + self.lmdbenv + .begin_ro_txn() + .map(RoTransactionImpl) + .map_err(ErrorImpl::LmdbError) + } + + fn begin_rw_txn(&'e self) -> Result<Self::RwTransaction, Self::Error> { + self.lmdbenv + .begin_rw_txn() + .map(RwTransactionImpl) + .map_err(ErrorImpl::LmdbError) + } + + fn sync(&self, force: bool) -> Result<(), Self::Error> { + self.lmdbenv.sync(force).map_err(ErrorImpl::LmdbError) + } + + fn stat(&self) -> Result<Self::Stat, Self::Error> { + self.lmdbenv + .stat() + .map(StatImpl) + .map_err(ErrorImpl::LmdbError) + } + + fn info(&self) -> Result<Self::Info, Self::Error> { + self.lmdbenv + .info() + .map(InfoImpl) + .map_err(ErrorImpl::LmdbError) + } + + fn freelist(&self) -> Result<usize, Self::Error> { + self.lmdbenv.freelist().map_err(ErrorImpl::LmdbError) + } + + fn load_ratio(&self) -> Result<Option<f32>, Self::Error> { + let stat = self.stat()?; + let info = self.info()?; + let freelist = self.freelist()?; + + let last_pgno = info.last_pgno() + 1; // pgno is 0 based. + let total_pgs = info.map_size() / stat.page_size(); + if freelist > last_pgno { + return Err(ErrorImpl::LmdbError(LmdbError::Corrupted)); + } + let used_pgs = last_pgno - freelist; + Ok(Some(used_pgs as f32 / total_pgs as f32)) + } + + fn set_map_size(&self, size: usize) -> Result<(), Self::Error> { + self.lmdbenv + .set_map_size(size) + .map_err(ErrorImpl::LmdbError) + } + + fn get_files_on_disk(&self) -> Vec<PathBuf> { + let mut store = vec![]; + + if self.env_path_type == EnvironmentPathType::NoSubDir { + // The option NO_SUB_DIR could change the default directory layout; therefore this should + // probably return the path used to create environment, along with the custom lockfile + // when available. + unimplemented!(); + } + + let mut db_filename = self.path.clone(); + db_filename.push("data.mdb"); + store.push(db_filename); + + if self.env_lock_type == EnvironmentLockType::Lockfile { + let mut lock_filename = self.path.clone(); + lock_filename.push("lock.mdb"); + store.push(lock_filename); + } + + store + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/error.rs b/third_party/rust/rkv/src/backend/impl_lmdb/error.rs new file mode 100644 index 0000000000..a2fd8a7859 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/error.rs @@ -0,0 +1,57 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{fmt, io, path::PathBuf}; + +use crate::{backend::traits::BackendError, error::StoreError}; + +#[derive(Debug)] +pub enum ErrorImpl { + LmdbError(lmdb::Error), + UnsuitableEnvironmentPath(PathBuf), + IoError(io::Error), +} + +impl BackendError for ErrorImpl {} + +impl fmt::Display for ErrorImpl { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match self { + ErrorImpl::LmdbError(e) => e.fmt(fmt), + ErrorImpl::UnsuitableEnvironmentPath(_) => write!(fmt, "UnsuitableEnvironmentPath"), + ErrorImpl::IoError(e) => e.fmt(fmt), + } + } +} + +impl Into<StoreError> for ErrorImpl { + fn into(self) -> StoreError { + match self { + ErrorImpl::LmdbError(lmdb::Error::Corrupted) => StoreError::DatabaseCorrupted, + ErrorImpl::LmdbError(lmdb::Error::NotFound) => StoreError::KeyValuePairNotFound, + ErrorImpl::LmdbError(lmdb::Error::BadValSize) => StoreError::KeyValuePairBadSize, + ErrorImpl::LmdbError(lmdb::Error::Invalid) => StoreError::FileInvalid, + ErrorImpl::LmdbError(lmdb::Error::MapFull) => StoreError::MapFull, + ErrorImpl::LmdbError(lmdb::Error::DbsFull) => StoreError::DbsFull, + ErrorImpl::LmdbError(lmdb::Error::ReadersFull) => StoreError::ReadersFull, + ErrorImpl::LmdbError(error) => StoreError::LmdbError(error), + ErrorImpl::UnsuitableEnvironmentPath(path) => { + StoreError::UnsuitableEnvironmentPath(path) + } + ErrorImpl::IoError(error) => StoreError::IoError(error), + } + } +} + +impl From<io::Error> for ErrorImpl { + fn from(e: io::Error) -> ErrorImpl { + ErrorImpl::IoError(e) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/flags.rs b/third_party/rust/rkv/src/backend/impl_lmdb/flags.rs new file mode 100644 index 0000000000..7a16bb7c46 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/flags.rs @@ -0,0 +1,123 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::backend::{ + common::{DatabaseFlags, EnvironmentFlags, WriteFlags}, + traits::{BackendDatabaseFlags, BackendEnvironmentFlags, BackendFlags, BackendWriteFlags}, +}; + +#[derive(Debug, Eq, PartialEq, Copy, Clone, Default)] +pub struct EnvironmentFlagsImpl(pub(crate) lmdb::EnvironmentFlags); + +impl BackendFlags for EnvironmentFlagsImpl { + fn empty() -> EnvironmentFlagsImpl { + EnvironmentFlagsImpl(lmdb::EnvironmentFlags::empty()) + } +} + +impl BackendEnvironmentFlags for EnvironmentFlagsImpl { + fn set(&mut self, flag: EnvironmentFlags, value: bool) { + self.0.set(flag.into(), value) + } +} + +impl Into<EnvironmentFlagsImpl> for EnvironmentFlags { + fn into(self) -> EnvironmentFlagsImpl { + EnvironmentFlagsImpl(self.into()) + } +} + +impl Into<lmdb::EnvironmentFlags> for EnvironmentFlags { + fn into(self) -> lmdb::EnvironmentFlags { + match self { + EnvironmentFlags::FIXED_MAP => lmdb::EnvironmentFlags::FIXED_MAP, + EnvironmentFlags::NO_SUB_DIR => lmdb::EnvironmentFlags::NO_SUB_DIR, + EnvironmentFlags::WRITE_MAP => lmdb::EnvironmentFlags::WRITE_MAP, + EnvironmentFlags::READ_ONLY => lmdb::EnvironmentFlags::READ_ONLY, + EnvironmentFlags::NO_META_SYNC => lmdb::EnvironmentFlags::NO_META_SYNC, + EnvironmentFlags::NO_SYNC => lmdb::EnvironmentFlags::NO_SYNC, + EnvironmentFlags::MAP_ASYNC => lmdb::EnvironmentFlags::MAP_ASYNC, + EnvironmentFlags::NO_TLS => lmdb::EnvironmentFlags::NO_TLS, + EnvironmentFlags::NO_LOCK => lmdb::EnvironmentFlags::NO_LOCK, + EnvironmentFlags::NO_READAHEAD => lmdb::EnvironmentFlags::NO_READAHEAD, + EnvironmentFlags::NO_MEM_INIT => lmdb::EnvironmentFlags::NO_MEM_INIT, + } + } +} + +#[derive(Debug, Eq, PartialEq, Copy, Clone, Default)] +pub struct DatabaseFlagsImpl(pub(crate) lmdb::DatabaseFlags); + +impl BackendFlags for DatabaseFlagsImpl { + fn empty() -> DatabaseFlagsImpl { + DatabaseFlagsImpl(lmdb::DatabaseFlags::empty()) + } +} + +impl BackendDatabaseFlags for DatabaseFlagsImpl { + fn set(&mut self, flag: DatabaseFlags, value: bool) { + self.0.set(flag.into(), value) + } +} + +impl Into<DatabaseFlagsImpl> for DatabaseFlags { + fn into(self) -> DatabaseFlagsImpl { + DatabaseFlagsImpl(self.into()) + } +} + +impl Into<lmdb::DatabaseFlags> for DatabaseFlags { + fn into(self) -> lmdb::DatabaseFlags { + match self { + DatabaseFlags::REVERSE_KEY => lmdb::DatabaseFlags::REVERSE_KEY, + #[cfg(feature = "db-dup-sort")] + DatabaseFlags::DUP_SORT => lmdb::DatabaseFlags::DUP_SORT, + #[cfg(feature = "db-dup-sort")] + DatabaseFlags::DUP_FIXED => lmdb::DatabaseFlags::DUP_FIXED, + #[cfg(feature = "db-int-key")] + DatabaseFlags::INTEGER_KEY => lmdb::DatabaseFlags::INTEGER_KEY, + DatabaseFlags::INTEGER_DUP => lmdb::DatabaseFlags::INTEGER_DUP, + DatabaseFlags::REVERSE_DUP => lmdb::DatabaseFlags::REVERSE_DUP, + } + } +} + +#[derive(Debug, Eq, PartialEq, Copy, Clone, Default)] +pub struct WriteFlagsImpl(pub(crate) lmdb::WriteFlags); + +impl BackendFlags for WriteFlagsImpl { + fn empty() -> WriteFlagsImpl { + WriteFlagsImpl(lmdb::WriteFlags::empty()) + } +} + +impl BackendWriteFlags for WriteFlagsImpl { + fn set(&mut self, flag: WriteFlags, value: bool) { + self.0.set(flag.into(), value) + } +} + +impl Into<WriteFlagsImpl> for WriteFlags { + fn into(self) -> WriteFlagsImpl { + WriteFlagsImpl(self.into()) + } +} + +impl Into<lmdb::WriteFlags> for WriteFlags { + fn into(self) -> lmdb::WriteFlags { + match self { + WriteFlags::NO_OVERWRITE => lmdb::WriteFlags::NO_OVERWRITE, + WriteFlags::NO_DUP_DATA => lmdb::WriteFlags::NO_DUP_DATA, + WriteFlags::CURRENT => lmdb::WriteFlags::CURRENT, + WriteFlags::APPEND => lmdb::WriteFlags::APPEND, + WriteFlags::APPEND_DUP => lmdb::WriteFlags::APPEND_DUP, + } + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/info.rs b/third_party/rust/rkv/src/backend/impl_lmdb/info.rs new file mode 100644 index 0000000000..6188065c07 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/info.rs @@ -0,0 +1,35 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::backend::traits::BackendInfo; + +pub struct InfoImpl(pub(crate) lmdb::Info); + +impl BackendInfo for InfoImpl { + fn map_size(&self) -> usize { + self.0.map_size() + } + + fn last_pgno(&self) -> usize { + self.0.last_pgno() + } + + fn last_txnid(&self) -> usize { + self.0.last_txnid() + } + + fn max_readers(&self) -> usize { + self.0.max_readers() as usize + } + + fn num_readers(&self) -> usize { + self.0.num_readers() as usize + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/iter.rs b/third_party/rust/rkv/src/backend/impl_lmdb/iter.rs new file mode 100644 index 0000000000..519d361d4f --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/iter.rs @@ -0,0 +1,41 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use super::ErrorImpl; +use crate::backend::traits::BackendIter; + +pub struct IterImpl<'i, C> { + // LMDB semantics dictate that a cursor must be valid for the entire lifetime + // of an iterator. In other words, cursors must not be dropped while an + // iterator built from it is alive. Unfortunately, the LMDB crate API does + // not express this through the type system, so we must enforce it somehow. + #[allow(dead_code)] + cursor: C, + iter: lmdb::Iter<'i>, +} + +impl<'i, C> IterImpl<'i, C> { + pub(crate) fn new( + mut cursor: C, + to_iter: impl FnOnce(&mut C) -> lmdb::Iter<'i>, + ) -> IterImpl<'i, C> { + let iter = to_iter(&mut cursor); + IterImpl { cursor, iter } + } +} + +impl<'i, C> BackendIter<'i> for IterImpl<'i, C> { + type Error = ErrorImpl; + + #[allow(clippy::type_complexity)] + fn next(&mut self) -> Option<Result<(&'i [u8], &'i [u8]), Self::Error>> { + self.iter.next().map(|e| e.map_err(ErrorImpl::LmdbError)) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/stat.rs b/third_party/rust/rkv/src/backend/impl_lmdb/stat.rs new file mode 100644 index 0000000000..b0de8c5051 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/stat.rs @@ -0,0 +1,39 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::backend::traits::BackendStat; + +pub struct StatImpl(pub(crate) lmdb::Stat); + +impl BackendStat for StatImpl { + fn page_size(&self) -> usize { + self.0.page_size() as usize + } + + fn depth(&self) -> usize { + self.0.depth() as usize + } + + fn branch_pages(&self) -> usize { + self.0.branch_pages() + } + + fn leaf_pages(&self) -> usize { + self.0.leaf_pages() + } + + fn overflow_pages(&self) -> usize { + self.0.overflow_pages() + } + + fn entries(&self) -> usize { + self.0.entries() + } +} diff --git a/third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs b/third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs new file mode 100644 index 0000000000..21752763fd --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_lmdb/transaction.rs @@ -0,0 +1,107 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use lmdb::Transaction; + +use super::{DatabaseImpl, ErrorImpl, RoCursorImpl, WriteFlagsImpl}; +use crate::backend::traits::{ + BackendRoCursorTransaction, BackendRoTransaction, BackendRwCursorTransaction, + BackendRwTransaction, +}; + +#[derive(Debug)] +pub struct RoTransactionImpl<'t>(pub(crate) lmdb::RoTransaction<'t>); + +impl<'t> BackendRoTransaction for RoTransactionImpl<'t> { + type Database = DatabaseImpl; + type Error = ErrorImpl; + + fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> { + self.0.get(db.0, &key).map_err(ErrorImpl::LmdbError) + } + + fn abort(self) { + self.0.abort() + } +} + +impl<'t> BackendRoCursorTransaction<'t> for RoTransactionImpl<'t> { + type RoCursor = RoCursorImpl<'t>; + + fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> { + self.0 + .open_ro_cursor(db.0) + .map(RoCursorImpl) + .map_err(ErrorImpl::LmdbError) + } +} + +#[derive(Debug)] +pub struct RwTransactionImpl<'t>(pub(crate) lmdb::RwTransaction<'t>); + +impl<'t> BackendRwTransaction for RwTransactionImpl<'t> { + type Database = DatabaseImpl; + type Error = ErrorImpl; + type Flags = WriteFlagsImpl; + + fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> { + self.0.get(db.0, &key).map_err(ErrorImpl::LmdbError) + } + + fn put( + &mut self, + db: &Self::Database, + key: &[u8], + value: &[u8], + flags: Self::Flags, + ) -> Result<(), Self::Error> { + self.0 + .put(db.0, &key, &value, flags.0) + .map_err(ErrorImpl::LmdbError) + } + + #[cfg(not(feature = "db-dup-sort"))] + fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error> { + self.0.del(db.0, &key, None).map_err(ErrorImpl::LmdbError) + } + + #[cfg(feature = "db-dup-sort")] + fn del( + &mut self, + db: &Self::Database, + key: &[u8], + value: Option<&[u8]>, + ) -> Result<(), Self::Error> { + self.0.del(db.0, &key, value).map_err(ErrorImpl::LmdbError) + } + + fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error> { + self.0.clear_db(db.0).map_err(ErrorImpl::LmdbError) + } + + fn commit(self) -> Result<(), Self::Error> { + self.0.commit().map_err(ErrorImpl::LmdbError) + } + + fn abort(self) { + self.0.abort() + } +} + +impl<'t> BackendRwCursorTransaction<'t> for RwTransactionImpl<'t> { + type RoCursor = RoCursorImpl<'t>; + + fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> { + self.0 + .open_ro_cursor(db.0) + .map(RoCursorImpl) + .map_err(ErrorImpl::LmdbError) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe.rs b/third_party/rust/rkv/src/backend/impl_safe.rs new file mode 100644 index 0000000000..66ca012973 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe.rs @@ -0,0 +1,30 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +mod cursor; +mod database; +mod environment; +mod error; +mod flags; +mod info; +mod iter; +mod snapshot; +mod stat; +mod transaction; + +pub use cursor::{RoCursorImpl, RwCursorImpl}; +pub use database::DatabaseImpl; +pub use environment::{EnvironmentBuilderImpl, EnvironmentImpl}; +pub use error::ErrorImpl; +pub use flags::{DatabaseFlagsImpl, EnvironmentFlagsImpl, WriteFlagsImpl}; +pub use info::InfoImpl; +pub use iter::IterImpl; +pub use stat::StatImpl; +pub use transaction::{RoTransactionImpl, RwTransactionImpl}; diff --git a/third_party/rust/rkv/src/backend/impl_safe/cursor.rs b/third_party/rust/rkv/src/backend/impl_safe/cursor.rs new file mode 100644 index 0000000000..c3bfefad94 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/cursor.rs @@ -0,0 +1,98 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use super::{snapshot::Snapshot, IterImpl}; +use crate::backend::traits::BackendRoCursor; + +#[derive(Debug)] +pub struct RoCursorImpl<'c>(pub(crate) &'c Snapshot); + +#[cfg(not(feature = "db-dup-sort"))] +impl<'c> BackendRoCursor<'c> for RoCursorImpl<'c> { + type Iter = IterImpl<'c>; + + fn into_iter(self) -> Self::Iter { + IterImpl(Box::new(self.0.iter())) + } + + fn into_iter_from<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + IterImpl(Box::new( + self.0.iter().skip_while(move |&(k, _)| k < key.as_ref()), + )) + } + + fn into_iter_dup_of<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + IterImpl(Box::new( + self.0.iter().filter(move |&(k, _)| k == key.as_ref()), + )) + } +} + +#[cfg(feature = "db-dup-sort")] +impl<'c> BackendRoCursor<'c> for RoCursorImpl<'c> { + type Iter = IterImpl<'c>; + + fn into_iter(self) -> Self::Iter { + let flattened = self + .0 + .iter() + .flat_map(|(key, values)| values.map(move |value| (key, value))); + IterImpl(Box::new(flattened)) + } + + fn into_iter_from<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + let skipped = self.0.iter().skip_while(move |&(k, _)| k < key.as_ref()); + let flattened = skipped.flat_map(|(key, values)| values.map(move |value| (key, value))); + IterImpl(Box::new(flattened)) + } + + fn into_iter_dup_of<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + let filtered = self.0.iter().filter(move |&(k, _)| k == key.as_ref()); + let flattened = filtered.flat_map(|(key, values)| values.map(move |value| (key, value))); + IterImpl(Box::new(flattened)) + } +} + +#[derive(Debug)] +pub struct RwCursorImpl<'c>(&'c mut Snapshot); + +impl<'c> BackendRoCursor<'c> for RwCursorImpl<'c> { + type Iter = IterImpl<'c>; + + fn into_iter(self) -> Self::Iter { + unimplemented!() + } + + fn into_iter_from<K>(self, _key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + unimplemented!() + } + + fn into_iter_dup_of<K>(self, _key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c, + { + unimplemented!() + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/database.rs b/third_party/rust/rkv/src/backend/impl_safe/database.rs new file mode 100644 index 0000000000..f8eb4aef76 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/database.rs @@ -0,0 +1,41 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use id_arena::Id; +use serde_derive::{Deserialize, Serialize}; + +use super::{snapshot::Snapshot, DatabaseFlagsImpl}; +use crate::backend::traits::BackendDatabase; + +#[derive(Debug, Eq, PartialEq, Copy, Clone, Hash)] +pub struct DatabaseImpl(pub(crate) Id<Database>); + +impl BackendDatabase for DatabaseImpl {} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Database { + snapshot: Snapshot, +} + +impl Database { + pub(crate) fn new(flags: Option<DatabaseFlagsImpl>, snapshot: Option<Snapshot>) -> Database { + Database { + snapshot: snapshot.unwrap_or_else(|| Snapshot::new(flags)), + } + } + + pub(crate) fn snapshot(&self) -> Snapshot { + self.snapshot.clone() + } + + pub(crate) fn replace(&mut self, snapshot: Snapshot) -> Snapshot { + std::mem::replace(&mut self.snapshot, snapshot) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/environment.rs b/third_party/rust/rkv/src/backend/impl_safe/environment.rs new file mode 100644 index 0000000000..f9c44cf664 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/environment.rs @@ -0,0 +1,351 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + borrow::Cow, + collections::HashMap, + fs, + ops::DerefMut, + path::{Path, PathBuf}, + sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}, +}; + +use id_arena::Arena; +use log::warn; + +use super::{ + database::Database, DatabaseFlagsImpl, DatabaseImpl, EnvironmentFlagsImpl, ErrorImpl, InfoImpl, + RoTransactionImpl, RwTransactionImpl, StatImpl, +}; +use crate::backend::common::RecoveryStrategy; +use crate::backend::traits::{BackendEnvironment, BackendEnvironmentBuilder}; + +const DEFAULT_DB_FILENAME: &str = "data.safe.bin"; +const DEFAULT_CORRUPT_DB_EXTENSION: &str = "bin.corrupt"; + +type DatabaseArena = Arena<Database>; +type DatabaseNameMap = HashMap<Option<String>, DatabaseImpl>; + +#[derive(Debug, PartialEq, Eq, Copy, Clone)] +pub struct EnvironmentBuilderImpl { + flags: EnvironmentFlagsImpl, + max_readers: Option<usize>, + max_dbs: Option<usize>, + map_size: Option<usize>, + make_dir_if_needed: bool, + corruption_recovery_strategy: RecoveryStrategy, +} + +impl<'b> BackendEnvironmentBuilder<'b> for EnvironmentBuilderImpl { + type Environment = EnvironmentImpl; + type Error = ErrorImpl; + type Flags = EnvironmentFlagsImpl; + + fn new() -> EnvironmentBuilderImpl { + EnvironmentBuilderImpl { + flags: EnvironmentFlagsImpl::empty(), + max_readers: None, + max_dbs: None, + map_size: None, + make_dir_if_needed: false, + corruption_recovery_strategy: RecoveryStrategy::Error, + } + } + + fn set_flags<T>(&mut self, flags: T) -> &mut Self + where + T: Into<Self::Flags>, + { + self.flags = flags.into(); + self + } + + fn set_max_readers(&mut self, max_readers: u32) -> &mut Self { + self.max_readers = Some(max_readers as usize); + self + } + + fn set_max_dbs(&mut self, max_dbs: u32) -> &mut Self { + self.max_dbs = Some(max_dbs as usize); + self + } + + fn set_map_size(&mut self, map_size: usize) -> &mut Self { + self.map_size = Some(map_size); + self + } + + fn set_make_dir_if_needed(&mut self, make_dir_if_needed: bool) -> &mut Self { + self.make_dir_if_needed = make_dir_if_needed; + self + } + + fn set_corruption_recovery_strategy(&mut self, strategy: RecoveryStrategy) -> &mut Self { + self.corruption_recovery_strategy = strategy; + self + } + + fn open(&self, path: &Path) -> Result<Self::Environment, Self::Error> { + // Technically NO_SUB_DIR should change these checks here, but they're both currently + // unimplemented with this storage backend. + if !path.is_dir() { + if !self.make_dir_if_needed { + return Err(ErrorImpl::UnsuitableEnvironmentPath(path.into())); + } + fs::create_dir_all(path)?; + } + let mut env = EnvironmentImpl::new( + path, + self.flags, + self.max_readers, + self.max_dbs, + self.map_size, + )?; + env.read_from_disk(self.corruption_recovery_strategy)?; + Ok(env) + } +} + +#[derive(Debug)] +pub(crate) struct EnvironmentDbs { + pub(crate) arena: DatabaseArena, + pub(crate) name_map: DatabaseNameMap, +} + +#[derive(Debug)] +pub(crate) struct EnvironmentDbsRefMut<'a> { + pub(crate) arena: &'a mut DatabaseArena, + pub(crate) name_map: &'a mut DatabaseNameMap, +} + +impl<'a> From<&'a mut EnvironmentDbs> for EnvironmentDbsRefMut<'a> { + fn from(dbs: &mut EnvironmentDbs) -> EnvironmentDbsRefMut { + EnvironmentDbsRefMut { + arena: &mut dbs.arena, + name_map: &mut dbs.name_map, + } + } +} + +#[derive(Debug)] +pub struct EnvironmentImpl { + path: PathBuf, + max_dbs: usize, + dbs: RwLock<EnvironmentDbs>, + ro_txns: Arc<()>, + rw_txns: Arc<()>, +} + +impl EnvironmentImpl { + fn serialize(&self) -> Result<Vec<u8>, ErrorImpl> { + let dbs = self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)?; + let data: HashMap<_, _> = dbs + .name_map + .iter() + .map(|(name, id)| (name, &dbs.arena[id.0])) + .collect(); + Ok(bincode::serialize(&data)?) + } + + fn load( + path: &Path, + strategy: RecoveryStrategy, + ) -> Result<(DatabaseArena, DatabaseNameMap), ErrorImpl> { + let bytes = fs::read(path)?; + + match Self::deserialize(&bytes) { + Ok((arena, name_map)) => Ok((arena, name_map)), + Err(err) => match strategy { + RecoveryStrategy::Error => Err(err), + RecoveryStrategy::Discard => Ok((DatabaseArena::new(), HashMap::new())), + RecoveryStrategy::Rename => { + let corrupted_path = path.with_extension(DEFAULT_CORRUPT_DB_EXTENSION); + fs::rename(path, corrupted_path)?; + + Ok((DatabaseArena::new(), HashMap::new())) + } + }, + } + } + + fn deserialize(bytes: &[u8]) -> Result<(DatabaseArena, DatabaseNameMap), ErrorImpl> { + let mut arena = DatabaseArena::new(); + let mut name_map = HashMap::new(); + let data: HashMap<_, _> = bincode::deserialize(bytes)?; + + for (name, db) in data { + name_map.insert(name, DatabaseImpl(arena.alloc(db))); + } + Ok((arena, name_map)) + } +} + +impl EnvironmentImpl { + pub(crate) fn new( + path: &Path, + flags: EnvironmentFlagsImpl, + max_readers: Option<usize>, + max_dbs: Option<usize>, + map_size: Option<usize>, + ) -> Result<EnvironmentImpl, ErrorImpl> { + if !flags.is_empty() { + warn!("Ignoring `flags={:?}`", flags); + } + if let Some(max_readers) = max_readers { + warn!("Ignoring `max_readers={}`", max_readers); + } + if let Some(map_size) = map_size { + warn!("Ignoring `map_size={}`", map_size); + } + + Ok(EnvironmentImpl { + path: path.to_path_buf(), + max_dbs: max_dbs.unwrap_or(std::usize::MAX), + dbs: RwLock::new(EnvironmentDbs { + arena: DatabaseArena::new(), + name_map: HashMap::new(), + }), + ro_txns: Arc::new(()), + rw_txns: Arc::new(()), + }) + } + + pub(crate) fn read_from_disk(&mut self, strategy: RecoveryStrategy) -> Result<(), ErrorImpl> { + let mut path = Cow::from(&self.path); + if fs::metadata(&path)?.is_dir() { + path.to_mut().push(DEFAULT_DB_FILENAME); + }; + if fs::metadata(&path).is_err() { + return Ok(()); + }; + let (arena, name_map) = Self::load(&path, strategy)?; + self.dbs = RwLock::new(EnvironmentDbs { arena, name_map }); + Ok(()) + } + + pub(crate) fn write_to_disk(&self) -> Result<(), ErrorImpl> { + let mut path = Cow::from(&self.path); + if fs::metadata(&path)?.is_dir() { + path.to_mut().push(DEFAULT_DB_FILENAME); + }; + + // Write to a temp file first. + let tmp_path = path.with_extension("tmp"); + fs::write(&tmp_path, self.serialize()?)?; + + // Atomically move that file to the database file. + fs::rename(tmp_path, path)?; + Ok(()) + } + + pub(crate) fn dbs(&self) -> Result<RwLockReadGuard<EnvironmentDbs>, ErrorImpl> { + self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError) + } + + pub(crate) fn dbs_mut(&self) -> Result<RwLockWriteGuard<EnvironmentDbs>, ErrorImpl> { + self.dbs.write().map_err(|_| ErrorImpl::EnvPoisonError) + } +} + +impl<'e> BackendEnvironment<'e> for EnvironmentImpl { + type Database = DatabaseImpl; + type Error = ErrorImpl; + type Flags = DatabaseFlagsImpl; + type Info = InfoImpl; + type RoTransaction = RoTransactionImpl<'e>; + type RwTransaction = RwTransactionImpl<'e>; + type Stat = StatImpl; + + fn get_dbs(&self) -> Result<Vec<Option<String>>, Self::Error> { + let dbs = self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)?; + Ok(dbs.name_map.keys().map(|key| key.to_owned()).collect()) + } + + fn open_db(&self, name: Option<&str>) -> Result<Self::Database, Self::Error> { + if Arc::strong_count(&self.ro_txns) > 1 { + return Err(ErrorImpl::DbsIllegalOpen); + } + // TOOD: don't reallocate `name`. + let key = name.map(String::from); + let dbs = self.dbs.read().map_err(|_| ErrorImpl::EnvPoisonError)?; + let db = dbs.name_map.get(&key).ok_or(ErrorImpl::DbNotFoundError)?; + Ok(*db) + } + + fn create_db( + &self, + name: Option<&str>, + flags: Self::Flags, + ) -> Result<Self::Database, Self::Error> { + if Arc::strong_count(&self.ro_txns) > 1 { + return Err(ErrorImpl::DbsIllegalOpen); + } + // TOOD: don't reallocate `name`. + let key = name.map(String::from); + let mut dbs = self.dbs.write().map_err(|_| ErrorImpl::EnvPoisonError)?; + if dbs.name_map.keys().filter_map(|k| k.as_ref()).count() >= self.max_dbs && name.is_some() + { + return Err(ErrorImpl::DbsFull); + } + let parts = EnvironmentDbsRefMut::from(dbs.deref_mut()); + let arena = parts.arena; + let name_map = parts.name_map; + let id = name_map + .entry(key) + .or_insert_with(|| DatabaseImpl(arena.alloc(Database::new(Some(flags), None)))); + Ok(*id) + } + + fn begin_ro_txn(&'e self) -> Result<Self::RoTransaction, Self::Error> { + RoTransactionImpl::new(self, self.ro_txns.clone()) + } + + fn begin_rw_txn(&'e self) -> Result<Self::RwTransaction, Self::Error> { + RwTransactionImpl::new(self, self.rw_txns.clone()) + } + + fn sync(&self, force: bool) -> Result<(), Self::Error> { + warn!("Ignoring `force={}`", force); + self.write_to_disk() + } + + fn stat(&self) -> Result<Self::Stat, Self::Error> { + Ok(StatImpl) + } + + fn info(&self) -> Result<Self::Info, Self::Error> { + Ok(InfoImpl) + } + + fn freelist(&self) -> Result<usize, Self::Error> { + unimplemented!() + } + + fn load_ratio(&self) -> Result<Option<f32>, Self::Error> { + warn!("`load_ratio()` is irrelevant for this storage backend."); + Ok(None) + } + + fn set_map_size(&self, size: usize) -> Result<(), Self::Error> { + warn!( + "`set_map_size({})` is ignored by this storage backend.", + size + ); + Ok(()) + } + + fn get_files_on_disk(&self) -> Vec<PathBuf> { + // Technically NO_SUB_DIR and NO_LOCK should change this output, but + // they're both currently unimplemented with this storage backend. + let mut db_filename = self.path.clone(); + db_filename.push(DEFAULT_DB_FILENAME); + vec![db_filename] + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/error.rs b/third_party/rust/rkv/src/backend/impl_safe/error.rs new file mode 100644 index 0000000000..f086cdc954 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/error.rs @@ -0,0 +1,79 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{fmt, io, path::PathBuf}; + +use bincode::Error as BincodeError; + +use crate::{backend::traits::BackendError, error::StoreError}; + +#[derive(Debug)] +pub enum ErrorImpl { + KeyValuePairNotFound, + EnvPoisonError, + DbsFull, + DbsIllegalOpen, + DbNotFoundError, + DbIsForeignError, + UnsuitableEnvironmentPath(PathBuf), + IoError(io::Error), + BincodeError(BincodeError), +} + +impl BackendError for ErrorImpl {} + +impl fmt::Display for ErrorImpl { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + match self { + ErrorImpl::KeyValuePairNotFound => write!(fmt, "KeyValuePairNotFound (safe mode)"), + ErrorImpl::EnvPoisonError => write!(fmt, "EnvPoisonError (safe mode)"), + ErrorImpl::DbsFull => write!(fmt, "DbsFull (safe mode)"), + ErrorImpl::DbsIllegalOpen => write!(fmt, "DbIllegalOpen (safe mode)"), + ErrorImpl::DbNotFoundError => write!(fmt, "DbNotFoundError (safe mode)"), + ErrorImpl::DbIsForeignError => write!(fmt, "DbIsForeignError (safe mode)"), + ErrorImpl::UnsuitableEnvironmentPath(_) => { + write!(fmt, "UnsuitableEnvironmentPath (safe mode)") + } + ErrorImpl::IoError(e) => e.fmt(fmt), + ErrorImpl::BincodeError(e) => e.fmt(fmt), + } + } +} + +impl Into<StoreError> for ErrorImpl { + fn into(self) -> StoreError { + // The `StoreError::KeyValuePairBadSize` error is unused, because this + // backend supports keys and values of arbitrary sizes. + // The `StoreError::MapFull` and `StoreError::ReadersFull` are + // unimplemented yet, but they should be in the future. + match self { + ErrorImpl::KeyValuePairNotFound => StoreError::KeyValuePairNotFound, + ErrorImpl::BincodeError(_) => StoreError::FileInvalid, + ErrorImpl::DbsFull => StoreError::DbsFull, + ErrorImpl::UnsuitableEnvironmentPath(path) => { + StoreError::UnsuitableEnvironmentPath(path) + } + ErrorImpl::IoError(error) => StoreError::IoError(error), + _ => StoreError::SafeModeError(self), + } + } +} + +impl From<io::Error> for ErrorImpl { + fn from(e: io::Error) -> ErrorImpl { + ErrorImpl::IoError(e) + } +} + +impl From<BincodeError> for ErrorImpl { + fn from(e: BincodeError) -> ErrorImpl { + ErrorImpl::BincodeError(e) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/flags.rs b/third_party/rust/rkv/src/backend/impl_safe/flags.rs new file mode 100644 index 0000000000..5469b44a25 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/flags.rs @@ -0,0 +1,124 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use bitflags::bitflags; +use serde_derive::{Deserialize, Serialize}; + +use crate::backend::{ + common::{DatabaseFlags, EnvironmentFlags, WriteFlags}, + traits::{BackendDatabaseFlags, BackendEnvironmentFlags, BackendFlags, BackendWriteFlags}, +}; + +bitflags! { + #[derive(Default, Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Copy)] + pub struct EnvironmentFlagsImpl: u32 { + const NIL = 0b0000_0000; + } +} + +impl BackendFlags for EnvironmentFlagsImpl { + fn empty() -> EnvironmentFlagsImpl { + EnvironmentFlagsImpl::empty() + } +} + +impl BackendEnvironmentFlags for EnvironmentFlagsImpl { + fn set(&mut self, flag: EnvironmentFlags, value: bool) { + self.set(flag.into(), value) + } +} + +impl Into<EnvironmentFlagsImpl> for EnvironmentFlags { + fn into(self) -> EnvironmentFlagsImpl { + match self { + EnvironmentFlags::FIXED_MAP => unimplemented!(), + EnvironmentFlags::NO_SUB_DIR => unimplemented!(), + EnvironmentFlags::WRITE_MAP => unimplemented!(), + EnvironmentFlags::READ_ONLY => unimplemented!(), + EnvironmentFlags::NO_META_SYNC => unimplemented!(), + EnvironmentFlags::NO_SYNC => unimplemented!(), + EnvironmentFlags::MAP_ASYNC => unimplemented!(), + EnvironmentFlags::NO_TLS => unimplemented!(), + EnvironmentFlags::NO_LOCK => unimplemented!(), + EnvironmentFlags::NO_READAHEAD => unimplemented!(), + EnvironmentFlags::NO_MEM_INIT => unimplemented!(), + } + } +} + +bitflags! { + #[derive(Default, Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Copy)] + pub struct DatabaseFlagsImpl: u32 { + const NIL = 0b0000_0000; + #[cfg(feature = "db-dup-sort")] + const DUP_SORT = 0b0000_0001; + #[cfg(feature = "db-int-key")] + const INTEGER_KEY = 0b0000_0010; + } +} + +impl BackendFlags for DatabaseFlagsImpl { + fn empty() -> DatabaseFlagsImpl { + DatabaseFlagsImpl::empty() + } +} + +impl BackendDatabaseFlags for DatabaseFlagsImpl { + fn set(&mut self, flag: DatabaseFlags, value: bool) { + self.set(flag.into(), value) + } +} + +impl Into<DatabaseFlagsImpl> for DatabaseFlags { + fn into(self) -> DatabaseFlagsImpl { + match self { + DatabaseFlags::REVERSE_KEY => unimplemented!(), + #[cfg(feature = "db-dup-sort")] + DatabaseFlags::DUP_SORT => DatabaseFlagsImpl::DUP_SORT, + #[cfg(feature = "db-dup-sort")] + DatabaseFlags::DUP_FIXED => unimplemented!(), + #[cfg(feature = "db-int-key")] + DatabaseFlags::INTEGER_KEY => DatabaseFlagsImpl::INTEGER_KEY, + DatabaseFlags::INTEGER_DUP => unimplemented!(), + DatabaseFlags::REVERSE_DUP => unimplemented!(), + } + } +} + +bitflags! { + #[derive(Default, Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Copy)] + pub struct WriteFlagsImpl: u32 { + const NIL = 0b0000_0000; + } +} + +impl BackendFlags for WriteFlagsImpl { + fn empty() -> WriteFlagsImpl { + WriteFlagsImpl::empty() + } +} + +impl BackendWriteFlags for WriteFlagsImpl { + fn set(&mut self, flag: WriteFlags, value: bool) { + self.set(flag.into(), value) + } +} + +impl Into<WriteFlagsImpl> for WriteFlags { + fn into(self) -> WriteFlagsImpl { + match self { + WriteFlags::NO_OVERWRITE => unimplemented!(), + WriteFlags::NO_DUP_DATA => unimplemented!(), + WriteFlags::CURRENT => unimplemented!(), + WriteFlags::APPEND => unimplemented!(), + WriteFlags::APPEND_DUP => unimplemented!(), + } + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/info.rs b/third_party/rust/rkv/src/backend/impl_safe/info.rs new file mode 100644 index 0000000000..18f0f51da3 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/info.rs @@ -0,0 +1,35 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::backend::traits::BackendInfo; + +pub struct InfoImpl; + +impl BackendInfo for InfoImpl { + fn map_size(&self) -> usize { + unimplemented!() + } + + fn last_pgno(&self) -> usize { + unimplemented!() + } + + fn last_txnid(&self) -> usize { + unimplemented!() + } + + fn max_readers(&self) -> usize { + unimplemented!() + } + + fn num_readers(&self) -> usize { + unimplemented!() + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/iter.rs b/third_party/rust/rkv/src/backend/impl_safe/iter.rs new file mode 100644 index 0000000000..a784c00a5b --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/iter.rs @@ -0,0 +1,24 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use super::ErrorImpl; +use crate::backend::traits::BackendIter; + +// FIXME: Use generics instead. +pub struct IterImpl<'i>(pub(crate) Box<dyn Iterator<Item = (&'i [u8], &'i [u8])> + 'i>); + +impl<'i> BackendIter<'i> for IterImpl<'i> { + type Error = ErrorImpl; + + #[allow(clippy::type_complexity)] + fn next(&mut self) -> Option<Result<(&'i [u8], &'i [u8]), Self::Error>> { + self.0.next().map(Ok) + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/snapshot.rs b/third_party/rust/rkv/src/backend/impl_safe/snapshot.rs new file mode 100644 index 0000000000..95f5690d34 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/snapshot.rs @@ -0,0 +1,141 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, +}; + +use serde_derive::{Deserialize, Serialize}; + +use super::DatabaseFlagsImpl; + +type Key = Box<[u8]>; +type Value = Box<[u8]>; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Snapshot { + flags: DatabaseFlagsImpl, + #[cfg(not(feature = "db-dup-sort"))] + map: Arc<BTreeMap<Key, Value>>, + #[cfg(feature = "db-dup-sort")] + map: Arc<BTreeMap<Key, BTreeSet<Value>>>, +} + +impl Snapshot { + pub(crate) fn new(flags: Option<DatabaseFlagsImpl>) -> Snapshot { + Snapshot { + flags: flags.unwrap_or_default(), + map: Default::default(), + } + } + + pub(crate) fn flags(&self) -> &DatabaseFlagsImpl { + &self.flags + } + + pub(crate) fn clear(&mut self) { + self.map = Default::default(); + } +} + +#[cfg(not(feature = "db-dup-sort"))] +impl Snapshot { + pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> { + self.map.get(key).map(|value| value.as_ref()) + } + + pub(crate) fn put(&mut self, key: &[u8], value: &[u8]) { + let map = Arc::make_mut(&mut self.map); + map.insert(Box::from(key), Box::from(value)); + } + + pub(crate) fn del(&mut self, key: &[u8]) -> Option<()> { + let map = Arc::make_mut(&mut self.map); + map.remove(key).map(|_| ()) + } + + pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], &[u8])> { + self.map + .iter() + .map(|(key, value)| (key.as_ref(), value.as_ref())) + } +} + +#[cfg(feature = "db-dup-sort")] +impl Snapshot { + pub(crate) fn get(&self, key: &[u8]) -> Option<&[u8]> { + self.map + .get(key) + .and_then(|v| v.iter().next()) + .map(|v| v.as_ref()) + } + + pub(crate) fn put(&mut self, key: &[u8], value: &[u8]) { + let map = Arc::make_mut(&mut self.map); + match map.get_mut(key) { + None => { + let mut values = BTreeSet::new(); + values.insert(Box::from(value)); + map.insert(Box::from(key), values); + } + Some(values) => { + values.clear(); + values.insert(Box::from(value)); + } + } + } + + pub(crate) fn del(&mut self, key: &[u8]) -> Option<()> { + let map = Arc::make_mut(&mut self.map); + match map.get_mut(key) { + None => None, + Some(values) => { + let was_empty = values.is_empty(); + values.clear(); + Some(()).filter(|_| !was_empty) + } + } + } + + pub(crate) fn iter(&self) -> impl Iterator<Item = (&[u8], impl Iterator<Item = &[u8]>)> { + self.map + .iter() + .map(|(key, values)| (key.as_ref(), values.iter().map(|value| value.as_ref()))) + } +} + +#[cfg(feature = "db-dup-sort")] +impl Snapshot { + pub(crate) fn put_dup(&mut self, key: &[u8], value: &[u8]) { + let map = Arc::make_mut(&mut self.map); + match map.get_mut(key) { + None => { + let mut values = BTreeSet::new(); + values.insert(Box::from(value)); + map.insert(Box::from(key), values); + } + Some(values) => { + values.insert(Box::from(value)); + } + } + } + + pub(crate) fn del_exact(&mut self, key: &[u8], value: &[u8]) -> Option<()> { + let map = Arc::make_mut(&mut self.map); + match map.get_mut(key) { + None => None, + Some(values) => { + let was_removed = values.remove(value); + Some(()).filter(|_| was_removed) + } + } + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/stat.rs b/third_party/rust/rkv/src/backend/impl_safe/stat.rs new file mode 100644 index 0000000000..c117b56833 --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/stat.rs @@ -0,0 +1,39 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::backend::traits::BackendStat; + +pub struct StatImpl; + +impl BackendStat for StatImpl { + fn page_size(&self) -> usize { + unimplemented!() + } + + fn depth(&self) -> usize { + unimplemented!() + } + + fn branch_pages(&self) -> usize { + unimplemented!() + } + + fn leaf_pages(&self) -> usize { + unimplemented!() + } + + fn overflow_pages(&self) -> usize { + unimplemented!() + } + + fn entries(&self) -> usize { + unimplemented!() + } +} diff --git a/third_party/rust/rkv/src/backend/impl_safe/transaction.rs b/third_party/rust/rkv/src/backend/impl_safe/transaction.rs new file mode 100644 index 0000000000..d37352a42d --- /dev/null +++ b/third_party/rust/rkv/src/backend/impl_safe/transaction.rs @@ -0,0 +1,208 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![allow(dead_code)] // TODO: Get rid of unused struct members + +use std::{collections::HashMap, sync::Arc}; + +use super::{ + snapshot::Snapshot, DatabaseImpl, EnvironmentImpl, ErrorImpl, RoCursorImpl, WriteFlagsImpl, +}; +use crate::backend::traits::{ + BackendRoCursorTransaction, BackendRoTransaction, BackendRwCursorTransaction, + BackendRwTransaction, +}; + +#[derive(Debug)] +pub struct RoTransactionImpl<'t> { + env: &'t EnvironmentImpl, + snapshots: HashMap<DatabaseImpl, Snapshot>, + idx: Arc<()>, +} + +impl<'t> RoTransactionImpl<'t> { + pub(crate) fn new( + env: &'t EnvironmentImpl, + idx: Arc<()>, + ) -> Result<RoTransactionImpl<'t>, ErrorImpl> { + let snapshots = env + .dbs()? + .arena + .iter() + .map(|(id, db)| (DatabaseImpl(id), db.snapshot())) + .collect(); + Ok(RoTransactionImpl { + env, + snapshots, + idx, + }) + } +} + +impl<'t> BackendRoTransaction for RoTransactionImpl<'t> { + type Database = DatabaseImpl; + type Error = ErrorImpl; + + fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> { + let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?; + snapshot.get(key).ok_or(ErrorImpl::KeyValuePairNotFound) + } + + fn abort(self) { + // noop + } +} + +impl<'t> BackendRoCursorTransaction<'t> for RoTransactionImpl<'t> { + type RoCursor = RoCursorImpl<'t>; + + fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> { + let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?; + Ok(RoCursorImpl(snapshot)) + } +} + +#[derive(Debug)] +pub struct RwTransactionImpl<'t> { + env: &'t EnvironmentImpl, + snapshots: HashMap<DatabaseImpl, Snapshot>, + idx: Arc<()>, +} + +impl<'t> RwTransactionImpl<'t> { + pub(crate) fn new( + env: &'t EnvironmentImpl, + idx: Arc<()>, + ) -> Result<RwTransactionImpl<'t>, ErrorImpl> { + let snapshots = env + .dbs()? + .arena + .iter() + .map(|(id, db)| (DatabaseImpl(id), db.snapshot())) + .collect(); + Ok(RwTransactionImpl { + env, + snapshots, + idx, + }) + } +} + +impl<'t> BackendRwTransaction for RwTransactionImpl<'t> { + type Database = DatabaseImpl; + type Error = ErrorImpl; + type Flags = WriteFlagsImpl; + + fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error> { + let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?; + snapshot.get(key).ok_or(ErrorImpl::KeyValuePairNotFound) + } + + #[cfg(not(feature = "db-dup-sort"))] + fn put( + &mut self, + db: &Self::Database, + key: &[u8], + value: &[u8], + _flags: Self::Flags, + ) -> Result<(), Self::Error> { + let snapshot = self + .snapshots + .get_mut(db) + .ok_or_else(|| ErrorImpl::DbIsForeignError)?; + snapshot.put(key, value); + Ok(()) + } + + #[cfg(feature = "db-dup-sort")] + fn put( + &mut self, + db: &Self::Database, + key: &[u8], + value: &[u8], + _flags: Self::Flags, + ) -> Result<(), Self::Error> { + use super::DatabaseFlagsImpl; + let snapshot = self + .snapshots + .get_mut(db) + .ok_or(ErrorImpl::DbIsForeignError)?; + if snapshot.flags().contains(DatabaseFlagsImpl::DUP_SORT) { + snapshot.put_dup(key, value); + } else { + snapshot.put(key, value); + } + Ok(()) + } + + #[cfg(not(feature = "db-dup-sort"))] + fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error> { + let snapshot = self + .snapshots + .get_mut(db) + .ok_or_else(|| ErrorImpl::DbIsForeignError)?; + let deleted = snapshot.del(key); + Ok(deleted.ok_or_else(|| ErrorImpl::KeyValuePairNotFound)?) + } + + #[cfg(feature = "db-dup-sort")] + fn del( + &mut self, + db: &Self::Database, + key: &[u8], + value: Option<&[u8]>, + ) -> Result<(), Self::Error> { + use super::DatabaseFlagsImpl; + let snapshot = self + .snapshots + .get_mut(db) + .ok_or(ErrorImpl::DbIsForeignError)?; + let deleted = match (value, snapshot.flags()) { + (Some(value), flags) if flags.contains(DatabaseFlagsImpl::DUP_SORT) => { + snapshot.del_exact(key, value) + } + _ => snapshot.del(key), + }; + deleted.ok_or(ErrorImpl::KeyValuePairNotFound) + } + + fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error> { + let snapshot = self + .snapshots + .get_mut(db) + .ok_or(ErrorImpl::DbIsForeignError)?; + snapshot.clear(); + Ok(()) + } + + fn commit(self) -> Result<(), Self::Error> { + let mut dbs = self.env.dbs_mut()?; + + for (id, snapshot) in self.snapshots { + let db = dbs.arena.get_mut(id.0).ok_or(ErrorImpl::DbIsForeignError)?; + db.replace(snapshot); + } + + drop(dbs); + self.env.write_to_disk() + } + + fn abort(self) { + // noop + } +} + +impl<'t> BackendRwCursorTransaction<'t> for RwTransactionImpl<'t> { + type RoCursor = RoCursorImpl<'t>; + + fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error> { + let snapshot = self.snapshots.get(db).ok_or(ErrorImpl::DbIsForeignError)?; + Ok(RoCursorImpl(snapshot)) + } +} diff --git a/third_party/rust/rkv/src/backend/traits.rs b/third_party/rust/rkv/src/backend/traits.rs new file mode 100644 index 0000000000..ac5d1e9e76 --- /dev/null +++ b/third_party/rust/rkv/src/backend/traits.rs @@ -0,0 +1,203 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + fmt::{Debug, Display}, + path::{Path, PathBuf}, +}; + +use crate::{ + backend::common::{DatabaseFlags, EnvironmentFlags, RecoveryStrategy, WriteFlags}, + error::StoreError, +}; + +pub trait BackendError: Debug + Display + Into<StoreError> {} + +pub trait BackendDatabase: Debug + Eq + PartialEq + Copy + Clone {} + +pub trait BackendFlags: Debug + Eq + PartialEq + Copy + Clone + Default { + fn empty() -> Self; +} + +pub trait BackendEnvironmentFlags: BackendFlags { + fn set(&mut self, flag: EnvironmentFlags, value: bool); +} + +pub trait BackendDatabaseFlags: BackendFlags { + fn set(&mut self, flag: DatabaseFlags, value: bool); +} + +pub trait BackendWriteFlags: BackendFlags { + fn set(&mut self, flag: WriteFlags, value: bool); +} + +pub trait BackendStat { + fn page_size(&self) -> usize; + + fn depth(&self) -> usize; + + fn branch_pages(&self) -> usize; + + fn leaf_pages(&self) -> usize; + + fn overflow_pages(&self) -> usize; + + fn entries(&self) -> usize; +} + +pub trait BackendInfo { + fn map_size(&self) -> usize; + + fn last_pgno(&self) -> usize; + + fn last_txnid(&self) -> usize; + + fn max_readers(&self) -> usize; + + fn num_readers(&self) -> usize; +} + +pub trait BackendEnvironmentBuilder<'b>: Debug + Eq + PartialEq + Copy + Clone { + type Error: BackendError; + type Environment: BackendEnvironment<'b>; + type Flags: BackendEnvironmentFlags; + + fn new() -> Self; + + fn set_flags<T>(&mut self, flags: T) -> &mut Self + where + T: Into<Self::Flags>; + + fn set_max_dbs(&mut self, max_dbs: u32) -> &mut Self; + + fn set_max_readers(&mut self, max_readers: u32) -> &mut Self; + + fn set_map_size(&mut self, size: usize) -> &mut Self; + + fn set_make_dir_if_needed(&mut self, make_dir_if_needed: bool) -> &mut Self; + + /// Set the corruption recovery strategy. See [`RecoveryStrategy`] for details. + fn set_corruption_recovery_strategy(&mut self, strategy: RecoveryStrategy) -> &mut Self; + + fn open(&self, path: &Path) -> Result<Self::Environment, Self::Error>; +} + +pub trait BackendEnvironment<'e>: Debug { + type Error: BackendError; + type Database: BackendDatabase; + type Flags: BackendDatabaseFlags; + type Stat: BackendStat; + type Info: BackendInfo; + type RoTransaction: BackendRoCursorTransaction<'e, Database = Self::Database>; + type RwTransaction: BackendRwCursorTransaction<'e, Database = Self::Database>; + + fn get_dbs(&self) -> Result<Vec<Option<String>>, Self::Error>; + + fn open_db(&self, name: Option<&str>) -> Result<Self::Database, Self::Error>; + + fn create_db( + &self, + name: Option<&str>, + flags: Self::Flags, + ) -> Result<Self::Database, Self::Error>; + + fn begin_ro_txn(&'e self) -> Result<Self::RoTransaction, Self::Error>; + + fn begin_rw_txn(&'e self) -> Result<Self::RwTransaction, Self::Error>; + + fn sync(&self, force: bool) -> Result<(), Self::Error>; + + fn stat(&self) -> Result<Self::Stat, Self::Error>; + + fn info(&self) -> Result<Self::Info, Self::Error>; + + fn freelist(&self) -> Result<usize, Self::Error>; + + fn load_ratio(&self) -> Result<Option<f32>, Self::Error>; + + fn set_map_size(&self, size: usize) -> Result<(), Self::Error>; + + fn get_files_on_disk(&self) -> Vec<PathBuf>; +} + +pub trait BackendRoTransaction: Debug { + type Error: BackendError; + type Database: BackendDatabase; + + fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error>; + + fn abort(self); +} + +pub trait BackendRwTransaction: Debug { + type Error: BackendError; + type Database: BackendDatabase; + type Flags: BackendWriteFlags; + + fn get(&self, db: &Self::Database, key: &[u8]) -> Result<&[u8], Self::Error>; + + fn put( + &mut self, + db: &Self::Database, + key: &[u8], + value: &[u8], + flags: Self::Flags, + ) -> Result<(), Self::Error>; + + #[cfg(not(feature = "db-dup-sort"))] + fn del(&mut self, db: &Self::Database, key: &[u8]) -> Result<(), Self::Error>; + + #[cfg(feature = "db-dup-sort")] + fn del( + &mut self, + db: &Self::Database, + key: &[u8], + value: Option<&[u8]>, + ) -> Result<(), Self::Error>; + + fn clear_db(&mut self, db: &Self::Database) -> Result<(), Self::Error>; + + fn commit(self) -> Result<(), Self::Error>; + + fn abort(self); +} + +pub trait BackendRoCursorTransaction<'t>: BackendRoTransaction { + type RoCursor: BackendRoCursor<'t>; + + fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error>; +} + +pub trait BackendRwCursorTransaction<'t>: BackendRwTransaction { + type RoCursor: BackendRoCursor<'t>; + + fn open_ro_cursor(&'t self, db: &Self::Database) -> Result<Self::RoCursor, Self::Error>; +} + +pub trait BackendRoCursor<'c>: Debug { + type Iter: BackendIter<'c>; + + fn into_iter(self) -> Self::Iter; + + fn into_iter_from<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c; + + fn into_iter_dup_of<K>(self, key: K) -> Self::Iter + where + K: AsRef<[u8]> + 'c; +} + +pub trait BackendIter<'i> { + type Error: BackendError; + + #[allow(clippy::type_complexity)] + fn next(&mut self) -> Option<Result<(&'i [u8], &'i [u8]), Self::Error>>; +} diff --git a/third_party/rust/rkv/src/bin/dump.rs b/third_party/rust/rkv/src/bin/dump.rs new file mode 100644 index 0000000000..c0e12b986f --- /dev/null +++ b/third_party/rust/rkv/src/bin/dump.rs @@ -0,0 +1,47 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{env::args, io, path::Path}; + +use rkv::migrator::{LmdbArchMigrateError, LmdbArchMigrator}; + +fn main() -> Result<(), LmdbArchMigrateError> { + let mut cli_args = args(); + let mut db_name = None; + let mut env_path = None; + + // The first arg is the name of the program, which we can ignore. + cli_args.next(); + + while let Some(arg) = cli_args.next() { + if &arg[0..1] == "-" { + match &arg[1..] { + "s" => { + db_name = match cli_args.next() { + None => return Err("-s must be followed by database name".into()), + Some(str) => Some(str), + }; + } + str => return Err(format!("arg -{str} not recognized").into()), + } + } else { + if env_path.is_some() { + return Err("must provide only one path to the LMDB environment".into()); + } + env_path = Some(arg); + } + } + + let env_path = env_path.ok_or("must provide a path to the LMDB environment")?; + let mut migrator = LmdbArchMigrator::new(Path::new(&env_path))?; + migrator.dump(db_name.as_deref(), io::stdout()).unwrap(); + + Ok(()) +} diff --git a/third_party/rust/rkv/src/bin/rand.rs b/third_party/rust/rkv/src/bin/rand.rs new file mode 100644 index 0000000000..56d578c22e --- /dev/null +++ b/third_party/rust/rkv/src/bin/rand.rs @@ -0,0 +1,106 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +//! A command-line utility to create an LMDB environment containing random data. +//! It requires one flag, `-s path/to/environment`, which specifies the location +//! where the tool should create the environment. Optionally, you may specify +//! the number of key/value pairs to create via the `-n <number>` flag +//! (for which the default value is 50). + +use std::{env::args, fs, fs::File, io::Read, path::Path}; + +use rkv::{ + backend::{BackendEnvironmentBuilder, Lmdb}, + Rkv, StoreOptions, Value, +}; + +fn main() { + let mut args = args(); + let mut database = None; + let mut path = None; + let mut num_pairs = 50; + + // The first arg is the name of the program, which we can ignore. + args.next(); + + while let Some(arg) = args.next() { + if &arg[0..1] == "-" { + match &arg[1..] { + "s" => { + database = match args.next() { + None => panic!("-s must be followed by database arg"), + Some(str) => Some(str), + }; + } + "n" => { + num_pairs = match args.next() { + None => panic!("-s must be followed by number of pairs"), + Some(str) => str.parse().expect("number"), + }; + } + str => panic!("arg -{} not recognized", str), + } + } else { + if path.is_some() { + panic!("must provide only one path to the LMDB environment"); + } + path = Some(arg); + } + } + + if path.is_none() { + panic!("must provide a path to the LMDB environment"); + } + + let path = path.unwrap(); + fs::create_dir_all(&path).expect("dir created"); + + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(2); + // Allocate enough map to accommodate the largest random collection. + // We currently do this by allocating twice the maximum possible size + // of the pairs (assuming maximum key and value sizes). + builder.set_map_size((511 + 65535) * num_pairs * 2); + let rkv = Rkv::from_builder(Path::new(&path), builder).expect("Rkv"); + let store = rkv + .open_single(database.as_deref(), StoreOptions::create()) + .expect("opened"); + let mut writer = rkv.write().expect("writer"); + + // Generate random values for the number of keys and key/value lengths. + // On Linux, "Just use /dev/urandom!" <https://www.2uo.de/myths-about-urandom/>. + // On macOS it doesn't matter (/dev/random and /dev/urandom are identical). + let mut random = File::open("/dev/urandom").unwrap(); + let mut nums = [0u8; 4]; + random.read_exact(&mut nums).unwrap(); + + // Generate 0–255 pairs. + for _ in 0..num_pairs { + // Generate key and value lengths. The key must be 1–511 bytes long. + // The value length can be 0 and is essentially unbounded; we generate + // value lengths of 0–0xffff (65535). + // NB: the modulus method for generating a random number within a range + // introduces distribution skew, but we don't need it to be perfect. + let key_len = ((u16::from(nums[0]) + (u16::from(nums[1]) << 8)) % 511 + 1) as usize; + let value_len = (u16::from(nums[2]) + (u16::from(nums[3]) << 8)) as usize; + + let mut key: Vec<u8> = vec![0; key_len]; + random.read_exact(&mut key[0..key_len]).unwrap(); + + let mut value: Vec<u8> = vec![0; value_len]; + random.read_exact(&mut value[0..value_len]).unwrap(); + + store + .put(&mut writer, key, &Value::Blob(&value)) + .expect("wrote"); + } + + writer.commit().expect("committed"); +} diff --git a/third_party/rust/rkv/src/env.rs b/third_party/rust/rkv/src/env.rs new file mode 100644 index 0000000000..32e4fe5035 --- /dev/null +++ b/third_party/rust/rkv/src/env.rs @@ -0,0 +1,320 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + fs, + os::raw::c_uint, + path::{Path, PathBuf}, +}; + +#[cfg(any(feature = "db-dup-sort", feature = "db-int-key"))] +use crate::backend::{BackendDatabaseFlags, DatabaseFlags}; +use crate::{ + backend::{ + BackendEnvironment, BackendEnvironmentBuilder, BackendRoCursorTransaction, + BackendRwCursorTransaction, SafeModeError, + }, + error::{CloseError, StoreError}, + readwrite::{Reader, Writer}, + store::{single::SingleStore, CloseOptions, Options as StoreOptions}, +}; + +#[cfg(feature = "db-dup-sort")] +use crate::store::multi::MultiStore; + +#[cfg(feature = "db-int-key")] +use crate::store::integer::IntegerStore; +#[cfg(feature = "db-int-key")] +use crate::store::keys::PrimitiveInt; + +#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))] +use crate::store::integermulti::MultiIntegerStore; + +pub static DEFAULT_MAX_DBS: c_uint = 5; + +/// Wrapper around an `Environment` (e.g. such as an `LMDB` or `SafeMode` environment). +#[derive(Debug)] +pub struct Rkv<E> { + _path: PathBuf, + env: E, +} + +/// Static methods. +impl<'e, E> Rkv<E> +where + E: BackendEnvironment<'e>, +{ + pub fn environment_builder<B>() -> B + where + B: BackendEnvironmentBuilder<'e, Environment = E>, + { + B::new() + } + + /// Return a new Rkv environment that supports up to `DEFAULT_MAX_DBS` open databases. + #[allow(clippy::new_ret_no_self)] + pub fn new<B>(path: &Path) -> Result<Rkv<E>, StoreError> + where + B: BackendEnvironmentBuilder<'e, Environment = E>, + { + Rkv::with_capacity::<B>(path, DEFAULT_MAX_DBS) + } + + /// Return a new Rkv environment that supports the specified number of open databases. + pub fn with_capacity<B>(path: &Path, max_dbs: c_uint) -> Result<Rkv<E>, StoreError> + where + B: BackendEnvironmentBuilder<'e, Environment = E>, + { + let mut builder = B::new(); + builder.set_max_dbs(max_dbs); + + // Future: set flags, maximum size, etc. here if necessary. + Rkv::from_builder(path, builder) + } + + /// Return a new Rkv environment from the provided builder. + pub fn from_builder<B>(path: &Path, builder: B) -> Result<Rkv<E>, StoreError> + where + B: BackendEnvironmentBuilder<'e, Environment = E>, + { + Ok(Rkv { + _path: path.into(), + env: builder.open(path).map_err(|e| e.into())?, + }) + } +} + +/// Store creation methods. +impl<'e, E> Rkv<E> +where + E: BackendEnvironment<'e>, +{ + /// Return all created databases. + pub fn get_dbs(&self) -> Result<Vec<Option<String>>, StoreError> { + self.env.get_dbs().map_err(|e| e.into()) + } + + /// Create or Open an existing database in (&[u8] -> Single Value) mode. + /// Note: that create=true cannot be called concurrently with other operations so if + /// you are sure that the database exists, call this with create=false. + pub fn open_single<'s, T>( + &self, + name: T, + opts: StoreOptions<E::Flags>, + ) -> Result<SingleStore<E::Database>, StoreError> + where + T: Into<Option<&'s str>>, + { + self.open(name, opts).map(SingleStore::new) + } + + /// Create or Open an existing database in (Integer -> Single Value) mode. + /// Note: that create=true cannot be called concurrently with other operations so if + /// you are sure that the database exists, call this with create=false. + #[cfg(feature = "db-int-key")] + pub fn open_integer<'s, T, K>( + &self, + name: T, + mut opts: StoreOptions<E::Flags>, + ) -> Result<IntegerStore<E::Database, K>, StoreError> + where + K: PrimitiveInt, + T: Into<Option<&'s str>>, + { + opts.flags.set(DatabaseFlags::INTEGER_KEY, true); + self.open(name, opts).map(IntegerStore::new) + } + + /// Create or Open an existing database in (&[u8] -> Multiple Values) mode. + /// Note: that create=true cannot be called concurrently with other operations so if + /// you are sure that the database exists, call this with create=false. + #[cfg(feature = "db-dup-sort")] + pub fn open_multi<'s, T>( + &self, + name: T, + mut opts: StoreOptions<E::Flags>, + ) -> Result<MultiStore<E::Database>, StoreError> + where + T: Into<Option<&'s str>>, + { + opts.flags.set(DatabaseFlags::DUP_SORT, true); + self.open(name, opts).map(MultiStore::new) + } + + /// Create or Open an existing database in (Integer -> Multiple Values) mode. + /// Note: that create=true cannot be called concurrently with other operations so if + /// you are sure that the database exists, call this with create=false. + #[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))] + pub fn open_multi_integer<'s, T, K>( + &self, + name: T, + mut opts: StoreOptions<E::Flags>, + ) -> Result<MultiIntegerStore<E::Database, K>, StoreError> + where + K: PrimitiveInt, + T: Into<Option<&'s str>>, + { + opts.flags.set(DatabaseFlags::INTEGER_KEY, true); + opts.flags.set(DatabaseFlags::DUP_SORT, true); + self.open(name, opts).map(MultiIntegerStore::new) + } + + fn open<'s, T>(&self, name: T, opts: StoreOptions<E::Flags>) -> Result<E::Database, StoreError> + where + T: Into<Option<&'s str>>, + { + if opts.create { + self.env + .create_db(name.into(), opts.flags) + .map_err(|e| match e.into() { + #[cfg(feature = "lmdb")] + StoreError::LmdbError(lmdb::Error::BadRslot) => { + StoreError::open_during_transaction() + } + StoreError::SafeModeError(SafeModeError::DbsIllegalOpen) => { + StoreError::open_during_transaction() + } + e => e, + }) + } else { + self.env.open_db(name.into()).map_err(|e| match e.into() { + #[cfg(feature = "lmdb")] + StoreError::LmdbError(lmdb::Error::BadRslot) => { + StoreError::open_during_transaction() + } + StoreError::SafeModeError(SafeModeError::DbsIllegalOpen) => { + StoreError::open_during_transaction() + } + e => e, + }) + } + } +} + +/// Read and write accessors. +impl<'e, E> Rkv<E> +where + E: BackendEnvironment<'e>, +{ + /// Create a read transaction. There can be multiple concurrent readers for an + /// environment, up to the maximum specified by LMDB (default 126), and you can open + /// readers while a write transaction is active. + pub fn read<T>(&'e self) -> Result<Reader<T>, StoreError> + where + E: BackendEnvironment<'e, RoTransaction = T>, + T: BackendRoCursorTransaction<'e, Database = E::Database>, + { + Ok(Reader::new(self.env.begin_ro_txn().map_err(|e| e.into())?)) + } + + /// Create a write transaction. There can be only one write transaction active at any + /// given time, so trying to create a second one will block until the first is + /// committed or aborted. + pub fn write<T>(&'e self) -> Result<Writer<T>, StoreError> + where + E: BackendEnvironment<'e, RwTransaction = T>, + T: BackendRwCursorTransaction<'e, Database = E::Database>, + { + Ok(Writer::new(self.env.begin_rw_txn().map_err(|e| e.into())?)) + } +} + +/// Other environment methods. +impl<'e, E> Rkv<E> +where + E: BackendEnvironment<'e>, +{ + /// Flush the data buffers to disk. This call is only useful, when the environment was + /// open with either `NO_SYNC`, `NO_META_SYNC` or `MAP_ASYNC` (see below). The call is + /// not valid if the environment was opened with `READ_ONLY`. + /// + /// Data is always written to disk when `transaction.commit()` is called, but the + /// operating system may keep it buffered. LMDB always flushes the OS buffers upon + /// commit as well, unless the environment was opened with `NO_SYNC` or in part + /// `NO_META_SYNC`. + /// + /// `force`: if true, force a synchronous flush. Otherwise if the environment has the + /// `NO_SYNC` flag set the flushes will be omitted, and with `MAP_ASYNC` they will + /// be asynchronous. + pub fn sync(&self, force: bool) -> Result<(), StoreError> { + self.env.sync(force).map_err(|e| e.into()) + } + + /// Retrieve statistics about this environment. + /// + /// It includes: + /// * Page size in bytes + /// * B-tree depth + /// * Number of internal (non-leaf) pages + /// * Number of leaf pages + /// * Number of overflow pages + /// * Number of data entries + pub fn stat(&self) -> Result<E::Stat, StoreError> { + self.env.stat().map_err(|e| e.into()) + } + + /// Retrieve information about this environment. + /// + /// It includes: + /// * Map size in bytes + /// * The last used page number + /// * The last transaction ID + /// * Max number of readers allowed + /// * Number of readers in use + pub fn info(&self) -> Result<E::Info, StoreError> { + self.env.info().map_err(|e| e.into()) + } + + /// Retrieve the load ratio (# of used pages / total pages) about this environment. + /// + /// With the formular: (last_page_no - freelist_pages) / total_pages. + /// A value of `None` means that the backend doesn't ever need to be resized. + pub fn load_ratio(&self) -> Result<Option<f32>, StoreError> { + self.env.load_ratio().map_err(|e| e.into()) + } + + /// Sets the size of the memory map to use for the environment. + /// + /// This can be used to resize the map when the environment is already open. You can + /// also use `Rkv::environment_builder()` to set the map size during the `Rkv` + /// initialization. + /// + /// Note: + /// + /// * No active transactions allowed when performing resizing in this process. It's up + /// to the consumer to enforce that. + /// + /// * The size should be a multiple of the OS page size. Any attempt to set a size + /// smaller than the space already consumed by the environment will be silently + /// changed to the current size of the used space. + /// + /// * In the multi-process case, once a process resizes the map, other processes need + /// to either re-open the environment, or call set_map_size with size 0 to update + /// the environment. Otherwise, new transaction creation will fail with + /// `LmdbError::MapResized`. + pub fn set_map_size(&self, size: usize) -> Result<(), StoreError> { + self.env.set_map_size(size).map_err(Into::into) + } + + /// Closes this environment and optionally deletes all its files from disk. Doesn't + /// delete the folder used when opening the environment. + pub fn close(self, options: CloseOptions) -> Result<(), CloseError> { + let files = self.env.get_files_on_disk(); + drop(self); + + if options.delete { + for file in files { + fs::remove_file(file)?; + } + } + + Ok(()) + } +} diff --git a/third_party/rust/rkv/src/error.rs b/third_party/rust/rkv/src/error.rs new file mode 100644 index 0000000000..f50dda73e4 --- /dev/null +++ b/third_party/rust/rkv/src/error.rs @@ -0,0 +1,150 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{io, path::PathBuf, sync, thread, thread::ThreadId}; + +use thiserror::Error; + +pub use crate::backend::SafeModeError; +use crate::value::Type; + +#[derive(Debug, Error)] +pub enum DataError { + #[error("unknown type tag: {0}")] + UnknownType(u8), + + #[error("unexpected type tag: expected {expected}, got {actual}")] + UnexpectedType { expected: Type, actual: Type }, + + #[error("empty data; expected tag")] + Empty, + + #[error("invalid value for type {value_type}: {err}")] + DecodingError { + value_type: Type, + err: Box<bincode::ErrorKind>, + }, + + #[error("couldn't encode value: {0}")] + EncodingError(#[from] Box<bincode::ErrorKind>), + + #[error("invalid uuid bytes")] + InvalidUuid, +} + +#[derive(Debug, Error)] +pub enum StoreError { + #[error("manager poisoned")] + ManagerPoisonError, + + #[error("database corrupted")] + DatabaseCorrupted, + + #[error("key/value pair not found")] + KeyValuePairNotFound, + + #[error("unsupported size of key/DB name/data")] + KeyValuePairBadSize, + + #[error("file is not a valid database")] + FileInvalid, + + #[error("environment mapsize reached")] + MapFull, + + #[error("environment maxdbs reached")] + DbsFull, + + #[error("environment maxreaders reached")] + ReadersFull, + + #[error("I/O error: {0:?}")] + IoError(#[from] io::Error), + + #[error("environment path does not exist or not the right type: {0:?}")] + UnsuitableEnvironmentPath(PathBuf), + + #[error("data error: {0:?}")] + DataError(#[from] DataError), + + #[cfg(feature = "lmdb")] + #[error("lmdb backend error: {0}")] + LmdbError(lmdb::Error), + + #[error("safe mode backend error: {0}")] + SafeModeError(SafeModeError), + + #[error("read transaction already exists in thread {0:?}")] + ReadTransactionAlreadyExists(ThreadId), + + #[error("attempted to open DB during transaction in thread {0:?}")] + OpenAttemptedDuringTransaction(ThreadId), +} + +impl StoreError { + pub fn open_during_transaction() -> StoreError { + StoreError::OpenAttemptedDuringTransaction(thread::current().id()) + } + + pub fn read_transaction_already_exists() -> StoreError { + StoreError::ReadTransactionAlreadyExists(thread::current().id()) + } +} + +impl<T> From<sync::PoisonError<T>> for StoreError { + fn from(_: sync::PoisonError<T>) -> StoreError { + StoreError::ManagerPoisonError + } +} + +#[derive(Debug, Error)] +pub enum CloseError { + #[error("manager poisoned")] + ManagerPoisonError, + + #[error("close attempted while manager has an environment still open")] + EnvironmentStillOpen, + + #[error("close attempted while an environment not known to the manager is still open")] + UnknownEnvironmentStillOpen, + + #[error("I/O error: {0:?}")] + IoError(#[from] io::Error), +} + +impl<T> From<sync::PoisonError<T>> for CloseError { + fn from(_: sync::PoisonError<T>) -> CloseError { + CloseError::ManagerPoisonError + } +} + +#[derive(Debug, Error)] +pub enum MigrateError { + #[error("store error: {0}")] + StoreError(#[from] StoreError), + + #[error("close error: {0}")] + CloseError(#[from] CloseError), + + #[error("manager poisoned")] + ManagerPoisonError, + + #[error("source is empty")] + SourceEmpty, + + #[error("destination is not empty")] + DestinationNotEmpty, +} + +impl<T> From<sync::PoisonError<T>> for MigrateError { + fn from(_: sync::PoisonError<T>) -> MigrateError { + MigrateError::ManagerPoisonError + } +} diff --git a/third_party/rust/rkv/src/helpers.rs b/third_party/rust/rkv/src/helpers.rs new file mode 100644 index 0000000000..89dcdf8dd7 --- /dev/null +++ b/third_party/rust/rkv/src/helpers.rs @@ -0,0 +1,43 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + io, + path::{Path, PathBuf}, +}; + +use url::Url; + +use crate::{error::StoreError, value::Value}; + +pub(crate) fn read_transform(value: Result<&[u8], StoreError>) -> Result<Value, StoreError> { + match value { + Ok(bytes) => Value::from_tagged_slice(bytes).map_err(StoreError::DataError), + Err(e) => Err(e), + } +} + +// Workaround the UNC path on Windows, see https://github.com/rust-lang/rust/issues/42869. +// Otherwise, `Env::from_builder()` will panic with error_no(123). +pub(crate) fn canonicalize_path<'p, P>(path: P) -> io::Result<PathBuf> +where + P: Into<&'p Path>, +{ + let canonical = path.into().canonicalize()?; + + Ok(if cfg!(target_os = "windows") { + let map_err = |_| io::Error::new(io::ErrorKind::Other, "path canonicalization error"); + Url::from_file_path(&canonical) + .and_then(|url| url.to_file_path()) + .map_err(map_err)? + } else { + canonical + }) +} diff --git a/third_party/rust/rkv/src/lib.rs b/third_party/rust/rkv/src/lib.rs new file mode 100644 index 0000000000..0c8951e191 --- /dev/null +++ b/third_party/rust/rkv/src/lib.rs @@ -0,0 +1,236 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![allow(clippy::from_over_into)] // TODO: `Into` implementations in [safe/lmdb]/flags.rs + +//! A simple, humane, typed key-value storage solution. It supports multiple backend +//! engines with varying guarantees, such as [LMDB](http://www.lmdb.tech/doc/) for +//! performance, or "SafeMode" for reliability. +//! +//! It aims to achieve the following: +//! +//! - Avoid sharp edges (e.g., obscure error codes for common situations). +//! - Correctly restrict access to one handle per process via a +//! [Manager](struct.Manager.html). +//! - Use Rust's type system to make single-typed key stores safe and ergonomic. +//! - Encode and decode values via [bincode](https://docs.rs/bincode/)/[serde](https://docs.rs/serde/) +//! and type tags, achieving platform-independent storage and input/output flexibility. +//! +//! It exposes these primary abstractions: +//! +//! - [Manager](struct.Manager.html): a singleton that controls access to environments +//! - [Rkv](struct.Rkv.html): an environment contains a set of key/value databases +//! - [SingleStore](store/single/struct.SingleStore.html): a database contains a set of +//! key/value pairs +//! +//! Keys can be anything that implements `AsRef<[u8]>` or integers +//! (when accessing an [IntegerStore](store/integer/struct.IntegerStore.html)). +//! +//! Values can be any of the types defined by the [Value](value/enum.Value.html) enum, +//! including: +//! +//! - booleans (`Value::Bool`) +//! - integers (`Value::I64`, `Value::U64`) +//! - floats (`Value::F64`) +//! - strings (`Value::Str`) +//! - blobs (`Value::Blob`) +//! +//! See [Value](value/enum.Value.html) for the complete list of supported types. +//! +//! ## Basic Usage +//! ``` +//! use rkv::{Manager, Rkv, SingleStore, Value, StoreOptions}; +//! use rkv::backend::{SafeMode, SafeModeEnvironment}; +//! use std::fs; +//! use tempfile::Builder; +//! +//! // First determine the path to the environment, which is represented on disk as a +//! // directory containing two files: +//! // +//! // * a data file containing the key/value stores +//! // * a lock file containing metadata about current transactions +//! // +//! // In this example, we use the `tempfile` crate to create the directory. +//! // +//! let root = Builder::new().prefix("simple-db").tempdir().unwrap(); +//! fs::create_dir_all(root.path()).unwrap(); +//! let path = root.path(); +//! +//! // The `Manager` enforces that each process opens the same environment at most once by +//! // caching a handle to each environment that it opens. Use it to retrieve the handle +//! // to an opened environment—or create one if it hasn't already been opened: +//! let mut manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap(); +//! let created_arc = manager.get_or_create(path, Rkv::new::<SafeMode>).unwrap(); +//! let env = created_arc.read().unwrap(); +//! +//! // Then you can use the environment handle to get a handle to a datastore: +//! let store = env.open_single("mydb", StoreOptions::create()).unwrap(); +//! +//! { +//! // Use a write transaction to mutate the store via a `Writer`. There can be only +//! // one writer for a given environment, so opening a second one will block until +//! // the first completes. +//! let mut writer = env.write().unwrap(); +//! +//! // Keys are `AsRef<[u8]>`, while values are `Value` enum instances. Use the `Blob` +//! // variant to store arbitrary collections of bytes. Putting data returns a +//! // `Result<(), StoreError>`, where StoreError is an enum identifying the reason +//! // for a failure. +//! store.put(&mut writer, "int", &Value::I64(1234)).unwrap(); +//! store.put(&mut writer, "uint", &Value::U64(1234_u64)).unwrap(); +//! store.put(&mut writer, "float", &Value::F64(1234.0.into())).unwrap(); +//! store.put(&mut writer, "instant", &Value::Instant(1528318073700)).unwrap(); +//! store.put(&mut writer, "boolean", &Value::Bool(true)).unwrap(); +//! store.put(&mut writer, "string", &Value::Str("Héllo, wörld!")).unwrap(); +//! store.put(&mut writer, "json", &Value::Json(r#"{"foo":"bar", "number": 1}"#)).unwrap(); +//! store.put(&mut writer, "blob", &Value::Blob(b"blob")).unwrap(); +//! +//! // You must commit a write transaction before the writer goes out of scope, or the +//! // transaction will abort and the data won't persist. +//! writer.commit().unwrap(); +//! } +//! +//! { +//! // Use a read transaction to query the store via a `Reader`. There can be multiple +//! // concurrent readers for a store, and readers never block on a writer nor other +//! // readers. +//! let reader = env.read().expect("reader"); +//! +//! // Keys are `AsRef<u8>`, and the return value is `Result<Option<Value>, StoreError>`. +//! println!("Get int {:?}", store.get(&reader, "int").unwrap()); +//! println!("Get uint {:?}", store.get(&reader, "uint").unwrap()); +//! println!("Get float {:?}", store.get(&reader, "float").unwrap()); +//! println!("Get instant {:?}", store.get(&reader, "instant").unwrap()); +//! println!("Get boolean {:?}", store.get(&reader, "boolean").unwrap()); +//! println!("Get string {:?}", store.get(&reader, "string").unwrap()); +//! println!("Get json {:?}", store.get(&reader, "json").unwrap()); +//! println!("Get blob {:?}", store.get(&reader, "blob").unwrap()); +//! +//! // Retrieving a non-existent value returns `Ok(None)`. +//! println!("Get non-existent value {:?}", store.get(&reader, "non-existent").unwrap()); +//! +//! // A read transaction will automatically close once the reader goes out of scope, +//! // so isn't necessary to close it explicitly, although you can do so by calling +//! // `Reader.abort()`. +//! } +//! +//! { +//! // Aborting a write transaction rolls back the change(s). +//! let mut writer = env.write().unwrap(); +//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); +//! writer.abort(); +//! let reader = env.read().expect("reader"); +//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap()); +//! } +//! +//! { +//! // Explicitly aborting a transaction is not required unless an early abort is +//! // desired, since both read and write transactions will implicitly be aborted once +//! // they go out of scope. +//! { +//! let mut writer = env.write().unwrap(); +//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); +//! } +//! let reader = env.read().expect("reader"); +//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap()); +//! } +//! +//! { +//! // Deleting a key/value pair also requires a write transaction. +//! let mut writer = env.write().unwrap(); +//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); +//! store.put(&mut writer, "bar", &Value::Str("baz")).unwrap(); +//! store.delete(&mut writer, "foo").unwrap(); +//! +//! // A write transaction also supports reading, and the version of the store that it +//! // reads includes the changes it has made regardless of the commit state of that +//! // transaction. + +//! // In the code above, "foo" and "bar" were put into the store, then "foo" was +//! // deleted so only "bar" will return a result when the database is queried via the +//! // writer. +//! println!("It should be None! ({:?})", store.get(&writer, "foo").unwrap()); +//! println!("Get bar ({:?})", store.get(&writer, "bar").unwrap()); +//! +//! // But a reader won't see that change until the write transaction is committed. +//! { +//! let reader = env.read().expect("reader"); +//! println!("Get foo {:?}", store.get(&reader, "foo").unwrap()); +//! println!("Get bar {:?}", store.get(&reader, "bar").unwrap()); +//! } +//! writer.commit().unwrap(); +//! { +//! let reader = env.read().expect("reader"); +//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap()); +//! println!("Get bar {:?}", store.get(&reader, "bar").unwrap()); +//! } +//! +//! // Committing a transaction consumes the writer, preventing you from reusing it by +//! // failing at compile time with an error. This line would report "error[E0382]: +//! // borrow of moved value: `writer`". +//! // store.put(&mut writer, "baz", &Value::Str("buz")).unwrap(); +//! } +//! +//! { +//! // Clearing all the entries in the store with a write transaction. +//! { +//! let mut writer = env.write().unwrap(); +//! store.put(&mut writer, "foo", &Value::Str("bar")).unwrap(); +//! store.put(&mut writer, "bar", &Value::Str("baz")).unwrap(); +//! writer.commit().unwrap(); +//! } +//! +//! { +//! let mut writer = env.write().unwrap(); +//! store.clear(&mut writer).unwrap(); +//! writer.commit().unwrap(); +//! } +//! +//! { +//! let reader = env.read().expect("reader"); +//! println!("It should be None! ({:?})", store.get(&reader, "foo").unwrap()); +//! println!("It should be None! ({:?})", store.get(&reader, "bar").unwrap()); +//! } +//! +//! } +//! +//! ``` + +mod env; +mod error; +mod helpers; +mod manager; +mod readwrite; + +pub mod backend; +#[cfg(feature = "lmdb")] +pub mod migrator; +pub mod store; +pub mod value; + +pub use backend::{DatabaseFlags, EnvironmentFlags, WriteFlags}; +pub use env::Rkv; +pub use error::{DataError, MigrateError, StoreError}; +pub use manager::Manager; +#[cfg(feature = "lmdb")] +pub use migrator::Migrator; +pub use readwrite::{Readable, Reader, Writer}; +pub use store::{keys::EncodableKey, single::SingleStore, CloseOptions, Options as StoreOptions}; +pub use value::{OwnedValue, Value}; + +#[cfg(feature = "db-dup-sort")] +pub use store::multi::MultiStore; + +#[cfg(feature = "db-int-key")] +pub use store::integer::IntegerStore; +#[cfg(feature = "db-int-key")] +pub use store::keys::PrimitiveInt; + +#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))] +pub use store::integermulti::MultiIntegerStore; diff --git a/third_party/rust/rkv/src/manager.rs b/third_party/rust/rkv/src/manager.rs new file mode 100644 index 0000000000..045564497c --- /dev/null +++ b/third_party/rust/rkv/src/manager.rs @@ -0,0 +1,247 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{ + collections::{btree_map::Entry, BTreeMap}, + os::raw::c_uint, + path::{Path, PathBuf}, + result, + sync::{Arc, RwLock}, +}; + +use lazy_static::lazy_static; + +#[cfg(feature = "lmdb")] +use crate::backend::LmdbEnvironment; +use crate::{ + backend::{BackendEnvironment, BackendEnvironmentBuilder, SafeModeEnvironment}, + error::{CloseError, StoreError}, + helpers::canonicalize_path, + store::CloseOptions, + Rkv, +}; + +type Result<T> = result::Result<T, StoreError>; +type CloseResult<T> = result::Result<T, CloseError>; +type SharedRkv<E> = Arc<RwLock<Rkv<E>>>; + +#[cfg(feature = "lmdb")] +lazy_static! { + static ref MANAGER_LMDB: RwLock<Manager<LmdbEnvironment>> = RwLock::new(Manager::new()); +} + +lazy_static! { + static ref MANAGER_SAFE_MODE: RwLock<Manager<SafeModeEnvironment>> = + RwLock::new(Manager::new()); +} + +/// A process is only permitted to have one open handle to each Rkv environment. This +/// manager exists to enforce that constraint: don't open environments directly. +/// +/// By default, path canonicalization is enabled for identifying RKV instances. This +/// is true by default, because it helps enforce the constraints guaranteed by +/// this manager. However, path canonicalization might crash in some fringe +/// circumstances, so the `no-canonicalize-path` feature offers the possibility of +/// disabling it. See: https://bugzilla.mozilla.org/show_bug.cgi?id=1531887 +/// +/// When path canonicalization is disabled, you *must* ensure an RKV environment is +/// always created or retrieved with the same path. +pub struct Manager<E> { + environments: BTreeMap<PathBuf, SharedRkv<E>>, +} + +impl<'e, E> Manager<E> +where + E: BackendEnvironment<'e>, +{ + fn new() -> Manager<E> { + Manager { + environments: Default::default(), + } + } + + /// Return the open env at `path`, returning `None` if it has not already been opened. + pub fn get<'p, P>(&self, path: P) -> Result<Option<SharedRkv<E>>> + where + P: Into<&'p Path>, + { + let canonical = if cfg!(feature = "no-canonicalize-path") { + path.into().to_path_buf() + } else { + canonicalize_path(path)? + }; + Ok(self.environments.get(&canonical).cloned()) + } + + /// Return the open env at `path`, or create it by calling `f`. + pub fn get_or_create<'p, F, P>(&mut self, path: P, f: F) -> Result<SharedRkv<E>> + where + F: FnOnce(&Path) -> Result<Rkv<E>>, + P: Into<&'p Path>, + { + let canonical = if cfg!(feature = "no-canonicalize-path") { + path.into().to_path_buf() + } else { + canonicalize_path(path)? + }; + Ok(match self.environments.entry(canonical) { + Entry::Occupied(e) => e.get().clone(), + Entry::Vacant(e) => { + let k = Arc::new(RwLock::new(f(e.key().as_path())?)); + e.insert(k).clone() + } + }) + } + + /// Return the open env at `path` with `capacity`, or create it by calling `f`. + pub fn get_or_create_with_capacity<'p, F, P>( + &mut self, + path: P, + capacity: c_uint, + f: F, + ) -> Result<SharedRkv<E>> + where + F: FnOnce(&Path, c_uint) -> Result<Rkv<E>>, + P: Into<&'p Path>, + { + let canonical = if cfg!(feature = "no-canonicalize-path") { + path.into().to_path_buf() + } else { + canonicalize_path(path)? + }; + Ok(match self.environments.entry(canonical) { + Entry::Occupied(e) => e.get().clone(), + Entry::Vacant(e) => { + let k = Arc::new(RwLock::new(f(e.key().as_path(), capacity)?)); + e.insert(k).clone() + } + }) + } + + /// Return a new Rkv environment from the builder, or create it by calling `f`. + pub fn get_or_create_from_builder<'p, F, P, B>( + &mut self, + path: P, + builder: B, + f: F, + ) -> Result<SharedRkv<E>> + where + F: FnOnce(&Path, B) -> Result<Rkv<E>>, + P: Into<&'p Path>, + B: BackendEnvironmentBuilder<'e, Environment = E>, + { + let canonical = if cfg!(feature = "no-canonicalize-path") { + path.into().to_path_buf() + } else { + canonicalize_path(path)? + }; + Ok(match self.environments.entry(canonical) { + Entry::Occupied(e) => e.get().clone(), + Entry::Vacant(e) => { + let k = Arc::new(RwLock::new(f(e.key().as_path(), builder)?)); + e.insert(k).clone() + } + }) + } + + /// Tries to close the specified environment. + /// Returns an error when other users of this environment still exist. + pub fn try_close<'p, P>(&mut self, path: P, options: CloseOptions) -> CloseResult<()> + where + P: Into<&'p Path>, + { + let canonical = if cfg!(feature = "no-canonicalize-path") { + path.into().to_path_buf() + } else { + canonicalize_path(path)? + }; + match self.environments.entry(canonical) { + Entry::Vacant(_) => Ok(()), + Entry::Occupied(e) if Arc::strong_count(e.get()) > 1 => { + Err(CloseError::EnvironmentStillOpen) + } + Entry::Occupied(e) => { + let env = Arc::try_unwrap(e.remove()) + .map_err(|_| CloseError::UnknownEnvironmentStillOpen)?; + env.into_inner()?.close(options)?; + Ok(()) + } + } + } +} + +#[cfg(feature = "lmdb")] +impl Manager<LmdbEnvironment> { + pub fn singleton() -> &'static RwLock<Manager<LmdbEnvironment>> { + &MANAGER_LMDB + } +} + +impl Manager<SafeModeEnvironment> { + pub fn singleton() -> &'static RwLock<Manager<SafeModeEnvironment>> { + &MANAGER_SAFE_MODE + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + use std::fs; + + use tempfile::Builder; + + #[cfg(feature = "lmdb")] + use backend::Lmdb; + + /// Test that one can mutate managed Rkv instances in surprising ways. + #[cfg(feature = "lmdb")] + #[test] + fn test_mutate_managed_rkv() { + let mut manager = Manager::<LmdbEnvironment>::new(); + + let root1 = Builder::new() + .prefix("test_mutate_managed_rkv_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root1.path()).expect("dir created"); + let path1 = root1.path(); + let arc = manager + .get_or_create(path1, Rkv::new::<Lmdb>) + .expect("created"); + + // Arc<RwLock<>> has interior mutability, so we can replace arc's Rkv instance with a new + // instance that has a different path. + let root2 = Builder::new() + .prefix("test_mutate_managed_rkv_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root2.path()).expect("dir created"); + let path2 = root2.path(); + { + let mut rkv = arc.write().expect("guard"); + let rkv2 = Rkv::new::<Lmdb>(path2).expect("Rkv"); + *rkv = rkv2; + } + + // Arc now has a different internal Rkv with path2, but it's still mapped to path1 in + // manager, so its pointer is equal to a new Arc for path1. + let path1_arc = manager.get(path1).expect("success").expect("existed"); + assert!(Arc::ptr_eq(&path1_arc, &arc)); + + // Meanwhile, a new Arc for path2 has a different pointer, even though its Rkv's path is + // the same as arc's current path. + let path2_arc = manager + .get_or_create(path2, Rkv::new::<Lmdb>) + .expect("success"); + assert!(!Arc::ptr_eq(&path2_arc, &arc)); + } +} diff --git a/third_party/rust/rkv/src/migrator.rs b/third_party/rust/rkv/src/migrator.rs new file mode 100644 index 0000000000..6dc8d60371 --- /dev/null +++ b/third_party/rust/rkv/src/migrator.rs @@ -0,0 +1,179 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +//! A simple utility for migrating data from one RVK environment to another. Notably, this +//! tool can migrate data from an enviroment created with a different backend than the +//! current RKV consumer (e.g from Lmdb to SafeMode). +//! +//! The utility doesn't support migrating between 32-bit and 64-bit LMDB environments yet, +//! see `arch_migrator` if this is needed. However, this utility is ultimately intended to +//! handle all possible migrations. +//! +//! The destination environment should be empty of data, otherwise an error is returned. +//! +//! There are 3 versions of the migration methods: +//! * `migrate_<src>_to_<dst>`, where `<src>` and `<dst>` are the source and destination +//! environment types. You're responsive with opening both these environments, handling +//! all errors, and performing any cleanup if necessary. +//! * `open_and_migrate_<src>_to_<dst>`, which is similar the the above, but automatically +//! attempts to open the source environment and delete all of its supporting files if +//! there's no other environment open at that path. You're still responsible with +//! handling all errors. +//! * `easy_migrate_<src>_to_<dst>` which is similar to the above, but ignores the +//! migration and doesn't delete any files if the source environment is invalid +//! (corrupted), unavailable (path not accessible or incompatible with configuration), +//! or empty (database has no records). +//! +//! The tool currently has these limitations: +//! +//! 1. It doesn't support migration from environments created with +//! `EnvironmentFlags::NO_SUB_DIR`. To migrate such an environment, create a temporary +//! directory, copy the environment's data files in the temporary directory, then +//! migrate the temporary directory as the source environment. +//! 2. It doesn't support migration from databases created with DatabaseFlags::DUP_SORT` +//! (with or without `DatabaseFlags::DUP_FIXED`) nor with `DatabaseFlags::INTEGER_KEY`. +//! This effectively means that migration is limited to `SingleStore`s. +//! 3. It doesn't allow for existing data in the destination environment, which means that +//! it cannot overwrite nor append data. + +use crate::{ + backend::{LmdbEnvironment, SafeModeEnvironment}, + error::MigrateError, + Rkv, StoreOptions, +}; + +pub use crate::backend::{LmdbArchMigrateError, LmdbArchMigrateResult, LmdbArchMigrator}; + +// FIXME: should parametrize this instead. + +macro_rules! fn_migrator { + ($name:tt, $src_env:ty, $dst_env:ty) => { + /// Migrate all data in all of databases from the source environment to the destination + /// environment. This includes all key/value pairs in the main database that aren't + /// metadata about subdatabases and all key/value pairs in all subdatabases. + /// + /// Other backend-specific metadata such as map size or maximum databases left intact on + /// the given environments. + /// + /// The destination environment should be empty of data, otherwise an error is returned. + pub fn $name<S, D>(src_env: S, dst_env: D) -> Result<(), MigrateError> + where + S: std::ops::Deref<Target = Rkv<$src_env>>, + D: std::ops::Deref<Target = Rkv<$dst_env>>, + { + let src_dbs = src_env.get_dbs().unwrap(); + if src_dbs.is_empty() { + return Err(MigrateError::SourceEmpty); + } + let dst_dbs = dst_env.get_dbs().unwrap(); + if !dst_dbs.is_empty() { + return Err(MigrateError::DestinationNotEmpty); + } + for name in src_dbs { + let src_store = src_env.open_single(name.as_deref(), StoreOptions::default())?; + let dst_store = dst_env.open_single(name.as_deref(), StoreOptions::create())?; + let reader = src_env.read()?; + let mut writer = dst_env.write()?; + let mut iter = src_store.iter_start(&reader)?; + while let Some(Ok((key, value))) = iter.next() { + dst_store.put(&mut writer, key, &value).expect("wrote"); + } + writer.commit()?; + } + Ok(()) + } + }; + + (open $migrate:tt, $name:tt, $builder:tt, $src_env:ty, $dst_env:ty) => { + /// Same as the the `migrate_x_to_y` migration method above, but automatically attempts + /// to open the source environment. Finally, deletes all of its supporting files if + /// there's no other environment open at that path and the migration succeeded. + pub fn $name<F, D>(path: &std::path::Path, build: F, dst_env: D) -> Result<(), MigrateError> + where + F: FnOnce(crate::backend::$builder) -> crate::backend::$builder, + D: std::ops::Deref<Target = Rkv<$dst_env>>, + { + use crate::{backend::*, CloseOptions}; + + let mut manager = crate::Manager::<$src_env>::singleton().write()?; + let mut builder = Rkv::<$src_env>::environment_builder::<$builder>(); + builder.set_max_dbs(crate::env::DEFAULT_MAX_DBS); + builder = build(builder); + + let src_env = + manager.get_or_create_from_builder(path, builder, Rkv::from_builder::<$builder>)?; + Migrator::$migrate(src_env.read()?, dst_env)?; + + drop(src_env); + manager.try_close(path, CloseOptions::delete_files_on_disk())?; + + Ok(()) + } + }; + + (easy $migrate:tt, $name:tt, $src_env:ty, $dst_env:ty) => { + /// Same as the `open_and_migrate_x_to_y` migration method above, but ignores the + /// migration and doesn't delete any files if the following conditions apply: + /// - Source environment is invalid/corrupted, unavailable, or empty. + /// - Destination environment is not empty. + /// Use this instead of the other migration methods if: + /// - You're not concerned by throwing away old data and starting fresh with a new store. + /// - You'll never want to overwrite data in the new store from the old store. + pub fn $name<D>(path: &std::path::Path, dst_env: D) -> Result<(), MigrateError> + where + D: std::ops::Deref<Target = Rkv<$dst_env>>, + { + match Migrator::$migrate(path, |builder| builder, dst_env) { + // Source environment is an invalid file or corrupted database. + Err(crate::MigrateError::StoreError(crate::StoreError::FileInvalid)) => Ok(()), + Err(crate::MigrateError::StoreError(crate::StoreError::DatabaseCorrupted)) => { + Ok(()) + } + // Path not accessible. + Err(crate::MigrateError::StoreError(crate::StoreError::IoError(_))) => Ok(()), + // Path accessible but incompatible for configuration. + Err(crate::MigrateError::StoreError( + crate::StoreError::UnsuitableEnvironmentPath(_), + )) => Ok(()), + // Couldn't close source environment and delete files on disk (e.g. other stores still open). + Err(crate::MigrateError::CloseError(_)) => Ok(()), + // Nothing to migrate. + Err(crate::MigrateError::SourceEmpty) => Ok(()), + // Migrating would overwrite. + Err(crate::MigrateError::DestinationNotEmpty) => Ok(()), + result => result, + }?; + + Ok(()) + } + }; +} + +macro_rules! fns_migrator { + ($src:tt, $dst:tt) => { + paste::item! { + fns_migrator!([<migrate_ $src _to_ $dst>], $src, $dst); + fns_migrator!([<migrate_ $dst _to_ $src>], $dst, $src); + } + }; + ($name:tt, $src:tt, $dst:tt) => { + paste::item! { + fn_migrator!($name, [<$src:camel Environment>], [<$dst:camel Environment>]); + fn_migrator!(open $name, [<open_and_ $name>], [<$src:camel>], [<$src:camel Environment>], [<$dst:camel Environment>]); + fn_migrator!(easy [<open_and_ $name>], [<easy_ $name>], [<$src:camel Environment>], [<$dst:camel Environment>]); + } + }; +} + +pub struct Migrator; + +impl Migrator { + fns_migrator!(lmdb, safe_mode); +} diff --git a/third_party/rust/rkv/src/readwrite.rs b/third_party/rust/rkv/src/readwrite.rs new file mode 100644 index 0000000000..38b4746210 --- /dev/null +++ b/third_party/rust/rkv/src/readwrite.rs @@ -0,0 +1,154 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::{ + backend::{ + BackendDatabase, BackendRoCursor, BackendRoCursorTransaction, BackendRoTransaction, + BackendRwCursorTransaction, BackendRwTransaction, + }, + error::StoreError, + helpers::read_transform, + value::Value, +}; + +pub struct Reader<T>(T); +pub struct Writer<T>(T); + +pub trait Readable<'r> { + type Database: BackendDatabase; + type RoCursor: BackendRoCursor<'r>; + + fn get<K>(&'r self, db: &Self::Database, k: &K) -> Result<Option<Value<'r>>, StoreError> + where + K: AsRef<[u8]>; + + fn open_ro_cursor(&'r self, db: &Self::Database) -> Result<Self::RoCursor, StoreError>; +} + +impl<'r, T> Readable<'r> for Reader<T> +where + T: BackendRoCursorTransaction<'r>, +{ + type Database = T::Database; + type RoCursor = T::RoCursor; + + fn get<K>(&'r self, db: &T::Database, k: &K) -> Result<Option<Value<'r>>, StoreError> + where + K: AsRef<[u8]>, + { + let bytes = self.0.get(db, k.as_ref()).map_err(|e| e.into()); + match read_transform(bytes).map(Some) { + Err(StoreError::KeyValuePairNotFound) => Ok(None), + result => result, + } + } + + fn open_ro_cursor(&'r self, db: &T::Database) -> Result<T::RoCursor, StoreError> { + self.0.open_ro_cursor(db).map_err(|e| e.into()) + } +} + +impl<T> Reader<T> { + pub(crate) fn new(txn: T) -> Reader<T> { + Reader(txn) + } +} + +impl<T> Reader<T> +where + T: BackendRoTransaction, +{ + pub fn abort(self) { + self.0.abort(); + } +} + +impl<'r, T> Readable<'r> for Writer<T> +where + T: BackendRwCursorTransaction<'r>, +{ + type Database = T::Database; + type RoCursor = T::RoCursor; + + fn get<K>(&'r self, db: &T::Database, k: &K) -> Result<Option<Value<'r>>, StoreError> + where + K: AsRef<[u8]>, + { + let bytes = self.0.get(db, k.as_ref()).map_err(|e| e.into()); + match read_transform(bytes).map(Some) { + Err(StoreError::KeyValuePairNotFound) => Ok(None), + result => result, + } + } + + fn open_ro_cursor(&'r self, db: &T::Database) -> Result<T::RoCursor, StoreError> { + self.0.open_ro_cursor(db).map_err(|e| e.into()) + } +} + +impl<T> Writer<T> { + pub(crate) fn new(txn: T) -> Writer<T> { + Writer(txn) + } +} + +impl<T> Writer<T> +where + T: BackendRwTransaction, +{ + pub fn commit(self) -> Result<(), StoreError> { + self.0.commit().map_err(|e| e.into()) + } + + pub fn abort(self) { + self.0.abort(); + } + + pub(crate) fn put<K>( + &mut self, + db: &T::Database, + k: &K, + v: &Value, + flags: T::Flags, + ) -> Result<(), StoreError> + where + K: AsRef<[u8]>, + { + // TODO: don't allocate twice. + self.0 + .put(db, k.as_ref(), &v.to_bytes()?, flags) + .map_err(|e| e.into()) + } + + #[cfg(not(feature = "db-dup-sort"))] + pub(crate) fn delete<K>(&mut self, db: &T::Database, k: &K) -> Result<(), StoreError> + where + K: AsRef<[u8]>, + { + self.0.del(db, k.as_ref()).map_err(|e| e.into()) + } + + #[cfg(feature = "db-dup-sort")] + pub(crate) fn delete<K>( + &mut self, + db: &T::Database, + k: &K, + v: Option<&[u8]>, + ) -> Result<(), StoreError> + where + K: AsRef<[u8]>, + { + self.0.del(db, k.as_ref(), v).map_err(|e| e.into()) + } + + pub(crate) fn clear(&mut self, db: &T::Database) -> Result<(), StoreError> { + self.0.clear_db(db).map_err(|e| e.into()) + } +} diff --git a/third_party/rust/rkv/src/store.rs b/third_party/rust/rkv/src/store.rs new file mode 100644 index 0000000000..c8ac2c583f --- /dev/null +++ b/third_party/rust/rkv/src/store.rs @@ -0,0 +1,52 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +pub mod keys; +pub mod single; + +#[cfg(feature = "db-dup-sort")] +pub mod multi; + +#[cfg(feature = "db-int-key")] +pub mod integer; + +#[cfg(all(feature = "db-dup-sort", feature = "db-int-key"))] +pub mod integermulti; + +use crate::backend::BackendDatabaseFlags; + +#[derive(Default, Debug, Copy, Clone)] +pub struct Options<F> { + pub create: bool, + pub flags: F, +} + +impl<F> Options<F> +where + F: BackendDatabaseFlags, +{ + pub fn create() -> Options<F> { + Options { + create: true, + flags: F::empty(), + } + } +} + +#[derive(Default, Debug, Copy, Clone)] +pub struct CloseOptions { + pub delete: bool, +} + +impl CloseOptions { + pub fn delete_files_on_disk() -> CloseOptions { + CloseOptions { delete: true } + } +} diff --git a/third_party/rust/rkv/src/store/integer.rs b/third_party/rust/rkv/src/store/integer.rs new file mode 100644 index 0000000000..f150f09c5f --- /dev/null +++ b/third_party/rust/rkv/src/store/integer.rs @@ -0,0 +1,637 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::marker::PhantomData; + +use crate::{ + backend::{BackendDatabase, BackendRwTransaction}, + error::StoreError, + readwrite::{Readable, Writer}, + store::{ + keys::{Key, PrimitiveInt}, + single::SingleStore, + }, + value::Value, +}; + +type EmptyResult = Result<(), StoreError>; + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub struct IntegerStore<D, K> { + inner: SingleStore<D>, + phantom: PhantomData<K>, +} + +impl<D, K> IntegerStore<D, K> +where + D: BackendDatabase, + K: PrimitiveInt, +{ + pub(crate) fn new(db: D) -> IntegerStore<D, K> { + IntegerStore { + inner: SingleStore::new(db), + phantom: PhantomData, + } + } + + pub fn get<'r, R>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError> + where + R: Readable<'r, Database = D>, + { + self.inner.get(reader, Key::new(&k)?) + } + + pub fn put<T>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.put(writer, Key::new(&k)?, v) + } + + pub fn delete<T>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.delete(writer, Key::new(&k)?) + } + + pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.clear(writer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + use std::fs; + + use tempfile::Builder; + + #[test] + fn test_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + macro_rules! test_integer_keys { + ($type:ty, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + s.put(&mut writer, $key, &Value::Str("hello!")) + .expect("write"); + assert_eq!( + s.get(&writer, $key).expect("read"), + Some(Value::Str("hello!")) + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!( + s.get(&reader, $key).expect("read"), + Some(Value::Str("hello!")) + ); + }}; + } + + test_integer_keys!(u32, std::u32::MIN); + test_integer_keys!(u32, std::u32::MAX); + } + + #[test] + fn test_clear() { + let root = Builder::new() + .prefix("test_integer_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 3, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!"))); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), None); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_dup() { + let root = Builder::new() + .prefix("test_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("foo!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("bar!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!"))); + assert_eq!(s.get(&writer, 2).expect("read"), None); + assert_eq!(s.get(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), None); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_del() { + let root = Builder::new() + .prefix("test_integer_del") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("foo!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("bar!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!"))); + assert_eq!(s.get(&writer, 2).expect("read"), None); + assert_eq!(s.get(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1).expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 2).expect_err("not deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 2).expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 3).expect_err("not deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_persist() { + let root = Builder::new() + .prefix("test_integer_persist") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 3, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!"))); + writer.commit().expect("committed"); + } + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 2).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 3).expect("read"), Some(Value::Str("hello!"))); + } + } + + #[test] + fn test_intertwine_read_write() { + let root = Builder::new() + .prefix("test_integer_intertwine_read_write") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 2).expect("read"), None); + assert_eq!(s.get(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + let reader = k.read().expect("reader"); + let mut writer = k.write().expect("writer"); + + { + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + + { + s.put(&mut writer, 1, &Value::Str("goodbye!")) + .expect("write"); + s.put(&mut writer, 2, &Value::Str("goodbye!")) + .expect("write"); + s.put(&mut writer, 3, &Value::Str("goodbye!")) + .expect("write"); + assert_eq!( + s.get(&writer, 1).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&writer, 2).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&writer, 3).expect("read"), + Some(Value::Str("goodbye!")) + ); + writer.commit().expect("committed"); + } + + { + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!( + s.get(&writer, 2).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&writer, 3).expect("read"), + Some(Value::Str("goodbye!")) + ); + writer.commit().expect("committed"); + } + + { + let reader = k.write().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!( + s.get(&reader, 2).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&reader, 3).expect("read"), + Some(Value::Str("goodbye!")) + ); + reader.commit().expect("committed"); + } + } +} + +#[cfg(test)] +mod tests_safe { + use super::*; + use crate::*; + + use std::fs; + + use tempfile::Builder; + + #[test] + fn test_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + macro_rules! test_integer_keys { + ($type:ty, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + s.put(&mut writer, $key, &Value::Str("hello!")) + .expect("write"); + assert_eq!( + s.get(&writer, $key).expect("read"), + Some(Value::Str("hello!")) + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!( + s.get(&reader, $key).expect("read"), + Some(Value::Str("hello!")) + ); + }}; + } + + test_integer_keys!(u32, std::u32::MIN); + test_integer_keys!(u32, std::u32::MAX); + } + + #[test] + fn test_clear() { + let root = Builder::new() + .prefix("test_integer_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 3, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!"))); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), None); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_dup() { + let root = Builder::new() + .prefix("test_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("foo!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("bar!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!"))); + assert_eq!(s.get(&writer, 2).expect("read"), None); + assert_eq!(s.get(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), None); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_del() { + let root = Builder::new() + .prefix("test_integer_del") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("foo!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("bar!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("bar!"))); + assert_eq!(s.get(&writer, 2).expect("read"), None); + assert_eq!(s.get(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1).expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 2).expect_err("not deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 2).expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 3).expect_err("not deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_persist() { + let root = Builder::new() + .prefix("test_integer_persist") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 3, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 2).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 3).expect("read"), Some(Value::Str("hello!"))); + writer.commit().expect("committed"); + } + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 2).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 3).expect("read"), Some(Value::Str("hello!"))); + } + } + + #[test] + fn test_intertwine_read_write() { + let root = Builder::new() + .prefix("test_integer_intertwine_read_write") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&writer, 2).expect("read"), None); + assert_eq!(s.get(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + let reader = k.read().expect("reader"); + let mut writer = k.write().expect("writer"); + + { + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + + { + s.put(&mut writer, 1, &Value::Str("goodbye!")) + .expect("write"); + s.put(&mut writer, 2, &Value::Str("goodbye!")) + .expect("write"); + s.put(&mut writer, 3, &Value::Str("goodbye!")) + .expect("write"); + assert_eq!( + s.get(&writer, 1).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&writer, 2).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&writer, 3).expect("read"), + Some(Value::Str("goodbye!")) + ); + writer.commit().expect("committed"); + } + + { + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!(s.get(&reader, 2).expect("read"), None); + assert_eq!(s.get(&reader, 3).expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + assert_eq!(s.get(&writer, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!( + s.get(&writer, 2).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&writer, 3).expect("read"), + Some(Value::Str("goodbye!")) + ); + writer.commit().expect("committed"); + } + + { + let reader = k.write().expect("reader"); + assert_eq!(s.get(&reader, 1).expect("read"), Some(Value::Str("hello!"))); + assert_eq!( + s.get(&reader, 2).expect("read"), + Some(Value::Str("goodbye!")) + ); + assert_eq!( + s.get(&reader, 3).expect("read"), + Some(Value::Str("goodbye!")) + ); + reader.commit().expect("committed"); + } + } +} diff --git a/third_party/rust/rkv/src/store/integermulti.rs b/third_party/rust/rkv/src/store/integermulti.rs new file mode 100644 index 0000000000..71dd0aae04 --- /dev/null +++ b/third_party/rust/rkv/src/store/integermulti.rs @@ -0,0 +1,709 @@ +// Copyright 2018 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::marker::PhantomData; + +use crate::{ + backend::{BackendDatabase, BackendIter, BackendRoCursor, BackendRwTransaction}, + error::StoreError, + readwrite::{Readable, Writer}, + store::{ + keys::{Key, PrimitiveInt}, + multi::{Iter, MultiStore}, + }, + value::Value, +}; + +type EmptyResult = Result<(), StoreError>; + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub struct MultiIntegerStore<D, K> { + inner: MultiStore<D>, + phantom: PhantomData<K>, +} + +impl<D, K> MultiIntegerStore<D, K> +where + D: BackendDatabase, + K: PrimitiveInt, +{ + pub(crate) fn new(db: D) -> MultiIntegerStore<D, K> { + MultiIntegerStore { + inner: MultiStore::new(db), + phantom: PhantomData, + } + } + + pub fn get<'r, R, I, C>(&self, reader: &'r R, k: K) -> Result<Iter<'r, I>, StoreError> + where + R: Readable<'r, Database = D, RoCursor = C>, + I: BackendIter<'r>, + C: BackendRoCursor<'r, Iter = I>, + K: 'r, + { + self.inner.get(reader, Key::new(&k)?) + } + + pub fn get_first<'r, R>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError> + where + R: Readable<'r, Database = D>, + { + self.inner.get_first(reader, Key::new(&k)?) + } + + pub fn put<T>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.put(writer, Key::new(&k)?, v) + } + + pub fn put_with_flags<T>( + &self, + writer: &mut Writer<T>, + k: K, + v: &Value, + flags: T::Flags, + ) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.put_with_flags(writer, Key::new(&k)?, v, flags) + } + + pub fn delete_all<T>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.delete_all(writer, Key::new(&k)?) + } + + pub fn delete<T>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.delete(writer, Key::new(&k)?, v) + } + + pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + self.inner.clear(writer) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::*; + + use std::fs; + + use tempfile::Builder; + + #[test] + fn test_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + macro_rules! test_integer_keys { + ($type:ty, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + s.put(&mut writer, $key, &Value::Str("hello!")) + .expect("write"); + assert_eq!( + s.get_first(&writer, $key).expect("read"), + Some(Value::Str("hello!")) + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!( + s.get_first(&reader, $key).expect("read"), + Some(Value::Str("hello!")) + ); + }}; + } + + test_integer_keys!(u32, std::u32::MIN); + test_integer_keys!(u32, std::u32::MAX); + } + + #[test] + fn test_clear() { + let root = Builder::new() + .prefix("test_multi_integer_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + assert_eq!( + s.get_first(&writer, 1).expect("read"), + Some(Value::Str("hello!")) + ); + assert_eq!( + s.get_first(&writer, 2).expect("read"), + Some(Value::Str("hello!")) + ); + assert_eq!(s.get_first(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get_first(&reader, 1).expect("read"), None); + assert_eq!(s.get_first(&reader, 2).expect("read"), None); + assert_eq!(s.get_first(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_dup() { + let root = Builder::new() + .prefix("test_multi_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + assert_eq!( + s.get_first(&writer, 1).expect("read"), + Some(Value::Str("hello!")) + ); + assert_eq!(s.get_first(&writer, 2).expect("read"), None); + assert_eq!(s.get_first(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get_first(&reader, 1).expect("read"), None); + assert_eq!(s.get_first(&reader, 2).expect("read"), None); + assert_eq!(s.get_first(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_dup_2() { + let root = Builder::new() + .prefix("test_multi_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + + let mut iter = s.get(&writer, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_del() { + let root = Builder::new() + .prefix("test_multi_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + { + let mut iter = s.get(&writer, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello!")) + .expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello!")) + .expect_err("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello1!")) + .expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert!(iter.next().is_none()); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello1!")) + .expect_err("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_persist() { + let root = Builder::new() + .prefix("test_multi_integer_persist") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + { + let mut iter = s.get(&writer, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + writer.commit().expect("committed"); + } + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + } +} + +#[cfg(test)] +mod tests_safe { + use super::*; + use crate::*; + + use std::fs; + + use tempfile::Builder; + + #[test] + fn test_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + macro_rules! test_integer_keys { + ($type:ty, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + s.put(&mut writer, $key, &Value::Str("hello!")) + .expect("write"); + assert_eq!( + s.get_first(&writer, $key).expect("read"), + Some(Value::Str("hello!")) + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!( + s.get_first(&reader, $key).expect("read"), + Some(Value::Str("hello!")) + ); + }}; + } + + test_integer_keys!(u32, std::u32::MIN); + test_integer_keys!(u32, std::u32::MAX); + } + + #[test] + fn test_clear() { + let root = Builder::new() + .prefix("test_multi_integer_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + assert_eq!( + s.get_first(&writer, 1).expect("read"), + Some(Value::Str("hello!")) + ); + assert_eq!( + s.get_first(&writer, 2).expect("read"), + Some(Value::Str("hello!")) + ); + assert_eq!(s.get_first(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get_first(&reader, 1).expect("read"), None); + assert_eq!(s.get_first(&reader, 2).expect("read"), None); + assert_eq!(s.get_first(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_dup() { + let root = Builder::new() + .prefix("test_multi_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + assert_eq!( + s.get_first(&writer, 1).expect("read"), + Some(Value::Str("hello!")) + ); + assert_eq!(s.get_first(&writer, 2).expect("read"), None); + assert_eq!(s.get_first(&writer, 3).expect("read"), None); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s.get_first(&reader, 1).expect("read"), None); + assert_eq!(s.get_first(&reader, 2).expect("read"), None); + assert_eq!(s.get_first(&reader, 3).expect("read"), None); + } + } + + #[test] + fn test_dup_2() { + let root = Builder::new() + .prefix("test_multi_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + + let mut iter = s.get(&writer, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_del() { + let root = Builder::new() + .prefix("test_multi_integer_dup") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + { + let mut iter = s.get(&writer, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello!")) + .expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello!")) + .expect_err("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello1!")) + .expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert!(iter.next().is_none()); + } + + { + let mut writer = k.write().expect("writer"); + s.delete(&mut writer, 1, &Value::Str("hello1!")) + .expect_err("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert!(iter.next().is_none()); + } + } + + #[test] + fn test_persist() { + let root = Builder::new() + .prefix("test_multi_integer_persist") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + let mut writer = k.write().expect("writer"); + s.put(&mut writer, 1, &Value::Str("hello!")).expect("write"); + s.put(&mut writer, 1, &Value::Str("hello1!")) + .expect("write"); + s.put(&mut writer, 2, &Value::Str("hello!")).expect("write"); + { + let mut iter = s.get(&writer, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + writer.commit().expect("committed"); + } + + { + let k = Rkv::new::<backend::SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + let reader = k.read().expect("reader"); + let mut iter = s.get(&reader, 1).expect("read"); + assert_eq!( + iter.next().expect("first").expect("ok").1, + Value::Str("hello!") + ); + assert_eq!( + iter.next().expect("second").expect("ok").1, + Value::Str("hello1!") + ); + assert!(iter.next().is_none()); + } + } +} diff --git a/third_party/rust/rkv/src/store/keys.rs b/third_party/rust/rkv/src/store/keys.rs new file mode 100644 index 0000000000..26a13db47a --- /dev/null +++ b/third_party/rust/rkv/src/store/keys.rs @@ -0,0 +1,46 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +mod encodables; +mod primitives; + +use std::marker::PhantomData; + +use crate::error::DataError; + +pub use encodables::*; +pub use primitives::*; + +pub(crate) struct Key<K> { + bytes: Vec<u8>, + phantom: PhantomData<K>, +} + +impl<K> AsRef<[u8]> for Key<K> +where + K: EncodableKey, +{ + fn as_ref(&self) -> &[u8] { + self.bytes.as_ref() + } +} + +impl<K> Key<K> +where + K: EncodableKey, +{ + #[allow(clippy::new_ret_no_self)] + pub fn new(k: &K) -> Result<Key<K>, DataError> { + Ok(Key { + bytes: k.to_bytes()?, + phantom: PhantomData, + }) + } +} diff --git a/third_party/rust/rkv/src/store/keys/encodables.rs b/third_party/rust/rkv/src/store/keys/encodables.rs new file mode 100644 index 0000000000..85e3eacdf5 --- /dev/null +++ b/third_party/rust/rkv/src/store/keys/encodables.rs @@ -0,0 +1,27 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use bincode::serialize; +use serde::Serialize; + +use crate::error::DataError; + +pub trait EncodableKey { + fn to_bytes(&self) -> Result<Vec<u8>, DataError>; +} + +impl<T> EncodableKey for T +where + T: Serialize, +{ + fn to_bytes(&self) -> Result<Vec<u8>, DataError> { + serialize(self).map_err(|e| e.into()) + } +} diff --git a/third_party/rust/rkv/src/store/keys/primitives.rs b/third_party/rust/rkv/src/store/keys/primitives.rs new file mode 100644 index 0000000000..26282b7cbb --- /dev/null +++ b/third_party/rust/rkv/src/store/keys/primitives.rs @@ -0,0 +1,15 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use crate::store::keys::EncodableKey; + +pub trait PrimitiveInt: EncodableKey {} + +impl PrimitiveInt for u32 {} diff --git a/third_party/rust/rkv/src/store/multi.rs b/third_party/rust/rkv/src/store/multi.rs new file mode 100644 index 0000000000..c7f816b1ec --- /dev/null +++ b/third_party/rust/rkv/src/store/multi.rs @@ -0,0 +1,133 @@ +// Copyright 2018 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::marker::PhantomData; + +use crate::{ + backend::{BackendDatabase, BackendFlags, BackendIter, BackendRoCursor, BackendRwTransaction}, + error::StoreError, + helpers::read_transform, + readwrite::{Readable, Writer}, + value::Value, +}; + +type EmptyResult = Result<(), StoreError>; + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub struct MultiStore<D> { + db: D, +} + +pub struct Iter<'i, I> { + iter: I, + phantom: PhantomData<&'i ()>, +} + +impl<D> MultiStore<D> +where + D: BackendDatabase, +{ + pub(crate) fn new(db: D) -> MultiStore<D> { + MultiStore { db } + } + + /// Provides a cursor to all of the values for the duplicate entries that match this + /// key + pub fn get<'r, R, I, C, K>(&self, reader: &'r R, k: K) -> Result<Iter<'r, I>, StoreError> + where + R: Readable<'r, Database = D, RoCursor = C>, + I: BackendIter<'r>, + C: BackendRoCursor<'r, Iter = I>, + K: AsRef<[u8]> + 'r, + { + let cursor = reader.open_ro_cursor(&self.db)?; + let iter = cursor.into_iter_dup_of(k); + + Ok(Iter { + iter, + phantom: PhantomData, + }) + } + + /// Provides the first value that matches this key + pub fn get_first<'r, R, K>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError> + where + R: Readable<'r, Database = D>, + K: AsRef<[u8]>, + { + reader.get(&self.db, &k) + } + + /// Insert a value at the specified key. + /// This put will allow duplicate entries. If you wish to have duplicate entries + /// rejected, use the `put_with_flags` function and specify NO_DUP_DATA + pub fn put<T, K>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.put(&self.db, &k, v, T::Flags::empty()) + } + + pub fn put_with_flags<T, K>( + &self, + writer: &mut Writer<T>, + k: K, + v: &Value, + flags: T::Flags, + ) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.put(&self.db, &k, v, flags) + } + + pub fn delete_all<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.delete(&self.db, &k, None) + } + + pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.delete(&self.db, &k, Some(&v.to_bytes()?)) + } + + pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + { + writer.clear(&self.db) + } +} + +impl<'i, I> Iterator for Iter<'i, I> +where + I: BackendIter<'i>, +{ + type Item = Result<(&'i [u8], Value<'i>), StoreError>; + + fn next(&mut self) -> Option<Self::Item> { + match self.iter.next() { + None => None, + Some(Ok((key, bytes))) => match read_transform(Ok(bytes)) { + Ok(val) => Some(Ok((key, val))), + Err(err) => Some(Err(err)), + }, + Some(Err(err)) => Some(Err(err.into())), + } + } +} diff --git a/third_party/rust/rkv/src/store/single.rs b/third_party/rust/rkv/src/store/single.rs new file mode 100644 index 0000000000..c456e6aaa6 --- /dev/null +++ b/third_party/rust/rkv/src/store/single.rs @@ -0,0 +1,132 @@ +// Copyright 2018 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::marker::PhantomData; + +use crate::{ + backend::{BackendDatabase, BackendFlags, BackendIter, BackendRoCursor, BackendRwTransaction}, + error::StoreError, + helpers::read_transform, + readwrite::{Readable, Writer}, + value::Value, +}; + +type EmptyResult = Result<(), StoreError>; + +#[derive(Debug, Eq, PartialEq, Copy, Clone)] +pub struct SingleStore<D> { + db: D, +} + +pub struct Iter<'i, I> { + iter: I, + phantom: PhantomData<&'i ()>, +} + +impl<D> SingleStore<D> +where + D: BackendDatabase, +{ + pub(crate) fn new(db: D) -> SingleStore<D> { + SingleStore { db } + } + + pub fn get<'r, R, K>(&self, reader: &'r R, k: K) -> Result<Option<Value<'r>>, StoreError> + where + R: Readable<'r, Database = D>, + K: AsRef<[u8]>, + { + reader.get(&self.db, &k) + } + + // TODO: flags + pub fn put<T, K>(&self, writer: &mut Writer<T>, k: K, v: &Value) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.put(&self.db, &k, v, T::Flags::empty()) + } + + #[cfg(not(feature = "db-dup-sort"))] + pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.delete(&self.db, &k) + } + + #[cfg(feature = "db-dup-sort")] + pub fn delete<T, K>(&self, writer: &mut Writer<T>, k: K) -> EmptyResult + where + T: BackendRwTransaction<Database = D>, + K: AsRef<[u8]>, + { + writer.delete(&self.db, &k, None) + } + + pub fn iter_start<'r, R, I, C>(&self, reader: &'r R) -> Result<Iter<'r, I>, StoreError> + where + R: Readable<'r, Database = D, RoCursor = C>, + I: BackendIter<'r>, + C: BackendRoCursor<'r, Iter = I>, + { + let cursor = reader.open_ro_cursor(&self.db)?; + let iter = cursor.into_iter(); + + Ok(Iter { + iter, + phantom: PhantomData, + }) + } + + pub fn iter_from<'r, R, I, C, K>(&self, reader: &'r R, k: K) -> Result<Iter<'r, I>, StoreError> + where + R: Readable<'r, Database = D, RoCursor = C>, + I: BackendIter<'r>, + C: BackendRoCursor<'r, Iter = I>, + K: AsRef<[u8]> + 'r, + { + let cursor = reader.open_ro_cursor(&self.db)?; + let iter = cursor.into_iter_from(k); + + Ok(Iter { + iter, + phantom: PhantomData, + }) + } + + pub fn clear<T>(&self, writer: &mut Writer<T>) -> EmptyResult + where + D: BackendDatabase, + T: BackendRwTransaction<Database = D>, + { + writer.clear(&self.db) + } +} + +impl<'i, I> Iterator for Iter<'i, I> +where + I: BackendIter<'i>, +{ + type Item = Result<(&'i [u8], Value<'i>), StoreError>; + + fn next(&mut self) -> Option<Self::Item> { + match self.iter.next() { + None => None, + Some(Ok((key, bytes))) => match read_transform(Ok(bytes)) { + Ok(val) => Some(Ok((key, val))), + Err(err) => Some(Err(err)), + }, + Some(Err(err)) => Some(Err(err.into())), + } + } +} diff --git a/third_party/rust/rkv/src/value.rs b/third_party/rust/rkv/src/value.rs new file mode 100644 index 0000000000..762607acd3 --- /dev/null +++ b/third_party/rust/rkv/src/value.rs @@ -0,0 +1,252 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::fmt; + +use arrayref::array_ref; +use bincode::{deserialize, serialize, serialized_size}; +use ordered_float::OrderedFloat; +use uuid::{Bytes, Uuid}; + +use crate::error::DataError; + +/// We define a set of types, associated with simple integers, to annotate values stored +/// in LMDB. This is to avoid an accidental 'cast' from a value of one type to another. +/// For this reason we don't simply use `deserialize` from the `bincode` crate. +#[repr(u8)] +#[derive(Debug, PartialEq, Eq)] +pub enum Type { + Bool = 1, + U64 = 2, + I64 = 3, + F64 = 4, + Instant = 5, // Millisecond-precision timestamp. + Uuid = 6, + Str = 7, + Json = 8, + Blob = 9, +} + +/// We use manual tagging, because <https://github.com/serde-rs/serde/issues/610>. +impl Type { + pub fn from_tag(tag: u8) -> Result<Type, DataError> { + #![allow(clippy::unnecessary_lazy_evaluations)] + Type::from_primitive(tag).ok_or_else(|| DataError::UnknownType(tag)) + } + + #[allow(clippy::wrong_self_convention)] + pub fn to_tag(self) -> u8 { + self as u8 + } + + fn from_primitive(p: u8) -> Option<Type> { + match p { + 1 => Some(Type::Bool), + 2 => Some(Type::U64), + 3 => Some(Type::I64), + 4 => Some(Type::F64), + 5 => Some(Type::Instant), + 6 => Some(Type::Uuid), + 7 => Some(Type::Str), + 8 => Some(Type::Json), + 9 => Some(Type::Blob), + _ => None, + } + } +} + +impl fmt::Display for Type { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + f.write_str(match *self { + Type::Bool => "bool", + Type::U64 => "u64", + Type::I64 => "i64", + Type::F64 => "f64", + Type::Instant => "instant", + Type::Uuid => "uuid", + Type::Str => "str", + Type::Json => "json", + Type::Blob => "blob", + }) + } +} + +#[derive(Debug, Eq, PartialEq)] +pub enum Value<'v> { + Bool(bool), + U64(u64), + I64(i64), + F64(OrderedFloat<f64>), + Instant(i64), // Millisecond-precision timestamp. + Uuid(&'v Bytes), + Str(&'v str), + Json(&'v str), + Blob(&'v [u8]), +} + +#[derive(Clone, Debug, PartialEq)] +pub enum OwnedValue { + Bool(bool), + U64(u64), + I64(i64), + F64(f64), + Instant(i64), // Millisecond-precision timestamp. + Uuid(Uuid), + Str(String), + Json(String), // TODO + Blob(Vec<u8>), +} + +fn uuid(bytes: &[u8]) -> Result<Value, DataError> { + if bytes.len() == 16 { + Ok(Value::Uuid(array_ref![bytes, 0, 16])) + } else { + Err(DataError::InvalidUuid) + } +} + +impl<'v> Value<'v> { + pub fn from_tagged_slice(slice: &'v [u8]) -> Result<Value<'v>, DataError> { + let (tag, data) = slice.split_first().ok_or(DataError::Empty)?; + let t = Type::from_tag(*tag)?; + Value::from_type_and_data(t, data) + } + + fn from_type_and_data(t: Type, data: &'v [u8]) -> Result<Value<'v>, DataError> { + if t == Type::Uuid { + return deserialize(data) + .map_err(|e| DataError::DecodingError { + value_type: t, + err: e, + }) + .map(uuid)?; + } + + match t { + Type::Bool => deserialize(data).map(Value::Bool), + Type::U64 => deserialize(data).map(Value::U64), + Type::I64 => deserialize(data).map(Value::I64), + Type::F64 => deserialize(data).map(OrderedFloat).map(Value::F64), + Type::Instant => deserialize(data).map(Value::Instant), + Type::Str => deserialize(data).map(Value::Str), + Type::Json => deserialize(data).map(Value::Json), + Type::Blob => deserialize(data).map(Value::Blob), + Type::Uuid => { + // Processed above to avoid verbose duplication of error transforms. + unreachable!() + } + } + .map_err(|e| DataError::DecodingError { + value_type: t, + err: e, + }) + } + + pub fn to_bytes(&self) -> Result<Vec<u8>, DataError> { + match self { + Value::Bool(v) => serialize(&(Type::Bool.to_tag(), *v)), + Value::U64(v) => serialize(&(Type::U64.to_tag(), *v)), + Value::I64(v) => serialize(&(Type::I64.to_tag(), *v)), + Value::F64(v) => serialize(&(Type::F64.to_tag(), v.0)), + Value::Instant(v) => serialize(&(Type::Instant.to_tag(), *v)), + Value::Str(v) => serialize(&(Type::Str.to_tag(), v)), + Value::Json(v) => serialize(&(Type::Json.to_tag(), v)), + Value::Blob(v) => serialize(&(Type::Blob.to_tag(), v)), + Value::Uuid(v) => serialize(&(Type::Uuid.to_tag(), v)), + } + .map_err(DataError::EncodingError) + } + + pub fn serialized_size(&self) -> Result<u64, DataError> { + match self { + Value::Bool(v) => serialized_size(&(Type::Bool.to_tag(), *v)), + Value::U64(v) => serialized_size(&(Type::U64.to_tag(), *v)), + Value::I64(v) => serialized_size(&(Type::I64.to_tag(), *v)), + Value::F64(v) => serialized_size(&(Type::F64.to_tag(), v.0)), + Value::Instant(v) => serialized_size(&(Type::Instant.to_tag(), *v)), + Value::Str(v) => serialized_size(&(Type::Str.to_tag(), v)), + Value::Json(v) => serialized_size(&(Type::Json.to_tag(), v)), + Value::Blob(v) => serialized_size(&(Type::Blob.to_tag(), v)), + Value::Uuid(v) => serialized_size(&(Type::Uuid.to_tag(), v)), + } + .map_err(DataError::EncodingError) + } +} + +impl<'v> From<&'v Value<'v>> for OwnedValue { + fn from(value: &Value) -> OwnedValue { + match value { + Value::Bool(v) => OwnedValue::Bool(*v), + Value::U64(v) => OwnedValue::U64(*v), + Value::I64(v) => OwnedValue::I64(*v), + Value::F64(v) => OwnedValue::F64(**v), + Value::Instant(v) => OwnedValue::Instant(*v), + Value::Uuid(v) => OwnedValue::Uuid(Uuid::from_bytes(**v)), + Value::Str(v) => OwnedValue::Str((*v).to_string()), + Value::Json(v) => OwnedValue::Json((*v).to_string()), + Value::Blob(v) => OwnedValue::Blob(v.to_vec()), + } + } +} + +impl<'v> From<&'v OwnedValue> for Value<'v> { + fn from(value: &OwnedValue) -> Value { + match value { + OwnedValue::Bool(v) => Value::Bool(*v), + OwnedValue::U64(v) => Value::U64(*v), + OwnedValue::I64(v) => Value::I64(*v), + OwnedValue::F64(v) => Value::F64(OrderedFloat::from(*v)), + OwnedValue::Instant(v) => Value::Instant(*v), + OwnedValue::Uuid(v) => Value::Uuid(v.as_bytes()), + OwnedValue::Str(v) => Value::Str(v), + OwnedValue::Json(v) => Value::Json(v), + OwnedValue::Blob(v) => Value::Blob(v), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_value_serialized_size() { + // | Value enum | tag: 1 byte | value_payload | + // |----------------------------------------------------------| + // | I64 | 1 | 8 | + // | U64 | 1 | 8 | + // | Bool | 1 | 1 | + // | Instant | 1 | 8 | + // | F64 | 1 | 8 | + // | Uuid | 1 | 16 | + // | Str/Blob/Json | 1 |(8: len + sizeof(payload))| + assert_eq!(Value::I64(-1000).serialized_size().unwrap(), 9); + assert_eq!(Value::U64(1000u64).serialized_size().unwrap(), 9); + assert_eq!(Value::Bool(true).serialized_size().unwrap(), 2); + assert_eq!( + Value::Instant(1_558_020_865_224).serialized_size().unwrap(), + 9 + ); + assert_eq!( + Value::F64(OrderedFloat(10000.1)).serialized_size().unwrap(), + 9 + ); + assert_eq!(Value::Str("hello!").serialized_size().unwrap(), 15); + assert_eq!(Value::Str("¡Hola").serialized_size().unwrap(), 15); + assert_eq!(Value::Blob(b"hello!").serialized_size().unwrap(), 15); + assert_eq!( + uuid(b"\x9f\xe2\xc4\xe9\x3f\x65\x4f\xdb\xb2\x4c\x02\xb1\x52\x59\x71\x6c") + .unwrap() + .serialized_size() + .unwrap(), + 17 + ); + } +} diff --git a/third_party/rust/rkv/tests/env-all.rs b/third_party/rust/rkv/tests/env-all.rs new file mode 100644 index 0000000000..a8d93c786e --- /dev/null +++ b/third_party/rust/rkv/tests/env-all.rs @@ -0,0 +1,328 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::fs; + +use tempfile::Builder; + +use rkv::{ + backend::{Lmdb, SafeMode}, + Rkv, StoreOptions, Value, +}; + +#[test] +fn test_open_safe_same_dir_as_lmdb() { + let root = Builder::new() + .prefix("test_open_safe_same_dir_as_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create database of type A and save to disk. + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type A was written to disk. + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and verify that it is empty. + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let _ = k + .open_single("sk", StoreOptions::default()) + .expect_err("not opened"); + } + // Verify that database of type A wasn't changed. + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and save to disk (type A exists at the same path). + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo1", &Value::I64(5678)) + .expect("wrote"); + sk.put(&mut writer, "bar1", &Value::Bool(false)) + .expect("wrote"); + sk.put(&mut writer, "baz1", &Value::Str("héllo~ yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&writer, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&writer, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type B was written to disk. + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&reader, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&reader, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + } + // Verify that database of type A still wasn't changed. + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } +} + +#[test] +fn test_open_lmdb_same_dir_as_safe() { + let root = Builder::new() + .prefix("test_open_lmdb_same_dir_as_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create database of type A and save to disk. + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type A was written to disk. + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and verify that it is empty. + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let _ = k + .open_single("sk", StoreOptions::default()) + .expect_err("not opened"); + } + // Verify that database of type A wasn't changed. + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Create database of type B and save to disk (type A exists at the same path). + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo1", &Value::I64(5678)) + .expect("wrote"); + sk.put(&mut writer, "bar1", &Value::Bool(false)) + .expect("wrote"); + sk.put(&mut writer, "baz1", &Value::Str("héllo~ yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&writer, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&writer, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + // Verify that database of type B was written to disk. + { + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo1").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + sk.get(&reader, "bar1").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + sk.get(&reader, "baz1").expect("read"), + Some(Value::Str("héllo~ yöu")) + ); + } + // Verify that database of type A still wasn't changed. + { + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } +} diff --git a/third_party/rust/rkv/tests/env-lmdb.rs b/third_party/rust/rkv/tests/env-lmdb.rs new file mode 100644 index 0000000000..e3aca394ab --- /dev/null +++ b/third_party/rust/rkv/tests/env-lmdb.rs @@ -0,0 +1,1647 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable +// deprecates `clippy::cyclomatic_complexity`. +#![allow(clippy::complexity)] + +use std::{ + fs, + path::Path, + str, + sync::{Arc, RwLock}, + thread, +}; + +use byteorder::{ByteOrder, LittleEndian}; +use tempfile::Builder; + +use rkv::{ + backend::{ + BackendEnvironmentBuilder, BackendInfo, BackendStat, Lmdb, LmdbDatabase, LmdbEnvironment, + LmdbRwTransaction, + }, + EnvironmentFlags, Rkv, SingleStore, StoreError, StoreOptions, Value, Writer, +}; + +fn check_rkv(k: &Rkv<LmdbEnvironment>) { + let _ = k + .open_single(None, StoreOptions::create()) + .expect("created default"); + + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + let reader = k.read().expect("reader"); + + let result = s.get(&reader, "foo"); + assert_eq!(None, result.expect("success but no value")); +} + +// The default size is 1MB. +const DEFAULT_SIZE: usize = 1024 * 1024; + +/// We can't open a directory that doesn't exist. +#[test] +fn test_open_fails() { + let root = Builder::new() + .prefix("test_open_fails") + .tempdir() + .expect("tempdir"); + assert!(root.path().exists()); + + let nope = root.path().join("nope/"); + assert!(!nope.exists()); + + let pb = nope.to_path_buf(); + match Rkv::new::<Lmdb>(nope.as_path()).err() { + Some(StoreError::UnsuitableEnvironmentPath(p)) => { + assert_eq!(pb, p); + } + _ => panic!("expected error"), + }; +} + +#[test] +fn test_open() { + let root = Builder::new() + .prefix("test_open") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder_with_no_subdir_1() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + { + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); + } + { + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_flags(EnvironmentFlags::NO_SUB_DIR); + builder.set_max_dbs(2); + + let mut datamdb = root.path().to_path_buf(); + datamdb.push("data.mdb"); + + let k = Rkv::from_builder(&datamdb, builder).expect("rkv"); + check_rkv(&k); + } +} + +#[test] +#[should_panic(expected = "rkv: UnsuitableEnvironmentPath")] +fn test_open_from_builder_with_no_subdir_2() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + { + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); + } + { + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_flags(EnvironmentFlags::NO_SUB_DIR); + builder.set_max_dbs(2); + + let mut datamdb = root.path().to_path_buf(); + datamdb.push("bogus.mdb"); + + let k = Rkv::from_builder(&datamdb, builder).expect("rkv"); + check_rkv(&k); + } +} + +#[test] +fn test_open_from_builder_with_dir_1() { + let root = Builder::new() + .prefix("test_open_from_builder") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(2); + builder.set_make_dir_if_needed(true); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "rkv: UnsuitableEnvironmentPath(\"bogus\")")] +fn test_open_from_builder_with_dir_2() { + let root = Path::new("bogus"); + println!("Root path: {root:?}"); + assert!(!root.is_dir()); + + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root, builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "opened: DbsFull")] +fn test_create_with_capacity_1() { + let root = Builder::new() + .prefix("test_create_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This errors with "opened: DbsFull" because we specified a capacity of one (database), + // and check_rkv already opened one (plus the default database, which doesn't count + // against the limit). + let _zzz = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); +} + +#[test] +fn test_create_with_capacity_2() { + let root = Builder::new() + .prefix("test_create_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This doesn't error with "opened: DbsFull" with because even though we specified a + // capacity of one (database), and check_rkv already opened one, the default database + // doesn't count against the limit. + let _zzz = k.open_single(None, StoreOptions::create()).expect("opened"); +} + +#[test] +#[should_panic(expected = "opened: DbsFull")] +fn test_open_with_capacity_1() { + let root = Builder::new() + .prefix("test_open_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single("zzz", StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_open_with_capacity_2() { + let root = Builder::new() + .prefix("test_open_with_capacity") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single(None, StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_list_dbs_1() { + let root = Builder::new() + .prefix("test_list_dbs") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let dbs = k.get_dbs().unwrap(); + assert_eq!(dbs, vec![Some("s".to_owned())]); +} + +#[test] +fn test_list_dbs_2() { + let root = Builder::new() + .prefix("test_list_dbs") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 2).expect("rkv"); + check_rkv(&k); + + let _ = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); + + let dbs = k.get_dbs().unwrap(); + assert_eq!(dbs, vec![Some("s".to_owned()), Some("zzz".to_owned())]); +} + +#[test] +fn test_list_dbs_3() { + let root = Builder::new() + .prefix("test_list_dbs") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<Lmdb>(root.path(), 0).expect("rkv"); + + let _ = k.open_single(None, StoreOptions::create()).expect("opened"); + + let dbs = k.get_dbs().unwrap(); + assert_eq!(dbs, vec![None]); +} + +fn get_larger_than_default_map_size_value() -> usize { + // The LMDB C library and lmdb Rust crate docs for setting the map size + // <http://www.lmdb.tech/doc/group__mdb.html#gaa2506ec8dab3d969b0e609cd82e619e5> + // <https://docs.rs/lmdb/0.8.0/lmdb/struct.EnvironmentBuilder.html#method.set_map_size> + // both say that the default map size is 10,485,760 bytes, i.e. 10MiB. + // + // But the DEFAULT_MAPSIZE define in the LMDB code + // https://github.com/LMDB/lmdb/blob/26c7df88e44e31623d0802a564f24781acdefde3/libraries/liblmdb/mdb.c#L729 + // sets the default map size to 1,048,576 bytes, i.e. 1MiB. + // + DEFAULT_SIZE + 1 /* 1,048,576 + 1 bytes, i.e. 1MiB + 1 byte */ +} + +#[test] +#[should_panic(expected = "wrote: MapFull")] +fn test_exceed_map_size() { + let root = Builder::new() + .prefix("test_exceed_map_size") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k + .open_single("test", StoreOptions::create()) + .expect("opened"); + + // Writing a large enough value should cause LMDB to fail on MapFull. + // We write a string that is larger than the default map size. + let val = "x".repeat(get_larger_than_default_map_size_value()); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str(&val)) + .expect("wrote"); +} + +#[test] +#[should_panic(expected = "wrote: KeyValuePairBadSize")] +fn test_exceed_key_size_limit() { + let root = Builder::new() + .prefix("test_exceed_key_size_limit") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k + .open_single("test", StoreOptions::create()) + .expect("opened"); + + let key = "k".repeat(512); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, key, &Value::Str("val")).expect("wrote"); +} + +#[test] +fn test_increase_map_size() { + let root = Builder::new() + .prefix("test_open_with_map_size") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let mut builder = Rkv::environment_builder::<Lmdb>(); + // Set the map size to the size of the value we'll store in it + 100KiB, + // which ensures that there's enough space for the value and metadata. + builder.set_map_size( + get_larger_than_default_map_size_value() + 100 * 1024, /* 100KiB */ + ); + builder.set_max_dbs(2); + let k = Rkv::from_builder(root.path(), builder).unwrap(); + let sk = k + .open_single("test", StoreOptions::create()) + .expect("opened"); + let val = "x".repeat(get_larger_than_default_map_size_value()); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str(&val)) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::Str(&val)) + ); +} + +#[test] +fn test_round_trip_and_transactions() { + let root = Builder::new() + .prefix("test_round_trip_and_transactions") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "noo").expect("read"), + Some(Value::F64(1234.0.into())) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + // Isolation. Reads won't return values. + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + // Dropped: tx rollback. Reads will still return nothing. + + { + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + // Isolation. Reads still return values. + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + // Dropped: tx rollback. Reads will still return values. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed but return None to indicate a missing value. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), None); + assert_eq!(sk.get(&r, "bar").expect("read"), None); + assert_eq!(sk.get(&r, "baz").expect("read"), None); + } +} + +#[test] +fn test_single_store_clear() { + let root = Builder::new() + .prefix("test_single_store_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + let iter = sk.iter_start(&r).expect("iter"); + assert_eq!(iter.count(), 0); + } +} + +#[test] +#[should_panic(expected = "KeyValuePairNotFound")] +fn test_single_store_delete_nonexistent() { + let root = Builder::new() + .prefix("test_single_store_delete_nonexistent") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "bogus").unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multi_put_get_del() { + let root = Builder::new() + .prefix("test_multi_put_get_del") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().unwrap(); + + let writer = k.write().unwrap(); + { + let mut iter = multistore.get(&writer, "str1").unwrap(); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 bar"))); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 foo"))); + } + writer.commit().unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .delete(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str1").unwrap(), + Some(Value::Str("str1 bar")) + ); + multistore + .delete(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str2").unwrap(), + Some(Value::Str("str2 foo")) + ); + multistore.delete_all(&mut writer, "str3").unwrap(); + assert_eq!(multistore.get_first(&writer, "str3").unwrap(), None); + writer.commit().unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multiple_store_clear() { + let root = Builder::new() + .prefix("test_multiple_store_clear") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let multistore = k + .open_multi("multistore", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = k.write().expect("writer"); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + multistore.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + assert_eq!(multistore.get_first(&r, "str1").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str2").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str3").expect("read"), None); + } +} + +#[test] +fn test_open_store_for_read() { + let root = Builder::new() + .prefix("test_open_store_for_read") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + + // First create the store, and start a write transaction on it. + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("write"); + + // Open the same store for read, note that the write transaction is still in progress, + // it should not block the reader though. + let sk_readonly = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + writer.commit().expect("commit"); + + // Now the write transaction is committed, any followed reads should see its change. + let reader = k.read().expect("reader"); + assert_eq!( + sk_readonly.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); +} + +#[test] +#[should_panic(expected = "open a missing store")] +fn test_open_a_missing_store() { + let root = Builder::new() + .prefix("test_open_a_missing_store") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let _sk = k + .open_single("sk", StoreOptions::default()) + .expect("open a missing store"); +} + +#[test] +#[should_panic(expected = "new failed: FileInvalid")] +fn test_open_a_broken_store() { + let root = Builder::new() + .prefix("test_open_a_missing_store") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.mdb"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + let _ = Rkv::new::<Lmdb>(root.path()).expect("new failed"); +} + +#[test] +fn test_open_fail_with_badrslot() { + let root = Builder::new() + .prefix("test_open_fail_with_badrslot") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + + // First create the store + let _sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Open a reader on this store + let _reader = k.read().expect("reader"); + + // Open the same store for read while the reader is in progress will panic + let store = k.open_single("sk", StoreOptions::default()); + match store { + Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (), + _ => panic!("should panic"), + } +} + +#[test] +fn test_read_before_write_num() { + let root = Builder::new() + .prefix("test_read_before_write_num") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a number, modifying it, and then writing it back. + // We have to be done with the Value::I64 before calling Writer::put, + // as the Value::I64 borrows an immutable reference to the Writer. + // So we extract and copy its primitive value. + + fn get_existing_foo( + store: SingleStore<LmdbDatabase>, + writer: &Writer<LmdbRwTransaction>, + ) -> Option<i64> { + match store.get(writer, "foo").expect("read") { + Some(Value::I64(val)) => Some(val), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or(99); + existing += 1; + sk.put(&mut writer, "foo", &Value::I64(existing)) + .expect("success"); + + let updated = get_existing_foo(sk, &writer).unwrap_or(99); + assert_eq!(updated, 100); + writer.commit().expect("commit"); +} + +#[test] +fn test_read_before_write_str() { + let root = Builder::new() + .prefix("test_read_before_write_str") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a string, modifying it, and then writing it back. + // We have to be done with the Value::Str before calling Writer::put, + // as the Value::Str (and its underlying &str) borrows an immutable + // reference to the Writer. So we copy it to a String. + + fn get_existing_foo( + store: SingleStore<LmdbDatabase>, + writer: &Writer<LmdbRwTransaction>, + ) -> Option<String> { + match store.get(writer, "foo").expect("read") { + Some(Value::Str(val)) => Some(val.to_string()), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or_default(); + existing.push('…'); + sk.put(&mut writer, "foo", &Value::Str(&existing)) + .expect("write"); + + let updated = get_existing_foo(sk, &writer).unwrap_or_default(); + assert_eq!(updated, "…"); + writer.commit().expect("commit"); +} + +#[test] +fn test_concurrent_read_transactions_prohibited() { + let root = Builder::new() + .prefix("test_concurrent_reads_prohibited") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let _first = k.read().expect("reader"); + let second = k.read(); + + match second { + Err(StoreError::ReadTransactionAlreadyExists(t)) => { + println!("Thread was {t:?}"); + } + Err(e) => { + println!("Got error {e:?}"); + } + _ => { + panic!("Expected error."); + } + } +} + +#[test] +fn test_isolation() { + let root = Builder::new() + .prefix("test_isolation") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + + // Add one field. + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote"); + writer.commit().expect("committed"); + } + + { + let reader = k.read().unwrap(); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + } + + // Establish a long-lived reader that outlasts a writer. + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // Start a write transaction. + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote"); + + // The reader and writer are isolated. + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999))); + + // If we commit the writer, we still have isolation. + writer.commit().expect("committed"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // A new reader sees the committed value. Note that LMDB doesn't allow two + // read transactions to exist in the same thread, so we abort the previous one. + reader.abort(); + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999))); +} + +#[test] +fn test_blob() { + let root = Builder::new() + .prefix("test_round_trip_blob") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4])) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::Blob(&[1, 2, 3, 4])) + ); + + fn u16_to_u8(src: &[u16]) -> Vec<u8> { + let mut dst = vec![0; 2 * src.len()]; + LittleEndian::write_u16_into(src, &mut dst); + dst + } + + fn u8_to_u16(src: &[u8]) -> Vec<u16> { + let mut dst = vec![0; src.len() / 2]; + LittleEndian::read_u16_into(src, &mut dst); + dst + } + + // When storing UTF-16 strings as blobs, we'll need to convert + // their [u16] backing storage to [u8]. Test that converting, writing, + // reading, and converting back works as expected. + let u16_array = [1000, 10000, 54321, 65535]; + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array))) + .expect("wrote"); + let u8_array = match sk.get(&writer, "bar").expect("read") { + Some(Value::Blob(val)) => val, + _ => &[], + }; + assert_eq!(u8_to_u16(u8_array), u16_array); +} + +#[test] +fn test_sync() { + let root = Builder::new() + .prefix("test_sync") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut builder = Rkv::environment_builder::<Lmdb>(); + builder.set_max_dbs(1); + builder.set_flags(EnvironmentFlags::NO_SYNC); + { + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + } + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); +} + +#[test] +#[cfg(feature = "db-int-key")] +fn test_stat() { + let root = Builder::new() + .prefix("test_stat") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + for i in 0..5 { + let sk = k + .open_integer(&format!("sk{i}")[..], StoreOptions::create()) + .expect("opened"); + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, i, &Value::I64(i64::from(i))) + .expect("wrote"); + writer.commit().expect("committed"); + } + } + assert_eq!(k.stat().expect("stat").depth(), 1); + assert_eq!(k.stat().expect("stat").entries(), 5); + assert_eq!(k.stat().expect("stat").branch_pages(), 0); + assert_eq!(k.stat().expect("stat").leaf_pages(), 1); +} + +#[test] +fn test_info() { + let root = Builder::new() + .prefix("test_info") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + writer.commit().expect("commited"); + + let info = k.info().expect("info"); + + // The default size is 1MB. + assert_eq!(info.map_size(), DEFAULT_SIZE); + // Should greater than 0 after the write txn. + assert!(info.last_pgno() > 0); + // A txn to open_single + a txn to write. + assert_eq!(info.last_txnid(), 2); + // The default max readers is 126. + assert_eq!(info.max_readers(), 126); + assert_eq!(info.num_readers(), 0); + + // A new reader should increment the reader counter. + let _reader = k.read().expect("reader"); + let info = k.info().expect("info"); + + assert_eq!(info.num_readers(), 1); +} + +#[test] +fn test_load_ratio() { + let root = Builder::new() + .prefix("test_load_ratio") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + writer.commit().expect("commited"); + let ratio = k.load_ratio().expect("ratio").unwrap(); + assert!(ratio > 0.0_f32 && ratio < 1.0_f32); + + // Put data to database should increase the load ratio. + let mut writer = k.write().expect("writer"); + sk.put( + &mut writer, + "bar", + &Value::Str(&"more-than-4KB".repeat(1000)), + ) + .expect("wrote"); + writer.commit().expect("commited"); + let new_ratio = k.load_ratio().expect("ratio").unwrap(); + assert!(new_ratio > ratio); + + // Clear the database so that all the used pages should go to freelist, hence the ratio + // should decrease. + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("clear"); + writer.commit().expect("commited"); + let after_clear_ratio = k.load_ratio().expect("ratio").unwrap(); + assert!(after_clear_ratio < new_ratio); +} + +#[test] +fn test_set_map_size() { + let root = Builder::new() + .prefix("test_size_map_size") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + assert_eq!(k.info().expect("info").map_size(), DEFAULT_SIZE); + + k.set_map_size(2 * DEFAULT_SIZE).expect("resized"); + + // Should be able to write. + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + writer.commit().expect("commited"); + + assert_eq!(k.info().expect("info").map_size(), 2 * DEFAULT_SIZE); +} + +#[test] +fn test_iter() { + let root = Builder::new() + .prefix("test_iter") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // An iterator over an empty store returns no values. + { + let reader = k.read().unwrap(); + let mut iter = sk.iter_start(&reader).unwrap(); + assert!(iter.next().is_none()); + } + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Reader.iter() returns (key, value) tuples ordered by key. + let mut iter = sk.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterators don't loop. Once one returns None, additional calls + // to its next() method will always return None. + assert!(iter.next().is_none()); + + // Reader.iter_from() begins iteration at the first key equal to + // or greater than the given key. + let mut iter = sk.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Reader.iter_from() works as expected when the given key is a prefix + // of a key in the store. + let mut iter = sk.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_iter_from_key_greater_than_existing() { + let root = Builder::new() + .prefix("test_iter_from_key_greater_than_existing") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + let mut iter = sk.iter_from(&reader, "nuu").unwrap(); + assert!(iter.next().is_none()); +} + +#[test] +fn test_multiple_store_read_write() { + let root = Builder::new() + .prefix("test_multiple_store_read_write") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + let s3 = k + .open_single("store_3", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + s1.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote"); + s3.put(&mut writer, "foo", &Value::Bool(true)) + .expect("wrote"); + + assert_eq!( + s1.get(&writer, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&writer, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&writer, "foo").expect("read"), + Some(Value::Bool(true)) + ); + + writer.commit().expect("committed"); + + let reader = k.read().expect("unbound_reader"); + assert_eq!( + s1.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&reader, "foo").expect("read"), + Some(Value::Bool(true)) + ); + reader.abort(); + + // test delete across multiple stores + let mut writer = k.write().expect("writer"); + s1.delete(&mut writer, "foo").expect("deleted"); + s2.delete(&mut writer, "foo").expect("deleted"); + s3.delete(&mut writer, "foo").expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s1.get(&reader, "key").expect("value"), None); + assert_eq!(s2.get(&reader, "key").expect("value"), None); + assert_eq!(s3.get(&reader, "key").expect("value"), None); +} + +#[test] +fn test_multiple_store_iter() { + let root = Builder::new() + .prefix("test_multiple_store_iter") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + // Write to "s1" + s1.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s1.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s1.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s1.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + // &mut writer to "s2" + s2.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s2.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s2.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s2.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Iterate through the whole store in "s1" + let mut iter = s1.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate through the whole store in "s2" + let mut iter = s2.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s1" + let mut iter = s1.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s2" + let mut iter = s2.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s1" + let mut iter = s1.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s2" + let mut iter = s2.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_store_multiple_thread() { + let root = Builder::new() + .prefix("test_multiple_thread") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let rkv_arc = Arc::new(RwLock::new( + Rkv::new::<Lmdb>(root.path()).expect("new succeeded"), + )); + let store = rkv_arc + .read() + .unwrap() + .open_single("test", StoreOptions::create()) + .expect("opened"); + + let num_threads = 10; + let mut write_handles = Vec::with_capacity(num_threads as usize); + let mut read_handles = Vec::with_capacity(num_threads as usize); + + // Note that this isn't intended to demonstrate a good use of threads. + // For this shape of data, it would be more performant to write/read + // all values using one transaction in a single thread. The point here + // is just to confirm that a store can be shared by multiple threads. + + // For each KV pair, spawn a thread that writes it to the store. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + write_handles.push(thread::spawn(move || { + let rkv = rkv_arc.write().expect("rkv"); + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, i.to_string(), &Value::U64(i)) + .expect("written"); + writer.commit().unwrap(); + })); + } + for handle in write_handles { + handle.join().expect("joined"); + } + + // For each KV pair, spawn a thread that reads it from the store + // and returns its value. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + read_handles.push(thread::spawn(move || { + let rkv = rkv_arc.read().expect("rkv"); + let reader = rkv.read().expect("reader"); + let value = match store.get(&reader, i.to_string()) { + Ok(Some(Value::U64(value))) => value, + Ok(Some(_)) => panic!("value type unexpected"), + Ok(None) => panic!("value not found"), + Err(err) => panic!("{}", err), + }; + assert_eq!(value, i); + value + })); + } + + // Sum the values returned from the threads and confirm that they're + // equal to the sum of values written to the threads. + let thread_sum: u64 = read_handles + .into_iter() + .map(|handle| handle.join().expect("value")) + .sum(); + assert_eq!(thread_sum, (0..num_threads).sum()); +} + +#[test] +fn test_use_value_as_key() { + let root = Builder::new() + .prefix("test_use_value_as_key") + .tempdir() + .expect("tempdir"); + let rkv = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let store = rkv + .open_single("store", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // It's possible to retrieve a value with a Reader and then use it + // as a key with a Writer. + { + let reader = &rkv.read().unwrap(); + if let Some(Value::Str(key)) = store.get(reader, "foo").expect("read") { + let mut writer = rkv.write().expect("writer"); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also retrieve a Value with a Writer and then use it as a key + // with the same Writer if you copy the value to an owned type + // so the Writer isn't still being borrowed by the retrieved value + // when you try to borrow the Writer again to modify that value. + { + let mut writer = rkv.write().expect("writer"); + if let Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") { + let key = value.to_owned(); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let name1 = rkv + .open_single("name1", StoreOptions::create()) + .expect("opened"); + let name2 = rkv + .open_single("name2", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv.write().expect("writer"); + name1 + .put(&mut writer, "key1", &Value::Str("bar")) + .expect("wrote"); + name1 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + name2 + .put(&mut writer, "key2", &Value::Str("bar")) + .expect("wrote"); + name2 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also iterate (store, key) pairs to retrieve foreign keys, + // then iterate those foreign keys to modify/delete them. + // + // You need to open the stores in advance, since opening a store + // uses a write transaction internally, so opening them while a writer + // is extant will hang. + // + // And you need to copy the values to an owned type so the Writer isn't + // still being borrowed by a retrieved value when you try to borrow + // the Writer again to modify another value. + let fields = vec![ + ( + rkv.open_single("name1", StoreOptions::create()) + .expect("opened"), + "key1", + ), + ( + rkv.open_single("name2", StoreOptions::create()) + .expect("opened"), + "key2", + ), + ]; + { + let mut foreignkeys = Vec::new(); + let mut writer = rkv.write().expect("writer"); + for (store, key) in fields.iter() { + if let Some(Value::Str(value)) = store.get(&writer, key).expect("read") { + foreignkeys.push((store, value.to_owned())); + } + } + for (store, key) in foreignkeys.iter() { + store.delete(&mut writer, key).expect("deleted"); + } + writer.commit().expect("committed"); + } +} diff --git a/third_party/rust/rkv/tests/env-migration.rs b/third_party/rust/rkv/tests/env-migration.rs new file mode 100644 index 0000000000..7026c25de8 --- /dev/null +++ b/third_party/rust/rkv/tests/env-migration.rs @@ -0,0 +1,607 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{fs, path::Path}; + +use tempfile::Builder; + +use rkv::{ + backend::{Lmdb, LmdbEnvironment, SafeMode, SafeModeEnvironment}, + Manager, Migrator, Rkv, StoreOptions, Value, +}; + +macro_rules! populate_store { + ($env:expr) => { + let store = $env + .open_single("store", StoreOptions::create()) + .expect("opened"); + let mut writer = $env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + store + .put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + }; +} + +#[test] +fn test_open_migrator_lmdb_to_safe() { + let root = Builder::new() + .prefix("test_open_migrator_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Populate source environment and persist to disk. + { + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + // Check if the files were written to disk. + { + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + } + // Verify that database was written to disk. + { + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let store = src_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = src_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Open and migrate. + { + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Verify that the database was indeed migrated. + { + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Check if the old files were deleted from disk. + { + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + } +} + +#[test] +fn test_open_migrator_safe_to_lmdb() { + let root = Builder::new() + .prefix("test_open_migrator_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Populate source environment and persist to disk. + { + let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + // Check if the files were written to disk. + { + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(safebin.exists()); + } + // Verify that database was written to disk. + { + let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let store = src_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = src_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Open and migrate. + { + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Verify that the database was indeed migrated. + { + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Check if the old files were deleted from disk. + { + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(!safebin.exists()); + } +} + +#[test] +fn test_open_migrator_round_trip() { + let root = Builder::new() + .prefix("test_open_migrator_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Populate source environment and persist to disk. + { + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + // Open and migrate. + { + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_lmdb_to_safe_mode(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Open and migrate back. + { + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::open_and_migrate_safe_mode_to_lmdb(root.path(), |builder| builder, &dst_env) + .expect("migrated"); + } + // Verify that the database was indeed migrated twice. + { + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + // Check if the right files are finally present on disk. + { + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); + assert!(lockmdb.exists()); + assert!(!safebin.exists()); + } +} + +#[test] +fn test_easy_migrator_no_dir_1() { + let root = Builder::new() + .prefix("test_easy_migrator_no_dir") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // This won't fail with IoError even though the path is a bogus path, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::easy_migrate_lmdb_to_safe_mode(Path::new("bogus"), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(!datamdb.exists()); + assert!(!lockmdb.exists()); + assert!(!safebin.exists()); // safe mode doesn't write an empty db to disk +} + +#[test] +fn test_easy_migrator_no_dir_2() { + let root = Builder::new() + .prefix("test_easy_migrator_no_dir") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // This won't fail with IoError even though the path is a bogus path, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::easy_migrate_safe_mode_to_lmdb(Path::new("bogus"), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); // lmdb writes an empty db to disk + assert!(lockmdb.exists()); + assert!(!safebin.exists()); +} + +#[test] +fn test_easy_migrator_invalid_1() { + let root = Builder::new() + .prefix("test_easy_migrator_invalid") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.mdb"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); // corrupted db isn't deleted + assert!(lockmdb.exists()); + assert!(!safebin.exists()); +} + +#[test] +fn test_easy_migrator_invalid_2() { + let root = Builder::new() + .prefix("test_easy_migrator_invalid") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.safe.bin"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated"); + + let mut datamdb = root.path().to_path_buf(); + let mut lockmdb = root.path().to_path_buf(); + let mut safebin = root.path().to_path_buf(); + datamdb.push("data.mdb"); + lockmdb.push("lock.mdb"); + safebin.push("data.safe.bin"); + assert!(datamdb.exists()); // lmdb writes an empty db to disk + assert!(lockmdb.exists()); + assert!(safebin.exists()); // corrupted db isn't deleted +} + +#[test] +#[should_panic(expected = "migrated: SourceEmpty")] +fn test_migrator_lmdb_to_safe_1() { + let root = Builder::new() + .prefix("test_migrate_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); +} + +#[test] +#[should_panic(expected = "migrated: DestinationNotEmpty")] +fn test_migrator_lmdb_to_safe_2() { + let root = Builder::new() + .prefix("test_migrate_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + populate_store!(&dst_env); + Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); +} + +#[test] +fn test_migrator_lmdb_to_safe_3() { + let root = Builder::new() + .prefix("test_migrate_lmdb_to_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::migrate_lmdb_to_safe_mode(&src_env, &dst_env).expect("migrated"); + + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); +} + +#[test] +#[should_panic(expected = "migrated: SourceEmpty")] +fn test_migrator_safe_to_lmdb_1() { + let root = Builder::new() + .prefix("test_migrate_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); +} + +#[test] +#[should_panic(expected = "migrated: DestinationNotEmpty")] +fn test_migrator_safe_to_lmdb_2() { + let root = Builder::new() + .prefix("test_migrate_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + populate_store!(&dst_env); + Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); +} + +#[test] +fn test_migrator_safe_to_lmdb_3() { + let root = Builder::new() + .prefix("test_migrate_safe_to_lmdb") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::migrate_safe_mode_to_lmdb(&src_env, &dst_env).expect("migrated"); + + let store = dst_env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = dst_env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); +} + +#[test] +fn test_easy_migrator_failed_migration_1() { + let root = Builder::new() + .prefix("test_easy_migrator_failed_migration_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.mdb"); + fs::write(&dbfile, "bogus").expect("bogus dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated"); + + // Populate destination environment and persist to disk. + populate_store!(&dst_env); + dst_env.sync(true).expect("synced"); + + // Delete bogus file and create a valid source environment in its place. + fs::remove_file(&dbfile).expect("bogus dbfile removed"); + let src_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + + // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty. + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), &dst_env).expect("migrated"); +} + +#[test] +fn test_easy_migrator_failed_migration_2() { + let root = Builder::new() + .prefix("test_easy_migrator_failed_migration_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.safe.bin"); + fs::write(&dbfile, "bogus").expect("bogus dbfile created"); + + // This won't fail with FileInvalid even though the database is a bogus file, because this + // is the "easy mode" migration which automatically handles (ignores) this error. + let dst_env = Rkv::new::<Lmdb>(root.path()).expect("new succeeded"); + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated"); + + // Populate destination environment and persist to disk. + populate_store!(&dst_env); + dst_env.sync(true).expect("synced"); + + // Delete bogus file and create a valid source environment in its place. + fs::remove_file(&dbfile).expect("bogus dbfile removed"); + let src_env = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + + // Attempt to migrate again. This should *NOT* fail with DestinationNotEmpty. + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), &dst_env).expect("migrated"); +} + +fn test_easy_migrator_from_manager_failed_migration_1() { + let root = Builder::new() + .prefix("test_easy_migrator_from_manager_failed_migration_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let mut src_manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); + let created_src_arc = src_manager + .get_or_create(root.path(), Rkv::new::<Lmdb>) + .unwrap(); + let src_env = created_src_arc.read().unwrap(); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + { + let mut dst_manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap(); + let created_dst_arc_1 = dst_manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .unwrap(); + let dst_env_1 = created_dst_arc_1.read().unwrap(); + populate_store!(&dst_env_1); + dst_env_1.sync(true).expect("synced"); + } + + // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty. + let dst_manager = Manager::<SafeModeEnvironment>::singleton().read().unwrap(); + let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap(); + let dst_env_2 = created_dst_arc_2.read().unwrap(); + Migrator::easy_migrate_lmdb_to_safe_mode(root.path(), dst_env_2).expect("migrated"); +} + +fn test_easy_migrator_from_manager_failed_migration_2() { + let root = Builder::new() + .prefix("test_easy_migrator_from_manager_failed_migration_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + { + let mut src_manager = Manager::<SafeModeEnvironment>::singleton().write().unwrap(); + let created_src_arc = src_manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .unwrap(); + let src_env = created_src_arc.read().unwrap(); + populate_store!(&src_env); + src_env.sync(true).expect("synced"); + } + { + let mut dst_manager = Manager::<LmdbEnvironment>::singleton().write().unwrap(); + let created_dst_arc_1 = dst_manager + .get_or_create(root.path(), Rkv::new::<Lmdb>) + .unwrap(); + let dst_env_1 = created_dst_arc_1.read().unwrap(); + populate_store!(&dst_env_1); + dst_env_1.sync(true).expect("synced"); + } + + // Attempt to migrate again in a new env. This should *NOT* fail with DestinationNotEmpty. + let dst_manager = Manager::<LmdbEnvironment>::singleton().read().unwrap(); + let created_dst_arc_2 = dst_manager.get(root.path()).unwrap().unwrap(); + let dst_env_2 = created_dst_arc_2.read().unwrap(); + Migrator::easy_migrate_safe_mode_to_lmdb(root.path(), dst_env_2).expect("migrated"); +} + +#[test] +fn test_easy_migrator_from_manager_failed_migration() { + test_easy_migrator_from_manager_failed_migration_1(); + test_easy_migrator_from_manager_failed_migration_2(); +} diff --git a/third_party/rust/rkv/tests/env-safe.rs b/third_party/rust/rkv/tests/env-safe.rs new file mode 100644 index 0000000000..9c20e39507 --- /dev/null +++ b/third_party/rust/rkv/tests/env-safe.rs @@ -0,0 +1,1370 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +// TODO: change this back to `clippy::cognitive_complexity` when Clippy stable +// deprecates `clippy::cyclomatic_complexity`. +#![allow(clippy::complexity)] + +use std::{ + fs, + path::Path, + str, + sync::{Arc, RwLock}, + thread, +}; + +use byteorder::{ByteOrder, LittleEndian}; +use tempfile::Builder; + +use rkv::{ + backend::{ + BackendEnvironmentBuilder, SafeMode, SafeModeDatabase, SafeModeEnvironment, + SafeModeRwTransaction, + }, + Rkv, SingleStore, StoreError, StoreOptions, Value, Writer, +}; + +fn check_rkv(k: &Rkv<SafeModeEnvironment>) { + let _ = k + .open_single(None, StoreOptions::create()) + .expect("created default"); + + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + let reader = k.read().expect("reader"); + + let result = s.get(&reader, "foo"); + assert_eq!(None, result.expect("success but no value")); +} + +/// We can't open a directory that doesn't exist. +#[test] +fn test_open_fails_safe() { + let root = Builder::new() + .prefix("test_open_fails_safe") + .tempdir() + .expect("tempdir"); + assert!(root.path().exists()); + + let nope = root.path().join("nope/"); + assert!(!nope.exists()); + + let pb = nope.to_path_buf(); + match Rkv::new::<SafeMode>(nope.as_path()).err() { + Some(StoreError::UnsuitableEnvironmentPath(p)) => { + assert_eq!(pb, p); + } + _ => panic!("expected error"), + }; +} + +#[test] +fn test_open_safe() { + let root = Builder::new() + .prefix("test_open_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder_safe() { + let root = Builder::new() + .prefix("test_open_from_builder_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let mut builder = Rkv::environment_builder::<SafeMode>(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +fn test_open_from_builder_with_dir_safe_1() { + let root = Builder::new() + .prefix("test_open_from_builder_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + + let mut builder = Rkv::environment_builder::<SafeMode>(); + builder.set_max_dbs(2); + builder.set_make_dir_if_needed(true); + + let k = Rkv::from_builder(root.path(), builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "rkv: UnsuitableEnvironmentPath(\"bogus\")")] +fn test_open_from_builder_with_dir_safe_2() { + let root = Path::new("bogus"); + println!("Root path: {root:?}"); + assert!(!root.is_dir()); + + let mut builder = Rkv::environment_builder::<SafeMode>(); + builder.set_max_dbs(2); + + let k = Rkv::from_builder(root, builder).expect("rkv"); + check_rkv(&k); +} + +#[test] +#[should_panic(expected = "opened: DbsFull")] +fn test_create_with_capacity_safe_1() { + let root = Builder::new() + .prefix("test_create_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This errors with "opened: DbsFull" because we specified a capacity of one (database), + // and check_rkv already opened one (plus the default database, which doesn't count + // against the limit). + let _zzz = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); +} + +#[test] +fn test_create_with_capacity_safe_2() { + let root = Builder::new() + .prefix("test_create_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + // This doesn't error with "opened: DbsFull" because even though we specified a capacity + // of one (database), and check_rkv already opened one, the default database doesn't + // count against the limit). + let _zzz = k.open_single(None, StoreOptions::create()).expect("opened"); +} + +#[test] +#[should_panic(expected = "opened: SafeModeError(DbNotFoundError)")] +fn test_open_with_capacity_safe_1() { + let root = Builder::new() + .prefix("test_open_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single("zzz", StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_open_with_capacity_safe_2() { + let root = Builder::new() + .prefix("test_open_with_capacity_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let _zzz = k + .open_single(None, StoreOptions::default()) + .expect("opened"); +} + +#[test] +fn test_list_dbs_safe_1() { + let root = Builder::new() + .prefix("test_list_dbs_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 1).expect("rkv"); + check_rkv(&k); + + let mut dbs = k.get_dbs().unwrap(); + dbs.sort(); + assert_eq!(dbs, vec![None, Some("s".to_owned())]); +} + +#[test] +fn test_list_dbs_safe_2() { + let root = Builder::new() + .prefix("test_list_dbs_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 2).expect("rkv"); + check_rkv(&k); + + let _ = k + .open_single("zzz", StoreOptions::create()) + .expect("opened"); + + let mut dbs = k.get_dbs().unwrap(); + dbs.sort(); + assert_eq!( + dbs, + vec![None, Some("s".to_owned()), Some("zzz".to_owned())] + ); +} + +#[test] +fn test_list_dbs_safe_3() { + let root = Builder::new() + .prefix("test_list_dbs_safe") + .tempdir() + .expect("tempdir"); + println!("Root path: {:?}", root.path()); + fs::create_dir_all(root.path()).expect("dir created"); + assert!(root.path().is_dir()); + + let k = Rkv::with_capacity::<SafeMode>(root.path(), 0).expect("rkv"); + + let _ = k.open_single(None, StoreOptions::create()).expect("opened"); + + let mut dbs = k.get_dbs().unwrap(); + dbs.sort(); + assert_eq!(dbs, vec![None]); +} + +#[test] +fn test_round_trip_and_transactions_safe() { + let root = Builder::new() + .prefix("test_round_trip_and_transactions_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "noo").expect("read"), + Some(Value::F64(1234.0.into())) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + // Isolation. Reads won't return values. + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + // Dropped: tx rollback. Reads will still return nothing. + + { + let r = &k.read().unwrap(); + assert_eq!(sk.get(r, "foo").expect("read"), None); + assert_eq!(sk.get(r, "bar").expect("read"), None); + assert_eq!(sk.get(r, "baz").expect("read"), None); + } + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + sk.get(&writer, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + sk.get(&writer, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + // Isolation. Reads still return values. + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + // Dropped: tx rollback. Reads will still return values. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(sk.get(&r, "bar").expect("read"), Some(Value::Bool(true))); + assert_eq!( + sk.get(&r, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + } + + { + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "foo").expect("deleted"); + sk.delete(&mut writer, "bar").expect("deleted"); + sk.delete(&mut writer, "baz").expect("deleted"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + assert_eq!(sk.get(&writer, "baz").expect("read"), None); + + writer.commit().expect("committed"); + } + + // Committed. Reads will succeed but return None to indicate a missing value. + + { + let r = k.read().unwrap(); + assert_eq!(sk.get(&r, "foo").expect("read"), None); + assert_eq!(sk.get(&r, "bar").expect("read"), None); + assert_eq!(sk.get(&r, "baz").expect("read"), None); + } +} + +#[test] +fn test_single_store_clear_safe() { + let root = Builder::new() + .prefix("test_single_store_clear_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + sk.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + let iter = sk.iter_start(&r).expect("iter"); + assert_eq!(iter.count(), 0); + } +} + +#[test] +#[should_panic(expected = "KeyValuePairNotFound")] +fn test_single_store_delete_nonexistent_safe() { + let root = Builder::new() + .prefix("test_single_store_delete_nonexistent_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.delete(&mut writer, "bogus").unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multi_put_get_del_safe() { + let root = Builder::new() + .prefix("test_multi_put_get_del_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let multistore = k.open_multi("multistore", StoreOptions::create()).unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().unwrap(); + + let writer = k.write().unwrap(); + { + let mut iter = multistore.get(&writer, "str1").unwrap(); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 bar"))); + let (id, val) = iter.next().unwrap().unwrap(); + assert_eq!((id, val), (&b"str1"[..], Value::Str("str1 foo"))); + } + writer.commit().unwrap(); + + let mut writer = k.write().unwrap(); + multistore + .delete(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str1").unwrap(), + Some(Value::Str("str1 bar")) + ); + multistore + .delete(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + assert_eq!( + multistore.get_first(&writer, "str2").unwrap(), + Some(Value::Str("str2 foo")) + ); + multistore.delete_all(&mut writer, "str3").unwrap(); + assert_eq!(multistore.get_first(&writer, "str3").unwrap(), None); + writer.commit().unwrap(); +} + +#[test] +#[cfg(feature = "db-dup-sort")] +fn test_multiple_store_clear_safe() { + let root = Builder::new() + .prefix("test_multiple_store_clear_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let multistore = k + .open_multi("multistore", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = k.write().expect("writer"); + multistore + .put(&mut writer, "str1", &Value::Str("str1 foo")) + .unwrap(); + multistore + .put(&mut writer, "str1", &Value::Str("str1 bar")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 foo")) + .unwrap(); + multistore + .put(&mut writer, "str2", &Value::Str("str2 bar")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 foo")) + .unwrap(); + multistore + .put(&mut writer, "str3", &Value::Str("str3 bar")) + .unwrap(); + writer.commit().expect("committed"); + } + + { + let mut writer = k.write().expect("writer"); + multistore.clear(&mut writer).expect("cleared"); + writer.commit().expect("committed"); + } + + { + let r = k.read().unwrap(); + assert_eq!(multistore.get_first(&r, "str1").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str2").expect("read"), None); + assert_eq!(multistore.get_first(&r, "str3").expect("read"), None); + } +} + +#[test] +fn test_open_store_for_read_safe() { + let root = Builder::new() + .prefix("test_open_store_for_read_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + + // First create the store, and start a write transaction on it. + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::Str("bar")) + .expect("write"); + + // Open the same store for read, note that the write transaction is still in progress, + // it should not block the reader though. + let sk_readonly = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + writer.commit().expect("commit"); + + // Now the write transaction is committed, any followed reads should see its change. + let reader = k.read().expect("reader"); + assert_eq!( + sk_readonly.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); +} + +#[test] +#[should_panic(expected = "open a missing store")] +fn test_open_a_missing_store_safe() { + let root = Builder::new() + .prefix("test_open_a_missing_store_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let _sk = k + .open_single("sk", StoreOptions::default()) + .expect("open a missing store"); +} + +#[test] +#[should_panic(expected = "new failed: FileInvalid")] +fn test_open_a_broken_store_safe() { + let root = Builder::new() + .prefix("test_open_a_missing_store_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let dbfile = root.path().join("data.safe.bin"); + fs::write(dbfile, "bogus").expect("dbfile created"); + + let _ = Rkv::new::<SafeMode>(root.path()).expect("new failed"); +} + +#[test] +fn test_open_fail_with_badrslot_safe() { + let root = Builder::new() + .prefix("test_open_fail_with_badrslot_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + + // First create the store + let _sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Open a reader on this store + let _reader = k.read().expect("reader"); + + // Open the same store for read while the reader is in progress will panic + let store = k.open_single("sk", StoreOptions::default()); + match store { + Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (), + _ => panic!("should panic"), + } +} + +#[test] +fn test_create_fail_with_badrslot_safe() { + let root = Builder::new() + .prefix("test_create_fail_with_badrslot_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + + // First create the store + let _sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Open a reader on this store + let _reader = k.read().expect("reader"); + + // Open the same store for read while the reader is in progress will panic + let store = k.open_single("sk", StoreOptions::create()); + match store { + Err(StoreError::OpenAttemptedDuringTransaction(_thread_id)) => (), + _ => panic!("should panic"), + } +} + +#[test] +fn test_read_before_write_num_safe() { + let root = Builder::new() + .prefix("test_read_before_write_num_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a number, modifying it, and then writing it back. + // We have to be done with the Value::I64 before calling Writer::put, + // as the Value::I64 borrows an immutable reference to the Writer. + // So we extract and copy its primitive value. + + fn get_existing_foo( + store: SingleStore<SafeModeDatabase>, + writer: &Writer<SafeModeRwTransaction>, + ) -> Option<i64> { + match store.get(writer, "foo").expect("read") { + Some(Value::I64(val)) => Some(val), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or(99); + existing += 1; + sk.put(&mut writer, "foo", &Value::I64(existing)) + .expect("success"); + + let updated = get_existing_foo(sk, &writer).unwrap_or(99); + assert_eq!(updated, 100); + writer.commit().expect("commit"); +} + +#[test] +fn test_read_before_write_str_safe() { + let root = Builder::new() + .prefix("test_read_before_write_str_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // Test reading a string, modifying it, and then writing it back. + // We have to be done with the Value::Str before calling Writer::put, + // as the Value::Str (and its underlying &str) borrows an immutable + // reference to the Writer. So we copy it to a String. + + fn get_existing_foo( + store: SingleStore<SafeModeDatabase>, + writer: &Writer<SafeModeRwTransaction>, + ) -> Option<String> { + match store.get(writer, "foo").expect("read") { + Some(Value::Str(val)) => Some(val.to_string()), + _ => None, + } + } + + let mut writer = k.write().expect("writer"); + let mut existing = get_existing_foo(sk, &writer).unwrap_or_default(); + existing.push('…'); + sk.put(&mut writer, "foo", &Value::Str(&existing)) + .expect("write"); + + let updated = get_existing_foo(sk, &writer).unwrap_or_default(); + assert_eq!(updated, "…"); + writer.commit().expect("commit"); +} + +#[test] +fn test_isolation_safe() { + let root = Builder::new() + .prefix("test_isolation_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_single("s", StoreOptions::create()).expect("opened"); + + // Add one field. + { + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(1234)).expect("wrote"); + writer.commit().expect("committed"); + } + + { + let reader = k.read().unwrap(); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + } + + // Establish a long-lived reader that outlasts a writer. + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // Start a write transaction. + let mut writer = k.write().expect("writer"); + s.put(&mut writer, "foo", &Value::I64(999)).expect("wrote"); + + // The reader and writer are isolated. + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + assert_eq!(s.get(&writer, "foo").expect("read"), Some(Value::I64(999))); + + // If we commit the writer, we still have isolation. + writer.commit().expect("committed"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(1234))); + + // A new reader sees the committed value. Note that LMDB doesn't allow two + // read transactions to exist in the same thread, so we abort the previous one. + reader.abort(); + let reader = k.read().expect("reader"); + assert_eq!(s.get(&reader, "foo").expect("read"), Some(Value::I64(999))); +} + +#[test] +fn test_blob_safe() { + let root = Builder::new() + .prefix("test_round_trip_blob_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + assert_eq!(sk.get(&writer, "foo").expect("read"), None); + sk.put(&mut writer, "foo", &Value::Blob(&[1, 2, 3, 4])) + .expect("wrote"); + assert_eq!( + sk.get(&writer, "foo").expect("read"), + Some(Value::Blob(&[1, 2, 3, 4])) + ); + + fn u16_to_u8(src: &[u16]) -> Vec<u8> { + let mut dst = vec![0; 2 * src.len()]; + LittleEndian::write_u16_into(src, &mut dst); + dst + } + + fn u8_to_u16(src: &[u8]) -> Vec<u16> { + let mut dst = vec![0; src.len() / 2]; + LittleEndian::read_u16_into(src, &mut dst); + dst + } + + // When storing UTF-16 strings as blobs, we'll need to convert + // their [u16] backing storage to [u8]. Test that converting, writing, + // reading, and converting back works as expected. + let u16_array = [1000, 10000, 54321, 65535]; + assert_eq!(sk.get(&writer, "bar").expect("read"), None); + sk.put(&mut writer, "bar", &Value::Blob(&u16_to_u8(&u16_array))) + .expect("wrote"); + let u8_array = match sk.get(&writer, "bar").expect("read") { + Some(Value::Blob(val)) => val, + _ => &[], + }; + assert_eq!(u8_to_u16(u8_array), u16_array); +} + +#[test] +fn test_sync_safe() { + let root = Builder::new() + .prefix("test_sync_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut builder = Rkv::environment_builder::<SafeMode>(); + builder.set_max_dbs(1); + { + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + { + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + writer.commit().expect("committed"); + k.sync(true).expect("synced"); + } + } + let k = Rkv::from_builder(root.path(), builder).expect("new succeeded"); + let sk = k + .open_single("sk", StoreOptions::default()) + .expect("opened"); + let reader = k.read().expect("reader"); + assert_eq!( + sk.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); +} + +#[test] +fn test_iter_safe() { + let root = Builder::new() + .prefix("test_iter_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + // An iterator over an empty store returns no values. + { + let reader = k.read().unwrap(); + let mut iter = sk.iter_start(&reader).unwrap(); + assert!(iter.next().is_none()); + } + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + sk.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + sk.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Reader.iter() returns (key, value) tuples ordered by key. + let mut iter = sk.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterators don't loop. Once one returns None, additional calls + // to its next() method will always return None. + assert!(iter.next().is_none()); + + // Reader.iter_from() begins iteration at the first key equal to + // or greater than the given key. + let mut iter = sk.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Reader.iter_from() works as expected when the given key is a prefix + // of a key in the store. + let mut iter = sk.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_iter_from_key_greater_than_existing_safe() { + let root = Builder::new() + .prefix("test_iter_from_key_greater_than_existing_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let sk = k.open_single("sk", StoreOptions::create()).expect("opened"); + + let mut writer = k.write().expect("writer"); + sk.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + sk.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + sk.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + sk.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + let mut iter = sk.iter_from(&reader, "nuu").unwrap(); + assert!(iter.next().is_none()); +} + +#[test] +fn test_multiple_store_read_write_safe() { + let root = Builder::new() + .prefix("test_multiple_store_read_write_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + let s3 = k + .open_single("store_3", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + s1.put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + s2.put(&mut writer, "foo", &Value::I64(123)).expect("wrote"); + s3.put(&mut writer, "foo", &Value::Bool(true)) + .expect("wrote"); + + assert_eq!( + s1.get(&writer, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&writer, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&writer, "foo").expect("read"), + Some(Value::Bool(true)) + ); + + writer.commit().expect("committed"); + + let reader = k.read().expect("unbound_reader"); + assert_eq!( + s1.get(&reader, "foo").expect("read"), + Some(Value::Str("bar")) + ); + assert_eq!(s2.get(&reader, "foo").expect("read"), Some(Value::I64(123))); + assert_eq!( + s3.get(&reader, "foo").expect("read"), + Some(Value::Bool(true)) + ); + reader.abort(); + + // test delete across multiple stores + let mut writer = k.write().expect("writer"); + s1.delete(&mut writer, "foo").expect("deleted"); + s2.delete(&mut writer, "foo").expect("deleted"); + s3.delete(&mut writer, "foo").expect("deleted"); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!(s1.get(&reader, "key").expect("value"), None); + assert_eq!(s2.get(&reader, "key").expect("value"), None); + assert_eq!(s3.get(&reader, "key").expect("value"), None); +} + +#[test] +fn test_multiple_store_iter_safe() { + let root = Builder::new() + .prefix("test_multiple_store_iter_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let s1 = k + .open_single("store_1", StoreOptions::create()) + .expect("opened"); + let s2 = k + .open_single("store_2", StoreOptions::create()) + .expect("opened"); + + let mut writer = k.write().expect("writer"); + // Write to "s1" + s1.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s1.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s1.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s1.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s1.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s1.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + // &mut writer to "s2" + s2.put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + s2.put(&mut writer, "noo", &Value::F64(1234.0.into())) + .expect("wrote"); + s2.put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + s2.put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + s2.put(&mut writer, "héllò, töűrîst", &Value::Str("Emil.RuleZ!")) + .expect("wrote"); + s2.put(&mut writer, "你好，遊客", &Value::Str("米克規則")) + .expect("wrote"); + writer.commit().expect("committed"); + + let reader = k.read().unwrap(); + + // Iterate through the whole store in "s1" + let mut iter = s1.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate through the whole store in "s2" + let mut iter = s2.iter_start(&reader).unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "bar"); + assert_eq!(val, Value::Bool(true)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "baz"); + assert_eq!(val, Value::Str("héllo, yöu")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "foo"); + assert_eq!(val, Value::I64(1234)); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "héllò, töűrîst"); + assert_eq!(val, Value::Str("Emil.RuleZ!")); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s1" + let mut iter = s1.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given key in "s2" + let mut iter = s2.iter_from(&reader, "moo").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s1" + let mut iter = s1.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); + + // Iterate from a given prefix in "s2" + let mut iter = s2.iter_from(&reader, "no").unwrap(); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "noo"); + assert_eq!(val, Value::F64(1234.0.into())); + let (key, val) = iter.next().unwrap().unwrap(); + assert_eq!(str::from_utf8(key).expect("key"), "你好，遊客"); + assert_eq!(val, Value::Str("米克規則")); + assert!(iter.next().is_none()); +} + +#[test] +fn test_store_multiple_thread_safe() { + let root = Builder::new() + .prefix("test_multiple_thread_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let rkv_arc = Arc::new(RwLock::new( + Rkv::new::<SafeMode>(root.path()).expect("new succeeded"), + )); + let store = rkv_arc + .read() + .unwrap() + .open_single("test", StoreOptions::create()) + .expect("opened"); + + let num_threads = 10; + let mut write_handles = Vec::with_capacity(num_threads as usize); + let mut read_handles = Vec::with_capacity(num_threads as usize); + + // Note that this isn't intended to demonstrate a good use of threads. + // For this shape of data, it would be more performant to write/read + // all values using one transaction in a single thread. The point here + // is just to confirm that a store can be shared by multiple threads. + + // For each KV pair, spawn a thread that writes it to the store. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + write_handles.push(thread::spawn(move || { + let rkv = rkv_arc.write().expect("rkv"); + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, i.to_string(), &Value::U64(i)) + .expect("written"); + writer.commit().unwrap(); + })); + } + for handle in write_handles { + handle.join().expect("joined"); + } + + // For each KV pair, spawn a thread that reads it from the store + // and returns its value. + for i in 0..num_threads { + let rkv_arc = rkv_arc.clone(); + read_handles.push(thread::spawn(move || { + let rkv = rkv_arc.read().expect("rkv"); + let reader = rkv.read().expect("reader"); + let value = match store.get(&reader, i.to_string()) { + Ok(Some(Value::U64(value))) => value, + Ok(Some(_)) => panic!("value type unexpected"), + Ok(None) => panic!("value not found"), + Err(err) => panic!("{}", err), + }; + assert_eq!(value, i); + value + })); + } + + // Sum the values returned from the threads and confirm that they're + // equal to the sum of values written to the threads. + let thread_sum: u64 = read_handles + .into_iter() + .map(|handle| handle.join().expect("value")) + .sum(); + assert_eq!(thread_sum, (0..num_threads).sum()); +} + +#[test] +fn test_use_value_as_key_safe() { + let root = Builder::new() + .prefix("test_use_value_as_key_safe") + .tempdir() + .expect("tempdir"); + let rkv = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let store = rkv + .open_single("store", StoreOptions::create()) + .expect("opened"); + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::Str("bar")) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // It's possible to retrieve a value with a Reader and then use it + // as a key with a Writer. + { + let reader = &rkv.read().unwrap(); + if let Some(Value::Str(key)) = store.get(reader, "foo").expect("read") { + let mut writer = rkv.write().expect("writer"); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let mut writer = rkv.write().expect("writer"); + store + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also retrieve a Value with a Writer and then use it as a key + // with the same Writer if you copy the value to an owned type + // so the Writer isn't still being borrowed by the retrieved value + // when you try to borrow the Writer again to modify that value. + { + let mut writer = rkv.write().expect("writer"); + if let Some(Value::Str(value)) = store.get(&writer, "foo").expect("read") { + let key = value.to_owned(); + store.delete(&mut writer, key).expect("deleted"); + writer.commit().expect("committed"); + } + } + + { + let name1 = rkv + .open_single("name1", StoreOptions::create()) + .expect("opened"); + let name2 = rkv + .open_single("name2", StoreOptions::create()) + .expect("opened"); + let mut writer = rkv.write().expect("writer"); + name1 + .put(&mut writer, "key1", &Value::Str("bar")) + .expect("wrote"); + name1 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + name2 + .put(&mut writer, "key2", &Value::Str("bar")) + .expect("wrote"); + name2 + .put(&mut writer, "bar", &Value::Str("baz")) + .expect("wrote"); + writer.commit().expect("committed"); + } + + // You can also iterate (store, key) pairs to retrieve foreign keys, + // then iterate those foreign keys to modify/delete them. + // + // You need to open the stores in advance, since opening a store + // uses a write transaction internally, so opening them while a writer + // is extant will hang. + // + // And you need to copy the values to an owned type so the Writer isn't + // still being borrowed by a retrieved value when you try to borrow + // the Writer again to modify another value. + let fields = vec![ + ( + rkv.open_single("name1", StoreOptions::create()) + .expect("opened"), + "key1", + ), + ( + rkv.open_single("name2", StoreOptions::create()) + .expect("opened"), + "key2", + ), + ]; + { + let mut foreignkeys = Vec::new(); + let mut writer = rkv.write().expect("writer"); + for (store, key) in fields.iter() { + if let Some(Value::Str(value)) = store.get(&writer, key).expect("read") { + foreignkeys.push((store, value.to_owned())); + } + } + for (store, key) in foreignkeys.iter() { + store.delete(&mut writer, key).expect("deleted"); + } + writer.commit().expect("committed"); + } +} diff --git a/third_party/rust/rkv/tests/integer-store.rs b/third_party/rust/rkv/tests/integer-store.rs new file mode 100644 index 0000000000..4f27776e79 --- /dev/null +++ b/third_party/rust/rkv/tests/integer-store.rs @@ -0,0 +1,88 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![cfg(feature = "db-int-key")] +#![allow(clippy::many_single_char_names)] + +use std::fs; + +use serde_derive::Serialize; +use tempfile::Builder; + +use rkv::{backend::SafeMode, PrimitiveInt, Rkv, StoreOptions, Value}; + +#[test] +fn test_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let s = k.open_integer("s", StoreOptions::create()).expect("open"); + + macro_rules! test_integer_keys { + ($store:expr, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + $store + .put(&mut writer, $key, &Value::Str("hello!")) + .expect("write"); + assert_eq!( + $store.get(&writer, $key).expect("read"), + Some(Value::Str("hello!")) + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + assert_eq!( + $store.get(&reader, $key).expect("read"), + Some(Value::Str("hello!")) + ); + }}; + } + + // The integer module provides only the u32 integer key variant + // of IntegerStore, so we can use it without further ado. + test_integer_keys!(s, std::u32::MIN); + test_integer_keys!(s, std::u32::MAX); + + // If you want to use another integer key variant, you need to implement + // a newtype, implement PrimitiveInt, and implement or derive Serialize + // for it. Here we do so for the i32 type. + + // DANGER! Doing this enables you to open a store with multiple, + // different integer key types, which may result in unexpected behavior. + // Make sure you know what you're doing! + + let t = k.open_integer("s", StoreOptions::create()).expect("open"); + + #[derive(Serialize)] + struct I32(i32); + impl PrimitiveInt for I32 {} + test_integer_keys!(t, I32(std::i32::MIN)); + test_integer_keys!(t, I32(std::i32::MAX)); + + let u = k.open_integer("s", StoreOptions::create()).expect("open"); + + #[derive(Serialize)] + struct U16(u16); + impl PrimitiveInt for U16 {} + test_integer_keys!(u, U16(std::u16::MIN)); + test_integer_keys!(u, U16(std::u16::MAX)); + + let v = k.open_integer("s", StoreOptions::create()).expect("open"); + + #[derive(Serialize)] + struct U64(u64); + impl PrimitiveInt for U64 {} + test_integer_keys!(v, U64(std::u64::MIN)); + test_integer_keys!(v, U64(std::u64::MAX)); +} diff --git a/third_party/rust/rkv/tests/manager.rs b/third_party/rust/rkv/tests/manager.rs new file mode 100644 index 0000000000..e4785e2461 --- /dev/null +++ b/third_party/rust/rkv/tests/manager.rs @@ -0,0 +1,471 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +use std::{fs, sync::Arc}; + +use tempfile::Builder; + +#[cfg(feature = "lmdb")] +use rkv::backend::{Lmdb, LmdbEnvironment}; +use rkv::{ + backend::{BackendEnvironmentBuilder, RecoveryStrategy, SafeMode, SafeModeEnvironment}, + CloseOptions, Rkv, StoreOptions, Value, +}; + +/// Test that a manager can be created with simple type inference. +#[cfg(feature = "lmdb")] +#[test] +#[allow(clippy::let_underscore_lock)] +fn test_simple() { + type Manager = rkv::Manager<LmdbEnvironment>; + + let _unused = Manager::singleton().write().unwrap(); +} + +/// Test that a manager can be created with simple type inference. +#[test] +#[allow(clippy::let_underscore_lock)] +fn test_simple_safe() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let _unused = Manager::singleton().write().unwrap(); +} + +/// Test that a shared Rkv instance can be created with simple type inference. +#[cfg(feature = "lmdb")] +#[test] +fn test_simple_2() { + type Manager = rkv::Manager<LmdbEnvironment>; + + let root = Builder::new() + .prefix("test_simple_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + let _ = manager + .get_or_create(root.path(), Rkv::new::<Lmdb>) + .unwrap(); +} + +/// Test that a shared Rkv instance can be created with simple type inference. +#[test] +fn test_simple_safe_2() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let root = Builder::new() + .prefix("test_simple_safe_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + let _ = manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .unwrap(); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[cfg(feature = "lmdb")] +#[test] +fn test_same() { + type Manager = rkv::Manager<LmdbEnvironment>; + + let root = Builder::new() + .prefix("test_same") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let p = root.path(); + assert!(Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .is_none()); + + let created_arc = Manager::singleton() + .write() + .unwrap() + .get_or_create(p, Rkv::new::<Lmdb>) + .expect("created"); + let fetched_arc = Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[test] +fn test_same_safe() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let root = Builder::new() + .prefix("test_same_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let p = root.path(); + assert!(Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .is_none()); + + let created_arc = Manager::singleton() + .write() + .unwrap() + .get_or_create(p, Rkv::new::<SafeMode>) + .expect("created"); + let fetched_arc = Manager::singleton() + .read() + .unwrap() + .get(p) + .expect("success") + .expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[cfg(feature = "lmdb")] +#[test] +fn test_same_with_capacity() { + type Manager = rkv::Manager<LmdbEnvironment>; + + let root = Builder::new() + .prefix("test_same_with_capacity") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + + let p = root.path(); + assert!(manager.get(p).expect("success").is_none()); + + let created_arc = manager + .get_or_create_with_capacity(p, 10, Rkv::with_capacity::<Lmdb>) + .expect("created"); + let fetched_arc = manager.get(p).expect("success").expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Test that the manager will return the same Rkv instance each time for each path. +#[test] +fn test_same_with_capacity_safe() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let root = Builder::new() + .prefix("test_same_with_capacity_safe") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut manager = Manager::singleton().write().unwrap(); + + let p = root.path(); + assert!(manager.get(p).expect("success").is_none()); + + let created_arc = manager + .get_or_create_with_capacity(p, 10, Rkv::with_capacity::<SafeMode>) + .expect("created"); + let fetched_arc = manager.get(p).expect("success").expect("existed"); + assert!(Arc::ptr_eq(&created_arc, &fetched_arc)); +} + +/// Some storage drivers are able to discard when the database is corrupted at runtime. +/// Test how these managers can discard corrupted databases and re-open. +#[test] +fn test_safe_mode_corrupt_while_open_1() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let root = Builder::new() + .prefix("test_safe_mode_corrupt_while_open_1") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create environment. + let mut manager = Manager::singleton().write().unwrap(); + let shared_env = manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Write some data. + let store = env + .open_single("store", StoreOptions::create()) + .expect("opened"); + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + store + .put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + // Verify it was flushed to disk. + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(safebin.exists()); + + // Oops, corruption. + fs::write(&safebin, "bogus").expect("dbfile corrupted"); + + // Close everything. + drop(env); + drop(shared_env); + manager + .try_close(root.path(), CloseOptions::default()) + .expect("closed without deleting"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // Recreating environment fails. + manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect_err("not created"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // But we can use a builder and pass `discard_if_corrupted` to deal with it. + let mut builder = Rkv::environment_builder::<SafeMode>(); + builder.set_corruption_recovery_strategy(RecoveryStrategy::Discard); + manager + .get_or_create_from_builder(root.path(), builder, Rkv::from_builder::<SafeMode>) + .expect("created"); + assert!(manager.get(root.path()).expect("success").is_some()); +} + +/// Some storage drivers are able to recover when the database is corrupted at runtime. +/// Test how these managers can recover corrupted databases while open. +#[test] +fn test_safe_mode_corrupt_while_open_2() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let root = Builder::new() + .prefix("test_safe_mode_corrupt_while_open_2") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + // Create environment. + let mut manager = Manager::singleton().write().unwrap(); + let shared_env = manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Write some data. + let store = env + .open_single("store", StoreOptions::create()) + .expect("opened"); + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(1234)) + .expect("wrote"); + store + .put(&mut writer, "bar", &Value::Bool(true)) + .expect("wrote"); + store + .put(&mut writer, "baz", &Value::Str("héllo, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + // Verify it was flushed to disk. + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + assert!(safebin.exists()); + + // Oops, corruption. + fs::write(&safebin, "bogus").expect("dbfile corrupted"); + + // Reading still works. Magic. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + reader.abort(); + + // Writing still works, dbfile will be un-corrupted. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo2", &Value::I64(5678)) + .expect("wrote"); + store + .put(&mut writer, "bar2", &Value::Bool(false)) + .expect("wrote"); + store + .put(&mut writer, "baz2", &Value::Str("byé, yöu")) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + // Close everything. + drop(env); + drop(shared_env); + manager + .try_close(root.path(), CloseOptions::default()) + .expect("closed without deleting"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // Recreate environment. + let shared_env = manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Verify that the dbfile is not corrupted. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(1234)) + ); + assert_eq!( + store.get(&reader, "bar").expect("read"), + Some(Value::Bool(true)) + ); + assert_eq!( + store.get(&reader, "baz").expect("read"), + Some(Value::Str("héllo, yöu")) + ); + assert_eq!( + store.get(&reader, "foo2").expect("read"), + Some(Value::I64(5678)) + ); + assert_eq!( + store.get(&reader, "bar2").expect("read"), + Some(Value::Bool(false)) + ); + assert_eq!( + store.get(&reader, "baz2").expect("read"), + Some(Value::Str("byé, yöu")) + ); +} + +/// Test how the manager can discard corrupted databases, while moving the corrupted one aside for +/// later inspection. +#[test] +fn test_safe_mode_corrupt_while_open_3() { + type Manager = rkv::Manager<SafeModeEnvironment>; + + let root = Builder::new() + .prefix("test_safe_mode_corrupt_while_open_3") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let mut safebin = root.path().to_path_buf(); + safebin.push("data.safe.bin"); + + // Oops, corruption. + fs::write(&safebin, "bogus").expect("dbfile corrupted"); + assert!(safebin.exists(), "Corrupted database file was written to"); + + // Create environment. + let mut manager = Manager::singleton().write().unwrap(); + + // Recreating environment fails. + manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect_err("not created"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // But we can use a builder and pass `RecoveryStrategy::Rename` to deal with it. + let mut builder = Rkv::environment_builder::<SafeMode>(); + builder.set_corruption_recovery_strategy(RecoveryStrategy::Rename); + manager + .get_or_create_from_builder(root.path(), builder, Rkv::from_builder::<SafeMode>) + .expect("created"); + assert!(manager.get(root.path()).expect("success").is_some()); + + assert!(!safebin.exists(), "Database file was moved out of the way"); + + let mut corruptbin = root.path().to_path_buf(); + corruptbin.push("data.safe.bin.corrupt"); + assert!(corruptbin.exists(), "Corrupted database file exists"); + + let shared_env = manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Writing still works. + let store = env + .open_single("store", StoreOptions::create()) + .expect("opened"); + + let reader = env.read().expect("reader"); + assert_eq!(store.get(&reader, "foo").expect("read"), None, "Nothing to be read"); + + // We can write. + let mut writer = env.write().expect("writer"); + store + .put(&mut writer, "foo", &Value::I64(5678)) + .expect("wrote"); + writer.commit().expect("committed"); + env.sync(true).expect("synced"); + + assert!(safebin.exists(), "Database file exists"); + + // Close everything. + drop(env); + drop(shared_env); + manager + .try_close(root.path(), CloseOptions::default()) + .expect("closed without deleting"); + assert!(manager.get(root.path()).expect("success").is_none()); + + // Recreate environment. + let shared_env = manager + .get_or_create(root.path(), Rkv::new::<SafeMode>) + .expect("created"); + let env = shared_env.read().unwrap(); + + // Verify that the dbfile is not corrupted. + let store = env + .open_single("store", StoreOptions::default()) + .expect("opened"); + let reader = env.read().expect("reader"); + assert_eq!( + store.get(&reader, "foo").expect("read"), + Some(Value::I64(5678)), + "Database contains expected value" + ); +} diff --git a/third_party/rust/rkv/tests/multi-integer-store.rs b/third_party/rust/rkv/tests/multi-integer-store.rs new file mode 100644 index 0000000000..bce0060288 --- /dev/null +++ b/third_party/rust/rkv/tests/multi-integer-store.rs @@ -0,0 +1,122 @@ +// Copyright 2018 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![cfg(all(feature = "db-dup-sort", feature = "db-int-key"))] +#![allow(clippy::many_single_char_names)] + +use std::fs; + +use serde_derive::Serialize; +use tempfile::Builder; + +use rkv::{backend::SafeMode, PrimitiveInt, Rkv, StoreOptions, Value}; + +#[test] +fn test_multi_integer_keys() { + let root = Builder::new() + .prefix("test_integer_keys") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let s = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + macro_rules! test_integer_keys { + ($store:expr, $key:expr) => {{ + let mut writer = k.write().expect("writer"); + + $store + .put(&mut writer, $key, &Value::Str("hello1")) + .expect("write"); + $store + .put(&mut writer, $key, &Value::Str("hello2")) + .expect("write"); + $store + .put(&mut writer, $key, &Value::Str("hello3")) + .expect("write"); + let vals = $store + .get(&writer, $key) + .expect("read") + .map(|result| result.expect("ok")) + .map(|(_, v)| v) + .collect::<Vec<Value>>(); + assert_eq!( + vals, + vec![ + Value::Str("hello1"), + Value::Str("hello2"), + Value::Str("hello3") + ] + ); + writer.commit().expect("committed"); + + let reader = k.read().expect("reader"); + let vals = $store + .get(&reader, $key) + .expect("read") + .map(|result| result.expect("ok")) + .map(|(_, v)| v) + .collect::<Vec<Value>>(); + assert_eq!( + vals, + vec![ + Value::Str("hello1"), + Value::Str("hello2"), + Value::Str("hello3") + ] + ); + }}; + } + + // The integer module provides only the u32 integer key variant + // of IntegerStore, so we can use it without further ado. + test_integer_keys!(s, std::u32::MIN); + test_integer_keys!(s, std::u32::MAX); + + // If you want to use another integer key variant, you need to implement + // a newtype, implement PrimitiveInt, and implement or derive Serialize + // for it. Here we do so for the i32 type. + + // DANGER! Doing this enables you to open a store with multiple, + // different integer key types, which may result in unexpected behavior. + // Make sure you know what you're doing! + + let t = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + #[derive(Serialize)] + struct I32(i32); + impl PrimitiveInt for I32 {} + test_integer_keys!(t, I32(std::i32::MIN)); + test_integer_keys!(t, I32(std::i32::MAX)); + + let u = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + #[derive(Serialize)] + struct U16(u16); + impl PrimitiveInt for U16 {} + test_integer_keys!(u, U16(std::u16::MIN)); + test_integer_keys!(u, U16(std::u16::MAX)); + + let v = k + .open_multi_integer("s", StoreOptions::create()) + .expect("open"); + + #[derive(Serialize)] + struct U64(u64); + impl PrimitiveInt for U64 {} + test_integer_keys!(v, U64(std::u64::MIN)); + test_integer_keys!(v, U64(std::u64::MAX)); +} diff --git a/third_party/rust/rkv/tests/test_txn.rs b/third_party/rust/rkv/tests/test_txn.rs new file mode 100644 index 0000000000..1e8fdfb416 --- /dev/null +++ b/third_party/rust/rkv/tests/test_txn.rs @@ -0,0 +1,131 @@ +// Copyright 2018-2019 Mozilla +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. +#![cfg(feature = "db-dup-sort")] + +use std::fs; + +use tempfile::Builder; + +use rkv::{ + backend::{SafeMode, SafeModeDatabase, SafeModeRoCursor, SafeModeRwTransaction}, + Readable, Rkv, StoreOptions, Value, Writer, +}; + +/// Consider a struct like this: +/// struct Sample { +/// id: u64, +/// value: String, +/// date: String, +/// } +/// We would like to index all of the fields so that we can search for the struct not only +/// by ID but also by value and date. When we index the fields individually in their own +/// tables, it is important that we run all operations within a single transaction to +/// ensure coherence of the indices. +/// This test features helper functions for reading and writing the parts of the struct. +/// Note that the reader functions take `Readable` because they might run within a Read +/// Transaction or a Write Transaction. The test demonstrates fetching values via both. + +type SingleStore = rkv::SingleStore<SafeModeDatabase>; +type MultiStore = rkv::MultiStore<SafeModeDatabase>; + +#[test] +fn read_many() { + let root = Builder::new() + .prefix("test_txns") + .tempdir() + .expect("tempdir"); + fs::create_dir_all(root.path()).expect("dir created"); + let k = Rkv::new::<SafeMode>(root.path()).expect("new succeeded"); + let samplestore = k.open_single("s", StoreOptions::create()).expect("open"); + let datestore = k.open_multi("m", StoreOptions::create()).expect("open"); + let valuestore = k.open_multi("m", StoreOptions::create()).expect("open"); + + { + let mut writer = k.write().expect("env write lock"); + + for id in 0..30_u64 { + let value = format!("value{id}"); + let date = format!("2019-06-{id}"); + put_id_field(&mut writer, datestore, &date, id); + put_id_field(&mut writer, valuestore, &value, id); + put_sample(&mut writer, samplestore, id, &value); + } + + // now we read in the same transaction + for id in 0..30_u64 { + let value = format!("value{id}"); + let date = format!("2019-06-{id}"); + let ids = get_ids_by_field(&writer, datestore, &date); + let ids2 = get_ids_by_field(&writer, valuestore, &value); + let samples = get_samples(&writer, samplestore, &ids); + let samples2 = get_samples(&writer, samplestore, &ids2); + println!("{samples:?}, {samples2:?}"); + } + } + + { + let reader = k.read().expect("env read lock"); + for id in 0..30_u64 { + let value = format!("value{id}"); + let date = format!("2019-06-{id}"); + let ids = get_ids_by_field(&reader, datestore, &date); + let ids2 = get_ids_by_field(&reader, valuestore, &value); + let samples = get_samples(&reader, samplestore, &ids); + let samples2 = get_samples(&reader, samplestore, &ids2); + println!("{samples:?}, {samples2:?}"); + } + } +} + +fn get_ids_by_field<'t, T>(txn: &'t T, store: MultiStore, field: &'t str) -> Vec<u64> +where + T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>, +{ + store + .get(txn, field) + .expect("get iterator") + .map(|id| match id.expect("field") { + (_, Value::U64(id)) => id, + _ => panic!("getting value in iter"), + }) + .collect::<Vec<u64>>() +} + +fn get_samples<'t, T>(txn: &'t T, samplestore: SingleStore, ids: &[u64]) -> Vec<String> +where + T: Readable<'t, Database = SafeModeDatabase, RoCursor = SafeModeRoCursor<'t>>, +{ + ids.iter() + .map(|id| { + let bytes = id.to_be_bytes(); + match samplestore.get(txn, &bytes).expect("fetch sample") { + Some(Value::Str(sample)) => String::from(sample), + Some(_) => panic!("wrong type"), + None => panic!("no sample for this id!"), + } + }) + .collect::<Vec<String>>() +} + +fn put_sample( + txn: &mut Writer<SafeModeRwTransaction>, + samplestore: SingleStore, + id: u64, + value: &str, +) { + let idbytes = id.to_be_bytes(); + samplestore + .put(txn, &idbytes, &Value::Str(value)) + .expect("put id"); +} + +fn put_id_field(txn: &mut Writer<SafeModeRwTransaction>, store: MultiStore, field: &str, id: u64) { + store.put(txn, field, &Value::U64(id)).expect("put id"); +} |