diff options
Diffstat (limited to 'third_party/rust/fluent-fallback')
27 files changed, 3144 insertions, 0 deletions
diff --git a/third_party/rust/fluent-fallback/.cargo-checksum.json b/third_party/rust/fluent-fallback/.cargo-checksum.json new file mode 100644 index 0000000000..fd4e643540 --- /dev/null +++ b/third_party/rust/fluent-fallback/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"CHANGELOG.md":"009535601424c3994d30788b8baa50b3527c5c36e26c882e87f023ac23feaee6","Cargo.lock":"ed57b6934296727509b6cf3379bc5039ff63a28f936eac8ec972428aec569793","Cargo.toml":"6ed10beb1162c35f9ff8d0027b1e4a89f4363fda0fe0a361627d3a8580161829","LICENSE-APACHE":"5db2b182453ff32ed40f7da63589c9667a3f8bd8b16b1471b152caae56f77e45","LICENSE-MIT":"49c0b000c03731d9e3970dc059ad4ca345d773681f4a612b0024435b663e0220","README.md":"f722df51c6c20153f073b5dda208b2a23e1d6a6351af0d5dac8dd35090c10b1f","examples/resources/en-US/simple.ftl":"55e8a72973d239c6ed3eb3d9cbc21d37dc90cea9fad85d1d8d73c96d63941629","examples/resources/pl/simple.ftl":"d63d7c62c225897d9f28f166c17e038b8f780dca9e9ee640e81360f23219a212","examples/simple-fallback.rs":"c61dc1a42b09bda3137bb72a08205db4630f47ce6faf9e0aceca4cade02422af","src/bundles.rs":"0696cff42b360fd018746d96136af2a3c38b7b3fdd681642168db8f915f729e3","src/cache.rs":"d0e886b95999120baf513d0438491c68cf37e9f99c515cdcf250ffc8f263f439","src/env.rs":"c11b27dcaf76a0f69d31007f8027cbdc6fb1330082afd28cf9f30aeb0352d127","src/errors.rs":"7424e9cc2cabc20cd987901a663671bfe063749547d271f47ddf67c3d12e1646","src/generator.rs":"f0c9f71baa3ef0e0f205692eb8719390234f7c4d609614b86969d69b8cee52de","src/lib.rs":"f8ee9e689af6d35faa8a2c49b73d3d41c6c27ac655ea0cce045d684e5c3307e4","src/localization.rs":"3fa84b81308e890d9a013e0b05cde7f7811e4f708ca5e5fa9e24ec67315de33c","src/pin_cell/README.md":"b230e479f0ce5de00ce6638aa47cdf1bd30a934df5f3ad33523c3b9f16ffb02b","src/pin_cell/mod.rs":"7247334eb4c6753babe8aeceacf1b36bdbbd60aed86754da61e09e87f1da24d1","src/pin_cell/pin_mut.rs":"116d0ac2353fbc4d2d1084610f90a9aa414b4607bb01595528025ed49889127c","src/pin_cell/pin_ref.rs":"e67ef14faf7d1d47e082732d3a5446e4ae78a25b9577f0819faa1705b236f01d","src/types.rs":"dbe93bf1cfec9b1e28612124c13d37cb0c1e1ba405ce4f4bd16cf3dde98cbb01","tests/localization_test.rs":"c98a27cf8fff60790a67447da584851201a4cadc51482bc81a8c950391513069","tests/resources/en-US/test.ftl":"1103dafb98582e728ddcd30b3fa8ffc1b9d4231cc86296f4e2d8865e9d40b25d","tests/resources/en-US/test2.ftl":"821c99ed74f57635e02877fccf6c2ac9e388997e646c850cd0f86cbd3238b490","tests/resources/pl/test.ftl":"5f7f7a9a8ef2c7175c2a9e2d68ff3748667e8ede877bb6b8a72337ac24e5dfeb","tests/resources/pl/test2.ftl":"68550e8e37adfb49c03a95e6b0a6501d58fbfb6e498cda00b52fc258758245b9"},"package":"08fdcccdeb6c01cb085f2bb3420506e6c67f025cee5db047529838c673a7d82b"}
\ No newline at end of file diff --git a/third_party/rust/fluent-fallback/CHANGELOG.md b/third_party/rust/fluent-fallback/CHANGELOG.md new file mode 100644 index 0000000000..1c6598c441 --- /dev/null +++ b/third_party/rust/fluent-fallback/CHANGELOG.md @@ -0,0 +1,68 @@ +# Changelog + +## Unreleased + + - … + +## fluent-fallback 0.7.0 (Nov 9, 2022) + - The `ResourceId`s are now stored as a `HashSet` rather than as a Vec. Adding a + duplicate `ResourceId` is now a noop. + +## fluent-fallback 0.6.0 (Dec 17, 2021) + - Add `ResourceId` struct which allows fluent resources to be optional. + +## fluent-fallback 0.5.0 (Jul 8, 2021) + - Separate out `Bundles` for state management. + +## fluent-fallback 0.4.4 (May 3, 2021) + - Fix waiting from multiple tasks. (#224) + - Bind locale iterator generics of `LocalesProvider` and `BundleGenerator`. + +## fluent-fallback 0.4.3 (April 26, 2021) + - Align errors even closer to fluent.js + +## fluent-fallback 0.4.2 (April 9, 2021) + - Align errors closer to fluent.js + +## fluent-fallback 0.4.0 (February 9, 2021) + - Use `fluent-bundle` 0.15. + +## fluent-fallback 0.3.0 (February 3, 2021) + - Handle locale management in `Localization`. + +## fluent-fallback 0.2.2 (January 16, 2021) + - Invalidate bundles on resource list change. + +## fluent-fallback 0.2.1 (January 15, 2021) + - Add `Localization::is_sync` + +## fluent-fallback 0.2.0 (January 12, 2021) + - Separate `Sync` and `Async` bundle generators. + - Reorganize fallback logic. + - Separate out prefetching trait. + - Vendor in pin-cell. + +## fluent-fallback 0.1.0 (January 3, 2021) + - Update `fluent-bundle` to 0.14. + - Switch from `elsa` to `chunky-vec`. + - Add `Localization::with_generator`. + - Add support for Streamed bundles. + - Add `LocalizationError`. + - Make `L10nKey`, `L10nMessage` and `L10nAttribute` types. + +## fluent-fallback 0.0.4 (May 6, 2020) + - Update `fluent-bundle` to 0.12. + - Update `unic-langid` to 0.9. + +## fluent-fallback 0.0.3 (February 13, 2020) + - Update `fluent-bundle` to 0.10. + - Update `unic-langid` to 0.8. + +## fluent-fallback 0.0.2 (November 26, 2019) + - Update `fluent-bundle` to 0.9. + - Update `unic-langid` to 0.7. + +## fluent-fallback 0.0.1 (August 1, 2019) + + - This is the first release to be listed in the CHANGELOG. + - Basic support for language fallbacking and runtime locale changes. diff --git a/third_party/rust/fluent-fallback/Cargo.lock b/third_party/rust/fluent-fallback/Cargo.lock new file mode 100644 index 0000000000..f16220d211 --- /dev/null +++ b/third_party/rust/fluent-fallback/Cargo.lock @@ -0,0 +1,415 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "async-trait" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "chunky-vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb7bdea464ae038f09197b82430b921c53619fc8d2bcaf7b151013b3ca008017" + +[[package]] +name = "displaydoc" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "fluent-bundle" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e242c601dec9711505f6d5bbff5bedd4b61b2469f2e8bb8e57ee7c9747a87ffd" +dependencies = [ + "fluent-langneg", + "fluent-syntax", + "intl-memoizer", + "intl_pluralrules", + "rustc-hash", + "self_cell", + "smallvec", + "unic-langid", +] + +[[package]] +name = "fluent-fallback" +version = "0.7.0" +dependencies = [ + "async-trait", + "chunky-vec", + "fluent-bundle", + "fluent-langneg", + "futures", + "once_cell", + "rustc-hash", + "tokio", + "unic-langid", +] + +[[package]] +name = "fluent-langneg" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4ad0989667548f06ccd0e306ed56b61bd4d35458d54df5ec7587c0e8ed5e94" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "fluent-syntax" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0abed97648395c902868fee9026de96483933faa54ea3b40d652f7dfe61ca78" +dependencies = [ + "thiserror", +] + +[[package]] +name = "futures" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" + +[[package]] +name = "futures-executor" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" + +[[package]] +name = "futures-macro" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" + +[[package]] +name = "futures-task" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" + +[[package]] +name = "futures-util" +version = "0.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "hermit-abi" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" +dependencies = [ + "libc", +] + +[[package]] +name = "intl-memoizer" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c310433e4a310918d6ed9243542a6b83ec1183df95dff8f23f87bb88a264a66f" +dependencies = [ + "type-map", + "unic-langid", +] + +[[package]] +name = "intl_pluralrules" +version = "7.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "078ea7b7c29a2b4df841a7f6ac8775ff6074020c6776d48491ce2268e068f972" +dependencies = [ + "unic-langid", +] + +[[package]] +name = "libc" +version = "0.2.137" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7fcc620a3bff7cdd7a365be3376c97191aeaccc2a603e600951e452615bf89" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "num_cpus" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro-hack" +version = "0.5.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" + +[[package]] +name = "proc-macro2" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ea3d908b0e36316caf9e9e2c4625cdde190a7e6f440d794667ed17a1855e725" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbe448f377a7d6961e30f5955f9b8d106c3f5e449d493ee1b125c1d43c2b5179" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "self_cell" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ef965a420fe14fdac7dd018862966a4c14094f900e1650bbc71ddd7d580c8af" + +[[package]] +name = "slab" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" + +[[package]] +name = "syn" +version = "1.0.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a864042229133ada95abf3b54fdc62ef5ccabe9515b64717bcb9a1919e59445d" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10deb33631e3c9018b9baf9dcbbc4f737320d2b576bac10f6aefa048fa407e3e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "982d17546b47146b28f7c22e3d08465f6b8903d0ea13c1660d9d84a6e7adcdbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tinystr" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8aeafdfd935e4a7fe16a91ab711fa52d54df84f9c8f7ca5837a9d1d902ef4c2" +dependencies = [ + "displaydoc", +] + +[[package]] +name = "tokio" +version = "1.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e03c497dc955702ba729190dc4aac6f2a0ce97f913e5b1b5912fc5039d9099" +dependencies = [ + "autocfg", + "num_cpus", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9724f9a975fb987ef7a3cd9be0350edcbe130698af5b8f7a631e23d42d052484" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "type-map" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d3364c5e96cb2ad1603037ab253ddd34d7fb72a58bdddf4b7350760fc69a46" +dependencies = [ + "rustc-hash", +] + +[[package]] +name = "unic-langid" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "398f9ad7239db44fd0f80fe068d12ff22d78354080332a5077dc6f52f14dcf2f" +dependencies = [ + "unic-langid-impl", + "unic-langid-macros", +] + +[[package]] +name = "unic-langid-impl" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e35bfd2f2b8796545b55d7d3fd3e89a0613f68a0d1c8bc28cb7ff96b411a35ff" +dependencies = [ + "tinystr", +] + +[[package]] +name = "unic-langid-macros" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "055e618bf694161ffff0466d95cef3e1a5edc59f6ba1888e97801f2b4ebdc4fe" +dependencies = [ + "proc-macro-hack", + "tinystr", + "unic-langid-impl", + "unic-langid-macros-impl", +] + +[[package]] +name = "unic-langid-macros-impl" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f5cdec05b907f4e2f6843f4354f4ce6a5bebe1a56df320a49134944477ce4d8" +dependencies = [ + "proc-macro-hack", + "quote", + "syn", + "unic-langid-impl", +] + +[[package]] +name = "unicode-ident" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ceab39d59e4c9499d4e5a8ee0e2735b891bb7308ac83dfb4e80cad195c9f6f3" diff --git a/third_party/rust/fluent-fallback/Cargo.toml b/third_party/rust/fluent-fallback/Cargo.toml new file mode 100644 index 0000000000..105de93998 --- /dev/null +++ b/third_party/rust/fluent-fallback/Cargo.toml @@ -0,0 +1,74 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "fluent-fallback" +version = "0.7.0" +authors = [ + "Zibi Braniecki <gandalf@mozilla.com>", + "Staś Małolepszy <stas@mozilla.com>", +] +description = """ +High-level abstraction model for managing localization resources +and runtime localization lifecycle. +""" +homepage = "http://www.projectfluent.org" +readme = "README.md" +keywords = [ + "localization", + "l10n", + "i18n", + "intl", + "internationalization", +] +categories = [ + "localization", + "internationalization", +] +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" +resolver = "1" + +[dependencies.async-trait] +version = "0.1" + +[dependencies.chunky-vec] +version = "0.1" + +[dependencies.fluent-bundle] +version = "0.15.2" + +[dependencies.futures] +version = "0.3" + +[dependencies.once_cell] +version = "1.9" + +[dependencies.rustc-hash] +version = "1" + +[dependencies.unic-langid] +version = "0.9" + +[dev-dependencies.fluent-langneg] +version = "0.13" + +[dev-dependencies.tokio] +version = "1.0" +features = [ + "rt-multi-thread", + "macros", +] + +[dev-dependencies.unic-langid] +version = "0.9" +features = ["macros"] diff --git a/third_party/rust/fluent-fallback/LICENSE-APACHE b/third_party/rust/fluent-fallback/LICENSE-APACHE new file mode 100644 index 0000000000..35582f166b --- /dev/null +++ b/third_party/rust/fluent-fallback/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Mozilla + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/rust/fluent-fallback/LICENSE-MIT b/third_party/rust/fluent-fallback/LICENSE-MIT new file mode 100644 index 0000000000..5655fa311c --- /dev/null +++ b/third_party/rust/fluent-fallback/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright 2017 Mozilla + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/rust/fluent-fallback/README.md b/third_party/rust/fluent-fallback/README.md new file mode 100644 index 0000000000..25f9350f2e --- /dev/null +++ b/third_party/rust/fluent-fallback/README.md @@ -0,0 +1,102 @@ +# Fluent + +`fluent-fallback` is a Rust implementation of the [Project Fluent][] higher level API. + +The `Localization` struct encapsulates a persistant localization context providing +language fallbacking. The instance remains available throughout the whole life cycle of +the corresponding UI, reacting to events such as locale changes, resource updates etc. + +The API can be used directly, or can serve as an example of state manager for `fluent-bundle` and `fluent-resmgr`. + +[](https://crates.io/crates/fluent-fallback) +[](https://github.com/projectfluent/fluent-rs/actions?query=branch%3Amaster+workflow%3A%22Build+and+test%22) +[](https://coveralls.io/github/projectfluent/fluent-rs?branch=master) + +Project Fluent keeps simple things simple and makes complex things possible. +The syntax used for describing translations is easy to read and understand. At +the same time it allows, when necessary, to represent complex concepts from +natural languages like gender, plurals, conjugations, and others. + +[Documentation][] + +[Project Fluent]: http://projectfluent.org +[Documentation]: https://docs.rs/fluent/ + +Usage +----- + +```rust +use fluent_fallback::Localization; + +fn main() { + // generate_messages is a closure that returns an iterator over FluentBundle + // instances. + let loc = Localization::new(vec!["simple.ftl".into()], generate_messages); + + let value = bundle.format_value("hello-world", None); + + assert_eq!(&value, "Hello, world!"); +} +``` + + +Status +------ + +The implementation is in its early stages and supports only some of the Project +Fluent's spec. Consult the [list of milestones][] for more information about +release planning and scope. + +[list of milestones]: https://github.com/projectfluent/fluent-rs/milestones + + +Local Development +----------------- + + cargo build + cargo test + cargo run --example simple-fallback + +When submitting a PR please use [`cargo fmt`][] (nightly). + +[`cargo fmt`]: https://github.com/rust-lang-nursery/rustfmt + + +Learn the FTL syntax +-------------------- + +FTL is a localization file format used for describing translation resources. +FTL stands for _Fluent Translation List_. + +FTL is designed to be simple to read, but at the same time allows to represent +complex concepts from natural languages like gender, plurals, conjugations, and +others. + + hello-user = Hello, { $username }! + +[Read the Fluent Syntax Guide][] in order to learn more about the syntax. If +you're a tool author you may be interested in the formal [EBNF grammar][]. + +[Read the Fluent Syntax Guide]: http://projectfluent.org/fluent/guide/ +[EBNF grammar]: https://github.com/projectfluent/fluent/tree/master/spec + + +Get Involved +------------ + +`fluent-rs` is open-source, licensed under the Apache License, Version 2.0. We +encourage everyone to take a look at our code and we'll listen to your +feedback. + + +Discuss +------- + +We'd love to hear your thoughts on Project Fluent! Whether you're a localizer +looking for a better way to express yourself in your language, or a developer +trying to make your app localizable and multilingual, or a hacker looking for +a project to contribute to, please do get in touch on the mailing list and the +IRC channel. + + - Discourse: https://discourse.mozilla.org/c/fluent + - IRC channel: [irc://irc.mozilla.org/l20n](irc://irc.mozilla.org/l20n) diff --git a/third_party/rust/fluent-fallback/examples/resources/en-US/simple.ftl b/third_party/rust/fluent-fallback/examples/resources/en-US/simple.ftl new file mode 100644 index 0000000000..99f0a6bb6f --- /dev/null +++ b/third_party/rust/fluent-fallback/examples/resources/en-US/simple.ftl @@ -0,0 +1,7 @@ +missing-arg-error = Error: Please provide a number as argument. +input-parse-error = Error: Could not parse input `{ $input }`. Reason: { $reason } +response-msg = + { $value -> + [one] "{ $input }" has one Collatz step. + *[other] "{ $input }" has { $value } Collatz steps. + } diff --git a/third_party/rust/fluent-fallback/examples/resources/pl/simple.ftl b/third_party/rust/fluent-fallback/examples/resources/pl/simple.ftl new file mode 100644 index 0000000000..16173dd92e --- /dev/null +++ b/third_party/rust/fluent-fallback/examples/resources/pl/simple.ftl @@ -0,0 +1,8 @@ +missing-arg-error = Błąd: Proszę wprowadzić liczbę jako argument. +input-parse-error = Błąd: Nie udało się sparsować `{ $input }`. Powód: { $reason } +response-msg = + { $value -> + [one] "{ $input }" ma jeden krok Collatza. + [few] "{ $input }" ma { $value } kroki Collatza. + *[many] "{ $input }" ma { $value } kroków Collatza. + } diff --git a/third_party/rust/fluent-fallback/examples/simple-fallback.rs b/third_party/rust/fluent-fallback/examples/simple-fallback.rs new file mode 100644 index 0000000000..efdc04af2c --- /dev/null +++ b/third_party/rust/fluent-fallback/examples/simple-fallback.rs @@ -0,0 +1,237 @@ +//! This is an example of a simple application +//! which calculates the Collatz conjecture. +//! +//! The function itself is trivial on purpose, +//! so that we can focus on understanding how +//! the application can be made localizable +//! via Fluent. +//! +//! To try the app launch `cargo run --example simple-fallback NUM (LOCALES)` +//! +//! NUM is a number to be calculated, and LOCALES is an optional +//! parameter with a comma-separated list of locales requested by the user. +//! +//! Example: +//! +//! cargo run --example simple-fallback 123 de,pl +//! +//! If the second argument is omitted, `en-US` locale is used as the +//! default one. + +use std::{env, fs, io, path::PathBuf, str::FromStr}; + +use fluent_bundle::{FluentArgs, FluentBundle, FluentResource}; +use fluent_fallback::{ + generator::{BundleGenerator, FluentBundleResult}, + types::ResourceId, + Localization, +}; +use fluent_langneg::{negotiate_languages, NegotiationStrategy}; + +use rustc_hash::FxHashSet; +use unic_langid::{langid, LanguageIdentifier}; + +/// This helper struct holds the scheme for converting +/// resource paths into full paths. It is used to customise +/// `fluent-fallback::SyncLocalization`. +struct Bundles { + res_path_scheme: PathBuf, +} + +/// This helper function allows us to read the list +/// of available locales by reading the list of +/// directories in `./examples/resources`. +/// +/// It is expected that every directory inside it +/// has a name that is a valid BCP47 language tag. +fn get_available_locales() -> io::Result<Vec<LanguageIdentifier>> { + let mut dir = env::current_dir()?; + if dir.to_string_lossy().ends_with("fluent-rs") { + dir.push("fluent-fallback"); + } + dir.push("examples"); + dir.push("resources"); + let res_dir = fs::read_dir(dir)?; + + let locales = res_dir + .into_iter() + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.path().is_dir()) + .filter_map(|dir| { + let file_name = dir.file_name(); + let name = file_name.to_str()?; + Some(name.parse().expect("Parsing failed.")) + }) + .collect(); + Ok(locales) +} + +fn resolve_app_locales<'l>(args: &[String]) -> Vec<LanguageIdentifier> { + let default_locale = langid!("en-US"); + let available = get_available_locales().expect("Retrieving available locales failed."); + + let requested: Vec<LanguageIdentifier> = args.get(2).map_or(vec![], |arg| { + arg.split(",") + .map(|s| s.parse().expect("Parsing locale failed.")) + .collect() + }); + + negotiate_languages( + &requested, + &available, + Some(&default_locale), + NegotiationStrategy::Filtering, + ) + .into_iter() + .cloned() + .collect() +} + +fn get_resource_manager() -> Bundles { + let mut res_path_scheme = env::current_dir().expect("Failed to retrieve current dir."); + + if res_path_scheme.to_string_lossy().ends_with("fluent-rs") { + res_path_scheme.push("fluent-fallback"); + } + res_path_scheme.push("examples"); + res_path_scheme.push("resources"); + + res_path_scheme.push("{locale}"); + res_path_scheme.push("{res_id}"); + + Bundles { res_path_scheme } +} + +static L10N_RESOURCES: &[&str] = &["simple.ftl"]; + +fn main() { + let args: Vec<String> = env::args().collect(); + + let app_locales: Vec<LanguageIdentifier> = resolve_app_locales(&args); + + let bundles = get_resource_manager(); + + let loc = Localization::with_env( + L10N_RESOURCES.iter().map(|&res| res.into()), + true, + app_locales, + bundles, + ); + let bundles = loc.bundles(); + + let mut errors = vec![]; + + match args.get(1) { + Some(input) => match isize::from_str(&input) { + Ok(i) => { + let mut args = FluentArgs::new(); + args.set("input", i); + args.set("value", collatz(i)); + let value = bundles + .format_value_sync("response-msg", Some(&args), &mut errors) + .unwrap() + .unwrap(); + println!("{}", value); + } + Err(err) => { + let mut args = FluentArgs::new(); + args.set("input", input.as_str()); + args.set("reason", err.to_string()); + let value = bundles + .format_value_sync("input-parse-error-msg", Some(&args), &mut errors) + .unwrap() + .unwrap(); + println!("{}", value); + } + }, + None => { + let value = bundles + .format_value_sync("missing-arg-error", None, &mut errors) + .unwrap() + .unwrap(); + println!("{}", value); + } + } +} + +/// Collatz conjecture calculating function. +fn collatz(n: isize) -> isize { + match n { + 1 => 0, + _ => match n % 2 { + 0 => 1 + collatz(n / 2), + _ => 1 + collatz(n * 3 + 1), + }, + } +} + +/// Bundle iterator used by BundleGeneratorSync implementation for Locales. +struct BundleIter { + res_path_scheme: String, + locales: <Vec<LanguageIdentifier> as IntoIterator>::IntoIter, + res_ids: FxHashSet<ResourceId>, +} + +impl Iterator for BundleIter { + type Item = FluentBundleResult<FluentResource>; + + fn next(&mut self) -> Option<Self::Item> { + let locale = self.locales.next()?; + let res_path_scheme = self + .res_path_scheme + .as_str() + .replace("{locale}", &locale.to_string()); + let mut bundle = FluentBundle::new(vec![locale]); + + let mut errors = vec![]; + + for res_id in &self.res_ids { + let res_path = res_path_scheme.as_str().replace("{res_id}", &res_id.value); + let source = fs::read_to_string(res_path).unwrap(); + let res = match FluentResource::try_new(source) { + Ok(res) => res, + Err((res, err)) => { + errors.extend(err.into_iter().map(Into::into)); + res + } + }; + bundle.add_resource(res).unwrap(); + } + + if errors.is_empty() { + Some(Ok(bundle)) + } else { + Some(Err((bundle, errors))) + } + } +} + +impl futures::Stream for BundleIter { + type Item = FluentBundleResult<FluentResource>; + + fn poll_next( + self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<Option<Self::Item>> { + todo!() + } +} + +impl BundleGenerator for Bundles { + type Resource = FluentResource; + type LocalesIter = std::vec::IntoIter<LanguageIdentifier>; + type Iter = BundleIter; + type Stream = BundleIter; + + fn bundles_iter( + &self, + locales: std::vec::IntoIter<LanguageIdentifier>, + res_ids: FxHashSet<ResourceId>, + ) -> Self::Iter { + BundleIter { + res_path_scheme: self.res_path_scheme.to_string_lossy().to_string(), + locales, + res_ids, + } + } +} diff --git a/third_party/rust/fluent-fallback/src/bundles.rs b/third_party/rust/fluent-fallback/src/bundles.rs new file mode 100644 index 0000000000..7ab726d684 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/bundles.rs @@ -0,0 +1,426 @@ +use crate::{ + cache::{AsyncCache, Cache}, + env::LocalesProvider, + errors::LocalizationError, + generator::{BundleGenerator, BundleIterator, BundleStream}, + types::{L10nAttribute, L10nKey, L10nMessage, ResourceId}, +}; +use fluent_bundle::{FluentArgs, FluentBundle, FluentError}; +use rustc_hash::FxHashSet; +use std::borrow::Cow; + +pub enum BundlesInner<G> +where + G: BundleGenerator, +{ + Iter(Cache<G::Iter, G::Resource>), + Stream(AsyncCache<G::Stream, G::Resource>), +} + +pub struct Bundles<G>(BundlesInner<G>) +where + G: BundleGenerator; + +impl<G> Bundles<G> +where + G: BundleGenerator, + G::Iter: BundleIterator, +{ + pub fn prefetch_sync(&self) { + match &self.0 { + BundlesInner::Iter(iter) => iter.prefetch(), + BundlesInner::Stream(_) => panic!("Can't prefetch a sync bundle set asynchronously"), + } + } +} + +impl<G> Bundles<G> +where + G: BundleGenerator, + G::Stream: BundleStream, +{ + pub async fn prefetch_async(&self) { + match &self.0 { + BundlesInner::Iter(_) => panic!("Can't prefetch a async bundle set synchronously"), + BundlesInner::Stream(stream) => stream.prefetch().await, + } + } +} + +impl<G> Bundles<G> +where + G: BundleGenerator, +{ + pub fn new<P>(sync: bool, res_ids: FxHashSet<ResourceId>, generator: &G, provider: &P) -> Self + where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, + { + Self(if sync { + BundlesInner::Iter(Cache::new( + generator.bundles_iter(provider.locales(), res_ids), + )) + } else { + BundlesInner::Stream(AsyncCache::new( + generator.bundles_stream(provider.locales(), res_ids), + )) + }) + } + + pub async fn format_value<'l>( + &'l self, + id: &'l str, + args: Option<&'l FluentArgs<'_>>, + errors: &mut Vec<LocalizationError>, + ) -> Option<Cow<'l, str>> { + match &self.0 { + BundlesInner::Iter(cache) => Self::format_value_from_iter(cache, id, args, errors), + BundlesInner::Stream(stream) => { + Self::format_value_from_stream(stream, id, args, errors).await + } + } + } + + pub async fn format_values<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<Cow<'l, str>>> { + match &self.0 { + BundlesInner::Iter(cache) => Self::format_values_from_iter(cache, keys, errors), + BundlesInner::Stream(stream) => { + Self::format_values_from_stream(stream, keys, errors).await + } + } + } + + pub async fn format_messages<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<L10nMessage<'l>>> { + match &self.0 { + BundlesInner::Iter(cache) => Self::format_messages_from_iter(cache, keys, errors), + BundlesInner::Stream(stream) => { + Self::format_messages_from_stream(stream, keys, errors).await + } + } + } + + pub fn format_value_sync<'l>( + &'l self, + id: &'l str, + args: Option<&'l FluentArgs>, + errors: &mut Vec<LocalizationError>, + ) -> Result<Option<Cow<'l, str>>, LocalizationError> { + match &self.0 { + BundlesInner::Iter(cache) => Ok(Self::format_value_from_iter(cache, id, args, errors)), + BundlesInner::Stream(_) => Err(LocalizationError::SyncRequestInAsyncMode), + } + } + + pub fn format_values_sync<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Result<Vec<Option<Cow<'l, str>>>, LocalizationError> { + match &self.0 { + BundlesInner::Iter(cache) => Ok(Self::format_values_from_iter(cache, keys, errors)), + BundlesInner::Stream(_) => Err(LocalizationError::SyncRequestInAsyncMode), + } + } + + pub fn format_messages_sync<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Result<Vec<Option<L10nMessage<'l>>>, LocalizationError> { + match &self.0 { + BundlesInner::Iter(cache) => Ok(Self::format_messages_from_iter(cache, keys, errors)), + BundlesInner::Stream(_) => Err(LocalizationError::SyncRequestInAsyncMode), + } + } +} + +macro_rules! format_value_from_inner { + ($step:expr, $id:expr, $args:expr, $errors:expr) => { + let mut found_message = false; + + while let Some(bundle) = $step { + let bundle = bundle.as_ref().unwrap_or_else(|(bundle, err)| { + $errors.extend(err.iter().cloned().map(Into::into)); + bundle + }); + + if let Some(msg) = bundle.get_message($id) { + found_message = true; + if let Some(value) = msg.value() { + let mut format_errors = vec![]; + let result = bundle.format_pattern(value, $args, &mut format_errors); + if !format_errors.is_empty() { + $errors.push(LocalizationError::Resolver { + id: $id.to_string(), + locale: bundle.locales[0].clone(), + errors: format_errors, + }); + } + return Some(result); + } else { + $errors.push(LocalizationError::MissingValue { + id: $id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } else { + $errors.push(LocalizationError::MissingMessage { + id: $id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } + if found_message { + $errors.push(LocalizationError::MissingValue { + id: $id.to_string(), + locale: None, + }); + } else { + $errors.push(LocalizationError::MissingMessage { + id: $id.to_string(), + locale: None, + }); + } + return None; + }; +} + +#[derive(Clone)] +enum Value<'l> { + Present(Cow<'l, str>), + Missing, + None, +} + +macro_rules! format_values_from_inner { + ($step:expr, $keys:expr, $errors:expr) => { + let mut cells = vec![Value::None; $keys.len()]; + + while let Some(bundle) = $step { + let bundle = bundle.as_ref().unwrap_or_else(|(bundle, err)| { + $errors.extend(err.iter().cloned().map(Into::into)); + bundle + }); + + let mut has_missing = false; + + for (key, cell) in $keys + .iter() + .zip(&mut cells) + .filter(|(_, cell)| !matches!(cell, Value::Present(_))) + { + if let Some(msg) = bundle.get_message(&key.id) { + if let Some(value) = msg.value() { + let mut format_errors = vec![]; + *cell = Value::Present(bundle.format_pattern( + value, + key.args.as_ref(), + &mut format_errors, + )); + if !format_errors.is_empty() { + $errors.push(LocalizationError::Resolver { + id: key.id.to_string(), + locale: bundle.locales[0].clone(), + errors: format_errors, + }); + } + } else { + *cell = Value::Missing; + has_missing = true; + $errors.push(LocalizationError::MissingValue { + id: key.id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } else { + has_missing = true; + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } + if !has_missing { + break; + } + } + + return $keys + .iter() + .zip(cells) + .map(|(key, value)| match value { + Value::Present(value) => Some(value), + Value::Missing => { + $errors.push(LocalizationError::MissingValue { + id: key.id.to_string(), + locale: None, + }); + None + } + Value::None => { + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: None, + }); + None + } + }) + .collect(); + }; +} + +macro_rules! format_messages_from_inner { + ($step:expr, $keys:expr, $errors:expr) => { + let mut result = vec![None; $keys.len()]; + + let mut is_complete = false; + + while let Some(bundle) = $step { + let bundle = bundle.as_ref().unwrap_or_else(|(bundle, err)| { + $errors.extend(err.iter().cloned().map(Into::into)); + bundle + }); + + let mut has_missing = false; + for (key, cell) in $keys + .iter() + .zip(&mut result) + .filter(|(_, cell)| cell.is_none()) + { + let mut format_errors = vec![]; + let msg = Self::format_message_from_bundle(bundle, key, &mut format_errors); + + if msg.is_none() { + has_missing = true; + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } else if !format_errors.is_empty() { + $errors.push(LocalizationError::Resolver { + id: key.id.to_string(), + locale: bundle.locales.get(0).cloned().unwrap(), + errors: format_errors, + }); + } + + *cell = msg; + } + if !has_missing { + is_complete = true; + break; + } + } + + if !is_complete { + for (key, _) in $keys + .iter() + .zip(&mut result) + .filter(|(_, cell)| cell.is_none()) + { + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: None, + }); + } + } + + return result; + }; +} + +impl<G> Bundles<G> +where + G: BundleGenerator, +{ + fn format_value_from_iter<'l>( + cache: &'l Cache<G::Iter, G::Resource>, + id: &'l str, + args: Option<&'l FluentArgs>, + errors: &mut Vec<LocalizationError>, + ) -> Option<Cow<'l, str>> { + let mut bundle_iter = cache.into_iter(); + format_value_from_inner!(bundle_iter.next(), id, args, errors); + } + + async fn format_value_from_stream<'l>( + stream: &'l AsyncCache<G::Stream, G::Resource>, + id: &'l str, + args: Option<&'l FluentArgs<'_>>, + errors: &mut Vec<LocalizationError>, + ) -> Option<Cow<'l, str>> { + use futures::StreamExt; + + let mut bundle_stream = stream.stream(); + format_value_from_inner!(bundle_stream.next().await, id, args, errors); + } + + async fn format_messages_from_stream<'l>( + stream: &'l AsyncCache<G::Stream, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<L10nMessage<'l>>> { + use futures::StreamExt; + let mut bundle_stream = stream.stream(); + format_messages_from_inner!(bundle_stream.next().await, keys, errors); + } + + async fn format_values_from_stream<'l>( + stream: &'l AsyncCache<G::Stream, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<Cow<'l, str>>> { + use futures::StreamExt; + let mut bundle_stream = stream.stream(); + + format_values_from_inner!(bundle_stream.next().await, keys, errors); + } + + fn format_message_from_bundle<'l>( + bundle: &'l FluentBundle<G::Resource>, + key: &'l L10nKey, + format_errors: &mut Vec<FluentError>, + ) -> Option<L10nMessage<'l>> { + let msg = bundle.get_message(&key.id)?; + let value = msg + .value() + .map(|pattern| bundle.format_pattern(pattern, key.args.as_ref(), format_errors)); + let attributes = msg + .attributes() + .map(|attr| { + let value = bundle.format_pattern(attr.value(), key.args.as_ref(), format_errors); + L10nAttribute { + name: attr.id().into(), + value, + } + }) + .collect(); + Some(L10nMessage { value, attributes }) + } + + fn format_messages_from_iter<'l>( + cache: &'l Cache<G::Iter, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<L10nMessage<'l>>> { + let mut bundle_iter = cache.into_iter(); + format_messages_from_inner!(bundle_iter.next(), keys, errors); + } + + fn format_values_from_iter<'l>( + cache: &'l Cache<G::Iter, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<Cow<'l, str>>> { + let mut bundle_iter = cache.into_iter(); + format_values_from_inner!(bundle_iter.next(), keys, errors); + } +} diff --git a/third_party/rust/fluent-fallback/src/cache.rs b/third_party/rust/fluent-fallback/src/cache.rs new file mode 100644 index 0000000000..32bc33fad1 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/cache.rs @@ -0,0 +1,253 @@ +use std::{ + cell::{RefCell, UnsafeCell}, + cmp::Ordering, + pin::Pin, + task::Context, + task::Poll, + task::Waker, +}; + +use crate::generator::{BundleIterator, BundleStream}; +use crate::pin_cell::{PinCell, PinMut}; +use chunky_vec::ChunkyVec; +use futures::{ready, Stream}; + +pub struct Cache<I, R> +where + I: Iterator, +{ + iter: RefCell<I>, + items: UnsafeCell<ChunkyVec<I::Item>>, + res: std::marker::PhantomData<R>, +} + +impl<I, R> Cache<I, R> +where + I: Iterator, +{ + pub fn new(iter: I) -> Self { + Self { + iter: RefCell::new(iter), + items: Default::default(), + res: std::marker::PhantomData, + } + } + + pub fn len(&self) -> usize { + unsafe { + let items = self.items.get(); + (*items).len() + } + } + + pub fn get(&self, index: usize) -> Option<&I::Item> { + unsafe { + let items = self.items.get(); + (*items).get(index) + } + } + + /// Push, immediately getting a reference to the element + pub fn push_get(&self, new_value: I::Item) -> &I::Item { + unsafe { + let items = self.items.get(); + (*items).push_get(new_value) + } + } +} + +impl<I, R> Cache<I, R> +where + I: BundleIterator + Iterator, +{ + pub fn prefetch(&self) { + self.iter.borrow_mut().prefetch_sync(); + } +} + +pub struct CacheIter<'a, I, R> +where + I: Iterator, +{ + cache: &'a Cache<I, R>, + curr: usize, +} + +impl<'a, I, R> Iterator for CacheIter<'a, I, R> +where + I: Iterator, +{ + type Item = &'a I::Item; + + fn next(&mut self) -> Option<Self::Item> { + let cache_len = self.cache.len(); + match self.curr.cmp(&cache_len) { + Ordering::Less => { + // Cached value + self.curr += 1; + self.cache.get(self.curr - 1) + } + Ordering::Equal => { + // Get the next item from the iterator + let item = self.cache.iter.borrow_mut().next(); + self.curr += 1; + if let Some(item) = item { + Some(self.cache.push_get(item)) + } else { + None + } + } + Ordering::Greater => { + // Ran off the end of the cache + None + } + } + } +} + +impl<'a, I, R> IntoIterator for &'a Cache<I, R> +where + I: Iterator, +{ + type Item = &'a I::Item; + type IntoIter = CacheIter<'a, I, R>; + + fn into_iter(self) -> Self::IntoIter { + CacheIter { + cache: self, + curr: 0, + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +pub struct AsyncCache<S, R> +where + S: Stream, +{ + stream: PinCell<S>, + items: UnsafeCell<ChunkyVec<S::Item>>, + // TODO: Should probably be an SmallVec<[Waker; 1]> or something? I guess + // multiple pending wakes are not really all that common. + pending_wakes: RefCell<Vec<Waker>>, + res: std::marker::PhantomData<R>, +} + +impl<S, R> AsyncCache<S, R> +where + S: Stream, +{ + pub fn new(stream: S) -> Self { + Self { + stream: PinCell::new(stream), + items: Default::default(), + pending_wakes: Default::default(), + res: std::marker::PhantomData, + } + } + + pub fn len(&self) -> usize { + unsafe { + let items = self.items.get(); + (*items).len() + } + } + + pub fn get(&self, index: usize) -> Poll<Option<&S::Item>> { + unsafe { + let items = self.items.get(); + (*items).get(index).into() + } + } + + /// Push, immediately getting a reference to the element + pub fn push_get(&self, new_value: S::Item) -> &S::Item { + unsafe { + let items = self.items.get(); + (*items).push_get(new_value) + } + } + + pub fn stream(&self) -> AsyncCacheStream<'_, S, R> { + AsyncCacheStream { + cache: self, + curr: 0, + } + } +} + +impl<S, R> AsyncCache<S, R> +where + S: BundleStream + Stream, +{ + pub async fn prefetch(&self) { + let pin = unsafe { Pin::new_unchecked(&self.stream) }; + unsafe { PinMut::as_mut(&mut pin.borrow_mut()).get_unchecked_mut() } + .prefetch_async() + .await + } +} + +impl<S, R> AsyncCache<S, R> +where + S: Stream, +{ + // Helper function that gets the next value from wrapped stream. + fn poll_next_item(&self, cx: &mut Context<'_>) -> Poll<Option<S::Item>> { + let pin = unsafe { Pin::new_unchecked(&self.stream) }; + let poll = PinMut::as_mut(&mut pin.borrow_mut()).poll_next(cx); + if poll.is_ready() { + let wakers = std::mem::take(&mut *self.pending_wakes.borrow_mut()); + for waker in wakers { + waker.wake(); + } + } else { + self.pending_wakes.borrow_mut().push(cx.waker().clone()); + } + poll + } +} + +pub struct AsyncCacheStream<'a, S, R> +where + S: Stream, +{ + cache: &'a AsyncCache<S, R>, + curr: usize, +} + +impl<'a, S, R> Stream for AsyncCacheStream<'a, S, R> +where + S: Stream, +{ + type Item = &'a S::Item; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll<Option<Self::Item>> { + let cache_len = self.cache.len(); + match self.curr.cmp(&cache_len) { + Ordering::Less => { + // Cached value + self.curr += 1; + self.cache.get(self.curr - 1) + } + Ordering::Equal => { + // Get the next item from the stream + let item = ready!(self.cache.poll_next_item(cx)); + self.curr += 1; + if let Some(item) = item { + Some(self.cache.push_get(item)).into() + } else { + None.into() + } + } + Ordering::Greater => { + // Ran off the end of the cache + None.into() + } + } + } +} diff --git a/third_party/rust/fluent-fallback/src/env.rs b/third_party/rust/fluent-fallback/src/env.rs new file mode 100644 index 0000000000..cf340fcfdf --- /dev/null +++ b/third_party/rust/fluent-fallback/src/env.rs @@ -0,0 +1,84 @@ +//! Traits required to provide environment driven data for [`Localization`](crate::Localization). +//! +//! Since [`Localization`](crate::Localization) is a long-lived structure, +//! the model in which the user provides ability for the system to react to changes +//! is by implementing the given environmental trait and triggering +//! [`Localization::on_change`](crate::Localization::on_change) method. +//! +//! At the moment just a single trait is provided, which allows the +//! environment to feed a selection of locales to be provided to the instance. +//! +//! The locales provided to [`Localization`](crate::Localization) should be +//! already negotiated to ensure that the resources in those locales +//! are available. The list should also be sorted according to the user +//! preference, as the order is significant for how [`Localization`](crate::Localization) performs +//! fallbacking. +use unic_langid::LanguageIdentifier; + +/// A trait used to provide a selection of locales to be used by the +/// [`Localization`](crate::Localization) instance for runtime +/// locale fallbacking. +/// +/// # Example +/// ``` +/// use fluent_fallback::{Localization, env::LocalesProvider}; +/// use fluent_resmgr::ResourceManager; +/// use unic_langid::LanguageIdentifier; +/// use std::{ +/// rc::Rc, +/// cell::RefCell +/// }; +/// +/// #[derive(Clone)] +/// struct Env { +/// locales: Rc<RefCell<Vec<LanguageIdentifier>>>, +/// } +/// +/// impl Env { +/// pub fn new(locales: Vec<LanguageIdentifier>) -> Self { +/// Self { locales: Rc::new(RefCell::new(locales)) } +/// } +/// +/// pub fn set_locales(&mut self, new_locales: Vec<LanguageIdentifier>) { +/// let mut locales = self.locales.borrow_mut(); +/// locales.clear(); +/// locales.extend(new_locales); +/// } +/// } +/// +/// impl LocalesProvider for Env { +/// type Iter = <Vec<LanguageIdentifier> as IntoIterator>::IntoIter; +/// fn locales(&self) -> Self::Iter { +/// self.locales.borrow().clone().into_iter() +/// } +/// } +/// +/// let res_mgr = ResourceManager::new("./path/{locale}/".to_string()); +/// +/// let mut env = Env::new(vec![ +/// "en-GB".parse().unwrap() +/// ]); +/// +/// let mut loc = Localization::with_env(vec![], true, env.clone(), res_mgr); +/// +/// env.set_locales(vec![ +/// "de".parse().unwrap(), +/// "en-GB".parse().unwrap(), +/// ]); +/// +/// loc.on_change(); +/// +/// // The next format call will attempt to localize to `de` first and +/// // fallback on `en-GB`. +/// ``` +pub trait LocalesProvider { + type Iter: Iterator<Item = LanguageIdentifier>; + fn locales(&self) -> Self::Iter; +} + +impl LocalesProvider for Vec<LanguageIdentifier> { + type Iter = <Vec<LanguageIdentifier> as IntoIterator>::IntoIter; + fn locales(&self) -> Self::Iter { + self.clone().into_iter() + } +} diff --git a/third_party/rust/fluent-fallback/src/errors.rs b/third_party/rust/fluent-fallback/src/errors.rs new file mode 100644 index 0000000000..9170fb1cb9 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/errors.rs @@ -0,0 +1,71 @@ +use fluent_bundle::FluentError; +use std::error::Error; +use unic_langid::LanguageIdentifier; + +#[derive(Debug, PartialEq)] +pub enum LocalizationError { + Bundle { + error: FluentError, + }, + Resolver { + id: String, + locale: LanguageIdentifier, + errors: Vec<FluentError>, + }, + MissingMessage { + id: String, + locale: Option<LanguageIdentifier>, + }, + MissingValue { + id: String, + locale: Option<LanguageIdentifier>, + }, + SyncRequestInAsyncMode, +} + +impl From<FluentError> for LocalizationError { + fn from(error: FluentError) -> Self { + Self::Bundle { error } + } +} + +impl std::fmt::Display for LocalizationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bundle { error } => write!(f, "[fluent][bundle] error: {}", error), + Self::Resolver { id, locale, errors } => { + let errors: Vec<String> = errors.iter().map(|err| err.to_string()).collect(); + write!( + f, + "[fluent][resolver] errors in {}/{}: {}", + locale, + id, + errors.join(", ") + ) + } + Self::MissingMessage { + id, + locale: Some(locale), + } => write!(f, "[fluent] Missing message in locale {}: {}", locale, id), + Self::MissingMessage { id, locale: None } => { + write!(f, "[fluent] Couldn't find a message: {}", id) + } + Self::MissingValue { + id, + locale: Some(locale), + } => write!( + f, + "[fluent] Message has no value in locale {}: {}", + locale, id + ), + Self::MissingValue { id, locale: None } => { + write!(f, "[fluent] Couldn't find a message with value: {}", id) + } + Self::SyncRequestInAsyncMode => { + write!(f, "Triggered synchronous format while in async mode") + } + } + } +} + +impl Error for LocalizationError {} diff --git a/third_party/rust/fluent-fallback/src/generator.rs b/third_party/rust/fluent-fallback/src/generator.rs new file mode 100644 index 0000000000..f13af63cfd --- /dev/null +++ b/third_party/rust/fluent-fallback/src/generator.rs @@ -0,0 +1,41 @@ +use fluent_bundle::{FluentBundle, FluentError, FluentResource}; +use futures::Stream; +use rustc_hash::FxHashSet; +use std::borrow::Borrow; +use unic_langid::LanguageIdentifier; + +use crate::types::ResourceId; + +pub type FluentBundleResult<R> = Result<FluentBundle<R>, (FluentBundle<R>, Vec<FluentError>)>; + +pub trait BundleIterator { + fn prefetch_sync(&mut self) {} +} + +#[async_trait::async_trait(?Send)] +pub trait BundleStream { + async fn prefetch_async(&mut self) {} +} + +pub trait BundleGenerator { + type Resource: Borrow<FluentResource>; + type LocalesIter: Iterator<Item = LanguageIdentifier>; + type Iter: Iterator<Item = FluentBundleResult<Self::Resource>>; + type Stream: Stream<Item = FluentBundleResult<Self::Resource>>; + + fn bundles_iter( + &self, + _locales: Self::LocalesIter, + _res_ids: FxHashSet<ResourceId>, + ) -> Self::Iter { + unimplemented!(); + } + + fn bundles_stream( + &self, + _locales: Self::LocalesIter, + _res_ids: FxHashSet<ResourceId>, + ) -> Self::Stream { + unimplemented!(); + } +} diff --git a/third_party/rust/fluent-fallback/src/lib.rs b/third_party/rust/fluent-fallback/src/lib.rs new file mode 100644 index 0000000000..9dbadc5b98 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/lib.rs @@ -0,0 +1,118 @@ +//! Fluent is a modern localization system designed to improve how software is translated. +//! +//! `fluent-fallback` is a high-level component of the [Fluent Localization +//! System](https://www.projectfluent.org). +//! +//! The crate builds on top of the mid-level [`fluent-bundle`](../fluent-bundle) package, and provides an ergonomic API for highly flexible localization. +//! +//! The functionality of this level is complete, but the API itself is in the +//! early stages and the goal of being ergonomic is yet to be achieved. +//! +//! If the user is willing to work through the challenge of setting up the +//! boiler-plate that will eventually go away, `fluent-fallback` provides +//! a powerful abstraction around [`FluentBundle`](fluent_bundle::FluentBundle) coupled +//! with a localization resource management system. +//! +//! The main struct, [`Localization`], is a long-lived, reactive, multi-lingual +//! struct which allows for strong error recovery and locale +//! fallbacking, exposing synchronous and asynchronous ergonomic methods +//! for [`L10nMessage`](types::L10nMessage) retrieval. +//! +//! [`Localization`] is also an API that is to be used when designing bindings +//! to user interface systems, such as DOM, React, and others. +//! +//! # Example +//! +//! ``` +//! use fluent_fallback::{Localization, types::{ResourceType, ToResourceId}}; +//! use fluent_resmgr::ResourceManager; +//! use unic_langid::langid; +//! +//! let res_mgr = ResourceManager::new("./tests/resources/{locale}/".to_string()); +//! +//! let loc = Localization::with_env( +//! vec![ +//! "test.ftl".into(), +//! "test2.ftl".to_resource_id(ResourceType::Optional), +//! ], +//! true, +//! vec![langid!("en-US")], +//! res_mgr, +//! ); +//! let bundles = loc.bundles(); +//! +//! let mut errors = vec![]; +//! let value = bundles.format_value_sync("hello-world", None, &mut errors) +//! .expect("Failed to format a value"); +//! +//! assert_eq!(value, Some("Hello World [en]".into())); +//! ``` +//! +//! The above example is far from the ergonomical API style the Fluent project +//! is aiming for, but it represents the full scope of functionality intended +//! for the model. +//! +//! # Resource Management +//! +//! Resource management is one of the most complicated parts of a localization system. +//! In particular, modern software may have needs for both synchronous +//! and asynchronous I/O. That, in turn has a large impact on what can happen +//! in case of missing resources, or errors. +//! +//! Resource identifiers can refer to resources that are either required or optional. +//! In the above example, `"test.ftl"` is a required resource (the default using `.into()`), +//! and `"test2.ftl" is an optional resource, which you can create via the +//! [`ToResourceId`](fluent_fallback::types::ToResourceId) trait. +//! +//! A required resource must be present in order for the a bundle to be considered valid. +//! If a required resource is missing for a given locale, a bundle will not be generated for that locale. +//! +//! A bundle is still considered valid if an optional resource is missing. A bundle will still be generated +//! and the entries for the missing optional resource will simply be missing from the bundle. This should be +//! used sparingly in exceptional cases where you do not want `Localization` to fall back to the next +//! locale if there is a missing resource. Marking all resources as optional will increase the state space +//! that the solver has to search through, and will have a negative impact on performance. +//! +//! Currently, [`Localization`] can be specialized over an implementation of +//! [`generator::BundleGenerator`] trait which provides a method to generate an +//! [`Iterator`] and [`Stream`](futures::stream::Stream). +//! +//! This is not very elegant and will likely be improved in the future, but for the time being, if +//! the customer doesn't need one of the modes, the unnecessary method should use the +//! `unimplemented!()` macro as its body. +//! +//! `fluent-resmgr` provides a simple resource manager which handles synchronous I/O +//! and uses local file system to store resources in a directory structure. +//! +//! That model is often sufficient and the user can either use `fluent-resmgr` or write +//! a similar API to provide the generator for [`Localization`]. +//! +//! Alternatively, a much more sophisticated resource manager can be used. Mozilla +//! for its needs in Firefox uses [`L10nRegistry`](https://github.com/zbraniecki/l10nregistry-rs) +//! library which implements [`BundleGenerator`](generator::BundleGenerator). +//! +//! # Locale Management +//! +//! As a long lived structure, the [`Localization`] is intended to handle runtime locale +//! management. +//! +//! In the example above, [`Vec<LagnuageIdentifier>`](unic_langid::LanguageIdentifier) +//! provides a static list of locales that the [`Localization`] handles, but that's just the +//! simplest implementation of the [`env::LocalesProvider`], and one can implement +//! a much more sophisticated one that reacts to user or environment driven changes, and +//! called [`Localization::on_change`] to trigger a new locales to be used for the +//! next translation request. +//! +//! See [`env::LocalesProvider`] trait for an example of a reactive system implementation. +mod bundles; +mod cache; +pub mod env; +mod errors; +pub mod generator; +mod localization; +mod pin_cell; +pub mod types; + +pub use bundles::Bundles; +pub use errors::LocalizationError; +pub use localization::Localization; diff --git a/third_party/rust/fluent-fallback/src/localization.rs b/third_party/rust/fluent-fallback/src/localization.rs new file mode 100644 index 0000000000..5424bcf311 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/localization.rs @@ -0,0 +1,137 @@ +use crate::{ + bundles::Bundles, + env::LocalesProvider, + generator::{BundleGenerator, BundleIterator, BundleStream}, + types::ResourceId, +}; +use once_cell::sync::OnceCell; +use rustc_hash::FxHashSet; +use std::rc::Rc; + +pub struct Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, +{ + bundles: OnceCell<Rc<Bundles<G>>>, + generator: G, + provider: P, + sync: bool, + res_ids: FxHashSet<ResourceId>, +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter> + Default, + P: LocalesProvider + Default, +{ + pub fn new<I>(res_ids: I, sync: bool) -> Self + where + I: IntoIterator<Item = ResourceId>, + { + Self { + bundles: OnceCell::new(), + generator: G::default(), + provider: P::default(), + sync, + res_ids: FxHashSet::from_iter(res_ids.into_iter()), + } + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, +{ + pub fn with_env<I>(res_ids: I, sync: bool, provider: P, generator: G) -> Self + where + I: IntoIterator<Item = ResourceId>, + { + Self { + bundles: OnceCell::new(), + generator, + provider, + sync, + res_ids: FxHashSet::from_iter(res_ids.into_iter()), + } + } + + pub fn is_sync(&self) -> bool { + self.sync + } + + pub fn add_resource_id<T: Into<ResourceId>>(&mut self, res_id: T) { + self.res_ids.insert(res_id.into()); + self.on_change(); + } + + pub fn add_resource_ids(&mut self, res_ids: Vec<ResourceId>) { + self.res_ids.extend(res_ids); + self.on_change(); + } + + pub fn remove_resource_id<T: PartialEq<ResourceId>>(&mut self, res_id: T) -> usize { + self.res_ids.retain(|x| !res_id.eq(x)); + self.on_change(); + self.res_ids.len() + } + + pub fn remove_resource_ids(&mut self, res_ids: Vec<ResourceId>) -> usize { + self.res_ids.retain(|x| !res_ids.contains(x)); + self.on_change(); + self.res_ids.len() + } + + pub fn set_async(&mut self) { + if self.sync { + self.sync = false; + self.on_change(); + } + } + + pub fn on_change(&mut self) { + self.bundles.take(); + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + G::Iter: BundleIterator, + P: LocalesProvider, +{ + pub fn prefetch_sync(&mut self) { + let bundles = self.bundles(); + bundles.prefetch_sync(); + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + G::Stream: BundleStream, + P: LocalesProvider, +{ + pub async fn prefetch_async(&mut self) { + let bundles = self.bundles(); + bundles.prefetch_async().await + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, +{ + pub fn bundles(&self) -> &Rc<Bundles<G>> { + self.bundles.get_or_init(|| { + Rc::new(Bundles::new( + self.sync, + self.res_ids.clone(), + &self.generator, + &self.provider, + )) + }) + } +} diff --git a/third_party/rust/fluent-fallback/src/pin_cell/README.md b/third_party/rust/fluent-fallback/src/pin_cell/README.md new file mode 100644 index 0000000000..b1c475f51b --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/README.md @@ -0,0 +1,2 @@ +This is a temporary fork of https://github.com/withoutboats/pin-cell until +https://github.com/withoutboats/pin-cell/issues/6 gets resolved. diff --git a/third_party/rust/fluent-fallback/src/pin_cell/mod.rs b/third_party/rust/fluent-fallback/src/pin_cell/mod.rs new file mode 100644 index 0000000000..175f9677e0 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/mod.rs @@ -0,0 +1,97 @@ +#![deny(missing_docs, missing_debug_implementations)] +//! This library defines the `PinCell` type, a pinning variant of the standard +//! library's `RefCell`. +//! +//! It is not safe to "pin project" through a `RefCell` - getting a pinned +//! reference to something inside the `RefCell` when you have a pinned +//! refernece to the `RefCell` - because `RefCell` is too powerful. +//! +//! A `PinCell` is slightly less powerful than `RefCell`: unlike a `RefCell`, +//! one cannot get a mutable reference into a `PinCell`, only a pinned mutable +//! reference (`Pin<&mut T>`). This makes pin projection safe, allowing you +//! to use interior mutability with the knowledge that `T` will never actually +//! be moved out of the `RefCell` that wraps it. + +mod pin_mut; +mod pin_ref; + +use core::cell::{BorrowMutError, RefCell, RefMut}; +use core::pin::Pin; + +pub use pin_mut::PinMut; +pub use pin_ref::PinRef; + +/// A mutable memory location with dynamically checked borrow rules +/// +/// Unlike `RefCell`, this type only allows *pinned* mutable access to the +/// inner value, enabling a "pin-safe" version of interior mutability. +/// +/// See the standard library documentation for more information. +#[derive(Default, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] +pub struct PinCell<T: ?Sized> { + inner: RefCell<T>, +} + +impl<T> PinCell<T> { + /// Creates a new `PinCell` containing `value`. + pub const fn new(value: T) -> PinCell<T> { + PinCell { + inner: RefCell::new(value), + } + } +} + +impl<T: ?Sized> PinCell<T> { + /// Mutably borrows the wrapped value, preserving its pinnedness. + /// + /// The borrow lasts until the returned `PinMut` or all `PinMut`s derived + /// from it exit scope. The value cannot be borrowed while this borrow is + /// active. + pub fn borrow_mut(self: Pin<&Self>) -> PinMut<'_, T> { + self.try_borrow_mut().expect("already borrowed") + } + + /// Mutably borrows the wrapped value, preserving its pinnedness, + /// returning an error if the value is currently borrowed. + /// + /// The borrow lasts until the returned `PinMut` or all `PinMut`s derived + /// from it exit scope. The value cannot be borrowed while this borrow is + /// active. + /// + /// This is the non-panicking variant of `borrow_mut`. + pub fn try_borrow_mut<'a>(self: Pin<&'a Self>) -> Result<PinMut<'a, T>, BorrowMutError> { + let ref_mut: RefMut<'a, T> = Pin::get_ref(self).inner.try_borrow_mut()?; + + // this is a pin projection from Pin<&PinCell<T>> to Pin<RefMut<T>> + // projecting is safe because: + // + // - for<T: ?Sized> (PinCell<T>: Unpin) imples (RefMut<T>: Unpin) + // holds true + // - PinCell does not implement Drop + // + // see discussion on tracking issue #49150 about pin projection + // invariants + let pin_ref_mut: Pin<RefMut<'a, T>> = unsafe { Pin::new_unchecked(ref_mut) }; + + Ok(PinMut { inner: pin_ref_mut }) + } +} + +impl<T> From<T> for PinCell<T> { + fn from(value: T) -> PinCell<T> { + PinCell::new(value) + } +} + +impl<T> From<RefCell<T>> for PinCell<T> { + fn from(cell: RefCell<T>) -> PinCell<T> { + PinCell { inner: cell } + } +} + +impl<T> From<PinCell<T>> for RefCell<T> { + fn from(input: PinCell<T>) -> Self { + input.inner + } +} +// TODO CoerceUnsized diff --git a/third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs b/third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs new file mode 100644 index 0000000000..09a4d4a6fb --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs @@ -0,0 +1,50 @@ +use core::cell::RefMut; +use core::fmt; +use core::ops::Deref; +use core::pin::Pin; + +#[derive(Debug)] +/// A wrapper type for a mutably borrowed value from a `PinCell<T>`. +pub struct PinMut<'a, T: ?Sized> { + pub(crate) inner: Pin<RefMut<'a, T>>, +} + +impl<'a, T: ?Sized> Deref for PinMut<'a, T> { + type Target = T; + + fn deref(&self) -> &T { + &*self.inner + } +} + +impl<'a, T: ?Sized> PinMut<'a, T> { + /// Get a pinned mutable reference to the value inside this wrapper. + pub fn as_mut<'b>(orig: &'b mut PinMut<'a, T>) -> Pin<&'b mut T> { + orig.inner.as_mut() + } +} + +/* TODO implement these APIs + +impl<'a, T: ?Sized> PinMut<'a, T> { + pub fn map<U, F>(orig: PinMut<'a, T>, f: F) -> PinMut<'a, U> where + F: FnOnce(Pin<&mut T>) -> Pin<&mut U>, + { + panic!() + } + + pub fn map_split<U, V, F>(orig: PinMut<'a, T>, f: F) -> (PinMut<'a, U>, PinMut<'a, V>) where + F: FnOnce(Pin<&mut T>) -> (Pin<&mut U>, Pin<&mut V>) + { + panic!() + } +} +*/ + +impl<'a, T: fmt::Display + ?Sized> fmt::Display for PinMut<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + <T as fmt::Display>::fmt(&**self, f) + } +} + +// TODO CoerceUnsized diff --git a/third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs b/third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs new file mode 100644 index 0000000000..46f9cfdabb --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs @@ -0,0 +1,47 @@ +use core::cell::Ref; +use core::fmt; +use core::ops::Deref; +use core::pin::Pin; + +#[derive(Debug)] +/// A wrapper type for a immutably borrowed value from a `PinCell<T>`. +pub struct PinRef<'a, T: ?Sized> { + pub(crate) inner: Pin<Ref<'a, T>>, +} + +impl<'a, T: ?Sized> Deref for PinRef<'a, T> { + type Target = T; + + fn deref(&self) -> &T { + &*self.inner + } +} + +/* TODO implement these APIs + +impl<'a, T: ?Sized> PinRef<'a, T> { + pub fn clone(orig: &PinRef<'a, T>) -> PinRef<'a, T> { + panic!() + } + + pub fn map<U, F>(orig: PinRef<'a, T>, f: F) -> PinRef<'a, U> where + F: FnOnce(Pin<&T>) -> Pin<&U>, + { + panic!() + } + + pub fn map_split<U, V, F>(orig: PinRef<'a, T>, f: F) -> (PinRef<'a, U>, PinRef<'a, V>) where + F: FnOnce(Pin<&T>) -> (Pin<&U>, Pin<&V>) + { + panic!() + } +} +*/ + +impl<'a, T: fmt::Display + ?Sized> fmt::Display for PinRef<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + <T as fmt::Display>::fmt(&**self, f) + } +} + +// TODO CoerceUnsized diff --git a/third_party/rust/fluent-fallback/src/types.rs b/third_party/rust/fluent-fallback/src/types.rs new file mode 100644 index 0000000000..6b87fa0522 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/types.rs @@ -0,0 +1,141 @@ +use fluent_bundle::FluentArgs; +use std::borrow::Cow; + +#[derive(Debug)] +pub struct L10nKey<'l> { + pub id: Cow<'l, str>, + pub args: Option<FluentArgs<'l>>, +} + +impl<'l> From<&'l str> for L10nKey<'l> { + fn from(id: &'l str) -> Self { + Self { + id: id.into(), + args: None, + } + } +} + +#[derive(Debug, Clone)] +pub struct L10nAttribute<'l> { + pub name: Cow<'l, str>, + pub value: Cow<'l, str>, +} + +#[derive(Debug, Clone)] +pub struct L10nMessage<'l> { + pub value: Option<Cow<'l, str>>, + pub attributes: Vec<L10nAttribute<'l>>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ResourceType { + /// This is a required resource. + /// + /// A bundle generator should not consider a solution as valid + /// if this resource is missing. + /// + /// This is the default when creating a [`ResourceId`]. + Required, + + /// This is an optional resource. + /// + /// A bundle generator should still populate a partial solution + /// even if this resource is missing. + /// + /// This is intended for experimental and/or under-development + /// resources that may not have content for all supported locales. + /// + /// This should be used sparingly, as it will greatly increase + /// the state space of the search for valid solutions which can + /// have a severe impact on performance. + Optional, +} + +/// A resource identifier for a localization resource. +#[derive(Debug, Clone, Hash)] +pub struct ResourceId { + /// The resource identifier. + pub value: String, + + /// The [`ResourceType`] for this resource. + /// + /// The default value (when converting from another type) is + /// [`ResourceType::Required`]. You should only set this to + /// [`ResourceType::Optional`] for experimental or under-development + /// features that may not yet have content in all eventually-supported locales. + /// + /// Setting this value to [`ResourceType::Optional`] for all resources + /// may have a severe impact on performance due to increasing the state space + /// of the solver. + pub resource_type: ResourceType, +} + +impl ResourceId { + pub fn new<S: Into<String>>(value: S, resource_type: ResourceType) -> Self { + Self { + value: value.into(), + resource_type, + } + } + + /// Returns [`true`] if the resource has [`ResourceType::Required`], + /// otherwise returns [`false`]. + pub fn is_required(&self) -> bool { + matches!(self.resource_type, ResourceType::Required) + } + + /// Returns [`true`] if the resource has [`ResourceType::Optional`], + /// otherwise returns [`false`]. + pub fn is_optional(&self) -> bool { + matches!(self.resource_type, ResourceType::Optional) + } +} + +impl<S: Into<String>> From<S> for ResourceId { + fn from(id: S) -> Self { + Self { + value: id.into(), + resource_type: ResourceType::Required, + } + } +} + +impl std::fmt::Display for ResourceId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.value) + } +} + +impl PartialEq<str> for ResourceId { + fn eq(&self, other: &str) -> bool { + self.value.as_str().eq(other) + } +} + +impl Eq for ResourceId {} +impl PartialEq for ResourceId { + fn eq(&self, other: &Self) -> bool { + self.value.eq(&other.value) + } +} + +/// A trait for creating a [`ResourceId`] from another type. +/// +/// This differs from the [`From`] trait in that the [`From`] trait +/// always takes the default resource type of [`ResourceType::Required`]. +/// +/// If you need to create a resource with a non-default [`ResourceType`], +/// such as [`ResourceType::Optional`], then use this trait. +/// +/// This trait is automatically implemented for types that implement [`Into<String>`]. +pub trait ToResourceId { + /// Creates a [`ResourceId`] from [`self`], given a [`ResourceType`]. + fn to_resource_id(self, resource_type: ResourceType) -> ResourceId; +} + +impl<S: Into<String>> ToResourceId for S { + fn to_resource_id(self, resource_type: ResourceType) -> ResourceId { + ResourceId::new(self.into(), resource_type) + } +} diff --git a/third_party/rust/fluent-fallback/tests/localization_test.rs b/third_party/rust/fluent-fallback/tests/localization_test.rs new file mode 100644 index 0000000000..b48f0a05b9 --- /dev/null +++ b/third_party/rust/fluent-fallback/tests/localization_test.rs @@ -0,0 +1,518 @@ +use std::borrow::Cow; +use std::fs; + +use fluent_bundle::{ + resolver::errors::{ReferenceKind, ResolverError}, + FluentArgs, FluentBundle, FluentError, FluentResource, +}; +use fluent_fallback::{ + env::LocalesProvider, + generator::{BundleGenerator, FluentBundleResult}, + types::{L10nKey, ResourceId}, + Localization, LocalizationError, +}; +use rustc_hash::FxHashSet; +use std::cell::RefCell; +use std::rc::Rc; +use unic_langid::{langid, LanguageIdentifier}; + +struct InnerLocales { + locales: RefCell<Vec<LanguageIdentifier>>, +} + +impl InnerLocales { + pub fn insert(&self, index: usize, element: LanguageIdentifier) { + self.locales.borrow_mut().insert(index, element); + } +} + +#[derive(Clone)] +struct Locales { + inner: Rc<InnerLocales>, +} + +impl Locales { + pub fn new(locales: Vec<LanguageIdentifier>) -> Self { + Self { + inner: Rc::new(InnerLocales { + locales: RefCell::new(locales), + }), + } + } + + pub fn insert(&mut self, index: usize, element: LanguageIdentifier) { + self.inner.insert(index, element); + } +} + +impl LocalesProvider for Locales { + type Iter = <Vec<LanguageIdentifier> as IntoIterator>::IntoIter; + fn locales(&self) -> Self::Iter { + self.inner.locales.borrow().clone().into_iter() + } +} + +// Due to limitation of trait, we need a nameable Iterator type. Due to the +// lack of GATs, these have to own members instead of taking slices. +struct BundleIter { + locales: <Vec<LanguageIdentifier> as IntoIterator>::IntoIter, + res_ids: FxHashSet<ResourceId>, +} + +impl Iterator for BundleIter { + type Item = FluentBundleResult<FluentResource>; + + fn next(&mut self) -> Option<Self::Item> { + let locale = self.locales.next()?; + + let mut bundle = FluentBundle::new(vec![locale.clone()]); + bundle.set_use_isolating(false); + + let mut errors = vec![]; + + for res_id in &self.res_ids { + let full_path = format!("./tests/resources/{}/{}", locale, res_id); + let source = fs::read_to_string(full_path).unwrap(); + let res = match FluentResource::try_new(source) { + Ok(res) => res, + Err((res, err)) => { + errors.extend(err.into_iter().map(Into::into)); + res + } + }; + bundle.add_resource(res).unwrap(); + } + if errors.is_empty() { + Some(Ok(bundle)) + } else { + Some(Err((bundle, errors))) + } + } +} + +impl futures::Stream for BundleIter { + type Item = FluentBundleResult<FluentResource>; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + _cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<Option<Self::Item>> { + if let Some(locale) = self.locales.next() { + let mut bundle = FluentBundle::new(vec![locale.clone()]); + bundle.set_use_isolating(false); + + let mut errors = vec![]; + for res_id in &self.res_ids { + let full_path = format!("./tests/resources/{}/{}", locale, res_id.value); + let source = fs::read_to_string(full_path).unwrap(); + let res = match FluentResource::try_new(source) { + Ok(res) => res, + Err((res, err)) => { + errors.extend(err.into_iter().map(Into::into)); + res + } + }; + bundle.add_resource(res).unwrap(); + } + if errors.is_empty() { + Some(Ok(bundle)).into() + } else { + Some(Err((bundle, errors))).into() + } + } else { + None.into() + } + } +} + +struct ResourceManager; + +impl BundleGenerator for ResourceManager { + type Resource = FluentResource; + type LocalesIter = std::vec::IntoIter<LanguageIdentifier>; + type Iter = BundleIter; + type Stream = BundleIter; + + fn bundles_iter( + &self, + locales: Self::LocalesIter, + res_ids: FxHashSet<ResourceId>, + ) -> Self::Iter { + BundleIter { locales, res_ids } + } + + fn bundles_stream( + &self, + locales: Self::LocalesIter, + res_ids: FxHashSet<ResourceId>, + ) -> Self::Stream { + BundleIter { locales, res_ids } + } +} + +#[test] +fn localization_format() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into(), "test2.ftl".into()]; + let locales = Locales::new(vec![langid!("pl"), langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales, res_mgr); + let bundles = loc.bundles(); + + let value = bundles + .format_value_sync("hello-world", None, &mut errors) + .unwrap(); + assert_eq!(value, Some(Cow::Borrowed("Hello World [pl]"))); + + let value = bundles + .format_value_sync("missing-message", None, &mut errors) + .unwrap(); + assert_eq!(value, None); + + let value = bundles + .format_value_sync("hello-world-3", None, &mut errors) + .unwrap(); + assert_eq!(value, Some(Cow::Borrowed("Hello World 3 [en]"))); + + assert_eq!(errors.len(), 4); +} + +#[test] +fn localization_on_change() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into(), "test2.ftl".into()]; + + let mut locales = Locales::new(vec![langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let mut loc = Localization::with_env(resource_ids, true, locales.clone(), res_mgr); + let bundles = loc.bundles(); + + let value = bundles + .format_value_sync("hello-world", None, &mut errors) + .unwrap(); + assert_eq!(value, Some(Cow::Borrowed("Hello World [en]"))); + + locales.insert(0, langid!("pl")); + loc.on_change(); + + let bundles = loc.bundles(); + let value = bundles + .format_value_sync("hello-world", None, &mut errors) + .unwrap(); + assert_eq!(value, Some(Cow::Borrowed("Hello World [pl]"))); +} + +#[test] +fn localization_format_value_missing_errors() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into(), "test2.ftl".into()]; + + let locales = Locales::new(vec![langid!("pl"), langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales.clone(), res_mgr); + let bundles = loc.bundles(); + + let _ = bundles + .format_value_sync("missing-message", None, &mut errors) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: None + }, + ] + ); + + errors.clear(); + + let _ = bundles + .format_value_sync("message-3", None, &mut errors) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: None + }, + ] + ); +} + +#[test] +fn localization_format_value_sync_missing_errors() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into(), "test2.ftl".into()]; + + let locales = Locales::new(vec![langid!("pl"), langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales.clone(), res_mgr); + let bundles = loc.bundles(); + + let _ = bundles + .format_value_sync("missing-message", None, &mut errors) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: None + }, + ] + ); + + errors.clear(); + + let _ = bundles + .format_value_sync("message-3", None, &mut errors) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: None + }, + ] + ); +} + +#[test] +fn localization_format_values_sync_missing_errors() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into(), "test2.ftl".into()]; + + let locales = Locales::new(vec![langid!("pl"), langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales.clone(), res_mgr); + let bundles = loc.bundles(); + + let _ = bundles + .format_values_sync( + &["missing-message".into(), "missing-message-2".into()], + &mut errors, + ) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingMessage { + id: "missing-message-2".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingMessage { + id: "missing-message-2".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: None + }, + LocalizationError::MissingMessage { + id: "missing-message-2".to_string(), + locale: None + }, + ] + ); + + errors.clear(); + + let _ = bundles + .format_values_sync(&["message-3".into()], &mut errors) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingValue { + id: "message-3".to_string(), + locale: None + }, + ] + ); +} + +#[test] +fn localization_format_messages_sync_missing_errors() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into(), "test2.ftl".into()]; + + let locales = Locales::new(vec![langid!("pl"), langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales.clone(), res_mgr); + let bundles = loc.bundles(); + + let _ = bundles + .format_messages_sync( + &["missing-message".into(), "missing-message-2".into()], + &mut errors, + ) + .unwrap(); + assert_eq!( + errors, + vec![ + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingMessage { + id: "missing-message-2".to_string(), + locale: Some(langid!("pl")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingMessage { + id: "missing-message-2".to_string(), + locale: Some(langid!("en-US")) + }, + LocalizationError::MissingMessage { + id: "missing-message".to_string(), + locale: None + }, + LocalizationError::MissingMessage { + id: "missing-message-2".to_string(), + locale: None + }, + ] + ); +} + +#[test] +fn localization_format_missing_argument_error() { + let resource_ids: Vec<ResourceId> = vec!["test2.ftl".into()]; + let locales = Locales::new(vec![langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales, res_mgr); + let bundles = loc.bundles(); + + let mut args = FluentArgs::new(); + args.set("userName", "John"); + let keys = vec![L10nKey { + id: "message-4".into(), + args: Some(args), + }]; + + let msgs = bundles.format_messages_sync(&keys, &mut errors).unwrap(); + assert_eq!( + msgs.get(0).unwrap().as_ref().unwrap().value, + Some(Cow::Borrowed("Hello, John. [en]")) + ); + assert_eq!(errors.len(), 0); + + let keys = vec![L10nKey { + id: "message-4".into(), + args: None, + }]; + let msgs = bundles.format_messages_sync(&keys, &mut errors).unwrap(); + assert_eq!( + msgs.get(0).unwrap().as_ref().unwrap().value, + Some(Cow::Borrowed("Hello, {$userName}. [en]")) + ); + assert_eq!( + errors, + vec![LocalizationError::Resolver { + id: "message-4".to_string(), + locale: langid!("en-US"), + errors: vec![FluentError::ResolverError(ResolverError::Reference( + ReferenceKind::Variable { + id: "userName".to_string(), + } + ))], + },] + ); +} + +#[tokio::test] +async fn localization_handle_state_changes_mid_async() { + let resource_ids: Vec<ResourceId> = vec!["test.ftl".into()]; + let locales = Locales::new(vec![langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let mut loc = Localization::with_env(resource_ids, false, locales, res_mgr); + + let bundles = loc.bundles().clone(); + + loc.add_resource_id("test2.ftl".to_string()); + + bundles.format_value("key", None, &mut errors).await; +} + +#[test] +fn localization_duplicate_resources() { + let resource_ids: Vec<ResourceId> = + vec!["test.ftl".into(), "test2.ftl".into(), "test2.ftl".into()]; + let locales = Locales::new(vec![langid!("pl"), langid!("en-US")]); + let res_mgr = ResourceManager; + let mut errors = vec![]; + + let loc = Localization::with_env(resource_ids, true, locales, res_mgr); + let bundles = loc.bundles(); + + let value = bundles + .format_value_sync("hello-world", None, &mut errors) + .unwrap(); + assert_eq!(value, Some(Cow::Borrowed("Hello World [pl]"))); + + assert_eq!(errors.len(), 0, "There were no errors"); +} diff --git a/third_party/rust/fluent-fallback/tests/resources/en-US/test.ftl b/third_party/rust/fluent-fallback/tests/resources/en-US/test.ftl new file mode 100644 index 0000000000..c6a7390acf --- /dev/null +++ b/third_party/rust/fluent-fallback/tests/resources/en-US/test.ftl @@ -0,0 +1,4 @@ +hello-world = Hello World [en] + +message-1 = Message 1 Value [en] + .attr1 = Message 1 Attribute [en] diff --git a/third_party/rust/fluent-fallback/tests/resources/en-US/test2.ftl b/third_party/rust/fluent-fallback/tests/resources/en-US/test2.ftl new file mode 100644 index 0000000000..faeae76cfe --- /dev/null +++ b/third_party/rust/fluent-fallback/tests/resources/en-US/test2.ftl @@ -0,0 +1,10 @@ +hello-world-2 = Hello World 2 [en] +hello-world-3 = Hello World 3 [en] + +message-2 = Message 2 Value [en] + .attr1 = Message 2 Attribute [en] + +message-3 = + .attr1 = Message 3 Attribute [en] + +message-4 = Hello, { $userName }. [en] diff --git a/third_party/rust/fluent-fallback/tests/resources/pl/test.ftl b/third_party/rust/fluent-fallback/tests/resources/pl/test.ftl new file mode 100644 index 0000000000..6e2f8835f1 --- /dev/null +++ b/third_party/rust/fluent-fallback/tests/resources/pl/test.ftl @@ -0,0 +1,4 @@ +hello-world = Hello World [pl] + +message-1 = Message 1 Value [pl] + .attr1 = Message 1 Attribute [pl] diff --git a/third_party/rust/fluent-fallback/tests/resources/pl/test2.ftl b/third_party/rust/fluent-fallback/tests/resources/pl/test2.ftl new file mode 100644 index 0000000000..35d646a520 --- /dev/null +++ b/third_party/rust/fluent-fallback/tests/resources/pl/test2.ftl @@ -0,0 +1,9 @@ +hello-world-2 = Hello World 2 [pl] + +message-2 = Message 2 Value [pl] + .attr1 = Message 2 Attribute [pl] + +message-3 = + .attr1 = Message 3 Attribute [pl] + +message-4 = Hello, { $userName }. [pl] |