diff options
Diffstat (limited to 'third_party/rust/fluent-bundle')
26 files changed, 3305 insertions, 0 deletions
diff --git a/third_party/rust/fluent-bundle/.cargo-checksum.json b/third_party/rust/fluent-bundle/.cargo-checksum.json new file mode 100644 index 0000000000..5ad5a9d239 --- /dev/null +++ b/third_party/rust/fluent-bundle/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"87a01e2e130c153cac13b916dba613ff4d9dde0795ebc607932d9ea9c960cf77","LICENSE-APACHE":"5db2b182453ff32ed40f7da63589c9667a3f8bd8b16b1471b152caae56f77e45","LICENSE-MIT":"49c0b000c03731d9e3970dc059ad4ca345d773681f4a612b0024435b663e0220","README.md":"444c8934a3bbec88cbf5e41d7b5fcb2d1f9c7e2c69a906f6df5fe18171942157","benches/resolver.rs":"bd46c8b710ac1898a0e69324a7ecf9aa38d577337bc5855a07ca0ad1043603a1","benches/resolver_iai.rs":"e9e940f4c09908069d474d379a0230dfc6fa44325300d72975e8f1d9ef64f6f1","examples/README.md":"99a51f7d388d2da3c5291e68de5264feaf642ba9a22f6f882c3b940c1b6429b2","src/args.rs":"51346e9ec84f2eeb4462e0e993b1bbb307585a2a40e41f6d0d745889bca56a7d","src/bundle.rs":"3da63b685acf559ee80fa489885da126f7c68405026ac065a07e559e2186a77f","src/concurrent.rs":"be77275513918809b98c554b26a65c6a9cf2a7bf52db3bbaf21ebdd34d94c651","src/entry.rs":"e1507b0e4c3e6d0d2efc5d622f4156a5156b9eeb40d9c5353cb7fdc236c38189","src/errors.rs":"a357f3a09335d31e362aa99a8d82eab4e238fdec8498141990f61ede58f4dabb","src/lib.rs":"58a0c929322f83aac41280da035b50adbcdb05d8a8376359d58c177cd9755eab","src/memoizer.rs":"922084f71f02d0532056db9b41cec4c1434001fe60215ee6f6ac8e3fd2518f12","src/message.rs":"4a3c95d3ecd016aeaa5da07e99d78de62f13aac8aa447818aabd0f63f2d143c4","src/resolver/errors.rs":"beaf41fabbfd11211cb2c3db6ca0ba26bccf75817bed05a92b980393edfb3f9f","src/resolver/expression.rs":"f18413de1a6b3ba43c062e24d58a60d63f4dad66bcec67ed55756ff5014f9347","src/resolver/inline_expression.rs":"089ad6745d0790478ea698fd530f2236c550889f9be75e245ed94bba4b883884","src/resolver/mod.rs":"d1b15ce110ea49876909412c12c4c1841052bb80f4838a934dfccd6a5264855c","src/resolver/pattern.rs":"64162a7e2ad0df82d463d14ac6a472005bba4cef4a7e73fe2a9529e811124a85","src/resolver/scope.rs":"816f51146c38affea54c6e0911e3522f998485829e619cca5f72cec05180de59","src/resource.rs":"cd56a14c01da2a689a408f0e5edd789e8557ae9327fa79788bf4ddb9c83431c7","src/types/mod.rs":"1cd65301fb32233fd241a79c6fa873edcb40a271330601360f72e2c452900509","src/types/number.rs":"2d3b403e5f545e2f4a3a16aec0bb019a4cc8e5ac0ab2db5642ba8039fbe203a8","src/types/plural.rs":"f28834e71d6970d5eb48089132f5242433b1e62b90765d85e3c76f805eecc92e"},"package":"e242c601dec9711505f6d5bbff5bedd4b61b2469f2e8bb8e57ee7c9747a87ffd"}
\ No newline at end of file diff --git a/third_party/rust/fluent-bundle/Cargo.toml b/third_party/rust/fluent-bundle/Cargo.toml new file mode 100644 index 0000000000..6d36ec3acd --- /dev/null +++ b/third_party/rust/fluent-bundle/Cargo.toml @@ -0,0 +1,78 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2018" +name = "fluent-bundle" +version = "0.15.2" +authors = ["Zibi Braniecki <gandalf@mozilla.com>", "Staś Małolepszy <stas@mozilla.com>"] +include = ["src/**/*", "benches/*.rs", "Cargo.toml", "README.md", "LICENSE-APACHE", "LICENSE-MIT"] +description = "A localization system designed to unleash the entire expressive power of\nnatural language translations.\n" +homepage = "http://www.projectfluent.org" +readme = "README.md" +keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] +categories = ["localization", "internationalization"] +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" + +[[bench]] +name = "resolver" +harness = false + +[[bench]] +name = "resolver_iai" +harness = false +[dependencies.fluent-langneg] +version = "0.13" + +[dependencies.fluent-syntax] +version = "0.11" + +[dependencies.intl-memoizer] +version = "0.5" + +[dependencies.intl_pluralrules] +version = "7.0.1" + +[dependencies.rustc-hash] +version = "1" + +[dependencies.self_cell] +version = "0.10" + +[dependencies.smallvec] +version = "1" + +[dependencies.unic-langid] +version = "0.9" +[dev-dependencies.criterion] +version = "0.3" + +[dev-dependencies.iai] +version = "0.1" + +[dev-dependencies.rand] +version = "0.8" + +[dev-dependencies.serde] +version = "1.0" +features = ["derive"] + +[dev-dependencies.serde_yaml] +version = "0.8" + +[dev-dependencies.unic-langid] +version = "0.9" +features = ["macros"] + +[features] +all-benchmarks = [] +default = [] diff --git a/third_party/rust/fluent-bundle/LICENSE-APACHE b/third_party/rust/fluent-bundle/LICENSE-APACHE new file mode 100644 index 0000000000..35582f166b --- /dev/null +++ b/third_party/rust/fluent-bundle/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Mozilla + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/rust/fluent-bundle/LICENSE-MIT b/third_party/rust/fluent-bundle/LICENSE-MIT new file mode 100644 index 0000000000..5655fa311c --- /dev/null +++ b/third_party/rust/fluent-bundle/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright 2017 Mozilla + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/rust/fluent-bundle/README.md b/third_party/rust/fluent-bundle/README.md new file mode 100644 index 0000000000..c8e1b1124a --- /dev/null +++ b/third_party/rust/fluent-bundle/README.md @@ -0,0 +1,111 @@ +# Fluent + +`fluent-rs` is a Rust implementation of [Project Fluent][], a localization +framework designed to unleash the entire expressive power of natural language +translations. + +[](https://crates.io/crates/fluent-bundle) +[](https://github.com/projectfluent/fluent-rs/actions?query=branch%3Amaster+workflow%3A%22Build+and+test%22) +[](https://coveralls.io/github/projectfluent/fluent-rs?branch=master) + +Project Fluent keeps simple things simple and makes complex things possible. +The syntax used for describing translations is easy to read and understand. At +the same time it allows, when necessary, to represent complex concepts from +natural languages like gender, plurals, conjugations, and others. + +[Documentation][] + +[Project Fluent]: http://projectfluent.org +[Documentation]: https://docs.rs/fluent/ + +Usage +----- + +```rust +use fluent_bundle::{FluentBundle, FluentResource}; +use unic_langid::langid; + +fn main() { + let ftl_string = "hello-world = Hello, world!".to_owned(); + let res = FluentResource::try_new(ftl_string) + .expect("Could not parse an FTL string."); + + let langid_en = langid!("en"); + let mut bundle = FluentBundle::new(vec![langid_en]); + + bundle.add_resource(&res) + .expect("Failed to add FTL resources to the bundle."); + + let msg = bundle.get_message("hello-world") + .expect("Failed to retrieve a message."); + let val = msg.value.expect("Message has no value."); + + let mut errors = vec![]; + let value = bundle.format_pattern(val, None, &mut errors); + + assert_eq!(&value, "Hello, world!"); +} +``` + + +Status +------ + +The implementation is in its early stages and supports only some of the Project +Fluent's spec. Consult the [list of milestones][] for more information about +release planning and scope. + +[list of milestones]: https://github.com/projectfluent/fluent-rs/milestones + + +Local Development +----------------- + + cargo build + cargo test + cargo bench + cargo run --example simple-app + +When submitting a PR please use [`cargo fmt`][] (nightly). + +[`cargo fmt`]: https://github.com/rust-lang-nursery/rustfmt + + +Learn the FTL syntax +-------------------- + +FTL is a localization file format used for describing translation resources. +FTL stands for _Fluent Translation List_. + +FTL is designed to be simple to read, but at the same time allows to represent +complex concepts from natural languages like gender, plurals, conjugations, and +others. + + hello-user = Hello, { $username }! + +[Read the Fluent Syntax Guide][] in order to learn more about the syntax. If +you're a tool author you may be interested in the formal [EBNF grammar][]. + +[Read the Fluent Syntax Guide]: http://projectfluent.org/fluent/guide/ +[EBNF grammar]: https://github.com/projectfluent/fluent/tree/master/spec + + +Get Involved +------------ + +`fluent-rs` is open-source, licensed under the Apache License, Version 2.0. We +encourage everyone to take a look at our code and we'll listen to your +feedback. + + +Discuss +------- + +We'd love to hear your thoughts on Project Fluent! Whether you're a localizer +looking for a better way to express yourself in your language, or a developer +trying to make your app localizable and multilingual, or a hacker looking for +a project to contribute to, please do get in touch on the mailing list and the +IRC channel. + + - Discourse: https://discourse.mozilla.org/c/fluent + - IRC channel: [irc://irc.mozilla.org/l20n](irc://irc.mozilla.org/l20n) diff --git a/third_party/rust/fluent-bundle/benches/resolver.rs b/third_party/rust/fluent-bundle/benches/resolver.rs new file mode 100644 index 0000000000..2116d1b91e --- /dev/null +++ b/third_party/rust/fluent-bundle/benches/resolver.rs @@ -0,0 +1,168 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::BenchmarkId; +use criterion::Criterion; +use std::collections::HashMap; +use std::fs::File; +use std::io; +use std::io::Read; +use std::rc::Rc; + +use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue}; +use fluent_syntax::ast; +use unic_langid::langid; + +fn read_file(path: &str) -> Result<String, io::Error> { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn get_strings(tests: &[&'static str]) -> HashMap<&'static str, String> { + let mut ftl_strings = HashMap::new(); + for test in tests { + let path = format!("./benches/{}.ftl", test); + ftl_strings.insert(*test, read_file(&path).expect("Couldn't load file")); + } + return ftl_strings; +} + +fn get_ids(res: &FluentResource) -> Vec<String> { + res.entries() + .filter_map(|entry| match entry { + ast::Entry::Message(ast::Message { id, .. }) => Some(id.name.to_owned()), + _ => None, + }) + .collect() +} + +fn get_args(name: &str) -> Option<FluentArgs> { + match name { + "preferences" => { + let mut prefs_args = FluentArgs::new(); + prefs_args.set("name", FluentValue::from("John")); + prefs_args.set("tabCount", FluentValue::from(5)); + prefs_args.set("count", FluentValue::from(3)); + prefs_args.set("version", FluentValue::from("65.0")); + prefs_args.set("path", FluentValue::from("/tmp")); + prefs_args.set("num", FluentValue::from(4)); + prefs_args.set("email", FluentValue::from("john@doe.com")); + prefs_args.set("value", FluentValue::from(4.5)); + prefs_args.set("unit", FluentValue::from("mb")); + prefs_args.set("service-name", FluentValue::from("Mozilla Disk")); + Some(prefs_args) + } + _ => None, + } +} + +fn add_functions<R>(name: &'static str, bundle: &mut FluentBundle<R>) { + match name { + "preferences" => { + bundle + .add_function("PLATFORM", |_args, _named_args| { + return "linux".into(); + }) + .expect("Failed to add a function to the bundle."); + } + _ => {} + } +} + +fn get_bundle(name: &'static str, source: &str) -> (FluentBundle<FluentResource>, Vec<String>) { + let res = FluentResource::try_new(source.to_owned()).expect("Couldn't parse an FTL source"); + let ids = get_ids(&res); + let lids = vec![langid!("en")]; + let mut bundle = FluentBundle::new(lids); + bundle + .add_resource(res) + .expect("Couldn't add FluentResource to the FluentBundle"); + add_functions(name, &mut bundle); + (bundle, ids) +} + +fn resolver_bench(c: &mut Criterion) { + let tests = &[ + #[cfg(feature = "all-benchmarks")] + "simple", + "preferences", + #[cfg(feature = "all-benchmarks")] + "menubar", + #[cfg(feature = "all-benchmarks")] + "unescape", + ]; + let ftl_strings = get_strings(tests); + + let mut group = c.benchmark_group("construct"); + for name in tests { + let source = ftl_strings.get(name).expect("Failed to find the source."); + group.bench_with_input(BenchmarkId::from_parameter(name), &source, |b, source| { + let res = Rc::new( + FluentResource::try_new(source.to_string()).expect("Couldn't parse an FTL source"), + ); + b.iter(|| { + let lids = vec![langid!("en")]; + let mut bundle = FluentBundle::new(lids); + bundle + .add_resource(res.clone()) + .expect("Couldn't add FluentResource to the FluentBundle"); + add_functions(name, &mut bundle); + }) + }); + } + group.finish(); + + let mut group = c.benchmark_group("resolve"); + for name in tests { + let source = ftl_strings.get(name).expect("Failed to find the source."); + group.bench_with_input(BenchmarkId::from_parameter(name), &source, |b, source| { + let (bundle, ids) = get_bundle(name, source); + let args = get_args(name); + b.iter(|| { + let mut s = String::new(); + for id in &ids { + let msg = bundle.get_message(id).expect("Message found"); + let mut errors = vec![]; + if let Some(value) = msg.value() { + let _ = bundle.write_pattern(&mut s, value, args.as_ref(), &mut errors); + s.clear(); + } + for attr in msg.attributes() { + let _ = + bundle.write_pattern(&mut s, attr.value(), args.as_ref(), &mut errors); + s.clear(); + } + assert!(errors.len() == 0, "Resolver errors: {:#?}", errors); + } + }) + }); + } + group.finish(); + + let mut group = c.benchmark_group("resolve_to_str"); + for name in tests { + let source = ftl_strings.get(name).expect("Failed to find the source."); + group.bench_with_input(BenchmarkId::from_parameter(name), &source, |b, source| { + let (bundle, ids) = get_bundle(name, source); + let args = get_args(name); + b.iter(|| { + for id in &ids { + let msg = bundle.get_message(id).expect("Message found"); + let mut errors = vec![]; + if let Some(value) = msg.value() { + let _ = bundle.format_pattern(value, args.as_ref(), &mut errors); + } + for attr in msg.attributes() { + let _ = bundle.format_pattern(attr.value(), args.as_ref(), &mut errors); + } + assert!(errors.len() == 0, "Resolver errors: {:#?}", errors); + } + }) + }); + } + group.finish(); +} + +criterion_group!(benches, resolver_bench); +criterion_main!(benches); diff --git a/third_party/rust/fluent-bundle/benches/resolver_iai.rs b/third_party/rust/fluent-bundle/benches/resolver_iai.rs new file mode 100644 index 0000000000..10212f6f39 --- /dev/null +++ b/third_party/rust/fluent-bundle/benches/resolver_iai.rs @@ -0,0 +1,79 @@ +use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue}; +use fluent_syntax::ast; +use unic_langid::{langid, LanguageIdentifier}; + +const LANG_EN: LanguageIdentifier = langid!("en"); + +fn add_functions<R>(name: &'static str, bundle: &mut FluentBundle<R>) { + match name { + "preferences" => { + bundle + .add_function("PLATFORM", |_args, _named_args| { + return "linux".into(); + }) + .expect("Failed to add a function to the bundle."); + } + _ => {} + } +} + +fn get_args(name: &str) -> Option<FluentArgs> { + match name { + "preferences" => { + let mut prefs_args = FluentArgs::new(); + prefs_args.set("name", FluentValue::from("John")); + prefs_args.set("tabCount", FluentValue::from(5)); + prefs_args.set("count", FluentValue::from(3)); + prefs_args.set("version", FluentValue::from("65.0")); + prefs_args.set("path", FluentValue::from("/tmp")); + prefs_args.set("num", FluentValue::from(4)); + prefs_args.set("email", FluentValue::from("john@doe.com")); + prefs_args.set("value", FluentValue::from(4.5)); + prefs_args.set("unit", FluentValue::from("mb")); + prefs_args.set("service-name", FluentValue::from("Mozilla Disk")); + Some(prefs_args) + } + _ => None, + } +} + +fn get_ids(res: &FluentResource) -> Vec<String> { + res.entries() + .filter_map(|entry| match entry { + ast::Entry::Message(ast::Message { id, .. }) => Some(id.name.to_owned()), + _ => None, + }) + .collect() +} + +fn iai_resolve_preferences() { + let files = &[include_str!("preferences.ftl")]; + for source in files { + let res = FluentResource::try_new(source.to_string()).expect("failed to parse FTL."); + let ids = get_ids(&res); + let mut bundle = FluentBundle::new(vec![LANG_EN]); + bundle.add_resource(res).expect("Failed to add a resource."); + add_functions("preferences", &mut bundle); + let args = get_args("preferences"); + let mut s = String::new(); + for id in &ids { + let msg = bundle.get_message(id).expect("Message found"); + let mut errors = vec![]; + if let Some(value) = msg.value() { + bundle + .write_pattern(&mut s, value, args.as_ref(), &mut errors) + .expect("Failed to write a pattern."); + s.clear(); + } + for attr in msg.attributes() { + bundle + .write_pattern(&mut s, attr.value(), args.as_ref(), &mut errors) + .expect("Failed to write a pattern."); + s.clear(); + } + assert!(errors.len() == 0, "Resolver errors: {:#?}", errors); + } + } +} + +iai::main!(iai_resolve_preferences); diff --git a/third_party/rust/fluent-bundle/examples/README.md b/third_party/rust/fluent-bundle/examples/README.md new file mode 100644 index 0000000000..2411aeb903 --- /dev/null +++ b/third_party/rust/fluent-bundle/examples/README.md @@ -0,0 +1,6 @@ +This directory contains a set of examples +of how to use Fluent. + +Start with the `simple-app.rs` which is a very +trivial example of a command line application +with localization handled by Fluent. diff --git a/third_party/rust/fluent-bundle/src/args.rs b/third_party/rust/fluent-bundle/src/args.rs new file mode 100644 index 0000000000..b2d17a84b6 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/args.rs @@ -0,0 +1,120 @@ +use std::borrow::Cow; +use std::iter::FromIterator; + +use crate::types::FluentValue; + +/// A map of arguments passed from the code to +/// the localization to be used for message +/// formatting. +/// +/// # Example +/// +/// ``` +/// use fluent_bundle::{FluentArgs, FluentBundle, FluentResource}; +/// +/// let mut args = FluentArgs::new(); +/// args.set("user", "John"); +/// args.set("emailCount", 5); +/// +/// let res = FluentResource::try_new(r#" +/// +/// msg-key = Hello, { $user }. You have { $emailCount } messages. +/// +/// "#.to_string()) +/// .expect("Failed to parse FTL."); +/// +/// let mut bundle = FluentBundle::default(); +/// +/// // For this example, we'll turn on BiDi support. +/// // Please, be careful when doing it, it's a risky move. +/// bundle.set_use_isolating(false); +/// +/// bundle.add_resource(res) +/// .expect("Failed to add a resource."); +/// +/// let mut err = vec![]; +/// +/// let msg = bundle.get_message("msg-key") +/// .expect("Failed to retrieve a message."); +/// let value = msg.value() +/// .expect("Failed to retrieve a value."); +/// +/// assert_eq!( +/// bundle.format_pattern(value, Some(&args), &mut err), +/// "Hello, John. You have 5 messages." +/// ); +/// ``` +#[derive(Debug, Default)] +pub struct FluentArgs<'args>(Vec<(Cow<'args, str>, FluentValue<'args>)>); + +impl<'args> FluentArgs<'args> { + pub fn new() -> Self { + Self::default() + } + + pub fn with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + pub fn get<K>(&self, key: K) -> Option<&FluentValue<'args>> + where + K: Into<Cow<'args, str>>, + { + let key = key.into(); + if let Ok(idx) = self.0.binary_search_by_key(&&key, |(k, _)| k) { + Some(&self.0[idx].1) + } else { + None + } + } + + pub fn set<K, V>(&mut self, key: K, value: V) + where + K: Into<Cow<'args, str>>, + V: Into<FluentValue<'args>>, + { + let key = key.into(); + let idx = match self.0.binary_search_by_key(&&key, |(k, _)| k) { + Ok(idx) => idx, + Err(idx) => idx, + }; + self.0.insert(idx, (key, value.into())); + } + + pub fn iter(&self) -> impl Iterator<Item = (&str, &FluentValue)> { + self.0.iter().map(|(k, v)| (k.as_ref(), v)) + } +} + +impl<'args, K, V> FromIterator<(K, V)> for FluentArgs<'args> +where + K: Into<Cow<'args, str>>, + V: Into<FluentValue<'args>>, +{ + fn from_iter<I>(iter: I) -> Self + where + I: IntoIterator<Item = (K, V)>, + { + let iter = iter.into_iter(); + let mut args = if let Some(size) = iter.size_hint().1 { + FluentArgs::with_capacity(size) + } else { + FluentArgs::new() + }; + + for (k, v) in iter { + args.set(k, v); + } + + args + } +} + +impl<'args> IntoIterator for FluentArgs<'args> { + type Item = (Cow<'args, str>, FluentValue<'args>); + type IntoIter = std::vec::IntoIter<Self::Item>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} diff --git a/third_party/rust/fluent-bundle/src/bundle.rs b/third_party/rust/fluent-bundle/src/bundle.rs new file mode 100644 index 0000000000..3d085cfee5 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/bundle.rs @@ -0,0 +1,615 @@ +//! `FluentBundle` is a collection of localization messages in Fluent. +//! +//! It stores a list of messages in a single locale which can reference one another, use the same +//! internationalization formatters, functions, scopeironmental variables and are expected to be used +//! together. + +use rustc_hash::FxHashMap; +use std::borrow::Borrow; +use std::borrow::Cow; +use std::collections::hash_map::Entry as HashEntry; +use std::default::Default; +use std::fmt; + +use fluent_syntax::ast; +use intl_memoizer::IntlLangMemoizer; +use unic_langid::LanguageIdentifier; + +use crate::args::FluentArgs; +use crate::entry::Entry; +use crate::entry::GetEntry; +use crate::errors::{EntryKind, FluentError}; +use crate::memoizer::MemoizerKind; +use crate::message::FluentMessage; +use crate::resolver::{ResolveValue, Scope, WriteValue}; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +/// A collection of localization messages for a single locale, which are meant +/// to be used together in a single view, widget or any other UI abstraction. +/// +/// # Examples +/// +/// ``` +/// use fluent_bundle::{FluentBundle, FluentResource, FluentValue, FluentArgs}; +/// use unic_langid::langid; +/// +/// // 1. Create a FluentResource +/// +/// let ftl_string = String::from("intro = Welcome, { $name }."); +/// let resource = FluentResource::try_new(ftl_string) +/// .expect("Could not parse an FTL string."); +/// +/// +/// // 2. Create a FluentBundle +/// +/// let langid_en = langid!("en-US"); +/// let mut bundle = FluentBundle::new(vec![langid_en]); +/// +/// +/// // 3. Add the resource to the bundle +/// +/// bundle.add_resource(&resource) +/// .expect("Failed to add FTL resources to the bundle."); +/// +/// +/// // 4. Retrieve a FluentMessage from the bundle +/// +/// let msg = bundle.get_message("intro") +/// .expect("Message doesn't exist."); +/// +/// let mut args = FluentArgs::new(); +/// args.set("name", "Rustacean"); +/// +/// +/// // 5. Format the value of the message +/// +/// let mut errors = vec![]; +/// +/// let pattern = msg.value() +/// .expect("Message has no value."); +/// +/// assert_eq!( +/// bundle.format_pattern(&pattern, Some(&args), &mut errors), +/// // The placeholder is wrapper in Unicode Directionality Marks +/// // to indicate that the placeholder may be of different direction +/// // than surrounding string. +/// "Welcome, \u{2068}Rustacean\u{2069}." +/// ); +/// +/// ``` +/// +/// # `FluentBundle` Life Cycle +/// +/// ## Create a bundle +/// +/// To create a bundle, call [`FluentBundle::new`] with a locale list that represents the best +/// possible fallback chain for a given locale. The simplest case is a one-locale list. +/// +/// Fluent uses [`LanguageIdentifier`] which can be created using `langid!` macro. +/// +/// ## Add Resources +/// +/// Next, call [`add_resource`](FluentBundle::add_resource) one or more times, supplying translations in the FTL syntax. +/// +/// Since [`FluentBundle`] is generic over anything that can borrow a [`FluentResource`], +/// one can use [`FluentBundle`] to own its resources, store references to them, +/// or even [`Rc<FluentResource>`](std::rc::Rc) or [`Arc<FluentResource>`](std::sync::Arc). +/// +/// The [`FluentBundle`] instance is now ready to be used for localization. +/// +/// ## Format +/// +/// To format a translation, call [`get_message`](FluentBundle::get_message) to retrieve a [`FluentMessage`], +/// and then call [`format_pattern`](FluentBundle::format_pattern) on the message value or attribute in order to +/// retrieve the translated string. +/// +/// The result of [`format_pattern`](FluentBundle::format_pattern) is an +/// [`Cow<str>`](std::borrow::Cow). It is +/// recommended to treat the result as opaque from the perspective of the program and use it only +/// to display localized messages. Do not examine it or alter in any way before displaying. This +/// is a general good practice as far as all internationalization operations are concerned. +/// +/// If errors were encountered during formatting, they will be +/// accumulated in the [`Vec<FluentError>`](FluentError) passed as the third argument. +/// +/// While they are not fatal, they usually indicate problems with the translation, +/// and should be logged or reported in a way that allows the developer to notice +/// and fix them. +/// +/// +/// # Locale Fallback Chain +/// +/// [`FluentBundle`] stores messages in a single locale, but keeps a locale fallback chain for the +/// purpose of language negotiation with i18n formatters. For instance, if date and time formatting +/// are not available in the first locale, [`FluentBundle`] will use its `locales` fallback chain +/// to negotiate a sensible fallback for date and time formatting. +/// +/// # Concurrency +/// +/// As you may have noticed, [`fluent_bundle::FluentBundle`](crate::FluentBundle) is a specialization of [`fluent_bundle::bundle::FluentBundle`](crate::bundle::FluentBundle) +/// which works with an [`IntlLangMemoizer`] over [`RefCell`](std::cell::RefCell). +/// In scenarios where the memoizer must work concurrently, there's an implementation of +/// [`IntlLangMemoizer`](intl_memoizer::concurrent::IntlLangMemoizer) that uses [`Mutex`](std::sync::Mutex) and there's [`FluentBundle::new_concurrent`] which works with that. +pub struct FluentBundle<R, M> { + pub locales: Vec<LanguageIdentifier>, + pub(crate) resources: Vec<R>, + pub(crate) entries: FxHashMap<String, Entry>, + pub(crate) intls: M, + pub(crate) use_isolating: bool, + pub(crate) transform: Option<fn(&str) -> Cow<str>>, + pub(crate) formatter: Option<fn(&FluentValue, &M) -> Option<String>>, +} + +impl<R, M> FluentBundle<R, M> { + /// Adds a resource to the bundle, returning an empty [`Result<T>`] on success. + /// + /// If any entry in the resource uses the same identifier as an already + /// existing key in the bundle, the new entry will be ignored and a + /// `FluentError::Overriding` will be added to the result. + /// + /// The method can take any type that can be borrowed to `FluentResource`: + /// - FluentResource + /// - &FluentResource + /// - Rc<FluentResource> + /// - Arc<FluentResurce> + /// + /// This allows the user to introduce custom resource management and share + /// resources between instances of `FluentBundle`. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from(" + /// hello = Hi! + /// goodbye = Bye! + /// "); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// assert_eq!(true, bundle.has_message("hello")); + /// ``` + /// + /// # Whitespace + /// + /// Message ids must have no leading whitespace. Message values that span + /// multiple lines must have leading whitespace on all but the first line. These + /// are standard FTL syntax rules that may prove a bit troublesome in source + /// code formatting. The [`indoc!`] crate can help with stripping extra indentation + /// if you wish to indent your entire message. + /// + /// [FTL syntax]: https://projectfluent.org/fluent/guide/ + /// [`indoc!`]: https://github.com/dtolnay/indoc + /// [`Result<T>`]: https://doc.rust-lang.org/std/result/enum.Result.html + pub fn add_resource(&mut self, r: R) -> Result<(), Vec<FluentError>> + where + R: Borrow<FluentResource>, + { + let mut errors = vec![]; + + let res = r.borrow(); + let res_pos = self.resources.len(); + + for (entry_pos, entry) in res.entries().enumerate() { + let (id, entry) = match entry { + ast::Entry::Message(ast::Message { ref id, .. }) => { + (id.name, Entry::Message((res_pos, entry_pos))) + } + ast::Entry::Term(ast::Term { ref id, .. }) => { + (id.name, Entry::Term((res_pos, entry_pos))) + } + _ => continue, + }; + + match self.entries.entry(id.to_string()) { + HashEntry::Vacant(empty) => { + empty.insert(entry); + } + HashEntry::Occupied(_) => { + let kind = match entry { + Entry::Message(..) => EntryKind::Message, + Entry::Term(..) => EntryKind::Term, + _ => unreachable!(), + }; + errors.push(FluentError::Overriding { + kind, + id: id.to_string(), + }); + } + } + } + self.resources.push(r); + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + + /// Adds a resource to the bundle, returning an empty [`Result<T>`] on success. + /// + /// If any entry in the resource uses the same identifier as an already + /// existing key in the bundle, the entry will override the previous one. + /// + /// The method can take any type that can be borrowed as FluentResource: + /// - FluentResource + /// - &FluentResource + /// - Rc<FluentResource> + /// - Arc<FluentResurce> + /// + /// This allows the user to introduce custom resource management and share + /// resources between instances of `FluentBundle`. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from(" + /// hello = Hi! + /// goodbye = Bye! + /// "); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// + /// let ftl_string = String::from(" + /// hello = Another Hi! + /// "); + /// let resource2 = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// bundle.add_resource_overriding(resource2); + /// + /// let mut errors = vec![]; + /// let msg = bundle.get_message("hello") + /// .expect("Failed to retrieve the message"); + /// let value = msg.value().expect("Failed to retrieve the value of the message"); + /// assert_eq!(bundle.format_pattern(value, None, &mut errors), "Another Hi!"); + /// ``` + /// + /// # Whitespace + /// + /// Message ids must have no leading whitespace. Message values that span + /// multiple lines must have leading whitespace on all but the first line. These + /// are standard FTL syntax rules that may prove a bit troublesome in source + /// code formatting. The [`indoc!`] crate can help with stripping extra indentation + /// if you wish to indent your entire message. + /// + /// [FTL syntax]: https://projectfluent.org/fluent/guide/ + /// [`indoc!`]: https://github.com/dtolnay/indoc + /// [`Result<T>`]: https://doc.rust-lang.org/std/result/enum.Result.html + pub fn add_resource_overriding(&mut self, r: R) + where + R: Borrow<FluentResource>, + { + let res = r.borrow(); + let res_pos = self.resources.len(); + + for (entry_pos, entry) in res.entries().enumerate() { + let (id, entry) = match entry { + ast::Entry::Message(ast::Message { ref id, .. }) => { + (id.name, Entry::Message((res_pos, entry_pos))) + } + ast::Entry::Term(ast::Term { ref id, .. }) => { + (id.name, Entry::Term((res_pos, entry_pos))) + } + _ => continue, + }; + + self.entries.insert(id.to_string(), entry); + } + self.resources.push(r); + } + + /// When formatting patterns, `FluentBundle` inserts + /// Unicode Directionality Isolation Marks to indicate + /// that the direction of a placeable may differ from + /// the surrounding message. + /// + /// This is important for cases such as when a + /// right-to-left user name is presented in the + /// left-to-right message. + /// + /// In some cases, such as testing, the user may want + /// to disable the isolating. + pub fn set_use_isolating(&mut self, value: bool) { + self.use_isolating = value; + } + + /// This method allows to specify a function that will + /// be called on all textual fragments of the pattern + /// during formatting. + /// + /// This is currently primarly used for pseudolocalization, + /// and `fluent-pseudo` crate provides a function + /// that can be passed here. + pub fn set_transform(&mut self, func: Option<fn(&str) -> Cow<str>>) { + self.transform = func; + } + + /// This method allows to specify a function that will + /// be called before any `FluentValue` is formatted + /// allowing overrides. + /// + /// It's particularly useful for plugging in an external + /// formatter for `FluentValue::Number`. + pub fn set_formatter(&mut self, func: Option<fn(&FluentValue, &M) -> Option<String>>) { + self.formatter = func; + } + + /// Returns true if this bundle contains a message with the given id. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello = Hi!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// assert_eq!(true, bundle.has_message("hello")); + /// + /// ``` + pub fn has_message(&self, id: &str) -> bool + where + R: Borrow<FluentResource>, + { + self.get_entry_message(id).is_some() + } + + /// Retrieves a `FluentMessage` from a bundle. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello-world = Hello World!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// let msg = bundle.get_message("hello-world"); + /// assert_eq!(msg.is_some(), true); + /// ``` + pub fn get_message<'l>(&'l self, id: &str) -> Option<FluentMessage<'l>> + where + R: Borrow<FluentResource>, + { + self.get_entry_message(id).map(Into::into) + } + + /// Writes a formatted pattern which comes from a `FluentMessage`. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello-world = Hello World!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a FluentMessage."); + /// + /// let pattern = msg.value() + /// .expect("Missing Value."); + /// let mut errors = vec![]; + /// + /// let mut s = String::new(); + /// bundle.write_pattern(&mut s, &pattern, None, &mut errors) + /// .expect("Failed to write."); + /// + /// assert_eq!(s, "Hello World!"); + /// ``` + pub fn write_pattern<'bundle, W>( + &'bundle self, + w: &mut W, + pattern: &'bundle ast::Pattern<&str>, + args: Option<&'bundle FluentArgs>, + errors: &mut Vec<FluentError>, + ) -> fmt::Result + where + R: Borrow<FluentResource>, + W: fmt::Write, + M: MemoizerKind, + { + let mut scope = Scope::new(self, args, Some(errors)); + pattern.write(w, &mut scope) + } + + /// Formats a pattern which comes from a `FluentMessage`. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello-world = Hello World!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a FluentMessage."); + /// + /// let pattern = msg.value() + /// .expect("Missing Value."); + /// let mut errors = vec![]; + /// + /// let result = bundle.format_pattern(&pattern, None, &mut errors); + /// + /// assert_eq!(result, "Hello World!"); + /// ``` + pub fn format_pattern<'bundle>( + &'bundle self, + pattern: &'bundle ast::Pattern<&str>, + args: Option<&'bundle FluentArgs>, + errors: &mut Vec<FluentError>, + ) -> Cow<'bundle, str> + where + R: Borrow<FluentResource>, + M: MemoizerKind, + { + let mut scope = Scope::new(self, args, Some(errors)); + let value = pattern.resolve(&mut scope); + value.as_string(&scope) + } + + /// Makes the provided rust function available to messages with the name `id`. See + /// the [FTL syntax guide] to learn how these are used in messages. + /// + /// FTL functions accept both positional and named args. The rust function you + /// provide therefore has two parameters: a slice of values for the positional + /// args, and a `FluentArgs` for named args. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource, FluentValue}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("length = { STRLEN(\"12345\") }"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// // Register a fn that maps from string to string length + /// bundle.add_function("STRLEN", |positional, _named| match positional { + /// [FluentValue::String(str)] => str.len().into(), + /// _ => FluentValue::Error, + /// }).expect("Failed to add a function to the bundle."); + /// + /// let msg = bundle.get_message("length").expect("Message doesn't exist."); + /// let mut errors = vec![]; + /// let pattern = msg.value().expect("Message has no value."); + /// let value = bundle.format_pattern(&pattern, None, &mut errors); + /// assert_eq!(&value, "5"); + /// ``` + /// + /// [FTL syntax guide]: https://projectfluent.org/fluent/guide/functions.html + pub fn add_function<F>(&mut self, id: &str, func: F) -> Result<(), FluentError> + where + F: for<'a> Fn(&[FluentValue<'a>], &FluentArgs) -> FluentValue<'a> + Sync + Send + 'static, + { + match self.entries.entry(id.to_owned()) { + HashEntry::Vacant(entry) => { + entry.insert(Entry::Function(Box::new(func))); + Ok(()) + } + HashEntry::Occupied(_) => Err(FluentError::Overriding { + kind: EntryKind::Function, + id: id.to_owned(), + }), + } + } +} + +impl<R> Default for FluentBundle<R, IntlLangMemoizer> { + fn default() -> Self { + Self::new(vec![LanguageIdentifier::default()]) + } +} + +impl<R> FluentBundle<R, IntlLangMemoizer> { + /// Constructs a FluentBundle. The first element in `locales` should be the + /// language this bundle represents, and will be used to determine the + /// correct plural rules for this bundle. You can optionally provide extra + /// languages in the list; they will be used as fallback date and time + /// formatters if a formatter for the primary language is unavailable. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::FluentBundle; + /// use fluent_bundle::FluentResource; + /// use unic_langid::langid; + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle: FluentBundle<FluentResource> = FluentBundle::new(vec![langid_en]); + /// ``` + /// + /// # Errors + /// + /// This will panic if no formatters can be found for the locales. + pub fn new(locales: Vec<LanguageIdentifier>) -> Self { + let first_locale = locales.get(0).cloned().unwrap_or_default(); + Self { + locales, + resources: vec![], + entries: FxHashMap::default(), + intls: IntlLangMemoizer::new(first_locale), + use_isolating: true, + transform: None, + formatter: None, + } + } +} + +impl crate::memoizer::MemoizerKind for IntlLangMemoizer { + fn new(lang: LanguageIdentifier) -> Self + where + Self: Sized, + { + Self::new(lang) + } + + fn with_try_get_threadsafe<I, R, U>(&self, args: I::Args, cb: U) -> Result<R, I::Error> + where + Self: Sized, + I: intl_memoizer::Memoizable + Send + Sync + 'static, + I::Args: Send + Sync + 'static, + U: FnOnce(&I) -> R, + { + self.with_try_get(args, cb) + } + + fn stringify_value( + &self, + value: &dyn crate::types::FluentType, + ) -> std::borrow::Cow<'static, str> { + value.as_string(self) + } +} diff --git a/third_party/rust/fluent-bundle/src/concurrent.rs b/third_party/rust/fluent-bundle/src/concurrent.rs new file mode 100644 index 0000000000..b87225efee --- /dev/null +++ b/third_party/rust/fluent-bundle/src/concurrent.rs @@ -0,0 +1,59 @@ +use intl_memoizer::{concurrent::IntlLangMemoizer, Memoizable}; +use rustc_hash::FxHashMap; +use unic_langid::LanguageIdentifier; + +use crate::bundle::FluentBundle; +use crate::memoizer::MemoizerKind; +use crate::types::FluentType; + +impl<R> FluentBundle<R, IntlLangMemoizer> { + /// A constructor analogous to [`FluentBundle::new`] but operating + /// on a concurrent version of [`IntlLangMemoizer`] over [`Mutex`](std::sync::Mutex). + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::bundle::FluentBundle; + /// use fluent_bundle::FluentResource; + /// use unic_langid::langid; + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle: FluentBundle<FluentResource, _> = + /// FluentBundle::new_concurrent(vec![langid_en]); + /// ``` + pub fn new_concurrent(locales: Vec<LanguageIdentifier>) -> Self { + let first_locale = locales.get(0).cloned().unwrap_or_default(); + Self { + locales, + resources: vec![], + entries: FxHashMap::default(), + intls: IntlLangMemoizer::new(first_locale), + use_isolating: true, + transform: None, + formatter: None, + } + } +} + +impl MemoizerKind for IntlLangMemoizer { + fn new(lang: LanguageIdentifier) -> Self + where + Self: Sized, + { + Self::new(lang) + } + + fn with_try_get_threadsafe<I, R, U>(&self, args: I::Args, cb: U) -> Result<R, I::Error> + where + Self: Sized, + I: Memoizable + Send + Sync + 'static, + I::Args: Send + Sync + 'static, + U: FnOnce(&I) -> R, + { + self.with_try_get(args, cb) + } + + fn stringify_value(&self, value: &dyn FluentType) -> std::borrow::Cow<'static, str> { + value.as_string_threadsafe(self) + } +} diff --git a/third_party/rust/fluent-bundle/src/entry.rs b/third_party/rust/fluent-bundle/src/entry.rs new file mode 100644 index 0000000000..1ac8ecf01b --- /dev/null +++ b/third_party/rust/fluent-bundle/src/entry.rs @@ -0,0 +1,62 @@ +//! `Entry` is used to store Messages, Terms and Functions in `FluentBundle` instances. + +use std::borrow::Borrow; + +use fluent_syntax::ast; + +use crate::args::FluentArgs; +use crate::bundle::FluentBundle; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +pub type FluentFunction = + Box<dyn for<'a> Fn(&[FluentValue<'a>], &FluentArgs) -> FluentValue<'a> + Send + Sync>; + +pub enum Entry { + Message((usize, usize)), + Term((usize, usize)), + Function(FluentFunction), +} + +pub trait GetEntry { + fn get_entry_message(&self, id: &str) -> Option<&ast::Message<&str>>; + fn get_entry_term(&self, id: &str) -> Option<&ast::Term<&str>>; + fn get_entry_function(&self, id: &str) -> Option<&FluentFunction>; +} + +impl<'bundle, R: Borrow<FluentResource>, M> GetEntry for FluentBundle<R, M> { + fn get_entry_message(&self, id: &str) -> Option<&ast::Message<&str>> { + self.entries.get(id).and_then(|ref entry| match entry { + Entry::Message(pos) => { + let res = self.resources.get(pos.0)?.borrow(); + if let ast::Entry::Message(ref msg) = res.get_entry(pos.1)? { + Some(msg) + } else { + None + } + } + _ => None, + }) + } + + fn get_entry_term(&self, id: &str) -> Option<&ast::Term<&str>> { + self.entries.get(id).and_then(|ref entry| match entry { + Entry::Term(pos) => { + let res = self.resources.get(pos.0)?.borrow(); + if let ast::Entry::Term(ref msg) = res.get_entry(pos.1)? { + Some(msg) + } else { + None + } + } + _ => None, + }) + } + + fn get_entry_function(&self, id: &str) -> Option<&FluentFunction> { + self.entries.get(id).and_then(|ref entry| match entry { + Entry::Function(function) => Some(function), + _ => None, + }) + } +} diff --git a/third_party/rust/fluent-bundle/src/errors.rs b/third_party/rust/fluent-bundle/src/errors.rs new file mode 100644 index 0000000000..ec4a02c4b4 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/errors.rs @@ -0,0 +1,86 @@ +use crate::resolver::ResolverError; +use fluent_syntax::parser::ParserError; +use std::error::Error; + +#[derive(Debug, PartialEq, Clone)] +pub enum EntryKind { + Message, + Term, + Function, +} + +impl std::fmt::Display for EntryKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Message => f.write_str("message"), + Self::Term => f.write_str("term"), + Self::Function => f.write_str("function"), + } + } +} + +/// Core error type for Fluent runtime system. +/// +/// It contains three main types of errors that may come up +/// during runtime use of the fluent-bundle crate. +#[derive(Debug, PartialEq, Clone)] +pub enum FluentError { + /// An error which occurs when + /// [`FluentBundle::add_resource`](crate::bundle::FluentBundle::add_resource) + /// adds entries that are already registered in a given [`FluentBundle`](crate::FluentBundle). + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("intro = Welcome, { $name }."); + /// let res1 = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// + /// let ftl_string = String::from("intro = Hi, { $name }."); + /// let res2 = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&res1) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// assert!(bundle.add_resource(&res2).is_err()); + /// ``` + Overriding { + kind: EntryKind, + id: String, + }, + ParserError(ParserError), + ResolverError(ResolverError), +} + +impl std::fmt::Display for FluentError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Overriding { kind, id } => { + write!(f, "Attempt to override an existing {}: \"{}\".", kind, id) + } + Self::ParserError(err) => write!(f, "Parser error: {}", err), + Self::ResolverError(err) => write!(f, "Resolver error: {}", err), + } + } +} + +impl Error for FluentError {} + +impl From<ResolverError> for FluentError { + fn from(error: ResolverError) -> Self { + Self::ResolverError(error) + } +} + +impl From<ParserError> for FluentError { + fn from(error: ParserError) -> Self { + Self::ParserError(error) + } +} diff --git a/third_party/rust/fluent-bundle/src/lib.rs b/third_party/rust/fluent-bundle/src/lib.rs new file mode 100644 index 0000000000..faf3e9ba60 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/lib.rs @@ -0,0 +1,127 @@ +//! Fluent is a modern localization system designed to improve how software is translated. +//! +//! `fluent-bundle` is a mid-level component of the [Fluent Localization +//! System](https://www.projectfluent.org). +//! +//! The crate builds on top of the low level [`fluent-syntax`](../fluent-syntax) package, and provides +//! foundational types and structures required for executing localization at runtime. +//! +//! There are four core concepts to understand Fluent runtime: +//! +//! * [`FluentMessage`] - A single translation unit +//! * [`FluentResource`] - A list of [`FluentMessage`] units +//! * [`FluentBundle`](crate::bundle::FluentBundle) - A collection of [`FluentResource`] lists +//! * [`FluentArgs`] - A list of elements used to resolve a [`FluentMessage`] value +//! +//! ## Example +//! +//! ``` +//! use fluent_bundle::{FluentBundle, FluentValue, FluentResource, FluentArgs}; +//! // Used to provide a locale for the bundle. +//! use unic_langid::langid; +//! +//! // 1. Crate a FluentResource +//! +//! let ftl_string = r#" +//! +//! hello-world = Hello, world! +//! intro = Welcome, { $name }. +//! +//! "#.to_string(); +//! +//! let res = FluentResource::try_new(ftl_string) +//! .expect("Failed to parse an FTL string."); +//! +//! +//! // 2. Crate a FluentBundle +//! +//! let langid_en = langid!("en-US"); +//! let mut bundle = FluentBundle::new(vec![langid_en]); +//! +//! +//! // 3. Add the resource to the bundle +//! +//! bundle +//! .add_resource(res) +//! .expect("Failed to add FTL resources to the bundle."); +//! +//! +//! // 4. Retrieve a FluentMessage from the bundle +//! +//! let msg = bundle.get_message("hello-world") +//! .expect("Message doesn't exist."); +//! +//! +//! // 5. Format the value of the simple message +//! +//! let mut errors = vec![]; +//! +//! let pattern = msg.value() +//! .expect("Message has no value."); +//! +//! let value = bundle.format_pattern(&pattern, None, &mut errors); +//! +//! assert_eq!( +//! bundle.format_pattern(&pattern, None, &mut errors), +//! "Hello, world!" +//! ); +//! +//! // 6. Format the value of the message with arguments +//! +//! let mut args = FluentArgs::new(); +//! args.set("name", "John"); +//! +//! let msg = bundle.get_message("intro") +//! .expect("Message doesn't exist."); +//! +//! let pattern = msg.value() +//! .expect("Message has no value."); +//! +//! // The FSI/PDI isolation marks ensure that the direction of +//! // the text from the variable is not affected by the translation. +//! assert_eq!( +//! bundle.format_pattern(&pattern, Some(&args), &mut errors), +//! "Welcome, \u{2068}John\u{2069}." +//! ); +//! ``` +//! +//! # Ergonomics & Higher Level APIs +//! +//! Reading the example, you may notice how verbose it feels. +//! Many core methods are fallible, others accumulate errors, and there +//! are intermediate structures used in operations. +//! +//! This is intentional as it serves as building blocks for variety of different +//! scenarios allowing implementations to handle errors, cache and +//! optimize results. +//! +//! At the moment it is expected that users will use +//! the `fluent-bundle` crate directly, while the ecosystem +//! matures and higher level APIs are being developed. +mod args; +pub mod bundle; +mod concurrent; +mod entry; +mod errors; +#[doc(hidden)] +pub mod memoizer; +mod message; +#[doc(hidden)] +pub mod resolver; +mod resource; +pub mod types; + +pub use args::FluentArgs; +/// Specialized [`FluentBundle`](crate::bundle::FluentBundle) over +/// non-concurrent [`IntlLangMemoizer`](intl_memoizer::IntlLangMemoizer). +/// +/// This is the basic variant of the [`FluentBundle`](crate::bundle::FluentBundle). +/// +/// The concurrent specialization, can be constructed with +/// [`FluentBundle::new_concurrent`](crate::bundle::FluentBundle::new_concurrent). +pub type FluentBundle<R> = bundle::FluentBundle<R, intl_memoizer::IntlLangMemoizer>; +pub use errors::FluentError; +pub use message::{FluentAttribute, FluentMessage}; +pub use resource::FluentResource; +#[doc(inline)] +pub use types::FluentValue; diff --git a/third_party/rust/fluent-bundle/src/memoizer.rs b/third_party/rust/fluent-bundle/src/memoizer.rs new file mode 100644 index 0000000000..c738a857b2 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/memoizer.rs @@ -0,0 +1,18 @@ +use crate::types::FluentType; +use intl_memoizer::Memoizable; +use unic_langid::LanguageIdentifier; + +pub trait MemoizerKind: 'static { + fn new(lang: LanguageIdentifier) -> Self + where + Self: Sized; + + fn with_try_get_threadsafe<I, R, U>(&self, args: I::Args, cb: U) -> Result<R, I::Error> + where + Self: Sized, + I: Memoizable + Send + Sync + 'static, + I::Args: Send + Sync + 'static, + U: FnOnce(&I) -> R; + + fn stringify_value(&self, value: &dyn FluentType) -> std::borrow::Cow<'static, str>; +} diff --git a/third_party/rust/fluent-bundle/src/message.rs b/third_party/rust/fluent-bundle/src/message.rs new file mode 100644 index 0000000000..a6cc00d77e --- /dev/null +++ b/third_party/rust/fluent-bundle/src/message.rs @@ -0,0 +1,274 @@ +use fluent_syntax::ast; + +/// [`FluentAttribute`] is a component of a compound [`FluentMessage`]. +/// +/// It represents a key-value pair providing a translation of a component +/// of a user interface widget localized by the given message. +/// +/// # Example +/// +/// ``` +/// use fluent_bundle::{FluentResource, FluentBundle}; +/// +/// let source = r#" +/// +/// confirm-modal = Are you sure? +/// .confirm = Yes +/// .cancel = No +/// .tooltip = Closing the window will lose all unsaved data. +/// +/// "#; +/// +/// let resource = FluentResource::try_new(source.to_string()) +/// .expect("Failed to parse the resource."); +/// +/// let mut bundle = FluentBundle::default(); +/// bundle.add_resource(resource) +/// .expect("Failed to add a resource."); +/// +/// let msg = bundle.get_message("confirm-modal") +/// .expect("Failed to retrieve a message."); +/// +/// let mut err = vec![]; +/// +/// let attributes = msg.attributes().map(|attr| { +/// bundle.format_pattern(attr.value(), None, &mut err) +/// }).collect::<Vec<_>>(); +/// +/// assert_eq!(attributes[0], "Yes"); +/// assert_eq!(attributes[1], "No"); +/// assert_eq!(attributes[2], "Closing the window will lose all unsaved data."); +/// ``` +#[derive(Debug, PartialEq)] +pub struct FluentAttribute<'m> { + node: &'m ast::Attribute<&'m str>, +} + +impl<'m> FluentAttribute<'m> { + /// Retrieves an id of an attribute. + /// + /// # Example + /// + /// ``` + /// # use fluent_bundle::{FluentResource, FluentBundle}; + /// # let source = r#" + /// # confirm-modal = + /// # .confirm = Yes + /// # "#; + /// # let resource = FluentResource::try_new(source.to_string()) + /// # .expect("Failed to parse the resource."); + /// # let mut bundle = FluentBundle::default(); + /// # bundle.add_resource(resource) + /// # .expect("Failed to add a resource."); + /// let msg = bundle.get_message("confirm-modal") + /// .expect("Failed to retrieve a message."); + /// + /// let attr1 = msg.attributes().next() + /// .expect("Failed to retrieve an attribute."); + /// + /// assert_eq!(attr1.id(), "confirm"); + /// ``` + pub fn id(&self) -> &'m str { + &self.node.id.name + } + + /// Retrieves an value of an attribute. + /// + /// # Example + /// + /// ``` + /// # use fluent_bundle::{FluentResource, FluentBundle}; + /// # let source = r#" + /// # confirm-modal = + /// # .confirm = Yes + /// # "#; + /// # let resource = FluentResource::try_new(source.to_string()) + /// # .expect("Failed to parse the resource."); + /// # let mut bundle = FluentBundle::default(); + /// # bundle.add_resource(resource) + /// # .expect("Failed to add a resource."); + /// let msg = bundle.get_message("confirm-modal") + /// .expect("Failed to retrieve a message."); + /// + /// let attr1 = msg.attributes().next() + /// .expect("Failed to retrieve an attribute."); + /// + /// let mut err = vec![]; + /// + /// let value = attr1.value(); + /// assert_eq!( + /// bundle.format_pattern(value, None, &mut err), + /// "Yes" + /// ); + /// ``` + pub fn value(&self) -> &'m ast::Pattern<&'m str> { + &self.node.value + } +} + +impl<'m> From<&'m ast::Attribute<&'m str>> for FluentAttribute<'m> { + fn from(attr: &'m ast::Attribute<&'m str>) -> Self { + FluentAttribute { node: attr } + } +} + +/// [`FluentMessage`] is a basic translation unit of the Fluent system. +/// +/// The instance of a message is returned from the +/// [`FluentBundle::get_message`](crate::bundle::FluentBundle::get_message) +/// method, for the lifetime of the [`FluentBundle`](crate::bundle::FluentBundle) instance. +/// +/// # Example +/// +/// ``` +/// use fluent_bundle::{FluentResource, FluentBundle}; +/// +/// let source = r#" +/// +/// hello-world = Hello World! +/// +/// "#; +/// +/// let resource = FluentResource::try_new(source.to_string()) +/// .expect("Failed to parse the resource."); +/// +/// let mut bundle = FluentBundle::default(); +/// bundle.add_resource(resource) +/// .expect("Failed to add a resource."); +/// +/// let msg = bundle.get_message("hello-world") +/// .expect("Failed to retrieve a message."); +/// +/// assert!(msg.value().is_some()); +/// ``` +/// +/// That value can be then passed to +/// [`FluentBundle::format_pattern`](crate::bundle::FluentBundle::format_pattern) to be formatted +/// within the context of a given [`FluentBundle`](crate::bundle::FluentBundle) instance. +/// +/// # Compound Message +/// +/// A message may contain a `value`, but it can also contain a list of [`FluentAttribute`] elements. +/// +/// If a message contains attributes, it is called a "compound" message. +/// +/// In such case, the message contains a list of key-value attributes that represent +/// different translation values associated with a single translation unit. +/// +/// This is useful for scenarios where a [`FluentMessage`] is associated with a +/// complex User Interface widget which has multiple attributes that need to be translated. +/// ```text +/// confirm-modal = Are you sure? +/// .confirm = Yes +/// .cancel = No +/// .tooltip = Closing the window will lose all unsaved data. +/// ``` +#[derive(Debug, PartialEq)] +pub struct FluentMessage<'m> { + node: &'m ast::Message<&'m str>, +} + +impl<'m> FluentMessage<'m> { + /// Retrieves an option of a [`ast::Pattern`](fluent_syntax::ast::Pattern). + /// + /// # Example + /// + /// ``` + /// # use fluent_bundle::{FluentResource, FluentBundle}; + /// # let source = r#" + /// # hello-world = Hello World! + /// # "#; + /// # let resource = FluentResource::try_new(source.to_string()) + /// # .expect("Failed to parse the resource."); + /// # let mut bundle = FluentBundle::default(); + /// # bundle.add_resource(resource) + /// # .expect("Failed to add a resource."); + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a message."); + /// + /// if let Some(value) = msg.value() { + /// let mut err = vec![]; + /// assert_eq!( + /// bundle.format_pattern(value, None, &mut err), + /// "Hello World!" + /// ); + /// # assert_eq!(err.len(), 0); + /// } + /// ``` + pub fn value(&self) -> Option<&'m ast::Pattern<&'m str>> { + self.node.value.as_ref() + } + + /// An iterator over [`FluentAttribute`] elements. + /// + /// # Example + /// + /// ``` + /// # use fluent_bundle::{FluentResource, FluentBundle}; + /// # let source = r#" + /// # hello-world = + /// # .label = This is a label + /// # .accesskey = C + /// # "#; + /// # let resource = FluentResource::try_new(source.to_string()) + /// # .expect("Failed to parse the resource."); + /// # let mut bundle = FluentBundle::default(); + /// # bundle.add_resource(resource) + /// # .expect("Failed to add a resource."); + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a message."); + /// + /// let mut err = vec![]; + /// + /// for attr in msg.attributes() { + /// let _ = bundle.format_pattern(attr.value(), None, &mut err); + /// } + /// # assert_eq!(err.len(), 0); + /// ``` + pub fn attributes(&self) -> impl Iterator<Item = FluentAttribute<'m>> { + self.node.attributes.iter().map(Into::into) + } + + /// Retrieve a single [`FluentAttribute`] element. + /// + /// # Example + /// + /// ``` + /// # use fluent_bundle::{FluentResource, FluentBundle}; + /// # let source = r#" + /// # hello-world = + /// # .label = This is a label + /// # .accesskey = C + /// # "#; + /// # let resource = FluentResource::try_new(source.to_string()) + /// # .expect("Failed to parse the resource."); + /// # let mut bundle = FluentBundle::default(); + /// # bundle.add_resource(resource) + /// # .expect("Failed to add a resource."); + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a message."); + /// + /// let mut err = vec![]; + /// + /// if let Some(attr) = msg.get_attribute("label") { + /// assert_eq!( + /// bundle.format_pattern(attr.value(), None, &mut err), + /// "This is a label" + /// ); + /// } + /// # assert_eq!(err.len(), 0); + /// ``` + pub fn get_attribute(&self, key: &str) -> Option<FluentAttribute<'m>> { + self.node + .attributes + .iter() + .find(|attr| attr.id.name == key) + .map(Into::into) + } +} + +impl<'m> From<&'m ast::Message<&'m str>> for FluentMessage<'m> { + fn from(msg: &'m ast::Message<&'m str>) -> Self { + FluentMessage { node: msg } + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/errors.rs b/third_party/rust/fluent-bundle/src/resolver/errors.rs new file mode 100644 index 0000000000..831d8474a5 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/errors.rs @@ -0,0 +1,96 @@ +use fluent_syntax::ast::InlineExpression; +use std::error::Error; + +#[derive(Debug, PartialEq, Clone)] +pub enum ReferenceKind { + Function { + id: String, + }, + Message { + id: String, + attribute: Option<String>, + }, + Term { + id: String, + attribute: Option<String>, + }, + Variable { + id: String, + }, +} + +impl<T> From<&InlineExpression<T>> for ReferenceKind +where + T: ToString, +{ + fn from(exp: &InlineExpression<T>) -> Self { + match exp { + InlineExpression::FunctionReference { id, .. } => Self::Function { + id: id.name.to_string(), + }, + InlineExpression::MessageReference { id, attribute } => Self::Message { + id: id.name.to_string(), + attribute: attribute.as_ref().map(|i| i.name.to_string()), + }, + InlineExpression::TermReference { id, attribute, .. } => Self::Term { + id: id.name.to_string(), + attribute: attribute.as_ref().map(|i| i.name.to_string()), + }, + InlineExpression::VariableReference { id, .. } => Self::Variable { + id: id.name.to_string(), + }, + _ => unreachable!(), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum ResolverError { + Reference(ReferenceKind), + NoValue(String), + MissingDefault, + Cyclic, + TooManyPlaceables, +} + +impl std::fmt::Display for ResolverError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Reference(exp) => match exp { + ReferenceKind::Function { id } => write!(f, "Unknown function: {}()", id), + ReferenceKind::Message { + id, + attribute: None, + } => write!(f, "Unknown message: {}", id), + ReferenceKind::Message { + id, + attribute: Some(attribute), + } => write!(f, "Unknown attribute: {}.{}", id, attribute), + ReferenceKind::Term { + id, + attribute: None, + } => write!(f, "Unknown term: -{}", id), + ReferenceKind::Term { + id, + attribute: Some(attribute), + } => write!(f, "Unknown attribute: -{}.{}", id, attribute), + ReferenceKind::Variable { id } => write!(f, "Unknown variable: ${}", id), + }, + Self::NoValue(id) => write!(f, "No value: {}", id), + Self::MissingDefault => f.write_str("No default"), + Self::Cyclic => f.write_str("Cyclical dependency detected"), + Self::TooManyPlaceables => f.write_str("Too many placeables"), + } + } +} + +impl<T> From<&InlineExpression<T>> for ResolverError +where + T: ToString, +{ + fn from(exp: &InlineExpression<T>) -> Self { + Self::Reference(exp.into()) + } +} + +impl Error for ResolverError {} diff --git a/third_party/rust/fluent-bundle/src/resolver/expression.rs b/third_party/rust/fluent-bundle/src/resolver/expression.rs new file mode 100644 index 0000000000..d0d02decd3 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/expression.rs @@ -0,0 +1,66 @@ +use super::scope::Scope; +use super::WriteValue; + +use std::borrow::Borrow; +use std::fmt; + +use fluent_syntax::ast; + +use crate::memoizer::MemoizerKind; +use crate::resolver::{ResolveValue, ResolverError}; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +impl<'p> WriteValue for ast::Expression<&'p str> { + fn write<'scope, 'errors, W, R, M>( + &'scope self, + w: &mut W, + scope: &mut Scope<'scope, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + M: MemoizerKind, + { + match self { + Self::Inline(exp) => exp.write(w, scope), + Self::Select { selector, variants } => { + let selector = selector.resolve(scope); + match selector { + FluentValue::String(_) | FluentValue::Number(_) => { + for variant in variants { + let key = match variant.key { + ast::VariantKey::Identifier { name } => name.into(), + ast::VariantKey::NumberLiteral { value } => { + FluentValue::try_number(value) + } + }; + if key.matches(&selector, scope) { + return variant.value.write(w, scope); + } + } + } + _ => {} + } + + for variant in variants { + if variant.default { + return variant.value.write(w, scope); + } + } + scope.add_error(ResolverError::MissingDefault); + Ok(()) + } + } + } + + fn write_error<W>(&self, w: &mut W) -> fmt::Result + where + W: fmt::Write, + { + match self { + Self::Inline(exp) => exp.write_error(w), + Self::Select { .. } => unreachable!(), + } + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/inline_expression.rs b/third_party/rust/fluent-bundle/src/resolver/inline_expression.rs new file mode 100644 index 0000000000..b9e89b6e8e --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/inline_expression.rs @@ -0,0 +1,181 @@ +use super::scope::Scope; +use super::{ResolveValue, ResolverError, WriteValue}; + +use std::borrow::Borrow; +use std::fmt; + +use fluent_syntax::ast; +use fluent_syntax::unicode::{unescape_unicode, unescape_unicode_to_string}; + +use crate::entry::GetEntry; +use crate::memoizer::MemoizerKind; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +impl<'p> WriteValue for ast::InlineExpression<&'p str> { + fn write<'scope, 'errors, W, R, M>( + &'scope self, + w: &mut W, + scope: &mut Scope<'scope, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + M: MemoizerKind, + { + match self { + Self::StringLiteral { value } => unescape_unicode(w, value), + Self::MessageReference { id, attribute } => { + if let Some(msg) = scope.bundle.get_entry_message(id.name) { + if let Some(attr) = attribute { + msg.attributes + .iter() + .find_map(|a| { + if a.id.name == attr.name { + Some(scope.track(w, &a.value, self)) + } else { + None + } + }) + .unwrap_or_else(|| scope.write_ref_error(w, self)) + } else { + msg.value + .as_ref() + .map(|value| scope.track(w, value, self)) + .unwrap_or_else(|| { + scope.add_error(ResolverError::NoValue(id.name.to_string())); + w.write_char('{')?; + self.write_error(w)?; + w.write_char('}') + }) + } + } else { + scope.write_ref_error(w, self) + } + } + Self::NumberLiteral { value } => FluentValue::try_number(*value).write(w, scope), + Self::TermReference { + id, + attribute, + arguments, + } => { + let (_, resolved_named_args) = scope.get_arguments(arguments.as_ref()); + + scope.local_args = Some(resolved_named_args); + let result = scope + .bundle + .get_entry_term(id.name) + .and_then(|term| { + if let Some(attr) = attribute { + term.attributes.iter().find_map(|a| { + if a.id.name == attr.name { + Some(scope.track(w, &a.value, self)) + } else { + None + } + }) + } else { + Some(scope.track(w, &term.value, self)) + } + }) + .unwrap_or_else(|| scope.write_ref_error(w, self)); + scope.local_args = None; + result + } + Self::FunctionReference { id, arguments } => { + let (resolved_positional_args, resolved_named_args) = + scope.get_arguments(Some(arguments)); + + let func = scope.bundle.get_entry_function(id.name); + + if let Some(func) = func { + let result = func(resolved_positional_args.as_slice(), &resolved_named_args); + if let FluentValue::Error = result { + self.write_error(w) + } else { + w.write_str(&result.as_string(scope)) + } + } else { + scope.write_ref_error(w, self) + } + } + Self::VariableReference { id } => { + let args = scope.local_args.as_ref().or(scope.args); + + if let Some(arg) = args.and_then(|args| args.get(id.name)) { + arg.write(w, scope) + } else { + if scope.local_args.is_none() { + scope.add_error(self.into()); + } + w.write_char('{')?; + self.write_error(w)?; + w.write_char('}') + } + } + Self::Placeable { expression } => expression.write(w, scope), + } + } + + fn write_error<W>(&self, w: &mut W) -> fmt::Result + where + W: fmt::Write, + { + match self { + Self::MessageReference { + id, + attribute: Some(attribute), + } => write!(w, "{}.{}", id.name, attribute.name), + Self::MessageReference { + id, + attribute: None, + } => w.write_str(id.name), + Self::TermReference { + id, + attribute: Some(attribute), + .. + } => write!(w, "-{}.{}", id.name, attribute.name), + Self::TermReference { + id, + attribute: None, + .. + } => write!(w, "-{}", id.name), + Self::FunctionReference { id, .. } => write!(w, "{}()", id.name), + Self::VariableReference { id } => write!(w, "${}", id.name), + _ => unreachable!(), + } + } +} + +impl<'p> ResolveValue for ast::InlineExpression<&'p str> { + fn resolve<'source, 'errors, R, M>( + &'source self, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> FluentValue<'source> + where + R: Borrow<FluentResource>, + M: MemoizerKind, + { + match self { + Self::StringLiteral { value } => unescape_unicode_to_string(value).into(), + Self::NumberLiteral { value } => FluentValue::try_number(*value), + Self::VariableReference { id } => { + let args = scope.local_args.as_ref().or(scope.args); + + if let Some(arg) = args.and_then(|args| args.get(id.name)) { + arg.clone() + } else { + if scope.local_args.is_none() { + scope.add_error(self.into()); + } + FluentValue::Error + } + } + _ => { + let mut result = String::new(); + self.write(&mut result, scope).expect("Failed to write"); + result.into() + } + } + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/mod.rs b/third_party/rust/fluent-bundle/src/resolver/mod.rs new file mode 100644 index 0000000000..f137bcc91b --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/mod.rs @@ -0,0 +1,42 @@ +pub mod errors; +mod expression; +mod inline_expression; +mod pattern; +mod scope; + +pub use errors::ResolverError; +pub use scope::Scope; + +use std::borrow::Borrow; +use std::fmt; + +use crate::memoizer::MemoizerKind; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +// Converts an AST node to a `FluentValue`. +pub(crate) trait ResolveValue { + fn resolve<'source, 'errors, R, M>( + &'source self, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> FluentValue<'source> + where + R: Borrow<FluentResource>, + M: MemoizerKind; +} + +pub(crate) trait WriteValue { + fn write<'source, 'errors, W, R, M>( + &'source self, + w: &mut W, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + M: MemoizerKind; + + fn write_error<W>(&self, _w: &mut W) -> fmt::Result + where + W: fmt::Write; +} diff --git a/third_party/rust/fluent-bundle/src/resolver/pattern.rs b/third_party/rust/fluent-bundle/src/resolver/pattern.rs new file mode 100644 index 0000000000..4e01d4ca47 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/pattern.rs @@ -0,0 +1,108 @@ +use super::scope::Scope; +use super::{ResolverError, WriteValue}; + +use std::borrow::Borrow; +use std::fmt; + +use fluent_syntax::ast; + +use crate::memoizer::MemoizerKind; +use crate::resolver::ResolveValue; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +const MAX_PLACEABLES: u8 = 100; + +impl<'p> WriteValue for ast::Pattern<&'p str> { + fn write<'scope, 'errors, W, R, M>( + &'scope self, + w: &mut W, + scope: &mut Scope<'scope, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + M: MemoizerKind, + { + let len = self.elements.len(); + + for elem in &self.elements { + if scope.dirty { + return Ok(()); + } + + match elem { + ast::PatternElement::TextElement { value } => { + if let Some(ref transform) = scope.bundle.transform { + w.write_str(&transform(value))?; + } else { + w.write_str(value)?; + } + } + ast::PatternElement::Placeable { ref expression } => { + scope.placeables += 1; + if scope.placeables > MAX_PLACEABLES { + scope.dirty = true; + scope.add_error(ResolverError::TooManyPlaceables); + return Ok(()); + } + + let needs_isolation = scope.bundle.use_isolating + && len > 1 + && !matches!( + expression, + ast::Expression::Inline(ast::InlineExpression::MessageReference { .. },) + | ast::Expression::Inline( + ast::InlineExpression::TermReference { .. }, + ) + | ast::Expression::Inline( + ast::InlineExpression::StringLiteral { .. }, + ) + ); + if needs_isolation { + w.write_char('\u{2068}')?; + } + scope.maybe_track(w, self, expression)?; + if needs_isolation { + w.write_char('\u{2069}')?; + } + } + } + } + Ok(()) + } + + fn write_error<W>(&self, _w: &mut W) -> fmt::Result + where + W: fmt::Write, + { + unreachable!() + } +} + +impl<'p> ResolveValue for ast::Pattern<&'p str> { + fn resolve<'source, 'errors, R, M>( + &'source self, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> FluentValue<'source> + where + R: Borrow<FluentResource>, + M: MemoizerKind, + { + let len = self.elements.len(); + + if len == 1 { + if let ast::PatternElement::TextElement { value } = self.elements[0] { + return scope + .bundle + .transform + .map_or_else(|| value.into(), |transform| transform(value).into()); + } + } + + let mut result = String::new(); + self.write(&mut result, scope) + .expect("Failed to write to a string."); + result.into() + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/scope.rs b/third_party/rust/fluent-bundle/src/resolver/scope.rs new file mode 100644 index 0000000000..004701137e --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/scope.rs @@ -0,0 +1,141 @@ +use crate::bundle::FluentBundle; +use crate::memoizer::MemoizerKind; +use crate::resolver::{ResolveValue, ResolverError, WriteValue}; +use crate::types::FluentValue; +use crate::{FluentArgs, FluentError, FluentResource}; +use fluent_syntax::ast; +use std::borrow::Borrow; +use std::fmt; + +/// State for a single `ResolveValue::to_value` call. +pub struct Scope<'scope, 'errors, R, M> { + /// The current `FluentBundle` instance. + pub bundle: &'scope FluentBundle<R, M>, + /// The current arguments passed by the developer. + pub(super) args: Option<&'scope FluentArgs<'scope>>, + /// Local args + pub(super) local_args: Option<FluentArgs<'scope>>, + /// The running count of resolved placeables. Used to detect the Billion + /// Laughs and Quadratic Blowup attacks. + pub(super) placeables: u8, + /// Tracks hashes to prevent infinite recursion. + travelled: smallvec::SmallVec<[&'scope ast::Pattern<&'scope str>; 2]>, + /// Track errors accumulated during resolving. + pub errors: Option<&'errors mut Vec<FluentError>>, + /// Makes the resolver bail. + pub dirty: bool, +} + +impl<'scope, 'errors, R, M> Scope<'scope, 'errors, R, M> { + pub fn new( + bundle: &'scope FluentBundle<R, M>, + args: Option<&'scope FluentArgs>, + errors: Option<&'errors mut Vec<FluentError>>, + ) -> Self { + Scope { + bundle, + args, + local_args: None, + placeables: 0, + travelled: Default::default(), + errors, + dirty: false, + } + } + + pub fn add_error(&mut self, error: ResolverError) { + if let Some(errors) = self.errors.as_mut() { + errors.push(error.into()); + } + } + + // This method allows us to lazily add Pattern on the stack, + // only if the Pattern::resolve has been called on an empty stack. + // + // This is the case when pattern is called from Bundle and it + // allows us to fast-path simple resolutions, and only use the stack + // for placeables. + pub fn maybe_track<W>( + &mut self, + w: &mut W, + pattern: &'scope ast::Pattern<&str>, + exp: &'scope ast::Expression<&str>, + ) -> fmt::Result + where + R: Borrow<FluentResource>, + W: fmt::Write, + M: MemoizerKind, + { + if self.travelled.is_empty() { + self.travelled.push(pattern); + } + exp.write(w, self)?; + if self.dirty { + w.write_char('{')?; + exp.write_error(w)?; + w.write_char('}') + } else { + Ok(()) + } + } + + pub fn track<W>( + &mut self, + w: &mut W, + pattern: &'scope ast::Pattern<&str>, + exp: &ast::InlineExpression<&str>, + ) -> fmt::Result + where + R: Borrow<FluentResource>, + W: fmt::Write, + M: MemoizerKind, + { + if self.travelled.contains(&pattern) { + self.add_error(ResolverError::Cyclic); + w.write_char('{')?; + exp.write_error(w)?; + w.write_char('}') + } else { + self.travelled.push(pattern); + let result = pattern.write(w, self); + self.travelled.pop(); + result + } + } + + pub fn write_ref_error<W>( + &mut self, + w: &mut W, + exp: &ast::InlineExpression<&str>, + ) -> fmt::Result + where + W: fmt::Write, + { + self.add_error(exp.into()); + w.write_char('{')?; + exp.write_error(w)?; + w.write_char('}') + } + + pub fn get_arguments( + &mut self, + arguments: Option<&'scope ast::CallArguments<&'scope str>>, + ) -> (Vec<FluentValue<'scope>>, FluentArgs<'scope>) + where + R: Borrow<FluentResource>, + M: MemoizerKind, + { + if let Some(ast::CallArguments { positional, named }) = arguments { + let positional = positional.iter().map(|expr| expr.resolve(self)).collect(); + + let named = named + .iter() + .map(|arg| (arg.name.name, arg.value.resolve(self))) + .collect(); + + (positional, named) + } else { + (Vec::new(), FluentArgs::new()) + } + } +} diff --git a/third_party/rust/fluent-bundle/src/resource.rs b/third_party/rust/fluent-bundle/src/resource.rs new file mode 100644 index 0000000000..0c39c838f4 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resource.rs @@ -0,0 +1,171 @@ +use fluent_syntax::ast; +use fluent_syntax::parser::{parse_runtime, ParserError}; + +use self_cell::self_cell; + +type Resource<'s> = ast::Resource<&'s str>; + +self_cell!( + pub struct InnerFluentResource { + owner: String, + + #[covariant] + dependent: Resource, + } + + impl {Debug} +); + +/// A resource containing a list of localization messages. +/// +/// [`FluentResource`] wraps an [`Abstract Syntax Tree`](../fluent_syntax/ast/index.html) produced by the +/// [`parser`](../fluent_syntax/parser/index.html) and provides an access to a list +/// of its entries. +/// +/// A good mental model for a resource is a single FTL file, but in the future +/// there's nothing preventing a resource from being stored in a data base, +/// pre-parsed format or in some other structured form. +/// +/// # Example +/// +/// ``` +/// use fluent_bundle::FluentResource; +/// +/// let source = r#" +/// +/// hello-world = Hello World! +/// +/// "#; +/// +/// let resource = FluentResource::try_new(source.to_string()) +/// .expect("Errors encountered while parsing a resource."); +/// +/// assert_eq!(resource.entries().count(), 1); +/// ``` +/// +/// # Ownership +/// +/// A resource owns the source string and the AST contains references +/// to the slices of the source. +#[derive(Debug)] +pub struct FluentResource(InnerFluentResource); + +impl FluentResource { + /// A fallible constructor of a new [`FluentResource`]. + /// + /// It takes an encoded `Fluent Translation List` string, parses + /// it and stores both, the input string and the AST view of it, + /// for runtime use. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::FluentResource; + /// + /// let source = r#" + /// + /// hello-world = Hello, { $user }! + /// + /// "#; + /// + /// let resource = FluentResource::try_new(source.to_string()); + /// + /// assert!(resource.is_ok()); + /// ``` + /// + /// # Errors + /// + /// The method will return the resource irrelevant of parse errors + /// encountered during parsing of the source, but in case of errors, + /// the `Err` variant will contain both the structure and a vector + /// of errors. + pub fn try_new(source: String) -> Result<Self, (Self, Vec<ParserError>)> { + let mut errors = None; + + let res = InnerFluentResource::new(source, |source| match parse_runtime(source.as_str()) { + Ok(ast) => ast, + Err((ast, err)) => { + errors = Some(err); + ast + } + }); + + match errors { + None => Ok(Self(res)), + Some(err) => Err((Self(res), err)), + } + } + + /// Returns a reference to the source string that was used + /// to construct the [`FluentResource`]. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::FluentResource; + /// + /// let source = "hello-world = Hello, { $user }!"; + /// + /// let resource = FluentResource::try_new(source.to_string()) + /// .expect("Failed to parse FTL."); + /// + /// assert_eq!( + /// resource.source(), + /// "hello-world = Hello, { $user }!" + /// ); + /// ``` + pub fn source(&self) -> &str { + &self.0.borrow_owner() + } + + /// Returns an iterator over [`entries`](fluent_syntax::ast::Entry) of the [`FluentResource`]. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::FluentResource; + /// use fluent_syntax::ast; + /// + /// let source = r#" + /// + /// hello-world = Hello, { $user }! + /// + /// "#; + /// + /// let resource = FluentResource::try_new(source.to_string()) + /// .expect("Failed to parse FTL."); + /// + /// assert_eq!( + /// resource.entries().count(), + /// 1 + /// ); + /// assert!(matches!(resource.entries().next(), Some(ast::Entry::Message(_)))); + /// ``` + pub fn entries(&self) -> impl Iterator<Item = &ast::Entry<&str>> { + self.0.borrow_dependent().body.iter() + } + + /// Returns an [`Entry`](fluent_syntax::ast::Entry) at the + /// given index out of the [`FluentResource`]. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::FluentResource; + /// use fluent_syntax::ast; + /// + /// let source = r#" + /// + /// hello-world = Hello, { $user }! + /// + /// "#; + /// + /// let resource = FluentResource::try_new(source.to_string()) + /// .expect("Failed to parse FTL."); + /// + /// assert!(matches!(resource.get_entry(0), Some(ast::Entry::Message(_)))); + /// ``` + pub fn get_entry(&self, idx: usize) -> Option<&ast::Entry<&str>> { + self.0.borrow_dependent().body.get(idx) + } +} diff --git a/third_party/rust/fluent-bundle/src/types/mod.rs b/third_party/rust/fluent-bundle/src/types/mod.rs new file mode 100644 index 0000000000..714fe4c76f --- /dev/null +++ b/third_party/rust/fluent-bundle/src/types/mod.rs @@ -0,0 +1,202 @@ +//! `types` module contains types necessary for Fluent runtime +//! value handling. +//! The core struct is [`FluentValue`] which is a type that can be passed +//! to the [`FluentBundle::format_pattern`](crate::bundle::FluentBundle) as an argument, it can be passed +//! to any Fluent Function, and any function may return it. +//! +//! This part of functionality is not fully hashed out yet, since we're waiting +//! for the internationalization APIs to mature, at which point all number +//! formatting operations will be moved out of Fluent. +//! +//! For now, [`FluentValue`] can be a string, a number, or a custom [`FluentType`] +//! which allows users of the library to implement their own types of values, +//! such as dates, or more complex structures needed for their bindings. +mod number; +mod plural; + +pub use number::*; +use plural::PluralRules; + +use std::any::Any; +use std::borrow::{Borrow, Cow}; +use std::fmt; +use std::str::FromStr; + +use intl_pluralrules::{PluralCategory, PluralRuleType}; + +use crate::memoizer::MemoizerKind; +use crate::resolver::Scope; +use crate::resource::FluentResource; + +pub trait FluentType: fmt::Debug + AnyEq + 'static { + fn duplicate(&self) -> Box<dyn FluentType + Send>; + fn as_string(&self, intls: &intl_memoizer::IntlLangMemoizer) -> Cow<'static, str>; + fn as_string_threadsafe( + &self, + intls: &intl_memoizer::concurrent::IntlLangMemoizer, + ) -> Cow<'static, str>; +} + +impl PartialEq for dyn FluentType + Send { + fn eq(&self, other: &Self) -> bool { + self.equals(other.as_any()) + } +} + +pub trait AnyEq: Any + 'static { + fn equals(&self, other: &dyn Any) -> bool; + fn as_any(&self) -> &dyn Any; +} + +impl<T: Any + PartialEq> AnyEq for T { + fn equals(&self, other: &dyn Any) -> bool { + other + .downcast_ref::<Self>() + .map_or(false, |that| self == that) + } + fn as_any(&self) -> &dyn Any { + self + } +} + +/// The `FluentValue` enum represents values which can be formatted to a String. +/// +/// Those values are either passed as arguments to [`FluentBundle::format_pattern`][] or +/// produced by functions, or generated in the process of pattern resolution. +/// +/// [`FluentBundle::format_pattern`]: ../bundle/struct.FluentBundle.html#method.format_pattern +#[derive(Debug)] +pub enum FluentValue<'source> { + String(Cow<'source, str>), + Number(FluentNumber), + Custom(Box<dyn FluentType + Send>), + None, + Error, +} + +impl<'s> PartialEq for FluentValue<'s> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (FluentValue::String(s), FluentValue::String(s2)) => s == s2, + (FluentValue::Number(s), FluentValue::Number(s2)) => s == s2, + (FluentValue::Custom(s), FluentValue::Custom(s2)) => s == s2, + _ => false, + } + } +} + +impl<'s> Clone for FluentValue<'s> { + fn clone(&self) -> Self { + match self { + FluentValue::String(s) => FluentValue::String(s.clone()), + FluentValue::Number(s) => FluentValue::Number(s.clone()), + FluentValue::Custom(s) => { + let new_value: Box<dyn FluentType + Send> = s.duplicate(); + FluentValue::Custom(new_value) + } + FluentValue::Error => FluentValue::Error, + FluentValue::None => FluentValue::None, + } + } +} + +impl<'source> FluentValue<'source> { + pub fn try_number<S: ToString>(v: S) -> Self { + let s = v.to_string(); + if let Ok(num) = FluentNumber::from_str(&s) { + num.into() + } else { + s.into() + } + } + + pub fn matches<R: Borrow<FluentResource>, M>( + &self, + other: &FluentValue, + scope: &Scope<R, M>, + ) -> bool + where + M: MemoizerKind, + { + match (self, other) { + (&FluentValue::String(ref a), &FluentValue::String(ref b)) => a == b, + (&FluentValue::Number(ref a), &FluentValue::Number(ref b)) => a == b, + (&FluentValue::String(ref a), &FluentValue::Number(ref b)) => { + let cat = match a.as_ref() { + "zero" => PluralCategory::ZERO, + "one" => PluralCategory::ONE, + "two" => PluralCategory::TWO, + "few" => PluralCategory::FEW, + "many" => PluralCategory::MANY, + "other" => PluralCategory::OTHER, + _ => return false, + }; + scope + .bundle + .intls + .with_try_get_threadsafe::<PluralRules, _, _>( + (PluralRuleType::CARDINAL,), + |pr| pr.0.select(b) == Ok(cat), + ) + .unwrap() + } + _ => false, + } + } + + pub fn write<W, R, M>(&self, w: &mut W, scope: &Scope<R, M>) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + M: MemoizerKind, + { + if let Some(formatter) = &scope.bundle.formatter { + if let Some(val) = formatter(self, &scope.bundle.intls) { + return w.write_str(&val); + } + } + match self { + FluentValue::String(s) => w.write_str(s), + FluentValue::Number(n) => w.write_str(&n.as_string()), + FluentValue::Custom(s) => w.write_str(&scope.bundle.intls.stringify_value(&**s)), + FluentValue::Error => Ok(()), + FluentValue::None => Ok(()), + } + } + + pub fn as_string<R: Borrow<FluentResource>, M>(&self, scope: &Scope<R, M>) -> Cow<'source, str> + where + M: MemoizerKind, + { + if let Some(formatter) = &scope.bundle.formatter { + if let Some(val) = formatter(self, &scope.bundle.intls) { + return val.into(); + } + } + match self { + FluentValue::String(s) => s.clone(), + FluentValue::Number(n) => n.as_string(), + FluentValue::Custom(s) => scope.bundle.intls.stringify_value(&**s), + FluentValue::Error => "".into(), + FluentValue::None => "".into(), + } + } +} + +impl<'source> From<String> for FluentValue<'source> { + fn from(s: String) -> Self { + FluentValue::String(s.into()) + } +} + +impl<'source> From<&'source str> for FluentValue<'source> { + fn from(s: &'source str) -> Self { + FluentValue::String(s.into()) + } +} + +impl<'source> From<Cow<'source, str>> for FluentValue<'source> { + fn from(s: Cow<'source, str>) -> Self { + FluentValue::String(s) + } +} diff --git a/third_party/rust/fluent-bundle/src/types/number.rs b/third_party/rust/fluent-bundle/src/types/number.rs new file mode 100644 index 0000000000..d39291ff46 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/types/number.rs @@ -0,0 +1,252 @@ +use std::borrow::Cow; +use std::convert::TryInto; +use std::default::Default; +use std::str::FromStr; + +use intl_pluralrules::operands::PluralOperands; + +use crate::args::FluentArgs; +use crate::types::FluentValue; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum FluentNumberStyle { + Decimal, + Currency, + Percent, +} + +impl std::default::Default for FluentNumberStyle { + fn default() -> Self { + Self::Decimal + } +} + +impl From<&str> for FluentNumberStyle { + fn from(input: &str) -> Self { + match input { + "decimal" => Self::Decimal, + "currency" => Self::Currency, + "percent" => Self::Percent, + _ => Self::default(), + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum FluentNumberCurrencyDisplayStyle { + Symbol, + Code, + Name, +} + +impl std::default::Default for FluentNumberCurrencyDisplayStyle { + fn default() -> Self { + Self::Symbol + } +} + +impl From<&str> for FluentNumberCurrencyDisplayStyle { + fn from(input: &str) -> Self { + match input { + "symbol" => Self::Symbol, + "code" => Self::Code, + "name" => Self::Name, + _ => Self::default(), + } + } +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct FluentNumberOptions { + pub style: FluentNumberStyle, + pub currency: Option<String>, + pub currency_display: FluentNumberCurrencyDisplayStyle, + pub use_grouping: bool, + pub minimum_integer_digits: Option<usize>, + pub minimum_fraction_digits: Option<usize>, + pub maximum_fraction_digits: Option<usize>, + pub minimum_significant_digits: Option<usize>, + pub maximum_significant_digits: Option<usize>, +} + +impl Default for FluentNumberOptions { + fn default() -> Self { + Self { + style: Default::default(), + currency: None, + currency_display: Default::default(), + use_grouping: true, + minimum_integer_digits: None, + minimum_fraction_digits: None, + maximum_fraction_digits: None, + minimum_significant_digits: None, + maximum_significant_digits: None, + } + } +} + +impl FluentNumberOptions { + pub fn merge(&mut self, opts: &FluentArgs) { + for (key, value) in opts.iter() { + match (key, value) { + ("style", FluentValue::String(n)) => { + self.style = n.as_ref().into(); + } + ("currency", FluentValue::String(n)) => { + self.currency = Some(n.to_string()); + } + ("currencyDisplay", FluentValue::String(n)) => { + self.currency_display = n.as_ref().into(); + } + ("useGrouping", FluentValue::String(n)) => { + self.use_grouping = n != "false"; + } + ("minimumIntegerDigits", FluentValue::Number(n)) => { + self.minimum_integer_digits = Some(n.into()); + } + ("minimumFractionDigits", FluentValue::Number(n)) => { + self.minimum_fraction_digits = Some(n.into()); + } + ("maximumFractionDigits", FluentValue::Number(n)) => { + self.maximum_fraction_digits = Some(n.into()); + } + ("minimumSignificantDigits", FluentValue::Number(n)) => { + self.minimum_significant_digits = Some(n.into()); + } + ("maximumSignificantDigits", FluentValue::Number(n)) => { + self.maximum_significant_digits = Some(n.into()); + } + _ => {} + } + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct FluentNumber { + pub value: f64, + pub options: FluentNumberOptions, +} + +impl FluentNumber { + pub const fn new(value: f64, options: FluentNumberOptions) -> Self { + Self { value, options } + } + + pub fn as_string(&self) -> Cow<'static, str> { + let mut val = self.value.to_string(); + if let Some(minfd) = self.options.minimum_fraction_digits { + if let Some(pos) = val.find('.') { + let frac_num = val.len() - pos - 1; + let missing = if frac_num > minfd { + 0 + } else { + minfd - frac_num + }; + val = format!("{}{}", val, "0".repeat(missing)); + } else { + val = format!("{}.{}", val, "0".repeat(minfd)); + } + } + val.into() + } +} + +impl FromStr for FluentNumber { + type Err = std::num::ParseFloatError; + + fn from_str(input: &str) -> Result<Self, Self::Err> { + f64::from_str(input).map(|n| { + let mfd = input.find('.').map(|pos| input.len() - pos - 1); + let opts = FluentNumberOptions { + minimum_fraction_digits: mfd, + ..Default::default() + }; + Self::new(n, opts) + }) + } +} + +impl<'l> From<FluentNumber> for FluentValue<'l> { + fn from(input: FluentNumber) -> Self { + FluentValue::Number(input) + } +} + +macro_rules! from_num { + ($num:ty) => { + impl From<$num> for FluentNumber { + fn from(n: $num) -> Self { + Self { + value: n as f64, + options: FluentNumberOptions::default(), + } + } + } + impl From<&$num> for FluentNumber { + fn from(n: &$num) -> Self { + Self { + value: *n as f64, + options: FluentNumberOptions::default(), + } + } + } + impl From<FluentNumber> for $num { + fn from(input: FluentNumber) -> Self { + input.value as $num + } + } + impl From<&FluentNumber> for $num { + fn from(input: &FluentNumber) -> Self { + input.value as $num + } + } + impl From<$num> for FluentValue<'_> { + fn from(n: $num) -> Self { + FluentValue::Number(n.into()) + } + } + impl From<&$num> for FluentValue<'_> { + fn from(n: &$num) -> Self { + FluentValue::Number(n.into()) + } + } + }; + ($($num:ty)+) => { + $(from_num!($num);)+ + }; +} + +impl From<&FluentNumber> for PluralOperands { + fn from(input: &FluentNumber) -> Self { + let mut operands: Self = input + .value + .try_into() + .expect("Failed to generate operands out of FluentNumber"); + if let Some(mfd) = input.options.minimum_fraction_digits { + if mfd > operands.v { + operands.f *= 10_u64.pow(mfd as u32 - operands.v as u32); + operands.v = mfd; + } + } + // XXX: Add support for other options. + operands + } +} + +from_num!(i8 i16 i32 i64 i128 isize); +from_num!(u8 u16 u32 u64 u128 usize); +from_num!(f32 f64); + +#[cfg(test)] +mod tests { + use crate::types::FluentValue; + + #[test] + fn value_from_copy_ref() { + let x = 1i16; + let y = &x; + let z: FluentValue = y.into(); + assert_eq!(z, FluentValue::try_number(1)); + } +} diff --git a/third_party/rust/fluent-bundle/src/types/plural.rs b/third_party/rust/fluent-bundle/src/types/plural.rs new file mode 100644 index 0000000000..1151fd6d36 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/types/plural.rs @@ -0,0 +1,22 @@ +use fluent_langneg::{negotiate_languages, NegotiationStrategy}; +use intl_memoizer::Memoizable; +use intl_pluralrules::{PluralRuleType, PluralRules as IntlPluralRules}; +use unic_langid::LanguageIdentifier; + +pub struct PluralRules(pub IntlPluralRules); + +impl Memoizable for PluralRules { + type Args = (PluralRuleType,); + type Error = &'static str; + fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> { + let default_lang: LanguageIdentifier = "en".parse().unwrap(); + let pr_lang = negotiate_languages( + &[lang], + &IntlPluralRules::get_locales(args.0), + Some(&default_lang), + NegotiationStrategy::Lookup, + )[0] + .clone(); + Ok(Self(IntlPluralRules::create(pr_lang, args.0)?)) + } +} |