diff options
Diffstat (limited to 'third_party/rust/fluent-bundle')
25 files changed, 2671 insertions, 0 deletions
diff --git a/third_party/rust/fluent-bundle/.cargo-checksum.json b/third_party/rust/fluent-bundle/.cargo-checksum.json new file mode 100644 index 0000000000..718e6fb35a --- /dev/null +++ b/third_party/rust/fluent-bundle/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.toml":"faf6b0a0061a6379af7f95060b77c2e51460d9b2e8f673570c81a67d855bbfa9","LICENSE-APACHE":"5db2b182453ff32ed40f7da63589c9667a3f8bd8b16b1471b152caae56f77e45","LICENSE-MIT":"49c0b000c03731d9e3970dc059ad4ca345d773681f4a612b0024435b663e0220","README.md":"4143ee70148d1e9a14131a0cc9a68c73cc43375bb3173110f516dc0bb2959640","benches/resolver.rs":"084684b22d5ccf86180354381b1be76245b36beb51ab5bbc44d9c94dacc0948f","examples/README.md":"99a51f7d388d2da3c5291e68de5264feaf642ba9a22f6f882c3b940c1b6429b2","src/args.rs":"3e04a53ea22cf86d4b2777942da3cbc183e25673928812884acaaf9a199ed769","src/bundle.rs":"aa59968ce98bba8d4a14b321d38f10938cc89535cb3f8519bce75ef7533c1912","src/concurrent.rs":"3a760f85d7373af13d8b1fdd2a03bd8aede8c0d08ec61589a8a0866df6a86e76","src/entry.rs":"8be695ed178d072d5471d18fd35d5a711db6e7d6cbcf651a5d9f1ce2ea55804f","src/errors.rs":"a924cdfcd3f76b42e162729d9131d7d528e49047cab38bf22cb3aefc8ad1b182","src/lib.rs":"a89ffa82eec432df803c741095563dae60784dedd77132e40c5529eba44a9c90","src/memoizer.rs":"922084f71f02d0532056db9b41cec4c1434001fe60215ee6f6ac8e3fd2518f12","src/message.rs":"dfb53488f82c425f97f99c5544e5410b465b55826423c35467c83e836d6d3464","src/resolver/errors.rs":"beaf41fabbfd11211cb2c3db6ca0ba26bccf75817bed05a92b980393edfb3f9f","src/resolver/expression.rs":"93f12328241161493371dee6a7d3e4c0baeb7d5db693ad45dec70f11ab766a8f","src/resolver/inline_expression.rs":"69052aa5a56bec6d41ae244c015202fae549112b985fff7e4dc79d94218b1af6","src/resolver/mod.rs":"3369d3792e987259ca572547c749930c24602fda48a1ed952f9cc6412a7e16b6","src/resolver/pattern.rs":"772ac9af07343e2cac2da63627b851aa442fe6172554f74879d3fed001f9483a","src/resolver/scope.rs":"8d193e390a356328fcec2b05eb51cf077ac1395c48d987b8dc544d7572b87fa2","src/resource.rs":"bcb5ec46f95ceca948f0d26b681c8569f677d9f9804f6fe5f85236407926b5c7","src/types/mod.rs":"48ab65ba9e9e11fb57fce30020ac69d2ec141c855950bed5bed36f9cd37835b3","src/types/number.rs":"d2073a2c03bcff3a55d9a8b8f206497bc2be11f56c598c448549b64bc2bd5d2d","src/types/plural.rs":"f28834e71d6970d5eb48089132f5242433b1e62b90765d85e3c76f805eecc92e"},"package":"4c0cd40fee9f6eb74dfb38f0fa700a92ac4fe702868b04e52018ead90d415332"}
\ No newline at end of file diff --git a/third_party/rust/fluent-bundle/Cargo.toml b/third_party/rust/fluent-bundle/Cargo.toml new file mode 100644 index 0000000000..d11babfc05 --- /dev/null +++ b/third_party/rust/fluent-bundle/Cargo.toml @@ -0,0 +1,65 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +edition = "2018" +name = "fluent-bundle" +version = "0.14.1" +authors = ["Zibi Braniecki <gandalf@mozilla.com>", "Staś Małolepszy <stas@mozilla.com>"] +include = ["src/**/*", "benches/*.rs", "Cargo.toml", "README.md", "LICENSE-APACHE", "LICENSE-MIT"] +description = "A localization system designed to unleash the entire expressive power of\nnatural language translations.\n" +homepage = "http://www.projectfluent.org" +readme = "README.md" +keywords = ["localization", "l10n", "i18n", "intl", "internationalization"] +categories = ["localization", "internationalization"] +license = "Apache-2.0/MIT" +repository = "https://github.com/projectfluent/fluent-rs" + +[[bench]] +name = "resolver" +harness = false +[dependencies.fluent-langneg] +version = "0.13" + +[dependencies.fluent-syntax] +version = "0.10.1" + +[dependencies.intl-memoizer] +version = "0.5" + +[dependencies.intl_pluralrules] +version = "7.0.1" + +[dependencies.ouroboros] +version = "0.7" + +[dependencies.smallvec] +version = "1" + +[dependencies.unic-langid] +version = "0.9" +[dev-dependencies.criterion] +version = "0.3" + +[dev-dependencies.rand] +version = "0.8" + +[dev-dependencies.serde] +version = "1.0" +features = ["derive"] + +[dev-dependencies.serde_yaml] +version = "0.8" + +[dev-dependencies.unic-langid] +version = "0.9" +features = ["macros"] diff --git a/third_party/rust/fluent-bundle/LICENSE-APACHE b/third_party/rust/fluent-bundle/LICENSE-APACHE new file mode 100644 index 0000000000..35582f166b --- /dev/null +++ b/third_party/rust/fluent-bundle/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Mozilla + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/rust/fluent-bundle/LICENSE-MIT b/third_party/rust/fluent-bundle/LICENSE-MIT new file mode 100644 index 0000000000..5655fa311c --- /dev/null +++ b/third_party/rust/fluent-bundle/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright 2017 Mozilla + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/third_party/rust/fluent-bundle/README.md b/third_party/rust/fluent-bundle/README.md new file mode 100644 index 0000000000..1cd0a75758 --- /dev/null +++ b/third_party/rust/fluent-bundle/README.md @@ -0,0 +1,111 @@ +# Fluent + +`fluent-rs` is a Rust implementation of [Project Fluent][], a localization +framework designed to unleash the entire expressive power of natural language +translations. + +[](https://crates.io/crates/fluent) +[](https://travis-ci.org/projectfluent/fluent-rs) +[](https://coveralls.io/github/projectfluent/fluent-rs?branch=master) + +Project Fluent keeps simple things simple and makes complex things possible. +The syntax used for describing translations is easy to read and understand. At +the same time it allows, when necessary, to represent complex concepts from +natural languages like gender, plurals, conjugations, and others. + +[Documentation][] + +[Project Fluent]: http://projectfluent.org +[Documentation]: https://docs.rs/fluent/ + +Usage +----- + +```rust +use fluent_bundle::{FluentBundle, FluentResource}; +use unic_langid::langid; + +fn main() { + let ftl_string = "hello-world = Hello, world!".to_owned(); + let res = FluentResource::try_new(ftl_string) + .expect("Could not parse an FTL string."); + + let langid_en = langid!("en"); + let mut bundle = FluentBundle::new(vec![langid_en]); + + bundle.add_resource(&res) + .expect("Failed to add FTL resources to the bundle."); + + let msg = bundle.get_message("hello-world") + .expect("Failed to retrieve a message."); + let val = msg.value.expect("Message has no value."); + + let mut errors = vec![]; + let value = bundle.format_pattern(val, None, &mut errors); + + assert_eq!(&value, "Hello, world!"); +} +``` + + +Status +------ + +The implementation is in its early stages and supports only some of the Project +Fluent's spec. Consult the [list of milestones][] for more information about +release planning and scope. + +[list of milestones]: https://github.com/projectfluent/fluent-rs/milestones + + +Local Development +----------------- + + cargo build + cargo test + cargo bench + cargo run --example simple-app + +When submitting a PR please use [`cargo fmt`][] (nightly). + +[`cargo fmt`]: https://github.com/rust-lang-nursery/rustfmt + + +Learn the FTL syntax +-------------------- + +FTL is a localization file format used for describing translation resources. +FTL stands for _Fluent Translation List_. + +FTL is designed to be simple to read, but at the same time allows to represent +complex concepts from natural languages like gender, plurals, conjugations, and +others. + + hello-user = Hello, { $username }! + +[Read the Fluent Syntax Guide][] in order to learn more about the syntax. If +you're a tool author you may be interested in the formal [EBNF grammar][]. + +[Read the Fluent Syntax Guide]: http://projectfluent.org/fluent/guide/ +[EBNF grammar]: https://github.com/projectfluent/fluent/tree/master/spec + + +Get Involved +------------ + +`fluent-rs` is open-source, licensed under the Apache License, Version 2.0. We +encourage everyone to take a look at our code and we'll listen to your +feedback. + + +Discuss +------- + +We'd love to hear your thoughts on Project Fluent! Whether you're a localizer +looking for a better way to express yourself in your language, or a developer +trying to make your app localizable and multilingual, or a hacker looking for +a project to contribute to, please do get in touch on the mailing list and the +IRC channel. + + - Discourse: https://discourse.mozilla.org/c/fluent + - IRC channel: [irc://irc.mozilla.org/l20n](irc://irc.mozilla.org/l20n) diff --git a/third_party/rust/fluent-bundle/benches/resolver.rs b/third_party/rust/fluent-bundle/benches/resolver.rs new file mode 100644 index 0000000000..31f084755e --- /dev/null +++ b/third_party/rust/fluent-bundle/benches/resolver.rs @@ -0,0 +1,142 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::BenchmarkId; +use criterion::Criterion; +use std::collections::HashMap; +use std::fs::File; +use std::io; +use std::io::Read; + +use fluent_bundle::{FluentArgs, FluentBundle, FluentResource, FluentValue}; +use fluent_syntax::ast; +use unic_langid::langid; + +fn read_file(path: &str) -> Result<String, io::Error> { + let mut f = File::open(path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + Ok(s) +} + +fn get_strings(tests: &[&'static str]) -> HashMap<&'static str, String> { + let mut ftl_strings = HashMap::new(); + for test in tests { + let path = format!("./benches/{}.ftl", test); + ftl_strings.insert(*test, read_file(&path).expect("Couldn't load file")); + } + return ftl_strings; +} + +fn get_ids(res: &FluentResource) -> Vec<String> { + res.ast() + .body + .iter() + .filter_map(|entry| match entry { + ast::Entry::Message(ast::Message { id, .. }) => Some(id.name.to_owned()), + _ => None, + }) + .collect() +} + +fn get_args(name: &str) -> Option<FluentArgs> { + match name { + "preferences" => { + let mut prefs_args = FluentArgs::new(); + prefs_args.add("name", FluentValue::from("John")); + prefs_args.add("tabCount", FluentValue::from(5)); + prefs_args.add("count", FluentValue::from(3)); + prefs_args.add("version", FluentValue::from("65.0")); + prefs_args.add("path", FluentValue::from("/tmp")); + prefs_args.add("num", FluentValue::from(4)); + prefs_args.add("email", FluentValue::from("john@doe.com")); + prefs_args.add("value", FluentValue::from(4.5)); + prefs_args.add("unit", FluentValue::from("mb")); + prefs_args.add("service-name", FluentValue::from("Mozilla Disk")); + Some(prefs_args) + } + _ => None, + } +} + +fn add_functions<R>(name: &'static str, bundle: &mut FluentBundle<R>) { + match name { + "preferences" => { + bundle + .add_function("PLATFORM", |_args, _named_args| { + return "linux".into(); + }) + .expect("Failed to add a function to the bundle."); + } + _ => {} + } +} + +fn get_bundle(name: &'static str, source: &str) -> (FluentBundle<FluentResource>, Vec<String>) { + let res = FluentResource::try_new(source.to_owned()).expect("Couldn't parse an FTL source"); + let ids = get_ids(&res); + let lids = vec![langid!("en")]; + let mut bundle = FluentBundle::new(lids); + bundle + .add_resource(res) + .expect("Couldn't add FluentResource to the FluentBundle"); + add_functions(name, &mut bundle); + (bundle, ids) +} + +fn resolver_bench(c: &mut Criterion) { + let tests = &["simple", "preferences", "menubar", "unescape"]; + let ftl_strings = get_strings(tests); + + let mut group = c.benchmark_group("resolve"); + for name in tests { + let source = ftl_strings.get(name).expect("Failed to find the source."); + group.bench_with_input(BenchmarkId::from_parameter(name), &source, |b, source| { + let (bundle, ids) = get_bundle(name, source); + let args = get_args(name); + b.iter(|| { + let mut s = String::new(); + for id in &ids { + let msg = bundle.get_message(id).expect("Message found"); + let mut errors = vec![]; + if let Some(value) = msg.value { + let _ = bundle.write_pattern(&mut s, value, args.as_ref(), &mut errors); + s.clear(); + } + for attr in msg.attributes { + let _ = + bundle.write_pattern(&mut s, attr.value, args.as_ref(), &mut errors); + s.clear(); + } + assert!(errors.len() == 0, "Resolver errors: {:#?}", errors); + } + }) + }); + } + group.finish(); + + let mut group = c.benchmark_group("resolve_to_str"); + for name in tests { + let source = ftl_strings.get(name).expect("Failed to find the source."); + group.bench_with_input(BenchmarkId::from_parameter(name), &source, |b, source| { + let (bundle, ids) = get_bundle(name, source); + let args = get_args(name); + b.iter(|| { + for id in &ids { + let msg = bundle.get_message(id).expect("Message found"); + let mut errors = vec![]; + if let Some(value) = msg.value { + let _ = bundle.format_pattern(value, args.as_ref(), &mut errors); + } + for attr in msg.attributes { + let _ = bundle.format_pattern(attr.value, args.as_ref(), &mut errors); + } + assert!(errors.len() == 0, "Resolver errors: {:#?}", errors); + } + }) + }); + } + group.finish(); +} + +criterion_group!(benches, resolver_bench); +criterion_main!(benches); diff --git a/third_party/rust/fluent-bundle/examples/README.md b/third_party/rust/fluent-bundle/examples/README.md new file mode 100644 index 0000000000..2411aeb903 --- /dev/null +++ b/third_party/rust/fluent-bundle/examples/README.md @@ -0,0 +1,6 @@ +This directory contains a set of examples +of how to use Fluent. + +Start with the `simple-app.rs` which is a very +trivial example of a command line application +with localization handled by Fluent. diff --git a/third_party/rust/fluent-bundle/src/args.rs b/third_party/rust/fluent-bundle/src/args.rs new file mode 100644 index 0000000000..368d70e4ed --- /dev/null +++ b/third_party/rust/fluent-bundle/src/args.rs @@ -0,0 +1,74 @@ +use std::borrow::Cow; +use std::iter::FromIterator; + +use crate::types::FluentValue; + +/// A map of arguments passed from the code to +/// the localization to be used for message +/// formatting. +#[derive(Debug, Default)] +pub struct FluentArgs<'args>(Vec<(Cow<'args, str>, FluentValue<'args>)>); + +impl<'args> FluentArgs<'args> { + pub fn new() -> Self { + Self(vec![]) + } + + pub fn with_capacity(capacity: usize) -> Self { + Self(Vec::with_capacity(capacity)) + } + + pub fn get(&self, key: &str) -> Option<&FluentValue<'args>> { + self.0.iter().find(|(k, _)| key == *k).map(|(_, v)| v) + } + + pub fn add<K>(&mut self, key: K, value: FluentValue<'args>) + where + K: Into<Cow<'args, str>>, + { + self.0.push((key.into(), value)); + } + + pub fn iter(&self) -> impl Iterator<Item = (&str, &FluentValue)> { + self.0.iter().map(|(k, v)| (k.as_ref(), v)) + } +} + +impl<'args> FromIterator<(&'args str, FluentValue<'args>)> for FluentArgs<'args> { + fn from_iter<I>(iter: I) -> Self + where + I: IntoIterator<Item = (&'args str, FluentValue<'args>)>, + { + let mut c = FluentArgs::new(); + + for (k, v) in iter { + c.add(k, v); + } + + c + } +} + +impl<'args> FromIterator<(String, FluentValue<'args>)> for FluentArgs<'args> { + fn from_iter<I>(iter: I) -> Self + where + I: IntoIterator<Item = (String, FluentValue<'args>)>, + { + let mut c = FluentArgs::new(); + + for (k, v) in iter { + c.add(k, v); + } + + c + } +} + +impl<'args> IntoIterator for FluentArgs<'args> { + type Item = (Cow<'args, str>, FluentValue<'args>); + type IntoIter = std::vec::IntoIter<Self::Item>; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} diff --git a/third_party/rust/fluent-bundle/src/bundle.rs b/third_party/rust/fluent-bundle/src/bundle.rs new file mode 100644 index 0000000000..091528b859 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/bundle.rs @@ -0,0 +1,510 @@ +//! `FluentBundle` is a collection of localization messages in Fluent. +//! +//! It stores a list of messages in a single locale which can reference one another, use the same +//! internationalization formatters, functions, scopeironmental variables and are expected to be used +//! together. + +use std::borrow::Borrow; +use std::borrow::Cow; +use std::collections::hash_map::{Entry as HashEntry, HashMap}; +use std::default::Default; +use std::fmt; + +use fluent_syntax::ast; +use unic_langid::LanguageIdentifier; + +use crate::args::FluentArgs; +use crate::entry::Entry; +use crate::entry::GetEntry; +use crate::errors::{EntryKind, FluentError}; +use crate::memoizer::MemoizerKind; +use crate::message::{FluentAttribute, FluentMessage}; +use crate::resolver::{ResolveValue, Scope, WriteValue}; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +/// Base class for a [`FluentBundle`] struct. See its docs for details. +/// It also is implemented for [`concurrent::FluentBundle`]. +/// +/// [`FluentBundle`]: ../type.FluentBundle.html +/// [`concurrent::FluentBundle`]: ../concurrent/type.FluentBundle.html +pub struct FluentBundleBase<R, M> { + pub locales: Vec<LanguageIdentifier>, + pub(crate) resources: Vec<R>, + pub(crate) entries: HashMap<String, Entry>, + pub(crate) intls: M, + pub(crate) use_isolating: bool, + pub(crate) transform: Option<fn(&str) -> Cow<str>>, + pub(crate) formatter: Option<fn(&FluentValue, &M) -> Option<String>>, +} + +impl<R, M: MemoizerKind> FluentBundleBase<R, M> { + /// Constructs a FluentBundle. The first element in `locales` should be the + /// language this bundle represents, and will be used to determine the + /// correct plural rules for this bundle. You can optionally provide extra + /// languages in the list; they will be used as fallback date and time + /// formatters if a formatter for the primary language is unavailable. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::FluentBundle; + /// use fluent_bundle::FluentResource; + /// use unic_langid::langid; + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle: FluentBundle<FluentResource> = FluentBundle::new(vec![langid_en]); + /// ``` + /// + /// # Errors + /// + /// This will panic if no formatters can be found for the locales. + pub fn new(locales: Vec<LanguageIdentifier>) -> Self { + let first_locale = locales.get(0).cloned().unwrap_or_default(); + Self { + locales, + resources: vec![], + entries: HashMap::new(), + intls: M::new(first_locale), + use_isolating: true, + transform: None, + formatter: None, + } + } + + /// Adds a resource to the bundle, returning an empty [`Result<T>`] on success. + /// + /// If any entry in the resource uses the same identifier as an already + /// existing key in the bundle, the new entry will be ignored and a + /// `FluentError::Overriding` will be added to the result. + /// + /// The method can take any type that can be borrowed to `FluentResource`: + /// - FluentResource + /// - &FluentResource + /// - Rc<FluentResource> + /// - Arc<FluentResurce> + /// + /// This allows the user to introduce custom resource management and share + /// resources between instances of `FluentBundle`. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from(" + /// hello = Hi! + /// goodbye = Bye! + /// "); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// assert_eq!(true, bundle.has_message("hello")); + /// ``` + /// + /// # Whitespace + /// + /// Message ids must have no leading whitespace. Message values that span + /// multiple lines must have leading whitespace on all but the first line. These + /// are standard FTL syntax rules that may prove a bit troublesome in source + /// code formatting. The [`indoc!`] crate can help with stripping extra indentation + /// if you wish to indent your entire message. + /// + /// [FTL syntax]: https://projectfluent.org/fluent/guide/ + /// [`indoc!`]: https://github.com/dtolnay/indoc + /// [`Result<T>`]: https://doc.rust-lang.org/std/result/enum.Result.html + pub fn add_resource(&mut self, r: R) -> Result<(), Vec<FluentError>> + where + R: Borrow<FluentResource>, + { + let mut errors = vec![]; + + let res = r.borrow(); + let res_pos = self.resources.len(); + + for (entry_pos, entry) in res.ast().body.iter().enumerate() { + let id = match entry { + ast::Entry::Message(ast::Message { ref id, .. }) + | ast::Entry::Term(ast::Term { ref id, .. }) => id.name, + _ => continue, + }; + + let (entry, kind) = match entry { + ast::Entry::Message(..) => { + (Entry::Message([res_pos, entry_pos]), EntryKind::Message) + } + ast::Entry::Term(..) => (Entry::Term([res_pos, entry_pos]), EntryKind::Term), + _ => continue, + }; + + match self.entries.entry(id.to_string()) { + HashEntry::Vacant(empty) => { + empty.insert(entry); + } + HashEntry::Occupied(_) => { + errors.push(FluentError::Overriding { + kind, + id: id.to_string(), + }); + } + } + } + self.resources.push(r); + + if errors.is_empty() { + Ok(()) + } else { + Err(errors) + } + } + + /// Adds a resource to the bundle, returning an empty [`Result<T>`] on success. + /// + /// If any entry in the resource uses the same identifier as an already + /// existing key in the bundle, the entry will override the previous one. + /// + /// The method can take any type that can be borrowed as FluentResource: + /// - FluentResource + /// - &FluentResource + /// - Rc<FluentResource> + /// - Arc<FluentResurce> + /// + /// This allows the user to introduce custom resource management and share + /// resources between instances of `FluentBundle`. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from(" + /// hello = Hi! + /// goodbye = Bye! + /// "); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// + /// let ftl_string = String::from(" + /// hello = Another Hi! + /// "); + /// let resource2 = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// bundle.add_resource_overriding(resource2); + /// + /// let mut errors = vec![]; + /// let msg = bundle.get_message("hello") + /// .expect("Failed to retrieve the message"); + /// let value = msg.value.expect("Failed to retrieve the value of the message"); + /// assert_eq!(bundle.format_pattern(value, None, &mut errors), "Another Hi!"); + /// ``` + /// + /// # Whitespace + /// + /// Message ids must have no leading whitespace. Message values that span + /// multiple lines must have leading whitespace on all but the first line. These + /// are standard FTL syntax rules that may prove a bit troublesome in source + /// code formatting. The [`indoc!`] crate can help with stripping extra indentation + /// if you wish to indent your entire message. + /// + /// [FTL syntax]: https://projectfluent.org/fluent/guide/ + /// [`indoc!`]: https://github.com/dtolnay/indoc + /// [`Result<T>`]: https://doc.rust-lang.org/std/result/enum.Result.html + pub fn add_resource_overriding(&mut self, r: R) + where + R: Borrow<FluentResource>, + { + let res = r.borrow(); + let res_pos = self.resources.len(); + + for (entry_pos, entry) in res.ast().body.iter().enumerate() { + let id = match entry { + ast::Entry::Message(ast::Message { ref id, .. }) + | ast::Entry::Term(ast::Term { ref id, .. }) => id.name, + _ => continue, + }; + + let entry = match entry { + ast::Entry::Message(..) => Entry::Message([res_pos, entry_pos]), + ast::Entry::Term(..) => Entry::Term([res_pos, entry_pos]), + _ => continue, + }; + + self.entries.insert(id.to_string(), entry); + } + self.resources.push(r); + } + + /// When formatting patterns, `FluentBundle` inserts + /// Unicode Directionality Isolation Marks to indicate + /// that the direction of a placeable may differ from + /// the surrounding message. + /// + /// This is important for cases such as when a + /// right-to-left user name is presented in the + /// left-to-right message. + /// + /// In some cases, such as testing, the user may want + /// to disable the isolating. + pub fn set_use_isolating(&mut self, value: bool) { + self.use_isolating = value; + } + + /// This method allows to specify a function that will + /// be called on all textual fragments of the pattern + /// during formatting. + /// + /// This is currently primarly used for pseudolocalization, + /// and `fluent-pseudo` crate provides a function + /// that can be passed here. + pub fn set_transform(&mut self, func: Option<fn(&str) -> Cow<str>>) { + if let Some(f) = func { + self.transform = Some(f); + } else { + self.transform = None; + } + } + + /// This method allows to specify a function that will + /// be called before any `FluentValue` is formatted + /// allowing overrides. + /// + /// It's particularly useful for plugging in an external + /// formatter for `FluentValue::Number`. + pub fn set_formatter(&mut self, func: Option<fn(&FluentValue, &M) -> Option<String>>) { + if let Some(f) = func { + self.formatter = Some(f); + } else { + self.formatter = None; + } + } + + /// Returns true if this bundle contains a message with the given id. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello = Hi!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// assert_eq!(true, bundle.has_message("hello")); + /// + /// ``` + pub fn has_message(&self, id: &str) -> bool + where + R: Borrow<FluentResource>, + { + self.get_entry_message(id).is_some() + } + + /// Retrieves a `FluentMessage` from a bundle. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello-world = Hello World!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// let msg = bundle.get_message("hello-world"); + /// assert_eq!(msg.is_some(), true); + /// ``` + pub fn get_message(&self, id: &str) -> Option<FluentMessage> + where + R: Borrow<FluentResource>, + { + let message = self.get_entry_message(id)?; + let value = message.value.as_ref(); + let mut attributes = Vec::with_capacity(message.attributes.len()); + + for attr in &message.attributes { + attributes.push(FluentAttribute { + id: attr.id.name, + value: &attr.value, + }); + } + Some(FluentMessage { value, attributes }) + } + + /// Writes a formatted pattern which comes from a `FluentMessage`. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello-world = Hello World!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a FluentMessage."); + /// + /// let pattern = msg.value + /// .expect("Missing Value."); + /// let mut errors = vec![]; + /// + /// let mut s = String::new(); + /// bundle.write_pattern(&mut s, &pattern, None, &mut errors) + /// .expect("Failed to write."); + /// + /// assert_eq!(s, "Hello World!"); + /// ``` + pub fn write_pattern<'bundle, W>( + &'bundle self, + w: &mut W, + pattern: &'bundle ast::Pattern<&str>, + args: Option<&'bundle FluentArgs>, + errors: &mut Vec<FluentError>, + ) -> fmt::Result + where + R: Borrow<FluentResource>, + W: fmt::Write, + { + let mut scope = Scope::new(self, args, Some(errors)); + pattern.write(w, &mut scope) + } + + /// Formats a pattern which comes from a `FluentMessage`. + /// + /// # Example + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("hello-world = Hello World!"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Failed to parse an FTL string."); + /// + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// let msg = bundle.get_message("hello-world") + /// .expect("Failed to retrieve a FluentMessage."); + /// + /// let pattern = msg.value + /// .expect("Missing Value."); + /// let mut errors = vec![]; + /// + /// let result = bundle.format_pattern(&pattern, None, &mut errors); + /// + /// assert_eq!(result, "Hello World!"); + /// ``` + pub fn format_pattern<'bundle>( + &'bundle self, + pattern: &'bundle ast::Pattern<&str>, + args: Option<&'bundle FluentArgs>, + errors: &mut Vec<FluentError>, + ) -> Cow<'bundle, str> + where + R: Borrow<FluentResource>, + { + let mut scope = Scope::new(self, args, Some(errors)); + let value = pattern.resolve(&mut scope); + value.as_string(&scope) + } + + /// Makes the provided rust function available to messages with the name `id`. See + /// the [FTL syntax guide] to learn how these are used in messages. + /// + /// FTL functions accept both positional and named args. The rust function you + /// provide therefore has two parameters: a slice of values for the positional + /// args, and a `FluentArgs` for named args. + /// + /// # Examples + /// + /// ``` + /// use fluent_bundle::{FluentBundle, FluentResource, FluentValue}; + /// use unic_langid::langid; + /// + /// let ftl_string = String::from("length = { STRLEN(\"12345\") }"); + /// let resource = FluentResource::try_new(ftl_string) + /// .expect("Could not parse an FTL string."); + /// let langid_en = langid!("en-US"); + /// let mut bundle = FluentBundle::new(vec![langid_en]); + /// bundle.add_resource(&resource) + /// .expect("Failed to add FTL resources to the bundle."); + /// + /// // Register a fn that maps from string to string length + /// bundle.add_function("STRLEN", |positional, _named| match positional { + /// [FluentValue::String(str)] => str.len().into(), + /// _ => FluentValue::Error, + /// }).expect("Failed to add a function to the bundle."); + /// + /// let msg = bundle.get_message("length").expect("Message doesn't exist."); + /// let mut errors = vec![]; + /// let pattern = msg.value.expect("Message has no value."); + /// let value = bundle.format_pattern(&pattern, None, &mut errors); + /// assert_eq!(&value, "5"); + /// ``` + /// + /// [FTL syntax guide]: https://projectfluent.org/fluent/guide/functions.html + pub fn add_function<F>(&mut self, id: &str, func: F) -> Result<(), FluentError> + where + F: for<'a> Fn(&[FluentValue<'a>], &FluentArgs) -> FluentValue<'a> + Sync + Send + 'static, + { + match self.entries.entry(id.to_owned()) { + HashEntry::Vacant(entry) => { + entry.insert(Entry::Function(Box::new(func))); + Ok(()) + } + HashEntry::Occupied(_) => Err(FluentError::Overriding { + kind: EntryKind::Function, + id: id.to_owned(), + }), + } + } +} + +impl<R, M: MemoizerKind> Default for FluentBundleBase<R, M> { + fn default() -> Self { + let langid = LanguageIdentifier::default(); + Self { + locales: vec![langid.clone()], + resources: vec![], + entries: Default::default(), + use_isolating: true, + intls: M::new(langid), + transform: None, + formatter: None, + } + } +} diff --git a/third_party/rust/fluent-bundle/src/concurrent.rs b/third_party/rust/fluent-bundle/src/concurrent.rs new file mode 100644 index 0000000000..33a19e56d1 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/concurrent.rs @@ -0,0 +1,34 @@ +use intl_memoizer::{concurrent::IntlLangMemoizer, Memoizable}; +use unic_langid::LanguageIdentifier; + +use crate::bundle::FluentBundleBase; +use crate::memoizer::MemoizerKind; +use crate::types::FluentType; + +/// Concurrent version of [`FluentBundle`] struct. See its docs for details. +/// +/// [`FluentBundle`]: ../type.FluentBundle.html +pub type FluentBundle<R> = FluentBundleBase<R, IntlLangMemoizer>; + +impl MemoizerKind for IntlLangMemoizer { + fn new(lang: LanguageIdentifier) -> Self + where + Self: Sized, + { + Self::new(lang) + } + + fn with_try_get_threadsafe<I, R, U>(&self, args: I::Args, cb: U) -> Result<R, I::Error> + where + Self: Sized, + I: Memoizable + Send + Sync + 'static, + I::Args: Send + Sync + 'static, + U: FnOnce(&I) -> R, + { + self.with_try_get(args, cb) + } + + fn stringify_value(&self, value: &dyn FluentType) -> std::borrow::Cow<'static, str> { + value.as_string_threadsafe(self) + } +} diff --git a/third_party/rust/fluent-bundle/src/entry.rs b/third_party/rust/fluent-bundle/src/entry.rs new file mode 100644 index 0000000000..8312537a62 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/entry.rs @@ -0,0 +1,62 @@ +//! `Entry` is used to store Messages, Terms and Functions in `FluentBundle` instances. + +use std::borrow::Borrow; + +use fluent_syntax::ast; + +use crate::args::FluentArgs; +use crate::bundle::FluentBundleBase; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +pub type FluentFunction = + Box<dyn for<'a> Fn(&[FluentValue<'a>], &FluentArgs) -> FluentValue<'a> + Send + Sync>; + +pub enum Entry { + Message([usize; 2]), + Term([usize; 2]), + Function(FluentFunction), +} + +pub trait GetEntry { + fn get_entry_message(&self, id: &str) -> Option<&ast::Message<&str>>; + fn get_entry_term(&self, id: &str) -> Option<&ast::Term<&str>>; + fn get_entry_function(&self, id: &str) -> Option<&FluentFunction>; +} + +impl<'bundle, R: Borrow<FluentResource>, M> GetEntry for FluentBundleBase<R, M> { + fn get_entry_message(&self, id: &str) -> Option<&ast::Message<&str>> { + self.entries.get(id).and_then(|entry| match *entry { + Entry::Message(pos) => { + let res = self.resources.get(pos[0])?.borrow(); + if let Some(ast::Entry::Message(ref msg)) = res.ast().body.get(pos[1]) { + Some(msg) + } else { + None + } + } + _ => None, + }) + } + + fn get_entry_term(&self, id: &str) -> Option<&ast::Term<&str>> { + self.entries.get(id).and_then(|entry| match *entry { + Entry::Term(pos) => { + let res = self.resources.get(pos[0])?.borrow(); + if let Some(ast::Entry::Term(ref msg)) = res.ast().body.get(pos[1]) { + Some(msg) + } else { + None + } + } + _ => None, + }) + } + + fn get_entry_function(&self, id: &str) -> Option<&FluentFunction> { + self.entries.get(id).and_then(|entry| match entry { + Entry::Function(function) => Some(function), + _ => None, + }) + } +} diff --git a/third_party/rust/fluent-bundle/src/errors.rs b/third_party/rust/fluent-bundle/src/errors.rs new file mode 100644 index 0000000000..239c0045e8 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/errors.rs @@ -0,0 +1,53 @@ +use crate::resolver::ResolverError; +use fluent_syntax::parser::ParserError; +use std::error::Error; + +#[derive(Debug, PartialEq, Clone)] +pub enum EntryKind { + Message, + Term, + Function, +} + +impl std::fmt::Display for EntryKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Message => f.write_str("message"), + Self::Term => f.write_str("term"), + Self::Function => f.write_str("function"), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum FluentError { + Overriding { kind: EntryKind, id: String }, + ParserError(ParserError), + ResolverError(ResolverError), +} + +impl std::fmt::Display for FluentError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Overriding { kind, id } => { + write!(f, "Attempt to override an existing {}: \"{}\".", kind, id) + } + Self::ParserError(err) => write!(f, "Parser error: {}", err), + Self::ResolverError(err) => write!(f, "Resolver error: {}", err), + } + } +} + +impl Error for FluentError {} + +impl From<ResolverError> for FluentError { + fn from(error: ResolverError) -> Self { + Self::ResolverError(error) + } +} + +impl From<ParserError> for FluentError { + fn from(error: ParserError) -> Self { + Self::ParserError(error) + } +} diff --git a/third_party/rust/fluent-bundle/src/lib.rs b/third_party/rust/fluent-bundle/src/lib.rs new file mode 100644 index 0000000000..de1e8708f5 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/lib.rs @@ -0,0 +1,221 @@ +//! Fluent is a modern localization system designed to improve how software is translated. +//! +//! The Rust implementation provides the low level components for syntax operations, like parser +//! and AST, and the core localization struct - [`FluentBundle`]. +//! +//! [`FluentBundle`] is the low level container for storing and formatting localization messages +//! in a single locale. +//! +//! This crate provides also a number of structures needed for a localization API such as [`FluentResource`], +//! [`FluentMessage`], [`FluentArgs`], and [`FluentValue`]. +//! +//! Together, they allow implementations to build higher-level APIs that use [`FluentBundle`] +//! and add user friendly helpers, framework bindings, error fallbacking, +//! language negotiation between user requested languages and available resources, +//! and I/O for loading selected resources. +//! +//! # Example +//! +//! ``` +//! use fluent_bundle::{FluentBundle, FluentValue, FluentResource, FluentArgs}; +//! +//! // Used to provide a locale for the bundle. +//! use unic_langid::langid; +//! +//! let ftl_string = String::from(" +//! hello-world = Hello, world! +//! intro = Welcome, { $name }. +//! "); +//! let res = FluentResource::try_new(ftl_string) +//! .expect("Failed to parse an FTL string."); +//! +//! let langid_en = langid!("en-US"); +//! let mut bundle = FluentBundle::new(vec![langid_en]); +//! +//! bundle +//! .add_resource(res) +//! .expect("Failed to add FTL resources to the bundle."); +//! +//! let msg = bundle.get_message("hello-world") +//! .expect("Message doesn't exist."); +//! let mut errors = vec![]; +//! let pattern = msg.value +//! .expect("Message has no value."); +//! let value = bundle.format_pattern(&pattern, None, &mut errors); +//! +//! assert_eq!(&value, "Hello, world!"); +//! +//! let mut args = FluentArgs::new(); +//! args.add("name", FluentValue::from("John")); +//! +//! let msg = bundle.get_message("intro") +//! .expect("Message doesn't exist."); +//! let mut errors = vec![]; +//! let pattern = msg.value.expect("Message has no value."); +//! let value = bundle.format_pattern(&pattern, Some(&args), &mut errors); +//! +//! // The FSI/PDI isolation marks ensure that the direction of +//! // the text from the variable is not affected by the translation. +//! assert_eq!(value, "Welcome, \u{2068}John\u{2069}."); +//! ``` +//! +//! # Ergonomics & Higher Level APIs +//! +//! Reading the example, you may notice how verbose it feels. +//! Many core methods are fallible, others accumulate errors, and there +//! are intermediate structures used in operations. +//! +//! This is intentional as it serves as building blocks for variety of different +//! scenarios allowing implementations to handle errors, cache and +//! optimize results. +//! +//! At the moment it is expected that users will use +//! the `fluent-bundle` crate directly, while the ecosystem +//! matures and higher level APIs are being developed. +//! +//! [`FluentBundle`]: ./type.FluentBundle.html +//! [`FluentResource`]: ./struct.FluentResource.html +//! [`FluentMessage`]: ./struct.FluentMessage.html +//! [`FluentValue`]: ./types/enum.FluentValue.html +//! [`FluentArgs`]: ./struct.FluentArgs.html + +use intl_memoizer::{IntlLangMemoizer, Memoizable}; +use unic_langid::LanguageIdentifier; + +mod args; +pub mod bundle; +pub mod concurrent; +mod entry; +mod errors; +pub mod memoizer; +mod message; +pub mod resolver; +mod resource; +pub mod types; + +pub use args::FluentArgs; +pub use errors::FluentError; +pub use message::{FluentAttribute, FluentMessage}; +pub use resource::FluentResource; +pub use types::FluentValue; + +/// A collection of localization messages for a single locale, which are meant +/// to be used together in a single view, widget or any other UI abstraction. +/// +/// # Examples +/// +/// ``` +/// use fluent_bundle::{FluentBundle, FluentResource, FluentValue, FluentArgs}; +/// use unic_langid::langid; +/// +/// let ftl_string = String::from("intro = Welcome, { $name }."); +/// let resource = FluentResource::try_new(ftl_string) +/// .expect("Could not parse an FTL string."); +/// +/// let langid_en = langid!("en-US"); +/// let mut bundle = FluentBundle::new(vec![langid_en]); +/// +/// bundle.add_resource(&resource) +/// .expect("Failed to add FTL resources to the bundle."); +/// +/// let mut args = FluentArgs::new(); +/// args.add("name", FluentValue::from("Rustacean")); +/// +/// let msg = bundle.get_message("intro").expect("Message doesn't exist."); +/// let mut errors = vec![]; +/// let pattern = msg.value.expect("Message has no value."); +/// let value = bundle.format_pattern(&pattern, Some(&args), &mut errors); +/// assert_eq!(&value, "Welcome, \u{2068}Rustacean\u{2069}."); +/// +/// ``` +/// +/// # `FluentBundle` Life Cycle +/// +/// ## Create a bundle +/// +/// To create a bundle, call [`FluentBundle::new`] with a locale list that represents the best +/// possible fallback chain for a given locale. The simplest case is a one-locale list. +/// +/// Fluent uses [`LanguageIdentifier`] which can be created using `langid!` macro. +/// +/// ## Add Resources +/// +/// Next, call [`add_resource`] one or more times, supplying translations in the FTL syntax. +/// +/// Since [`FluentBundle`] is generic over anything that can borrow a [`FluentResource`], +/// one can use [`FluentBundle`] to own its resources, store references to them, +/// or even [`Rc<FluentResource>`] or [`Arc<FluentResource>`]. +/// +/// The [`FluentBundle`] instance is now ready to be used for localization. +/// +/// ## Format +/// +/// To format a translation, call [`get_message`] to retrieve a [`FluentMessage`], +/// and then call [`format_pattern`] on the message value or attribute in order to +/// retrieve the translated string. +/// +/// The result of [`format_pattern`] is an [`Cow<str>`]. It is +/// recommended to treat the result as opaque from the perspective of the program and use it only +/// to display localized messages. Do not examine it or alter in any way before displaying. This +/// is a general good practice as far as all internationalization operations are concerned. +/// +/// If errors were encountered during formatting, they will be +/// accumulated in the [`Vec<FluentError>`] passed as the third argument. +/// +/// While they are not fatal, they usually indicate problems with the translation, +/// and should be logged or reported in a way that allows the developer to notice +/// and fix them. +/// +/// +/// # Locale Fallback Chain +/// +/// [`FluentBundle`] stores messages in a single locale, but keeps a locale fallback chain for the +/// purpose of language negotiation with i18n formatters. For instance, if date and time formatting +/// are not available in the first locale, [`FluentBundle`] will use its `locales` fallback chain +/// to negotiate a sensible fallback for date and time formatting. +/// +/// # Concurrency +/// +/// As you may have noticed, `FluentBundle` is a specialization of [`FluentBundleBase`] +/// which works with an [`IntlMemoizer`][] over `RefCell`. +/// In scenarios where the memoizer must work concurrently, there's an implementation of +/// `IntlMemoizer` that uses `Mutex` and there's [`concurrent::FluentBundle`] which works with that. +/// +/// [`add_resource`]: ./bundle/struct.FluentBundleBase.html#method.add_resource +/// [`FluentBundle::new`]: ./bundle/struct.FluentBundleBase.html#method.new +/// [`FluentMessage`]: ./struct.FluentMessage.html +/// [`FluentBundle`]: ./type.FluentBundle.html +/// [`FluentResource`]: ./struct.FluentResource.html +/// [`get_message`]: ./bundle/struct.FluentBundleBase.html#method.get_message +/// [`format_pattern`]: ./bundle/struct.FluentBundleBase.html#method.format_pattern +/// [`Cow<str>`]: http://doc.rust-lang.org/std/borrow/enum.Cow.html +/// [`Rc<FluentResource>`]: https://doc.rust-lang.org/std/rc/struct.Rc.html +/// [`Arc<FluentResource>`]: https://doc.rust-lang.org/std/sync/struct.Arc.html +/// [`LanguageIdentifier`]: https://crates.io/crates/unic-langid +/// [`IntlMemoizer`]: https://github.com/projectfluent/fluent-rs/tree/master/intl-memoizer +/// [`Vec<FluentError>`]: ./enum.FluentError.html +/// [`concurrent::FluentBundle`]: ./concurrent/type.FluentBundle.html +pub type FluentBundle<R> = bundle::FluentBundleBase<R, IntlLangMemoizer>; + +impl memoizer::MemoizerKind for IntlLangMemoizer { + fn new(lang: LanguageIdentifier) -> Self + where + Self: Sized, + { + Self::new(lang) + } + + fn with_try_get_threadsafe<I, R, U>(&self, args: I::Args, cb: U) -> Result<R, I::Error> + where + Self: Sized, + I: Memoizable + Send + Sync + 'static, + I::Args: Send + Sync + 'static, + U: FnOnce(&I) -> R, + { + self.with_try_get(args, cb) + } + + fn stringify_value(&self, value: &dyn types::FluentType) -> std::borrow::Cow<'static, str> { + value.as_string(self) + } +} diff --git a/third_party/rust/fluent-bundle/src/memoizer.rs b/third_party/rust/fluent-bundle/src/memoizer.rs new file mode 100644 index 0000000000..c738a857b2 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/memoizer.rs @@ -0,0 +1,18 @@ +use crate::types::FluentType; +use intl_memoizer::Memoizable; +use unic_langid::LanguageIdentifier; + +pub trait MemoizerKind: 'static { + fn new(lang: LanguageIdentifier) -> Self + where + Self: Sized; + + fn with_try_get_threadsafe<I, R, U>(&self, args: I::Args, cb: U) -> Result<R, I::Error> + where + Self: Sized, + I: Memoizable + Send + Sync + 'static, + I::Args: Send + Sync + 'static, + U: FnOnce(&I) -> R; + + fn stringify_value(&self, value: &dyn FluentType) -> std::borrow::Cow<'static, str>; +} diff --git a/third_party/rust/fluent-bundle/src/message.rs b/third_party/rust/fluent-bundle/src/message.rs new file mode 100644 index 0000000000..4b0a84769e --- /dev/null +++ b/third_party/rust/fluent-bundle/src/message.rs @@ -0,0 +1,20 @@ +use fluent_syntax::ast; + +#[derive(Debug, PartialEq)] +pub struct FluentAttribute<'m> { + pub id: &'m str, + pub value: &'m ast::Pattern<&'m str>, +} +/// A single localization unit composed of an identifier, +/// value, and attributes. +#[derive(Debug, PartialEq)] +pub struct FluentMessage<'m> { + pub value: Option<&'m ast::Pattern<&'m str>>, + pub attributes: Vec<FluentAttribute<'m>>, +} + +impl<'m> FluentMessage<'m> { + pub fn get_attribute(&self, key: &str) -> Option<&FluentAttribute> { + self.attributes.iter().find(|attr| attr.id == key) + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/errors.rs b/third_party/rust/fluent-bundle/src/resolver/errors.rs new file mode 100644 index 0000000000..831d8474a5 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/errors.rs @@ -0,0 +1,96 @@ +use fluent_syntax::ast::InlineExpression; +use std::error::Error; + +#[derive(Debug, PartialEq, Clone)] +pub enum ReferenceKind { + Function { + id: String, + }, + Message { + id: String, + attribute: Option<String>, + }, + Term { + id: String, + attribute: Option<String>, + }, + Variable { + id: String, + }, +} + +impl<T> From<&InlineExpression<T>> for ReferenceKind +where + T: ToString, +{ + fn from(exp: &InlineExpression<T>) -> Self { + match exp { + InlineExpression::FunctionReference { id, .. } => Self::Function { + id: id.name.to_string(), + }, + InlineExpression::MessageReference { id, attribute } => Self::Message { + id: id.name.to_string(), + attribute: attribute.as_ref().map(|i| i.name.to_string()), + }, + InlineExpression::TermReference { id, attribute, .. } => Self::Term { + id: id.name.to_string(), + attribute: attribute.as_ref().map(|i| i.name.to_string()), + }, + InlineExpression::VariableReference { id, .. } => Self::Variable { + id: id.name.to_string(), + }, + _ => unreachable!(), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum ResolverError { + Reference(ReferenceKind), + NoValue(String), + MissingDefault, + Cyclic, + TooManyPlaceables, +} + +impl std::fmt::Display for ResolverError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Reference(exp) => match exp { + ReferenceKind::Function { id } => write!(f, "Unknown function: {}()", id), + ReferenceKind::Message { + id, + attribute: None, + } => write!(f, "Unknown message: {}", id), + ReferenceKind::Message { + id, + attribute: Some(attribute), + } => write!(f, "Unknown attribute: {}.{}", id, attribute), + ReferenceKind::Term { + id, + attribute: None, + } => write!(f, "Unknown term: -{}", id), + ReferenceKind::Term { + id, + attribute: Some(attribute), + } => write!(f, "Unknown attribute: -{}.{}", id, attribute), + ReferenceKind::Variable { id } => write!(f, "Unknown variable: ${}", id), + }, + Self::NoValue(id) => write!(f, "No value: {}", id), + Self::MissingDefault => f.write_str("No default"), + Self::Cyclic => f.write_str("Cyclical dependency detected"), + Self::TooManyPlaceables => f.write_str("Too many placeables"), + } + } +} + +impl<T> From<&InlineExpression<T>> for ResolverError +where + T: ToString, +{ + fn from(exp: &InlineExpression<T>) -> Self { + Self::Reference(exp.into()) + } +} + +impl Error for ResolverError {} diff --git a/third_party/rust/fluent-bundle/src/resolver/expression.rs b/third_party/rust/fluent-bundle/src/resolver/expression.rs new file mode 100644 index 0000000000..84de7248d3 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/expression.rs @@ -0,0 +1,65 @@ +use super::scope::Scope; +use super::WriteValue; + +use std::borrow::Borrow; +use std::fmt; + +use fluent_syntax::ast; + +use crate::memoizer::MemoizerKind; +use crate::resolver::{ResolveValue, ResolverError}; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +impl<'p> WriteValue for ast::Expression<&'p str> { + fn write<'scope, 'errors, W, R, M: MemoizerKind>( + &'scope self, + w: &mut W, + scope: &mut Scope<'scope, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + { + match self { + Self::InlineExpression(exp) => exp.write(w, scope), + Self::SelectExpression { selector, variants } => { + let selector = selector.resolve(scope); + match selector { + FluentValue::String(_) | FluentValue::Number(_) => { + for variant in variants { + let key = match variant.key { + ast::VariantKey::Identifier { name } => name.into(), + ast::VariantKey::NumberLiteral { value } => { + FluentValue::try_number(value) + } + }; + if key.matches(&selector, scope) { + return variant.value.write(w, scope); + } + } + } + _ => {} + } + + for variant in variants { + if variant.default { + return variant.value.write(w, scope); + } + } + scope.add_error(ResolverError::MissingDefault); + Ok(()) + } + } + } + + fn write_error<W>(&self, w: &mut W) -> fmt::Result + where + W: fmt::Write, + { + match self { + Self::InlineExpression(exp) => exp.write_error(w), + Self::SelectExpression { .. } => unreachable!(), + } + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/inline_expression.rs b/third_party/rust/fluent-bundle/src/resolver/inline_expression.rs new file mode 100644 index 0000000000..83cd622cc8 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/inline_expression.rs @@ -0,0 +1,179 @@ +use super::scope::Scope; +use super::{ResolveValue, ResolverError, WriteValue}; + +use std::borrow::Borrow; +use std::fmt; + +use fluent_syntax::ast; +use fluent_syntax::unicode::{unescape_unicode, unescape_unicode_to_string}; + +use crate::entry::GetEntry; +use crate::memoizer::MemoizerKind; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +impl<'p> WriteValue for ast::InlineExpression<&'p str> { + fn write<'scope, 'errors, W, R, M: MemoizerKind>( + &'scope self, + w: &mut W, + scope: &mut Scope<'scope, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + { + match self { + Self::StringLiteral { value } => unescape_unicode(w, value), + Self::MessageReference { id, attribute } => { + if let Some(msg) = scope.bundle.get_entry_message(id.name) { + if let Some(attr) = attribute { + msg.attributes + .iter() + .find_map(|a| { + if a.id.name == attr.name { + Some(scope.track(w, &a.value, self)) + } else { + None + } + }) + .unwrap_or_else(|| scope.write_ref_error(w, self)) + } else { + msg.value + .as_ref() + .map(|value| scope.track(w, value, self)) + .unwrap_or_else(|| { + scope.add_error(ResolverError::NoValue(id.name.to_string())); + w.write_char('{')?; + self.write_error(w)?; + w.write_char('}') + }) + } + } else { + scope.write_ref_error(w, self) + } + } + Self::NumberLiteral { value } => FluentValue::try_number(*value).write(w, scope), + Self::TermReference { + id, + attribute, + arguments, + } => { + let (_, resolved_named_args) = scope.get_arguments(arguments); + + scope.local_args = Some(resolved_named_args); + let result = scope + .bundle + .get_entry_term(id.name) + .and_then(|term| { + if let Some(attr) = attribute { + term.attributes.iter().find_map(|a| { + if a.id.name == attr.name { + Some(scope.track(w, &a.value, self)) + } else { + None + } + }) + } else { + Some(scope.track(w, &term.value, self)) + } + }) + .unwrap_or_else(|| scope.write_ref_error(w, self)); + scope.local_args = None; + result + } + Self::FunctionReference { id, arguments } => { + let (resolved_positional_args, resolved_named_args) = + scope.get_arguments(arguments); + + let func = scope.bundle.get_entry_function(id.name); + + if let Some(func) = func { + let result = func(resolved_positional_args.as_slice(), &resolved_named_args); + if let FluentValue::Error = result { + self.write_error(w) + } else { + w.write_str(&result.as_string(scope)) + } + } else { + scope.write_ref_error(w, self) + } + } + Self::VariableReference { id } => { + let args = scope.local_args.as_ref().or(scope.args); + + if let Some(arg) = args.and_then(|args| args.get(id.name)) { + arg.write(w, scope) + } else { + if scope.local_args.is_none() { + scope.add_error(self.into()); + } + w.write_char('{')?; + self.write_error(w)?; + w.write_char('}') + } + } + Self::Placeable { expression } => expression.write(w, scope), + } + } + + fn write_error<W>(&self, w: &mut W) -> fmt::Result + where + W: fmt::Write, + { + match self { + Self::MessageReference { + id, + attribute: Some(attribute), + } => write!(w, "{}.{}", id.name, attribute.name), + Self::MessageReference { + id, + attribute: None, + } => w.write_str(id.name), + Self::TermReference { + id, + attribute: Some(attribute), + .. + } => write!(w, "-{}.{}", id.name, attribute.name), + Self::TermReference { + id, + attribute: None, + .. + } => write!(w, "-{}", id.name), + Self::FunctionReference { id, .. } => write!(w, "{}()", id.name), + Self::VariableReference { id } => write!(w, "${}", id.name), + _ => unreachable!(), + } + } +} + +impl<'p> ResolveValue for ast::InlineExpression<&'p str> { + fn resolve<'source, 'errors, R, M: MemoizerKind>( + &'source self, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> FluentValue<'source> + where + R: Borrow<FluentResource>, + { + match self { + Self::StringLiteral { value } => unescape_unicode_to_string(value).into(), + Self::NumberLiteral { value } => FluentValue::try_number(*value), + Self::VariableReference { id } => { + let args = scope.local_args.as_ref().or(scope.args); + + if let Some(arg) = args.and_then(|args| args.get(id.name)) { + arg.clone() + } else { + if scope.local_args.is_none() { + scope.add_error(self.into()); + } + FluentValue::Error + } + } + _ => { + let mut result = String::new(); + self.write(&mut result, scope).expect("Failed to write"); + result.into() + } + } + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/mod.rs b/third_party/rust/fluent-bundle/src/resolver/mod.rs new file mode 100644 index 0000000000..4413737bc1 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/mod.rs @@ -0,0 +1,48 @@ +//! The `ResolveValue` trait resolves Fluent AST nodes to [`FluentValues`]. +//! +//! This is an internal API used by [`FluentBundle`] to evaluate Messages, Attributes and other +//! AST nodes to [`FluentValues`] which can be then formatted to strings. +//! +//! [`FluentValues`]: ../types/enum.FluentValue.html +//! [`FluentBundle`]: ../bundle/struct.FluentBundle.html + +mod errors; +mod expression; +mod inline_expression; +mod pattern; +mod scope; + +pub use errors::ResolverError; +pub use scope::Scope; + +use std::borrow::Borrow; +use std::fmt; + +use crate::memoizer::MemoizerKind; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +// Converts an AST node to a `FluentValue`. +pub(crate) trait ResolveValue { + fn resolve<'source, 'errors, R, M: MemoizerKind>( + &'source self, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> FluentValue<'source> + where + R: Borrow<FluentResource>; +} + +pub(crate) trait WriteValue { + fn write<'source, 'errors, W, R, M: MemoizerKind>( + &'source self, + w: &mut W, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>; + + fn write_error<W>(&self, _w: &mut W) -> fmt::Result + where + W: fmt::Write; +} diff --git a/third_party/rust/fluent-bundle/src/resolver/pattern.rs b/third_party/rust/fluent-bundle/src/resolver/pattern.rs new file mode 100644 index 0000000000..33d8e118a5 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/pattern.rs @@ -0,0 +1,105 @@ +use super::scope::Scope; +use super::{ResolverError, WriteValue}; + +use std::borrow::Borrow; +use std::fmt; + +use fluent_syntax::ast; + +use crate::memoizer::MemoizerKind; +use crate::resolver::ResolveValue; +use crate::resource::FluentResource; +use crate::types::FluentValue; + +const MAX_PLACEABLES: u8 = 100; + +impl<'p> WriteValue for ast::Pattern<&'p str> { + fn write<'scope, 'errors, W, R, M: MemoizerKind>( + &'scope self, + w: &mut W, + scope: &mut Scope<'scope, 'errors, R, M>, + ) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + { + let len = self.elements.len(); + + for elem in &self.elements { + if scope.dirty { + return Ok(()); + } + + match elem { + ast::PatternElement::TextElement { value } => { + if let Some(ref transform) = scope.bundle.transform { + w.write_str(&transform(value))?; + } else { + w.write_str(value)?; + } + } + ast::PatternElement::Placeable { ref expression } => { + scope.placeables += 1; + if scope.placeables > MAX_PLACEABLES { + scope.dirty = true; + scope.add_error(ResolverError::TooManyPlaceables); + return Ok(()); + } + + let needs_isolation = scope.bundle.use_isolating + && len > 1 + && !matches!(expression, ast::Expression::InlineExpression( + ast::InlineExpression::MessageReference { .. }, + ) + | ast::Expression::InlineExpression( + ast::InlineExpression::TermReference { .. }, + ) + | ast::Expression::InlineExpression( + ast::InlineExpression::StringLiteral { .. }, + )); + if needs_isolation { + w.write_char('\u{2068}')?; + } + scope.maybe_track(w, self, expression)?; + if needs_isolation { + w.write_char('\u{2069}')?; + } + } + } + } + Ok(()) + } + + fn write_error<W>(&self, _w: &mut W) -> fmt::Result + where + W: fmt::Write, + { + unreachable!() + } +} + +impl<'p> ResolveValue for ast::Pattern<&'p str> { + fn resolve<'source, 'errors, R, M: MemoizerKind>( + &'source self, + scope: &mut Scope<'source, 'errors, R, M>, + ) -> FluentValue<'source> + where + R: Borrow<FluentResource>, + { + let len = self.elements.len(); + + if len == 1 { + if let ast::PatternElement::TextElement { value } = self.elements[0] { + return scope + .bundle + .transform + .map_or_else(|| value.into(), |transform| transform(value).into()); + } + } + + let mut result = String::new(); + self.write(&mut result, scope) + .expect("Failed to write to a string."); + result.into() + } +} diff --git a/third_party/rust/fluent-bundle/src/resolver/scope.rs b/third_party/rust/fluent-bundle/src/resolver/scope.rs new file mode 100644 index 0000000000..6cbfb19cf2 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resolver/scope.rs @@ -0,0 +1,140 @@ +use crate::bundle::FluentBundleBase; +use crate::memoizer::MemoizerKind; +use crate::resolver::{ResolveValue, ResolverError, WriteValue}; +use crate::types::FluentValue; +use crate::{FluentArgs, FluentError, FluentResource}; +use fluent_syntax::ast; +use std::borrow::Borrow; +use std::fmt; + +/// State for a single `ResolveValue::to_value` call. +pub struct Scope<'scope, 'errors, R, M> { + /// The current `FluentBundleBase` instance. + pub bundle: &'scope FluentBundleBase<R, M>, + /// The current arguments passed by the developer. + pub(super) args: Option<&'scope FluentArgs<'scope>>, + /// Local args + pub(super) local_args: Option<FluentArgs<'scope>>, + /// The running count of resolved placeables. Used to detect the Billion + /// Laughs and Quadratic Blowup attacks. + pub(super) placeables: u8, + /// Tracks hashes to prevent infinite recursion. + travelled: smallvec::SmallVec<[&'scope ast::Pattern<&'scope str>; 2]>, + /// Track errors accumulated during resolving. + pub errors: Option<&'errors mut Vec<FluentError>>, + /// Makes the resolver bail. + pub dirty: bool, +} + +impl<'scope, 'errors, R, M: MemoizerKind> Scope<'scope, 'errors, R, M> { + pub fn new( + bundle: &'scope FluentBundleBase<R, M>, + args: Option<&'scope FluentArgs>, + errors: Option<&'errors mut Vec<FluentError>>, + ) -> Self { + Scope { + bundle, + args, + local_args: None, + placeables: 0, + travelled: Default::default(), + errors, + dirty: false, + } + } + + pub fn add_error(&mut self, error: ResolverError) { + if let Some(errors) = self.errors.as_mut() { + errors.push(error.into()); + } + } + + // This method allows us to lazily add Pattern on the stack, + // only if the Pattern::resolve has been called on an empty stack. + // + // This is the case when pattern is called from Bundle and it + // allows us to fast-path simple resolutions, and only use the stack + // for placeables. + pub fn maybe_track<W>( + &mut self, + w: &mut W, + pattern: &'scope ast::Pattern<&str>, + exp: &'scope ast::Expression<&str>, + ) -> fmt::Result + where + R: Borrow<FluentResource>, + W: fmt::Write, + { + if self.travelled.is_empty() { + self.travelled.push(pattern); + } + exp.write(w, self)?; + if self.dirty { + w.write_char('{')?; + exp.write_error(w)?; + w.write_char('}') + } else { + Ok(()) + } + } + + pub fn track<W>( + &mut self, + w: &mut W, + pattern: &'scope ast::Pattern<&str>, + exp: &ast::InlineExpression<&str>, + ) -> fmt::Result + where + R: Borrow<FluentResource>, + W: fmt::Write, + { + if self.travelled.contains(&pattern) { + self.add_error(ResolverError::Cyclic); + w.write_char('{')?; + exp.write_error(w)?; + w.write_char('}') + } else { + self.travelled.push(pattern); + let result = pattern.write(w, self); + self.travelled.pop(); + result + } + } + + pub fn write_ref_error<W>( + &mut self, + w: &mut W, + exp: &ast::InlineExpression<&str>, + ) -> fmt::Result + where + W: fmt::Write, + { + self.add_error(exp.into()); + w.write_char('{')?; + exp.write_error(w)?; + w.write_char('}') + } + + pub fn get_arguments( + &mut self, + arguments: &'scope Option<ast::CallArguments<&'scope str>>, + ) -> (Vec<FluentValue<'scope>>, FluentArgs<'scope>) + where + R: Borrow<FluentResource>, + { + let mut resolved_positional_args = Vec::new(); + let mut resolved_named_args = FluentArgs::new(); + + if let Some(ast::CallArguments { named, positional }) = arguments { + for expression in positional { + resolved_positional_args.push(expression.resolve(self)); + } + + for arg in named { + resolved_named_args.add(arg.name.name, arg.value.resolve(self)); + } + } + + (resolved_positional_args, resolved_named_args) + } +} diff --git a/third_party/rust/fluent-bundle/src/resource.rs b/third_party/rust/fluent-bundle/src/resource.rs new file mode 100644 index 0000000000..ef5dada797 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/resource.rs @@ -0,0 +1,44 @@ +use fluent_syntax::ast; +use fluent_syntax::parser::Parser; +use fluent_syntax::parser::ParserError; +use ouroboros::self_referencing; + +#[self_referencing] +#[derive(Debug)] +pub struct InnerFluentResource { + string: String, + #[borrows(string)] + ast: ast::Resource<&'this str>, +} + +/// A resource containing a list of localization messages. +#[derive(Debug)] +pub struct FluentResource(InnerFluentResource); + +impl FluentResource { + pub fn try_new(source: String) -> Result<Self, (Self, Vec<ParserError>)> { + let mut errors = None; + + let res = InnerFluentResourceBuilder { + string: source, + ast_builder: |string: &str| match Parser::new(string).parse() { + Ok(ast) => ast, + Err((ast, err)) => { + errors = Some(err); + ast + } + }, + } + .build(); + + if let Some(errors) = errors { + Err((Self(res), errors)) + } else { + Ok(Self(res)) + } + } + + pub fn ast(&self) -> &ast::Resource<&str> { + self.0.borrow_ast() + } +} diff --git a/third_party/rust/fluent-bundle/src/types/mod.rs b/third_party/rust/fluent-bundle/src/types/mod.rs new file mode 100644 index 0000000000..d906591ee3 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/types/mod.rs @@ -0,0 +1,186 @@ +mod number; +mod plural; + +pub use number::*; +use plural::PluralRules; + +use std::any::Any; +use std::borrow::{Borrow, Cow}; +use std::fmt; +use std::str::FromStr; + +use intl_pluralrules::{PluralCategory, PluralRuleType}; + +use crate::memoizer::MemoizerKind; +use crate::resolver::Scope; +use crate::resource::FluentResource; + +pub trait FluentType: fmt::Debug + AnyEq + 'static { + fn duplicate(&self) -> Box<dyn FluentType + Send>; + fn as_string(&self, intls: &intl_memoizer::IntlLangMemoizer) -> Cow<'static, str>; + fn as_string_threadsafe( + &self, + intls: &intl_memoizer::concurrent::IntlLangMemoizer, + ) -> Cow<'static, str>; +} + +impl PartialEq for dyn FluentType + Send { + fn eq(&self, other: &Self) -> bool { + self.equals(other.as_any()) + } +} + +pub trait AnyEq: Any + 'static { + fn equals(&self, other: &dyn Any) -> bool; + fn as_any(&self) -> &dyn Any; +} + +impl<T: Any + PartialEq> AnyEq for T { + fn equals(&self, other: &dyn Any) -> bool { + other + .downcast_ref::<Self>() + .map_or(false, |that| self == that) + } + fn as_any(&self) -> &dyn Any { + self + } +} + +/// The `FluentValue` enum represents values which can be formatted to a String. +/// +/// Those values are either passed as arguments to [`FluentBundle::format_pattern`][] or +/// produced by functions, or generated in the process of pattern resolution. +/// +/// [`FluentBundle::format_pattern`]: ../bundle/struct.FluentBundleBase.html#method.format_pattern +#[derive(Debug)] +pub enum FluentValue<'source> { + String(Cow<'source, str>), + Number(FluentNumber), + Custom(Box<dyn FluentType + Send>), + None, + Error, +} + +impl<'s> PartialEq for FluentValue<'s> { + fn eq(&self, other: &Self) -> bool { + match (self, other) { + (FluentValue::String(s), FluentValue::String(s2)) => s == s2, + (FluentValue::Number(s), FluentValue::Number(s2)) => s == s2, + (FluentValue::Custom(s), FluentValue::Custom(s2)) => s == s2, + _ => false, + } + } +} + +impl<'s> Clone for FluentValue<'s> { + fn clone(&self) -> Self { + match self { + FluentValue::String(s) => FluentValue::String(s.clone()), + FluentValue::Number(s) => FluentValue::Number(s.clone()), + FluentValue::Custom(s) => { + let new_value: Box<dyn FluentType + Send> = s.duplicate(); + FluentValue::Custom(new_value) + } + FluentValue::Error => FluentValue::Error, + FluentValue::None => FluentValue::None, + } + } +} + +impl<'source> FluentValue<'source> { + pub fn try_number<S: ToString>(v: S) -> Self { + let s = v.to_string(); + if let Ok(num) = FluentNumber::from_str(&s) { + num.into() + } else { + s.into() + } + } + + pub fn matches<R: Borrow<FluentResource>, M: MemoizerKind>( + &self, + other: &FluentValue, + scope: &Scope<R, M>, + ) -> bool { + match (self, other) { + (&FluentValue::String(ref a), &FluentValue::String(ref b)) => a == b, + (&FluentValue::Number(ref a), &FluentValue::Number(ref b)) => a == b, + (&FluentValue::String(ref a), &FluentValue::Number(ref b)) => { + let cat = match a.as_ref() { + "zero" => PluralCategory::ZERO, + "one" => PluralCategory::ONE, + "two" => PluralCategory::TWO, + "few" => PluralCategory::FEW, + "many" => PluralCategory::MANY, + "other" => PluralCategory::OTHER, + _ => return false, + }; + scope + .bundle + .intls + .with_try_get_threadsafe::<PluralRules, _, _>( + (PluralRuleType::CARDINAL,), + |pr| pr.0.select(b) == Ok(cat), + ) + .unwrap() + } + _ => false, + } + } + + pub fn write<W, R, M>(&self, w: &mut W, scope: &Scope<R, M>) -> fmt::Result + where + W: fmt::Write, + R: Borrow<FluentResource>, + M: MemoizerKind, + { + if let Some(formatter) = &scope.bundle.formatter { + if let Some(val) = formatter(self, &scope.bundle.intls) { + return w.write_str(&val); + } + } + match self { + FluentValue::String(s) => w.write_str(s), + FluentValue::Number(n) => w.write_str(&n.as_string()), + FluentValue::Custom(s) => w.write_str(&scope.bundle.intls.stringify_value(&**s)), + FluentValue::Error => Ok(()), + FluentValue::None => Ok(()), + } + } + + pub fn as_string<R: Borrow<FluentResource>, M: MemoizerKind>( + &self, + scope: &Scope<R, M>, + ) -> Cow<'source, str> { + if let Some(formatter) = &scope.bundle.formatter { + if let Some(val) = formatter(self, &scope.bundle.intls) { + return val.into(); + } + } + match self { + FluentValue::String(s) => s.clone(), + FluentValue::Number(n) => n.as_string(), + FluentValue::Custom(s) => scope.bundle.intls.stringify_value(&**s), + FluentValue::Error => "".into(), + FluentValue::None => "".into(), + } + } +} + +impl<'source> From<String> for FluentValue<'source> { + fn from(s: String) -> Self { + FluentValue::String(s.into()) + } +} + +impl<'source> From<&'source str> for FluentValue<'source> { + fn from(s: &'source str) -> Self { + FluentValue::String(s.into()) + } +} + +impl<'source> From<Cow<'source, str>> for FluentValue<'source> { + fn from(s: Cow<'source, str>) -> Self { + FluentValue::String(s) + } +} diff --git a/third_party/rust/fluent-bundle/src/types/number.rs b/third_party/rust/fluent-bundle/src/types/number.rs new file mode 100644 index 0000000000..e2f813941b --- /dev/null +++ b/third_party/rust/fluent-bundle/src/types/number.rs @@ -0,0 +1,249 @@ +use std::borrow::Cow; +use std::convert::TryInto; +use std::default::Default; +use std::str::FromStr; + +use intl_pluralrules::operands::PluralOperands; + +use crate::args::FluentArgs; +use crate::types::FluentValue; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum FluentNumberStyle { + Decimal, + Currency, + Percent, +} + +impl std::default::Default for FluentNumberStyle { + fn default() -> Self { + Self::Decimal + } +} + +impl From<&str> for FluentNumberStyle { + fn from(input: &str) -> Self { + match input { + "decimal" => Self::Decimal, + "currency" => Self::Currency, + "percent" => Self::Percent, + _ => Self::default(), + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum FluentNumberCurrencyDisplayStyle { + Symbol, + Code, + Name, +} + +impl std::default::Default for FluentNumberCurrencyDisplayStyle { + fn default() -> Self { + Self::Symbol + } +} + +impl From<&str> for FluentNumberCurrencyDisplayStyle { + fn from(input: &str) -> Self { + match input { + "symbol" => Self::Symbol, + "code" => Self::Code, + "name" => Self::Name, + _ => Self::default(), + } + } +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct FluentNumberOptions { + pub style: FluentNumberStyle, + pub currency: Option<String>, + pub currency_display: FluentNumberCurrencyDisplayStyle, + pub use_grouping: bool, + pub minimum_integer_digits: Option<usize>, + pub minimum_fraction_digits: Option<usize>, + pub maximum_fraction_digits: Option<usize>, + pub minimum_significant_digits: Option<usize>, + pub maximum_significant_digits: Option<usize>, +} + +impl Default for FluentNumberOptions { + fn default() -> Self { + Self { + style: Default::default(), + currency: None, + currency_display: Default::default(), + use_grouping: true, + minimum_integer_digits: None, + minimum_fraction_digits: None, + maximum_fraction_digits: None, + minimum_significant_digits: None, + maximum_significant_digits: None, + } + } +} + +impl FluentNumberOptions { + pub fn merge(&mut self, opts: &FluentArgs) { + for (key, value) in opts.iter() { + match (key, value) { + ("style", FluentValue::String(n)) => { + self.style = n.as_ref().into(); + } + ("currency", FluentValue::String(n)) => { + self.currency = Some(n.to_string()); + } + ("currencyDisplay", FluentValue::String(n)) => { + self.currency_display = n.as_ref().into(); + } + ("minimumIntegerDigits", FluentValue::Number(n)) => { + self.minimum_integer_digits = Some(n.into()); + } + ("minimumFractionDigits", FluentValue::Number(n)) => { + self.minimum_fraction_digits = Some(n.into()); + } + ("maximumFractionDigits", FluentValue::Number(n)) => { + self.maximum_fraction_digits = Some(n.into()); + } + ("minimumSignificantDigits", FluentValue::Number(n)) => { + self.minimum_significant_digits = Some(n.into()); + } + ("maximumSignificantDigits", FluentValue::Number(n)) => { + self.maximum_significant_digits = Some(n.into()); + } + _ => {} + } + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct FluentNumber { + pub value: f64, + pub options: FluentNumberOptions, +} + +impl FluentNumber { + pub const fn new(value: f64, options: FluentNumberOptions) -> Self { + Self { value, options } + } + + pub fn as_string(&self) -> Cow<'static, str> { + let mut val = self.value.to_string(); + if let Some(minfd) = self.options.minimum_fraction_digits { + if let Some(pos) = val.find('.') { + let frac_num = val.len() - pos - 1; + let missing = if frac_num > minfd { + 0 + } else { + minfd - frac_num + }; + val = format!("{}{}", val, "0".repeat(missing)); + } else { + val = format!("{}.{}", val, "0".repeat(minfd)); + } + } + val.into() + } +} + +impl FromStr for FluentNumber { + type Err = std::num::ParseFloatError; + + fn from_str(input: &str) -> Result<Self, Self::Err> { + f64::from_str(input).map(|n| { + let mfd = input.find('.').map(|pos| input.len() - pos - 1); + let opts = FluentNumberOptions { + minimum_fraction_digits: mfd, + ..Default::default() + }; + Self::new(n, opts) + }) + } +} + +impl<'l> From<FluentNumber> for FluentValue<'l> { + fn from(input: FluentNumber) -> Self { + FluentValue::Number(input) + } +} + +macro_rules! from_num { + ($num:ty) => { + impl From<$num> for FluentNumber { + fn from(n: $num) -> Self { + Self { + value: n as f64, + options: FluentNumberOptions::default(), + } + } + } + impl From<&$num> for FluentNumber { + fn from(n: &$num) -> Self { + Self { + value: *n as f64, + options: FluentNumberOptions::default(), + } + } + } + impl From<FluentNumber> for $num { + fn from(input: FluentNumber) -> Self { + input.value as $num + } + } + impl From<&FluentNumber> for $num { + fn from(input: &FluentNumber) -> Self { + input.value as $num + } + } + impl From<$num> for FluentValue<'_> { + fn from(n: $num) -> Self { + FluentValue::Number(n.into()) + } + } + impl From<&$num> for FluentValue<'_> { + fn from(n: &$num) -> Self { + FluentValue::Number(n.into()) + } + } + }; + ($($num:ty)+) => { + $(from_num!($num);)+ + }; +} + +impl From<&FluentNumber> for PluralOperands { + fn from(input: &FluentNumber) -> Self { + let mut operands: Self = input + .value + .try_into() + .expect("Failed to generate operands out of FluentNumber"); + if let Some(mfd) = input.options.minimum_fraction_digits { + if mfd > operands.v { + operands.f *= 10_u64.pow(mfd as u32 - operands.v as u32); + operands.v = mfd; + } + } + // XXX: Add support for other options. + operands + } +} + +from_num!(i8 i16 i32 i64 i128 isize); +from_num!(u8 u16 u32 u64 u128 usize); +from_num!(f32 f64); + +#[cfg(test)] +mod tests { + use crate::types::FluentValue; + + #[test] + fn value_from_copy_ref() { + let x = 1i16; + let y = &x; + let z: FluentValue = y.into(); + assert_eq!(z, FluentValue::try_number(1)); + } +} diff --git a/third_party/rust/fluent-bundle/src/types/plural.rs b/third_party/rust/fluent-bundle/src/types/plural.rs new file mode 100644 index 0000000000..1151fd6d36 --- /dev/null +++ b/third_party/rust/fluent-bundle/src/types/plural.rs @@ -0,0 +1,22 @@ +use fluent_langneg::{negotiate_languages, NegotiationStrategy}; +use intl_memoizer::Memoizable; +use intl_pluralrules::{PluralRuleType, PluralRules as IntlPluralRules}; +use unic_langid::LanguageIdentifier; + +pub struct PluralRules(pub IntlPluralRules); + +impl Memoizable for PluralRules { + type Args = (PluralRuleType,); + type Error = &'static str; + fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> { + let default_lang: LanguageIdentifier = "en".parse().unwrap(); + let pr_lang = negotiate_languages( + &[lang], + &IntlPluralRules::get_locales(args.0), + Some(&default_lang), + NegotiationStrategy::Lookup, + )[0] + .clone(); + Ok(Self(IntlPluralRules::create(pr_lang, args.0)?)) + } +} |