diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-07 19:33:14 +0000 |
commit | 36d22d82aa202bb199967e9512281e9a53db42c9 (patch) | |
tree | 105e8c98ddea1c1e4784a60a5a6410fa416be2de /third_party/rust/fluent-fallback/src | |
parent | Initial commit. (diff) | |
download | firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.tar.xz firefox-esr-36d22d82aa202bb199967e9512281e9a53db42c9.zip |
Adding upstream version 115.7.0esr.upstream/115.7.0esrupstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/fluent-fallback/src')
-rw-r--r-- | third_party/rust/fluent-fallback/src/bundles.rs | 426 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/cache.rs | 253 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/env.rs | 84 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/errors.rs | 71 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/generator.rs | 41 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/lib.rs | 118 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/localization.rs | 137 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/pin_cell/README.md | 2 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/pin_cell/mod.rs | 97 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs | 50 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs | 47 | ||||
-rw-r--r-- | third_party/rust/fluent-fallback/src/types.rs | 141 |
12 files changed, 1467 insertions, 0 deletions
diff --git a/third_party/rust/fluent-fallback/src/bundles.rs b/third_party/rust/fluent-fallback/src/bundles.rs new file mode 100644 index 0000000000..7ab726d684 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/bundles.rs @@ -0,0 +1,426 @@ +use crate::{ + cache::{AsyncCache, Cache}, + env::LocalesProvider, + errors::LocalizationError, + generator::{BundleGenerator, BundleIterator, BundleStream}, + types::{L10nAttribute, L10nKey, L10nMessage, ResourceId}, +}; +use fluent_bundle::{FluentArgs, FluentBundle, FluentError}; +use rustc_hash::FxHashSet; +use std::borrow::Cow; + +pub enum BundlesInner<G> +where + G: BundleGenerator, +{ + Iter(Cache<G::Iter, G::Resource>), + Stream(AsyncCache<G::Stream, G::Resource>), +} + +pub struct Bundles<G>(BundlesInner<G>) +where + G: BundleGenerator; + +impl<G> Bundles<G> +where + G: BundleGenerator, + G::Iter: BundleIterator, +{ + pub fn prefetch_sync(&self) { + match &self.0 { + BundlesInner::Iter(iter) => iter.prefetch(), + BundlesInner::Stream(_) => panic!("Can't prefetch a sync bundle set asynchronously"), + } + } +} + +impl<G> Bundles<G> +where + G: BundleGenerator, + G::Stream: BundleStream, +{ + pub async fn prefetch_async(&self) { + match &self.0 { + BundlesInner::Iter(_) => panic!("Can't prefetch a async bundle set synchronously"), + BundlesInner::Stream(stream) => stream.prefetch().await, + } + } +} + +impl<G> Bundles<G> +where + G: BundleGenerator, +{ + pub fn new<P>(sync: bool, res_ids: FxHashSet<ResourceId>, generator: &G, provider: &P) -> Self + where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, + { + Self(if sync { + BundlesInner::Iter(Cache::new( + generator.bundles_iter(provider.locales(), res_ids), + )) + } else { + BundlesInner::Stream(AsyncCache::new( + generator.bundles_stream(provider.locales(), res_ids), + )) + }) + } + + pub async fn format_value<'l>( + &'l self, + id: &'l str, + args: Option<&'l FluentArgs<'_>>, + errors: &mut Vec<LocalizationError>, + ) -> Option<Cow<'l, str>> { + match &self.0 { + BundlesInner::Iter(cache) => Self::format_value_from_iter(cache, id, args, errors), + BundlesInner::Stream(stream) => { + Self::format_value_from_stream(stream, id, args, errors).await + } + } + } + + pub async fn format_values<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<Cow<'l, str>>> { + match &self.0 { + BundlesInner::Iter(cache) => Self::format_values_from_iter(cache, keys, errors), + BundlesInner::Stream(stream) => { + Self::format_values_from_stream(stream, keys, errors).await + } + } + } + + pub async fn format_messages<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<L10nMessage<'l>>> { + match &self.0 { + BundlesInner::Iter(cache) => Self::format_messages_from_iter(cache, keys, errors), + BundlesInner::Stream(stream) => { + Self::format_messages_from_stream(stream, keys, errors).await + } + } + } + + pub fn format_value_sync<'l>( + &'l self, + id: &'l str, + args: Option<&'l FluentArgs>, + errors: &mut Vec<LocalizationError>, + ) -> Result<Option<Cow<'l, str>>, LocalizationError> { + match &self.0 { + BundlesInner::Iter(cache) => Ok(Self::format_value_from_iter(cache, id, args, errors)), + BundlesInner::Stream(_) => Err(LocalizationError::SyncRequestInAsyncMode), + } + } + + pub fn format_values_sync<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Result<Vec<Option<Cow<'l, str>>>, LocalizationError> { + match &self.0 { + BundlesInner::Iter(cache) => Ok(Self::format_values_from_iter(cache, keys, errors)), + BundlesInner::Stream(_) => Err(LocalizationError::SyncRequestInAsyncMode), + } + } + + pub fn format_messages_sync<'l>( + &'l self, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Result<Vec<Option<L10nMessage<'l>>>, LocalizationError> { + match &self.0 { + BundlesInner::Iter(cache) => Ok(Self::format_messages_from_iter(cache, keys, errors)), + BundlesInner::Stream(_) => Err(LocalizationError::SyncRequestInAsyncMode), + } + } +} + +macro_rules! format_value_from_inner { + ($step:expr, $id:expr, $args:expr, $errors:expr) => { + let mut found_message = false; + + while let Some(bundle) = $step { + let bundle = bundle.as_ref().unwrap_or_else(|(bundle, err)| { + $errors.extend(err.iter().cloned().map(Into::into)); + bundle + }); + + if let Some(msg) = bundle.get_message($id) { + found_message = true; + if let Some(value) = msg.value() { + let mut format_errors = vec![]; + let result = bundle.format_pattern(value, $args, &mut format_errors); + if !format_errors.is_empty() { + $errors.push(LocalizationError::Resolver { + id: $id.to_string(), + locale: bundle.locales[0].clone(), + errors: format_errors, + }); + } + return Some(result); + } else { + $errors.push(LocalizationError::MissingValue { + id: $id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } else { + $errors.push(LocalizationError::MissingMessage { + id: $id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } + if found_message { + $errors.push(LocalizationError::MissingValue { + id: $id.to_string(), + locale: None, + }); + } else { + $errors.push(LocalizationError::MissingMessage { + id: $id.to_string(), + locale: None, + }); + } + return None; + }; +} + +#[derive(Clone)] +enum Value<'l> { + Present(Cow<'l, str>), + Missing, + None, +} + +macro_rules! format_values_from_inner { + ($step:expr, $keys:expr, $errors:expr) => { + let mut cells = vec![Value::None; $keys.len()]; + + while let Some(bundle) = $step { + let bundle = bundle.as_ref().unwrap_or_else(|(bundle, err)| { + $errors.extend(err.iter().cloned().map(Into::into)); + bundle + }); + + let mut has_missing = false; + + for (key, cell) in $keys + .iter() + .zip(&mut cells) + .filter(|(_, cell)| !matches!(cell, Value::Present(_))) + { + if let Some(msg) = bundle.get_message(&key.id) { + if let Some(value) = msg.value() { + let mut format_errors = vec![]; + *cell = Value::Present(bundle.format_pattern( + value, + key.args.as_ref(), + &mut format_errors, + )); + if !format_errors.is_empty() { + $errors.push(LocalizationError::Resolver { + id: key.id.to_string(), + locale: bundle.locales[0].clone(), + errors: format_errors, + }); + } + } else { + *cell = Value::Missing; + has_missing = true; + $errors.push(LocalizationError::MissingValue { + id: key.id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } else { + has_missing = true; + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } + } + if !has_missing { + break; + } + } + + return $keys + .iter() + .zip(cells) + .map(|(key, value)| match value { + Value::Present(value) => Some(value), + Value::Missing => { + $errors.push(LocalizationError::MissingValue { + id: key.id.to_string(), + locale: None, + }); + None + } + Value::None => { + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: None, + }); + None + } + }) + .collect(); + }; +} + +macro_rules! format_messages_from_inner { + ($step:expr, $keys:expr, $errors:expr) => { + let mut result = vec![None; $keys.len()]; + + let mut is_complete = false; + + while let Some(bundle) = $step { + let bundle = bundle.as_ref().unwrap_or_else(|(bundle, err)| { + $errors.extend(err.iter().cloned().map(Into::into)); + bundle + }); + + let mut has_missing = false; + for (key, cell) in $keys + .iter() + .zip(&mut result) + .filter(|(_, cell)| cell.is_none()) + { + let mut format_errors = vec![]; + let msg = Self::format_message_from_bundle(bundle, key, &mut format_errors); + + if msg.is_none() { + has_missing = true; + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: Some(bundle.locales[0].clone()), + }); + } else if !format_errors.is_empty() { + $errors.push(LocalizationError::Resolver { + id: key.id.to_string(), + locale: bundle.locales.get(0).cloned().unwrap(), + errors: format_errors, + }); + } + + *cell = msg; + } + if !has_missing { + is_complete = true; + break; + } + } + + if !is_complete { + for (key, _) in $keys + .iter() + .zip(&mut result) + .filter(|(_, cell)| cell.is_none()) + { + $errors.push(LocalizationError::MissingMessage { + id: key.id.to_string(), + locale: None, + }); + } + } + + return result; + }; +} + +impl<G> Bundles<G> +where + G: BundleGenerator, +{ + fn format_value_from_iter<'l>( + cache: &'l Cache<G::Iter, G::Resource>, + id: &'l str, + args: Option<&'l FluentArgs>, + errors: &mut Vec<LocalizationError>, + ) -> Option<Cow<'l, str>> { + let mut bundle_iter = cache.into_iter(); + format_value_from_inner!(bundle_iter.next(), id, args, errors); + } + + async fn format_value_from_stream<'l>( + stream: &'l AsyncCache<G::Stream, G::Resource>, + id: &'l str, + args: Option<&'l FluentArgs<'_>>, + errors: &mut Vec<LocalizationError>, + ) -> Option<Cow<'l, str>> { + use futures::StreamExt; + + let mut bundle_stream = stream.stream(); + format_value_from_inner!(bundle_stream.next().await, id, args, errors); + } + + async fn format_messages_from_stream<'l>( + stream: &'l AsyncCache<G::Stream, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<L10nMessage<'l>>> { + use futures::StreamExt; + let mut bundle_stream = stream.stream(); + format_messages_from_inner!(bundle_stream.next().await, keys, errors); + } + + async fn format_values_from_stream<'l>( + stream: &'l AsyncCache<G::Stream, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<Cow<'l, str>>> { + use futures::StreamExt; + let mut bundle_stream = stream.stream(); + + format_values_from_inner!(bundle_stream.next().await, keys, errors); + } + + fn format_message_from_bundle<'l>( + bundle: &'l FluentBundle<G::Resource>, + key: &'l L10nKey, + format_errors: &mut Vec<FluentError>, + ) -> Option<L10nMessage<'l>> { + let msg = bundle.get_message(&key.id)?; + let value = msg + .value() + .map(|pattern| bundle.format_pattern(pattern, key.args.as_ref(), format_errors)); + let attributes = msg + .attributes() + .map(|attr| { + let value = bundle.format_pattern(attr.value(), key.args.as_ref(), format_errors); + L10nAttribute { + name: attr.id().into(), + value, + } + }) + .collect(); + Some(L10nMessage { value, attributes }) + } + + fn format_messages_from_iter<'l>( + cache: &'l Cache<G::Iter, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<L10nMessage<'l>>> { + let mut bundle_iter = cache.into_iter(); + format_messages_from_inner!(bundle_iter.next(), keys, errors); + } + + fn format_values_from_iter<'l>( + cache: &'l Cache<G::Iter, G::Resource>, + keys: &'l [L10nKey<'l>], + errors: &mut Vec<LocalizationError>, + ) -> Vec<Option<Cow<'l, str>>> { + let mut bundle_iter = cache.into_iter(); + format_values_from_inner!(bundle_iter.next(), keys, errors); + } +} diff --git a/third_party/rust/fluent-fallback/src/cache.rs b/third_party/rust/fluent-fallback/src/cache.rs new file mode 100644 index 0000000000..32bc33fad1 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/cache.rs @@ -0,0 +1,253 @@ +use std::{ + cell::{RefCell, UnsafeCell}, + cmp::Ordering, + pin::Pin, + task::Context, + task::Poll, + task::Waker, +}; + +use crate::generator::{BundleIterator, BundleStream}; +use crate::pin_cell::{PinCell, PinMut}; +use chunky_vec::ChunkyVec; +use futures::{ready, Stream}; + +pub struct Cache<I, R> +where + I: Iterator, +{ + iter: RefCell<I>, + items: UnsafeCell<ChunkyVec<I::Item>>, + res: std::marker::PhantomData<R>, +} + +impl<I, R> Cache<I, R> +where + I: Iterator, +{ + pub fn new(iter: I) -> Self { + Self { + iter: RefCell::new(iter), + items: Default::default(), + res: std::marker::PhantomData, + } + } + + pub fn len(&self) -> usize { + unsafe { + let items = self.items.get(); + (*items).len() + } + } + + pub fn get(&self, index: usize) -> Option<&I::Item> { + unsafe { + let items = self.items.get(); + (*items).get(index) + } + } + + /// Push, immediately getting a reference to the element + pub fn push_get(&self, new_value: I::Item) -> &I::Item { + unsafe { + let items = self.items.get(); + (*items).push_get(new_value) + } + } +} + +impl<I, R> Cache<I, R> +where + I: BundleIterator + Iterator, +{ + pub fn prefetch(&self) { + self.iter.borrow_mut().prefetch_sync(); + } +} + +pub struct CacheIter<'a, I, R> +where + I: Iterator, +{ + cache: &'a Cache<I, R>, + curr: usize, +} + +impl<'a, I, R> Iterator for CacheIter<'a, I, R> +where + I: Iterator, +{ + type Item = &'a I::Item; + + fn next(&mut self) -> Option<Self::Item> { + let cache_len = self.cache.len(); + match self.curr.cmp(&cache_len) { + Ordering::Less => { + // Cached value + self.curr += 1; + self.cache.get(self.curr - 1) + } + Ordering::Equal => { + // Get the next item from the iterator + let item = self.cache.iter.borrow_mut().next(); + self.curr += 1; + if let Some(item) = item { + Some(self.cache.push_get(item)) + } else { + None + } + } + Ordering::Greater => { + // Ran off the end of the cache + None + } + } + } +} + +impl<'a, I, R> IntoIterator for &'a Cache<I, R> +where + I: Iterator, +{ + type Item = &'a I::Item; + type IntoIter = CacheIter<'a, I, R>; + + fn into_iter(self) -> Self::IntoIter { + CacheIter { + cache: self, + curr: 0, + } + } +} + +//////////////////////////////////////////////////////////////////////////////// + +pub struct AsyncCache<S, R> +where + S: Stream, +{ + stream: PinCell<S>, + items: UnsafeCell<ChunkyVec<S::Item>>, + // TODO: Should probably be an SmallVec<[Waker; 1]> or something? I guess + // multiple pending wakes are not really all that common. + pending_wakes: RefCell<Vec<Waker>>, + res: std::marker::PhantomData<R>, +} + +impl<S, R> AsyncCache<S, R> +where + S: Stream, +{ + pub fn new(stream: S) -> Self { + Self { + stream: PinCell::new(stream), + items: Default::default(), + pending_wakes: Default::default(), + res: std::marker::PhantomData, + } + } + + pub fn len(&self) -> usize { + unsafe { + let items = self.items.get(); + (*items).len() + } + } + + pub fn get(&self, index: usize) -> Poll<Option<&S::Item>> { + unsafe { + let items = self.items.get(); + (*items).get(index).into() + } + } + + /// Push, immediately getting a reference to the element + pub fn push_get(&self, new_value: S::Item) -> &S::Item { + unsafe { + let items = self.items.get(); + (*items).push_get(new_value) + } + } + + pub fn stream(&self) -> AsyncCacheStream<'_, S, R> { + AsyncCacheStream { + cache: self, + curr: 0, + } + } +} + +impl<S, R> AsyncCache<S, R> +where + S: BundleStream + Stream, +{ + pub async fn prefetch(&self) { + let pin = unsafe { Pin::new_unchecked(&self.stream) }; + unsafe { PinMut::as_mut(&mut pin.borrow_mut()).get_unchecked_mut() } + .prefetch_async() + .await + } +} + +impl<S, R> AsyncCache<S, R> +where + S: Stream, +{ + // Helper function that gets the next value from wrapped stream. + fn poll_next_item(&self, cx: &mut Context<'_>) -> Poll<Option<S::Item>> { + let pin = unsafe { Pin::new_unchecked(&self.stream) }; + let poll = PinMut::as_mut(&mut pin.borrow_mut()).poll_next(cx); + if poll.is_ready() { + let wakers = std::mem::take(&mut *self.pending_wakes.borrow_mut()); + for waker in wakers { + waker.wake(); + } + } else { + self.pending_wakes.borrow_mut().push(cx.waker().clone()); + } + poll + } +} + +pub struct AsyncCacheStream<'a, S, R> +where + S: Stream, +{ + cache: &'a AsyncCache<S, R>, + curr: usize, +} + +impl<'a, S, R> Stream for AsyncCacheStream<'a, S, R> +where + S: Stream, +{ + type Item = &'a S::Item; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> Poll<Option<Self::Item>> { + let cache_len = self.cache.len(); + match self.curr.cmp(&cache_len) { + Ordering::Less => { + // Cached value + self.curr += 1; + self.cache.get(self.curr - 1) + } + Ordering::Equal => { + // Get the next item from the stream + let item = ready!(self.cache.poll_next_item(cx)); + self.curr += 1; + if let Some(item) = item { + Some(self.cache.push_get(item)).into() + } else { + None.into() + } + } + Ordering::Greater => { + // Ran off the end of the cache + None.into() + } + } + } +} diff --git a/third_party/rust/fluent-fallback/src/env.rs b/third_party/rust/fluent-fallback/src/env.rs new file mode 100644 index 0000000000..cf340fcfdf --- /dev/null +++ b/third_party/rust/fluent-fallback/src/env.rs @@ -0,0 +1,84 @@ +//! Traits required to provide environment driven data for [`Localization`](crate::Localization). +//! +//! Since [`Localization`](crate::Localization) is a long-lived structure, +//! the model in which the user provides ability for the system to react to changes +//! is by implementing the given environmental trait and triggering +//! [`Localization::on_change`](crate::Localization::on_change) method. +//! +//! At the moment just a single trait is provided, which allows the +//! environment to feed a selection of locales to be provided to the instance. +//! +//! The locales provided to [`Localization`](crate::Localization) should be +//! already negotiated to ensure that the resources in those locales +//! are available. The list should also be sorted according to the user +//! preference, as the order is significant for how [`Localization`](crate::Localization) performs +//! fallbacking. +use unic_langid::LanguageIdentifier; + +/// A trait used to provide a selection of locales to be used by the +/// [`Localization`](crate::Localization) instance for runtime +/// locale fallbacking. +/// +/// # Example +/// ``` +/// use fluent_fallback::{Localization, env::LocalesProvider}; +/// use fluent_resmgr::ResourceManager; +/// use unic_langid::LanguageIdentifier; +/// use std::{ +/// rc::Rc, +/// cell::RefCell +/// }; +/// +/// #[derive(Clone)] +/// struct Env { +/// locales: Rc<RefCell<Vec<LanguageIdentifier>>>, +/// } +/// +/// impl Env { +/// pub fn new(locales: Vec<LanguageIdentifier>) -> Self { +/// Self { locales: Rc::new(RefCell::new(locales)) } +/// } +/// +/// pub fn set_locales(&mut self, new_locales: Vec<LanguageIdentifier>) { +/// let mut locales = self.locales.borrow_mut(); +/// locales.clear(); +/// locales.extend(new_locales); +/// } +/// } +/// +/// impl LocalesProvider for Env { +/// type Iter = <Vec<LanguageIdentifier> as IntoIterator>::IntoIter; +/// fn locales(&self) -> Self::Iter { +/// self.locales.borrow().clone().into_iter() +/// } +/// } +/// +/// let res_mgr = ResourceManager::new("./path/{locale}/".to_string()); +/// +/// let mut env = Env::new(vec![ +/// "en-GB".parse().unwrap() +/// ]); +/// +/// let mut loc = Localization::with_env(vec![], true, env.clone(), res_mgr); +/// +/// env.set_locales(vec![ +/// "de".parse().unwrap(), +/// "en-GB".parse().unwrap(), +/// ]); +/// +/// loc.on_change(); +/// +/// // The next format call will attempt to localize to `de` first and +/// // fallback on `en-GB`. +/// ``` +pub trait LocalesProvider { + type Iter: Iterator<Item = LanguageIdentifier>; + fn locales(&self) -> Self::Iter; +} + +impl LocalesProvider for Vec<LanguageIdentifier> { + type Iter = <Vec<LanguageIdentifier> as IntoIterator>::IntoIter; + fn locales(&self) -> Self::Iter { + self.clone().into_iter() + } +} diff --git a/third_party/rust/fluent-fallback/src/errors.rs b/third_party/rust/fluent-fallback/src/errors.rs new file mode 100644 index 0000000000..9170fb1cb9 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/errors.rs @@ -0,0 +1,71 @@ +use fluent_bundle::FluentError; +use std::error::Error; +use unic_langid::LanguageIdentifier; + +#[derive(Debug, PartialEq)] +pub enum LocalizationError { + Bundle { + error: FluentError, + }, + Resolver { + id: String, + locale: LanguageIdentifier, + errors: Vec<FluentError>, + }, + MissingMessage { + id: String, + locale: Option<LanguageIdentifier>, + }, + MissingValue { + id: String, + locale: Option<LanguageIdentifier>, + }, + SyncRequestInAsyncMode, +} + +impl From<FluentError> for LocalizationError { + fn from(error: FluentError) -> Self { + Self::Bundle { error } + } +} + +impl std::fmt::Display for LocalizationError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bundle { error } => write!(f, "[fluent][bundle] error: {}", error), + Self::Resolver { id, locale, errors } => { + let errors: Vec<String> = errors.iter().map(|err| err.to_string()).collect(); + write!( + f, + "[fluent][resolver] errors in {}/{}: {}", + locale, + id, + errors.join(", ") + ) + } + Self::MissingMessage { + id, + locale: Some(locale), + } => write!(f, "[fluent] Missing message in locale {}: {}", locale, id), + Self::MissingMessage { id, locale: None } => { + write!(f, "[fluent] Couldn't find a message: {}", id) + } + Self::MissingValue { + id, + locale: Some(locale), + } => write!( + f, + "[fluent] Message has no value in locale {}: {}", + locale, id + ), + Self::MissingValue { id, locale: None } => { + write!(f, "[fluent] Couldn't find a message with value: {}", id) + } + Self::SyncRequestInAsyncMode => { + write!(f, "Triggered synchronous format while in async mode") + } + } + } +} + +impl Error for LocalizationError {} diff --git a/third_party/rust/fluent-fallback/src/generator.rs b/third_party/rust/fluent-fallback/src/generator.rs new file mode 100644 index 0000000000..f13af63cfd --- /dev/null +++ b/third_party/rust/fluent-fallback/src/generator.rs @@ -0,0 +1,41 @@ +use fluent_bundle::{FluentBundle, FluentError, FluentResource}; +use futures::Stream; +use rustc_hash::FxHashSet; +use std::borrow::Borrow; +use unic_langid::LanguageIdentifier; + +use crate::types::ResourceId; + +pub type FluentBundleResult<R> = Result<FluentBundle<R>, (FluentBundle<R>, Vec<FluentError>)>; + +pub trait BundleIterator { + fn prefetch_sync(&mut self) {} +} + +#[async_trait::async_trait(?Send)] +pub trait BundleStream { + async fn prefetch_async(&mut self) {} +} + +pub trait BundleGenerator { + type Resource: Borrow<FluentResource>; + type LocalesIter: Iterator<Item = LanguageIdentifier>; + type Iter: Iterator<Item = FluentBundleResult<Self::Resource>>; + type Stream: Stream<Item = FluentBundleResult<Self::Resource>>; + + fn bundles_iter( + &self, + _locales: Self::LocalesIter, + _res_ids: FxHashSet<ResourceId>, + ) -> Self::Iter { + unimplemented!(); + } + + fn bundles_stream( + &self, + _locales: Self::LocalesIter, + _res_ids: FxHashSet<ResourceId>, + ) -> Self::Stream { + unimplemented!(); + } +} diff --git a/third_party/rust/fluent-fallback/src/lib.rs b/third_party/rust/fluent-fallback/src/lib.rs new file mode 100644 index 0000000000..9dbadc5b98 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/lib.rs @@ -0,0 +1,118 @@ +//! Fluent is a modern localization system designed to improve how software is translated. +//! +//! `fluent-fallback` is a high-level component of the [Fluent Localization +//! System](https://www.projectfluent.org). +//! +//! The crate builds on top of the mid-level [`fluent-bundle`](../fluent-bundle) package, and provides an ergonomic API for highly flexible localization. +//! +//! The functionality of this level is complete, but the API itself is in the +//! early stages and the goal of being ergonomic is yet to be achieved. +//! +//! If the user is willing to work through the challenge of setting up the +//! boiler-plate that will eventually go away, `fluent-fallback` provides +//! a powerful abstraction around [`FluentBundle`](fluent_bundle::FluentBundle) coupled +//! with a localization resource management system. +//! +//! The main struct, [`Localization`], is a long-lived, reactive, multi-lingual +//! struct which allows for strong error recovery and locale +//! fallbacking, exposing synchronous and asynchronous ergonomic methods +//! for [`L10nMessage`](types::L10nMessage) retrieval. +//! +//! [`Localization`] is also an API that is to be used when designing bindings +//! to user interface systems, such as DOM, React, and others. +//! +//! # Example +//! +//! ``` +//! use fluent_fallback::{Localization, types::{ResourceType, ToResourceId}}; +//! use fluent_resmgr::ResourceManager; +//! use unic_langid::langid; +//! +//! let res_mgr = ResourceManager::new("./tests/resources/{locale}/".to_string()); +//! +//! let loc = Localization::with_env( +//! vec![ +//! "test.ftl".into(), +//! "test2.ftl".to_resource_id(ResourceType::Optional), +//! ], +//! true, +//! vec![langid!("en-US")], +//! res_mgr, +//! ); +//! let bundles = loc.bundles(); +//! +//! let mut errors = vec![]; +//! let value = bundles.format_value_sync("hello-world", None, &mut errors) +//! .expect("Failed to format a value"); +//! +//! assert_eq!(value, Some("Hello World [en]".into())); +//! ``` +//! +//! The above example is far from the ergonomical API style the Fluent project +//! is aiming for, but it represents the full scope of functionality intended +//! for the model. +//! +//! # Resource Management +//! +//! Resource management is one of the most complicated parts of a localization system. +//! In particular, modern software may have needs for both synchronous +//! and asynchronous I/O. That, in turn has a large impact on what can happen +//! in case of missing resources, or errors. +//! +//! Resource identifiers can refer to resources that are either required or optional. +//! In the above example, `"test.ftl"` is a required resource (the default using `.into()`), +//! and `"test2.ftl" is an optional resource, which you can create via the +//! [`ToResourceId`](fluent_fallback::types::ToResourceId) trait. +//! +//! A required resource must be present in order for the a bundle to be considered valid. +//! If a required resource is missing for a given locale, a bundle will not be generated for that locale. +//! +//! A bundle is still considered valid if an optional resource is missing. A bundle will still be generated +//! and the entries for the missing optional resource will simply be missing from the bundle. This should be +//! used sparingly in exceptional cases where you do not want `Localization` to fall back to the next +//! locale if there is a missing resource. Marking all resources as optional will increase the state space +//! that the solver has to search through, and will have a negative impact on performance. +//! +//! Currently, [`Localization`] can be specialized over an implementation of +//! [`generator::BundleGenerator`] trait which provides a method to generate an +//! [`Iterator`] and [`Stream`](futures::stream::Stream). +//! +//! This is not very elegant and will likely be improved in the future, but for the time being, if +//! the customer doesn't need one of the modes, the unnecessary method should use the +//! `unimplemented!()` macro as its body. +//! +//! `fluent-resmgr` provides a simple resource manager which handles synchronous I/O +//! and uses local file system to store resources in a directory structure. +//! +//! That model is often sufficient and the user can either use `fluent-resmgr` or write +//! a similar API to provide the generator for [`Localization`]. +//! +//! Alternatively, a much more sophisticated resource manager can be used. Mozilla +//! for its needs in Firefox uses [`L10nRegistry`](https://github.com/zbraniecki/l10nregistry-rs) +//! library which implements [`BundleGenerator`](generator::BundleGenerator). +//! +//! # Locale Management +//! +//! As a long lived structure, the [`Localization`] is intended to handle runtime locale +//! management. +//! +//! In the example above, [`Vec<LagnuageIdentifier>`](unic_langid::LanguageIdentifier) +//! provides a static list of locales that the [`Localization`] handles, but that's just the +//! simplest implementation of the [`env::LocalesProvider`], and one can implement +//! a much more sophisticated one that reacts to user or environment driven changes, and +//! called [`Localization::on_change`] to trigger a new locales to be used for the +//! next translation request. +//! +//! See [`env::LocalesProvider`] trait for an example of a reactive system implementation. +mod bundles; +mod cache; +pub mod env; +mod errors; +pub mod generator; +mod localization; +mod pin_cell; +pub mod types; + +pub use bundles::Bundles; +pub use errors::LocalizationError; +pub use localization::Localization; diff --git a/third_party/rust/fluent-fallback/src/localization.rs b/third_party/rust/fluent-fallback/src/localization.rs new file mode 100644 index 0000000000..5424bcf311 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/localization.rs @@ -0,0 +1,137 @@ +use crate::{ + bundles::Bundles, + env::LocalesProvider, + generator::{BundleGenerator, BundleIterator, BundleStream}, + types::ResourceId, +}; +use once_cell::sync::OnceCell; +use rustc_hash::FxHashSet; +use std::rc::Rc; + +pub struct Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, +{ + bundles: OnceCell<Rc<Bundles<G>>>, + generator: G, + provider: P, + sync: bool, + res_ids: FxHashSet<ResourceId>, +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter> + Default, + P: LocalesProvider + Default, +{ + pub fn new<I>(res_ids: I, sync: bool) -> Self + where + I: IntoIterator<Item = ResourceId>, + { + Self { + bundles: OnceCell::new(), + generator: G::default(), + provider: P::default(), + sync, + res_ids: FxHashSet::from_iter(res_ids.into_iter()), + } + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, +{ + pub fn with_env<I>(res_ids: I, sync: bool, provider: P, generator: G) -> Self + where + I: IntoIterator<Item = ResourceId>, + { + Self { + bundles: OnceCell::new(), + generator, + provider, + sync, + res_ids: FxHashSet::from_iter(res_ids.into_iter()), + } + } + + pub fn is_sync(&self) -> bool { + self.sync + } + + pub fn add_resource_id<T: Into<ResourceId>>(&mut self, res_id: T) { + self.res_ids.insert(res_id.into()); + self.on_change(); + } + + pub fn add_resource_ids(&mut self, res_ids: Vec<ResourceId>) { + self.res_ids.extend(res_ids); + self.on_change(); + } + + pub fn remove_resource_id<T: PartialEq<ResourceId>>(&mut self, res_id: T) -> usize { + self.res_ids.retain(|x| !res_id.eq(x)); + self.on_change(); + self.res_ids.len() + } + + pub fn remove_resource_ids(&mut self, res_ids: Vec<ResourceId>) -> usize { + self.res_ids.retain(|x| !res_ids.contains(x)); + self.on_change(); + self.res_ids.len() + } + + pub fn set_async(&mut self) { + if self.sync { + self.sync = false; + self.on_change(); + } + } + + pub fn on_change(&mut self) { + self.bundles.take(); + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + G::Iter: BundleIterator, + P: LocalesProvider, +{ + pub fn prefetch_sync(&mut self) { + let bundles = self.bundles(); + bundles.prefetch_sync(); + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + G::Stream: BundleStream, + P: LocalesProvider, +{ + pub async fn prefetch_async(&mut self) { + let bundles = self.bundles(); + bundles.prefetch_async().await + } +} + +impl<G, P> Localization<G, P> +where + G: BundleGenerator<LocalesIter = P::Iter>, + P: LocalesProvider, +{ + pub fn bundles(&self) -> &Rc<Bundles<G>> { + self.bundles.get_or_init(|| { + Rc::new(Bundles::new( + self.sync, + self.res_ids.clone(), + &self.generator, + &self.provider, + )) + }) + } +} diff --git a/third_party/rust/fluent-fallback/src/pin_cell/README.md b/third_party/rust/fluent-fallback/src/pin_cell/README.md new file mode 100644 index 0000000000..b1c475f51b --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/README.md @@ -0,0 +1,2 @@ +This is a temporary fork of https://github.com/withoutboats/pin-cell until +https://github.com/withoutboats/pin-cell/issues/6 gets resolved. diff --git a/third_party/rust/fluent-fallback/src/pin_cell/mod.rs b/third_party/rust/fluent-fallback/src/pin_cell/mod.rs new file mode 100644 index 0000000000..175f9677e0 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/mod.rs @@ -0,0 +1,97 @@ +#![deny(missing_docs, missing_debug_implementations)] +//! This library defines the `PinCell` type, a pinning variant of the standard +//! library's `RefCell`. +//! +//! It is not safe to "pin project" through a `RefCell` - getting a pinned +//! reference to something inside the `RefCell` when you have a pinned +//! refernece to the `RefCell` - because `RefCell` is too powerful. +//! +//! A `PinCell` is slightly less powerful than `RefCell`: unlike a `RefCell`, +//! one cannot get a mutable reference into a `PinCell`, only a pinned mutable +//! reference (`Pin<&mut T>`). This makes pin projection safe, allowing you +//! to use interior mutability with the knowledge that `T` will never actually +//! be moved out of the `RefCell` that wraps it. + +mod pin_mut; +mod pin_ref; + +use core::cell::{BorrowMutError, RefCell, RefMut}; +use core::pin::Pin; + +pub use pin_mut::PinMut; +pub use pin_ref::PinRef; + +/// A mutable memory location with dynamically checked borrow rules +/// +/// Unlike `RefCell`, this type only allows *pinned* mutable access to the +/// inner value, enabling a "pin-safe" version of interior mutability. +/// +/// See the standard library documentation for more information. +#[derive(Default, Clone, Ord, PartialOrd, Eq, PartialEq, Debug)] +pub struct PinCell<T: ?Sized> { + inner: RefCell<T>, +} + +impl<T> PinCell<T> { + /// Creates a new `PinCell` containing `value`. + pub const fn new(value: T) -> PinCell<T> { + PinCell { + inner: RefCell::new(value), + } + } +} + +impl<T: ?Sized> PinCell<T> { + /// Mutably borrows the wrapped value, preserving its pinnedness. + /// + /// The borrow lasts until the returned `PinMut` or all `PinMut`s derived + /// from it exit scope. The value cannot be borrowed while this borrow is + /// active. + pub fn borrow_mut(self: Pin<&Self>) -> PinMut<'_, T> { + self.try_borrow_mut().expect("already borrowed") + } + + /// Mutably borrows the wrapped value, preserving its pinnedness, + /// returning an error if the value is currently borrowed. + /// + /// The borrow lasts until the returned `PinMut` or all `PinMut`s derived + /// from it exit scope. The value cannot be borrowed while this borrow is + /// active. + /// + /// This is the non-panicking variant of `borrow_mut`. + pub fn try_borrow_mut<'a>(self: Pin<&'a Self>) -> Result<PinMut<'a, T>, BorrowMutError> { + let ref_mut: RefMut<'a, T> = Pin::get_ref(self).inner.try_borrow_mut()?; + + // this is a pin projection from Pin<&PinCell<T>> to Pin<RefMut<T>> + // projecting is safe because: + // + // - for<T: ?Sized> (PinCell<T>: Unpin) imples (RefMut<T>: Unpin) + // holds true + // - PinCell does not implement Drop + // + // see discussion on tracking issue #49150 about pin projection + // invariants + let pin_ref_mut: Pin<RefMut<'a, T>> = unsafe { Pin::new_unchecked(ref_mut) }; + + Ok(PinMut { inner: pin_ref_mut }) + } +} + +impl<T> From<T> for PinCell<T> { + fn from(value: T) -> PinCell<T> { + PinCell::new(value) + } +} + +impl<T> From<RefCell<T>> for PinCell<T> { + fn from(cell: RefCell<T>) -> PinCell<T> { + PinCell { inner: cell } + } +} + +impl<T> From<PinCell<T>> for RefCell<T> { + fn from(input: PinCell<T>) -> Self { + input.inner + } +} +// TODO CoerceUnsized diff --git a/third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs b/third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs new file mode 100644 index 0000000000..09a4d4a6fb --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/pin_mut.rs @@ -0,0 +1,50 @@ +use core::cell::RefMut; +use core::fmt; +use core::ops::Deref; +use core::pin::Pin; + +#[derive(Debug)] +/// A wrapper type for a mutably borrowed value from a `PinCell<T>`. +pub struct PinMut<'a, T: ?Sized> { + pub(crate) inner: Pin<RefMut<'a, T>>, +} + +impl<'a, T: ?Sized> Deref for PinMut<'a, T> { + type Target = T; + + fn deref(&self) -> &T { + &*self.inner + } +} + +impl<'a, T: ?Sized> PinMut<'a, T> { + /// Get a pinned mutable reference to the value inside this wrapper. + pub fn as_mut<'b>(orig: &'b mut PinMut<'a, T>) -> Pin<&'b mut T> { + orig.inner.as_mut() + } +} + +/* TODO implement these APIs + +impl<'a, T: ?Sized> PinMut<'a, T> { + pub fn map<U, F>(orig: PinMut<'a, T>, f: F) -> PinMut<'a, U> where + F: FnOnce(Pin<&mut T>) -> Pin<&mut U>, + { + panic!() + } + + pub fn map_split<U, V, F>(orig: PinMut<'a, T>, f: F) -> (PinMut<'a, U>, PinMut<'a, V>) where + F: FnOnce(Pin<&mut T>) -> (Pin<&mut U>, Pin<&mut V>) + { + panic!() + } +} +*/ + +impl<'a, T: fmt::Display + ?Sized> fmt::Display for PinMut<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + <T as fmt::Display>::fmt(&**self, f) + } +} + +// TODO CoerceUnsized diff --git a/third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs b/third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs new file mode 100644 index 0000000000..46f9cfdabb --- /dev/null +++ b/third_party/rust/fluent-fallback/src/pin_cell/pin_ref.rs @@ -0,0 +1,47 @@ +use core::cell::Ref; +use core::fmt; +use core::ops::Deref; +use core::pin::Pin; + +#[derive(Debug)] +/// A wrapper type for a immutably borrowed value from a `PinCell<T>`. +pub struct PinRef<'a, T: ?Sized> { + pub(crate) inner: Pin<Ref<'a, T>>, +} + +impl<'a, T: ?Sized> Deref for PinRef<'a, T> { + type Target = T; + + fn deref(&self) -> &T { + &*self.inner + } +} + +/* TODO implement these APIs + +impl<'a, T: ?Sized> PinRef<'a, T> { + pub fn clone(orig: &PinRef<'a, T>) -> PinRef<'a, T> { + panic!() + } + + pub fn map<U, F>(orig: PinRef<'a, T>, f: F) -> PinRef<'a, U> where + F: FnOnce(Pin<&T>) -> Pin<&U>, + { + panic!() + } + + pub fn map_split<U, V, F>(orig: PinRef<'a, T>, f: F) -> (PinRef<'a, U>, PinRef<'a, V>) where + F: FnOnce(Pin<&T>) -> (Pin<&U>, Pin<&V>) + { + panic!() + } +} +*/ + +impl<'a, T: fmt::Display + ?Sized> fmt::Display for PinRef<'a, T> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + <T as fmt::Display>::fmt(&**self, f) + } +} + +// TODO CoerceUnsized diff --git a/third_party/rust/fluent-fallback/src/types.rs b/third_party/rust/fluent-fallback/src/types.rs new file mode 100644 index 0000000000..6b87fa0522 --- /dev/null +++ b/third_party/rust/fluent-fallback/src/types.rs @@ -0,0 +1,141 @@ +use fluent_bundle::FluentArgs; +use std::borrow::Cow; + +#[derive(Debug)] +pub struct L10nKey<'l> { + pub id: Cow<'l, str>, + pub args: Option<FluentArgs<'l>>, +} + +impl<'l> From<&'l str> for L10nKey<'l> { + fn from(id: &'l str) -> Self { + Self { + id: id.into(), + args: None, + } + } +} + +#[derive(Debug, Clone)] +pub struct L10nAttribute<'l> { + pub name: Cow<'l, str>, + pub value: Cow<'l, str>, +} + +#[derive(Debug, Clone)] +pub struct L10nMessage<'l> { + pub value: Option<Cow<'l, str>>, + pub attributes: Vec<L10nAttribute<'l>>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ResourceType { + /// This is a required resource. + /// + /// A bundle generator should not consider a solution as valid + /// if this resource is missing. + /// + /// This is the default when creating a [`ResourceId`]. + Required, + + /// This is an optional resource. + /// + /// A bundle generator should still populate a partial solution + /// even if this resource is missing. + /// + /// This is intended for experimental and/or under-development + /// resources that may not have content for all supported locales. + /// + /// This should be used sparingly, as it will greatly increase + /// the state space of the search for valid solutions which can + /// have a severe impact on performance. + Optional, +} + +/// A resource identifier for a localization resource. +#[derive(Debug, Clone, Hash)] +pub struct ResourceId { + /// The resource identifier. + pub value: String, + + /// The [`ResourceType`] for this resource. + /// + /// The default value (when converting from another type) is + /// [`ResourceType::Required`]. You should only set this to + /// [`ResourceType::Optional`] for experimental or under-development + /// features that may not yet have content in all eventually-supported locales. + /// + /// Setting this value to [`ResourceType::Optional`] for all resources + /// may have a severe impact on performance due to increasing the state space + /// of the solver. + pub resource_type: ResourceType, +} + +impl ResourceId { + pub fn new<S: Into<String>>(value: S, resource_type: ResourceType) -> Self { + Self { + value: value.into(), + resource_type, + } + } + + /// Returns [`true`] if the resource has [`ResourceType::Required`], + /// otherwise returns [`false`]. + pub fn is_required(&self) -> bool { + matches!(self.resource_type, ResourceType::Required) + } + + /// Returns [`true`] if the resource has [`ResourceType::Optional`], + /// otherwise returns [`false`]. + pub fn is_optional(&self) -> bool { + matches!(self.resource_type, ResourceType::Optional) + } +} + +impl<S: Into<String>> From<S> for ResourceId { + fn from(id: S) -> Self { + Self { + value: id.into(), + resource_type: ResourceType::Required, + } + } +} + +impl std::fmt::Display for ResourceId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.value) + } +} + +impl PartialEq<str> for ResourceId { + fn eq(&self, other: &str) -> bool { + self.value.as_str().eq(other) + } +} + +impl Eq for ResourceId {} +impl PartialEq for ResourceId { + fn eq(&self, other: &Self) -> bool { + self.value.eq(&other.value) + } +} + +/// A trait for creating a [`ResourceId`] from another type. +/// +/// This differs from the [`From`] trait in that the [`From`] trait +/// always takes the default resource type of [`ResourceType::Required`]. +/// +/// If you need to create a resource with a non-default [`ResourceType`], +/// such as [`ResourceType::Optional`], then use this trait. +/// +/// This trait is automatically implemented for types that implement [`Into<String>`]. +pub trait ToResourceId { + /// Creates a [`ResourceId`] from [`self`], given a [`ResourceType`]. + fn to_resource_id(self, resource_type: ResourceType) -> ResourceId; +} + +impl<S: Into<String>> ToResourceId for S { + fn to_resource_id(self, resource_type: ResourceType) -> ResourceId { + ResourceId::new(self.into(), resource_type) + } +} |