diff options
Diffstat (limited to 'intl/l10n/rust/l10nregistry-rs/src/source')
| -rw-r--r-- | intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs | 30 | ||||
| -rw-r--r-- | intl/l10n/rust/l10nregistry-rs/src/source/mod.rs | 558 |
2 files changed, 588 insertions, 0 deletions
diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs new file mode 100644 index 0000000000..3a022990a6 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs @@ -0,0 +1,30 @@ +use async_trait::async_trait; +use fluent_fallback::types::ResourceId; +use std::io; + +/// The users of [`FileSource`] implement this trait to provide loading of +/// resources, returning the contents of a resource as a +/// `String`. [`FileSource`] handles the conversion from string representation +/// into `FluentResource`. +/// +/// [`FileSource`]: source/struct.FileSource.html +#[async_trait(?Send)] +pub trait FileFetcher { + /// Return the `String` representation for `path`. This version is + /// blocking. + /// + /// See [`fetch`](#tymethod.fetch). + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String>; + + /// Return the `String` representation for `path`. + /// + /// On success, returns `Poll::Ready(Ok(..))`. + /// + /// If no resource is available to be fetched, the method returns + /// `Poll::Pending` and arranges for the current task (via + /// `cx.waker().wake_by_ref()`) to receive a notification when the resource + /// is available. + /// + /// See [`fetch_sync`](#tymethod.fetch_sync) + async fn fetch(&self, path: &ResourceId) -> io::Result<String>; +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs new file mode 100644 index 0000000000..1c72065b38 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs @@ -0,0 +1,558 @@ +mod fetcher; +pub use fetcher::FileFetcher; +pub use fluent_fallback::types::{ResourceId, ToResourceId}; + +use crate::env::ErrorReporter; +use crate::errors::L10nRegistryError; +use crate::fluent::FluentResource; + +use std::{ + borrow::Borrow, + cell::RefCell, + fmt, + hash::{Hash, Hasher}, + pin::Pin, + rc::Rc, + task::Poll, +}; + +use futures::{future::Shared, Future, FutureExt}; +use rustc_hash::FxHashMap; +use unic_langid::LanguageIdentifier; + +pub type RcResource = Rc<FluentResource>; + +/// An option type whose None variant is either optional or required. +/// +/// This behaves similarly to the standard-library [`Option`] type +/// except that there are two [`None`]-like variants: +/// [`ResourceOption::MissingOptional`] and [`ResourceOption::MissingRequired`]. +#[derive(Clone, Debug)] +pub enum ResourceOption { + /// An available resource. + Some(RcResource), + /// A missing optional resource. + MissingOptional, + /// A missing required resource. + MissingRequired, +} + +impl ResourceOption { + /// Creates a resource option that is either [`ResourceOption::MissingRequired`] + /// or [`ResourceOption::MissingOptional`] based on whether the given [`ResourceId`] + /// is required or optional. + pub fn missing_resource(resource_id: &ResourceId) -> Self { + if resource_id.is_required() { + Self::MissingRequired + } else { + Self::MissingOptional + } + } + + /// Returns [`true`] if this option contains a recource, otherwise [`false`]. + pub fn is_some(&self) -> bool { + matches!(self, Self::Some(_)) + } + + /// Resource [`true`] if this option is missing a resource of any type, otherwise [`false`]. + pub fn is_none(&self) -> bool { + matches!(self, Self::MissingOptional | Self::MissingRequired) + } + + /// Returns [`true`] if this option is missing a required resource, otherwise [`false`]. + pub fn is_required_and_missing(&self) -> bool { + matches!(self, Self::MissingRequired) + } +} + +impl From<ResourceOption> for Option<RcResource> { + fn from(other: ResourceOption) -> Self { + match other { + ResourceOption::Some(id) => Some(id), + _ => None, + } + } +} + +pub type ResourceFuture = Shared<Pin<Box<dyn Future<Output = ResourceOption>>>>; + +#[derive(Debug, Clone)] +pub enum ResourceStatus { + /// The resource is missing. Don't bother trying to fetch. + MissingRequired, + MissingOptional, + /// The resource is loading and future will deliver the result. + Loading(ResourceFuture), + /// The resource is loaded and parsed. + Loaded(RcResource), +} + +impl From<ResourceOption> for ResourceStatus { + fn from(input: ResourceOption) -> Self { + match input { + ResourceOption::Some(res) => Self::Loaded(res), + ResourceOption::MissingOptional => Self::MissingOptional, + ResourceOption::MissingRequired => Self::MissingRequired, + } + } +} + +impl Future for ResourceStatus { + type Output = ResourceOption; + + fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> { + use ResourceStatus::*; + + let this = &mut *self; + + match this { + MissingRequired => ResourceOption::MissingRequired.into(), + MissingOptional => ResourceOption::MissingOptional.into(), + Loaded(res) => ResourceOption::Some(res.clone()).into(), + Loading(res) => Pin::new(res).poll(cx), + } + } +} + +/// `FileSource` provides a generic fetching and caching of fluent resources. +/// The user of `FileSource` provides a [`FileFetcher`](trait.FileFetcher.html) +/// implementation and `FileSource` takes care of the rest. +#[derive(Clone)] +pub struct FileSource { + /// Name of the FileSource, e.g. "browser" + pub name: String, + /// Pre-formatted path for the FileSource, e.g. "/browser/data/locale/{locale}/" + pub pre_path: String, + /// Metasource name for the FileSource, e.g. "app", "langpack" + /// Only sources from the same metasource are passed into the solver. + pub metasource: String, + /// The locales for which data is present in the FileSource, e.g. ["en-US", "pl"] + locales: Vec<LanguageIdentifier>, + shared: Rc<Inner>, + index: Option<Vec<String>>, + pub options: FileSourceOptions, +} + +struct Inner { + fetcher: Box<dyn FileFetcher>, + error_reporter: Option<RefCell<Box<dyn ErrorReporter>>>, + entries: RefCell<FxHashMap<String, ResourceStatus>>, +} + +impl fmt::Display for FileSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name) + } +} + +impl PartialEq<FileSource> for FileSource { + fn eq(&self, other: &Self) -> bool { + self.name == other.name && self.metasource == other.metasource + } +} + +impl Eq for FileSource {} + +impl Hash for FileSource { + fn hash<H: Hasher>(&self, state: &mut H) { + self.name.hash(state) + } +} + +#[derive(PartialEq, Clone, Debug)] +pub struct FileSourceOptions { + pub allow_override: bool, +} + +impl Default for FileSourceOptions { + fn default() -> Self { + Self { + allow_override: false, + } + } +} + +impl FileSource { + /// Create a `FileSource` using the provided [`FileFetcher`](../trait.FileFetcher.html). + pub fn new( + name: String, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + pre_path: String, + options: FileSourceOptions, + fetcher: impl FileFetcher + 'static, + ) -> Self { + FileSource { + name, + metasource: metasource.unwrap_or_default(), + pre_path, + locales, + index: None, + shared: Rc::new(Inner { + entries: RefCell::new(FxHashMap::default()), + fetcher: Box::new(fetcher), + error_reporter: None, + }), + options, + } + } + + pub fn new_with_index( + name: String, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + pre_path: String, + options: FileSourceOptions, + fetcher: impl FileFetcher + 'static, + index: Vec<String>, + ) -> Self { + FileSource { + name, + metasource: metasource.unwrap_or_default(), + pre_path, + locales, + index: Some(index), + shared: Rc::new(Inner { + entries: RefCell::new(FxHashMap::default()), + fetcher: Box::new(fetcher), + error_reporter: None, + }), + options, + } + } + + pub fn set_reporter(&mut self, reporter: impl ErrorReporter + 'static) { + let mut shared = Rc::get_mut(&mut self.shared).unwrap(); + shared.error_reporter = Some(RefCell::new(Box::new(reporter))); + } +} + +fn calculate_pos_in_source(source: &str, idx: usize) -> (usize, usize) { + let mut ptr = 0; + let mut result = (1, 1); + for line in source.lines() { + let bytes = line.as_bytes().len(); + if ptr + bytes < idx { + ptr += bytes + 1; + result.0 += 1; + } else { + result.1 = idx - ptr + 1; + break; + } + } + result +} + +impl FileSource { + fn get_path(&self, locale: &LanguageIdentifier, resource_id: &ResourceId) -> String { + format!( + "{}{}", + self.pre_path.replace("{locale}", &locale.to_string()), + resource_id.value, + ) + } + + fn fetch_sync(&self, resource_id: &ResourceId) -> ResourceOption { + self.shared + .fetcher + .fetch_sync(resource_id) + .ok() + .map(|source| match FluentResource::try_new(source) { + Ok(res) => ResourceOption::Some(Rc::new(res)), + Err((res, errors)) => { + if let Some(reporter) = &self.shared.error_reporter { + reporter.borrow().report_errors( + errors + .into_iter() + .map(|e| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), + error: e.into(), + }) + .collect(), + ); + } + ResourceOption::Some(Rc::new(res)) + } + }) + .unwrap_or_else(|| ResourceOption::missing_resource(resource_id)) + } + + /// Attempt to synchronously fetch resource for the combination of `locale` + /// and `path`. Returns `Some(ResourceResult)` if the resource is available, + /// else `None`. + pub fn fetch_file_sync( + &self, + locale: &LanguageIdentifier, + resource_id: &ResourceId, + overload: bool, + ) -> ResourceOption { + use ResourceStatus::*; + + if self.has_file(locale, resource_id) == Some(false) { + return ResourceOption::missing_resource(resource_id); + } + + let full_path_id = self + .get_path(locale, resource_id) + .to_resource_id(resource_id.resource_type); + + let res = self.shared.lookup_resource(full_path_id.clone(), || { + self.fetch_sync(&full_path_id).into() + }); + + match res { + MissingRequired => ResourceOption::MissingRequired, + MissingOptional => ResourceOption::MissingOptional, + Loaded(res) => ResourceOption::Some(res), + Loading(..) if overload => { + // A sync load has been requested for the same resource that has + // a pending async load in progress. How do we handle this? + // + // Ideally, we would sync load and resolve all the pending + // futures with the result. With the current Futures and + // combinators, it's unclear how to proceed. One potential + // solution is to store a oneshot::Sender and + // Shared<oneshot::Receiver>. When the async loading future + // resolves it would check that the state is still `Loading`, + // and if so, send the result. The sync load would do the same + // send on the oneshot::Sender. + // + // For now, we warn and return the resource, paying the cost of + // duplication of the resource. + self.fetch_sync(&full_path_id) + } + Loading(..) => { + panic!("[l10nregistry] Attempting to synchronously load file {} while it's being loaded asynchronously.", &full_path_id.value); + } + } + } + + /// Attempt to fetch resource for the combination of `locale` and `path`. + /// Returns [`ResourceStatus`](enum.ResourceStatus.html) which is + /// a `Future` that can be polled. + pub fn fetch_file( + &self, + locale: &LanguageIdentifier, + resource_id: &ResourceId, + ) -> ResourceStatus { + use ResourceStatus::*; + + if self.has_file(locale, resource_id) == Some(false) { + return ResourceOption::missing_resource(resource_id).into(); + } + + let full_path_id = self + .get_path(locale, resource_id) + .to_resource_id(resource_id.resource_type); + + self.shared.lookup_resource(full_path_id.clone(), || { + let shared = self.shared.clone(); + Loading(read_resource(full_path_id, shared).boxed_local().shared()) + }) + } + + /// Determine if the `FileSource` has a loaded resource for the combination + /// of `locale` and `path`. Returns `Some(true)` if the file is loaded, else + /// `Some(false)`. `None` is returned if there is an outstanding async fetch + /// pending and the status is yet to be determined. + pub fn has_file<L: Borrow<LanguageIdentifier>>( + &self, + locale: L, + path: &ResourceId, + ) -> Option<bool> { + let locale = locale.borrow(); + if !self.locales.contains(locale) { + Some(false) + } else { + let full_path = self.get_path(locale, path); + if let Some(index) = &self.index { + return Some(index.iter().any(|p| p == &full_path)); + } + self.shared.has_file(&full_path) + } + } + + pub fn locales(&self) -> &[LanguageIdentifier] { + &self.locales + } + + pub fn get_index(&self) -> Option<&Vec<String>> { + self.index.as_ref() + } +} + +impl std::fmt::Debug for FileSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { + if let Some(index) = &self.index { + f.debug_struct("FileSource") + .field("name", &self.name) + .field("metasource", &self.metasource) + .field("locales", &self.locales) + .field("pre_path", &self.pre_path) + .field("index", index) + .finish() + } else { + f.debug_struct("FileSource") + .field("name", &self.name) + .field("metasource", &self.metasource) + .field("locales", &self.locales) + .field("pre_path", &self.pre_path) + .finish() + } + } +} + +impl Inner { + fn lookup_resource<F>(&self, resource_id: ResourceId, f: F) -> ResourceStatus + where + F: FnOnce() -> ResourceStatus, + { + let mut lock = self.entries.borrow_mut(); + lock.entry(resource_id.value).or_insert_with(|| f()).clone() + } + + fn update_resource(&self, resource_id: ResourceId, resource: ResourceOption) -> ResourceOption { + let mut lock = self.entries.borrow_mut(); + let entry = lock.get_mut(&resource_id.value); + match entry { + Some(entry) => *entry = resource.clone().into(), + _ => panic!("Expected "), + } + resource + } + + pub fn has_file(&self, full_path: &str) -> Option<bool> { + match self.entries.borrow().get(full_path) { + Some(ResourceStatus::MissingRequired) => Some(false), + Some(ResourceStatus::MissingOptional) => Some(false), + Some(ResourceStatus::Loaded(_)) => Some(true), + Some(ResourceStatus::Loading(_)) | None => None, + } + } +} + +async fn read_resource(resource_id: ResourceId, shared: Rc<Inner>) -> ResourceOption { + let resource = shared + .fetcher + .fetch(&resource_id) + .await + .ok() + .map(|source| match FluentResource::try_new(source) { + Ok(res) => ResourceOption::Some(Rc::new(res)), + Err((res, errors)) => { + if let Some(reporter) = &shared.error_reporter.borrow() { + reporter.borrow().report_errors( + errors + .into_iter() + .map(|e| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), + error: e.into(), + }) + .collect(), + ); + } + ResourceOption::Some(Rc::new(res)) + } + }) + .unwrap_or_else(|| ResourceOption::missing_resource(&resource_id)); + // insert the resource into the cache + shared.update_resource(resource_id, resource) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calculate_source_pos() { + let source = r#" +key = Value + +key2 = Value 2 +"# + .trim(); + let result = calculate_pos_in_source(source, 0); + assert_eq!(result, (1, 1)); + + let result = calculate_pos_in_source(source, 1); + assert_eq!(result, (1, 2)); + + let result = calculate_pos_in_source(source, 12); + assert_eq!(result, (2, 1)); + + let result = calculate_pos_in_source(source, 13); + assert_eq!(result, (3, 1)); + } +} + +#[cfg(test)] +#[cfg(all(feature = "tokio", feature = "test-fluent"))] +mod tests_tokio { + use super::*; + use crate::testing::TestFileFetcher; + + static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl"; + static FTL_RESOURCE_MISSING: &str = "missing.ftl"; + + #[tokio::test] + async fn file_source_fetch() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + } + + #[tokio::test] + async fn file_source_fetch_missing() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await; + assert!(file.is_none()); + } + + #[tokio::test] + async fn file_source_already_loaded() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + } + + #[tokio::test] + async fn file_source_concurrent() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file1 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + let file2 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + assert!(file1.await.is_some()); + assert!(file2.await.is_some()); + } + + #[test] + fn file_source_sync_after_async_fail() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let _ = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + let file2 = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), true); + assert!(file2.is_some()); + } +} |