mod fetcher; pub use fetcher::FileFetcher; pub use fluent_fallback::types::{ResourceId, ToResourceId}; use crate::env::ErrorReporter; use crate::errors::L10nRegistryError; use crate::fluent::FluentResource; use std::{ borrow::Borrow, cell::RefCell, fmt, hash::{Hash, Hasher}, pin::Pin, rc::Rc, task::Poll, }; use futures::{future::Shared, Future, FutureExt}; use rustc_hash::FxHashMap; use unic_langid::LanguageIdentifier; pub type RcResource = Rc; /// An option type whose None variant is either optional or required. /// /// This behaves similarly to the standard-library [`Option`] type /// except that there are two [`None`]-like variants: /// [`ResourceOption::MissingOptional`] and [`ResourceOption::MissingRequired`]. #[derive(Clone, Debug)] pub enum ResourceOption { /// An available resource. Some(RcResource), /// A missing optional resource. MissingOptional, /// A missing required resource. MissingRequired, } impl ResourceOption { /// Creates a resource option that is either [`ResourceOption::MissingRequired`] /// or [`ResourceOption::MissingOptional`] based on whether the given [`ResourceId`] /// is required or optional. pub fn missing_resource(resource_id: &ResourceId) -> Self { if resource_id.is_required() { Self::MissingRequired } else { Self::MissingOptional } } /// Returns [`true`] if this option contains a recource, otherwise [`false`]. pub fn is_some(&self) -> bool { matches!(self, Self::Some(_)) } /// Resource [`true`] if this option is missing a resource of any type, otherwise [`false`]. pub fn is_none(&self) -> bool { matches!(self, Self::MissingOptional | Self::MissingRequired) } /// Returns [`true`] if this option is missing a required resource, otherwise [`false`]. pub fn is_required_and_missing(&self) -> bool { matches!(self, Self::MissingRequired) } } impl From for Option { fn from(other: ResourceOption) -> Self { match other { ResourceOption::Some(id) => Some(id), _ => None, } } } pub type ResourceFuture = Shared>>>; #[derive(Debug, Clone)] pub enum ResourceStatus { /// The resource is missing. Don't bother trying to fetch. MissingRequired, MissingOptional, /// The resource is loading and future will deliver the result. Loading(ResourceFuture), /// The resource is loaded and parsed. Loaded(RcResource), } impl From for ResourceStatus { fn from(input: ResourceOption) -> Self { match input { ResourceOption::Some(res) => Self::Loaded(res), ResourceOption::MissingOptional => Self::MissingOptional, ResourceOption::MissingRequired => Self::MissingRequired, } } } impl Future for ResourceStatus { type Output = ResourceOption; fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll { use ResourceStatus::*; let this = &mut *self; match this { MissingRequired => ResourceOption::MissingRequired.into(), MissingOptional => ResourceOption::MissingOptional.into(), Loaded(res) => ResourceOption::Some(res.clone()).into(), Loading(res) => Pin::new(res).poll(cx), } } } /// `FileSource` provides a generic fetching and caching of fluent resources. /// The user of `FileSource` provides a [`FileFetcher`](trait.FileFetcher.html) /// implementation and `FileSource` takes care of the rest. #[derive(Clone)] pub struct FileSource { /// Name of the FileSource, e.g. "browser" pub name: String, /// Pre-formatted path for the FileSource, e.g. "/browser/data/locale/{locale}/" pub pre_path: String, /// Metasource name for the FileSource, e.g. "app", "langpack" /// Only sources from the same metasource are passed into the solver. pub metasource: String, /// The locales for which data is present in the FileSource, e.g. ["en-US", "pl"] locales: Vec, shared: Rc, index: Option>, pub options: FileSourceOptions, } struct Inner { fetcher: Box, error_reporter: Option>>, entries: RefCell>, } impl fmt::Display for FileSource { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.name) } } impl PartialEq for FileSource { fn eq(&self, other: &Self) -> bool { self.name == other.name && self.metasource == other.metasource } } impl Eq for FileSource {} impl Hash for FileSource { fn hash(&self, state: &mut H) { self.name.hash(state) } } #[derive(PartialEq, Clone, Debug)] pub struct FileSourceOptions { pub allow_override: bool, } impl Default for FileSourceOptions { fn default() -> Self { Self { allow_override: false, } } } impl FileSource { /// Create a `FileSource` using the provided [`FileFetcher`](../trait.FileFetcher.html). pub fn new( name: String, metasource: Option, locales: Vec, pre_path: String, options: FileSourceOptions, fetcher: impl FileFetcher + 'static, ) -> Self { FileSource { name, metasource: metasource.unwrap_or_default(), pre_path, locales, index: None, shared: Rc::new(Inner { entries: RefCell::new(FxHashMap::default()), fetcher: Box::new(fetcher), error_reporter: None, }), options, } } pub fn new_with_index( name: String, metasource: Option, locales: Vec, pre_path: String, options: FileSourceOptions, fetcher: impl FileFetcher + 'static, index: Vec, ) -> Self { FileSource { name, metasource: metasource.unwrap_or_default(), pre_path, locales, index: Some(index), shared: Rc::new(Inner { entries: RefCell::new(FxHashMap::default()), fetcher: Box::new(fetcher), error_reporter: None, }), options, } } pub fn set_reporter(&mut self, reporter: impl ErrorReporter + 'static) { let mut shared = Rc::get_mut(&mut self.shared).unwrap(); shared.error_reporter = Some(RefCell::new(Box::new(reporter))); } } fn calculate_pos_in_source(source: &str, idx: usize) -> (usize, usize) { let mut ptr = 0; let mut result = (1, 1); for line in source.lines() { let bytes = line.as_bytes().len(); if ptr + bytes < idx { ptr += bytes + 1; result.0 += 1; } else { result.1 = idx - ptr + 1; break; } } result } impl FileSource { fn get_path(&self, locale: &LanguageIdentifier, resource_id: &ResourceId) -> String { format!( "{}{}", self.pre_path.replace("{locale}", &locale.to_string()), resource_id.value, ) } fn fetch_sync(&self, resource_id: &ResourceId) -> ResourceOption { self.shared .fetcher .fetch_sync(resource_id) .ok() .map(|source| match FluentResource::try_new(source) { Ok(res) => ResourceOption::Some(Rc::new(res)), Err((res, errors)) => { if let Some(reporter) = &self.shared.error_reporter { reporter.borrow().report_errors( errors .into_iter() .map(|e| L10nRegistryError::FluentError { resource_id: resource_id.clone(), loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), error: e.into(), }) .collect(), ); } ResourceOption::Some(Rc::new(res)) } }) .unwrap_or_else(|| ResourceOption::missing_resource(resource_id)) } /// Attempt to synchronously fetch resource for the combination of `locale` /// and `path`. Returns `Some(ResourceResult)` if the resource is available, /// else `None`. pub fn fetch_file_sync( &self, locale: &LanguageIdentifier, resource_id: &ResourceId, overload: bool, ) -> ResourceOption { use ResourceStatus::*; if self.has_file(locale, resource_id) == Some(false) { return ResourceOption::missing_resource(resource_id); } let full_path_id = self .get_path(locale, resource_id) .to_resource_id(resource_id.resource_type); let res = self.shared.lookup_resource(full_path_id.clone(), || { self.fetch_sync(&full_path_id).into() }); match res { MissingRequired => ResourceOption::MissingRequired, MissingOptional => ResourceOption::MissingOptional, Loaded(res) => ResourceOption::Some(res), Loading(..) if overload => { // A sync load has been requested for the same resource that has // a pending async load in progress. How do we handle this? // // Ideally, we would sync load and resolve all the pending // futures with the result. With the current Futures and // combinators, it's unclear how to proceed. One potential // solution is to store a oneshot::Sender and // Shared. When the async loading future // resolves it would check that the state is still `Loading`, // and if so, send the result. The sync load would do the same // send on the oneshot::Sender. // // For now, we warn and return the resource, paying the cost of // duplication of the resource. self.fetch_sync(&full_path_id) } Loading(..) => { panic!("[l10nregistry] Attempting to synchronously load file {} while it's being loaded asynchronously.", &full_path_id.value); } } } /// Attempt to fetch resource for the combination of `locale` and `path`. /// Returns [`ResourceStatus`](enum.ResourceStatus.html) which is /// a `Future` that can be polled. pub fn fetch_file( &self, locale: &LanguageIdentifier, resource_id: &ResourceId, ) -> ResourceStatus { use ResourceStatus::*; if self.has_file(locale, resource_id) == Some(false) { return ResourceOption::missing_resource(resource_id).into(); } let full_path_id = self .get_path(locale, resource_id) .to_resource_id(resource_id.resource_type); self.shared.lookup_resource(full_path_id.clone(), || { let shared = self.shared.clone(); Loading(read_resource(full_path_id, shared).boxed_local().shared()) }) } /// Determine if the `FileSource` has a loaded resource for the combination /// of `locale` and `path`. Returns `Some(true)` if the file is loaded, else /// `Some(false)`. `None` is returned if there is an outstanding async fetch /// pending and the status is yet to be determined. pub fn has_file>( &self, locale: L, path: &ResourceId, ) -> Option { let locale = locale.borrow(); if !self.locales.contains(locale) { Some(false) } else { let full_path = self.get_path(locale, path); if let Some(index) = &self.index { return Some(index.iter().any(|p| p == &full_path)); } self.shared.has_file(&full_path) } } pub fn locales(&self) -> &[LanguageIdentifier] { &self.locales } pub fn get_index(&self) -> Option<&Vec> { self.index.as_ref() } } impl std::fmt::Debug for FileSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { if let Some(index) = &self.index { f.debug_struct("FileSource") .field("name", &self.name) .field("metasource", &self.metasource) .field("locales", &self.locales) .field("pre_path", &self.pre_path) .field("index", index) .finish() } else { f.debug_struct("FileSource") .field("name", &self.name) .field("metasource", &self.metasource) .field("locales", &self.locales) .field("pre_path", &self.pre_path) .finish() } } } impl Inner { fn lookup_resource(&self, resource_id: ResourceId, f: F) -> ResourceStatus where F: FnOnce() -> ResourceStatus, { let mut lock = self.entries.borrow_mut(); lock.entry(resource_id.value).or_insert_with(|| f()).clone() } fn update_resource(&self, resource_id: ResourceId, resource: ResourceOption) -> ResourceOption { let mut lock = self.entries.borrow_mut(); let entry = lock.get_mut(&resource_id.value); match entry { Some(entry) => *entry = resource.clone().into(), _ => panic!("Expected "), } resource } pub fn has_file(&self, full_path: &str) -> Option { match self.entries.borrow().get(full_path) { Some(ResourceStatus::MissingRequired) => Some(false), Some(ResourceStatus::MissingOptional) => Some(false), Some(ResourceStatus::Loaded(_)) => Some(true), Some(ResourceStatus::Loading(_)) | None => None, } } } async fn read_resource(resource_id: ResourceId, shared: Rc) -> ResourceOption { let resource = shared .fetcher .fetch(&resource_id) .await .ok() .map(|source| match FluentResource::try_new(source) { Ok(res) => ResourceOption::Some(Rc::new(res)), Err((res, errors)) => { if let Some(reporter) = &shared.error_reporter.borrow() { reporter.borrow().report_errors( errors .into_iter() .map(|e| L10nRegistryError::FluentError { resource_id: resource_id.clone(), loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), error: e.into(), }) .collect(), ); } ResourceOption::Some(Rc::new(res)) } }) .unwrap_or_else(|| ResourceOption::missing_resource(&resource_id)); // insert the resource into the cache shared.update_resource(resource_id, resource) } #[cfg(test)] mod tests { use super::*; #[test] fn calculate_source_pos() { let source = r#" key = Value key2 = Value 2 "# .trim(); let result = calculate_pos_in_source(source, 0); assert_eq!(result, (1, 1)); let result = calculate_pos_in_source(source, 1); assert_eq!(result, (1, 2)); let result = calculate_pos_in_source(source, 12); assert_eq!(result, (2, 1)); let result = calculate_pos_in_source(source, 13); assert_eq!(result, (3, 1)); } } #[cfg(test)] #[cfg(all(feature = "tokio", feature = "test-fluent"))] mod tests_tokio { use super::*; use crate::testing::TestFileFetcher; static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl"; static FTL_RESOURCE_MISSING: &str = "missing.ftl"; #[tokio::test] async fn file_source_fetch() { let fetcher = TestFileFetcher::new(); let en_us: LanguageIdentifier = "en-US".parse().unwrap(); let fs1 = fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; assert!(file.is_some()); } #[tokio::test] async fn file_source_fetch_missing() { let fetcher = TestFileFetcher::new(); let en_us: LanguageIdentifier = "en-US".parse().unwrap(); let fs1 = fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await; assert!(file.is_none()); } #[tokio::test] async fn file_source_already_loaded() { let fetcher = TestFileFetcher::new(); let en_us: LanguageIdentifier = "en-US".parse().unwrap(); let fs1 = fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; assert!(file.is_some()); let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; assert!(file.is_some()); } #[tokio::test] async fn file_source_concurrent() { let fetcher = TestFileFetcher::new(); let en_us: LanguageIdentifier = "en-US".parse().unwrap(); let fs1 = fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); let file1 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); let file2 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); assert!(file1.await.is_some()); assert!(file2.await.is_some()); } #[test] fn file_source_sync_after_async_fail() { let fetcher = TestFileFetcher::new(); let en_us: LanguageIdentifier = "en-US".parse().unwrap(); let fs1 = fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); let _ = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); let file2 = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), true); assert!(file2.is_some()); } }