summaryrefslogtreecommitdiffstats
path: root/intl/l10n/rust/l10nregistry-rs/src/source
diff options
context:
space:
mode:
Diffstat (limited to 'intl/l10n/rust/l10nregistry-rs/src/source')
-rw-r--r--intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs30
-rw-r--r--intl/l10n/rust/l10nregistry-rs/src/source/mod.rs558
2 files changed, 588 insertions, 0 deletions
diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs
new file mode 100644
index 0000000000..3a022990a6
--- /dev/null
+++ b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs
@@ -0,0 +1,30 @@
+use async_trait::async_trait;
+use fluent_fallback::types::ResourceId;
+use std::io;
+
+/// The users of [`FileSource`] implement this trait to provide loading of
+/// resources, returning the contents of a resource as a
+/// `String`. [`FileSource`] handles the conversion from string representation
+/// into `FluentResource`.
+///
+/// [`FileSource`]: source/struct.FileSource.html
+#[async_trait(?Send)]
+pub trait FileFetcher {
+ /// Return the `String` representation for `path`. This version is
+ /// blocking.
+ ///
+ /// See [`fetch`](#tymethod.fetch).
+ fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String>;
+
+ /// Return the `String` representation for `path`.
+ ///
+ /// On success, returns `Poll::Ready(Ok(..))`.
+ ///
+ /// If no resource is available to be fetched, the method returns
+ /// `Poll::Pending` and arranges for the current task (via
+ /// `cx.waker().wake_by_ref()`) to receive a notification when the resource
+ /// is available.
+ ///
+ /// See [`fetch_sync`](#tymethod.fetch_sync)
+ async fn fetch(&self, path: &ResourceId) -> io::Result<String>;
+}
diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs
new file mode 100644
index 0000000000..1c72065b38
--- /dev/null
+++ b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs
@@ -0,0 +1,558 @@
+mod fetcher;
+pub use fetcher::FileFetcher;
+pub use fluent_fallback::types::{ResourceId, ToResourceId};
+
+use crate::env::ErrorReporter;
+use crate::errors::L10nRegistryError;
+use crate::fluent::FluentResource;
+
+use std::{
+ borrow::Borrow,
+ cell::RefCell,
+ fmt,
+ hash::{Hash, Hasher},
+ pin::Pin,
+ rc::Rc,
+ task::Poll,
+};
+
+use futures::{future::Shared, Future, FutureExt};
+use rustc_hash::FxHashMap;
+use unic_langid::LanguageIdentifier;
+
+pub type RcResource = Rc<FluentResource>;
+
+/// An option type whose None variant is either optional or required.
+///
+/// This behaves similarly to the standard-library [`Option`] type
+/// except that there are two [`None`]-like variants:
+/// [`ResourceOption::MissingOptional`] and [`ResourceOption::MissingRequired`].
+#[derive(Clone, Debug)]
+pub enum ResourceOption {
+ /// An available resource.
+ Some(RcResource),
+ /// A missing optional resource.
+ MissingOptional,
+ /// A missing required resource.
+ MissingRequired,
+}
+
+impl ResourceOption {
+ /// Creates a resource option that is either [`ResourceOption::MissingRequired`]
+ /// or [`ResourceOption::MissingOptional`] based on whether the given [`ResourceId`]
+ /// is required or optional.
+ pub fn missing_resource(resource_id: &ResourceId) -> Self {
+ if resource_id.is_required() {
+ Self::MissingRequired
+ } else {
+ Self::MissingOptional
+ }
+ }
+
+ /// Returns [`true`] if this option contains a recource, otherwise [`false`].
+ pub fn is_some(&self) -> bool {
+ matches!(self, Self::Some(_))
+ }
+
+ /// Resource [`true`] if this option is missing a resource of any type, otherwise [`false`].
+ pub fn is_none(&self) -> bool {
+ matches!(self, Self::MissingOptional | Self::MissingRequired)
+ }
+
+ /// Returns [`true`] if this option is missing a required resource, otherwise [`false`].
+ pub fn is_required_and_missing(&self) -> bool {
+ matches!(self, Self::MissingRequired)
+ }
+}
+
+impl From<ResourceOption> for Option<RcResource> {
+ fn from(other: ResourceOption) -> Self {
+ match other {
+ ResourceOption::Some(id) => Some(id),
+ _ => None,
+ }
+ }
+}
+
+pub type ResourceFuture = Shared<Pin<Box<dyn Future<Output = ResourceOption>>>>;
+
+#[derive(Debug, Clone)]
+pub enum ResourceStatus {
+ /// The resource is missing. Don't bother trying to fetch.
+ MissingRequired,
+ MissingOptional,
+ /// The resource is loading and future will deliver the result.
+ Loading(ResourceFuture),
+ /// The resource is loaded and parsed.
+ Loaded(RcResource),
+}
+
+impl From<ResourceOption> for ResourceStatus {
+ fn from(input: ResourceOption) -> Self {
+ match input {
+ ResourceOption::Some(res) => Self::Loaded(res),
+ ResourceOption::MissingOptional => Self::MissingOptional,
+ ResourceOption::MissingRequired => Self::MissingRequired,
+ }
+ }
+}
+
+impl Future for ResourceStatus {
+ type Output = ResourceOption;
+
+ fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
+ use ResourceStatus::*;
+
+ let this = &mut *self;
+
+ match this {
+ MissingRequired => ResourceOption::MissingRequired.into(),
+ MissingOptional => ResourceOption::MissingOptional.into(),
+ Loaded(res) => ResourceOption::Some(res.clone()).into(),
+ Loading(res) => Pin::new(res).poll(cx),
+ }
+ }
+}
+
+/// `FileSource` provides a generic fetching and caching of fluent resources.
+/// The user of `FileSource` provides a [`FileFetcher`](trait.FileFetcher.html)
+/// implementation and `FileSource` takes care of the rest.
+#[derive(Clone)]
+pub struct FileSource {
+ /// Name of the FileSource, e.g. "browser"
+ pub name: String,
+ /// Pre-formatted path for the FileSource, e.g. "/browser/data/locale/{locale}/"
+ pub pre_path: String,
+ /// Metasource name for the FileSource, e.g. "app", "langpack"
+ /// Only sources from the same metasource are passed into the solver.
+ pub metasource: String,
+ /// The locales for which data is present in the FileSource, e.g. ["en-US", "pl"]
+ locales: Vec<LanguageIdentifier>,
+ shared: Rc<Inner>,
+ index: Option<Vec<String>>,
+ pub options: FileSourceOptions,
+}
+
+struct Inner {
+ fetcher: Box<dyn FileFetcher>,
+ error_reporter: Option<RefCell<Box<dyn ErrorReporter>>>,
+ entries: RefCell<FxHashMap<String, ResourceStatus>>,
+}
+
+impl fmt::Display for FileSource {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "{}", self.name)
+ }
+}
+
+impl PartialEq<FileSource> for FileSource {
+ fn eq(&self, other: &Self) -> bool {
+ self.name == other.name && self.metasource == other.metasource
+ }
+}
+
+impl Eq for FileSource {}
+
+impl Hash for FileSource {
+ fn hash<H: Hasher>(&self, state: &mut H) {
+ self.name.hash(state)
+ }
+}
+
+#[derive(PartialEq, Clone, Debug)]
+pub struct FileSourceOptions {
+ pub allow_override: bool,
+}
+
+impl Default for FileSourceOptions {
+ fn default() -> Self {
+ Self {
+ allow_override: false,
+ }
+ }
+}
+
+impl FileSource {
+ /// Create a `FileSource` using the provided [`FileFetcher`](../trait.FileFetcher.html).
+ pub fn new(
+ name: String,
+ metasource: Option<String>,
+ locales: Vec<LanguageIdentifier>,
+ pre_path: String,
+ options: FileSourceOptions,
+ fetcher: impl FileFetcher + 'static,
+ ) -> Self {
+ FileSource {
+ name,
+ metasource: metasource.unwrap_or_default(),
+ pre_path,
+ locales,
+ index: None,
+ shared: Rc::new(Inner {
+ entries: RefCell::new(FxHashMap::default()),
+ fetcher: Box::new(fetcher),
+ error_reporter: None,
+ }),
+ options,
+ }
+ }
+
+ pub fn new_with_index(
+ name: String,
+ metasource: Option<String>,
+ locales: Vec<LanguageIdentifier>,
+ pre_path: String,
+ options: FileSourceOptions,
+ fetcher: impl FileFetcher + 'static,
+ index: Vec<String>,
+ ) -> Self {
+ FileSource {
+ name,
+ metasource: metasource.unwrap_or_default(),
+ pre_path,
+ locales,
+ index: Some(index),
+ shared: Rc::new(Inner {
+ entries: RefCell::new(FxHashMap::default()),
+ fetcher: Box::new(fetcher),
+ error_reporter: None,
+ }),
+ options,
+ }
+ }
+
+ pub fn set_reporter(&mut self, reporter: impl ErrorReporter + 'static) {
+ let mut shared = Rc::get_mut(&mut self.shared).unwrap();
+ shared.error_reporter = Some(RefCell::new(Box::new(reporter)));
+ }
+}
+
+fn calculate_pos_in_source(source: &str, idx: usize) -> (usize, usize) {
+ let mut ptr = 0;
+ let mut result = (1, 1);
+ for line in source.lines() {
+ let bytes = line.as_bytes().len();
+ if ptr + bytes < idx {
+ ptr += bytes + 1;
+ result.0 += 1;
+ } else {
+ result.1 = idx - ptr + 1;
+ break;
+ }
+ }
+ result
+}
+
+impl FileSource {
+ fn get_path(&self, locale: &LanguageIdentifier, resource_id: &ResourceId) -> String {
+ format!(
+ "{}{}",
+ self.pre_path.replace("{locale}", &locale.to_string()),
+ resource_id.value,
+ )
+ }
+
+ fn fetch_sync(&self, resource_id: &ResourceId) -> ResourceOption {
+ self.shared
+ .fetcher
+ .fetch_sync(resource_id)
+ .ok()
+ .map(|source| match FluentResource::try_new(source) {
+ Ok(res) => ResourceOption::Some(Rc::new(res)),
+ Err((res, errors)) => {
+ if let Some(reporter) = &self.shared.error_reporter {
+ reporter.borrow().report_errors(
+ errors
+ .into_iter()
+ .map(|e| L10nRegistryError::FluentError {
+ resource_id: resource_id.clone(),
+ loc: Some(calculate_pos_in_source(res.source(), e.pos.start)),
+ error: e.into(),
+ })
+ .collect(),
+ );
+ }
+ ResourceOption::Some(Rc::new(res))
+ }
+ })
+ .unwrap_or_else(|| ResourceOption::missing_resource(resource_id))
+ }
+
+ /// Attempt to synchronously fetch resource for the combination of `locale`
+ /// and `path`. Returns `Some(ResourceResult)` if the resource is available,
+ /// else `None`.
+ pub fn fetch_file_sync(
+ &self,
+ locale: &LanguageIdentifier,
+ resource_id: &ResourceId,
+ overload: bool,
+ ) -> ResourceOption {
+ use ResourceStatus::*;
+
+ if self.has_file(locale, resource_id) == Some(false) {
+ return ResourceOption::missing_resource(resource_id);
+ }
+
+ let full_path_id = self
+ .get_path(locale, resource_id)
+ .to_resource_id(resource_id.resource_type);
+
+ let res = self.shared.lookup_resource(full_path_id.clone(), || {
+ self.fetch_sync(&full_path_id).into()
+ });
+
+ match res {
+ MissingRequired => ResourceOption::MissingRequired,
+ MissingOptional => ResourceOption::MissingOptional,
+ Loaded(res) => ResourceOption::Some(res),
+ Loading(..) if overload => {
+ // A sync load has been requested for the same resource that has
+ // a pending async load in progress. How do we handle this?
+ //
+ // Ideally, we would sync load and resolve all the pending
+ // futures with the result. With the current Futures and
+ // combinators, it's unclear how to proceed. One potential
+ // solution is to store a oneshot::Sender and
+ // Shared<oneshot::Receiver>. When the async loading future
+ // resolves it would check that the state is still `Loading`,
+ // and if so, send the result. The sync load would do the same
+ // send on the oneshot::Sender.
+ //
+ // For now, we warn and return the resource, paying the cost of
+ // duplication of the resource.
+ self.fetch_sync(&full_path_id)
+ }
+ Loading(..) => {
+ panic!("[l10nregistry] Attempting to synchronously load file {} while it's being loaded asynchronously.", &full_path_id.value);
+ }
+ }
+ }
+
+ /// Attempt to fetch resource for the combination of `locale` and `path`.
+ /// Returns [`ResourceStatus`](enum.ResourceStatus.html) which is
+ /// a `Future` that can be polled.
+ pub fn fetch_file(
+ &self,
+ locale: &LanguageIdentifier,
+ resource_id: &ResourceId,
+ ) -> ResourceStatus {
+ use ResourceStatus::*;
+
+ if self.has_file(locale, resource_id) == Some(false) {
+ return ResourceOption::missing_resource(resource_id).into();
+ }
+
+ let full_path_id = self
+ .get_path(locale, resource_id)
+ .to_resource_id(resource_id.resource_type);
+
+ self.shared.lookup_resource(full_path_id.clone(), || {
+ let shared = self.shared.clone();
+ Loading(read_resource(full_path_id, shared).boxed_local().shared())
+ })
+ }
+
+ /// Determine if the `FileSource` has a loaded resource for the combination
+ /// of `locale` and `path`. Returns `Some(true)` if the file is loaded, else
+ /// `Some(false)`. `None` is returned if there is an outstanding async fetch
+ /// pending and the status is yet to be determined.
+ pub fn has_file<L: Borrow<LanguageIdentifier>>(
+ &self,
+ locale: L,
+ path: &ResourceId,
+ ) -> Option<bool> {
+ let locale = locale.borrow();
+ if !self.locales.contains(locale) {
+ Some(false)
+ } else {
+ let full_path = self.get_path(locale, path);
+ if let Some(index) = &self.index {
+ return Some(index.iter().any(|p| p == &full_path));
+ }
+ self.shared.has_file(&full_path)
+ }
+ }
+
+ pub fn locales(&self) -> &[LanguageIdentifier] {
+ &self.locales
+ }
+
+ pub fn get_index(&self) -> Option<&Vec<String>> {
+ self.index.as_ref()
+ }
+}
+
+impl std::fmt::Debug for FileSource {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result {
+ if let Some(index) = &self.index {
+ f.debug_struct("FileSource")
+ .field("name", &self.name)
+ .field("metasource", &self.metasource)
+ .field("locales", &self.locales)
+ .field("pre_path", &self.pre_path)
+ .field("index", index)
+ .finish()
+ } else {
+ f.debug_struct("FileSource")
+ .field("name", &self.name)
+ .field("metasource", &self.metasource)
+ .field("locales", &self.locales)
+ .field("pre_path", &self.pre_path)
+ .finish()
+ }
+ }
+}
+
+impl Inner {
+ fn lookup_resource<F>(&self, resource_id: ResourceId, f: F) -> ResourceStatus
+ where
+ F: FnOnce() -> ResourceStatus,
+ {
+ let mut lock = self.entries.borrow_mut();
+ lock.entry(resource_id.value).or_insert_with(|| f()).clone()
+ }
+
+ fn update_resource(&self, resource_id: ResourceId, resource: ResourceOption) -> ResourceOption {
+ let mut lock = self.entries.borrow_mut();
+ let entry = lock.get_mut(&resource_id.value);
+ match entry {
+ Some(entry) => *entry = resource.clone().into(),
+ _ => panic!("Expected "),
+ }
+ resource
+ }
+
+ pub fn has_file(&self, full_path: &str) -> Option<bool> {
+ match self.entries.borrow().get(full_path) {
+ Some(ResourceStatus::MissingRequired) => Some(false),
+ Some(ResourceStatus::MissingOptional) => Some(false),
+ Some(ResourceStatus::Loaded(_)) => Some(true),
+ Some(ResourceStatus::Loading(_)) | None => None,
+ }
+ }
+}
+
+async fn read_resource(resource_id: ResourceId, shared: Rc<Inner>) -> ResourceOption {
+ let resource = shared
+ .fetcher
+ .fetch(&resource_id)
+ .await
+ .ok()
+ .map(|source| match FluentResource::try_new(source) {
+ Ok(res) => ResourceOption::Some(Rc::new(res)),
+ Err((res, errors)) => {
+ if let Some(reporter) = &shared.error_reporter.borrow() {
+ reporter.borrow().report_errors(
+ errors
+ .into_iter()
+ .map(|e| L10nRegistryError::FluentError {
+ resource_id: resource_id.clone(),
+ loc: Some(calculate_pos_in_source(res.source(), e.pos.start)),
+ error: e.into(),
+ })
+ .collect(),
+ );
+ }
+ ResourceOption::Some(Rc::new(res))
+ }
+ })
+ .unwrap_or_else(|| ResourceOption::missing_resource(&resource_id));
+ // insert the resource into the cache
+ shared.update_resource(resource_id, resource)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn calculate_source_pos() {
+ let source = r#"
+key = Value
+
+key2 = Value 2
+"#
+ .trim();
+ let result = calculate_pos_in_source(source, 0);
+ assert_eq!(result, (1, 1));
+
+ let result = calculate_pos_in_source(source, 1);
+ assert_eq!(result, (1, 2));
+
+ let result = calculate_pos_in_source(source, 12);
+ assert_eq!(result, (2, 1));
+
+ let result = calculate_pos_in_source(source, 13);
+ assert_eq!(result, (3, 1));
+ }
+}
+
+#[cfg(test)]
+#[cfg(all(feature = "tokio", feature = "test-fluent"))]
+mod tests_tokio {
+ use super::*;
+ use crate::testing::TestFileFetcher;
+
+ static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl";
+ static FTL_RESOURCE_MISSING: &str = "missing.ftl";
+
+ #[tokio::test]
+ async fn file_source_fetch() {
+ let fetcher = TestFileFetcher::new();
+ let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+ let fs1 =
+ fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");
+
+ let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await;
+ assert!(file.is_some());
+ }
+
+ #[tokio::test]
+ async fn file_source_fetch_missing() {
+ let fetcher = TestFileFetcher::new();
+ let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+ let fs1 =
+ fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");
+
+ let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await;
+ assert!(file.is_none());
+ }
+
+ #[tokio::test]
+ async fn file_source_already_loaded() {
+ let fetcher = TestFileFetcher::new();
+ let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+ let fs1 =
+ fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");
+
+ let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await;
+ assert!(file.is_some());
+ let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await;
+ assert!(file.is_some());
+ }
+
+ #[tokio::test]
+ async fn file_source_concurrent() {
+ let fetcher = TestFileFetcher::new();
+ let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+ let fs1 =
+ fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");
+
+ let file1 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into());
+ let file2 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into());
+ assert!(file1.await.is_some());
+ assert!(file2.await.is_some());
+ }
+
+ #[test]
+ fn file_source_sync_after_async_fail() {
+ let fetcher = TestFileFetcher::new();
+ let en_us: LanguageIdentifier = "en-US".parse().unwrap();
+ let fs1 =
+ fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");
+
+ let _ = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into());
+ let file2 = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), true);
+ assert!(file2.is_some());
+ }
+}