path: root/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs

mod fetcher;
pub use fetcher::FileFetcher;
pub use fluent_fallback::types::{ResourceId, ToResourceId};

use crate::env::ErrorReporter;
use crate::errors::L10nRegistryError;
use crate::fluent::FluentResource;

use std::{
    borrow::Borrow,
    cell::RefCell,
    fmt,
    hash::{Hash, Hasher},
    pin::Pin,
    rc::Rc,
    task::Poll,
};

use futures::{future::Shared, Future, FutureExt};
use rustc_hash::FxHashMap;
use unic_langid::LanguageIdentifier;

pub type RcResource = Rc<FluentResource>;

/// An option type whose None variant is either optional or required.
///
/// This behaves similarly to the standard-library [`Option`] type
/// except that there are two [`None`]-like variants:
/// [`ResourceOption::MissingOptional`] and [`ResourceOption::MissingRequired`].
#[derive(Clone, Debug)]
pub enum ResourceOption {
    /// An available resource.
    Some(RcResource),
    /// A missing optional resource.
    MissingOptional,
    /// A missing required resource.
    MissingRequired,
}

impl ResourceOption {
    /// Creates a resource option that is either [`ResourceOption::MissingRequired`]
    /// or [`ResourceOption::MissingOptional`] based on whether the given [`ResourceId`]
    /// is required or optional.
    pub fn missing_resource(resource_id: &ResourceId) -> Self {
        if resource_id.is_required() {
            Self::MissingRequired
        } else {
            Self::MissingOptional
        }
    }

    /// Returns [`true`] if this option contains a recource, otherwise [`false`].
    pub fn is_some(&self) -> bool {
        matches!(self, Self::Some(_))
    }

    /// Resource [`true`] if this option is missing a resource of any type, otherwise [`false`].
    pub fn is_none(&self) -> bool {
        matches!(self, Self::MissingOptional | Self::MissingRequired)
    }

    /// Returns [`true`] if this option is missing a required resource, otherwise [`false`].
    pub fn is_required_and_missing(&self) -> bool {
        matches!(self, Self::MissingRequired)
    }
}

impl From<ResourceOption> for Option<RcResource> {
    fn from(other: ResourceOption) -> Self {
        match other {
            ResourceOption::Some(id) => Some(id),
            _ => None,
        }
    }
}

pub type ResourceFuture = Shared<Pin<Box<dyn Future<Output = ResourceOption>>>>;

#[derive(Debug, Clone)]
pub enum ResourceStatus {
    /// The resource is missing.  Don't bother trying to fetch.
    MissingRequired,
    MissingOptional,
    /// The resource is loading and future will deliver the result.
    Loading(ResourceFuture),
    /// The resource is loaded and parsed.
    Loaded(RcResource),
}

impl From<ResourceOption> for ResourceStatus {
    fn from(input: ResourceOption) -> Self {
        match input {
            ResourceOption::Some(res) => Self::Loaded(res),
            ResourceOption::MissingOptional => Self::MissingOptional,
            ResourceOption::MissingRequired => Self::MissingRequired,
        }
    }
}

impl Future for ResourceStatus {
    type Output = ResourceOption;

    fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
        use ResourceStatus::*;

        let this = &mut *self;

        match this {
            MissingRequired => ResourceOption::MissingRequired.into(),
            MissingOptional => ResourceOption::MissingOptional.into(),
            Loaded(res) => ResourceOption::Some(res.clone()).into(),
            Loading(res) => Pin::new(res).poll(cx),
        }
    }
}

/// `FileSource` provides a generic fetching and caching of fluent resources.
/// The user of `FileSource` provides a [`FileFetcher`](trait.FileFetcher.html)
/// implementation and `FileSource` takes care of the rest.
#[derive(Clone)]
pub struct FileSource {
    /// Name of the FileSource, e.g. "browser"
    pub name: String,
    /// Pre-formatted path for the FileSource, e.g. "/browser/data/locale/{locale}/"
    pub pre_path: String,
    /// Metasource name for the FileSource, e.g. "app", "langpack"
    /// Only sources from the same metasource are passed into the solver.
    pub metasource: String,
    /// The locales for which data is present in the FileSource, e.g. ["en-US", "pl"]
    locales: Vec<LanguageIdentifier>,
    shared: Rc<Inner>,
    index: Option<Vec<String>>,
    pub options: FileSourceOptions,
}

struct Inner {
    fetcher: Box<dyn FileFetcher>,
    error_reporter: Option<RefCell<Box<dyn ErrorReporter>>>,
    entries: RefCell<FxHashMap<String, ResourceStatus>>,
}

impl fmt::Display for FileSource {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        write!(f, "{}", self.name)
    }
}

impl PartialEq<FileSource> for FileSource {
    fn eq(&self, other: &Self) -> bool {
        self.name == other.name && self.metasource == other.metasource
    }
}

impl Eq for FileSource {}

impl Hash for FileSource {
    fn hash<H: Hasher>(&self, state: &mut H) {
        self.name.hash(state)
    }
}

#[derive(PartialEq, Clone, Debug)]
pub struct FileSourceOptions {
    pub allow_override: bool,
}

impl Default for FileSourceOptions {
    fn default() -> Self {
        Self {
            allow_override: false,
        }
    }
}

impl FileSource {
    /// Create a `FileSource` using the provided [`FileFetcher`](../trait.FileFetcher.html).
    pub fn new(
        name: String,
        metasource: Option<String>,
        locales: Vec<LanguageIdentifier>,
        pre_path: String,
        options: FileSourceOptions,
        fetcher: impl FileFetcher + 'static,
    ) -> Self {
        FileSource {
            name,
            metasource: metasource.unwrap_or_default(),
            pre_path,
            locales,
            index: None,
            shared: Rc::new(Inner {
                entries: RefCell::new(FxHashMap::default()),
                fetcher: Box::new(fetcher),
                error_reporter: None,
            }),
            options,
        }
    }

    pub fn new_with_index(
        name: String,
        metasource: Option<String>,
        locales: Vec<LanguageIdentifier>,
        pre_path: String,
        options: FileSourceOptions,
        fetcher: impl FileFetcher + 'static,
        index: Vec<String>,
    ) -> Self {
        FileSource {
            name,
            metasource: metasource.unwrap_or_default(),
            pre_path,
            locales,
            index: Some(index),
            shared: Rc::new(Inner {
                entries: RefCell::new(FxHashMap::default()),
                fetcher: Box::new(fetcher),
                error_reporter: None,
            }),
            options,
        }
    }

    pub fn set_reporter(&mut self, reporter: impl ErrorReporter + 'static) {
        let mut shared = Rc::get_mut(&mut self.shared).unwrap();
        shared.error_reporter = Some(RefCell::new(Box::new(reporter)));
    }
}

fn calculate_pos_in_source(source: &str, idx: usize) -> (usize, usize) {
    let mut ptr = 0;
    let mut result = (1, 1);
    for line in source.lines() {
        let bytes = line.as_bytes().len();
        if ptr + bytes < idx {
            ptr += bytes + 1;
            result.0 += 1;
        } else {
            result.1 = idx - ptr + 1;
            break;
        }
    }
    result
}

impl FileSource {
    fn get_path(&self, locale: &LanguageIdentifier, resource_id: &ResourceId) -> String {
        format!(
            "{}{}",
            self.pre_path.replace("{locale}", &locale.to_string()),
            resource_id.value,
        )
    }

    fn fetch_sync(&self, resource_id: &ResourceId) -> ResourceOption {
        self.shared
            .fetcher
            .fetch_sync(resource_id)
            .ok()
            .map(|source| match FluentResource::try_new(source) {
                Ok(res) => ResourceOption::Some(Rc::new(res)),
                Err((res, errors)) => {
                    if let Some(reporter) = &self.shared.error_reporter {
                        reporter.borrow().report_errors(
                            errors
                                .into_iter()
                                .map(|e| L10nRegistryError::FluentError {
                                    resource_id: resource_id.clone(),
                                    loc: Some(calculate_pos_in_source(res.source(), e.pos.start)),
                                    error: e.into(),
                                })
                                .collect(),
                        );
                    }
                    ResourceOption::Some(Rc::new(res))
                }
            })
            .unwrap_or_else(|| ResourceOption::missing_resource(resource_id))
    }

    /// Attempt to synchronously fetch resource for the combination of `locale`
    /// and `path`. Returns `Some(ResourceResult)` if the resource is available,
    /// else `None`.
    pub fn fetch_file_sync(
        &self,
        locale: &LanguageIdentifier,
        resource_id: &ResourceId,
        overload: bool,
    ) -> ResourceOption {
        use ResourceStatus::*;

        if self.has_file(locale, resource_id) == Some(false) {
            return ResourceOption::missing_resource(resource_id);
        }

        let full_path_id = self
            .get_path(locale, resource_id)
            .to_resource_id(resource_id.resource_type);

        let res = self.shared.lookup_resource(full_path_id.clone(), || {
            self.fetch_sync(&full_path_id).into()
        });

        match res {
            MissingRequired => ResourceOption::MissingRequired,
            MissingOptional => ResourceOption::MissingOptional,
            Loaded(res) => ResourceOption::Some(res),
            Loading(..) if overload => {
                // A sync load has been requested for the same resource that has
                // a pending async load in progress. How do we handle this?
                //
                // Ideally, we would sync load and resolve all the pending
                // futures with the result. With the current Futures and
                // combinators, it's unclear how to proceed. One potential
                // solution is to store a oneshot::Sender and
                // Shared<oneshot::Receiver>. When the async loading future
                // resolves it would check that the state is still `Loading`,
                // and if so, send the result. The sync load would do the same
                // send on the oneshot::Sender.
                //
                // For now, we warn and return the resource, paying the cost of
                // duplication of the resource.
                self.fetch_sync(&full_path_id)
            }
            Loading(..) => {
                panic!("[l10nregistry] Attempting to synchronously load file {} while it's being loaded asynchronously.", &full_path_id.value);
            }
        }
    }

    /// Attempt to fetch resource for the combination of `locale` and `path`.
    /// Returns [`ResourceStatus`](enum.ResourceStatus.html) which is
    /// a `Future` that can be polled.
    pub fn fetch_file(
        &self,
        locale: &LanguageIdentifier,
        resource_id: &ResourceId,
    ) -> ResourceStatus {
        use ResourceStatus::*;

        if self.has_file(locale, resource_id) == Some(false) {
            return ResourceOption::missing_resource(resource_id).into();
        }

        let full_path_id = self
            .get_path(locale, resource_id)
            .to_resource_id(resource_id.resource_type);

        self.shared.lookup_resource(full_path_id.clone(), || {
            let shared = self.shared.clone();
            Loading(read_resource(full_path_id, shared).boxed_local().shared())
        })
    }

    /// Determine if the `FileSource` has a loaded resource for the combination
    /// of `locale` and `path`. Returns `Some(true)` if the file is loaded, else
    /// `Some(false)`. `None` is returned if there is an outstanding async fetch
    /// pending and the status is yet to be determined.
    pub fn has_file<L: Borrow<LanguageIdentifier>>(
        &self,
        locale: L,
        path: &ResourceId,
    ) -> Option<bool> {
        let locale = locale.borrow();
        if !self.locales.contains(locale) {
            Some(false)
        } else {
            let full_path = self.get_path(locale, path);
            if let Some(index) = &self.index {
                return Some(index.iter().any(|p| p == &full_path));
            }
            self.shared.has_file(&full_path)
        }
    }

    pub fn locales(&self) -> &[LanguageIdentifier] {
        &self.locales
    }

    pub fn get_index(&self) -> Option<&Vec<String>> {
        self.index.as_ref()
    }
}

impl std::fmt::Debug for FileSource {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result {
        if let Some(index) = &self.index {
            f.debug_struct("FileSource")
                .field("name", &self.name)
                .field("metasource", &self.metasource)
                .field("locales", &self.locales)
                .field("pre_path", &self.pre_path)
                .field("index", index)
                .finish()
        } else {
            f.debug_struct("FileSource")
                .field("name", &self.name)
                .field("metasource", &self.metasource)
                .field("locales", &self.locales)
                .field("pre_path", &self.pre_path)
                .finish()
        }
    }
}

impl Inner {
    fn lookup_resource<F>(&self, resource_id: ResourceId, f: F) -> ResourceStatus
    where
        F: FnOnce() -> ResourceStatus,
    {
        let mut lock = self.entries.borrow_mut();
        lock.entry(resource_id.value).or_insert_with(|| f()).clone()
    }

    fn update_resource(&self, resource_id: ResourceId, resource: ResourceOption) -> ResourceOption {
        let mut lock = self.entries.borrow_mut();
        let entry = lock.get_mut(&resource_id.value);
        match entry {
            Some(entry) => *entry = resource.clone().into(),
            _ => panic!("Expected "),
        }
        resource
    }

    pub fn has_file(&self, full_path: &str) -> Option<bool> {
        match self.entries.borrow().get(full_path) {
            Some(ResourceStatus::MissingRequired) => Some(false),
            Some(ResourceStatus::MissingOptional) => Some(false),
            Some(ResourceStatus::Loaded(_)) => Some(true),
            Some(ResourceStatus::Loading(_)) | None => None,
        }
    }
}

async fn read_resource(resource_id: ResourceId, shared: Rc<Inner>) -> ResourceOption {
    let resource = shared
        .fetcher
        .fetch(&resource_id)
        .await
        .ok()
        .map(|source| match FluentResource::try_new(source) {
            Ok(res) => ResourceOption::Some(Rc::new(res)),
            Err((res, errors)) => {
                if let Some(reporter) = &shared.error_reporter.borrow() {
                    reporter.borrow().report_errors(
                        errors
                            .into_iter()
                            .map(|e| L10nRegistryError::FluentError {
                                resource_id: resource_id.clone(),
                                loc: Some(calculate_pos_in_source(res.source(), e.pos.start)),
                                error: e.into(),
                            })
                            .collect(),
                    );
                }
                ResourceOption::Some(Rc::new(res))
            }
        })
        .unwrap_or_else(|| ResourceOption::missing_resource(&resource_id));
    // insert the resource into the cache
    shared.update_resource(resource_id, resource)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn calculate_source_pos() {
        let source = r#"
key = Value

key2 = Value 2
"#
        .trim();
        let result = calculate_pos_in_source(source, 0);
        assert_eq!(result, (1, 1));

        let result = calculate_pos_in_source(source, 1);
        assert_eq!(result, (1, 2));

        let result = calculate_pos_in_source(source, 12);
        assert_eq!(result, (2, 1));

        let result = calculate_pos_in_source(source, 13);
        assert_eq!(result, (3, 1));
    }
}

#[cfg(test)]
#[cfg(all(feature = "tokio", feature = "test-fluent"))]
mod tests_tokio {
    use super::*;
    use crate::testing::TestFileFetcher;

    static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl";
    static FTL_RESOURCE_MISSING: &str = "missing.ftl";

    #[tokio::test]
    async fn file_source_fetch() {
        let fetcher = TestFileFetcher::new();
        let en_us: LanguageIdentifier = "en-US".parse().unwrap();
        let fs1 =
            fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");

        let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await;
        assert!(file.is_some());
    }

    #[tokio::test]
    async fn file_source_fetch_missing() {
        let fetcher = TestFileFetcher::new();
        let en_us: LanguageIdentifier = "en-US".parse().unwrap();
        let fs1 =
            fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");

        let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await;
        assert!(file.is_none());
    }

    #[tokio::test]
    async fn file_source_already_loaded() {
        let fetcher = TestFileFetcher::new();
        let en_us: LanguageIdentifier = "en-US".parse().unwrap();
        let fs1 =
            fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");

        let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await;
        assert!(file.is_some());
        let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await;
        assert!(file.is_some());
    }

    #[tokio::test]
    async fn file_source_concurrent() {
        let fetcher = TestFileFetcher::new();
        let en_us: LanguageIdentifier = "en-US".parse().unwrap();
        let fs1 =
            fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");

        let file1 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into());
        let file2 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into());
        assert!(file1.await.is_some());
        assert!(file2.await.is_some());
    }

    #[test]
    fn file_source_sync_after_async_fail() {
        let fetcher = TestFileFetcher::new();
        let en_us: LanguageIdentifier = "en-US".parse().unwrap();
        let fs1 =
            fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/");

        let _ = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into());
        let file2 = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), true);
        assert!(file2.is_some());
    }
}