diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-19 00:47:55 +0000 |
commit | 26a029d407be480d791972afb5975cf62c9360a6 (patch) | |
tree | f435a8308119effd964b339f76abb83a57c29483 /intl/l10n/rust | |
parent | Initial commit. (diff) | |
download | firefox-26a029d407be480d791972afb5975cf62c9360a6.tar.xz firefox-26a029d407be480d791972afb5975cf62c9360a6.zip |
Adding upstream version 124.0.1.upstream/124.0.1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'intl/l10n/rust')
57 files changed, 7828 insertions, 0 deletions
diff --git a/intl/l10n/rust/fluent-ffi/Cargo.toml b/intl/l10n/rust/fluent-ffi/Cargo.toml new file mode 100644 index 0000000000..1f922822d7 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "fluent-ffi" +version = "0.1.0" +authors = ["Zibi Braniecki <zibi@braniecki.net>"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +fluent = { version = "0.16.0", features = ["fluent-pseudo"] } +fluent-syntax = "0.11.0" +fluent-pseudo = "0.3.1" +intl-memoizer = "0.5.1" +unic-langid = "0.9" +nsstring = { path = "../../../../xpcom/rust/nsstring" } +cstr = "0.2" +xpcom = { path = "../../../../xpcom/rust/xpcom" } +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } diff --git a/intl/l10n/rust/fluent-ffi/cbindgen.toml b/intl/l10n/rust/fluent-ffi/cbindgen.toml new file mode 100644 index 0000000000..5384a81b0a --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/cbindgen.toml @@ -0,0 +1,24 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef mozilla_intl_l10n_FluentBindings_h +#error "Don't include this file directly, instead include FluentBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +namespaces = ["mozilla", "intl", "ffi"] + +[parse] +parse_deps = true +include = ["fluent", "fluent-bundle", "intl-memoizer"] + +[enum] +derive_helper_methods = true + +[export.rename] +"ThinVec" = "nsTArray" diff --git a/intl/l10n/rust/fluent-ffi/src/builtins.rs b/intl/l10n/rust/fluent-ffi/src/builtins.rs new file mode 100644 index 0000000000..c7ffe8c3ee --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/builtins.rs @@ -0,0 +1,389 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::ffi; +use fluent::types::{FluentNumberOptions, FluentType, FluentValue}; +use fluent::FluentArgs; +use intl_memoizer::IntlLangMemoizer; +use intl_memoizer::Memoizable; +use nsstring::nsCString; +use std::borrow::Cow; +use std::ptr::NonNull; +use unic_langid::LanguageIdentifier; + +pub struct NumberFormat { + raw: Option<NonNull<ffi::RawNumberFormatter>>, +} + +/** + * According to http://userguide.icu-project.org/design, as long as we constrain + * ourselves to const APIs ICU is const-correct. + */ +unsafe impl Send for NumberFormat {} +unsafe impl Sync for NumberFormat {} + +impl NumberFormat { + pub fn new(locale: LanguageIdentifier, options: &FluentNumberOptions) -> Self { + let loc: String = locale.to_string(); + Self { + raw: unsafe { + NonNull::new(ffi::FluentBuiltInNumberFormatterCreate( + &loc.into(), + &options.into(), + )) + }, + } + } + + pub fn format(&self, input: f64) -> String { + if let Some(raw) = self.raw { + unsafe { + let mut byte_count = 0; + let mut capacity = 0; + let buffer = ffi::FluentBuiltInNumberFormatterFormat( + raw.as_ptr(), + input, + &mut byte_count, + &mut capacity, + ); + if buffer.is_null() { + return String::new(); + } + String::from_raw_parts(buffer, byte_count, capacity) + } + } else { + String::new() + } + } +} + +impl Drop for NumberFormat { + fn drop(&mut self) { + if let Some(raw) = self.raw { + unsafe { ffi::FluentBuiltInNumberFormatterDestroy(raw.as_ptr()) }; + } + } +} + +impl Memoizable for NumberFormat { + type Args = (FluentNumberOptions,); + type Error = &'static str; + fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> { + Ok(NumberFormat::new(lang, &args.0)) + } +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub enum FluentDateTimeStyle { + Full, + Long, + Medium, + Short, + None, +} + +impl Default for FluentDateTimeStyle { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeStyle { + fn from(input: &str) -> Self { + match input { + "full" => Self::Full, + "long" => Self::Long, + "medium" => Self::Medium, + "short" => Self::Short, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeHourCycle { + H24, + H23, + H12, + H11, + None, +} + +impl Default for FluentDateTimeHourCycle { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeHourCycle { + fn from(input: &str) -> Self { + match input { + "h24" => Self::H24, + "h23" => Self::H23, + "h12" => Self::H12, + "h11" => Self::H11, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeTextComponent { + Long, + Short, + Narrow, + None, +} + +impl Default for FluentDateTimeTextComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeTextComponent { + fn from(input: &str) -> Self { + match input { + "long" => Self::Long, + "short" => Self::Short, + "narrow" => Self::Narrow, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeNumericComponent { + Numeric, + TwoDigit, + None, +} + +impl Default for FluentDateTimeNumericComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeNumericComponent { + fn from(input: &str) -> Self { + match input { + "numeric" => Self::Numeric, + "2-digit" => Self::TwoDigit, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeMonthComponent { + Numeric, + TwoDigit, + Long, + Short, + Narrow, + None, +} + +impl Default for FluentDateTimeMonthComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeMonthComponent { + fn from(input: &str) -> Self { + match input { + "numeric" => Self::Numeric, + "2-digit" => Self::TwoDigit, + "long" => Self::Long, + "short" => Self::Short, + "narrow" => Self::Narrow, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeTimeZoneNameComponent { + Long, + Short, + None, +} + +impl Default for FluentDateTimeTimeZoneNameComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeTimeZoneNameComponent { + fn from(input: &str) -> Self { + match input { + "long" => Self::Long, + "short" => Self::Short, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Default, Debug, Clone, Hash, PartialEq, Eq)] +pub struct FluentDateTimeOptions { + pub date_style: FluentDateTimeStyle, + pub time_style: FluentDateTimeStyle, + pub hour_cycle: FluentDateTimeHourCycle, + pub weekday: FluentDateTimeTextComponent, + pub era: FluentDateTimeTextComponent, + pub year: FluentDateTimeNumericComponent, + pub month: FluentDateTimeMonthComponent, + pub day: FluentDateTimeNumericComponent, + pub hour: FluentDateTimeNumericComponent, + pub minute: FluentDateTimeNumericComponent, + pub second: FluentDateTimeNumericComponent, + pub time_zone_name: FluentDateTimeTimeZoneNameComponent, +} + +impl FluentDateTimeOptions { + pub fn merge(&mut self, opts: &FluentArgs) { + for (key, value) in opts.iter() { + match (key, value) { + ("dateStyle", FluentValue::String(n)) => { + self.date_style = n.as_ref().into(); + } + ("timeStyle", FluentValue::String(n)) => { + self.time_style = n.as_ref().into(); + } + ("hourCycle", FluentValue::String(n)) => { + self.hour_cycle = n.as_ref().into(); + } + ("weekday", FluentValue::String(n)) => { + self.weekday = n.as_ref().into(); + } + ("era", FluentValue::String(n)) => { + self.era = n.as_ref().into(); + } + ("year", FluentValue::String(n)) => { + self.year = n.as_ref().into(); + } + ("month", FluentValue::String(n)) => { + self.month = n.as_ref().into(); + } + ("day", FluentValue::String(n)) => { + self.day = n.as_ref().into(); + } + ("hour", FluentValue::String(n)) => { + self.hour = n.as_ref().into(); + } + ("minute", FluentValue::String(n)) => { + self.minute = n.as_ref().into(); + } + ("second", FluentValue::String(n)) => { + self.second = n.as_ref().into(); + } + ("timeZoneName", FluentValue::String(n)) => { + self.time_zone_name = n.as_ref().into(); + } + _ => {} + } + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct FluentDateTime { + epoch: f64, + options: FluentDateTimeOptions, +} + +impl FluentType for FluentDateTime { + fn duplicate(&self) -> Box<dyn FluentType + Send> { + Box::new(self.clone()) + } + fn as_string(&self, intls: &IntlLangMemoizer) -> Cow<'static, str> { + let result = intls + .with_try_get::<DateTimeFormat, _, _>((self.options.clone(),), |dtf| { + dtf.format(self.epoch) + }) + .expect("Failed to retrieve a DateTimeFormat instance."); + result.into() + } + fn as_string_threadsafe( + &self, + _: &intl_memoizer::concurrent::IntlLangMemoizer, + ) -> Cow<'static, str> { + unimplemented!() + } +} + +impl std::fmt::Display for FluentDateTime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DATETIME: {}", self.epoch) + } +} + +impl FluentDateTime { + pub fn new(epoch: f64, options: FluentDateTimeOptions) -> Self { + Self { epoch, options } + } +} + +pub struct DateTimeFormat { + raw: Option<NonNull<ffi::RawDateTimeFormatter>>, +} + +/** + * According to http://userguide.icu-project.org/design, as long as we constrain + * ourselves to const APIs ICU is const-correct. + */ +unsafe impl Send for DateTimeFormat {} +unsafe impl Sync for DateTimeFormat {} + +impl DateTimeFormat { + pub fn new(locale: LanguageIdentifier, options: FluentDateTimeOptions) -> Self { + // ICU needs null-termination here, otherwise we could use nsCStr. + let loc: nsCString = locale.to_string().into(); + Self { + raw: unsafe { NonNull::new(ffi::FluentBuiltInDateTimeFormatterCreate(&loc, options)) }, + } + } + + pub fn format(&self, input: f64) -> String { + if let Some(raw) = self.raw { + unsafe { + let mut byte_count = 0; + let buffer = + ffi::FluentBuiltInDateTimeFormatterFormat(raw.as_ptr(), input, &mut byte_count); + if buffer.is_null() { + return String::new(); + } + String::from_raw_parts(buffer, byte_count as usize, byte_count as usize) + } + } else { + String::new() + } + } +} + +impl Drop for DateTimeFormat { + fn drop(&mut self) { + if let Some(raw) = self.raw { + unsafe { ffi::FluentBuiltInDateTimeFormatterDestroy(raw.as_ptr()) }; + } + } +} + +impl Memoizable for DateTimeFormat { + type Args = (FluentDateTimeOptions,); + type Error = &'static str; + fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> { + Ok(DateTimeFormat::new(lang, args.0)) + } +} diff --git a/intl/l10n/rust/fluent-ffi/src/bundle.rs b/intl/l10n/rust/fluent-ffi/src/bundle.rs new file mode 100644 index 0000000000..21bf0d52e9 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/bundle.rs @@ -0,0 +1,331 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::builtins::{FluentDateTime, FluentDateTimeOptions, NumberFormat}; +use cstr::cstr; +pub use fluent::{FluentArgs, FluentBundle, FluentError, FluentResource, FluentValue}; +use fluent_pseudo::transform_dom; +pub use intl_memoizer::IntlLangMemoizer; +use nsstring::{nsACString, nsCString}; +use std::borrow::Cow; +use std::ffi::CStr; +use std::mem; +use std::rc::Rc; +use thin_vec::ThinVec; +use unic_langid::LanguageIdentifier; +use xpcom::interfaces::nsIPrefBranch; + +pub type FluentBundleRc = FluentBundle<Rc<FluentResource>>; + +#[derive(Debug)] +#[repr(C, u8)] +pub enum FluentArgument<'s> { + Double_(f64), + String(&'s nsACString), +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nArg<'s> { + pub id: &'s nsACString, + pub value: FluentArgument<'s>, +} + +fn transform_accented(s: &str) -> Cow<str> { + transform_dom(s, false, true, true) +} + +fn transform_bidi(s: &str) -> Cow<str> { + transform_dom(s, false, false, false) +} + +fn format_numbers(num: &FluentValue, intls: &IntlLangMemoizer) -> Option<String> { + match num { + FluentValue::Number(n) => { + let result = intls + .with_try_get::<NumberFormat, _, _>((n.options.clone(),), |nf| nf.format(n.value)) + .expect("Failed to retrieve a NumberFormat instance."); + Some(result) + } + _ => None, + } +} + +fn get_string_pref(name: &CStr) -> Option<nsCString> { + let mut value = nsCString::new(); + let prefs_service = + xpcom::get_service::<nsIPrefBranch>(cstr!("@mozilla.org/preferences-service;1"))?; + unsafe { + prefs_service + .GetCharPref(name.as_ptr(), &mut *value) + .to_result() + .ok()?; + } + Some(value) +} + +fn get_bool_pref(name: &CStr) -> Option<bool> { + let mut value = false; + let prefs_service = + xpcom::get_service::<nsIPrefBranch>(cstr!("@mozilla.org/preferences-service;1"))?; + unsafe { + prefs_service + .GetBoolPref(name.as_ptr(), &mut value) + .to_result() + .ok()?; + } + Some(value) +} + +pub fn adapt_bundle_for_gecko(bundle: &mut FluentBundleRc, pseudo_strategy: Option<&nsACString>) { + bundle.set_formatter(Some(format_numbers)); + + bundle + .add_function("PLATFORM", |_args, _named_args| { + if cfg!(target_os = "linux") { + "linux".into() + } else if cfg!(target_os = "windows") { + "windows".into() + } else if cfg!(target_os = "macos") { + "macos".into() + } else if cfg!(target_os = "android") { + "android".into() + } else { + "other".into() + } + }) + .expect("Failed to add a function to the bundle."); + bundle + .add_function("NUMBER", |args, named| { + if let Some(FluentValue::Number(n)) = args.get(0) { + let mut num = n.clone(); + num.options.merge(named); + FluentValue::Number(num) + } else { + FluentValue::None + } + }) + .expect("Failed to add a function to the bundle."); + bundle + .add_function("DATETIME", |args, named| { + if let Some(FluentValue::Number(n)) = args.get(0) { + let mut options = FluentDateTimeOptions::default(); + options.merge(&named); + FluentValue::Custom(Box::new(FluentDateTime::new(n.value, options))) + } else { + FluentValue::None + } + }) + .expect("Failed to add a function to the bundle."); + + enum PseudoStrategy { + Accented, + Bidi, + None, + } + // This is quirky because we can't coerce Option<&nsACString> and Option<nsCString> + // into bytes easily without allocating. + let strategy_kind = match pseudo_strategy.map(|s| &s[..]) { + Some(b"accented") => PseudoStrategy::Accented, + Some(b"bidi") => PseudoStrategy::Bidi, + _ => { + if let Some(pseudo_strategy) = get_string_pref(cstr!("intl.l10n.pseudo")) { + match &pseudo_strategy[..] { + b"accented" => PseudoStrategy::Accented, + b"bidi" => PseudoStrategy::Bidi, + _ => PseudoStrategy::None, + } + } else { + PseudoStrategy::None + } + } + }; + match strategy_kind { + PseudoStrategy::Accented => bundle.set_transform(Some(transform_accented)), + PseudoStrategy::Bidi => bundle.set_transform(Some(transform_bidi)), + PseudoStrategy::None => bundle.set_transform(None), + } + + // Temporarily disable bidi isolation due to Microsoft not supporting FSI/PDI. + // See bug 1439018 for details. + let default_use_isolating = false; + let use_isolating = + get_bool_pref(cstr!("intl.l10n.enable-bidi-marks")).unwrap_or(default_use_isolating); + bundle.set_use_isolating(use_isolating); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_new_single( + locale: &nsACString, + use_isolating: bool, + pseudo_strategy: &nsACString, +) -> *mut FluentBundleRc { + let id = match locale.to_utf8().parse::<LanguageIdentifier>() { + Ok(id) => id, + Err(..) => return std::ptr::null_mut(), + }; + + Box::into_raw(fluent_bundle_new_internal( + &[id], + use_isolating, + pseudo_strategy, + )) +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_new( + locales: *const nsCString, + locale_count: usize, + use_isolating: bool, + pseudo_strategy: &nsACString, +) -> *mut FluentBundleRc { + let mut langids = Vec::with_capacity(locale_count); + let locales = std::slice::from_raw_parts(locales, locale_count); + for locale in locales { + let id = match locale.to_utf8().parse::<LanguageIdentifier>() { + Ok(id) => id, + Err(..) => return std::ptr::null_mut(), + }; + langids.push(id); + } + + Box::into_raw(fluent_bundle_new_internal( + &langids, + use_isolating, + pseudo_strategy, + )) +} + +fn fluent_bundle_new_internal( + langids: &[LanguageIdentifier], + use_isolating: bool, + pseudo_strategy: &nsACString, +) -> Box<FluentBundleRc> { + let mut bundle = FluentBundle::new(langids.to_vec()); + bundle.set_use_isolating(use_isolating); + + bundle.set_formatter(Some(format_numbers)); + + adapt_bundle_for_gecko(&mut bundle, Some(pseudo_strategy)); + + Box::new(bundle) +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_get_locales( + bundle: &FluentBundleRc, + result: &mut ThinVec<nsCString>, +) { + for locale in &bundle.locales { + result.push(locale.to_string().as_str().into()); + } +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_destroy(bundle: *mut FluentBundleRc) { + let _ = Box::from_raw(bundle); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_has_message(bundle: &FluentBundleRc, id: &nsACString) -> bool { + bundle.has_message(id.to_string().as_str()) +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_get_message( + bundle: &FluentBundleRc, + id: &nsACString, + has_value: &mut bool, + attrs: &mut ThinVec<nsCString>, +) -> bool { + match bundle.get_message(&id.to_utf8()) { + Some(message) => { + attrs.reserve(message.attributes().count()); + *has_value = message.value().is_some(); + for attr in message.attributes() { + attrs.push(attr.id().into()); + } + true + } + None => { + *has_value = false; + false + } + } +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_format_pattern( + bundle: &FluentBundleRc, + id: &nsACString, + attr: &nsACString, + args: &ThinVec<L10nArg>, + ret_val: &mut nsACString, + ret_errors: &mut ThinVec<nsCString>, +) -> bool { + let args = convert_args(&args); + + let message = match bundle.get_message(&id.to_utf8()) { + Some(message) => message, + None => return false, + }; + + let pattern = if !attr.is_empty() { + match message.get_attribute(&attr.to_utf8()) { + Some(attr) => attr.value(), + None => return false, + } + } else { + match message.value() { + Some(value) => value, + None => return false, + } + }; + + let mut errors = vec![]; + bundle + .write_pattern(ret_val, pattern, args.as_ref(), &mut errors) + .expect("Failed to write to a nsCString."); + append_fluent_errors_to_ret_errors(ret_errors, &errors); + true +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_add_resource( + bundle: &mut FluentBundleRc, + r: *const FluentResource, + allow_overrides: bool, + ret_errors: &mut ThinVec<nsCString>, +) { + // we don't own the resource + let r = mem::ManuallyDrop::new(Rc::from_raw(r)); + + if allow_overrides { + bundle.add_resource_overriding(Rc::clone(&r)); + } else if let Err(errors) = bundle.add_resource(Rc::clone(&r)) { + append_fluent_errors_to_ret_errors(ret_errors, &errors); + } +} + +pub fn convert_args<'s>(args: &[L10nArg<'s>]) -> Option<FluentArgs<'s>> { + if args.is_empty() { + return None; + } + + let mut result = FluentArgs::with_capacity(args.len()); + for arg in args { + let val = match arg.value { + FluentArgument::Double_(d) => FluentValue::from(d), + FluentArgument::String(s) => FluentValue::from(s.to_utf8()), + }; + result.set(arg.id.to_string(), val); + } + Some(result) +} + +fn append_fluent_errors_to_ret_errors(ret_errors: &mut ThinVec<nsCString>, errors: &[FluentError]) { + for error in errors { + ret_errors.push(error.to_string().into()); + } +} diff --git a/intl/l10n/rust/fluent-ffi/src/ffi.rs b/intl/l10n/rust/fluent-ffi/src/ffi.rs new file mode 100644 index 0000000000..a264ad11b7 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/ffi.rs @@ -0,0 +1,154 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::builtins::FluentDateTimeOptions; +use fluent::types::FluentNumberCurrencyDisplayStyle; +use fluent::types::FluentNumberOptions; +use fluent::types::FluentNumberStyle; +use nsstring::nsCString; + +pub enum RawNumberFormatter {} + +#[repr(C)] +pub enum FluentNumberStyleRaw { + Decimal, + Currency, + Percent, +} + +impl From<FluentNumberStyle> for FluentNumberStyleRaw { + fn from(input: FluentNumberStyle) -> Self { + match input { + FluentNumberStyle::Decimal => Self::Decimal, + FluentNumberStyle::Currency => Self::Currency, + FluentNumberStyle::Percent => Self::Percent, + } + } +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub enum FluentNumberCurrencyDisplayStyleRaw { + Symbol, + Code, + Name, +} + +impl From<FluentNumberCurrencyDisplayStyle> for FluentNumberCurrencyDisplayStyleRaw { + fn from(input: FluentNumberCurrencyDisplayStyle) -> Self { + match input { + FluentNumberCurrencyDisplayStyle::Symbol => Self::Symbol, + FluentNumberCurrencyDisplayStyle::Code => Self::Code, + FluentNumberCurrencyDisplayStyle::Name => Self::Name, + } + } +} + +#[repr(C)] +pub struct FluentNumberOptionsRaw { + pub style: FluentNumberStyleRaw, + pub currency: nsCString, + pub currency_display: FluentNumberCurrencyDisplayStyleRaw, + pub use_grouping: bool, + pub minimum_integer_digits: usize, + pub minimum_fraction_digits: usize, + pub maximum_fraction_digits: usize, + pub minimum_significant_digits: isize, + pub maximum_significant_digits: isize, +} + +fn get_number_option(val: Option<usize>, min: usize, max: usize, default: usize) -> usize { + if let Some(val) = val { + if val >= min && val <= max { + val + } else { + default + } + } else { + default + } +} + +impl From<&FluentNumberOptions> for FluentNumberOptionsRaw { + fn from(input: &FluentNumberOptions) -> Self { + let currency: nsCString = if let Some(ref currency) = input.currency { + currency.into() + } else { + nsCString::new() + }; + + //XXX: This should be fetched from currency table. + let currency_digits = 2; + + // Keep it aligned with ECMA402 NumberFormat logic. + let minfd_default = if input.style == FluentNumberStyle::Currency { + currency_digits + } else { + 0 + }; + let maxfd_default = match input.style { + FluentNumberStyle::Decimal => 3, + FluentNumberStyle::Currency => currency_digits, + FluentNumberStyle::Percent => 0, + }; + let minid = get_number_option(input.minimum_integer_digits, 1, 21, 1); + let minfd = get_number_option(input.minimum_fraction_digits, 0, 20, minfd_default); + let maxfd_actual_default = std::cmp::max(minfd, maxfd_default); + let maxfd = get_number_option( + input.maximum_fraction_digits, + minfd, + 20, + maxfd_actual_default, + ); + + let (minsd, maxsd) = if input.minimum_significant_digits.is_some() + || input.maximum_significant_digits.is_some() + { + let minsd = get_number_option(input.minimum_significant_digits, 1, 21, 1); + let maxsd = get_number_option(input.maximum_significant_digits, minsd, 21, 21); + (minsd as isize, maxsd as isize) + } else { + (-1, -1) + }; + + Self { + style: input.style.into(), + currency, + currency_display: input.currency_display.into(), + use_grouping: input.use_grouping, + minimum_integer_digits: minid, + minimum_fraction_digits: minfd, + maximum_fraction_digits: maxfd, + minimum_significant_digits: minsd, + maximum_significant_digits: maxsd, + } + } +} + +pub enum RawDateTimeFormatter {} + +extern "C" { + pub fn FluentBuiltInNumberFormatterCreate( + locale: &nsCString, + options: &FluentNumberOptionsRaw, + ) -> *mut RawNumberFormatter; + pub fn FluentBuiltInNumberFormatterFormat( + formatter: *const RawNumberFormatter, + input: f64, + out_count: &mut usize, + out_capacity: &mut usize, + ) -> *mut u8; + pub fn FluentBuiltInNumberFormatterDestroy(formatter: *mut RawNumberFormatter); + + pub fn FluentBuiltInDateTimeFormatterCreate( + locale: &nsCString, + options: FluentDateTimeOptions, + ) -> *mut RawDateTimeFormatter; + pub fn FluentBuiltInDateTimeFormatterFormat( + formatter: *const RawDateTimeFormatter, + input: f64, + out_count: &mut u32, + ) -> *mut u8; + pub fn FluentBuiltInDateTimeFormatterDestroy(formatter: *mut RawDateTimeFormatter); +} diff --git a/intl/l10n/rust/fluent-ffi/src/lib.rs b/intl/l10n/rust/fluent-ffi/src/lib.rs new file mode 100644 index 0000000000..bb671f4b17 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/lib.rs @@ -0,0 +1,12 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +mod builtins; +mod bundle; +mod ffi; +mod resource; +mod text_elements; + +pub use bundle::*; +pub use resource::*; diff --git a/intl/l10n/rust/fluent-ffi/src/resource.rs b/intl/l10n/rust/fluent-ffi/src/resource.rs new file mode 100644 index 0000000000..dc011b9462 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/resource.rs @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub use fluent::FluentResource; +use nsstring::nsACString; +use std::{ + mem::{self, ManuallyDrop}, + rc::Rc, +}; + +#[no_mangle] +pub extern "C" fn fluent_resource_new( + name: &nsACString, + has_errors: &mut bool, +) -> *const FluentResource { + let res = match FluentResource::try_new(name.to_string()) { + Ok(res) => { + *has_errors = false; + res + } + Err((res, _)) => { + *has_errors = true; + res + } + }; + Rc::into_raw(Rc::new(res)) +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_resource_addref(res: *const FluentResource) { + let raw = ManuallyDrop::new(Rc::from_raw(res)); + mem::forget(Rc::clone(&raw)); +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_resource_release(res: *const FluentResource) { + let _ = Rc::from_raw(res); +} diff --git a/intl/l10n/rust/fluent-ffi/src/text_elements.rs b/intl/l10n/rust/fluent-ffi/src/text_elements.rs new file mode 100644 index 0000000000..0ffeffd4c7 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/text_elements.rs @@ -0,0 +1,164 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use fluent::FluentResource; +use fluent_syntax::ast; +use nsstring::nsCString; +use thin_vec::ThinVec; + +#[repr(C)] +pub struct TextElementInfo { + id: nsCString, + attr: nsCString, + text: nsCString, +} + +struct TextElementsCollector<'a> { + current_id: Option<String>, + current_attr: Option<String>, + elements: &'a mut ThinVec<TextElementInfo>, +} + +impl<'a> TextElementsCollector<'a> { + pub fn new(elements: &'a mut ThinVec<TextElementInfo>) -> Self { + Self { + current_id: None, + current_attr: None, + elements: elements, + } + } + + fn collect_inline_expression(&mut self, x: &ast::InlineExpression<&str>) { + match x { + ast::InlineExpression::StringLiteral { .. } => {} + ast::InlineExpression::NumberLiteral { .. } => {} + ast::InlineExpression::FunctionReference { arguments, .. } => { + self.collect_call_arguments(arguments); + } + ast::InlineExpression::MessageReference { .. } => {} + ast::InlineExpression::TermReference { arguments, .. } => { + if let Some(y) = arguments { + self.collect_call_arguments(y); + } + } + ast::InlineExpression::VariableReference { .. } => {} + ast::InlineExpression::Placeable { expression } => { + self.collect_expression(expression.as_ref()); + } + } + } + + fn collect_named_argument(&mut self, x: &ast::NamedArgument<&str>) { + self.collect_inline_expression(&x.value); + } + + fn collect_call_arguments(&mut self, x: &ast::CallArguments<&str>) { + for y in x.positional.iter() { + self.collect_inline_expression(y); + } + for y in x.named.iter() { + self.collect_named_argument(y); + } + } + + fn collect_variant(&mut self, x: &ast::Variant<&str>) { + self.collect_pattern(&x.value); + } + + fn collect_expression(&mut self, x: &ast::Expression<&str>) { + match x { + ast::Expression::Select { selector, variants } => { + self.collect_inline_expression(selector); + for y in variants.iter() { + self.collect_variant(y); + } + } + ast::Expression::Inline(i) => { + self.collect_inline_expression(i); + } + } + } + + fn collect_pattern_element(&mut self, x: &ast::PatternElement<&str>) { + match x { + ast::PatternElement::TextElement { value } => { + self.elements.push(TextElementInfo { + id: self + .current_id + .as_ref() + .map_or_else(|| nsCString::new(), nsCString::from), + attr: self + .current_attr + .as_ref() + .map_or_else(|| nsCString::new(), nsCString::from), + text: nsCString::from(*value), + }); + } + ast::PatternElement::Placeable { expression } => { + self.collect_expression(expression); + } + } + } + + fn collect_pattern(&mut self, x: &ast::Pattern<&str>) { + for y in x.elements.iter() { + self.collect_pattern_element(y); + } + } + + fn collect_attribute(&mut self, x: &ast::Attribute<&str>) { + self.current_attr = Some(x.id.name.to_string()); + + self.collect_pattern(&x.value); + } + + fn collect_message(&mut self, x: &ast::Message<&str>) { + self.current_id = Some(x.id.name.to_string()); + self.current_attr = None; + + if let Some(ref y) = x.value { + self.collect_pattern(y); + } + for y in x.attributes.iter() { + self.collect_attribute(y); + } + } + + fn collect_term(&mut self, x: &ast::Term<&str>) { + self.current_id = Some(x.id.name.to_string()); + self.current_attr = None; + + self.collect_pattern(&x.value); + for y in x.attributes.iter() { + self.collect_attribute(y); + } + } + + pub fn collect_entry(&mut self, x: &ast::Entry<&str>) { + match x { + ast::Entry::Message(m) => { + self.collect_message(m); + } + ast::Entry::Term(t) => { + self.collect_term(t); + } + ast::Entry::Comment(_) => {} + ast::Entry::GroupComment(_) => {} + ast::Entry::ResourceComment(_) => {} + ast::Entry::Junk { .. } => {} + } + } +} + +#[no_mangle] +pub extern "C" fn fluent_resource_get_text_elements( + res: &FluentResource, + elements: &mut ThinVec<TextElementInfo>, +) { + let mut collector = TextElementsCollector::new(elements); + + for entry in res.entries() { + collector.collect_entry(entry); + } +} diff --git a/intl/l10n/rust/gtest/Cargo.toml b/intl/l10n/rust/gtest/Cargo.toml new file mode 100644 index 0000000000..ebc532f599 --- /dev/null +++ b/intl/l10n/rust/gtest/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "l10nregistry-ffi-gtest" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +license = "MPL-2.0" +description = "Tests for rust bindings to l10nRegistry" +edition = "2018" + +[dependencies] +l10nregistry-ffi = { path = "../l10nregistry-ffi" } +moz_task = { path = "../../../../xpcom/rust/moz_task" } + +[lib] +path = "test.rs" diff --git a/intl/l10n/rust/gtest/Test.cpp b/intl/l10n/rust/gtest/Test.cpp new file mode 100644 index 0000000000..98e7a8b5c6 --- /dev/null +++ b/intl/l10n/rust/gtest/Test.cpp @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" + +extern "C" void Rust_L10NLoadAsync(bool* aItWorked); + +TEST(RustL10N, LoadAsync) +{ + bool itWorked = false; + Rust_L10NLoadAsync(&itWorked); + EXPECT_TRUE(itWorked); +} + +extern "C" void Rust_L10NLoadSync(bool* aItWorked); + +TEST(RustL10N, LoadSync) +{ + bool itWorked = false; + Rust_L10NLoadSync(&itWorked); + EXPECT_TRUE(itWorked); +} diff --git a/intl/l10n/rust/gtest/moz.build b/intl/l10n/rust/gtest/moz.build new file mode 100644 index 0000000000..7c73e04fc8 --- /dev/null +++ b/intl/l10n/rust/gtest/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "Test.cpp", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/intl/l10n/rust/gtest/test.rs b/intl/l10n/rust/gtest/test.rs new file mode 100644 index 0000000000..3e993f4789 --- /dev/null +++ b/intl/l10n/rust/gtest/test.rs @@ -0,0 +1,58 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use l10nregistry_ffi::load::{load_async, load_sync}; +use moz_task; +use std::borrow::Borrow; + +// We want to test a file that ships in every platform configuration, so we take +// something from `toolkit/`. But we don't want to depend on the specifics of +// the text, or the packaging of that text, since those can change. It would be +// best to register an untranslated `.ftl` for this test, but that's difficult. +// Second best is to ship an untranslated `.ftl`, but that is not well-supported +// by existing processes either. So we settle for depending on the form of +// specific identifiers, whose names will appear in future searches, while +// depending on the specific messages or the file packaging. +fn assert_about_about_correct<T: Borrow<[u8]>> (res: T) { + assert!(res.borrow().len() > 0); + + // `windows` is a convenient, if inefficient, way to look for a subslice. + let needle = b"about-about-title"; + assert!(res.borrow().windows(needle.len()).position(|window| window == needle).is_some()); + + let needle = b"about-about-note"; + assert!(res.borrow().windows(needle.len()).position(|window| window == needle).is_some()); +} + +#[no_mangle] +pub extern "C" fn Rust_L10NLoadAsync(it_worked: *mut bool) { + let future = async move { + match load_async("resource://gre/localization/en-US/toolkit/about/aboutAbout.ftl").await { + Ok(res) => { + assert_about_about_correct(res); + unsafe { + *it_worked = true; + } + } + Err(err) => panic!("{:?}", err), + } + }; + + unsafe { + moz_task::gtest_only::spin_event_loop_until("Rust_L10NLoadAsync", future).unwrap(); + } +} + +#[no_mangle] +pub extern "C" fn Rust_L10NLoadSync(it_worked: *mut bool) { + match load_sync("resource://gre/localization/en-US/toolkit/about/aboutAbout.ftl") { + Ok(res) => { + assert_about_about_correct(res); + unsafe { + *it_worked = true; + } + } + Err(err) => panic!("{:?}", err), + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/Cargo.toml b/intl/l10n/rust/l10nregistry-ffi/Cargo.toml new file mode 100644 index 0000000000..8d5d575074 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "l10nregistry-ffi" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +futures-channel = "0.3" +futures = "0.3" +libc = "0.2" +cstr = "0.2" +log = "0.4" +nserror = { path = "../../../../xpcom/rust/nserror" } +nsstring = { path = "../../../../xpcom/rust/nsstring" } +l10nregistry = { path = "../l10nregistry-rs" } +fluent = { version = "0.16.0", features = ["fluent-pseudo"] } +unic-langid = "0.9" +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } +async-trait = "0.1" +moz_task = { path = "../../../../xpcom/rust/moz_task" } +xpcom = { path = "../../../../xpcom/rust/xpcom" } +fluent-ffi = { path = "../fluent-ffi" } +fluent-fallback = "0.7.0" diff --git a/intl/l10n/rust/l10nregistry-ffi/cbindgen.toml b/intl/l10n/rust/l10nregistry-ffi/cbindgen.toml new file mode 100644 index 0000000000..9e7127eeaa --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/cbindgen.toml @@ -0,0 +1,26 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef mozilla_intl_l10n_RegistryBindings_h +#error "Don't include this file directly, instead include RegistryBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +namespaces = ["mozilla", "intl", "ffi"] +includes = ["nsIStreamLoader.h"] + +[parse] +parse_deps = true +include = ["fluent-bundle", "fluent-fallback", "l10nregistry"] + +[enum] +derive_helper_methods = true + +[export.rename] +"ThinVec" = "nsTArray" +"Promise" = "dom::Promise" diff --git a/intl/l10n/rust/l10nregistry-ffi/src/env.rs b/intl/l10n/rust/l10nregistry-ffi/src/env.rs new file mode 100644 index 0000000000..7a77af2176 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/env.rs @@ -0,0 +1,132 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::xpcom_utils::get_app_locales; +use cstr::cstr; +use fluent_fallback::env::LocalesProvider; +use l10nregistry::{ + env::ErrorReporter, + errors::{L10nRegistryError, L10nRegistrySetupError}, +}; +use log::warn; +use nserror::{nsresult, NS_ERROR_NOT_AVAILABLE}; +use nsstring::{nsCStr, nsString}; +use std::fmt::{self, Write}; +use unic_langid::LanguageIdentifier; +use xpcom::interfaces; + +#[derive(Clone)] +pub struct GeckoEnvironment { + custom_locales: Option<Vec<LanguageIdentifier>>, +} + +impl GeckoEnvironment { + pub fn new(custom_locales: Option<Vec<LanguageIdentifier>>) -> Self { + Self { custom_locales } + } + + pub fn report_l10nregistry_setup_error(error: &L10nRegistrySetupError) { + warn!("L10nRegistry setup error: {}", error); + let result = log_simple_console_error( + &error.to_string(), + false, + true, + None, + (0, 0), + interfaces::nsIScriptError::errorFlag as u32, + ); + if let Err(err) = result { + warn!("Error while reporting an error: {}", err); + } + } +} + +impl ErrorReporter for GeckoEnvironment { + fn report_errors(&self, errors: Vec<L10nRegistryError>) { + for error in errors { + warn!("L10nRegistry error: {}", error); + let result = match error { + L10nRegistryError::FluentError { + resource_id, + loc, + error, + } => log_simple_console_error( + &error.to_string(), + false, + true, + Some(nsString::from(&resource_id.value)), + loc.map_or((0, 0), |(l, c)| (l as u32, c as u32)), + interfaces::nsIScriptError::errorFlag as u32, + ), + L10nRegistryError::MissingResource { .. } => log_simple_console_error( + &error.to_string(), + false, + true, + None, + (0, 0), + interfaces::nsIScriptError::warningFlag as u32, + ), + }; + if let Err(err) = result { + warn!("Error while reporting an error: {}", err); + } + } + } +} + +impl LocalesProvider for GeckoEnvironment { + type Iter = std::vec::IntoIter<unic_langid::LanguageIdentifier>; + fn locales(&self) -> Self::Iter { + if let Some(custom_locales) = &self.custom_locales { + custom_locales.clone().into_iter() + } else { + let result = get_app_locales() + .expect("Failed to retrieve app locales") + .into_iter() + .map(|s| LanguageIdentifier::from_bytes(&s).expect("Failed to parse a locale")) + .collect::<Vec<_>>(); + result.into_iter() + } + } +} + +fn log_simple_console_error( + error: &impl fmt::Display, + from_private_window: bool, + from_chrome_context: bool, + path: Option<nsString>, + pos: (u32, u32), + error_flags: u32, +) -> Result<(), nsresult> { + // Format whatever error argument into a wide string with `Display`. + let mut error_str = nsString::new(); + write!(&mut error_str, "{}", error).expect("nsString has an infallible Write impl"); + + // Get the relevant services, and create the script error object. + let console_service = + xpcom::get_service::<interfaces::nsIConsoleService>(cstr!("@mozilla.org/consoleservice;1")) + .ok_or(NS_ERROR_NOT_AVAILABLE)?; + let script_error = + xpcom::create_instance::<interfaces::nsIScriptError>(cstr!("@mozilla.org/scripterror;1")) + .ok_or(NS_ERROR_NOT_AVAILABLE)?; + let category = nsCStr::from("l10n"); + unsafe { + script_error + .Init( + &*error_str, + &*path.unwrap_or(nsString::new()), /* aSourceName */ + &*nsString::new(), /* aSourceLine */ + pos.0, /* aLineNumber */ + pos.1, /* aColNumber */ + error_flags, + &*category, + from_private_window, + from_chrome_context, + ) + .to_result()?; + + console_service.LogMessage(&**script_error).to_result()?; + } + Ok(()) +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs b/intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs new file mode 100644 index 0000000000..aba8a81470 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use l10nregistry::source::{FileFetcher, ResourceId}; + +use std::{borrow::Cow, io}; + +pub struct GeckoFileFetcher; + +fn try_string_from_box_u8(input: Box<[u8]>) -> io::Result<String> { + String::from_utf8(input.into()) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.utf8_error())) +} + +// For historical reasons we maintain a locale in Firefox with a codename `ja-JP-mac`. +// This string is an invalid BCP-47 language tag, so we don't store it in Gecko, which uses +// valid BCP-47 tags only, but rather keep that quirk local to Gecko L10nRegistry file fetcher. +// +// Here, if we encounter `ja-JP-macos` (valid BCP-47), we swap it for `ja-JP-mac`. +// +// See bug 1726586 for details, and source::get_locale_from_gecko. +fn get_path_for_gecko<'s>(input: &'s str) -> Cow<'s, str> { + if input.contains("ja-JP-macos") { + input.replace("ja-JP-macos", "ja-JP-mac").into() + } else { + input.into() + } +} + +#[async_trait::async_trait(?Send)] +impl FileFetcher for GeckoFileFetcher { + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String> { + let path = get_path_for_gecko(&resource_id.value); + crate::load::load_sync(path).and_then(try_string_from_box_u8) + } + + async fn fetch(&self, resource_id: &ResourceId) -> io::Result<String> { + let path = get_path_for_gecko(&resource_id.value); + crate::load::load_async(path) + .await + .and_then(try_string_from_box_u8) + } +} + +pub struct MockFileFetcher { + fs: Vec<(String, String)>, +} + +impl MockFileFetcher { + pub fn new(fs: Vec<(String, String)>) -> Self { + Self { fs } + } +} + +#[async_trait::async_trait(?Send)] +impl FileFetcher for MockFileFetcher { + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String> { + for (p, source) in &self.fs { + if p == &resource_id.value { + return Ok(source.clone()); + } + } + Err(io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + async fn fetch(&self, resource_id: &ResourceId) -> io::Result<String> { + self.fetch_sync(resource_id) + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/lib.rs b/intl/l10n/rust/l10nregistry-ffi/src/lib.rs new file mode 100644 index 0000000000..843860abf9 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/lib.rs @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub mod env; +mod fetcher; +pub mod load; +pub mod registry; +mod source; +mod xpcom_utils; diff --git a/intl/l10n/rust/l10nregistry-ffi/src/load.rs b/intl/l10n/rust/l10nregistry-ffi/src/load.rs new file mode 100644 index 0000000000..04a041246f --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/load.rs @@ -0,0 +1,113 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use futures_channel::oneshot; +use nserror::{nsresult, NS_OK, NS_SUCCESS_ADOPTED_DATA}; +use nsstring::{nsACString, nsCStringLike}; +use std::{ + cell::Cell, + ffi::c_void, + io::{self, Error, ErrorKind}, + ptr, +}; +use xpcom::{ + interfaces::{nsIStreamLoader, nsIStreamLoaderObserver, nsISupports}, + xpcom, +}; + +unsafe fn boxed_slice_from_raw(ptr: *mut u8, len: usize) -> Box<[u8]> { + if ptr.is_null() { + // It is undefined behaviour to create a `Box<[u8]>` with a null pointer, + // so avoid that case. + assert_eq!(len, 0); + Box::new([]) + } else { + Box::from_raw(ptr::slice_from_raw_parts_mut(ptr, len)) + } +} + +#[xpcom(implement(nsIStreamLoaderObserver), nonatomic)] +struct StreamLoaderObserver { + sender: Cell<Option<oneshot::Sender<Result<Box<[u8]>, nsresult>>>>, +} + +impl StreamLoaderObserver { + #[allow(non_snake_case)] + unsafe fn OnStreamComplete( + &self, + _loader: *const nsIStreamLoader, + _ctxt: *const nsISupports, + status: nsresult, + result_length: u32, + result: *const u8, + ) -> nsresult { + let sender = match self.sender.take() { + Some(sender) => sender, + None => return NS_OK, + }; + + if status.failed() { + sender.send(Err(status)).expect("Failed to send data"); + return NS_OK; + } + + // safety: take ownership of the data passed in. This is OK because we + // have configured Rust and C++ to use the same allocator, and our + // caller won't free the `result` pointer if we return + // NS_SUCCESS_ADOPTED_DATA. + sender + .send(Ok(boxed_slice_from_raw( + result as *mut u8, + result_length as usize, + ))) + .expect("Failed to send data"); + NS_SUCCESS_ADOPTED_DATA + } +} + +extern "C" { + fn L10nRegistryLoad( + path: *const nsACString, + observer: *const nsIStreamLoaderObserver, + ) -> nsresult; + + fn L10nRegistryLoadSync( + aPath: *const nsACString, + aData: *mut *mut c_void, + aSize: *mut u64, + ) -> nsresult; +} + +pub async fn load_async(path: impl nsCStringLike) -> io::Result<Box<[u8]>> { + let (sender, receiver) = oneshot::channel::<Result<Box<[u8]>, nsresult>>(); + let observer = StreamLoaderObserver::allocate(InitStreamLoaderObserver { + sender: Cell::new(Some(sender)), + }); + unsafe { + L10nRegistryLoad(&*path.adapt(), observer.coerce()) + .to_result() + .map_err(|err| Error::new(ErrorKind::Other, err))?; + } + receiver + .await + .expect("Failed to receive from observer.") + .map_err(|err| Error::new(ErrorKind::Other, err)) +} + +pub fn load_sync(path: impl nsCStringLike) -> io::Result<Box<[u8]>> { + let mut data_ptr: *mut c_void = ptr::null_mut(); + let mut data_length: u64 = 0; + unsafe { + L10nRegistryLoadSync(&*path.adapt(), &mut data_ptr, &mut data_length) + .to_result() + .map_err(|err| Error::new(ErrorKind::Other, err))?; + + // The call succeeded, meaning `data_ptr` and `size` have been filled in with owning pointers to actual data payloads (or null). + // If we get a null, return a successful read of the empty file. + Ok(boxed_slice_from_raw( + data_ptr as *mut u8, + data_length as usize, + )) + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/registry.rs b/intl/l10n/rust/l10nregistry-ffi/src/registry.rs new file mode 100644 index 0000000000..846f12273c --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/registry.rs @@ -0,0 +1,519 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use fluent_ffi::{adapt_bundle_for_gecko, FluentBundleRc}; +use nsstring::{nsACString, nsCString}; +use std::mem; +use std::rc::Rc; +use thin_vec::ThinVec; + +use crate::{env::GeckoEnvironment, fetcher::GeckoFileFetcher, xpcom_utils::is_parent_process}; +use fluent_fallback::{generator::BundleGenerator, types::ResourceType}; +use futures_channel::mpsc::{unbounded, UnboundedSender}; +pub use l10nregistry::{ + errors::L10nRegistrySetupError, + registry::{BundleAdapter, GenerateBundles, GenerateBundlesSync, L10nRegistry}, + source::{FileSource, ResourceId, ToResourceId}, +}; +use unic_langid::LanguageIdentifier; +use xpcom::RefPtr; + +#[derive(Clone)] +pub struct GeckoBundleAdapter { + use_isolating: bool, +} + +impl Default for GeckoBundleAdapter { + fn default() -> Self { + Self { + use_isolating: true, + } + } +} + +impl BundleAdapter for GeckoBundleAdapter { + fn adapt_bundle(&self, bundle: &mut l10nregistry::fluent::FluentBundle) { + bundle.set_use_isolating(self.use_isolating); + adapt_bundle_for_gecko(bundle, None); + } +} + +thread_local!(static L10N_REGISTRY: Rc<GeckoL10nRegistry> = { + let sources = if is_parent_process() { + let packaged_locales = get_packaged_locales(); + let entries = get_l10n_registry_category_entries(); + + Some(entries + .into_iter() + .map(|entry| { + FileSource::new( + entry.entry.to_string(), + Some("app".to_string()), + packaged_locales.clone(), + entry.value.to_string(), + Default::default(), + GeckoFileFetcher, + ) + }) + .collect()) + + } else { + None + }; + + create_l10n_registry(sources) +}); + +pub type GeckoL10nRegistry = L10nRegistry<GeckoEnvironment, GeckoBundleAdapter>; +pub type GeckoFluentBundleIterator = GenerateBundlesSync<GeckoEnvironment, GeckoBundleAdapter>; + +trait GeckoReportError<V, E> { + fn report_error(self) -> Result<V, E>; +} + +impl<V> GeckoReportError<V, L10nRegistrySetupError> for Result<V, L10nRegistrySetupError> { + fn report_error(self) -> Self { + if let Err(ref err) = self { + GeckoEnvironment::report_l10nregistry_setup_error(err); + } + self + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nFileSourceDescriptor { + name: nsCString, + metasource: nsCString, + locales: ThinVec<nsCString>, + pre_path: nsCString, + index: ThinVec<nsCString>, +} + +fn get_l10n_registry_category_entries() -> Vec<crate::xpcom_utils::CategoryEntry> { + crate::xpcom_utils::get_category_entries(&nsCString::from("l10n-registry")).unwrap_or_default() +} + +fn get_packaged_locales() -> Vec<LanguageIdentifier> { + crate::xpcom_utils::get_packaged_locales() + .map(|locales| { + locales + .into_iter() + .map(|s| s.to_utf8().parse().expect("Failed to parse locale.")) + .collect() + }) + .unwrap_or_default() +} + +fn create_l10n_registry(sources: Option<Vec<FileSource>>) -> Rc<GeckoL10nRegistry> { + let env = GeckoEnvironment::new(None); + let mut reg = L10nRegistry::with_provider(env); + + reg.set_bundle_adapter(GeckoBundleAdapter::default()) + .expect("Failed to set bundle adaptation closure."); + + if let Some(sources) = sources { + reg.register_sources(sources) + .expect("Failed to register sources."); + } + Rc::new(reg) +} + +pub fn set_l10n_registry(new_sources: &ThinVec<L10nFileSourceDescriptor>) { + L10N_REGISTRY.with(|reg| { + let new_source_names: Vec<_> = new_sources + .iter() + .map(|d| d.name.to_utf8().to_string()) + .collect(); + let old_sources = reg.get_source_names().unwrap(); + + let mut sources_to_be_removed = vec![]; + for name in &old_sources { + if !new_source_names.contains(&name) { + sources_to_be_removed.push(name); + } + } + reg.remove_sources(sources_to_be_removed).unwrap(); + + let mut add_sources = vec![]; + for desc in new_sources { + if !old_sources.contains(&desc.name.to_string()) { + add_sources.push(FileSource::new( + desc.name.to_string(), + Some(desc.metasource.to_string()), + desc.locales + .iter() + .map(|s| s.to_utf8().parse().unwrap()) + .collect(), + desc.pre_path.to_string(), + Default::default(), + GeckoFileFetcher, + )); + } + } + reg.register_sources(add_sources).unwrap(); + }); +} + +pub fn get_l10n_registry() -> Rc<GeckoL10nRegistry> { + L10N_REGISTRY.with(|reg| reg.clone()) +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub enum GeckoResourceType { + Optional, + Required, +} + +#[repr(C)] +pub struct GeckoResourceId { + value: nsCString, + resource_type: GeckoResourceType, +} + +impl From<&GeckoResourceId> for ResourceId { + fn from(resource_id: &GeckoResourceId) -> Self { + resource_id + .value + .to_string() + .to_resource_id(match resource_id.resource_type { + GeckoResourceType::Optional => ResourceType::Optional, + GeckoResourceType::Required => ResourceType::Required, + }) + } +} + +#[repr(C)] +pub enum L10nRegistryStatus { + None, + EmptyName, + InvalidLocaleCode, +} + +#[no_mangle] +pub extern "C" fn l10nregistry_new(use_isolating: bool) -> *const GeckoL10nRegistry { + let env = GeckoEnvironment::new(None); + let mut reg = L10nRegistry::with_provider(env); + let _ = reg + .set_bundle_adapter(GeckoBundleAdapter { use_isolating }) + .report_error(); + Rc::into_raw(Rc::new(reg)) +} + +#[no_mangle] +pub extern "C" fn l10nregistry_instance_get() -> *const GeckoL10nRegistry { + let reg = get_l10n_registry(); + Rc::into_raw(reg) +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_get_parent_process_sources( + sources: &mut ThinVec<L10nFileSourceDescriptor>, +) { + debug_assert!( + is_parent_process(), + "This should be called only in parent process." + ); + + // If at the point when the first content process is being initialized, the parent + // process `L10nRegistryService` has not been initialized yet, this will trigger it. + // + // This is architecturally imperfect, but acceptable for simplicity reasons because + // `L10nRegistry` instance is cheap and mainly servers as a store of state. + let reg = get_l10n_registry(); + for name in reg.get_source_names().unwrap() { + let source = reg.file_source_by_name(&name).unwrap().unwrap(); + let descriptor = L10nFileSourceDescriptor { + name: source.name.as_str().into(), + metasource: source.metasource.as_str().into(), + locales: source + .locales() + .iter() + .map(|l| l.to_string().into()) + .collect(), + pre_path: source.pre_path.as_str().into(), + index: source + .get_index() + .map(|index| index.into_iter().map(|s| s.into()).collect()) + .unwrap_or_default(), + }; + sources.push(descriptor); + } +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_register_parent_process_sources( + sources: &ThinVec<L10nFileSourceDescriptor>, +) { + debug_assert!( + !is_parent_process(), + "This should be called only in content process." + ); + set_l10n_registry(sources); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_addref(reg: *const GeckoL10nRegistry) { + let raw = Rc::from_raw(reg); + mem::forget(Rc::clone(&raw)); + mem::forget(raw); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_release(reg: *const GeckoL10nRegistry) { + let _ = Rc::from_raw(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_get_available_locales( + reg: &GeckoL10nRegistry, + result: &mut ThinVec<nsCString>, +) { + if let Ok(locales) = reg.get_available_locales().report_error() { + result.extend(locales.into_iter().map(|locale| locale.to_string().into())); + } +} + +fn broadcast_settings_if_parent(reg: &GeckoL10nRegistry) { + if !is_parent_process() { + return; + } + + L10N_REGISTRY.with(|reg_service| { + if std::ptr::eq(Rc::as_ptr(reg_service), reg) { + let locales = reg + .get_available_locales() + .unwrap() + .iter() + .map(|loc| loc.to_string().into()) + .collect(); + + unsafe { + crate::xpcom_utils::set_available_locales(&locales); + L10nRegistrySendUpdateL10nFileSources(); + } + } + }); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_register_sources( + reg: &GeckoL10nRegistry, + sources: &ThinVec<&FileSource>, +) { + let _ = reg + .register_sources(sources.iter().map(|&s| s.clone()).collect()) + .report_error(); + + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_update_sources( + reg: &GeckoL10nRegistry, + sources: &mut ThinVec<&FileSource>, +) { + let _ = reg + .update_sources(sources.iter().map(|&s| s.clone()).collect()) + .report_error(); + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_remove_sources( + reg: &GeckoL10nRegistry, + sources_elements: *const nsCString, + sources_length: usize, +) { + if sources_elements.is_null() { + return; + } + + let sources = std::slice::from_raw_parts(sources_elements, sources_length); + let _ = reg.remove_sources(sources.to_vec()).report_error(); + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_has_source( + reg: &GeckoL10nRegistry, + name: &nsACString, + status: &mut L10nRegistryStatus, +) -> bool { + if name.is_empty() { + *status = L10nRegistryStatus::EmptyName; + return false; + } + *status = L10nRegistryStatus::None; + reg.has_source(&name.to_utf8()) + .report_error() + .unwrap_or(false) +} + +#[no_mangle] +pub extern "C" fn l10nregistry_get_source( + reg: &GeckoL10nRegistry, + name: &nsACString, + status: &mut L10nRegistryStatus, +) -> *mut FileSource { + if name.is_empty() { + *status = L10nRegistryStatus::EmptyName; + return std::ptr::null_mut(); + } + + *status = L10nRegistryStatus::None; + + if let Ok(Some(source)) = reg.file_source_by_name(&name.to_utf8()).report_error() { + Box::into_raw(Box::new(source)) + } else { + std::ptr::null_mut() + } +} + +#[no_mangle] +pub extern "C" fn l10nregistry_clear_sources(reg: &GeckoL10nRegistry) { + let _ = reg.clear_sources().report_error(); + + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_get_source_names( + reg: &GeckoL10nRegistry, + result: &mut ThinVec<nsCString>, +) { + if let Ok(names) = reg.get_source_names().report_error() { + result.extend(names.into_iter().map(|name| nsCString::from(name))); + } +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_generate_bundles_sync( + reg: &GeckoL10nRegistry, + locales_elements: *const nsCString, + locales_length: usize, + res_ids_elements: *const GeckoResourceId, + res_ids_length: usize, + status: &mut L10nRegistryStatus, +) -> *mut GeckoFluentBundleIterator { + let locales = std::slice::from_raw_parts(locales_elements, locales_length); + let res_ids = std::slice::from_raw_parts(res_ids_elements, res_ids_length) + .into_iter() + .map(ResourceId::from) + .collect(); + let locales: Result<Vec<LanguageIdentifier>, _> = + locales.into_iter().map(|s| s.to_utf8().parse()).collect(); + + match locales { + Ok(locales) => { + *status = L10nRegistryStatus::None; + let iter = reg.bundles_iter(locales.into_iter(), res_ids); + Box::into_raw(Box::new(iter)) + } + Err(_) => { + *status = L10nRegistryStatus::InvalidLocaleCode; + std::ptr::null_mut() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_iterator_destroy(iter: *mut GeckoFluentBundleIterator) { + let _ = Box::from_raw(iter); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_iterator_next( + iter: &mut GeckoFluentBundleIterator, +) -> *mut FluentBundleRc { + if let Some(Ok(result)) = iter.next() { + Box::into_raw(Box::new(result)) + } else { + std::ptr::null_mut() + } +} + +pub struct NextRequest { + promise: RefPtr<xpcom::Promise>, + // Ownership is transferred here. + callback: unsafe extern "C" fn(&xpcom::Promise, *mut FluentBundleRc), +} + +pub struct GeckoFluentBundleAsyncIteratorWrapper(UnboundedSender<NextRequest>); + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_generate_bundles( + reg: &GeckoL10nRegistry, + locales_elements: *const nsCString, + locales_length: usize, + res_ids_elements: *const GeckoResourceId, + res_ids_length: usize, + status: &mut L10nRegistryStatus, +) -> *mut GeckoFluentBundleAsyncIteratorWrapper { + let locales = std::slice::from_raw_parts(locales_elements, locales_length); + let res_ids = std::slice::from_raw_parts(res_ids_elements, res_ids_length) + .into_iter() + .map(ResourceId::from) + .collect(); + let locales: Result<Vec<LanguageIdentifier>, _> = + locales.into_iter().map(|s| s.to_utf8().parse()).collect(); + + match locales { + Ok(locales) => { + *status = L10nRegistryStatus::None; + let mut iter = reg.bundles_stream(locales.into_iter(), res_ids); + + // Immediately spawn the task which will handle the async calls, and use an `UnboundedSender` + // to send callbacks for specific `next()` calls to it. + let (sender, mut receiver) = unbounded::<NextRequest>(); + moz_task::spawn_local("l10nregistry_generate_bundles", async move { + use futures::StreamExt; + while let Some(req) = receiver.next().await { + let result = match iter.next().await { + Some(Ok(result)) => Box::into_raw(Box::new(result)), + _ => std::ptr::null_mut(), + }; + (req.callback)(&req.promise, result); + } + }) + .detach(); + let iter = GeckoFluentBundleAsyncIteratorWrapper(sender); + Box::into_raw(Box::new(iter)) + } + Err(_) => { + *status = L10nRegistryStatus::InvalidLocaleCode; + std::ptr::null_mut() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_async_iterator_destroy( + iter: *mut GeckoFluentBundleAsyncIteratorWrapper, +) { + let _ = Box::from_raw(iter); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_async_iterator_next( + iter: &GeckoFluentBundleAsyncIteratorWrapper, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, *mut FluentBundleRc), +) { + if iter + .0 + .unbounded_send(NextRequest { + promise: RefPtr::new(promise), + callback, + }) + .is_err() + { + callback(promise, std::ptr::null_mut()); + } +} + +extern "C" { + pub fn L10nRegistrySendUpdateL10nFileSources(); +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/source.rs b/intl/l10n/rust/l10nregistry-ffi/src/source.rs new file mode 100644 index 0000000000..7003d8da97 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/source.rs @@ -0,0 +1,359 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use super::fetcher::{GeckoFileFetcher, MockFileFetcher}; +use crate::env::GeckoEnvironment; + +use fluent::FluentResource; +use l10nregistry::source::{FileSource, FileSourceOptions, ResourceOption, ResourceStatus, RcResource}; + +use nsstring::{nsACString, nsCString}; +use thin_vec::ThinVec; +use unic_langid::LanguageIdentifier; + +use std::{borrow::Cow, mem, rc::Rc}; +use xpcom::RefPtr; + +#[repr(C)] +pub enum L10nFileSourceStatus { + None, + EmptyName, + EmptyPrePath, + EmptyResId, + InvalidLocaleCode, +} + +// For historical reasons we maintain a locale in Firefox with a codename `ja-JP-mac`. +// This string is an invalid BCP-47 language tag, so we don't store it in Gecko, which uses +// valid BCP-47 tags only, but rather keep that quirk local to Gecko L10nRegistry file fetcher. +// +// Here, if we encounter `ja-JP-mac` (invalid BCP-47), we swap it for a valid equivalent: `ja-JP-macos`. +// +// See bug 1726586 for details and fetcher::get_locale_for_gecko. +fn get_locale_from_gecko<'s>(input: Cow<'s, str>) -> Cow<'s, str> { + if input == "ja-JP-mac" { + "ja-JP-macos".into() + } else { + input + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_new( + name: &nsACString, + metasource: &nsACString, + locales: &ThinVec<nsCString>, + pre_path: &nsACString, + allow_override: bool, + status: &mut L10nFileSourceStatus, +) -> *const FileSource { + if name.is_empty() { + *status = L10nFileSourceStatus::EmptyName; + return std::ptr::null(); + } + if pre_path.is_empty() { + *status = L10nFileSourceStatus::EmptyPrePath; + return std::ptr::null(); + } + + let locales: Result<Vec<LanguageIdentifier>, _> = locales + .iter() + .map(|l| get_locale_from_gecko(l.to_utf8()).parse()) + .collect(); + + let locales = match locales { + Ok(locales) => locales, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + let mut source = FileSource::new( + name.to_string(), + Some(metasource.to_string()), + locales, + pre_path.to_string(), + FileSourceOptions { allow_override }, + GeckoFileFetcher, + ); + source.set_reporter(GeckoEnvironment::new(None)); + + *status = L10nFileSourceStatus::None; + Rc::into_raw(Rc::new(source)) +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_new_with_index( + name: &nsACString, + metasource: &nsACString, + locales: &ThinVec<nsCString>, + pre_path: &nsACString, + index_elements: *const nsCString, + index_length: usize, + allow_override: bool, + status: &mut L10nFileSourceStatus, +) -> *const FileSource { + if name.is_empty() { + *status = L10nFileSourceStatus::EmptyName; + return std::ptr::null(); + } + if pre_path.is_empty() { + *status = L10nFileSourceStatus::EmptyPrePath; + return std::ptr::null(); + } + + let locales: Result<Vec<LanguageIdentifier>, _> = locales + .iter() + .map(|l| get_locale_from_gecko(l.to_utf8()).parse()) + .collect(); + + let index = if index_length > 0 { + assert!(!index_elements.is_null()); + std::slice::from_raw_parts(index_elements, index_length) + } else { + &[] + } + .into_iter() + .map(|s| s.to_string()) + .collect(); + + let locales = match locales { + Ok(locales) => locales, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + let mut source = FileSource::new_with_index( + name.to_string(), + Some(metasource.to_string()), + locales, + pre_path.to_string(), + FileSourceOptions { allow_override }, + GeckoFileFetcher, + index, + ); + source.set_reporter(GeckoEnvironment::new(None)); + + *status = L10nFileSourceStatus::None; + Rc::into_raw(Rc::new(source)) +} + +#[repr(C)] +pub struct L10nFileSourceMockFile { + path: nsCString, + source: nsCString, +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_new_mock( + name: &nsACString, + metasource: &nsACString, + locales: &ThinVec<nsCString>, + pre_path: &nsACString, + fs: &ThinVec<L10nFileSourceMockFile>, + status: &mut L10nFileSourceStatus, +) -> *const FileSource { + if name.is_empty() { + *status = L10nFileSourceStatus::EmptyName; + return std::ptr::null(); + } + if pre_path.is_empty() { + *status = L10nFileSourceStatus::EmptyPrePath; + return std::ptr::null(); + } + + let locales: Result<Vec<LanguageIdentifier>, _> = locales + .iter() + .map(|l| get_locale_from_gecko(l.to_utf8()).parse()) + .collect(); + + let locales = match locales { + Ok(locales) => locales, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + let fs = fs + .iter() + .map(|mock| (mock.path.to_string(), mock.source.to_string())) + .collect(); + let fetcher = MockFileFetcher::new(fs); + let mut source = FileSource::new( + name.to_string(), + Some(metasource.to_string()), + locales, + pre_path.to_string(), + Default::default(), + fetcher, + ); + source.set_reporter(GeckoEnvironment::new(None)); + + *status = L10nFileSourceStatus::None; + Rc::into_raw(Rc::new(source)) +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_addref(source: *const FileSource) { + let raw = Rc::from_raw(source); + mem::forget(Rc::clone(&raw)); + mem::forget(raw); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_release(source: *const FileSource) { + let _ = Rc::from_raw(source); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_name(source: &FileSource, ret_val: &mut nsACString) { + ret_val.assign(&source.name); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_metasource(source: &FileSource, ret_val: &mut nsACString) { + ret_val.assign(&source.metasource); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_locales( + source: &FileSource, + ret_val: &mut ThinVec<nsCString>, +) { + for locale in source.locales() { + ret_val.push(locale.to_string().into()); + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_prepath(source: &FileSource, ret_val: &mut nsACString) { + ret_val.assign(&source.pre_path); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_index( + source: &FileSource, + ret_val: &mut ThinVec<nsCString>, +) -> bool { + if let Some(index) = source.get_index() { + for entry in index { + ret_val.push(entry.to_string().into()); + } + true + } else { + false + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_has_file( + source: &FileSource, + locale: &nsACString, + path: &nsACString, + status: &mut L10nFileSourceStatus, + present: &mut bool, +) -> bool { + if path.is_empty() { + *status = L10nFileSourceStatus::EmptyResId; + return false; + } + + let locale = match locale.to_utf8().parse() { + Ok(locale) => locale, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return false; + } + }; + + *status = L10nFileSourceStatus::None; + // To work around Option<bool> we return bool for the option, + // and the `present` argument is the value of it. + if let Some(val) = source.has_file(&locale, &path.to_utf8().into()) { + *present = val; + true + } else { + false + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_fetch_file_sync( + source: &FileSource, + locale: &nsACString, + path: &nsACString, + status: &mut L10nFileSourceStatus, +) -> *const FluentResource { + if path.is_empty() { + *status = L10nFileSourceStatus::EmptyResId; + return std::ptr::null(); + } + + let locale = match locale.to_utf8().parse() { + Ok(locale) => locale, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + *status = L10nFileSourceStatus::None; + //XXX: Bug 1723191 - if we encounter a request for sync load while async load is in progress + // we will discard the async load and force the sync load instead. + // There may be a better option but we haven't had time to explore it. + if let ResourceOption::Some(res) = + source.fetch_file_sync(&locale, &path.to_utf8().into(), /* overload */ true) + { + Rc::into_raw(res) + } else { + std::ptr::null() + } +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_fetch_file( + source: &FileSource, + locale: &nsACString, + path: &nsACString, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, Option<&FluentResource>), + status: &mut L10nFileSourceStatus, +) { + if path.is_empty() { + *status = L10nFileSourceStatus::EmptyResId; + return; + } + + let locale = match locale.to_utf8().parse() { + Ok(locale) => locale, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return; + } + }; + + *status = L10nFileSourceStatus::None; + + let path = path.to_utf8().into(); + + match source.fetch_file(&locale, &path) { + ResourceStatus::MissingOptional => callback(promise, None), + ResourceStatus::MissingRequired => callback(promise, None), + ResourceStatus::Loaded(res) => callback(promise, Some(&res)), + res @ ResourceStatus::Loading(_) => { + let strong_promise = RefPtr::new(promise); + moz_task::spawn_local("l10nfilesource_fetch_file", async move { + callback( + &strong_promise, + Option::<RcResource>::from(res.await).as_ref().map(|r| &**r), + ); + }) + .detach(); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs b/intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs new file mode 100644 index 0000000000..7c83088075 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs @@ -0,0 +1,129 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use cstr::cstr; +use nsstring::{nsACString, nsCString}; +use std::marker::PhantomData; +use thin_vec::ThinVec; +use xpcom::{ + get_service, getter_addrefs, + interfaces::{ + mozILocaleService, nsICategoryEntry, nsICategoryManager, nsISimpleEnumerator, nsIXULRuntime, + }, + RefPtr, XpCom, +}; + +pub struct IterSimpleEnumerator<T> { + enumerator: RefPtr<nsISimpleEnumerator>, + phantom: PhantomData<T>, +} + +impl<T: XpCom> IterSimpleEnumerator<T> { + /// Convert a `nsISimpleEnumerator` into a rust `Iterator` type. + pub fn new(enumerator: RefPtr<nsISimpleEnumerator>) -> Self { + IterSimpleEnumerator { + enumerator, + phantom: PhantomData, + } + } +} + +impl<T: XpCom + 'static> Iterator for IterSimpleEnumerator<T> { + type Item = RefPtr<T>; + + fn next(&mut self) -> Option<Self::Item> { + let mut more = false; + unsafe { + self.enumerator + .HasMoreElements(&mut more) + .to_result() + .ok()? + } + if !more { + return None; + } + + let element = getter_addrefs(|p| unsafe { self.enumerator.GetNext(p) }).ok()?; + element.query_interface::<T>() + } +} + +fn process_type() -> u32 { + if let Ok(appinfo) = xpcom::components::XULRuntime::service::<nsIXULRuntime>() { + let mut process_type = nsIXULRuntime::PROCESS_TYPE_DEFAULT; + if unsafe { appinfo.GetProcessType(&mut process_type).succeeded() } { + return process_type; + } + } + nsIXULRuntime::PROCESS_TYPE_DEFAULT +} + +pub fn is_parent_process() -> bool { + process_type() == nsIXULRuntime::PROCESS_TYPE_DEFAULT +} + +pub fn get_packaged_locales() -> Option<ThinVec<nsCString>> { + let locale_service = + get_service::<mozILocaleService>(cstr!("@mozilla.org/intl/localeservice;1"))?; + let mut locales = ThinVec::new(); + unsafe { + locale_service + .GetPackagedLocales(&mut locales) + .to_result() + .ok()?; + } + Some(locales) +} + +pub fn get_app_locales() -> Option<ThinVec<nsCString>> { + let locale_service = + get_service::<mozILocaleService>(cstr!("@mozilla.org/intl/localeservice;1"))?; + let mut locales = ThinVec::new(); + unsafe { + locale_service + .GetAppLocalesAsBCP47(&mut locales) + .to_result() + .ok()?; + } + Some(locales) +} + +pub fn set_available_locales(locales: &ThinVec<nsCString>) { + let locale_service = + get_service::<mozILocaleService>(cstr!("@mozilla.org/intl/localeservice;1")) + .expect("Failed to get a service."); + unsafe { + locale_service + .SetAvailableLocales(locales) + .to_result() + .expect("Failed to set locales."); + } +} + +pub struct CategoryEntry { + pub entry: nsCString, + pub value: nsCString, +} + +pub fn get_category_entries(category: &nsACString) -> Option<Vec<CategoryEntry>> { + let category_manager = + get_service::<nsICategoryManager>(cstr!("@mozilla.org/categorymanager;1"))?; + + let enumerator = + getter_addrefs(|p| unsafe { category_manager.EnumerateCategory(category, p) }).ok()?; + + Some( + IterSimpleEnumerator::<nsICategoryEntry>::new(enumerator) + .map(|ientry| { + let mut entry = nsCString::new(); + let mut value = nsCString::new(); + unsafe { + let _ = ientry.GetEntry(&mut *entry); + let _ = ientry.GetValue(&mut *value); + } + CategoryEntry { entry, value } + }) + .collect(), + ) +} diff --git a/intl/l10n/rust/l10nregistry-rs/.gitignore b/intl/l10n/rust/l10nregistry-rs/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/intl/l10n/rust/l10nregistry-rs/Cargo.toml b/intl/l10n/rust/l10nregistry-rs/Cargo.toml new file mode 100644 index 0000000000..5bc3656d6e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "l10nregistry" +version = "0.3.0" +authors = ["Zibi Braniecki <gandalf@mozilla.com>"] +license = "Apache-2.0/MIT" +edition = "2018" + +[dependencies] +async-trait = "0.1" +fluent-bundle = "0.15.2" +fluent-fallback = "0.7.0" +futures = "0.3" +pin-project-lite = "0.2" +replace_with = "0.1" +rustc-hash = "1" +unic-langid = "0.9" + +[features] +default = [] +test-fluent = [] diff --git a/intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE b/intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE new file mode 100644 index 0000000000..35582f166b --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Mozilla + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/intl/l10n/rust/l10nregistry-rs/LICENSE-MIT b/intl/l10n/rust/l10nregistry-rs/LICENSE-MIT new file mode 100644 index 0000000000..5655fa311c --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright 2017 Mozilla + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/intl/l10n/rust/l10nregistry-rs/README.md b/intl/l10n/rust/l10nregistry-rs/README.md new file mode 100644 index 0000000000..873555df89 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/README.md @@ -0,0 +1,17 @@ +# l10nregistry-rs + +The `L10nRegistry` is responsible for taking `FileSources` across the app, and turning them into bundles. It is hooked into the `L10nRegistry` global available from privileged JavaScript. See the [L10nRegistry.webidl](https://searchfox.org/mozilla-central/source/dom/chrome-webidl/L10nRegistry.webidl#100) for detailed information about this API, and `intl/l10n/test/test_l10nregistry.js` for integration tests with examples of how it can be used. + +## Testing + +Tests can be run directly from this directory via: + +``` +cargo test --all-features +``` + +Benchmarks are also available. First uncomment the `criterion` dependency in the `Cargo.toml` and then run. + +``` +cargo test bench --all-features +``` diff --git a/intl/l10n/rust/l10nregistry-rs/src/env.rs b/intl/l10n/rust/l10nregistry-rs/src/env.rs new file mode 100644 index 0000000000..7cd1ff30f4 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/env.rs @@ -0,0 +1,5 @@ +use crate::errors::L10nRegistryError; + +pub trait ErrorReporter { + fn report_errors(&self, errors: Vec<L10nRegistryError>); +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/errors.rs b/intl/l10n/rust/l10nregistry-rs/src/errors.rs new file mode 100644 index 0000000000..d58f02ea8e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/errors.rs @@ -0,0 +1,74 @@ +use fluent_bundle::FluentError; +use fluent_fallback::types::ResourceId; +use std::error::Error; +use unic_langid::LanguageIdentifier; + +#[derive(Debug, Clone, PartialEq)] +pub enum L10nRegistryError { + FluentError { + resource_id: ResourceId, + loc: Option<(usize, usize)>, + error: FluentError, + }, + MissingResource { + locale: LanguageIdentifier, + resource_id: ResourceId, + }, +} + +impl std::fmt::Display for L10nRegistryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingResource { + locale, + resource_id, + } => { + write!( + f, + "Missing resource in locale {}: {}", + locale, resource_id.value + ) + } + Self::FluentError { + resource_id, + loc, + error, + } => { + if let Some(loc) = loc { + write!( + f, + "Fluent Error in {}[line: {}, col: {}]: {}", + resource_id.value, loc.0, loc.1, error + ) + } else { + write!(f, "Fluent Error in {}: {}", resource_id.value, error) + } + } + } + } +} + +impl Error for L10nRegistryError {} + +#[derive(Debug, Clone, PartialEq)] +pub enum L10nRegistrySetupError { + RegistryLocked, + DuplicatedSource { name: String }, + MissingSource { name: String }, +} + +impl std::fmt::Display for L10nRegistrySetupError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::RegistryLocked => write!(f, "Can't modify a registry when locked."), + Self::DuplicatedSource { name } => { + write!(f, "Source with a name {} is already registered.", &name) + } + Self::MissingSource { name } => { + write!(f, "Cannot find a source with a name {}.", &name) + } + } + } +} + +impl Error for L10nRegistrySetupError {} diff --git a/intl/l10n/rust/l10nregistry-rs/src/fluent.rs b/intl/l10n/rust/l10nregistry-rs/src/fluent.rs new file mode 100644 index 0000000000..b6ac2a12ab --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/fluent.rs @@ -0,0 +1,5 @@ +use fluent_bundle::FluentBundle as FluentBundleBase; +pub use fluent_bundle::{FluentError, FluentResource}; +use std::rc::Rc; + +pub type FluentBundle = FluentBundleBase<Rc<FluentResource>>; diff --git a/intl/l10n/rust/l10nregistry-rs/src/lib.rs b/intl/l10n/rust/l10nregistry-rs/src/lib.rs new file mode 100644 index 0000000000..42d1b10f62 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/lib.rs @@ -0,0 +1,6 @@ +pub mod env; +pub mod errors; +pub mod fluent; +pub mod registry; +pub mod solver; +pub mod source; diff --git a/intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs b/intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs new file mode 100644 index 0000000000..bfcff941b5 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs @@ -0,0 +1,294 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use crate::{ + env::ErrorReporter, + errors::{L10nRegistryError, L10nRegistrySetupError}, + fluent::{FluentBundle, FluentError}, + registry::{BundleAdapter, L10nRegistry, MetaSources}, + solver::{AsyncTester, ParallelProblemSolver}, + source::{ResourceOption, ResourceStatus}, +}; + +use fluent_fallback::{generator::BundleStream, types::ResourceId}; +use futures::{ + stream::{Collect, FuturesOrdered}, + Stream, StreamExt, +}; +use std::future::Future; +use unic_langid::LanguageIdentifier; + +impl<P, B> L10nRegistry<P, B> +where + P: Clone, + B: Clone, +{ + /// This method is useful for testing various configurations. + #[cfg(feature = "test-fluent")] + pub fn generate_bundles_for_lang( + &self, + langid: LanguageIdentifier, + resource_ids: Vec<ResourceId>, + ) -> Result<GenerateBundles<P, B>, L10nRegistrySetupError> { + let lang_ids = vec![langid]; + + Ok(GenerateBundles::new( + self.clone(), + lang_ids.into_iter(), + resource_ids, + // Cheaply create an immutable shallow copy of the [MetaSources]. + self.try_borrow_metasources()?.clone(), + )) + } + + // Asynchronously generate the bundles. + pub fn generate_bundles( + &self, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + ) -> Result<GenerateBundles<P, B>, L10nRegistrySetupError> { + Ok(GenerateBundles::new( + self.clone(), + locales, + resource_ids, + // Cheaply create an immutable shallow copy of the [MetaSources]. + self.try_borrow_metasources()?.clone(), + )) + } +} + +/// This enum contains the various states the [GenerateBundles] can be in during the +/// asynchronous generation step. +enum State<P, B> { + Empty, + Locale(LanguageIdentifier), + Solver { + locale: LanguageIdentifier, + solver: ParallelProblemSolver<GenerateBundles<P, B>>, + }, +} + +impl<P, B> Default for State<P, B> { + fn default() -> Self { + Self::Empty + } +} + +impl<P, B> State<P, B> { + fn get_locale(&self) -> &LanguageIdentifier { + match self { + Self::Locale(locale) => locale, + Self::Solver { locale, .. } => locale, + Self::Empty => unreachable!("Attempting to get a locale for an empty state."), + } + } + + fn take_solver(&mut self) -> ParallelProblemSolver<GenerateBundles<P, B>> { + replace_with::replace_with_or_default_and_return(self, |self_| match self_ { + Self::Solver { locale, solver } => (solver, Self::Locale(locale)), + _ => unreachable!("Attempting to take a solver in an invalid state."), + }) + } + + fn put_back_solver(&mut self, solver: ParallelProblemSolver<GenerateBundles<P, B>>) { + replace_with::replace_with_or_default(self, |self_| match self_ { + Self::Locale(locale) => Self::Solver { locale, solver }, + _ => unreachable!("Attempting to put back a solver in an invalid state."), + }) + } +} + +pub struct GenerateBundles<P, B> { + /// Do not access the metasources in the registry, as they may be mutated between + /// async iterations. + reg: L10nRegistry<P, B>, + /// This is an immutable shallow copy of the MetaSources that should not be mutated + /// during the iteration process. This ensures that the iterator will still be + /// valid if the L10nRegistry is mutated while iterating through the sources. + metasources: MetaSources, + locales: std::vec::IntoIter<LanguageIdentifier>, + current_metasource: usize, + resource_ids: Vec<ResourceId>, + state: State<P, B>, +} + +impl<P, B> GenerateBundles<P, B> { + fn new( + reg: L10nRegistry<P, B>, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + metasources: MetaSources, + ) -> Self { + Self { + reg, + metasources, + locales, + current_metasource: 0, + resource_ids, + state: State::Empty, + } + } +} + +pub type ResourceSetStream = Collect<FuturesOrdered<ResourceStatus>, Vec<ResourceOption>>; +pub struct TestResult(ResourceSetStream); +impl std::marker::Unpin for TestResult {} + +impl Future for TestResult { + type Output = Vec<bool>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + let pinned = Pin::new(&mut self.0); + pinned + .poll(cx) + .map(|set| set.iter().map(|c| !c.is_required_and_missing()).collect()) + } +} + +impl<'l, P, B> AsyncTester for GenerateBundles<P, B> { + type Result = TestResult; + + fn test_async(&self, query: Vec<(usize, usize)>) -> Self::Result { + let locale = self.state.get_locale(); + + let stream = query + .iter() + .map(|(res_idx, source_idx)| { + let resource_id = &self.resource_ids[*res_idx]; + self.metasources + .filesource(self.current_metasource, *source_idx) + .fetch_file(locale, resource_id) + }) + .collect::<FuturesOrdered<_>>(); + TestResult(stream.collect::<_>()) + } +} + +#[async_trait::async_trait(?Send)] +impl<P, B> BundleStream for GenerateBundles<P, B> { + async fn prefetch_async(&mut self) { + todo!(); + } +} + +/// Generate [FluentBundles](FluentBundle) asynchronously. +impl<P, B> Stream for GenerateBundles<P, B> +where + P: ErrorReporter, + B: BundleAdapter, +{ + type Item = Result<FluentBundle, (FluentBundle, Vec<FluentError>)>; + + /// Asynchronously try and get a solver, and then with the solver generate a bundle. + /// If the solver is not ready yet, then this function will return as `Pending`, and + /// the Future runner will need to re-enter at a later point to try again. + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> { + if self.metasources.is_empty() { + // There are no metasources available, so no bundles can be generated. + return None.into(); + } + loop { + if let State::Solver { .. } = self.state { + // A solver has already been set up, continue iterating through the + // resources and generating a bundle. + + // Pin the solver so that the async try_poll_next can be called. + let mut solver = self.state.take_solver(); + let pinned_solver = Pin::new(&mut solver); + + if let std::task::Poll::Ready(solver_result) = + pinned_solver.try_poll_next(cx, &self, false) + { + // The solver is ready, but may not have generated an ordering. + + if let Ok(Some(order)) = solver_result { + // The solver resolved an ordering, and a bundle may be able + // to be generated. + + let bundle = self.metasources.bundle_from_order( + self.current_metasource, + self.state.get_locale().clone(), + &order, + &self.resource_ids, + &self.reg.shared.provider, + self.reg.shared.bundle_adapter.as_ref(), + ); + + self.state.put_back_solver(solver); + + if bundle.is_some() { + // The bundle was successfully generated. + return bundle.into(); + } + + // No bundle was generated, continue on. + continue; + } + + // There is no bundle ordering available. + + if self.current_metasource > 0 { + // There are more metasources, create a new solver and try the + // next metasource. If there is an error in the solver_result + // ignore it for now, since there are more metasources. + self.current_metasource -= 1; + let solver = ParallelProblemSolver::new( + self.resource_ids.len(), + self.metasources.get(self.current_metasource).len(), + ); + self.state = State::Solver { + locale: self.state.get_locale().clone(), + solver, + }; + continue; + } + + if let Err(idx) = solver_result { + // Since there are no more metasources, and there is an error, + // report it instead of ignoring it. + self.reg.shared.provider.report_errors(vec![ + L10nRegistryError::MissingResource { + locale: self.state.get_locale().clone(), + resource_id: self.resource_ids[idx].clone(), + }, + ]); + } + + // There are no more metasources. + self.state = State::Empty; + continue; + } + + // The solver is not ready yet, so exit out of this async task + // and mark it as pending. It can be tried again later. + self.state.put_back_solver(solver); + return std::task::Poll::Pending; + } + + // There are no more metasources to search. + + // Try the next locale. + if let Some(locale) = self.locales.next() { + // Restart at the end of the metasources for this locale, and iterate + // backwards. + let last_metasource_idx = self.metasources.len() - 1; + self.current_metasource = last_metasource_idx; + + let solver = ParallelProblemSolver::new( + self.resource_ids.len(), + self.metasources.get(self.current_metasource).len(), + ); + self.state = State::Solver { locale, solver }; + + // Continue iterating on the next solver. + continue; + } + + // There are no more locales or metasources to search. This iterator + // is done. + return None.into(); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs new file mode 100644 index 0000000000..c342aa55aa --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs @@ -0,0 +1,363 @@ +mod asynchronous; +mod synchronous; + +use crate::{ + env::ErrorReporter, + errors::L10nRegistrySetupError, + fluent::FluentBundle, + source::{FileSource, ResourceId}, +}; +use fluent_bundle::FluentResource; +use fluent_fallback::generator::BundleGenerator; +use rustc_hash::FxHashSet; +use std::{ + cell::{Ref, RefCell, RefMut}, + collections::HashSet, + rc::Rc, +}; +use unic_langid::LanguageIdentifier; + +pub use asynchronous::GenerateBundles; +pub use synchronous::GenerateBundlesSync; + +pub type FluentResourceSet = Vec<Rc<FluentResource>>; + +/// The shared information that makes up the configuration the L10nRegistry. It is +/// broken out into a separate struct so that it can be shared via an Rc pointer. +#[derive(Default)] +struct Shared<P, B> { + metasources: RefCell<MetaSources>, + provider: P, + bundle_adapter: Option<B>, +} + +/// [FileSources](FileSource) represent a single directory location to look for .ftl +/// files. These are Stored in a [Vec]. For instance, in a built version of Firefox with +/// the en-US locale, each [FileSource] may represent a different folder with many +/// different files. +/// +/// Firefox supports other *meta sources* for localization files in the form of language +/// packs which can be downloaded from the addon store. These language packs then would +/// be a separate metasource than the app' language. This [MetaSources] adds another [Vec] +/// over the [Vec] of [FileSources](FileSource) in order to provide a unified way to +/// iterate over all possible [FileSource] locations to finally obtain the final bundle. +/// +/// This structure uses an [Rc] to point to the [FileSource] so that a shallow copy +/// of these [FileSources](FileSource) can be obtained for iteration. This makes +/// it quick to copy the list of [MetaSources] for iteration, and guards against +/// invalidating that async nature of iteration when the underlying data mutates. +/// +/// Note that the async iteration of bundles is still only happening in one thread, +/// and is not multi-threaded. The processing is just split over time. +/// +/// The [MetaSources] are ultimately owned by the [Shared] in a [RefCell] so that the +/// source of truth can be mutated, and shallow copies of the [MetaSources] used for +/// iteration will be unaffected. +/// +/// Deriving [Clone] here is a relatively cheap operation, since the [Rc] will be cloned, +/// and point to the original [FileSource]. +#[derive(Default, Clone)] +pub struct MetaSources(Vec<Vec<Rc<FileSource>>>); + +impl MetaSources { + /// Iterate over all FileSources in all MetaSources. + pub fn filesources(&self) -> impl Iterator<Item = &Rc<FileSource>> { + self.0.iter().flatten() + } + + /// Iterate over all FileSources in all MetaSources. + pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Vec<Rc<FileSource>>> { + self.0.iter_mut() + } + + /// The number of metasources. + pub fn len(&self) -> usize { + self.0.len() + } + + /// If there are no metasources. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Clears out all metasources. + pub fn clear(&mut self) { + self.0.clear(); + } + + /// Clears out only empty metasources. + pub fn clear_empty_metasources(&mut self) { + self.0.retain(|metasource| !metasource.is_empty()); + } + + /// Adds a [FileSource] to its appropriate metasource. + pub fn add_filesource(&mut self, new_source: FileSource) { + if let Some(metasource) = self + .0 + .iter_mut() + .find(|source| source[0].metasource == new_source.metasource) + { + // A metasource was found, add to the existing one. + metasource.push(Rc::new(new_source)); + } else { + // Create a new metasource. + self.0.push(vec![Rc::new(new_source)]); + } + } + + /// Adds a [FileSource] to its appropriate metasource. + pub fn update_filesource(&mut self, new_source: &FileSource) -> bool { + if let Some(metasource) = self + .0 + .iter_mut() + .find(|source| source[0].metasource == new_source.metasource) + { + if let Some(idx) = metasource.iter().position(|source| **source == *new_source) { + *metasource.get_mut(idx).unwrap() = Rc::new(new_source.clone()); + return true; + } + } + false + } + + /// Get a metasource by index, but provide a nice error message if the index + /// is out of bounds. + pub fn get(&self, metasource_idx: usize) -> &Vec<Rc<FileSource>> { + if let Some(metasource) = self.0.get(metasource_idx) { + return &metasource; + } + panic!( + "Metasource index of {} is out of range of the list of {} meta sources.", + metasource_idx, + self.0.len() + ); + } + + /// Get a [FileSource] from a metasource, but provide a nice error message if the + /// index is out of bounds. + pub fn filesource(&self, metasource_idx: usize, filesource_idx: usize) -> &FileSource { + let metasource = self.get(metasource_idx); + let reversed_idx = metasource.len() - 1 - filesource_idx; + if let Some(file_source) = metasource.get(reversed_idx) { + return file_source; + } + panic!( + "File source index of {} is out of range of the list of {} file sources.", + filesource_idx, + metasource.len() + ); + } + + /// Get a [FileSource] by name from a metasource. This is useful for testing. + #[cfg(feature = "test-fluent")] + pub fn file_source_by_name(&self, metasource_idx: usize, name: &str) -> Option<&FileSource> { + use std::borrow::Borrow; + self.get(metasource_idx) + .iter() + .find(|&source| source.name == name) + .map(|source| source.borrow()) + } + + /// Get an iterator for the [FileSources](FileSource) that match the [LanguageIdentifier] + /// and [ResourceId]. + #[cfg(feature = "test-fluent")] + pub fn get_sources_for_resource<'l>( + &'l self, + metasource_idx: usize, + langid: &'l LanguageIdentifier, + resource_id: &'l ResourceId, + ) -> impl Iterator<Item = &FileSource> { + use std::borrow::Borrow; + self.get(metasource_idx) + .iter() + .filter(move |source| source.has_file(langid, resource_id) != Some(false)) + .map(|source| source.borrow()) + } +} + +/// The [BundleAdapter] can adapt the bundle to the environment with such actions as +/// setting the platform, and hooking up functions such as Fluent's DATETIME and +/// NUMBER formatting functions. +pub trait BundleAdapter { + fn adapt_bundle(&self, bundle: &mut FluentBundle); +} + +/// The L10nRegistry is the main struct for owning the registry information. +/// +/// `P` - A provider +/// `B` - A bundle adapter +#[derive(Clone)] +pub struct L10nRegistry<P, B> { + shared: Rc<Shared<P, B>>, +} + +impl<P, B> L10nRegistry<P, B> { + /// Create a new [L10nRegistry] from a provider. + pub fn with_provider(provider: P) -> Self { + Self { + shared: Rc::new(Shared { + metasources: Default::default(), + provider, + bundle_adapter: None, + }), + } + } + + /// Set the bundle adapter. See [BundleAdapter] for more information. + pub fn set_bundle_adapter(&mut self, bundle_adapter: B) -> Result<(), L10nRegistrySetupError> + where + B: BundleAdapter, + { + let shared = Rc::get_mut(&mut self.shared).ok_or(L10nRegistrySetupError::RegistryLocked)?; + shared.bundle_adapter = Some(bundle_adapter); + Ok(()) + } + + pub fn try_borrow_metasources(&self) -> Result<Ref<MetaSources>, L10nRegistrySetupError> { + self.shared + .metasources + .try_borrow() + .map_err(|_| L10nRegistrySetupError::RegistryLocked) + } + + pub fn try_borrow_metasources_mut( + &self, + ) -> Result<RefMut<MetaSources>, L10nRegistrySetupError> { + self.shared + .metasources + .try_borrow_mut() + .map_err(|_| L10nRegistrySetupError::RegistryLocked) + } + + /// Adds a new [FileSource] to the registry and to its appropriate metasource. If the + /// metasource for this [FileSource] does not exist, then it is created. + pub fn register_sources( + &self, + new_sources: Vec<FileSource>, + ) -> Result<(), L10nRegistrySetupError> { + for new_source in new_sources { + self.try_borrow_metasources_mut()? + .add_filesource(new_source); + } + Ok(()) + } + + /// Update the information about sources already stored in the registry. Each + /// [FileSource] provided must exist, or else a [L10nRegistrySetupError] will + /// be returned. + pub fn update_sources( + &self, + new_sources: Vec<FileSource>, + ) -> Result<(), L10nRegistrySetupError> { + for new_source in new_sources { + if !self + .try_borrow_metasources_mut()? + .update_filesource(&new_source) + { + return Err(L10nRegistrySetupError::MissingSource { + name: new_source.name, + }); + } + } + Ok(()) + } + + /// Remove the provided sources. If a metasource becomes empty after this operation, + /// the metasource is also removed. + pub fn remove_sources<S>(&self, del_sources: Vec<S>) -> Result<(), L10nRegistrySetupError> + where + S: ToString, + { + let del_sources: Vec<String> = del_sources.into_iter().map(|s| s.to_string()).collect(); + + for metasource in self.try_borrow_metasources_mut()?.iter_mut() { + metasource.retain(|source| !del_sources.contains(&source.name)); + } + + self.try_borrow_metasources_mut()?.clear_empty_metasources(); + + Ok(()) + } + + /// Clears out all metasources and sources. + pub fn clear_sources(&self) -> Result<(), L10nRegistrySetupError> { + self.try_borrow_metasources_mut()?.clear(); + Ok(()) + } + + /// Flattens out all metasources and returns the complete list of source names. + pub fn get_source_names(&self) -> Result<Vec<String>, L10nRegistrySetupError> { + Ok(self + .try_borrow_metasources()? + .filesources() + .map(|s| s.name.clone()) + .collect()) + } + + /// Checks if any metasources has a source, by the name. + pub fn has_source(&self, name: &str) -> Result<bool, L10nRegistrySetupError> { + Ok(self + .try_borrow_metasources()? + .filesources() + .any(|source| source.name == name)) + } + + /// Get a [FileSource] by name by searching through all meta sources. + pub fn file_source_by_name( + &self, + name: &str, + ) -> Result<Option<FileSource>, L10nRegistrySetupError> { + Ok(self + .try_borrow_metasources()? + .filesources() + .find(|source| source.name == name) + .map(|source| (**source).clone())) + } + + /// Returns a unique list of locale names from all sources. + pub fn get_available_locales(&self) -> Result<Vec<LanguageIdentifier>, L10nRegistrySetupError> { + let mut result = HashSet::new(); + let metasources = self.try_borrow_metasources()?; + for source in metasources.filesources() { + for locale in source.locales() { + result.insert(locale); + } + } + Ok(result.into_iter().map(|l| l.to_owned()).collect()) + } +} + +/// Defines how to generate bundles synchronously and asynchronously. +impl<P, B> BundleGenerator for L10nRegistry<P, B> +where + P: ErrorReporter + Clone, + B: BundleAdapter + Clone, +{ + type Resource = Rc<FluentResource>; + type Iter = GenerateBundlesSync<P, B>; + type Stream = GenerateBundles<P, B>; + type LocalesIter = std::vec::IntoIter<LanguageIdentifier>; + + /// The synchronous version of the bundle generator. This is hooked into Gecko + /// code via the `l10nregistry_generate_bundles_sync` function. + fn bundles_iter( + &self, + locales: Self::LocalesIter, + resource_ids: FxHashSet<ResourceId>, + ) -> Self::Iter { + let resource_ids = resource_ids.into_iter().collect(); + self.generate_bundles_sync(locales, resource_ids) + } + + /// The asynchronous version of the bundle generator. This is hooked into Gecko + /// code via the `l10nregistry_generate_bundles` function. + fn bundles_stream( + &self, + locales: Self::LocalesIter, + resource_ids: FxHashSet<ResourceId>, + ) -> Self::Stream { + let resource_ids = resource_ids.into_iter().collect(); + self.generate_bundles(locales, resource_ids) + .expect("Unable to get the MetaSources.") + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs b/intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs new file mode 100644 index 0000000000..097ca68eee --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs @@ -0,0 +1,307 @@ +use super::{BundleAdapter, L10nRegistry, MetaSources}; +use crate::env::ErrorReporter; +use crate::errors::L10nRegistryError; +use crate::fluent::{FluentBundle, FluentError}; +use crate::solver::{SerialProblemSolver, SyncTester}; +use crate::source::ResourceOption; +use fluent_fallback::{generator::BundleIterator, types::ResourceId}; +use unic_langid::LanguageIdentifier; + +impl MetaSources { + pub(crate) fn bundle_from_order<P, B>( + &self, + metasource: usize, + locale: LanguageIdentifier, + source_order: &[usize], + resource_ids: &[ResourceId], + error_reporter: &P, + bundle_adapter: Option<&B>, + ) -> Option<Result<FluentBundle, (FluentBundle, Vec<FluentError>)>> + where + P: ErrorReporter, + B: BundleAdapter, + { + let mut bundle = FluentBundle::new(vec![locale.clone()]); + + if let Some(bundle_adapter) = bundle_adapter { + bundle_adapter.adapt_bundle(&mut bundle); + } + + let mut errors = vec![]; + + for (&source_idx, resource_id) in source_order.iter().zip(resource_ids.iter()) { + let source = self.filesource(metasource, source_idx); + if let ResourceOption::Some(res) = + source.fetch_file_sync(&locale, resource_id, /* overload */ true) + { + if source.options.allow_override { + bundle.add_resource_overriding(res); + } else if let Err(err) = bundle.add_resource(res) { + errors.extend(err.into_iter().map(|error| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: None, + error, + })); + } + } else if resource_id.is_required() { + return None; + } + } + + if !errors.is_empty() { + error_reporter.report_errors(errors); + } + Some(Ok(bundle)) + } +} + +impl<P, B> L10nRegistry<P, B> +where + P: Clone, + B: Clone, +{ + /// A test-only function for easily generating bundles for a single langid. + #[cfg(feature = "test-fluent")] + pub fn generate_bundles_for_lang_sync( + &self, + langid: LanguageIdentifier, + resource_ids: Vec<ResourceId>, + ) -> GenerateBundlesSync<P, B> { + let lang_ids = vec![langid]; + + GenerateBundlesSync::new(self.clone(), lang_ids.into_iter(), resource_ids) + } + + /// Wiring for hooking up the synchronous bundle generation to the + /// [BundleGenerator] trait. + pub fn generate_bundles_sync( + &self, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + ) -> GenerateBundlesSync<P, B> { + GenerateBundlesSync::new(self.clone(), locales, resource_ids) + } +} + +enum State { + Empty, + Locale(LanguageIdentifier), + Solver { + locale: LanguageIdentifier, + solver: SerialProblemSolver, + }, +} + +impl Default for State { + fn default() -> Self { + Self::Empty + } +} + +impl State { + fn get_locale(&self) -> &LanguageIdentifier { + match self { + Self::Locale(locale) => locale, + Self::Solver { locale, .. } => locale, + Self::Empty => unreachable!("Attempting to get a locale for an empty state."), + } + } + + fn take_solver(&mut self) -> SerialProblemSolver { + replace_with::replace_with_or_default_and_return(self, |self_| match self_ { + Self::Solver { locale, solver } => (solver, Self::Locale(locale)), + _ => unreachable!("Attempting to take a solver in an invalid state."), + }) + } + + fn put_back_solver(&mut self, solver: SerialProblemSolver) { + replace_with::replace_with_or_default(self, |self_| match self_ { + Self::Locale(locale) => Self::Solver { locale, solver }, + _ => unreachable!("Attempting to put back a solver in an invalid state."), + }) + } +} + +pub struct GenerateBundlesSync<P, B> { + reg: L10nRegistry<P, B>, + locales: std::vec::IntoIter<LanguageIdentifier>, + current_metasource: usize, + resource_ids: Vec<ResourceId>, + state: State, +} + +impl<P, B> GenerateBundlesSync<P, B> { + fn new( + reg: L10nRegistry<P, B>, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + ) -> Self { + Self { + reg, + locales, + current_metasource: 0, + resource_ids, + state: State::Empty, + } + } +} + +impl<P, B> SyncTester for GenerateBundlesSync<P, B> { + fn test_sync(&self, res_idx: usize, source_idx: usize) -> bool { + let locale = self.state.get_locale(); + let resource_id = &self.resource_ids[res_idx]; + !self + .reg + .try_borrow_metasources() + .expect("Unable to get the MetaSources.") + .filesource(self.current_metasource, source_idx) + .fetch_file_sync(locale, resource_id, /* overload */ true) + .is_required_and_missing() + } +} + +impl<P, B> BundleIterator for GenerateBundlesSync<P, B> +where + P: ErrorReporter, +{ + fn prefetch_sync(&mut self) { + if let State::Solver { .. } = self.state { + let mut solver = self.state.take_solver(); + if let Err(idx) = solver.try_next(self, true) { + self.reg + .shared + .provider + .report_errors(vec![L10nRegistryError::MissingResource { + locale: self.state.get_locale().clone(), + resource_id: self.resource_ids[idx].clone(), + }]); + } + self.state.put_back_solver(solver); + return; + } + + if let Some(locale) = self.locales.next() { + let mut solver = SerialProblemSolver::new( + self.resource_ids.len(), + self.reg + .try_borrow_metasources() + .expect("Unable to get the MetaSources.") + .get(self.current_metasource) + .len(), + ); + self.state = State::Locale(locale.clone()); + if let Err(idx) = solver.try_next(self, true) { + self.reg + .shared + .provider + .report_errors(vec![L10nRegistryError::MissingResource { + locale, + resource_id: self.resource_ids[idx].clone(), + }]); + } + self.state.put_back_solver(solver); + } + } +} + +impl<P, B> Iterator for GenerateBundlesSync<P, B> +where + P: ErrorReporter, + B: BundleAdapter, +{ + type Item = Result<FluentBundle, (FluentBundle, Vec<FluentError>)>; + + /// Synchronously generate a bundle based on a solver. + fn next(&mut self) -> Option<Self::Item> { + let metasources = self + .reg + .try_borrow_metasources() + .expect("Unable to get the MetaSources."); + + if metasources.is_empty() { + // There are no metasources available, so no bundles can be generated. + return None; + } + + loop { + if let State::Solver { .. } = self.state { + // A solver has already been set up, continue iterating through the + // resources and generating a bundle. + let mut solver = self.state.take_solver(); + let solver_result = solver.try_next(self, false); + + if let Ok(Some(order)) = solver_result { + // The solver resolved an ordering, and a bundle may be able + // to be generated. + + let bundle = metasources.bundle_from_order( + self.current_metasource, + self.state.get_locale().clone(), + &order, + &self.resource_ids, + &self.reg.shared.provider, + self.reg.shared.bundle_adapter.as_ref(), + ); + + self.state.put_back_solver(solver); + + if bundle.is_some() { + // The bundle was successfully generated. + return bundle; + } + + // No bundle was generated, continue on. + continue; + } + + // There is no bundle ordering available. + + if self.current_metasource > 0 { + // There are more metasources, create a new solver and try the + // next metasource. If there is an error in the solver_result + // ignore it for now, since there are more metasources. + self.current_metasource -= 1; + let solver = SerialProblemSolver::new( + self.resource_ids.len(), + metasources.get(self.current_metasource).len(), + ); + self.state = State::Solver { + locale: self.state.get_locale().clone(), + solver, + }; + continue; + } + + if let Err(idx) = solver_result { + // Since there are no more metasources, and there is an error, + // report it instead of ignoring it. + self.reg.shared.provider.report_errors(vec![ + L10nRegistryError::MissingResource { + locale: self.state.get_locale().clone(), + resource_id: self.resource_ids[idx].clone(), + }, + ]); + } + + self.state = State::Empty; + continue; + } + + // Try the next locale, or break out of the loop if there are none left. + let locale = self.locales.next()?; + + // Restart at the end of the metasources for this locale, and iterate + // backwards. + let last_metasource_idx = metasources.len() - 1; + self.current_metasource = last_metasource_idx; + + let solver = SerialProblemSolver::new( + self.resource_ids.len(), + metasources.get(self.current_metasource).len(), + ); + + // Continue iterating on the next solver. + self.state = State::Solver { locale, solver }; + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/README.md b/intl/l10n/rust/l10nregistry-rs/src/solver/README.md new file mode 100644 index 0000000000..acd56b52b4 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/README.md @@ -0,0 +1,239 @@ + +Source Order Problem Solver +====================== + +This module contains an algorithm used to power the `FluentBundle` generator in `L10nRegistry`. + +The main concept behind it is a problem solver which takes a list of resources and a list of sources and computes all possible iterations of valid combinations of source orders that allow for creation of `FluentBundle` with the requested resources. + +The algorithm is notoriously hard to read, write, and modify, which prompts this documentation to be extensive and provide an example with diagram presentations to aid the reader. + +# Example +For the purpose of a graphical illustration of the example, we will evaluate a scenario with two sources and three resources. + +The sources and resource identifiers will be named in concise way (*1* or *A*) to simplify diagrams, while a more tangible names derived from real-world examples in Firefox use-case will be listed in their initial definition. + +### Sources +A source can be a packaged directory, and a language pack, or any other directory, zip file, or remote source which contains localization resource files. +In the example, we have two sources: +* Source 1 named ***0*** (e.g. `browser`) +* Source 2 named ***1*** (e.g. `toolkit`) + +### Resources +A resource is a single Fluent Translation List file. `FluentBundle` is a combination of such resources used together to resolve translations. This algorithm operates on lists of resource identifiers which represent relative paths within the source. +In the example we have three resources: +* Resource 1 named ***A*** (e.g. `branding/brand.ftl`) +* Resource 2 named ***B*** (e.g. `errors/common.ftl`) +* Resource 3 named ***C*** (e.g. `menu/list.ftl`) + +## Task +The task in this example is to generate all possible iterations of the three resources from the given two sources. Since I/O is expensive, and in most production scenarios all necessary translations are available in the first set, the iterator is used to lazily fallback on the alternative sets only in case of missing translations. + +If all resources are available in both sources, the iterator should produce the following results: +1. `[A0, B0, C0]` +2. `[A0, B0, C1]` +3. `[A0, B1, C0]` +4. `[A0, B1, C1]` +5. `[A1, B0, C0]` +6. `[A1, B0, C1]` +7. `[A1, B1, C0]` +8. `[A1, B1, C1]` + +Since the resources are defined by their column, we can store the resources as `[A, B, C]` separately and simplify the notation to just: +1. `[0, 0, 0]` +2. `[0, 0, 1]` +3. `[0, 1, 0]` +4. `[0, 1, 1]` +5. `[1, 0, 0]` +6. `[1, 0, 1]` +7. `[1, 1, 0]` +8. `[1, 1, 1]` + +This notation will be used from now on. + +## State + +For the in-detail diagrams on the algorithm, we'll use another way to look at the iterator - by evaluating it state. At every point of the algorithm, there is a *partial solution* which may lead to a *complete solution*. It is encoded as: + +```rust +struct Solution { + candidate: Vec<usize>, + idx: usize, +} +``` + +and which starting point can be visualized as: + +```text + ▼ +┌┲━┱┬───┬───┐ +│┃0┃│ │ │ +└╂─╂┴───┴───┘ + ┃ ┃ + ┗━┛ +``` +###### Diagrams generated with use of http://marklodato.github.io/js-boxdrawing/ + +where the horizontal block is a candidate, vertical block is a set of sources possible for each resource, and the arrow represents the index of a resource the iterator is currently evaluating. + +With those tools introduced, we can now guide the reader through how the algorithm works. +But before we do that, it is important to justify writing a custom algorithm in place of existing generic solutions, and explain the two testing strategies which heavily impact the algorithm. + +# Existing libraries +Intuitively, the starting point to exploration of the problem scope would be to look at it as some variation of the [Cartesian Product](https://en.wikipedia.org/wiki/Cartesian_product) iterator. + +#### Python + +In Python, `itertools` package provides a function [`itertools::product`](https://docs.python.org/3/library/itertools.html#itertools.product) which can be used to generate such iterator: +```python +import itertools + +for set in itertools.product(range(2), repeat=3): + print(set) +``` + +#### Rust + +In Rust, crate [`itertools`](https://crates.io/crates/itertools) provides, [`multi_cartesian_product`](https://docs.rs/itertools/0.9.0/itertools/trait.Itertools.html#method.multi_cartesian_product) which can be used like this: +```rust +use itertools::Itertools; + +let multi_prod = (0..3).map(|i| 0..2) + .multi_cartesian_product(); + +for set in multi_prod { + println!("{:?}", set); +} +``` +([playground](https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=6ef231f6b011b234babb0aa3e68b78ab)) + +#### Reasons for a custom algorithm + +Unfortunately, the computational complexity of generating all possible sets is growing exponentially, both in the cost of CPU and memory use. +On a high-end laptop, computing the sets for all possible variations of the example above generates *8* sets and takes only *700 nanoseconds*, but computing the same for four sources and 16 resources (a scenario theoretically possible in Firefox with one language pack and Preferences UI for example) generates over *4 billion* sets and takes over *2 minutes*. + +Since one part of static cost is the I/O, the application of a [Memoization](https://en.wikipedia.org/wiki/Memoization) technique allows us to minimize the cost of constructing, storing and retrieving sets. + +Second important observation is that in most scenarios any resource exists in only some of the sources, and ability to bail out from a branch of candidates that cannot lead to a solution yields significantly fewer permutations in result. + +## Optimizations + +The algorithm used here is highly efficient. For the conservative scenario listed above, where 4 sources and 15 resources are all present in every source, the total time on the reference hardware is cut from *2 minutes* to *24 seconds*, while generating the same *4 billion* sets for a **5x** performance improvement. + +### Streaming Iterator +Unline regular iterator, a streaming iterator allows a borrowed reference to be returned, which in this case, where the solver yields a read-only "view" of a solution, allows us to avoid having to clone it. + +### Cache +Memory is much less of a problem for the algorithm than CPU usage, so the solver uses a matrix of source/resource `Option` to memoize visited cells. This allows for each source/resource combination to be tested only once after which all future tests can be skipped. + +### Backtracking +This optimization allows to benefit from the recognition of the fact that most resources are only available in some sources. +Instead of generating all possible sets and then ignoring ones which are incomplete, it allows the algorithm to [backtrack](https://en.wikipedia.org/wiki/Backtracking) from partial candidates that cannot lead to a complete solution. + +That technique is very powerful in the `L10nRegistry` use case and in many scenarios leads to 10-100x speed ups even in cases where all sets have to be generated. + +# Serial vs Parallel Testing +At the core of the solver is a *tester* component which is responsible for eagerly evaluating candidates to allow for early bailouts from partial solutions which cannot lead to a complete solution. + +This can be performed in one of two ways: + +### Serial + The algorithm is synchronous and each extension of the candidate is evaluated serially, one by one, allowing the for *backtracking* as soon as a given extension of a partial solution is confirmed to not lead to a complete solution. + +Bringing back the initial state of the solver: + +```text + ▼ +┌┲━┱┬───┬───┐ +│┃0┃│ │ │ +└╂─╂┴───┴───┘ + ┃ ┃ + ┗━┛ +``` + +The tester will evaluate whether the first resource **A** is available in the first source **0**. The testing will be performed synchronously, and the result will inform the algorithm on whether the candidate may lead to a complete solution, or this branch should be bailed out from, and the next candidate must be tried. + +#### Success case + +If the test returns a success, the extensions of the candidate is generated: +```text + ▼ +┌┲━┱┬┲━┱┬───┐ +│┃0┃│┃0┃│ │ +└╂─╂┴╂─╂┴───┘ + ┃ ┃ ┃ ┃ + ┗━┛ ┗━┛ +``` + +When a candidate is complete, in other words, when the last cell of a candidate has been tested and did not lead to a backtrack, we know that the candidate is a solution to the problem, and we can yield it from the iterator. + +#### Failure case + +If the test returns a failure, the next step is to evaluate alternative source for the same resource. Let's assume that *Source 0* had *Resource A* but it does not have *Resource B*. In such case, the algorithm will increment the second cell's source index: + +```text + ▼ + ┏━┓ + ┃0┃ +┌┲━┱┬╂─╂┬───┐ +│┃0┃│┃1┃│ │ +└╂─╂┴┺━┹┴───┘ + ┃ ┃ + ┗━┛ + ``` + +and that will potentially lead to a partial solution `[0, 1, ]` to be stored for the next iteration. + +If the test fails and no more sources can be generated, the algorithm will *backtrack* from the current cell looking for a cell with the **highest** index prior to the cell that was being evaluated which is not yet on the last source. If such cell is found, the results of all cells **to the right** of the newfound cell are **erased** and the next branch can be evaluated. + +If no such cell can be found, that means that the iterator is complete. + +### Parallel + +If the testing can be performed in parallel, like an asynchronous I/O, the above *serial* solution is sub-optimal as it misses on the benefit of testing multiple cells at once. + +In such a scenario, the algorithm will construct a candidate that *can* be valid (bailing only from candidates that have been already memoized as unavailable), and then test all of the untested cells in that candidate at once. + +```text + ▼ +┌┲━┱┬┲━┱┬┲━┱┐ +│┃0┃│┃0┃│┃0┃│ +└╂─╂┴╂─╂┴╂─╂┘ + ┃ ┃ ┃ ┃ ┃ ┃ + ┗━┛ ┗━┛ ┗━┛ +``` + +When the parallel execution returns, the algorithm memoizes all new cell results and tests if the candidate is now a valid complete solution. + +#### Success case + +If the result a set of successes, the candidate is returned as a solution, and the algorithm proceeds to the same operation as if it was a failure. + +#### Failure case +If the result contains failures, the iterator will now backtrack to find the closest lower or equal cell to the current index which can be advanced to the next source. +In the example state above, the current cell can be advanced to *source 1* and then just a set of `[None, None, 1]` is to be evaluated by the tester (since we know that *A0* and *B0* are valid). + +If that is successful, the `[0, 0, 1]` set is a complete solution and is yielded. + +Then, if the iterator is resumed, the next state to be tested is: + +```text + ▼ + ┏━┓ + ┃0┃ +┌┲━┱┬╂─╂┬┲━┱┐ +│┃0┃│┃1┃│┃0┃│ +└╂─╂┴┺━┹┴╂─╂┘ + ┃ ┃ ┃ ┃ + ┗━┛ ┗━┛ +``` + +since cell *2* was at the highest index, cell *1* is the highest lower than *2* that was not at the highest source index position. That cell is advanced, and all cells after it are *pruned* (in this case, cell *2* is the only one). Then, the memoization kicks in, and since *A0* and *C0* are already cached as valid, the tester receives just `[None, 1, None]` to be tested and the algorithm continues. + +# Summary + +The algorithm explained above is tailored to the problem domain of `L10nRegistry` and is designed to be further extended in the future. + +It is important to maintain this guide up to date as any changes to the algorithm are to be made. + +Good luck. diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs new file mode 100644 index 0000000000..8357ac49fc --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs @@ -0,0 +1,121 @@ +mod parallel; +mod serial; + +pub use parallel::{AsyncTester, ParallelProblemSolver}; +pub use serial::{SerialProblemSolver, SyncTester}; + +pub struct ProblemSolver { + width: usize, + depth: usize, + + cache: Vec<Vec<Option<bool>>>, + + solution: Vec<usize>, + idx: usize, + + dirty: bool, +} + +impl ProblemSolver { + pub fn new(width: usize, depth: usize) -> Self { + Self { + width, + depth, + cache: vec![vec![None; depth]; width], + + solution: vec![0; width], + idx: 0, + + dirty: false, + } + } +} + +impl ProblemSolver { + pub fn bail(&mut self) -> bool { + if self.try_advance_source() { + true + } else { + self.try_backtrack() + } + } + + pub fn has_missing_cell(&self) -> Option<usize> { + for res_idx in 0..self.width { + if self.cache[res_idx].iter().all(|c| *c == Some(false)) { + return Some(res_idx); + } + } + None + } + + fn is_cell_missing(&self, res_idx: usize, source_idx: usize) -> bool { + if let Some(false) = self.cache[res_idx][source_idx] { + return true; + } + false + } + + fn is_current_cell_missing(&self) -> bool { + let res_idx = self.idx; + let source_idx = self.solution[res_idx]; + let cell = &self.cache[res_idx][source_idx]; + if let Some(false) = cell { + return true; + } + false + } + + pub fn try_advance_resource(&mut self) -> bool { + if self.idx >= self.width - 1 { + false + } else { + self.idx += 1; + while self.is_current_cell_missing() { + if !self.try_advance_source() { + return false; + } + } + true + } + } + + pub fn try_advance_source(&mut self) -> bool { + while self.solution[self.idx] < self.depth - 1 { + self.solution[self.idx] += 1; + if !self.is_current_cell_missing() { + return true; + } + } + false + } + + pub fn try_backtrack(&mut self) -> bool { + while self.solution[self.idx] == self.depth - 1 { + if self.idx == 0 { + return false; + } + self.idx -= 1; + } + self.solution[self.idx] += 1; + self.prune() + } + + pub fn prune(&mut self) -> bool { + for i in self.idx + 1..self.width { + let mut source_idx = 0; + while self.is_cell_missing(i, source_idx) { + if source_idx >= self.depth - 1 { + return false; + } + source_idx += 1; + } + self.solution[i] = source_idx; + } + true + } + + pub fn is_complete(&self) -> bool { + self.idx == self.width - 1 + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs new file mode 100644 index 0000000000..320ad65c89 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs @@ -0,0 +1,175 @@ +use super::ProblemSolver; +use std::ops::{Deref, DerefMut}; + +use futures::ready; +use std::future::Future; +use std::pin::Pin; + +pub trait AsyncTester { + type Result: Future<Output = Vec<bool>>; + + fn test_async(&self, query: Vec<(usize, usize)>) -> Self::Result; +} + +pub struct ParallelProblemSolver<T> +where + T: AsyncTester, +{ + solver: ProblemSolver, + current_test: Option<(T::Result, Vec<usize>)>, +} + +impl<T: AsyncTester> Deref for ParallelProblemSolver<T> { + type Target = ProblemSolver; + + fn deref(&self) -> &Self::Target { + &self.solver + } +} + +impl<T: AsyncTester> DerefMut for ParallelProblemSolver<T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.solver + } +} + +impl<T: AsyncTester> ParallelProblemSolver<T> { + pub fn new(width: usize, depth: usize) -> Self { + Self { + solver: ProblemSolver::new(width, depth), + current_test: None, + } + } +} + +type TestQuery = (Vec<(usize, usize)>, Vec<usize>); + +impl<T: AsyncTester> ParallelProblemSolver<T> { + pub fn try_generate_complete_candidate(&mut self) -> bool { + while !self.is_complete() { + while self.is_current_cell_missing() { + if !self.try_advance_source() { + return false; + } + } + if !self.try_advance_resource() { + return false; + } + } + true + } + + fn try_generate_test_query(&mut self) -> Result<TestQuery, usize> { + let mut test_cells = vec![]; + let query = self + .solution + .iter() + .enumerate() + .filter_map(|(res_idx, source_idx)| { + let cell = self.cache[res_idx][*source_idx]; + match cell { + None => { + test_cells.push(res_idx); + Some(Ok((res_idx, *source_idx))) + } + Some(false) => Some(Err(res_idx)), + Some(true) => None, + } + }) + .collect::<Result<_, _>>()?; + Ok((query, test_cells)) + } + + fn apply_test_result( + &mut self, + resources: Vec<bool>, + testing_cells: Vec<usize>, + ) -> Result<(), usize> { + let mut first_missing = None; + for (result, res_idx) in resources.into_iter().zip(testing_cells) { + let source_idx = self.solution[res_idx]; + self.cache[res_idx][source_idx] = Some(result); + if !result && first_missing.is_none() { + first_missing = Some(res_idx); + } + } + if let Some(idx) = first_missing { + Err(idx) + } else { + Ok(()) + } + } + + pub fn try_poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + tester: &T, + prefetch: bool, + ) -> std::task::Poll<Result<Option<Vec<usize>>, usize>> + where + <T as AsyncTester>::Result: Unpin, + { + if self.width == 0 || self.depth == 0 { + return Ok(None).into(); + } + + 'outer: loop { + if let Some((test, testing_cells)) = &mut self.current_test { + let pinned = Pin::new(test); + let set = ready!(pinned.poll(cx)); + let testing_cells = testing_cells.clone(); + + if let Err(res_idx) = self.apply_test_result(set, testing_cells) { + self.idx = res_idx; + self.prune(); + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx).into(); + } else { + return Ok(None).into(); + } + } + self.current_test = None; + continue 'outer; + } else { + self.current_test = None; + if !prefetch { + self.dirty = true; + } + return Ok(Some(self.solution.clone())).into(); + } + } else { + if self.dirty { + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx).into(); + } else { + return Ok(None).into(); + } + } + self.dirty = false; + } + while self.try_generate_complete_candidate() { + match self.try_generate_test_query() { + Ok((query, testing_cells)) => { + self.current_test = Some((tester.test_async(query), testing_cells)); + continue 'outer; + } + Err(res_idx) => { + self.idx = res_idx; + self.prune(); + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx).into(); + } else { + return Ok(None).into(); + } + } + } + } + } + return Ok(None).into(); + } + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs new file mode 100644 index 0000000000..9368c12c9e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs @@ -0,0 +1,76 @@ +use super::ProblemSolver; +use std::ops::{Deref, DerefMut}; + +pub trait SyncTester { + fn test_sync(&self, res_idx: usize, source_idx: usize) -> bool; +} + +pub struct SerialProblemSolver(ProblemSolver); + +impl Deref for SerialProblemSolver { + type Target = ProblemSolver; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for SerialProblemSolver { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl SerialProblemSolver { + pub fn new(width: usize, depth: usize) -> Self { + Self(ProblemSolver::new(width, depth)) + } +} + +impl SerialProblemSolver { + fn test_current_cell<T>(&mut self, tester: &T) -> bool + where + T: SyncTester, + { + let res_idx = self.idx; + let source_idx = self.solution[res_idx]; + let cell = &mut self.cache[res_idx][source_idx]; + *cell.get_or_insert_with(|| tester.test_sync(res_idx, source_idx)) + } + + pub fn try_next<T>(&mut self, tester: &T, prefetch: bool) -> Result<Option<&[usize]>, usize> + where + T: SyncTester, + { + if self.width == 0 || self.depth == 0 { + return Ok(None); + } + if self.dirty { + if !self.bail() { + return Ok(None); + } + self.dirty = false; + } + loop { + if !self.test_current_cell(tester) { + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx); + } else { + return Ok(None); + } + } + continue; + } + if self.is_complete() { + if !prefetch { + self.dirty = true; + } + return Ok(Some(&self.solution)); + } + if !self.try_advance_resource() { + return Ok(None); + } + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs new file mode 100644 index 0000000000..3a022990a6 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs @@ -0,0 +1,30 @@ +use async_trait::async_trait; +use fluent_fallback::types::ResourceId; +use std::io; + +/// The users of [`FileSource`] implement this trait to provide loading of +/// resources, returning the contents of a resource as a +/// `String`. [`FileSource`] handles the conversion from string representation +/// into `FluentResource`. +/// +/// [`FileSource`]: source/struct.FileSource.html +#[async_trait(?Send)] +pub trait FileFetcher { + /// Return the `String` representation for `path`. This version is + /// blocking. + /// + /// See [`fetch`](#tymethod.fetch). + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String>; + + /// Return the `String` representation for `path`. + /// + /// On success, returns `Poll::Ready(Ok(..))`. + /// + /// If no resource is available to be fetched, the method returns + /// `Poll::Pending` and arranges for the current task (via + /// `cx.waker().wake_by_ref()`) to receive a notification when the resource + /// is available. + /// + /// See [`fetch_sync`](#tymethod.fetch_sync) + async fn fetch(&self, path: &ResourceId) -> io::Result<String>; +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs new file mode 100644 index 0000000000..0408727fb7 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs @@ -0,0 +1,480 @@ +mod fetcher; +pub use fetcher::FileFetcher; +pub use fluent_fallback::types::{ResourceId, ToResourceId}; + +use crate::env::ErrorReporter; +use crate::errors::L10nRegistryError; +use crate::fluent::FluentResource; + +use std::{ + borrow::Borrow, + cell::RefCell, + fmt, + hash::{Hash, Hasher}, + pin::Pin, + rc::Rc, + task::Poll, +}; + +use futures::{future::Shared, Future, FutureExt}; +use rustc_hash::FxHashMap; +use unic_langid::LanguageIdentifier; + +pub type RcResource = Rc<FluentResource>; + +/// An option type whose None variant is either optional or required. +/// +/// This behaves similarly to the standard-library [`Option`] type +/// except that there are two [`None`]-like variants: +/// [`ResourceOption::MissingOptional`] and [`ResourceOption::MissingRequired`]. +#[derive(Clone, Debug)] +pub enum ResourceOption { + /// An available resource. + Some(RcResource), + /// A missing optional resource. + MissingOptional, + /// A missing required resource. + MissingRequired, +} + +impl ResourceOption { + /// Creates a resource option that is either [`ResourceOption::MissingRequired`] + /// or [`ResourceOption::MissingOptional`] based on whether the given [`ResourceId`] + /// is required or optional. + pub fn missing_resource(resource_id: &ResourceId) -> Self { + if resource_id.is_required() { + Self::MissingRequired + } else { + Self::MissingOptional + } + } + + /// Returns [`true`] if this option contains a recource, otherwise [`false`]. + pub fn is_some(&self) -> bool { + matches!(self, Self::Some(_)) + } + + /// Resource [`true`] if this option is missing a resource of any type, otherwise [`false`]. + pub fn is_none(&self) -> bool { + matches!(self, Self::MissingOptional | Self::MissingRequired) + } + + /// Returns [`true`] if this option is missing a required resource, otherwise [`false`]. + pub fn is_required_and_missing(&self) -> bool { + matches!(self, Self::MissingRequired) + } +} + +impl From<ResourceOption> for Option<RcResource> { + fn from(other: ResourceOption) -> Self { + match other { + ResourceOption::Some(id) => Some(id), + _ => None, + } + } +} + +pub type ResourceFuture = Shared<Pin<Box<dyn Future<Output = ResourceOption>>>>; + +#[derive(Debug, Clone)] +pub enum ResourceStatus { + /// The resource is missing. Don't bother trying to fetch. + MissingRequired, + MissingOptional, + /// The resource is loading and future will deliver the result. + Loading(ResourceFuture), + /// The resource is loaded and parsed. + Loaded(RcResource), +} + +impl From<ResourceOption> for ResourceStatus { + fn from(input: ResourceOption) -> Self { + match input { + ResourceOption::Some(res) => Self::Loaded(res), + ResourceOption::MissingOptional => Self::MissingOptional, + ResourceOption::MissingRequired => Self::MissingRequired, + } + } +} + +impl Future for ResourceStatus { + type Output = ResourceOption; + + fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> { + use ResourceStatus::*; + + let this = &mut *self; + + match this { + MissingRequired => ResourceOption::MissingRequired.into(), + MissingOptional => ResourceOption::MissingOptional.into(), + Loaded(res) => ResourceOption::Some(res.clone()).into(), + Loading(res) => Pin::new(res).poll(cx), + } + } +} + +/// `FileSource` provides a generic fetching and caching of fluent resources. +/// The user of `FileSource` provides a [`FileFetcher`](trait.FileFetcher.html) +/// implementation and `FileSource` takes care of the rest. +#[derive(Clone)] +pub struct FileSource { + /// Name of the FileSource, e.g. "browser" + pub name: String, + /// Pre-formatted path for the FileSource, e.g. "/browser/data/locale/{locale}/" + pub pre_path: String, + /// Metasource name for the FileSource, e.g. "app", "langpack" + /// Only sources from the same metasource are passed into the solver. + pub metasource: String, + /// The locales for which data is present in the FileSource, e.g. ["en-US", "pl"] + locales: Vec<LanguageIdentifier>, + shared: Rc<Inner>, + index: Option<Vec<String>>, + pub options: FileSourceOptions, +} + +struct Inner { + fetcher: Box<dyn FileFetcher>, + error_reporter: Option<RefCell<Box<dyn ErrorReporter>>>, + entries: RefCell<FxHashMap<String, ResourceStatus>>, +} + +impl fmt::Display for FileSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name) + } +} + +impl PartialEq<FileSource> for FileSource { + fn eq(&self, other: &Self) -> bool { + self.name == other.name && self.metasource == other.metasource + } +} + +impl Eq for FileSource {} + +impl Hash for FileSource { + fn hash<H: Hasher>(&self, state: &mut H) { + self.name.hash(state) + } +} + +#[derive(PartialEq, Clone, Debug, Default)] +pub struct FileSourceOptions { + pub allow_override: bool, +} + +impl FileSource { + /// Create a `FileSource` using the provided [`FileFetcher`](../trait.FileFetcher.html). + pub fn new( + name: String, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + pre_path: String, + options: FileSourceOptions, + fetcher: impl FileFetcher + 'static, + ) -> Self { + FileSource { + name, + metasource: metasource.unwrap_or_default(), + pre_path, + locales, + index: None, + shared: Rc::new(Inner { + entries: RefCell::new(FxHashMap::default()), + fetcher: Box::new(fetcher), + error_reporter: None, + }), + options, + } + } + + pub fn new_with_index( + name: String, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + pre_path: String, + options: FileSourceOptions, + fetcher: impl FileFetcher + 'static, + index: Vec<String>, + ) -> Self { + FileSource { + name, + metasource: metasource.unwrap_or_default(), + pre_path, + locales, + index: Some(index), + shared: Rc::new(Inner { + entries: RefCell::new(FxHashMap::default()), + fetcher: Box::new(fetcher), + error_reporter: None, + }), + options, + } + } + + pub fn set_reporter(&mut self, reporter: impl ErrorReporter + 'static) { + let shared = Rc::get_mut(&mut self.shared).unwrap(); + shared.error_reporter = Some(RefCell::new(Box::new(reporter))); + } +} + +fn calculate_pos_in_source(source: &str, idx: usize) -> (usize, usize) { + let mut ptr = 0; + let mut result = (1, 1); + for line in source.lines() { + let bytes = line.as_bytes().len(); + if ptr + bytes < idx { + ptr += bytes + 1; + result.0 += 1; + } else { + result.1 = idx - ptr + 1; + break; + } + } + result +} + +impl FileSource { + fn get_path(&self, locale: &LanguageIdentifier, resource_id: &ResourceId) -> String { + format!( + "{}{}", + self.pre_path.replace("{locale}", &locale.to_string()), + resource_id.value, + ) + } + + fn fetch_sync(&self, resource_id: &ResourceId) -> ResourceOption { + self.shared + .fetcher + .fetch_sync(resource_id) + .ok() + .map(|source| match FluentResource::try_new(source) { + Ok(res) => ResourceOption::Some(Rc::new(res)), + Err((res, errors)) => { + if let Some(reporter) = &self.shared.error_reporter { + reporter.borrow().report_errors( + errors + .into_iter() + .map(|e| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), + error: e.into(), + }) + .collect(), + ); + } + ResourceOption::Some(Rc::new(res)) + } + }) + .unwrap_or_else(|| ResourceOption::missing_resource(resource_id)) + } + + /// Attempt to synchronously fetch resource for the combination of `locale` + /// and `path`. Returns `Some(ResourceResult)` if the resource is available, + /// else `None`. + pub fn fetch_file_sync( + &self, + locale: &LanguageIdentifier, + resource_id: &ResourceId, + overload: bool, + ) -> ResourceOption { + use ResourceStatus::*; + + if self.has_file(locale, resource_id) == Some(false) { + return ResourceOption::missing_resource(resource_id); + } + + let full_path_id = self + .get_path(locale, resource_id) + .to_resource_id(resource_id.resource_type); + + let res = self.shared.lookup_resource(full_path_id.clone(), || { + self.fetch_sync(&full_path_id).into() + }); + + match res { + MissingRequired => ResourceOption::MissingRequired, + MissingOptional => ResourceOption::MissingOptional, + Loaded(res) => ResourceOption::Some(res), + Loading(..) if overload => { + // A sync load has been requested for the same resource that has + // a pending async load in progress. How do we handle this? + // + // Ideally, we would sync load and resolve all the pending + // futures with the result. With the current Futures and + // combinators, it's unclear how to proceed. One potential + // solution is to store a oneshot::Sender and + // Shared<oneshot::Receiver>. When the async loading future + // resolves it would check that the state is still `Loading`, + // and if so, send the result. The sync load would do the same + // send on the oneshot::Sender. + // + // For now, we warn and return the resource, paying the cost of + // duplication of the resource. + self.fetch_sync(&full_path_id) + } + Loading(..) => { + panic!("[l10nregistry] Attempting to synchronously load file {} while it's being loaded asynchronously.", &full_path_id.value); + } + } + } + + /// Attempt to fetch resource for the combination of `locale` and `path`. + /// Returns [`ResourceStatus`](enum.ResourceStatus.html) which is + /// a `Future` that can be polled. + pub fn fetch_file( + &self, + locale: &LanguageIdentifier, + resource_id: &ResourceId, + ) -> ResourceStatus { + use ResourceStatus::*; + + if self.has_file(locale, resource_id) == Some(false) { + return ResourceOption::missing_resource(resource_id).into(); + } + + let full_path_id = self + .get_path(locale, resource_id) + .to_resource_id(resource_id.resource_type); + + self.shared.lookup_resource(full_path_id.clone(), || { + let shared = self.shared.clone(); + Loading(read_resource(full_path_id, shared).boxed_local().shared()) + }) + } + + /// Determine if the `FileSource` has a loaded resource for the combination + /// of `locale` and `path`. Returns `Some(true)` if the file is loaded, else + /// `Some(false)`. `None` is returned if there is an outstanding async fetch + /// pending and the status is yet to be determined. + pub fn has_file<L: Borrow<LanguageIdentifier>>( + &self, + locale: L, + path: &ResourceId, + ) -> Option<bool> { + let locale = locale.borrow(); + if !self.locales.contains(locale) { + Some(false) + } else { + let full_path = self.get_path(locale, path); + if let Some(index) = &self.index { + return Some(index.iter().any(|p| p == &full_path)); + } + self.shared.has_file(&full_path) + } + } + + pub fn locales(&self) -> &[LanguageIdentifier] { + &self.locales + } + + pub fn get_index(&self) -> Option<&Vec<String>> { + self.index.as_ref() + } +} + +impl std::fmt::Debug for FileSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { + if let Some(index) = &self.index { + f.debug_struct("FileSource") + .field("name", &self.name) + .field("metasource", &self.metasource) + .field("locales", &self.locales) + .field("pre_path", &self.pre_path) + .field("index", index) + .finish() + } else { + f.debug_struct("FileSource") + .field("name", &self.name) + .field("metasource", &self.metasource) + .field("locales", &self.locales) + .field("pre_path", &self.pre_path) + .finish() + } + } +} + +impl Inner { + fn lookup_resource<F>(&self, resource_id: ResourceId, f: F) -> ResourceStatus + where + F: FnOnce() -> ResourceStatus, + { + let mut lock = self.entries.borrow_mut(); + lock.entry(resource_id.value).or_insert_with(f).clone() + } + + fn update_resource(&self, resource_id: ResourceId, resource: ResourceOption) -> ResourceOption { + let mut lock = self.entries.borrow_mut(); + let entry = lock.get_mut(&resource_id.value); + match entry { + Some(entry) => *entry = resource.clone().into(), + _ => panic!("Expected "), + } + resource + } + + pub fn has_file(&self, full_path: &str) -> Option<bool> { + match self.entries.borrow().get(full_path) { + Some(ResourceStatus::MissingRequired) => Some(false), + Some(ResourceStatus::MissingOptional) => Some(false), + Some(ResourceStatus::Loaded(_)) => Some(true), + Some(ResourceStatus::Loading(_)) | None => None, + } + } +} + +async fn read_resource(resource_id: ResourceId, shared: Rc<Inner>) -> ResourceOption { + let resource = shared + .fetcher + .fetch(&resource_id) + .await + .ok() + .map(|source| match FluentResource::try_new(source) { + Ok(res) => ResourceOption::Some(Rc::new(res)), + Err((res, errors)) => { + if let Some(reporter) = &shared.error_reporter { + reporter.borrow().report_errors( + errors + .into_iter() + .map(|e| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), + error: e.into(), + }) + .collect(), + ); + } + ResourceOption::Some(Rc::new(res)) + } + }) + .unwrap_or_else(|| ResourceOption::missing_resource(&resource_id)); + // insert the resource into the cache + shared.update_resource(resource_id, resource) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calculate_source_pos() { + let source = r#" +key = Value + +key2 = Value 2 +"# + .trim(); + let result = calculate_pos_in_source(source, 0); + assert_eq!(result, (1, 1)); + + let result = calculate_pos_in_source(source, 1); + assert_eq!(result, (1, 2)); + + let result = calculate_pos_in_source(source, 12); + assert_eq!(result, (2, 1)); + + let result = calculate_pos_in_source(source, 13); + assert_eq!(result, (3, 1)); + } +} diff --git a/intl/l10n/rust/l10nregistry-tests/Cargo.toml b/intl/l10n/rust/l10nregistry-tests/Cargo.toml new file mode 100644 index 0000000000..12ffbfdc66 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "l10nregistry-tests" +version = "0.3.0" +authors = ["Zibi Braniecki <gandalf@mozilla.com>"] +license = "Apache-2.0/MIT" +edition = "2018" + +[dependencies] +l10nregistry = { path = "../l10nregistry-rs", features = ["test-fluent"] } +async-trait = "0.1" +fluent-fallback = "0.7.0" +fluent-testing = { version = "0.0.3", features = ["sync", "async"] } +tokio = { version = "1.0", features = ["rt-multi-thread", "macros"] } +unic-langid = { version = "0.9", features = ["macros"] } + +[dev-dependencies] +criterion = "0.3" +fluent-bundle = "0.15.2" +futures = "0.3" +serial_test = "0.6" + +[features] +default = [] + +[[bench]] +name = "preferences" +harness = false + +[[bench]] +name = "localization" +harness = false + +[[bench]] +name = "source" +harness = false + +[[bench]] +name = "solver" +harness = false + +[[bench]] +name = "registry" +harness = false diff --git a/intl/l10n/rust/l10nregistry-tests/benches/localization.rs b/intl/l10n/rust/l10nregistry-tests/benches/localization.rs new file mode 100644 index 0000000000..b946d2cf6c --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/benches/localization.rs @@ -0,0 +1,70 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use fluent_bundle::FluentArgs; +use fluent_fallback::{types::L10nKey, Localization}; +use fluent_testing::get_scenarios; +use l10nregistry_tests::TestFileFetcher; + +fn preferences_bench(c: &mut Criterion) { + let fetcher = TestFileFetcher::new(); + + let mut group = c.benchmark_group("localization/scenarios"); + + for scenario in get_scenarios() { + let res_ids = scenario.res_ids.clone(); + let l10n_keys: Vec<(String, Option<FluentArgs>)> = scenario + .queries + .iter() + .map(|q| { + ( + q.input.id.clone(), + q.input.args.as_ref().map(|args| { + let mut result = FluentArgs::new(); + for arg in args.as_slice() { + result.set(arg.id.clone(), arg.value.clone()); + } + result + }), + ) + }) + .collect(); + + group.bench_function(format!("{}/format_value_sync", scenario.name), |b| { + b.iter(|| { + let (env, reg) = fetcher.get_registry_and_environment(&scenario); + let mut errors = vec![]; + + let loc = Localization::with_env(res_ids.clone(), true, env.clone(), reg.clone()); + let bundles = loc.bundles(); + + for key in l10n_keys.iter() { + bundles.format_value_sync(&key.0, key.1.as_ref(), &mut errors); + } + }) + }); + + let keys: Vec<L10nKey> = l10n_keys + .into_iter() + .map(|key| L10nKey { + id: key.0.into(), + args: key.1, + }) + .collect(); + group.bench_function(format!("{}/format_messages_sync", scenario.name), |b| { + b.iter(|| { + let (env, reg) = fetcher.get_registry_and_environment(&scenario); + let mut errors = vec![]; + let loc = Localization::with_env(res_ids.clone(), true, env.clone(), reg.clone()); + let bundles = loc.bundles(); + bundles.format_messages_sync(&keys, &mut errors); + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, preferences_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-tests/benches/preferences.rs b/intl/l10n/rust/l10nregistry-tests/benches/preferences.rs new file mode 100644 index 0000000000..ea657ad474 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/benches/preferences.rs @@ -0,0 +1,65 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use fluent_testing::get_scenarios; +use l10nregistry_tests::TestFileFetcher; + +use unic_langid::LanguageIdentifier; + +fn preferences_bench(c: &mut Criterion) { + let fetcher = TestFileFetcher::new(); + + let mut group = c.benchmark_group("registry/scenarios"); + + for scenario in get_scenarios() { + let res_ids = scenario.res_ids.clone(); + + let locales: Vec<LanguageIdentifier> = scenario + .locales + .iter() + .map(|l| l.parse().unwrap()) + .collect(); + + group.bench_function(format!("{}/sync/first_bundle", scenario.name), |b| { + b.iter(|| { + let reg = fetcher.get_registry(&scenario); + let mut bundles = + reg.generate_bundles_sync(locales.clone().into_iter(), res_ids.clone()); + for _ in 0..locales.len() { + if bundles.next().is_some() { + break; + } + } + }) + }); + + #[cfg(feature = "tokio")] + { + use futures::stream::StreamExt; + + let rt = tokio::runtime::Runtime::new().unwrap(); + + group.bench_function(&format!("{}/async/first_bundle", scenario.name), |b| { + b.iter(|| { + rt.block_on(async { + let reg = fetcher.get_registry(&scenario); + + let mut bundles = + reg.generate_bundles(locales.clone().into_iter(), res_ids.clone()); + for _ in 0..locales.len() { + if bundles.next().await.is_some() { + break; + } + } + }); + }) + }); + } + } + + group.finish(); +} + +criterion_group!(benches, preferences_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-tests/benches/registry.rs b/intl/l10n/rust/l10nregistry-tests/benches/registry.rs new file mode 100644 index 0000000000..3d456092f1 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/benches/registry.rs @@ -0,0 +1,133 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use futures::stream::StreamExt; +use l10nregistry::source::ResourceId; +use l10nregistry_tests::{FileSource, RegistrySetup, TestFileFetcher}; +use unic_langid::LanguageIdentifier; + +fn get_paths() -> Vec<ResourceId> { + let paths: Vec<&'static str> = vec![ + "branding/brand.ftl", + "browser/sanitize.ftl", + "browser/preferences/blocklists.ftl", + "browser/preferences/colors.ftl", + "browser/preferences/selectBookmark.ftl", + "browser/preferences/connection.ftl", + "browser/preferences/addEngine.ftl", + "browser/preferences/siteDataSettings.ftl", + "browser/preferences/fonts.ftl", + "browser/preferences/languages.ftl", + "browser/preferences/preferences.ftl", + "security/certificates/certManager.ftl", + "security/certificates/deviceManager.ftl", + "toolkit/global/textActions.ftl", + "toolkit/printing/printUI.ftl", + "toolkit/updates/history.ftl", + "toolkit/featuregates/features.ftl", + ]; + + paths.into_iter().map(ResourceId::from).collect() +} + +fn registry_bench(c: &mut Criterion) { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let mut group = c.benchmark_group("non-metasource"); + + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + group.bench_function(&format!("serial",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), get_paths()); + while let Some(_) = i.next() {} + }) + }); + + let rt = tokio::runtime::Runtime::new().unwrap(); + group.bench_function(&format!("parallel",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles(lang_ids.into_iter(), get_paths()); + rt.block_on(async { while let Some(_) = i.as_mut().unwrap().next().await {} }); + }) + }); + + group.finish(); +} + +fn registry_metasource_bench(c: &mut Criterion) { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let mut group = c.benchmark_group("metasource"); + + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + Some("app"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("app"), + vec![en_us.clone()], + "browser/{locale}/", + ), + FileSource::new( + "toolkit", + Some("langpack"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("langpack"), + vec![en_us.clone()], + "browser/{locale}/", + ), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + group.bench_function(&format!("serial",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), get_paths()); + while let Some(_) = i.next() {} + }) + }); + + let rt = tokio::runtime::Runtime::new().unwrap(); + group.bench_function(&format!("parallel",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles(lang_ids.into_iter(), get_paths()); + rt.block_on(async { while let Some(_) = i.as_mut().unwrap().next().await {} }); + }) + }); + + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = registry_bench, registry_metasource_bench +); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-tests/benches/solver.rs b/intl/l10n/rust/l10nregistry-tests/benches/solver.rs new file mode 100644 index 0000000000..515b5ef1f9 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/benches/solver.rs @@ -0,0 +1,120 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use futures::stream::Collect; +use futures::stream::FuturesOrdered; +use futures::StreamExt; +use l10nregistry_tests::solver::get_scenarios; +use l10nregistry::solver::{AsyncTester, ParallelProblemSolver, SerialProblemSolver, SyncTester}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +pub struct MockTester { + values: Vec<Vec<bool>>, +} + +impl SyncTester for MockTester { + fn test_sync(&self, res_idx: usize, source_idx: usize) -> bool { + self.values[res_idx][source_idx] + } +} + +pub struct SingleTestResult(bool); + +impl Future for SingleTestResult { + type Output = bool; + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> { + self.0.into() + } +} + +pub type ResourceSetStream = Collect<FuturesOrdered<SingleTestResult>, Vec<bool>>; +pub struct TestResult(ResourceSetStream); + +impl std::marker::Unpin for TestResult {} + +impl Future for TestResult { + type Output = Vec<bool>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + let pinned = Pin::new(&mut self.0); + pinned.poll(cx) + } +} + +impl AsyncTester for MockTester { + type Result = TestResult; + + fn test_async(&self, query: Vec<(usize, usize)>) -> Self::Result { + let futures = query + .into_iter() + .map(|(res_idx, source_idx)| SingleTestResult(self.test_sync(res_idx, source_idx))) + .collect::<Vec<_>>(); + TestResult(futures.into_iter().collect::<FuturesOrdered<_>>().collect()) + } +} + +struct TestStream<'t> { + solver: ParallelProblemSolver<MockTester>, + tester: &'t MockTester, +} + +impl<'t> TestStream<'t> { + pub fn new(solver: ParallelProblemSolver<MockTester>, tester: &'t MockTester) -> Self { + Self { solver, tester } + } +} + +impl<'t> futures::stream::Stream for TestStream<'t> { + type Item = Vec<usize>; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<Option<Self::Item>> { + let tester = self.tester; + let solver = &mut self.solver; + let pinned = std::pin::Pin::new(solver); + pinned + .try_poll_next(cx, tester, false) + .map(|v| v.ok().flatten()) + } +} + +fn solver_bench(c: &mut Criterion) { + let scenarios = get_scenarios(); + + let mut group = c.benchmark_group("solver"); + + for scenario in scenarios { + let tester = MockTester { + values: scenario.values.clone(), + }; + + group.bench_function(&format!("serial/{}", &scenario.name), |b| { + b.iter(|| { + let mut gen = SerialProblemSolver::new(scenario.width, scenario.depth); + while let Ok(Some(_)) = gen.try_next(&tester, false) {} + }) + }); + + { + let rt = tokio::runtime::Runtime::new().unwrap(); + + group.bench_function(&format!("parallel/{}", &scenario.name), |b| { + b.iter(|| { + let gen = ParallelProblemSolver::new(scenario.width, scenario.depth); + let mut t = TestStream::new(gen, &tester); + rt.block_on(async { while let Some(_) = t.next().await {} }); + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, solver_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-tests/benches/source.rs b/intl/l10n/rust/l10nregistry-tests/benches/source.rs new file mode 100644 index 0000000000..040bcc8d28 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/benches/source.rs @@ -0,0 +1,60 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use fluent_testing::get_scenarios; +use l10nregistry_tests::TestFileFetcher; + +use unic_langid::LanguageIdentifier; + +fn get_locales<S>(input: &[S]) -> Vec<LanguageIdentifier> +where + S: AsRef<str>, +{ + input.iter().map(|s| s.as_ref().parse().unwrap()).collect() +} + +fn source_bench(c: &mut Criterion) { + let fetcher = TestFileFetcher::new(); + + let mut group = c.benchmark_group("source/scenarios"); + + for scenario in get_scenarios() { + let res_ids = scenario.res_ids.clone(); + + let locales: Vec<LanguageIdentifier> = get_locales(&scenario.locales); + + let sources: Vec<_> = scenario + .file_sources + .iter() + .map(|s| { + fetcher.get_test_file_source(&s.name, None, get_locales(&s.locales), &s.path_scheme) + }) + .collect(); + + group.bench_function(format!("{}/has_file", scenario.name), |b| { + b.iter(|| { + for source in &sources { + for res_id in &res_ids { + source.has_file(&locales[0], &res_id); + } + } + }) + }); + + group.bench_function(format!("{}/sync/fetch_file_sync", scenario.name), |b| { + b.iter(|| { + for source in &sources { + for res_id in &res_ids { + source.fetch_file_sync(&locales[0], &res_id, false); + } + } + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, source_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-tests/src/lib.rs b/intl/l10n/rust/l10nregistry-tests/src/lib.rs new file mode 100644 index 0000000000..0194775b22 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/src/lib.rs @@ -0,0 +1,324 @@ +use l10nregistry::env::ErrorReporter; +use l10nregistry::errors::L10nRegistryError; +use l10nregistry::fluent::FluentBundle; +use l10nregistry::registry::BundleAdapter; +use l10nregistry::registry::L10nRegistry; +use l10nregistry::source::FileFetcher; +use async_trait::async_trait; +use fluent_fallback::{env::LocalesProvider, types::ResourceId}; +use fluent_testing::MockFileSystem; +use std::cell::RefCell; +use std::rc::Rc; +use unic_langid::LanguageIdentifier; + +pub mod solver; + +pub struct RegistrySetup { + pub name: String, + pub file_sources: Vec<FileSource>, + pub locales: Vec<LanguageIdentifier>, +} + +pub struct FileSource { + pub name: String, + pub metasource: String, + pub locales: Vec<LanguageIdentifier>, + pub path_scheme: String, +} + +#[derive(Clone)] +pub struct MockBundleAdapter; + +impl BundleAdapter for MockBundleAdapter { + fn adapt_bundle(&self, _bundle: &mut FluentBundle) {} +} + +impl FileSource { + pub fn new<S>( + name: S, + metasource: Option<S>, + locales: Vec<LanguageIdentifier>, + path_scheme: S, + ) -> Self + where + S: ToString, + { + let metasource = match metasource { + Some(s) => s.to_string(), + None => String::default(), + }; + + Self { + name: name.to_string(), + metasource, + locales, + path_scheme: path_scheme.to_string(), + } + } +} + +impl RegistrySetup { + pub fn new( + name: &str, + file_sources: Vec<FileSource>, + locales: Vec<LanguageIdentifier>, + ) -> Self { + Self { + name: name.to_string(), + file_sources, + locales, + } + } +} + +impl From<fluent_testing::scenarios::structs::Scenario> for RegistrySetup { + fn from(s: fluent_testing::scenarios::structs::Scenario) -> Self { + Self { + name: s.name, + file_sources: s + .file_sources + .into_iter() + .map(|source| { + FileSource::new( + source.name, + None, + source + .locales + .into_iter() + .map(|l| l.parse().unwrap()) + .collect(), + source.path_scheme, + ) + }) + .collect(), + locales: s + .locales + .into_iter() + .map(|loc| loc.parse().unwrap()) + .collect(), + } + } +} + +impl From<&fluent_testing::scenarios::structs::Scenario> for RegistrySetup { + fn from(s: &fluent_testing::scenarios::structs::Scenario) -> Self { + Self { + name: s.name.clone(), + file_sources: s + .file_sources + .iter() + .map(|source| { + FileSource::new( + source.name.clone(), + None, + source.locales.iter().map(|l| l.parse().unwrap()).collect(), + source.path_scheme.clone(), + ) + }) + .collect(), + locales: s.locales.iter().map(|loc| loc.parse().unwrap()).collect(), + } + } +} + +#[derive(Default)] +struct InnerFileFetcher { + fs: MockFileSystem, +} + +#[derive(Clone)] +pub struct TestFileFetcher { + inner: Rc<InnerFileFetcher>, +} + +impl TestFileFetcher { + pub fn new() -> Self { + Self { + inner: Rc::new(InnerFileFetcher::default()), + } + } + + pub fn get_test_file_source( + &self, + name: &str, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + path: &str, + ) -> l10nregistry::source::FileSource { + l10nregistry::source::FileSource::new( + name.to_string(), + metasource, + locales, + path.to_string(), + Default::default(), + self.clone(), + ) + } + + pub fn get_test_file_source_with_index( + &self, + name: &str, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + path: &str, + index: Vec<&str>, + ) -> l10nregistry::source::FileSource { + l10nregistry::source::FileSource::new_with_index( + name.to_string(), + metasource, + locales, + path.to_string(), + Default::default(), + self.clone(), + index.into_iter().map(|s| s.to_string()).collect(), + ) + } + + pub fn get_registry<S>(&self, setup: S) -> L10nRegistry<TestEnvironment, MockBundleAdapter> + where + S: Into<RegistrySetup>, + { + self.get_registry_and_environment(setup).1 + } + + pub fn get_registry_and_environment<S>( + &self, + setup: S, + ) -> ( + TestEnvironment, + L10nRegistry<TestEnvironment, MockBundleAdapter>, + ) + where + S: Into<RegistrySetup>, + { + let setup: RegistrySetup = setup.into(); + let provider = TestEnvironment::new(setup.locales); + + let reg = L10nRegistry::with_provider(provider.clone()); + let sources = setup + .file_sources + .into_iter() + .map(|source| { + let mut s = self.get_test_file_source( + &source.name, + Some(source.metasource), + source.locales, + &source.path_scheme, + ); + s.set_reporter(provider.clone()); + s + }) + .collect(); + reg.register_sources(sources).unwrap(); + (provider, reg) + } + + pub fn get_registry_and_environment_with_adapter<S, B>( + &self, + setup: S, + bundle_adapter: B, + ) -> (TestEnvironment, L10nRegistry<TestEnvironment, B>) + where + S: Into<RegistrySetup>, + B: BundleAdapter, + { + let setup: RegistrySetup = setup.into(); + let provider = TestEnvironment::new(setup.locales); + + let mut reg = L10nRegistry::with_provider(provider.clone()); + let sources = setup + .file_sources + .into_iter() + .map(|source| { + let mut s = self.get_test_file_source( + &source.name, + None, + source.locales, + &source.path_scheme, + ); + s.set_reporter(provider.clone()); + s + }) + .collect(); + reg.register_sources(sources).unwrap(); + reg.set_bundle_adapter(bundle_adapter) + .expect("Failed to set bundle adapter."); + (provider, reg) + } +} + +#[async_trait(?Send)] +impl FileFetcher for TestFileFetcher { + fn fetch_sync(&self, resource_id: &ResourceId) -> std::io::Result<String> { + self.inner.fs.get_test_file_sync(&resource_id.value) + } + + async fn fetch(&self, resource_id: &ResourceId) -> std::io::Result<String> { + self.inner.fs.get_test_file_async(&resource_id.value).await + } +} + +pub enum ErrorStrategy { + Panic, + Report, + Nothing, +} + +pub struct InnerTestEnvironment { + locales: Vec<LanguageIdentifier>, + errors: Vec<L10nRegistryError>, + error_strategy: ErrorStrategy, +} + +#[derive(Clone)] +pub struct TestEnvironment { + inner: Rc<RefCell<InnerTestEnvironment>>, +} + +impl TestEnvironment { + pub fn new(locales: Vec<LanguageIdentifier>) -> Self { + Self { + inner: Rc::new(RefCell::new(InnerTestEnvironment { + locales, + errors: vec![], + error_strategy: ErrorStrategy::Report, + })), + } + } + + pub fn set_locales(&self, locales: Vec<LanguageIdentifier>) { + self.inner.borrow_mut().locales = locales; + } + + pub fn errors(&self) -> Vec<L10nRegistryError> { + self.inner.borrow().errors.clone() + } + + pub fn clear_errors(&self) { + self.inner.borrow_mut().errors.clear() + } +} + +impl LocalesProvider for TestEnvironment { + type Iter = std::vec::IntoIter<LanguageIdentifier>; + + fn locales(&self) -> Self::Iter { + self.inner.borrow().locales.clone().into_iter() + } +} + +impl ErrorReporter for TestEnvironment { + fn report_errors(&self, errors: Vec<L10nRegistryError>) { + match self.inner.borrow().error_strategy { + ErrorStrategy::Panic => { + panic!("Errors: {:#?}", errors); + } + ErrorStrategy::Report => { + #[cfg(test)] // Don't let printing affect benchmarks + eprintln!("Errors: {:#?}", errors); + } + ErrorStrategy::Nothing => {} + } + self.inner.borrow_mut().errors.extend(errors); + } +} diff --git a/intl/l10n/rust/l10nregistry-tests/src/solver/mod.rs b/intl/l10n/rust/l10nregistry-tests/src/solver/mod.rs new file mode 100644 index 0000000000..68f566250e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/src/solver/mod.rs @@ -0,0 +1,38 @@ +mod scenarios; + +pub use scenarios::get_scenarios; + +/// Define a testing scenario. +pub struct Scenario { + /// Name of the scenario. + pub name: String, + /// Number of resources. + pub width: usize, + /// Number of sources. + pub depth: usize, + /// Vector of resources, containing a vector of sources, with true indicating + /// whether the resource is present in that source. + pub values: Vec<Vec<bool>>, + /// Vector of solutions, each containing a vector of resources, with the index + /// indicating from which source the resource is chosen. + /// TODO(issue#17): This field is currently unused! + pub solutions: Vec<Vec<usize>>, +} + +impl Scenario { + pub fn new<S: ToString>( + name: S, + width: usize, + depth: usize, + values: Vec<Vec<bool>>, + solutions: Vec<Vec<usize>>, + ) -> Self { + Self { + name: name.to_string(), + width, + depth, + values, + solutions, + } + } +} diff --git a/intl/l10n/rust/l10nregistry-tests/src/solver/scenarios.rs b/intl/l10n/rust/l10nregistry-tests/src/solver/scenarios.rs new file mode 100644 index 0000000000..8addec979b --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/src/solver/scenarios.rs @@ -0,0 +1,151 @@ +use super::*; + +pub fn get_scenarios() -> Vec<Scenario> { + vec![ + Scenario::new("no-sources", 1, 0, vec![], vec![]), + Scenario::new("no-resources", 1, 0, vec![vec![true]], vec![]), + Scenario::new("no-keys", 0, 1, vec![], vec![]), + Scenario::new( + "one-res-two-sources", + 1, + 2, + vec![vec![true, true]], + vec![vec![0], vec![1]], + ), + Scenario::new( + "two-res-two-sources", + 2, + 2, + vec![vec![false, true], vec![true, false]], + vec![vec![1, 0]], + ), + Scenario::new( + "small", + 3, + 2, + vec![vec![true, true], vec![true, true], vec![true, true]], + vec![ + vec![0, 0, 0], + vec![0, 0, 1], + vec![0, 1, 0], + vec![0, 1, 1], + vec![1, 0, 0], + vec![1, 0, 1], + vec![1, 1, 0], + vec![1, 1, 1], + ], + ), + Scenario::new( + "incomplete", + 3, + 2, + vec![vec![true, false], vec![false, true], vec![true, true]], + vec![vec![0, 1, 0], vec![0, 1, 1]], + ), + Scenario::new( + "preferences", + 19, + 2, + vec![ + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![false, true], + vec![false, true], + vec![false, true], + ], + vec![vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, + ]], + ), + Scenario::new( + "langpack", + 3, + 4, + vec![ + vec![true, true, true, true], + vec![true, true, true, true], + vec![true, true, true, true], + ], + vec![ + vec![0, 0, 0], + vec![0, 0, 1], + vec![0, 0, 2], + vec![0, 0, 3], + vec![0, 1, 0], + vec![0, 1, 1], + vec![0, 1, 2], + vec![0, 1, 3], + vec![0, 2, 0], + vec![0, 2, 1], + vec![0, 2, 2], + vec![0, 2, 3], + vec![0, 3, 0], + vec![0, 3, 1], + vec![0, 3, 2], + vec![0, 3, 3], + vec![1, 0, 0], + vec![1, 0, 1], + vec![1, 0, 2], + vec![1, 0, 3], + vec![1, 1, 0], + vec![1, 1, 1], + vec![1, 1, 2], + vec![1, 1, 3], + vec![1, 2, 0], + vec![1, 2, 1], + vec![1, 2, 2], + vec![1, 2, 3], + vec![1, 3, 0], + vec![1, 3, 1], + vec![1, 3, 2], + vec![1, 3, 3], + vec![2, 0, 0], + vec![2, 0, 1], + vec![2, 0, 2], + vec![2, 0, 3], + vec![2, 1, 0], + vec![2, 1, 1], + vec![2, 1, 2], + vec![2, 1, 3], + vec![2, 2, 0], + vec![2, 2, 1], + vec![2, 2, 2], + vec![2, 2, 3], + vec![2, 3, 0], + vec![2, 3, 1], + vec![2, 3, 2], + vec![2, 3, 3], + vec![3, 0, 0], + vec![3, 0, 1], + vec![3, 0, 2], + vec![3, 0, 3], + vec![3, 1, 0], + vec![3, 1, 1], + vec![3, 1, 2], + vec![3, 1, 3], + vec![3, 2, 0], + vec![3, 2, 1], + vec![3, 2, 2], + vec![3, 2, 3], + vec![3, 3, 0], + vec![3, 3, 1], + vec![3, 3, 2], + vec![3, 3, 3], + ], + ), + ] +} diff --git a/intl/l10n/rust/l10nregistry-tests/tests/localization.rs b/intl/l10n/rust/l10nregistry-tests/tests/localization.rs new file mode 100644 index 0000000000..1362fc030e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/tests/localization.rs @@ -0,0 +1,201 @@ +use std::borrow::Cow; + +use fluent_fallback::{ + env::LocalesProvider, + types::{L10nKey, ResourceId}, + Localization, +}; +use l10nregistry_tests::{ + FileSource, MockBundleAdapter, RegistrySetup, TestEnvironment, TestFileFetcher, +}; +use serial_test::serial; +use unic_langid::{langid, LanguageIdentifier}; + +type L10nRegistry = l10nregistry::registry::L10nRegistry<TestEnvironment, MockBundleAdapter>; + +static LOCALES: &[LanguageIdentifier] = &[langid!("pl"), langid!("en-US")]; +static mut FILE_FETCHER: Option<TestFileFetcher> = None; +static mut L10N_REGISTRY: Option<L10nRegistry> = None; + +const FTL_RESOURCE: &str = "toolkit/updates/history.ftl"; +const L10N_ID_PL_EN: (&str, Option<&str>) = ("history-title", Some("Historia aktualizacji")); +const L10N_ID_MISSING: (&str, Option<&str>) = ("missing-id", None); +const L10N_ID_ONLY_EN: (&str, Option<&str>) = ( + "history-intro", + Some("The following updates have been installed"), +); + +fn get_file_fetcher() -> &'static TestFileFetcher { + let fetcher: &mut Option<TestFileFetcher> = unsafe { &mut FILE_FETCHER }; + + fetcher.get_or_insert_with(|| TestFileFetcher::new()) +} + +fn get_l10n_registry() -> &'static L10nRegistry { + let reg: &mut Option<L10nRegistry> = unsafe { &mut L10N_REGISTRY }; + + reg.get_or_insert_with(|| { + let fetcher = get_file_fetcher(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + None, + get_app_locales().to_vec(), + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + None, + get_app_locales().to_vec(), + "browser/{locale}/", + ), + ], + get_app_locales().to_vec(), + ); + fetcher.get_registry_and_environment(setup).1 + }) +} + +fn get_app_locales() -> &'static [LanguageIdentifier] { + LOCALES +} + +struct LocalesService; + +impl LocalesProvider for LocalesService { + type Iter = std::vec::IntoIter<LanguageIdentifier>; + + fn locales(&self) -> Self::Iter { + get_app_locales().to_vec().into_iter() + } +} + +fn sync_localization( + reg: &'static L10nRegistry, + res_ids: Vec<ResourceId>, +) -> Localization<L10nRegistry, LocalesService> { + Localization::with_env(res_ids, true, LocalesService, reg.clone()) +} + +fn async_localization( + reg: &'static L10nRegistry, + res_ids: Vec<ResourceId>, +) -> Localization<L10nRegistry, LocalesService> { + Localization::with_env(res_ids, false, LocalesService, reg.clone()) +} + +fn setup_sync_test() -> Localization<L10nRegistry, LocalesService> { + sync_localization(get_l10n_registry(), vec![FTL_RESOURCE.into()]) +} + +fn setup_async_test() -> Localization<L10nRegistry, LocalesService> { + async_localization(get_l10n_registry(), vec![FTL_RESOURCE.into()]) +} + +#[test] +#[serial] +fn localization_format_value_sync() { + let loc = setup_sync_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + for query in &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN] { + let value = bundles + .format_value_sync(query.0, None, &mut errors) + .unwrap(); + assert_eq!(value, query.1.map(|s| Cow::Borrowed(s))); + } + + assert_eq!(errors.len(), 4); +} + +#[test] +#[serial] +fn localization_format_values_sync() { + let loc = setup_sync_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + let ids = &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN]; + let keys = ids + .iter() + .map(|query| L10nKey { + id: query.0.into(), + args: None, + }) + .collect::<Vec<_>>(); + + let values = bundles.format_values_sync(&keys, &mut errors).unwrap(); + + assert_eq!(values.len(), ids.len()); + + for (value, query) in values.iter().zip(ids) { + if let Some(expected) = query.1 { + assert_eq!(*value, Some(Cow::Borrowed(expected))); + } + } + assert_eq!(errors.len(), 4); +} + +#[tokio::test] +#[serial] +async fn localization_format_value_async() { + let loc = setup_async_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + for query in &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN] { + let value = bundles.format_value(query.0, None, &mut errors).await; + if let Some(expected) = query.1 { + assert_eq!(value, Some(Cow::Borrowed(expected))); + } + } +} + +#[tokio::test] +#[serial] +async fn localization_format_values_async() { + let loc = setup_async_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + let ids = &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN]; + let keys = ids + .iter() + .map(|query| L10nKey { + id: query.0.into(), + args: None, + }) + .collect::<Vec<_>>(); + + let values = bundles.format_values(&keys, &mut errors).await; + + assert_eq!(values.len(), ids.len()); + + for (value, query) in values.iter().zip(ids) { + if let Some(expected) = query.1 { + assert_eq!(*value, Some(Cow::Borrowed(expected))); + } + } +} + +#[tokio::test] +#[serial] +async fn localization_upgrade() { + let mut loc = setup_sync_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + let value = bundles + .format_value_sync(L10N_ID_PL_EN.0, None, &mut errors) + .unwrap(); + assert_eq!(value, L10N_ID_PL_EN.1.map(|s| Cow::Borrowed(s))); + + loc.set_async(); + let bundles = loc.bundles(); + let value = bundles + .format_value(L10N_ID_PL_EN.0, None, &mut errors) + .await; + assert_eq!(value, L10N_ID_PL_EN.1.map(|s| Cow::Borrowed(s))); +} diff --git a/intl/l10n/rust/l10nregistry-tests/tests/registry.rs b/intl/l10n/rust/l10nregistry-tests/tests/registry.rs new file mode 100644 index 0000000000..72d6e6ed8b --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/tests/registry.rs @@ -0,0 +1,304 @@ +use l10nregistry_tests::{FileSource, RegistrySetup, TestFileFetcher}; +use unic_langid::LanguageIdentifier; + +static FTL_RESOURCE_TOOLKIT: &str = "toolkit/global/textActions.ftl"; +static FTL_RESOURCE_BROWSER: &str = "branding/brand.ftl"; + +#[test] +fn test_get_sources_for_resource() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + { + let metasources = reg + .try_borrow_metasources() + .expect("Unable to borrow metasources."); + + let toolkit = metasources.file_source_by_name(0, "toolkit").unwrap(); + let browser = metasources.file_source_by_name(0, "browser").unwrap(); + let toolkit_resource_id = FTL_RESOURCE_TOOLKIT.into(); + + let mut i = metasources.get_sources_for_resource(0, &en_us, &toolkit_resource_id); + + assert_eq!(i.next(), Some(toolkit)); + assert_eq!(i.next(), Some(browser)); + assert_eq!(i.next(), None); + + assert!(browser + .fetch_file_sync(&en_us, &FTL_RESOURCE_TOOLKIT.into(), false) + .is_none()); + + let mut i = metasources.get_sources_for_resource(0, &en_us, &toolkit_resource_id); + assert_eq!(i.next(), Some(toolkit)); + assert_eq!(i.next(), None); + + assert!(toolkit + .fetch_file_sync(&en_us, &FTL_RESOURCE_TOOLKIT.into(), false) + .is_some()); + + let mut i = metasources.get_sources_for_resource(0, &en_us, &toolkit_resource_id); + assert_eq!(i.next(), Some(toolkit)); + assert_eq!(i.next(), None); + } +} + +#[test] +fn test_generate_bundles_for_lang_sync() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_for_lang_sync(en_us.clone(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[test] +fn test_generate_bundles_sync() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let lang_ids = vec![en_us]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[tokio::test] +async fn test_generate_bundles_for_lang() { + use futures::stream::StreamExt; + + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg + .generate_bundles_for_lang(en_us, paths) + .expect("Failed to get GenerateBundles."); + + assert!(i.next().await.is_some()); + assert!(i.next().await.is_none()); +} + +#[tokio::test] +async fn test_generate_bundles() { + use futures::stream::StreamExt; + + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let langs = vec![en_us]; + let mut i = reg + .generate_bundles(langs.into_iter(), paths) + .expect("Failed to get GenerateBundles."); + + assert!(i.next().await.is_some()); + assert!(i.next().await.is_none()); +} + +#[test] +fn test_manage_sources() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let lang_ids = vec![en_us]; + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_none()); + + reg.clone() + .remove_sources(vec!["toolkit"]) + .expect("Failed to remove a source."); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_none()); + + let paths = vec![FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_some()); + assert!(i.next().is_none()); + + reg.register_sources(vec![fetcher.get_test_file_source( + "toolkit", + None, + lang_ids.clone(), + "browser/{locale}/", + )]) + .expect("Failed to register a source."); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_none()); + + reg.update_sources(vec![fetcher.get_test_file_source( + "toolkit", + None, + lang_ids.clone(), + "toolkit/{locale}/", + )]) + .expect("Failed to update a source."); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[test] +fn test_generate_bundles_with_metasources_sync() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + Some("app"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("app"), + vec![en_us.clone()], + "browser/{locale}/", + ), + FileSource::new( + "toolkit", + Some("langpack"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("langpack"), + vec![en_us.clone()], + "browser/{locale}/", + ), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let lang_ids = vec![en_us]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[tokio::test] +async fn test_generate_bundles_with_metasources() { + use futures::stream::StreamExt; + + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + Some("app"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("app"), + vec![en_us.clone()], + "browser/{locale}/", + ), + FileSource::new( + "toolkit", + Some("langpack"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("langpack"), + vec![en_us.clone()], + "browser/{locale}/", + ), + ], + vec![en_us.clone()], + ); + + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let langs = vec![en_us]; + let mut i = reg + .generate_bundles(langs.into_iter(), paths) + .expect("Failed to get GenerateBundles."); + + assert!(i.next().await.is_some()); + assert!(i.next().await.is_some()); + assert!(i.next().await.is_none()); +} diff --git a/intl/l10n/rust/l10nregistry-tests/tests/scenarios_async.rs b/intl/l10n/rust/l10nregistry-tests/tests/scenarios_async.rs new file mode 100644 index 0000000000..cdbf373d84 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/tests/scenarios_async.rs @@ -0,0 +1,109 @@ +use fluent_bundle::FluentArgs; +use fluent_fallback::Localization; +use fluent_testing::get_scenarios; +use l10nregistry::fluent::FluentBundle; +use l10nregistry::registry::BundleAdapter; +use l10nregistry_tests::{RegistrySetup, TestFileFetcher}; + +#[derive(Clone)] +struct ScenarioBundleAdapter {} + +impl BundleAdapter for ScenarioBundleAdapter { + fn adapt_bundle(&self, bundle: &mut FluentBundle) { + bundle.set_use_isolating(false); + bundle + .add_function("PLATFORM", |_positional, _named| "linux".into()) + .expect("Failed to add a function to the bundle."); + } +} + +#[tokio::test] +async fn scenarios_async() { + use fluent_testing::scenarios::structs::Scenario; + let fetcher = TestFileFetcher::new(); + + let scenarios = get_scenarios(); + + let adapter = ScenarioBundleAdapter {}; + + let cannot_produce_bundle = |scenario: &Scenario| { + scenario + .queries + .iter() + .any(|query| query.exceptional_context.blocks_bundle_generation()) + }; + + for scenario in scenarios { + println!("scenario: {}", scenario.name); + let setup: RegistrySetup = (&scenario).into(); + let (env, reg) = fetcher.get_registry_and_environment_with_adapter(setup, adapter.clone()); + + let loc = Localization::with_env(scenario.res_ids.clone(), false, env.clone(), reg); + let bundles = loc.bundles(); + let no_bundles = cannot_produce_bundle(&scenario); + + let mut errors = vec![]; + + for query in scenario.queries.iter() { + let errors_start_len = errors.len(); + + let args = query.input.args.as_ref().map(|args| { + let mut result = FluentArgs::new(); + for arg in args.as_slice() { + result.set(arg.id.clone(), arg.value.clone()); + } + result + }); + + if let Some(output) = &query.output { + if let Some(value) = &output.value { + let v = bundles + .format_value(&query.input.id, args.as_ref(), &mut errors) + .await; + if no_bundles || query.exceptional_context.causes_failed_value_lookup() { + assert!(v.is_none()); + if no_bundles { + continue; + } + } else { + assert_eq!(v.unwrap(), value.as_str()) + } + } + } + + if query.exceptional_context.causes_reported_format_error() { + assert!( + errors.len() > errors_start_len, + "expected reported errors for query {:#?}", + query + ); + } else { + assert_eq!( + errors.len(), + errors_start_len, + "expected no reported errors for query {:#?}", + query + ); + } + } + + if scenario + .queries + .iter() + .any(|query| query.exceptional_context.missing_required_resource()) + { + assert!( + !env.errors().is_empty(), + "expected errors for scenario {{ {} }}, but found none", + scenario.name + ); + } else { + assert!( + env.errors().is_empty(), + "expected no errors for scenario {{ {} }}, but found {:#?}", + scenario.name, + env.errors() + ); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-tests/tests/scenarios_sync.rs b/intl/l10n/rust/l10nregistry-tests/tests/scenarios_sync.rs new file mode 100644 index 0000000000..ce13b241fe --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/tests/scenarios_sync.rs @@ -0,0 +1,107 @@ +use fluent_bundle::FluentArgs; +use fluent_fallback::Localization; +use fluent_testing::get_scenarios; +use l10nregistry::fluent::FluentBundle; +use l10nregistry::registry::BundleAdapter; +use l10nregistry_tests::{RegistrySetup, TestFileFetcher}; + +#[derive(Clone)] +struct ScenarioBundleAdapter {} + +impl BundleAdapter for ScenarioBundleAdapter { + fn adapt_bundle(&self, bundle: &mut FluentBundle) { + bundle.set_use_isolating(false); + bundle + .add_function("PLATFORM", |_positional, _named| "linux".into()) + .expect("Failed to add a function to the bundle."); + } +} + +#[test] +fn scenarios_sync() { + use fluent_testing::scenarios::structs::Scenario; + let fetcher = TestFileFetcher::new(); + + let scenarios = get_scenarios(); + + let adapter = ScenarioBundleAdapter {}; + + let cannot_produce_bundle = |scenario: &Scenario| { + scenario + .queries + .iter() + .any(|query| query.exceptional_context.blocks_bundle_generation()) + }; + + for scenario in scenarios { + println!("scenario: {}", scenario.name); + let setup: RegistrySetup = (&scenario).into(); + let (env, reg) = fetcher.get_registry_and_environment_with_adapter(setup, adapter.clone()); + + let loc = Localization::with_env(scenario.res_ids.clone(), true, env.clone(), reg); + let bundles = loc.bundles(); + let no_bundles = cannot_produce_bundle(&scenario); + + let mut errors = vec![]; + + for query in scenario.queries.iter() { + let errors_start_len = errors.len(); + + let args = query.input.args.as_ref().map(|args| { + let mut result = FluentArgs::new(); + for arg in args.as_slice() { + result.set(arg.id.clone(), arg.value.clone()); + } + result + }); + + if let Some(output) = &query.output { + if let Some(value) = &output.value { + let v = bundles.format_value_sync(&query.input.id, args.as_ref(), &mut errors); + if no_bundles || query.exceptional_context.causes_failed_value_lookup() { + assert!(v.is_err() || v.unwrap().is_none()); + if no_bundles { + continue; + } + } else { + assert_eq!(v.unwrap().unwrap(), value.as_str()) + } + } + } + + if query.exceptional_context.causes_reported_format_error() { + assert!( + errors.len() > errors_start_len, + "expected reported errors for query {:#?}", + query + ); + } else { + assert_eq!( + errors.len(), + errors_start_len, + "expected no reported errors for query {:#?}", + query + ); + } + } + + if scenario + .queries + .iter() + .any(|query| query.exceptional_context.missing_required_resource()) + { + assert!( + !env.errors().is_empty(), + "expected errors for scenario {{ {} }}, but found none", + scenario.name + ); + } else { + assert!( + env.errors().is_empty(), + "expected no errors for scenario {{ {} }}, but found {:#?}", + scenario.name, + env.errors() + ); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-tests/tests/source.rs b/intl/l10n/rust/l10nregistry-tests/tests/source.rs new file mode 100644 index 0000000000..63104db8eb --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/tests/source.rs @@ -0,0 +1,305 @@ +use fluent_fallback::types::{ResourceType, ToResourceId}; +use futures::future::join_all; +use l10nregistry_tests::TestFileFetcher; +use unic_langid::LanguageIdentifier; + +static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl"; +static FTL_RESOURCE_MISSING: &str = "missing.ftl"; + +#[test] +fn test_fetch_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false); + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Required), + false, + ); + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional), + false, + ); + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_MISSING.into(), false); + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Required), + false, + ); + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional), + false, + ); + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(!file.is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Required), + ) + .await; + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional), + ) + .await; + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await; + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Required), + ) + .await; + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional), + ) + .await; + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(!file.is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_sync_2_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert!(fs1 + .fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false) + .is_some()); + assert!(fs1 + .fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()) + .await + .is_some()); + assert!(fs1 + .fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false) + .is_some()); +} + +#[tokio::test] +async fn test_fetch_async_2_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert!(fs1 + .fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()) + .await + .is_some()); + assert!(fs1 + .fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false) + .is_some()); +} + +#[test] +fn test_fetch_has_value_required_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.into(); + let path_missing = FTL_RESOURCE_MISSING.into(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file_sync(&en_us, &path, false).is_some()); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); +} + +#[test] +fn test_fetch_has_value_optional_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional); + let path_missing = FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file_sync(&en_us, &path, false).is_some()); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); +} + +#[tokio::test] +async fn test_fetch_has_value_required_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.into(); + let path_missing = FTL_RESOURCE_MISSING.into(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file(&en_us, &path).await.is_some()); + println!("Completed"); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + + assert!(fs1.fetch_file(&en_us, &path_missing).await.is_none()); + assert!(fs1 + .fetch_file(&en_us, &path_missing) + .await + .is_required_and_missing()); + + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); + + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert!(fs1 + .fetch_file_sync(&en_us, &path_missing, false) + .is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_has_value_optional_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional); + let path_missing = FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file(&en_us, &path).await.is_some()); + println!("Completed"); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + assert!(fs1.fetch_file(&en_us, &path_missing).await.is_none()); + assert!(!fs1 + .fetch_file(&en_us, &path_missing) + .await + .is_required_and_missing()); + + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); + + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert!(!fs1 + .fetch_file_sync(&en_us, &path_missing, false) + .is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_async_consecutive() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let results = join_all(vec![ + fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()), + fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()), + ]) + .await; + assert!(results[0].is_some()); + assert!(results[1].is_some()); + + assert!(fs1 + .fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()) + .await + .is_some()); +} + +#[test] +fn test_indexed() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT; + let path_missing = FTL_RESOURCE_MISSING; + + let fs1 = fetcher.get_test_file_source_with_index( + "toolkit", + None, + vec![en_us.clone()], + "toolkit/{locale}/", + vec!["toolkit/en-US/toolkit/global/textActions.ftl"], + ); + + assert_eq!(fs1.has_file(&en_us, &path.into()), Some(true)); + assert!(fs1.fetch_file_sync(&en_us, &path.into(), false).is_some()); + assert_eq!(fs1.has_file(&en_us, &path.into()), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing.into()), Some(false)); + assert!(fs1 + .fetch_file_sync(&en_us, &path_missing.into(), false) + .is_none()); + assert_eq!(fs1.has_file(&en_us, &path_missing.into()), Some(false)); +} diff --git a/intl/l10n/rust/l10nregistry-tests/tests/tokio.rs b/intl/l10n/rust/l10nregistry-tests/tests/tokio.rs new file mode 100644 index 0000000000..0f404c3a08 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-tests/tests/tokio.rs @@ -0,0 +1,65 @@ +use l10nregistry_tests::TestFileFetcher; +use unic_langid::LanguageIdentifier; + +static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl"; +static FTL_RESOURCE_MISSING: &str = "missing.ftl"; + +#[tokio::test] +async fn file_source_fetch() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); +} + +#[tokio::test] +async fn file_source_fetch_missing() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await; + assert!(file.is_none()); +} + +#[tokio::test] +async fn file_source_already_loaded() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); +} + +#[tokio::test] +async fn file_source_concurrent() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file1 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + let file2 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + assert!(file1.await.is_some()); + assert!(file2.await.is_some()); +} + +#[test] +fn file_source_sync_after_async_fail() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let _ = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + let file2 = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), true); + assert!(file2.is_some()); +} diff --git a/intl/l10n/rust/localization-ffi/Cargo.toml b/intl/l10n/rust/localization-ffi/Cargo.toml new file mode 100644 index 0000000000..59445bdc5f --- /dev/null +++ b/intl/l10n/rust/localization-ffi/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "localization-ffi" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +futures-channel = "0.3" +futures = "0.3" +nserror = { path = "../../../../xpcom/rust/nserror" } +nsstring = { path = "../../../../xpcom/rust/nsstring" } +l10nregistry = { path = "../l10nregistry-rs" } +fluent = "0.16.0" +unic-langid = "0.9" +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } +async-trait = "0.1" +moz_task = { path = "../../../../xpcom/rust/moz_task" } +fluent-ffi = { path = "../fluent-ffi" } +fluent-fallback = "0.7.0" +l10nregistry-ffi = { path = "../l10nregistry-ffi" } +xpcom = { path = "../../../../xpcom/rust/xpcom" } +cstr = "0.2" diff --git a/intl/l10n/rust/localization-ffi/cbindgen.toml b/intl/l10n/rust/localization-ffi/cbindgen.toml new file mode 100644 index 0000000000..584ce6e9cc --- /dev/null +++ b/intl/l10n/rust/localization-ffi/cbindgen.toml @@ -0,0 +1,33 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef mozilla_intl_l10n_LocalizationBindings_h +#error "Don't include this file directly, instead include LocalizationBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +namespaces = ["mozilla", "intl", "ffi"] +includes = ["mozilla/intl/RegistryBindings.h"] + +[parse] +parse_deps = true +include = ["fluent-fallback", "l10nregistry-ffi"] + +[enum] +derive_helper_methods = true + +[export.rename] +"ThinVec" = "nsTArray" +"Promise" = "dom::Promise" + +[export] +# These are already exported by l10nregistry-ffi. +exclude = [ + "GeckoResourceId", + "GeckoResourceType", +] diff --git a/intl/l10n/rust/localization-ffi/src/lib.rs b/intl/l10n/rust/localization-ffi/src/lib.rs new file mode 100644 index 0000000000..8896231786 --- /dev/null +++ b/intl/l10n/rust/localization-ffi/src/lib.rs @@ -0,0 +1,625 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use fluent::FluentValue; +use fluent_fallback::{ + types::{ + L10nAttribute as FluentL10nAttribute, L10nKey as FluentL10nKey, + L10nMessage as FluentL10nMessage, ResourceId, + }, + Localization, +}; +use fluent_ffi::{convert_args, FluentArgs, FluentArgument, L10nArg}; +use l10nregistry_ffi::{ + env::GeckoEnvironment, + registry::{get_l10n_registry, GeckoL10nRegistry, GeckoResourceId}, +}; +use nsstring::{nsACString, nsCString}; +use std::os::raw::c_void; +use std::{borrow::Cow, cell::RefCell}; +use thin_vec::ThinVec; +use unic_langid::LanguageIdentifier; +use xpcom::{ + interfaces::{nsIRunnablePriority}, + RefCounted, RefPtr, Refcnt, +}; + +#[derive(Debug)] +#[repr(C)] +pub struct L10nKey<'s> { + id: &'s nsACString, + args: ThinVec<L10nArg<'s>>, +} + +impl<'s> From<&'s L10nKey<'s>> for FluentL10nKey<'static> { + fn from(input: &'s L10nKey<'s>) -> Self { + FluentL10nKey { + id: input.id.to_utf8().to_string().into(), + args: convert_args_to_owned(&input.args), + } + } +} + +// This is a variant of `convert_args` from `fluent-ffi` with a 'static constrain +// put on the resulting `FluentArgs` to make it acceptable into `spqwn_current_thread`. +pub fn convert_args_to_owned(args: &[L10nArg]) -> Option<FluentArgs<'static>> { + if args.is_empty() { + return None; + } + + let mut result = FluentArgs::with_capacity(args.len()); + for arg in args { + let val = match arg.value { + FluentArgument::Double_(d) => FluentValue::from(d), + // We need this to be owned because we pass the result into `spawn_local`. + FluentArgument::String(s) => FluentValue::from(Cow::Owned(s.to_utf8().to_string())), + }; + result.set(arg.id.to_string(), val); + } + Some(result) +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nAttribute { + name: nsCString, + value: nsCString, +} + +impl From<FluentL10nAttribute<'_>> for L10nAttribute { + fn from(attr: FluentL10nAttribute<'_>) -> Self { + Self { + name: nsCString::from(&*attr.name), + value: nsCString::from(&*attr.value), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nMessage { + value: nsCString, + attributes: ThinVec<L10nAttribute>, +} + +impl std::default::Default for L10nMessage { + fn default() -> Self { + Self { + value: nsCString::new(), + attributes: ThinVec::new(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct OptionalL10nMessage { + is_present: bool, + message: L10nMessage, +} + +impl From<FluentL10nMessage<'_>> for L10nMessage { + fn from(input: FluentL10nMessage) -> Self { + let value = if let Some(value) = input.value { + value.to_string().into() + } else { + let mut s = nsCString::new(); + s.set_is_void(true); + s + }; + Self { + value, + attributes: input.attributes.into_iter().map(Into::into).collect(), + } + } +} + +pub struct LocalizationRc { + inner: RefCell<Localization<GeckoL10nRegistry, GeckoEnvironment>>, + refcnt: Refcnt, +} + +// xpcom::RefPtr support +unsafe impl RefCounted for LocalizationRc { + unsafe fn addref(&self) { + localization_addref(self); + } + unsafe fn release(&self) { + localization_release(self); + } +} + +impl LocalizationRc { + pub fn new( + res_ids: Vec<ResourceId>, + is_sync: bool, + registry: Option<&GeckoL10nRegistry>, + locales: Option<Vec<LanguageIdentifier>>, + ) -> RefPtr<Self> { + let env = GeckoEnvironment::new(locales); + let inner = if let Some(reg) = registry { + Localization::with_env(res_ids, is_sync, env, reg.clone()) + } else { + let reg = (*get_l10n_registry()).clone(); + Localization::with_env(res_ids, is_sync, env, reg) + }; + + let loc = Box::new(LocalizationRc { + inner: RefCell::new(inner), + refcnt: unsafe { Refcnt::new() }, + }); + + unsafe { + RefPtr::from_raw(Box::into_raw(loc)) + .expect("Failed to create RefPtr<LocalizationRc> from Box<LocalizationRc>") + } + } + + pub fn add_resource_id(&self, res_id: ResourceId) { + self.inner.borrow_mut().add_resource_id(res_id); + } + + pub fn add_resource_ids(&self, res_ids: Vec<ResourceId>) { + self.inner.borrow_mut().add_resource_ids(res_ids); + } + + pub fn remove_resource_id(&self, res_id: ResourceId) -> usize { + self.inner.borrow_mut().remove_resource_id(res_id) + } + + pub fn remove_resource_ids(&self, res_ids: Vec<ResourceId>) -> usize { + self.inner.borrow_mut().remove_resource_ids(res_ids) + } + + pub fn set_async(&self) { + if self.is_sync() { + self.inner.borrow_mut().set_async(); + } + } + + pub fn is_sync(&self) -> bool { + self.inner.borrow().is_sync() + } + + pub fn on_change(&self) { + self.inner.borrow_mut().on_change(); + } + + pub fn format_value_sync( + &self, + id: &nsACString, + args: &ThinVec<L10nArg>, + ret_val: &mut nsACString, + ret_err: &mut ThinVec<nsCString>, + ) -> bool { + let mut errors = vec![]; + let args = convert_args(&args); + if let Ok(value) = self.inner.borrow().bundles().format_value_sync( + &id.to_utf8(), + args.as_ref(), + &mut errors, + ) { + if let Some(value) = value { + ret_val.assign(&value); + } else { + ret_val.set_is_void(true); + } + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &[id], |id| id.to_string()); + ret_err.extend(errors.into_iter().map(|err| err.to_string().into())); + true + } else { + false + } + } + + pub fn format_values_sync( + &self, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<nsCString>, + ret_err: &mut ThinVec<nsCString>, + ) -> bool { + ret_val.reserve(keys.len()); + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + let mut errors = vec![]; + if let Ok(values) = self + .inner + .borrow() + .bundles() + .format_values_sync(&keys, &mut errors) + { + for value in values.iter() { + if let Some(value) = value { + ret_val.push(value.as_ref().into()); + } else { + let mut void_string = nsCString::new(); + void_string.set_is_void(true); + ret_val.push(void_string); + } + } + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + ret_err.extend(errors.into_iter().map(|err| err.to_string().into())); + true + } else { + false + } + } + + pub fn format_messages_sync( + &self, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<OptionalL10nMessage>, + ret_err: &mut ThinVec<nsCString>, + ) -> bool { + ret_val.reserve(keys.len()); + let mut errors = vec![]; + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + if let Ok(messages) = self + .inner + .borrow() + .bundles() + .format_messages_sync(&keys, &mut errors) + { + for msg in messages { + ret_val.push(if let Some(msg) = msg { + OptionalL10nMessage { + is_present: true, + message: msg.into(), + } + } else { + OptionalL10nMessage { + is_present: false, + message: L10nMessage::default(), + } + }); + } + assert_eq!(keys.len(), ret_val.len()); + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + ret_err.extend(errors.into_iter().map(|err| err.to_string().into())); + true + } else { + false + } + } + + pub fn format_value( + &self, + id: &nsACString, + args: &ThinVec<L10nArg>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &nsACString, &ThinVec<nsCString>), + ) { + let bundles = self.inner.borrow().bundles().clone(); + + let args = convert_args_to_owned(&args); + + let id = nsCString::from(id); + let strong_promise = RefPtr::new(promise); + + moz_task::TaskBuilder::new("LocalizationRc::format_value", async move { + let mut errors = vec![]; + let value = if let Some(value) = bundles + .format_value(&id.to_utf8(), args.as_ref(), &mut errors) + .await + { + let v: nsCString = value.to_string().into(); + v + } else { + let mut v = nsCString::new(); + v.set_is_void(true); + v + }; + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &[id], |id| id.to_string()); + let errors = errors + .into_iter() + .map(|err| err.to_string().into()) + .collect(); + callback(&strong_promise, &value, &errors); + }) + .priority(nsIRunnablePriority::PRIORITY_RENDER_BLOCKING as u32) + .spawn_local() + .detach(); + } + + pub fn format_values( + &self, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &ThinVec<nsCString>, &ThinVec<nsCString>), + ) { + let bundles = self.inner.borrow().bundles().clone(); + + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + + let strong_promise = RefPtr::new(promise); + + moz_task::TaskBuilder::new("LocalizationRc::format_values", async move { + let mut errors = vec![]; + let ret_val = bundles + .format_values(&keys, &mut errors) + .await + .into_iter() + .map(|value| { + if let Some(value) = value { + nsCString::from(value.as_ref()) + } else { + let mut v = nsCString::new(); + v.set_is_void(true); + v + } + }) + .collect::<ThinVec<_>>(); + + assert_eq!(keys.len(), ret_val.len()); + + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + let errors = errors + .into_iter() + .map(|err| err.to_string().into()) + .collect(); + + callback(&strong_promise, &ret_val, &errors); + }) + .priority(nsIRunnablePriority::PRIORITY_RENDER_BLOCKING as u32) + .spawn_local() + .detach(); + } + + pub fn format_messages( + &self, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn( + &xpcom::Promise, + &ThinVec<OptionalL10nMessage>, + &ThinVec<nsCString>, + ), + ) { + let bundles = self.inner.borrow().bundles().clone(); + + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + + let strong_promise = RefPtr::new(promise); + + moz_task::TaskBuilder::new("LocalizationRc::format_messages", async move { + let mut errors = vec![]; + let ret_val = bundles + .format_messages(&keys, &mut errors) + .await + .into_iter() + .map(|msg| { + if let Some(msg) = msg { + OptionalL10nMessage { + is_present: true, + message: msg.into(), + } + } else { + OptionalL10nMessage { + is_present: false, + message: L10nMessage::default(), + } + } + }) + .collect::<ThinVec<_>>(); + + assert_eq!(keys.len(), ret_val.len()); + + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + + let errors = errors + .into_iter() + .map(|err| err.to_string().into()) + .collect(); + + callback(&strong_promise, &ret_val, &errors); + }) + .priority(nsIRunnablePriority::PRIORITY_RENDER_BLOCKING as u32) + .spawn_local() + .detach(); + } +} + +#[no_mangle] +pub extern "C" fn localization_parse_locale(input: &nsCString) -> *const c_void { + let l: LanguageIdentifier = input.to_utf8().parse().unwrap(); + Box::into_raw(Box::new(l)) as *const c_void +} + +#[no_mangle] +pub extern "C" fn localization_new( + res_ids: &ThinVec<GeckoResourceId>, + is_sync: bool, + reg: Option<&GeckoL10nRegistry>, + result: &mut *const LocalizationRc, +) { + *result = std::ptr::null(); + let res_ids: Vec<ResourceId> = res_ids.iter().map(ResourceId::from).collect(); + *result = RefPtr::forget_into_raw(LocalizationRc::new(res_ids, is_sync, reg, None)); +} + +#[no_mangle] +pub extern "C" fn localization_new_with_locales( + res_ids: &ThinVec<GeckoResourceId>, + is_sync: bool, + reg: Option<&GeckoL10nRegistry>, + locales: Option<&ThinVec<nsCString>>, + result: &mut *const LocalizationRc, +) -> bool { + *result = std::ptr::null(); + let res_ids: Vec<ResourceId> = res_ids.iter().map(ResourceId::from).collect(); + let locales: Result<Option<Vec<LanguageIdentifier>>, _> = locales + .map(|locales| { + locales + .iter() + .map(|s| LanguageIdentifier::from_bytes(&s)) + .collect() + }) + .transpose(); + + if let Ok(locales) = locales { + *result = RefPtr::forget_into_raw(LocalizationRc::new(res_ids, is_sync, reg, locales)); + true + } else { + false + } +} + +#[no_mangle] +pub unsafe extern "C" fn localization_addref(loc: &LocalizationRc) { + loc.refcnt.inc(); +} + +#[no_mangle] +pub unsafe extern "C" fn localization_release(loc: *const LocalizationRc) { + let rc = (*loc).refcnt.dec(); + if rc == 0 { + std::mem::drop(Box::from_raw(loc as *const _ as *mut LocalizationRc)); + } +} + +#[no_mangle] +pub extern "C" fn localization_add_res_id(loc: &LocalizationRc, res_id: &GeckoResourceId) { + loc.add_resource_id(res_id.into()); +} + +#[no_mangle] +pub extern "C" fn localization_add_res_ids(loc: &LocalizationRc, res_ids: &ThinVec<GeckoResourceId>) { + let res_ids = res_ids.iter().map(ResourceId::from).collect(); + loc.add_resource_ids(res_ids); +} + +#[no_mangle] +pub extern "C" fn localization_remove_res_id(loc: &LocalizationRc, res_id: &GeckoResourceId) -> usize { + loc.remove_resource_id(res_id.into()) +} + +#[no_mangle] +pub extern "C" fn localization_remove_res_ids( + loc: &LocalizationRc, + res_ids: &ThinVec<GeckoResourceId>, +) -> usize { + let res_ids = res_ids.iter().map(ResourceId::from).collect(); + loc.remove_resource_ids(res_ids) +} + +#[no_mangle] +pub extern "C" fn localization_format_value_sync( + loc: &LocalizationRc, + id: &nsACString, + args: &ThinVec<L10nArg>, + ret_val: &mut nsACString, + ret_err: &mut ThinVec<nsCString>, +) -> bool { + loc.format_value_sync(id, args, ret_val, ret_err) +} + +#[no_mangle] +pub extern "C" fn localization_format_values_sync( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<nsCString>, + ret_err: &mut ThinVec<nsCString>, +) -> bool { + loc.format_values_sync(keys, ret_val, ret_err) +} + +#[no_mangle] +pub extern "C" fn localization_format_messages_sync( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<OptionalL10nMessage>, + ret_err: &mut ThinVec<nsCString>, +) -> bool { + loc.format_messages_sync(keys, ret_val, ret_err) +} + +#[no_mangle] +pub extern "C" fn localization_format_value( + loc: &LocalizationRc, + id: &nsACString, + args: &ThinVec<L10nArg>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &nsACString, &ThinVec<nsCString>), +) { + loc.format_value(id, args, promise, callback); +} + +#[no_mangle] +pub extern "C" fn localization_format_values( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &ThinVec<nsCString>, &ThinVec<nsCString>), +) { + loc.format_values(keys, promise, callback); +} + +#[no_mangle] +pub extern "C" fn localization_format_messages( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &ThinVec<OptionalL10nMessage>, &ThinVec<nsCString>), +) { + loc.format_messages(keys, promise, callback); +} + +#[no_mangle] +pub extern "C" fn localization_set_async(loc: &LocalizationRc) { + loc.set_async(); +} + +#[no_mangle] +pub extern "C" fn localization_is_sync(loc: &LocalizationRc) -> bool { + loc.is_sync() +} + +#[no_mangle] +pub extern "C" fn localization_on_change(loc: &LocalizationRc) { + loc.on_change(); +} + +#[cfg(debug_assertions)] +fn debug_assert_variables_exist<K, F>( + errors: &[fluent_fallback::LocalizationError], + keys: &[K], + to_string: F, +) where + F: Fn(&K) -> String, +{ + for error in errors { + if let fluent_fallback::LocalizationError::Resolver { errors, .. } = error { + use fluent::{ + resolver::{errors::ReferenceKind, ResolverError}, + FluentError, + }; + for error in errors { + if let FluentError::ResolverError(ResolverError::Reference( + ReferenceKind::Variable { id }, + )) = error + { + // This error needs to be actionable for Firefox engineers to fix + // their Fluent issues. It might be nicer to share the specific + // message, but at this point we don't have that information. + eprintln!( + "Fluent error, the argument \"${}\" was not provided a value.", + id + ); + eprintln!("This error happened while formatting the following messages:"); + for key in keys { + eprintln!(" {:?}", to_string(key)) + } + + // Panic with the slightly more cryptic ResolverError. + panic!("{}", error.to_string()); + } + } + } + } +} |