diff options
Diffstat (limited to '')
102 files changed, 17371 insertions, 0 deletions
diff --git a/intl/l10n/FileSource.cpp b/intl/l10n/FileSource.cpp new file mode 100644 index 0000000000..2469f597c6 --- /dev/null +++ b/intl/l10n/FileSource.cpp @@ -0,0 +1,192 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FileSource.h" +#include "mozilla/dom/Promise.h" + +using namespace mozilla::dom; + +namespace mozilla::intl { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(L10nFileSource, mGlobal) + +L10nFileSource::L10nFileSource(RefPtr<const ffi::FileSource> aRaw, + nsIGlobalObject* aGlobal) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +/* static */ +already_AddRefed<L10nFileSource> L10nFileSource::Constructor( + const GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray<nsCString>& aLocales, + const nsACString& aPrePath, const dom::FileSourceOptions& aOptions, + const Optional<Sequence<nsCString>>& aIndex, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + + ffi::L10nFileSourceStatus status; + + bool allowOverrides = aOptions.mAddResourceOptions.mAllowOverrides; + + RefPtr<const ffi::FileSource> raw; + if (aIndex.WasPassed()) { + raw = dont_AddRef(ffi::l10nfilesource_new_with_index( + &aName, &aMetaSource, &aLocales, &aPrePath, aIndex.Value().Elements(), + aIndex.Value().Length(), allowOverrides, &status)); + } else { + raw = dont_AddRef(ffi::l10nfilesource_new( + &aName, &aMetaSource, &aLocales, &aPrePath, allowOverrides, &status)); + } + + if (PopulateError(aRv, status)) { + return nullptr; + } + return MakeAndAddRef<L10nFileSource>(std::move(raw), global); +} + +/* static */ +already_AddRefed<L10nFileSource> L10nFileSource::CreateMock( + const GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray<nsCString>& aLocales, + const nsACString& aPrePath, const nsTArray<L10nFileSourceMockFile>& aFS, + ErrorResult& aRv) { + nsTArray<ffi::L10nFileSourceMockFile> fs(aFS.Length()); + for (const auto& file : aFS) { + auto f = fs.AppendElement(); + f->path = file.mPath; + f->source = file.mSource; + } + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + + ffi::L10nFileSourceStatus status; + + RefPtr<const ffi::FileSource> raw(dont_AddRef(ffi::l10nfilesource_new_mock( + &aName, &aMetaSource, &aLocales, &aPrePath, &fs, &status))); + + if (PopulateError(aRv, status)) { + return nullptr; + } + return MakeAndAddRef<L10nFileSource>(std::move(raw), global); +} + +JSObject* L10nFileSource::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return L10nFileSource_Binding::Wrap(aCx, this, aGivenProto); +} + +void L10nFileSource::GetName(nsCString& aRetVal) { + ffi::l10nfilesource_get_name(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetMetaSource(nsCString& aRetVal) { + ffi::l10nfilesource_get_metasource(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetLocales(nsTArray<nsCString>& aRetVal) { + ffi::l10nfilesource_get_locales(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetPrePath(nsCString& aRetVal) { + ffi::l10nfilesource_get_prepath(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetIndex(Nullable<nsTArray<nsCString>>& aRetVal) { + bool hasIndex = + ffi::l10nfilesource_get_index(mRaw.get(), &aRetVal.SetValue()); + if (!hasIndex) { + aRetVal.SetNull(); + } +} + +L10nFileSourceHasFileStatus L10nFileSource::HasFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv) { + ffi::L10nFileSourceStatus status; + + bool isPresent = false; + bool hasValue = ffi::l10nfilesource_has_file(mRaw.get(), &aLocale, &aPath, + &status, &isPresent); + + if (!PopulateError(aRv, status) && hasValue) { + if (isPresent) { + return L10nFileSourceHasFileStatus::Present; + } + + return L10nFileSourceHasFileStatus::Missing; + } + return L10nFileSourceHasFileStatus::Unknown; +} + +already_AddRefed<FluentResource> L10nFileSource::FetchFileSync( + const nsACString& aLocale, const nsACString& aPath, ErrorResult& aRv) { + ffi::L10nFileSourceStatus status; + + RefPtr<const ffi::FluentResource> raw = + dont_AddRef(ffi::l10nfilesource_fetch_file_sync(mRaw.get(), &aLocale, + &aPath, &status)); + + if (!PopulateError(aRv, status) && raw) { + return MakeAndAddRef<FluentResource>(mGlobal, raw); + } + + return nullptr; +} + +already_AddRefed<Promise> L10nFileSource::FetchFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv) { + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (aRv.Failed()) { + return nullptr; + } + + ffi::L10nFileSourceStatus status; + + ffi::l10nfilesource_fetch_file( + mRaw.get(), &aLocale, &aPath, promise, + [](const Promise* aPromise, const ffi::FluentResource* aRes) { + Promise* promise = const_cast<Promise*>(aPromise); + + if (aRes) { + nsIGlobalObject* global = promise->GetGlobalObject(); + RefPtr<FluentResource> res = new FluentResource(global, aRes); + promise->MaybeResolve(res); + } else { + promise->MaybeResolve(JS::NullHandleValue); + } + }, + &status); + + if (PopulateError(aRv, status)) { + return nullptr; + } + + return promise.forget(); +} + +/* static */ +bool L10nFileSource::PopulateError(ErrorResult& aError, + ffi::L10nFileSourceStatus& aStatus) { + switch (aStatus) { + case ffi::L10nFileSourceStatus::InvalidLocaleCode: + aError.ThrowTypeError("Invalid locale code"); + return true; + case ffi::L10nFileSourceStatus::EmptyName: + aError.ThrowTypeError("Name cannot be empty."); + return true; + case ffi::L10nFileSourceStatus::EmptyPrePath: + aError.ThrowTypeError("prePath cannot be empty."); + return true; + case ffi::L10nFileSourceStatus::EmptyResId: + aError.ThrowTypeError("resId cannot be empty."); + return true; + + case ffi::L10nFileSourceStatus::None: + return false; + } + MOZ_ASSERT_UNREACHABLE("Unknown status"); + return false; +} + +} // namespace mozilla::intl diff --git a/intl/l10n/FileSource.h b/intl/l10n/FileSource.h new file mode 100644 index 0000000000..e719dff74f --- /dev/null +++ b/intl/l10n/FileSource.h @@ -0,0 +1,73 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FileSource_h +#define mozilla_intl_l10n_FileSource_h + +#include "nsWrapperCache.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/L10nRegistryBinding.h" +#include "mozilla/dom/FluentBinding.h" +#include "mozilla/intl/RegistryBindings.h" + +class nsIGlobalObject; + +namespace mozilla::intl { + +class L10nFileSource : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(L10nFileSource) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(L10nFileSource) + + explicit L10nFileSource(RefPtr<const ffi::FileSource> aRaw, + nsIGlobalObject* aGlobal = nullptr); + + static already_AddRefed<L10nFileSource> Constructor( + const dom::GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray<nsCString>& aLocales, + const nsACString& aPrePath, const dom::FileSourceOptions& aOptions, + const dom::Optional<dom::Sequence<nsCString>>& aIndex, ErrorResult& aRv); + + static already_AddRefed<L10nFileSource> CreateMock( + const dom::GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray<nsCString>& aLocales, + const nsACString& aPrePath, + const nsTArray<dom::L10nFileSourceMockFile>& aFS, ErrorResult& aRv); + + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + void GetName(nsCString& aRetVal); + void GetMetaSource(nsCString& aRetVal); + void GetLocales(nsTArray<nsCString>& aRetVal); + void GetPrePath(nsCString& aRetVal); + void GetIndex(dom::Nullable<nsTArray<nsCString>>& aRetVal); + + dom::L10nFileSourceHasFileStatus HasFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv); + already_AddRefed<dom::Promise> FetchFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv); + already_AddRefed<FluentResource> FetchFileSync(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv); + + const ffi::FileSource* Raw() const { return mRaw; } + + protected: + virtual ~L10nFileSource() = default; + nsCOMPtr<nsIGlobalObject> mGlobal; + const RefPtr<const ffi::FileSource> mRaw; + static bool PopulateError(ErrorResult& aError, + ffi::L10nFileSourceStatus& aStatus); +}; + +} // namespace mozilla::intl + +#endif diff --git a/intl/l10n/FluentBindings.h b/intl/l10n/FluentBindings.h new file mode 100644 index 0000000000..836929eef0 --- /dev/null +++ b/intl/l10n/FluentBindings.h @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FluentBindings_h +#define mozilla_intl_l10n_FluentBindings_h + +#include "mozilla/intl/fluent_ffi_generated.h" + +#include "mozilla/RefPtr.h" + +namespace mozilla { + +template <> +struct RefPtrTraits<intl::ffi::FluentResource> { + static void AddRef(const intl::ffi::FluentResource* aPtr) { + intl::ffi::fluent_resource_addref(aPtr); + } + static void Release(const intl::ffi::FluentResource* aPtr) { + intl::ffi::fluent_resource_release(aPtr); + } +}; + +template <> +class DefaultDelete<intl::ffi::FluentBundleRc> { + public: + void operator()(intl::ffi::FluentBundleRc* aPtr) const { + fluent_bundle_destroy(aPtr); + } +}; + +} // namespace mozilla + +#endif diff --git a/intl/l10n/FluentBundle.cpp b/intl/l10n/FluentBundle.cpp new file mode 100644 index 0000000000..a20fca5564 --- /dev/null +++ b/intl/l10n/FluentBundle.cpp @@ -0,0 +1,506 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FluentBundle.h" +#include "nsContentUtils.h" +#include "mozilla/dom/ToJSValue.h" +#include "mozilla/dom/UnionTypes.h" +#include "mozilla/intl/NumberFormat.h" +#include "mozilla/intl/DateTimeFormat.h" +#include "mozilla/intl/DateTimePatternGenerator.h" +#include "nsIInputStream.h" +#include "nsStringFwd.h" +#include "nsTArray.h" +#include "js/PropertyAndElement.h" // JS_DefineElement + +using namespace mozilla::dom; + +namespace mozilla { +namespace intl { + +class SizeableUTF8Buffer { + public: + using CharType = char; + + bool reserve(size_t size) { + mBuffer.reset(reinterpret_cast<CharType*>(malloc(size))); + mCapacity = size; + return true; + } + + CharType* data() { return mBuffer.get(); } + + size_t capacity() const { return mCapacity; } + + void written(size_t amount) { mWritten = amount; } + + size_t mWritten = 0; + size_t mCapacity = 0; + + struct FreePolicy { + void operator()(const void* ptr) { free(const_cast<void*>(ptr)); } + }; + + UniquePtr<CharType[], FreePolicy> mBuffer; +}; + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentPattern, mParent) + +FluentPattern::FluentPattern(nsISupports* aParent, const nsACString& aId) + : mId(aId), mParent(aParent) { + MOZ_COUNT_CTOR(FluentPattern); +} +FluentPattern::FluentPattern(nsISupports* aParent, const nsACString& aId, + const nsACString& aAttrName) + : mId(aId), mAttrName(aAttrName), mParent(aParent) { + MOZ_COUNT_CTOR(FluentPattern); +} + +JSObject* FluentPattern::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return FluentPattern_Binding::Wrap(aCx, this, aGivenProto); +} + +FluentPattern::~FluentPattern() { MOZ_COUNT_DTOR(FluentPattern); }; + +/* FluentBundle */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentBundle, mParent) + +FluentBundle::FluentBundle(nsISupports* aParent, + UniquePtr<ffi::FluentBundleRc> aRaw) + : mParent(aParent), mRaw(std::move(aRaw)) { + MOZ_COUNT_CTOR(FluentBundle); +} + +already_AddRefed<FluentBundle> FluentBundle::Constructor( + const dom::GlobalObject& aGlobal, + const UTF8StringOrUTF8StringSequence& aLocales, + const dom::FluentBundleOptions& aOptions, ErrorResult& aRv) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + bool useIsolating = aOptions.mUseIsolating; + + nsAutoCString pseudoStrategy; + if (aOptions.mPseudoStrategy.WasPassed()) { + pseudoStrategy = aOptions.mPseudoStrategy.Value(); + } + + UniquePtr<ffi::FluentBundleRc> raw; + + if (aLocales.IsUTF8String()) { + const nsACString& locale = aLocales.GetAsUTF8String(); + raw.reset( + ffi::fluent_bundle_new_single(&locale, useIsolating, &pseudoStrategy)); + } else { + const auto& locales = aLocales.GetAsUTF8StringSequence(); + raw.reset(ffi::fluent_bundle_new(locales.Elements(), locales.Length(), + useIsolating, &pseudoStrategy)); + } + + if (!raw) { + aRv.ThrowInvalidStateError( + "Failed to create the FluentBundle. Check the " + "locales and pseudo strategy arguments."); + return nullptr; + } + + return do_AddRef(new FluentBundle(global, std::move(raw))); +} + +JSObject* FluentBundle::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return FluentBundle_Binding::Wrap(aCx, this, aGivenProto); +} + +FluentBundle::~FluentBundle() { MOZ_COUNT_DTOR(FluentBundle); }; + +void FluentBundle::GetLocales(nsTArray<nsCString>& aLocales) { + fluent_bundle_get_locales(mRaw.get(), &aLocales); +} + +void FluentBundle::AddResource( + FluentResource& aResource, + const dom::FluentBundleAddResourceOptions& aOptions) { + bool allowOverrides = aOptions.mAllowOverrides; + nsTArray<nsCString> errors; + + fluent_bundle_add_resource(mRaw.get(), aResource.Raw(), allowOverrides, + &errors); + + for (auto& err : errors) { + nsContentUtils::LogSimpleConsoleError(NS_ConvertUTF8toUTF16(err), "L10n"_ns, + false, true, + nsIScriptError::warningFlag); + } +} + +bool FluentBundle::HasMessage(const nsACString& aId) { + return fluent_bundle_has_message(mRaw.get(), &aId); +} + +void FluentBundle::GetMessage(const nsACString& aId, + Nullable<FluentMessage>& aRetVal) { + bool hasValue = false; + nsTArray<nsCString> attributes; + bool exists = + fluent_bundle_get_message(mRaw.get(), &aId, &hasValue, &attributes); + if (exists) { + FluentMessage& msg = aRetVal.SetValue(); + if (hasValue) { + msg.mValue = new FluentPattern(mParent, aId); + } + for (auto& name : attributes) { + auto newEntry = msg.mAttributes.Entries().AppendElement(fallible); + newEntry->mKey = name; + newEntry->mValue = new FluentPattern(mParent, aId, name); + } + } +} + +bool extendJSArrayWithErrors(JSContext* aCx, JS::Handle<JSObject*> aErrors, + nsTArray<nsCString>& aInput) { + uint32_t length; + if (NS_WARN_IF(!JS::GetArrayLength(aCx, aErrors, &length))) { + return false; + } + + for (auto& err : aInput) { + JS::Rooted<JS::Value> jsval(aCx); + if (!ToJSValue(aCx, NS_ConvertUTF8toUTF16(err), &jsval)) { + return false; + } + if (!JS_DefineElement(aCx, aErrors, length++, jsval, JSPROP_ENUMERATE)) { + return false; + } + } + return true; +} + +/* static */ +void FluentBundle::ConvertArgs(const L10nArgs& aArgs, + nsTArray<ffi::L10nArg>& aRetVal) { + aRetVal.SetCapacity(aArgs.Entries().Length()); + for (const auto& entry : aArgs.Entries()) { + if (!entry.mValue.IsNull()) { + const auto& value = entry.mValue.Value(); + + if (value.IsUTF8String()) { + aRetVal.AppendElement(ffi::L10nArg{ + &entry.mKey, + ffi::FluentArgument::String(&value.GetAsUTF8String())}); + } else { + aRetVal.AppendElement(ffi::L10nArg{ + &entry.mKey, ffi::FluentArgument::Double_(value.GetAsDouble())}); + } + } + } +} + +void FluentBundle::FormatPattern(JSContext* aCx, const FluentPattern& aPattern, + const Nullable<L10nArgs>& aArgs, + const Optional<JS::Handle<JSObject*>>& aErrors, + nsACString& aRetVal, ErrorResult& aRv) { + nsTArray<ffi::L10nArg> l10nArgs; + + if (!aArgs.IsNull()) { + const L10nArgs& args = aArgs.Value(); + ConvertArgs(args, l10nArgs); + } + + nsTArray<nsCString> errors; + bool succeeded = fluent_bundle_format_pattern(mRaw.get(), &aPattern.mId, + &aPattern.mAttrName, &l10nArgs, + &aRetVal, &errors); + + if (!succeeded) { + return aRv.ThrowInvalidStateError( + "Failed to format the FluentPattern. Likely the " + "pattern could not be retrieved from the bundle."); + } + + if (aErrors.WasPassed()) { + if (!extendJSArrayWithErrors(aCx, aErrors.Value(), errors)) { + aRv.ThrowUnknownError("Failed to add errors to an error array."); + } + } +} + +// FFI + +extern "C" { +ffi::RawNumberFormatter* FluentBuiltInNumberFormatterCreate( + const nsCString* aLocale, const ffi::FluentNumberOptionsRaw* aOptions) { + NumberFormatOptions options; + switch (aOptions->style) { + case ffi::FluentNumberStyleRaw::Decimal: + break; + case ffi::FluentNumberStyleRaw::Currency: { + std::string currency = aOptions->currency.get(); + switch (aOptions->currency_display) { + case ffi::FluentNumberCurrencyDisplayStyleRaw::Symbol: + options.mCurrency = Some(std::make_pair( + currency, NumberFormatOptions::CurrencyDisplay::Symbol)); + break; + case ffi::FluentNumberCurrencyDisplayStyleRaw::Code: + options.mCurrency = Some(std::make_pair( + currency, NumberFormatOptions::CurrencyDisplay::Code)); + break; + case ffi::FluentNumberCurrencyDisplayStyleRaw::Name: + options.mCurrency = Some(std::make_pair( + currency, NumberFormatOptions::CurrencyDisplay::Name)); + break; + default: + MOZ_ASSERT_UNREACHABLE(); + break; + } + } break; + case ffi::FluentNumberStyleRaw::Percent: + options.mPercent = true; + break; + default: + MOZ_ASSERT_UNREACHABLE(); + break; + } + + options.mGrouping = aOptions->use_grouping + ? NumberFormatOptions::Grouping::Auto + : NumberFormatOptions::Grouping::Never; + options.mMinIntegerDigits = Some(aOptions->minimum_integer_digits); + + if (aOptions->minimum_significant_digits >= 0 || + aOptions->maximum_significant_digits >= 0) { + options.mSignificantDigits = + Some(std::make_pair(aOptions->minimum_significant_digits, + aOptions->maximum_significant_digits)); + } else { + options.mFractionDigits = Some(std::make_pair( + aOptions->minimum_fraction_digits, aOptions->maximum_fraction_digits)); + } + + Result<UniquePtr<NumberFormat>, ICUError> result = + NumberFormat::TryCreate(aLocale->get(), options); + + MOZ_ASSERT(result.isOk()); + + if (result.isOk()) { + return reinterpret_cast<ffi::RawNumberFormatter*>( + result.unwrap().release()); + } + + return nullptr; +} + +uint8_t* FluentBuiltInNumberFormatterFormat( + const ffi::RawNumberFormatter* aFormatter, double input, size_t* aOutCount, + size_t* aOutCapacity) { + const NumberFormat* nf = reinterpret_cast<const NumberFormat*>(aFormatter); + + SizeableUTF8Buffer buffer; + if (nf->format(input, buffer).isOk()) { + *aOutCount = buffer.mWritten; + *aOutCapacity = buffer.mCapacity; + return reinterpret_cast<uint8_t*>(buffer.mBuffer.release()); + } + + return nullptr; +} + +void FluentBuiltInNumberFormatterDestroy(ffi::RawNumberFormatter* aFormatter) { + delete reinterpret_cast<NumberFormat*>(aFormatter); +} + +/* DateTime */ + +static Maybe<DateTimeFormat::Style> GetStyle(ffi::FluentDateTimeStyle aStyle) { + switch (aStyle) { + case ffi::FluentDateTimeStyle::Full: + return Some(DateTimeFormat::Style::Full); + case ffi::FluentDateTimeStyle::Long: + return Some(DateTimeFormat::Style::Long); + case ffi::FluentDateTimeStyle::Medium: + return Some(DateTimeFormat::Style::Medium); + case ffi::FluentDateTimeStyle::Short: + return Some(DateTimeFormat::Style::Short); + case ffi::FluentDateTimeStyle::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe<DateTimeFormat::Text> GetText( + ffi::FluentDateTimeTextComponent aText) { + switch (aText) { + case ffi::FluentDateTimeTextComponent::Long: + return Some(DateTimeFormat::Text::Long); + case ffi::FluentDateTimeTextComponent::Short: + return Some(DateTimeFormat::Text::Short); + case ffi::FluentDateTimeTextComponent::Narrow: + return Some(DateTimeFormat::Text::Narrow); + case ffi::FluentDateTimeTextComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe<DateTimeFormat::Month> GetMonth( + ffi::FluentDateTimeMonthComponent aMonth) { + switch (aMonth) { + case ffi::FluentDateTimeMonthComponent::Numeric: + return Some(DateTimeFormat::Month::Numeric); + case ffi::FluentDateTimeMonthComponent::TwoDigit: + return Some(DateTimeFormat::Month::TwoDigit); + case ffi::FluentDateTimeMonthComponent::Long: + return Some(DateTimeFormat::Month::Long); + case ffi::FluentDateTimeMonthComponent::Short: + return Some(DateTimeFormat::Month::Short); + case ffi::FluentDateTimeMonthComponent::Narrow: + return Some(DateTimeFormat::Month::Narrow); + case ffi::FluentDateTimeMonthComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe<DateTimeFormat::Numeric> GetNumeric( + ffi::FluentDateTimeNumericComponent aNumeric) { + switch (aNumeric) { + case ffi::FluentDateTimeNumericComponent::Numeric: + return Some(DateTimeFormat::Numeric::Numeric); + case ffi::FluentDateTimeNumericComponent::TwoDigit: + return Some(DateTimeFormat::Numeric::TwoDigit); + case ffi::FluentDateTimeNumericComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe<DateTimeFormat::TimeZoneName> GetTimeZoneName( + ffi::FluentDateTimeTimeZoneNameComponent aTimeZoneName) { + switch (aTimeZoneName) { + case ffi::FluentDateTimeTimeZoneNameComponent::Long: + return Some(DateTimeFormat::TimeZoneName::Long); + case ffi::FluentDateTimeTimeZoneNameComponent::Short: + return Some(DateTimeFormat::TimeZoneName::Short); + case ffi::FluentDateTimeTimeZoneNameComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe<DateTimeFormat::HourCycle> GetHourCycle( + ffi::FluentDateTimeHourCycle aHourCycle) { + switch (aHourCycle) { + case ffi::FluentDateTimeHourCycle::H24: + return Some(DateTimeFormat::HourCycle::H24); + case ffi::FluentDateTimeHourCycle::H23: + return Some(DateTimeFormat::HourCycle::H23); + case ffi::FluentDateTimeHourCycle::H12: + return Some(DateTimeFormat::HourCycle::H12); + case ffi::FluentDateTimeHourCycle::H11: + return Some(DateTimeFormat::HourCycle::H11); + case ffi::FluentDateTimeHourCycle::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe<DateTimeFormat::ComponentsBag> GetComponentsBag( + ffi::FluentDateTimeOptions aOptions) { + if (GetStyle(aOptions.date_style) || GetStyle(aOptions.time_style)) { + return Nothing(); + } + + DateTimeFormat::ComponentsBag components; + components.era = GetText(aOptions.era); + components.year = GetNumeric(aOptions.year); + components.month = GetMonth(aOptions.month); + components.day = GetNumeric(aOptions.day); + components.weekday = GetText(aOptions.weekday); + components.hour = GetNumeric(aOptions.hour); + components.minute = GetNumeric(aOptions.minute); + components.second = GetNumeric(aOptions.second); + components.timeZoneName = GetTimeZoneName(aOptions.time_zone_name); + components.hourCycle = GetHourCycle(aOptions.hour_cycle); + + if (!components.era && !components.year && !components.month && + !components.day && !components.weekday && !components.hour && + !components.minute && !components.second && !components.timeZoneName) { + return Nothing(); + } + + return Some(components); +} + +ffi::RawDateTimeFormatter* FluentBuiltInDateTimeFormatterCreate( + const nsCString* aLocale, ffi::FluentDateTimeOptions aOptions) { + auto genResult = DateTimePatternGenerator::TryCreate(aLocale->get()); + if (genResult.isErr()) { + MOZ_ASSERT_UNREACHABLE("There was an error in DateTimeFormat"); + return nullptr; + } + UniquePtr<DateTimePatternGenerator> dateTimePatternGenerator = + genResult.unwrap(); + + if (auto components = GetComponentsBag(aOptions)) { + auto result = DateTimeFormat::TryCreateFromComponents( + Span(*aLocale), *components, dateTimePatternGenerator.get()); + if (result.isErr()) { + MOZ_ASSERT_UNREACHABLE("There was an error in DateTimeFormat"); + return nullptr; + } + + return reinterpret_cast<ffi::RawDateTimeFormatter*>( + result.unwrap().release()); + } + + DateTimeFormat::StyleBag style; + style.date = GetStyle(aOptions.date_style); + style.time = GetStyle(aOptions.time_style); + + auto result = DateTimeFormat::TryCreateFromStyle( + Span(*aLocale), style, dateTimePatternGenerator.get()); + + if (result.isErr()) { + MOZ_ASSERT_UNREACHABLE("There was an error in DateTimeFormat"); + return nullptr; + } + + return reinterpret_cast<ffi::RawDateTimeFormatter*>( + result.unwrap().release()); +} + +uint8_t* FluentBuiltInDateTimeFormatterFormat( + const ffi::RawDateTimeFormatter* aFormatter, double aUnixEpoch, + uint32_t* aOutCount) { + const auto* dtFormat = reinterpret_cast<const DateTimeFormat*>(aFormatter); + + SizeableUTF8Buffer buffer; + dtFormat->TryFormat(aUnixEpoch, buffer).unwrap(); + + *aOutCount = buffer.mWritten; + + return reinterpret_cast<uint8_t*>(buffer.mBuffer.release()); +} + +void FluentBuiltInDateTimeFormatterDestroy( + ffi::RawDateTimeFormatter* aFormatter) { + delete reinterpret_cast<const DateTimeFormat*>(aFormatter); +} +} + +} // namespace intl +} // namespace mozilla diff --git a/intl/l10n/FluentBundle.h b/intl/l10n/FluentBundle.h new file mode 100644 index 0000000000..5c28d30290 --- /dev/null +++ b/intl/l10n/FluentBundle.h @@ -0,0 +1,99 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FluentBundle_h +#define mozilla_intl_l10n_FluentBundle_h + +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" +#include "mozilla/dom/FluentBinding.h" +#include "mozilla/dom/LocalizationBinding.h" +#include "mozilla/intl/FluentResource.h" +#include "mozilla/intl/FluentBindings.h" + +class nsIGlobalObject; + +namespace mozilla { +class ErrorResult; + +namespace dom { +struct FluentMessage; +struct L10nMessage; +class OwningUTF8StringOrDouble; +class UTF8StringOrUTF8StringSequence; +struct FluentBundleOptions; +struct FluentBundleAddResourceOptions; +} // namespace dom + +namespace intl { + +class FluentResource; + +using L10nArgs = + dom::Record<nsCString, dom::Nullable<dom::OwningUTF8StringOrDouble>>; + +class FluentPattern : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentPattern) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentPattern) + + FluentPattern(nsISupports* aParent, const nsACString& aId); + FluentPattern(nsISupports* aParent, const nsACString& aId, + const nsACString& aAttrName); + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + nsISupports* GetParentObject() const { return mParent; } + + nsCString mId; + nsCString mAttrName; + + protected: + virtual ~FluentPattern(); + + nsCOMPtr<nsISupports> mParent; +}; + +class FluentBundle final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentBundle) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentBundle) + + FluentBundle(nsISupports* aParent, UniquePtr<ffi::FluentBundleRc> aRaw); + + static already_AddRefed<FluentBundle> Constructor( + const dom::GlobalObject& aGlobal, + const dom::UTF8StringOrUTF8StringSequence& aLocales, + const dom::FluentBundleOptions& aOptions, ErrorResult& aRv); + JSObject* WrapObject(JSContext* aCx, JS::Handle<JSObject*> aGivenProto) final; + nsISupports* GetParentObject() const { return mParent; } + + void GetLocales(nsTArray<nsCString>& aLocales); + + void AddResource(FluentResource& aResource, + const dom::FluentBundleAddResourceOptions& aOptions); + bool HasMessage(const nsACString& aId); + void GetMessage(const nsACString& aId, + dom::Nullable<dom::FluentMessage>& aRetVal); + void FormatPattern(JSContext* aCx, const FluentPattern& aPattern, + const dom::Nullable<L10nArgs>& aArgs, + const dom::Optional<JS::Handle<JSObject*>>& aErrors, + nsACString& aRetVal, ErrorResult& aRv); + + static void ConvertArgs(const L10nArgs& aArgs, + nsTArray<ffi::L10nArg>& aRetVal); + + protected: + virtual ~FluentBundle(); + + nsCOMPtr<nsISupports> mParent; + UniquePtr<ffi::FluentBundleRc> mRaw; +}; + +} // namespace intl +} // namespace mozilla + +#endif diff --git a/intl/l10n/FluentResource.cpp b/intl/l10n/FluentResource.cpp new file mode 100644 index 0000000000..bef7d7adf4 --- /dev/null +++ b/intl/l10n/FluentResource.cpp @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsContentUtils.h" +#include "FluentResource.h" + +using namespace mozilla::dom; + +namespace mozilla { +namespace intl { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentResource, mParent) + +FluentResource::FluentResource(nsISupports* aParent, + const ffi::FluentResource* aRaw) + : mParent(aParent), mRaw(std::move(aRaw)), mHasErrors(false) {} + +FluentResource::FluentResource(nsISupports* aParent, const nsACString& aSource) + : mParent(aParent), mHasErrors(false) { + mRaw = dont_AddRef(ffi::fluent_resource_new(&aSource, &mHasErrors)); +} + +already_AddRefed<FluentResource> FluentResource::Constructor( + const GlobalObject& aGlobal, const nsACString& aSource) { + RefPtr<FluentResource> res = + new FluentResource(aGlobal.GetAsSupports(), aSource); + + if (res->mHasErrors) { + nsContentUtils::LogSimpleConsoleError( + u"Errors encountered while parsing Fluent Resource."_ns, "chrome"_ns, + false, true /* from chrome context*/); + } + return res.forget(); +} + +JSObject* FluentResource::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return FluentResource_Binding::Wrap(aCx, this, aGivenProto); +} + +} // namespace intl +} // namespace mozilla diff --git a/intl/l10n/FluentResource.h b/intl/l10n/FluentResource.h new file mode 100644 index 0000000000..56e7f8198b --- /dev/null +++ b/intl/l10n/FluentResource.h @@ -0,0 +1,47 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FluentResource_h +#define mozilla_intl_l10n_FluentResource_h + +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" +#include "mozilla/dom/FluentBinding.h" +#include "mozilla/intl/FluentBindings.h" + +namespace mozilla { +namespace intl { + +class FluentResource : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentResource) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentResource) + + FluentResource(nsISupports* aParent, const ffi::FluentResource* aRaw); + FluentResource(nsISupports* aParent, const nsACString& aSource); + + static already_AddRefed<FluentResource> Constructor( + const dom::GlobalObject& aGlobal, const nsACString& aSource); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + nsISupports* GetParentObject() const { return mParent; } + + const ffi::FluentResource* Raw() const { return mRaw; } + + protected: + virtual ~FluentResource() = default; + + nsCOMPtr<nsISupports> mParent; + RefPtr<const ffi::FluentResource> mRaw; + bool mHasErrors; +}; + +} // namespace intl +} // namespace mozilla + +#endif diff --git a/intl/l10n/FluentSyntax.jsm b/intl/l10n/FluentSyntax.jsm new file mode 100644 index 0000000000..ec60dd5283 --- /dev/null +++ b/intl/l10n/FluentSyntax.jsm @@ -0,0 +1,1957 @@ +/* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */ + +/* Copyright 2019 Mozilla Foundation and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* fluent-syntax@0.12.0 */ + +/* + * Base class for all Fluent AST nodes. + * + * All productions described in the ASDL subclass BaseNode, including Span and + * Annotation. + * + */ +class BaseNode { + constructor() {} + + equals(other, ignoredFields = ["span"]) { + const thisKeys = new Set(Object.keys(this)); + const otherKeys = new Set(Object.keys(other)); + if (ignoredFields) { + for (const fieldName of ignoredFields) { + thisKeys.delete(fieldName); + otherKeys.delete(fieldName); + } + } + if (thisKeys.size !== otherKeys.size) { + return false; + } + for (const fieldName of thisKeys) { + if (!otherKeys.has(fieldName)) { + return false; + } + const thisVal = this[fieldName]; + const otherVal = other[fieldName]; + if (typeof thisVal !== typeof otherVal) { + return false; + } + if (thisVal instanceof Array) { + if (thisVal.length !== otherVal.length) { + return false; + } + for (let i = 0; i < thisVal.length; ++i) { + if (!scalarsEqual(thisVal[i], otherVal[i], ignoredFields)) { + return false; + } + } + } else if (!scalarsEqual(thisVal, otherVal, ignoredFields)) { + return false; + } + } + return true; + } + + clone() { + function visit(value) { + if (value instanceof BaseNode) { + return value.clone(); + } + if (Array.isArray(value)) { + return value.map(visit); + } + return value; + } + const clone = Object.create(this.constructor.prototype); + for (const prop of Object.keys(this)) { + clone[prop] = visit(this[prop]); + } + return clone; + } +} + +function scalarsEqual(thisVal, otherVal, ignoredFields) { + if (thisVal instanceof BaseNode) { + return thisVal.equals(otherVal, ignoredFields); + } + return thisVal === otherVal; +} + +/* + * Base class for AST nodes which can have Spans. + */ +class SyntaxNode extends BaseNode { + addSpan(start, end) { + this.span = new Span(start, end); + } +} + +class Resource extends SyntaxNode { + constructor(body = []) { + super(); + this.type = "Resource"; + this.body = body; + } +} + +/* + * An abstract base class for useful elements of Resource.body. + */ +class Entry extends SyntaxNode {} + +class Message extends Entry { + constructor(id, value = null, attributes = [], comment = null) { + super(); + this.type = "Message"; + this.id = id; + this.value = value; + this.attributes = attributes; + this.comment = comment; + } +} + +class Term extends Entry { + constructor(id, value, attributes = [], comment = null) { + super(); + this.type = "Term"; + this.id = id; + this.value = value; + this.attributes = attributes; + this.comment = comment; + } +} + +class Pattern extends SyntaxNode { + constructor(elements) { + super(); + this.type = "Pattern"; + this.elements = elements; + } +} + +/* + * An abstract base class for elements of Patterns. + */ +class PatternElement extends SyntaxNode {} + +class TextElement extends PatternElement { + constructor(value) { + super(); + this.type = "TextElement"; + this.value = value; + } +} + +class Placeable extends PatternElement { + constructor(expression) { + super(); + this.type = "Placeable"; + this.expression = expression; + } +} + +/* + * An abstract base class for expressions. + */ +class Expression extends SyntaxNode {} + +// An abstract base class for Literals. +class Literal extends Expression { + constructor(value) { + super(); + // The "value" field contains the exact contents of the literal, + // character-for-character. + this.value = value; + } + + parse() { + return {value: this.value}; + } +} + +class StringLiteral extends Literal { + constructor(value) { + super(value); + this.type = "StringLiteral"; + } + + parse() { + // Backslash backslash, backslash double quote, uHHHH, UHHHHHH. + const KNOWN_ESCAPES = + /(?:\\\\|\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g; + + function from_escape_sequence(match, codepoint4, codepoint6) { + switch (match) { + case "\\\\": + return "\\"; + case "\\\"": + return "\""; + default: + let codepoint = parseInt(codepoint4 || codepoint6, 16); + if (codepoint <= 0xD7FF || 0xE000 <= codepoint) { + // It's a Unicode scalar value. + return String.fromCodePoint(codepoint); + } + // Escape sequences reresenting surrogate code points are + // well-formed but invalid in Fluent. Replace them with U+FFFD + // REPLACEMENT CHARACTER. + return "�"; + } + } + + let value = this.value.replace(KNOWN_ESCAPES, from_escape_sequence); + return {value}; + } +} + +class NumberLiteral extends Literal { + constructor(value) { + super(value); + this.type = "NumberLiteral"; + } + + parse() { + let value = parseFloat(this.value); + let decimal_position = this.value.indexOf("."); + let precision = decimal_position > 0 + ? this.value.length - decimal_position - 1 + : 0; + return {value, precision}; + } +} + +class MessageReference extends Expression { + constructor(id, attribute = null) { + super(); + this.type = "MessageReference"; + this.id = id; + this.attribute = attribute; + } +} + +class TermReference extends Expression { + constructor(id, attribute = null, args = null) { + super(); + this.type = "TermReference"; + this.id = id; + this.attribute = attribute; + this.arguments = args; + } +} + +class VariableReference extends Expression { + constructor(id) { + super(); + this.type = "VariableReference"; + this.id = id; + } +} + +class FunctionReference extends Expression { + constructor(id, args) { + super(); + this.type = "FunctionReference"; + this.id = id; + this.arguments = args; + } +} + +class SelectExpression extends Expression { + constructor(selector, variants) { + super(); + this.type = "SelectExpression"; + this.selector = selector; + this.variants = variants; + } +} + +class CallArguments extends SyntaxNode { + constructor(positional = [], named = []) { + super(); + this.type = "CallArguments"; + this.positional = positional; + this.named = named; + } +} + +class Attribute extends SyntaxNode { + constructor(id, value) { + super(); + this.type = "Attribute"; + this.id = id; + this.value = value; + } +} + +class Variant extends SyntaxNode { + constructor(key, value, def = false) { + super(); + this.type = "Variant"; + this.key = key; + this.value = value; + this.default = def; + } +} + +class NamedArgument extends SyntaxNode { + constructor(name, value) { + super(); + this.type = "NamedArgument"; + this.name = name; + this.value = value; + } +} + +class Identifier extends SyntaxNode { + constructor(name) { + super(); + this.type = "Identifier"; + this.name = name; + } +} + +class BaseComment extends Entry { + constructor(content) { + super(); + this.type = "BaseComment"; + this.content = content; + } +} + +class Comment extends BaseComment { + constructor(content) { + super(content); + this.type = "Comment"; + } +} + +class GroupComment extends BaseComment { + constructor(content) { + super(content); + this.type = "GroupComment"; + } +} +class ResourceComment extends BaseComment { + constructor(content) { + super(content); + this.type = "ResourceComment"; + } +} + +class Junk extends SyntaxNode { + constructor(content) { + super(); + this.type = "Junk"; + this.annotations = []; + this.content = content; + } + + addAnnotation(annot) { + this.annotations.push(annot); + } +} + +class Span extends BaseNode { + constructor(start, end) { + super(); + this.type = "Span"; + this.start = start; + this.end = end; + } +} + +class Annotation extends SyntaxNode { + constructor(code, args = [], message) { + super(); + this.type = "Annotation"; + this.code = code; + this.arguments = args; + this.message = message; + } +} + +const ast = ({ + BaseNode: BaseNode, + Resource: Resource, + Entry: Entry, + Message: Message, + Term: Term, + Pattern: Pattern, + PatternElement: PatternElement, + TextElement: TextElement, + Placeable: Placeable, + Expression: Expression, + Literal: Literal, + StringLiteral: StringLiteral, + NumberLiteral: NumberLiteral, + MessageReference: MessageReference, + TermReference: TermReference, + VariableReference: VariableReference, + FunctionReference: FunctionReference, + SelectExpression: SelectExpression, + CallArguments: CallArguments, + Attribute: Attribute, + Variant: Variant, + NamedArgument: NamedArgument, + Identifier: Identifier, + BaseComment: BaseComment, + Comment: Comment, + GroupComment: GroupComment, + ResourceComment: ResourceComment, + Junk: Junk, + Span: Span, + Annotation: Annotation +}); + +class ParseError extends Error { + constructor(code, ...args) { + super(); + this.code = code; + this.args = args; + this.message = getErrorMessage(code, args); + } +} + +/* eslint-disable complexity */ +function getErrorMessage(code, args) { + switch (code) { + case "E0001": + return "Generic error"; + case "E0002": + return "Expected an entry start"; + case "E0003": { + const [token] = args; + return `Expected token: "${token}"`; + } + case "E0004": { + const [range] = args; + return `Expected a character from range: "${range}"`; + } + case "E0005": { + const [id] = args; + return `Expected message "${id}" to have a value or attributes`; + } + case "E0006": { + const [id] = args; + return `Expected term "-${id}" to have a value`; + } + case "E0007": + return "Keyword cannot end with a whitespace"; + case "E0008": + return "The callee has to be an upper-case identifier or a term"; + case "E0009": + return "The argument name has to be a simple identifier"; + case "E0010": + return "Expected one of the variants to be marked as default (*)"; + case "E0011": + return 'Expected at least one variant after "->"'; + case "E0012": + return "Expected value"; + case "E0013": + return "Expected variant key"; + case "E0014": + return "Expected literal"; + case "E0015": + return "Only one variant can be marked as default (*)"; + case "E0016": + return "Message references cannot be used as selectors"; + case "E0017": + return "Terms cannot be used as selectors"; + case "E0018": + return "Attributes of messages cannot be used as selectors"; + case "E0019": + return "Attributes of terms cannot be used as placeables"; + case "E0020": + return "Unterminated string expression"; + case "E0021": + return "Positional arguments must not follow named arguments"; + case "E0022": + return "Named arguments must be unique"; + case "E0024": + return "Cannot access variants of a message."; + case "E0025": { + const [char] = args; + return `Unknown escape sequence: \\${char}.`; + } + case "E0026": { + const [sequence] = args; + return `Invalid Unicode escape sequence: ${sequence}.`; + } + case "E0027": + return "Unbalanced closing brace in TextElement."; + case "E0028": + return "Expected an inline expression"; + default: + return code; + } +} + +function includes(arr, elem) { + return arr.indexOf(elem) > -1; +} + +/* eslint no-magic-numbers: "off" */ + +class ParserStream { + constructor(string) { + this.string = string; + this.index = 0; + this.peekOffset = 0; + } + + charAt(offset) { + // When the cursor is at CRLF, return LF but don't move the cursor. + // The cursor still points to the EOL position, which in this case is the + // beginning of the compound CRLF sequence. This ensures slices of + // [inclusive, exclusive) continue to work properly. + if (this.string[offset] === "\r" + && this.string[offset + 1] === "\n") { + return "\n"; + } + + return this.string[offset]; + } + + get currentChar() { + return this.charAt(this.index); + } + + get currentPeek() { + return this.charAt(this.index + this.peekOffset); + } + + next() { + this.peekOffset = 0; + // Skip over the CRLF as if it was a single character. + if (this.string[this.index] === "\r" + && this.string[this.index + 1] === "\n") { + this.index++; + } + this.index++; + return this.string[this.index]; + } + + peek() { + // Skip over the CRLF as if it was a single character. + if (this.string[this.index + this.peekOffset] === "\r" + && this.string[this.index + this.peekOffset + 1] === "\n") { + this.peekOffset++; + } + this.peekOffset++; + return this.string[this.index + this.peekOffset]; + } + + resetPeek(offset = 0) { + this.peekOffset = offset; + } + + skipToPeek() { + this.index += this.peekOffset; + this.peekOffset = 0; + } +} + +const EOL = "\n"; +const EOF = undefined; +const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; + +class FluentParserStream extends ParserStream { + peekBlankInline() { + const start = this.index + this.peekOffset; + while (this.currentPeek === " ") { + this.peek(); + } + return this.string.slice(start, this.index + this.peekOffset); + } + + skipBlankInline() { + const blank = this.peekBlankInline(); + this.skipToPeek(); + return blank; + } + + peekBlankBlock() { + let blank = ""; + while (true) { + const lineStart = this.peekOffset; + this.peekBlankInline(); + if (this.currentPeek === EOL) { + blank += EOL; + this.peek(); + continue; + } + if (this.currentPeek === EOF) { + // Treat the blank line at EOF as a blank block. + return blank; + } + // Any other char; reset to column 1 on this line. + this.resetPeek(lineStart); + return blank; + } + } + + skipBlankBlock() { + const blank = this.peekBlankBlock(); + this.skipToPeek(); + return blank; + } + + peekBlank() { + while (this.currentPeek === " " || this.currentPeek === EOL) { + this.peek(); + } + } + + skipBlank() { + this.peekBlank(); + this.skipToPeek(); + } + + expectChar(ch) { + if (this.currentChar === ch) { + this.next(); + return true; + } + + throw new ParseError("E0003", ch); + } + + expectLineEnd() { + if (this.currentChar === EOF) { + // EOF is a valid line end in Fluent. + return true; + } + + if (this.currentChar === EOL) { + this.next(); + return true; + } + + // Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) + throw new ParseError("E0003", "\u2424"); + } + + takeChar(f) { + const ch = this.currentChar; + if (ch === EOF) { + return EOF; + } + if (f(ch)) { + this.next(); + return ch; + } + return null; + } + + isCharIdStart(ch) { + if (ch === EOF) { + return false; + } + + const cc = ch.charCodeAt(0); + return (cc >= 97 && cc <= 122) || // a-z + (cc >= 65 && cc <= 90); // A-Z + } + + isIdentifierStart() { + return this.isCharIdStart(this.currentPeek); + } + + isNumberStart() { + const ch = this.currentChar === "-" + ? this.peek() + : this.currentChar; + + if (ch === EOF) { + this.resetPeek(); + return false; + } + + const cc = ch.charCodeAt(0); + const isDigit = cc >= 48 && cc <= 57; // 0-9 + this.resetPeek(); + return isDigit; + } + + isCharPatternContinuation(ch) { + if (ch === EOF) { + return false; + } + + return !includes(SPECIAL_LINE_START_CHARS, ch); + } + + isValueStart() { + // Inline Patterns may start with any char. + const ch = this.currentPeek; + return ch !== EOL && ch !== EOF; + } + + isValueContinuation() { + const column1 = this.peekOffset; + this.peekBlankInline(); + + if (this.currentPeek === "{") { + this.resetPeek(column1); + return true; + } + + if (this.peekOffset - column1 === 0) { + return false; + } + + if (this.isCharPatternContinuation(this.currentPeek)) { + this.resetPeek(column1); + return true; + } + + return false; + } + + // -1 - any + // 0 - comment + // 1 - group comment + // 2 - resource comment + isNextLineComment(level = -1) { + if (this.currentChar !== EOL) { + return false; + } + + let i = 0; + + while (i <= level || (level === -1 && i < 3)) { + if (this.peek() !== "#") { + if (i <= level && level !== -1) { + this.resetPeek(); + return false; + } + break; + } + i++; + } + + // The first char after #, ## or ###. + const ch = this.peek(); + if (ch === " " || ch === EOL) { + this.resetPeek(); + return true; + } + + this.resetPeek(); + return false; + } + + isVariantStart() { + const currentPeekOffset = this.peekOffset; + if (this.currentPeek === "*") { + this.peek(); + } + if (this.currentPeek === "[") { + this.resetPeek(currentPeekOffset); + return true; + } + this.resetPeek(currentPeekOffset); + return false; + } + + isAttributeStart() { + return this.currentPeek === "."; + } + + skipToNextEntryStart(junkStart) { + let lastNewline = this.string.lastIndexOf(EOL, this.index); + if (junkStart < lastNewline) { + // Last seen newline is _after_ the junk start. It's safe to rewind + // without the risk of resuming at the same broken entry. + this.index = lastNewline; + } + while (this.currentChar) { + // We're only interested in beginnings of line. + if (this.currentChar !== EOL) { + this.next(); + continue; + } + + // Break if the first char in this line looks like an entry start. + const first = this.next(); + if (this.isCharIdStart(first) || first === "-" || first === "#") { + break; + } + } + } + + takeIDStart() { + if (this.isCharIdStart(this.currentChar)) { + const ret = this.currentChar; + this.next(); + return ret; + } + + throw new ParseError("E0004", "a-zA-Z"); + } + + takeIDChar() { + const closure = ch => { + const cc = ch.charCodeAt(0); + return ((cc >= 97 && cc <= 122) || // a-z + (cc >= 65 && cc <= 90) || // A-Z + (cc >= 48 && cc <= 57) || // 0-9 + cc === 95 || cc === 45); // _- + }; + + return this.takeChar(closure); + } + + takeDigit() { + const closure = ch => { + const cc = ch.charCodeAt(0); + return (cc >= 48 && cc <= 57); // 0-9 + }; + + return this.takeChar(closure); + } + + takeHexDigit() { + const closure = ch => { + const cc = ch.charCodeAt(0); + return (cc >= 48 && cc <= 57) // 0-9 + || (cc >= 65 && cc <= 70) // A-F + || (cc >= 97 && cc <= 102); // a-f + }; + + return this.takeChar(closure); + } +} + +/* eslint no-magic-numbers: [0] */ + + +const trailingWSRe = /[ \t\n\r]+$/; + + +function withSpan(fn) { + return function(ps, ...args) { + if (!this.withSpans) { + return fn.call(this, ps, ...args); + } + + const start = ps.index; + const node = fn.call(this, ps, ...args); + + // Don't re-add the span if the node already has it. This may happen when + // one decorated function calls another decorated function. + if (node.span) { + return node; + } + + const end = ps.index; + node.addSpan(start, end); + return node; + }; +} + + +class FluentParser { + constructor({ + withSpans = true, + } = {}) { + this.withSpans = withSpans; + + // Poor man's decorators. + const methodNames = [ + "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", + "getVariant", "getNumber", "getPattern", "getTextElement", + "getPlaceable", "getExpression", "getInlineExpression", + "getCallArgument", "getCallArguments", "getString", "getLiteral", + ]; + for (const name of methodNames) { + this[name] = withSpan(this[name]); + } + } + + parse(source) { + const ps = new FluentParserStream(source); + ps.skipBlankBlock(); + + const entries = []; + let lastComment = null; + + while (ps.currentChar) { + const entry = this.getEntryOrJunk(ps); + const blankLines = ps.skipBlankBlock(); + + // Regular Comments require special logic. Comments may be attached to + // Messages or Terms if they are followed immediately by them. However + // they should parse as standalone when they're followed by Junk. + // Consequently, we only attach Comments once we know that the Message + // or the Term parsed successfully. + if (entry.type === "Comment" + && blankLines.length === 0 + && ps.currentChar) { + // Stash the comment and decide what to do with it in the next pass. + lastComment = entry; + continue; + } + + if (lastComment) { + if (entry.type === "Message" || entry.type === "Term") { + entry.comment = lastComment; + if (this.withSpans) { + entry.span.start = entry.comment.span.start; + } + } else { + entries.push(lastComment); + } + // In either case, the stashed comment has been dealt with; clear it. + lastComment = null; + } + + // No special logic for other types of entries. + entries.push(entry); + } + + const res = new Resource(entries); + + if (this.withSpans) { + res.addSpan(0, ps.index); + } + + return res; + } + + /* + * Parse the first Message or Term in `source`. + * + * Skip all encountered comments and start parsing at the first Message or + * Term start. Return Junk if the parsing is not successful. + * + * Preceding comments are ignored unless they contain syntax errors + * themselves, in which case Junk for the invalid comment is returned. + */ + parseEntry(source) { + const ps = new FluentParserStream(source); + ps.skipBlankBlock(); + + while (ps.currentChar === "#") { + const skipped = this.getEntryOrJunk(ps); + if (skipped.type === "Junk") { + // Don't skip Junk comments. + return skipped; + } + ps.skipBlankBlock(); + } + + return this.getEntryOrJunk(ps); + } + + getEntryOrJunk(ps) { + const entryStartPos = ps.index; + + try { + const entry = this.getEntry(ps); + ps.expectLineEnd(); + return entry; + } catch (err) { + if (!(err instanceof ParseError)) { + throw err; + } + + let errorIndex = ps.index; + ps.skipToNextEntryStart(entryStartPos); + const nextEntryStart = ps.index; + if (nextEntryStart < errorIndex) { + // The position of the error must be inside of the Junk's span. + errorIndex = nextEntryStart; + } + + // Create a Junk instance + const slice = ps.string.substring(entryStartPos, nextEntryStart); + const junk = new Junk(slice); + if (this.withSpans) { + junk.addSpan(entryStartPos, nextEntryStart); + } + const annot = new Annotation(err.code, err.args, err.message); + annot.addSpan(errorIndex, errorIndex); + junk.addAnnotation(annot); + return junk; + } + } + + getEntry(ps) { + if (ps.currentChar === "#") { + return this.getComment(ps); + } + + if (ps.currentChar === "-") { + return this.getTerm(ps); + } + + if (ps.isIdentifierStart()) { + return this.getMessage(ps); + } + + throw new ParseError("E0002"); + } + + getComment(ps) { + // 0 - comment + // 1 - group comment + // 2 - resource comment + let level = -1; + let content = ""; + + while (true) { + let i = -1; + while (ps.currentChar === "#" && (i < (level === -1 ? 2 : level))) { + ps.next(); + i++; + } + + if (level === -1) { + level = i; + } + + if (ps.currentChar !== EOL) { + ps.expectChar(" "); + let ch; + while ((ch = ps.takeChar(x => x !== EOL))) { + content += ch; + } + } + + if (ps.isNextLineComment(level)) { + content += ps.currentChar; + ps.next(); + } else { + break; + } + } + + let Comment$$1; + switch (level) { + case 0: + Comment$$1 = Comment; + break; + case 1: + Comment$$1 = GroupComment; + break; + case 2: + Comment$$1 = ResourceComment; + break; + } + return new Comment$$1(content); + } + + getMessage(ps) { + const id = this.getIdentifier(ps); + + ps.skipBlankInline(); + ps.expectChar("="); + + const value = this.maybeGetPattern(ps); + const attrs = this.getAttributes(ps); + + if (value === null && attrs.length === 0) { + throw new ParseError("E0005", id.name); + } + + return new Message(id, value, attrs); + } + + getTerm(ps) { + ps.expectChar("-"); + const id = this.getIdentifier(ps); + + ps.skipBlankInline(); + ps.expectChar("="); + + const value = this.maybeGetPattern(ps); + if (value === null) { + throw new ParseError("E0006", id.name); + } + + const attrs = this.getAttributes(ps); + return new Term(id, value, attrs); + } + + getAttribute(ps) { + ps.expectChar("."); + + const key = this.getIdentifier(ps); + + ps.skipBlankInline(); + ps.expectChar("="); + + const value = this.maybeGetPattern(ps); + if (value === null) { + throw new ParseError("E0012"); + } + + return new Attribute(key, value); + } + + getAttributes(ps) { + const attrs = []; + ps.peekBlank(); + while (ps.isAttributeStart()) { + ps.skipToPeek(); + const attr = this.getAttribute(ps); + attrs.push(attr); + ps.peekBlank(); + } + return attrs; + } + + getIdentifier(ps) { + let name = ps.takeIDStart(); + + let ch; + while ((ch = ps.takeIDChar())) { + name += ch; + } + + return new Identifier(name); + } + + getVariantKey(ps) { + const ch = ps.currentChar; + + if (ch === EOF) { + throw new ParseError("E0013"); + } + + const cc = ch.charCodeAt(0); + + if ((cc >= 48 && cc <= 57) || cc === 45) { // 0-9, - + return this.getNumber(ps); + } + + return this.getIdentifier(ps); + } + + getVariant(ps, {hasDefault}) { + let defaultIndex = false; + + if (ps.currentChar === "*") { + if (hasDefault) { + throw new ParseError("E0015"); + } + ps.next(); + defaultIndex = true; + } + + ps.expectChar("["); + + ps.skipBlank(); + + const key = this.getVariantKey(ps); + + ps.skipBlank(); + ps.expectChar("]"); + + const value = this.maybeGetPattern(ps); + if (value === null) { + throw new ParseError("E0012"); + } + + return new Variant(key, value, defaultIndex); + } + + getVariants(ps) { + const variants = []; + let hasDefault = false; + + ps.skipBlank(); + while (ps.isVariantStart()) { + const variant = this.getVariant(ps, {hasDefault}); + + if (variant.default) { + hasDefault = true; + } + + variants.push(variant); + ps.expectLineEnd(); + ps.skipBlank(); + } + + if (variants.length === 0) { + throw new ParseError("E0011"); + } + + if (!hasDefault) { + throw new ParseError("E0010"); + } + + return variants; + } + + getDigits(ps) { + let num = ""; + + let ch; + while ((ch = ps.takeDigit())) { + num += ch; + } + + if (num.length === 0) { + throw new ParseError("E0004", "0-9"); + } + + return num; + } + + getNumber(ps) { + let value = ""; + + if (ps.currentChar === "-") { + ps.next(); + value += `-${this.getDigits(ps)}`; + } else { + value += this.getDigits(ps); + } + + if (ps.currentChar === ".") { + ps.next(); + value += `.${this.getDigits(ps)}`; + } + + return new NumberLiteral(value); + } + + // maybeGetPattern distinguishes between patterns which start on the same line + // as the identifier (a.k.a. inline signleline patterns and inline multiline + // patterns) and patterns which start on a new line (a.k.a. block multiline + // patterns). The distinction is important for the dedentation logic: the + // indent of the first line of a block pattern must be taken into account when + // calculating the maximum common indent. + maybeGetPattern(ps) { + ps.peekBlankInline(); + if (ps.isValueStart()) { + ps.skipToPeek(); + return this.getPattern(ps, {isBlock: false}); + } + + ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + return this.getPattern(ps, {isBlock: true}); + } + + return null; + } + + getPattern(ps, {isBlock}) { + const elements = []; + if (isBlock) { + // A block pattern is a pattern which starts on a new line. Store and + // measure the indent of this first line for the dedentation logic. + const blankStart = ps.index; + const firstIndent = ps.skipBlankInline(); + elements.push(this.getIndent(ps, firstIndent, blankStart)); + var commonIndentLength = firstIndent.length; + } else { + commonIndentLength = Infinity; + } + + let ch; + elements: while ((ch = ps.currentChar)) { + switch (ch) { + case EOL: { + const blankStart = ps.index; + const blankLines = ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + const indent = ps.skipBlankInline(); + commonIndentLength = Math.min(commonIndentLength, indent.length); + elements.push(this.getIndent(ps, blankLines + indent, blankStart)); + continue elements; + } + + // The end condition for getPattern's while loop is a newline + // which is not followed by a valid pattern continuation. + ps.resetPeek(); + break elements; + } + case "{": + elements.push(this.getPlaceable(ps)); + continue elements; + case "}": + throw new ParseError("E0027"); + default: + const element = this.getTextElement(ps); + elements.push(element); + } + } + + const dedented = this.dedent(elements, commonIndentLength); + return new Pattern(dedented); + } + + // Create a token representing an indent. It's not part of the AST and it will + // be trimmed and merged into adjacent TextElements, or turned into a new + // TextElement, if it's surrounded by two Placeables. + getIndent(ps, value, start) { + return { + type: "Indent", + span: {start, end: ps.index}, + value, + }; + } + + // Dedent a list of elements by removing the maximum common indent from the + // beginning of text lines. The common indent is calculated in getPattern. + dedent(elements, commonIndent) { + const trimmed = []; + + for (let element of elements) { + if (element.type === "Placeable") { + trimmed.push(element); + continue; + } + + if (element.type === "Indent") { + // Strip common indent. + element.value = element.value.slice( + 0, element.value.length - commonIndent); + if (element.value.length === 0) { + continue; + } + } + + let prev = trimmed[trimmed.length - 1]; + if (prev && prev.type === "TextElement") { + // Join adjacent TextElements by replacing them with their sum. + const sum = new TextElement(prev.value + element.value); + if (this.withSpans) { + sum.addSpan(prev.span.start, element.span.end); + } + trimmed[trimmed.length - 1] = sum; + continue; + } + + if (element.type === "Indent") { + // If the indent hasn't been merged into a preceding TextElement, + // convert it into a new TextElement. + const textElement = new TextElement(element.value); + if (this.withSpans) { + textElement.addSpan(element.span.start, element.span.end); + } + element = textElement; + } + + trimmed.push(element); + } + + // Trim trailing whitespace from the Pattern. + const lastElement = trimmed[trimmed.length - 1]; + if (lastElement.type === "TextElement") { + lastElement.value = lastElement.value.replace(trailingWSRe, ""); + if (lastElement.value.length === 0) { + trimmed.pop(); + } + } + + return trimmed; + } + + getTextElement(ps) { + let buffer = ""; + + let ch; + while ((ch = ps.currentChar)) { + if (ch === "{" || ch === "}") { + return new TextElement(buffer); + } + + if (ch === EOL) { + return new TextElement(buffer); + } + + buffer += ch; + ps.next(); + } + + return new TextElement(buffer); + } + + getEscapeSequence(ps) { + const next = ps.currentChar; + + switch (next) { + case "\\": + case "\"": + ps.next(); + return `\\${next}`; + case "u": + return this.getUnicodeEscapeSequence(ps, next, 4); + case "U": + return this.getUnicodeEscapeSequence(ps, next, 6); + default: + throw new ParseError("E0025", next); + } + } + + getUnicodeEscapeSequence(ps, u, digits) { + ps.expectChar(u); + + let sequence = ""; + for (let i = 0; i < digits; i++) { + const ch = ps.takeHexDigit(); + + if (!ch) { + throw new ParseError( + "E0026", `\\${u}${sequence}${ps.currentChar}`); + } + + sequence += ch; + } + + return `\\${u}${sequence}`; + } + + getPlaceable(ps) { + ps.expectChar("{"); + ps.skipBlank(); + const expression = this.getExpression(ps); + ps.expectChar("}"); + return new Placeable(expression); + } + + getExpression(ps) { + const selector = this.getInlineExpression(ps); + ps.skipBlank(); + + if (ps.currentChar === "-") { + if (ps.peek() !== ">") { + ps.resetPeek(); + return selector; + } + + if (selector.type === "MessageReference") { + if (selector.attribute === null) { + throw new ParseError("E0016"); + } else { + throw new ParseError("E0018"); + } + } + + if (selector.type === "TermReference" && selector.attribute === null) { + throw new ParseError("E0017"); + } + + ps.next(); + ps.next(); + + ps.skipBlankInline(); + ps.expectLineEnd(); + + const variants = this.getVariants(ps); + return new SelectExpression(selector, variants); + } + + if (selector.type === "TermReference" && selector.attribute !== null) { + throw new ParseError("E0019"); + } + + return selector; + } + + getInlineExpression(ps) { + if (ps.currentChar === "{") { + return this.getPlaceable(ps); + } + + if (ps.isNumberStart()) { + return this.getNumber(ps); + } + + if (ps.currentChar === '"') { + return this.getString(ps); + } + + if (ps.currentChar === "$") { + ps.next(); + const id = this.getIdentifier(ps); + return new VariableReference(id); + } + + if (ps.currentChar === "-") { + ps.next(); + const id = this.getIdentifier(ps); + + let attr; + if (ps.currentChar === ".") { + ps.next(); + attr = this.getIdentifier(ps); + } + + let args; + if (ps.currentChar === "(") { + args = this.getCallArguments(ps); + } + + return new TermReference(id, attr, args); + } + + if (ps.isIdentifierStart()) { + const id = this.getIdentifier(ps); + + if (ps.currentChar === "(") { + // It's a Function. Ensure it's all upper-case. + if (!/^[A-Z][A-Z0-9_-]*$/.test(id.name)) { + throw new ParseError("E0008"); + } + + let args = this.getCallArguments(ps); + return new FunctionReference(id, args); + } + + let attr; + if (ps.currentChar === ".") { + ps.next(); + attr = this.getIdentifier(ps); + } + + return new MessageReference(id, attr); + } + + + throw new ParseError("E0028"); + } + + getCallArgument(ps) { + const exp = this.getInlineExpression(ps); + + ps.skipBlank(); + + if (ps.currentChar !== ":") { + return exp; + } + + if (exp.type === "MessageReference" && exp.attribute === null) { + ps.next(); + ps.skipBlank(); + + const value = this.getLiteral(ps); + return new NamedArgument(exp.id, value); + } + + throw new ParseError("E0009"); + } + + getCallArguments(ps) { + const positional = []; + const named = []; + const argumentNames = new Set(); + + ps.expectChar("("); + ps.skipBlank(); + + while (true) { + if (ps.currentChar === ")") { + break; + } + + const arg = this.getCallArgument(ps); + if (arg.type === "NamedArgument") { + if (argumentNames.has(arg.name.name)) { + throw new ParseError("E0022"); + } + named.push(arg); + argumentNames.add(arg.name.name); + } else if (argumentNames.size > 0) { + throw new ParseError("E0021"); + } else { + positional.push(arg); + } + + ps.skipBlank(); + + if (ps.currentChar === ",") { + ps.next(); + ps.skipBlank(); + continue; + } + + break; + } + + ps.expectChar(")"); + return new CallArguments(positional, named); + } + + getString(ps) { + ps.expectChar("\""); + let value = ""; + + let ch; + while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { + if (ch === "\\") { + value += this.getEscapeSequence(ps); + } else { + value += ch; + } + } + + if (ps.currentChar === EOL) { + throw new ParseError("E0020"); + } + + ps.expectChar("\""); + + return new StringLiteral(value); + } + + getLiteral(ps) { + if (ps.isNumberStart()) { + return this.getNumber(ps); + } + + if (ps.currentChar === '"') { + return this.getString(ps); + } + + throw new ParseError("E0014"); + } +} + +function indent(content) { + return content.split("\n").join("\n "); +} + +function includesNewLine(elem) { + return elem.type === "TextElement" && includes(elem.value, "\n"); +} + +function isSelectExpr(elem) { + return elem.type === "Placeable" + && elem.expression.type === "SelectExpression"; +} + +const HAS_ENTRIES = 1; + +class FluentSerializer { + constructor({ withJunk = false } = {}) { + this.withJunk = withJunk; + } + + serialize(resource) { + if (resource.type !== "Resource") { + throw new Error(`Unknown resource type: ${resource.type}`); + } + + let state = 0; + const parts = []; + + for (const entry of resource.body) { + if (entry.type !== "Junk" || this.withJunk) { + parts.push(this.serializeEntry(entry, state)); + if (!(state & HAS_ENTRIES)) { + state |= HAS_ENTRIES; + } + } + } + + return parts.join(""); + } + + serializeEntry(entry, state = 0) { + switch (entry.type) { + case "Message": + return serializeMessage(entry); + case "Term": + return serializeTerm(entry); + case "Comment": + if (state & HAS_ENTRIES) { + return `\n${serializeComment(entry, "#")}\n`; + } + return `${serializeComment(entry, "#")}\n`; + case "GroupComment": + if (state & HAS_ENTRIES) { + return `\n${serializeComment(entry, "##")}\n`; + } + return `${serializeComment(entry, "##")}\n`; + case "ResourceComment": + if (state & HAS_ENTRIES) { + return `\n${serializeComment(entry, "###")}\n`; + } + return `${serializeComment(entry, "###")}\n`; + case "Junk": + return serializeJunk(entry); + default : + throw new Error(`Unknown entry type: ${entry.type}`); + } + } +} + + +function serializeComment(comment, prefix = "#") { + const prefixed = comment.content.split("\n").map( + line => line.length ? `${prefix} ${line}` : prefix + ).join("\n"); + // Add the trailing newline. + return `${prefixed}\n`; +} + + +function serializeJunk(junk) { + return junk.content; +} + + +function serializeMessage(message) { + const parts = []; + + if (message.comment) { + parts.push(serializeComment(message.comment)); + } + + parts.push(`${message.id.name} =`); + + if (message.value) { + parts.push(serializePattern(message.value)); + } + + for (const attribute of message.attributes) { + parts.push(serializeAttribute(attribute)); + } + + parts.push("\n"); + return parts.join(""); +} + + +function serializeTerm(term) { + const parts = []; + + if (term.comment) { + parts.push(serializeComment(term.comment)); + } + + parts.push(`-${term.id.name} =`); + parts.push(serializePattern(term.value)); + + for (const attribute of term.attributes) { + parts.push(serializeAttribute(attribute)); + } + + parts.push("\n"); + return parts.join(""); +} + + +function serializeAttribute(attribute) { + const value = indent(serializePattern(attribute.value)); + return `\n .${attribute.id.name} =${value}`; +} + + +function serializePattern(pattern) { + const content = pattern.elements.map(serializeElement).join(""); + const startOnNewLine = + pattern.elements.some(isSelectExpr) || + pattern.elements.some(includesNewLine); + + if (startOnNewLine) { + return `\n ${indent(content)}`; + } + + return ` ${content}`; +} + + +function serializeElement(element) { + switch (element.type) { + case "TextElement": + return element.value; + case "Placeable": + return serializePlaceable(element); + default: + throw new Error(`Unknown element type: ${element.type}`); + } +} + + +function serializePlaceable(placeable) { + const expr = placeable.expression; + switch (expr.type) { + case "Placeable": + return `{${serializePlaceable(expr)}}`; + case "SelectExpression": + // Special-case select expression to control the whitespace around the + // opening and the closing brace. + return `{ ${serializeExpression(expr)}}`; + default: + return `{ ${serializeExpression(expr)} }`; + } +} + + +function serializeExpression(expr) { + switch (expr.type) { + case "StringLiteral": + return `"${expr.value}"`; + case "NumberLiteral": + return expr.value; + case "VariableReference": + return `$${expr.id.name}`; + case "TermReference": { + let out = `-${expr.id.name}`; + if (expr.attribute) { + out += `.${expr.attribute.name}`; + } + if (expr.arguments) { + out += serializeCallArguments(expr.arguments); + } + return out; + } + case "MessageReference": { + let out = expr.id.name; + if (expr.attribute) { + out += `.${expr.attribute.name}`; + } + return out; + } + case "FunctionReference": + return `${expr.id.name}${serializeCallArguments(expr.arguments)}`; + case "SelectExpression": { + let out = `${serializeExpression(expr.selector)} ->`; + for (let variant of expr.variants) { + out += serializeVariant(variant); + } + return `${out}\n`; + } + case "Placeable": + return serializePlaceable(expr); + default: + throw new Error(`Unknown expression type: ${expr.type}`); + } +} + + +function serializeVariant(variant) { + const key = serializeVariantKey(variant.key); + const value = indent(serializePattern(variant.value)); + + if (variant.default) { + return `\n *[${key}]${value}`; + } + + return `\n [${key}]${value}`; +} + + +function serializeCallArguments(expr) { + const positional = expr.positional.map(serializeExpression).join(", "); + const named = expr.named.map(serializeNamedArgument).join(", "); + if (expr.positional.length > 0 && expr.named.length > 0) { + return `(${positional}, ${named})`; + } + return `(${positional || named})`; +} + + +function serializeNamedArgument(arg) { + const value = serializeExpression(arg.value); + return `${arg.name.name}: ${value}`; +} + + +function serializeVariantKey(key) { + switch (key.type) { + case "Identifier": + return key.name; + case "NumberLiteral": + return key.value; + default: + throw new Error(`Unknown variant key type: ${key.type}`); + } +} + +/* + * Abstract Visitor pattern + */ +class Visitor { + visit(node) { + if (Array.isArray(node)) { + node.forEach(child => this.visit(child)); + return; + } + if (!(node instanceof BaseNode)) { + return; + } + const visit = this[`visit${node.type}`] || this.genericVisit; + visit.call(this, node); + } + + genericVisit(node) { + for (const propname of Object.keys(node)) { + this.visit(node[propname]); + } + } +} + +/* + * Abstract Transformer pattern + */ +class Transformer extends Visitor { + visit(node) { + if (!(node instanceof BaseNode)) { + return node; + } + const visit = this[`visit${node.type}`] || this.genericVisit; + return visit.call(this, node); + } + + genericVisit(node) { + for (const propname of Object.keys(node)) { + const propvalue = node[propname]; + if (Array.isArray(propvalue)) { + const newvals = propvalue + .map(child => this.visit(child)) + .filter(newchild => newchild !== undefined); + node[propname] = newvals; + } + if (propvalue instanceof BaseNode) { + const new_val = this.visit(propvalue); + if (new_val === undefined) { + delete node[propname]; + } else { + node[propname] = new_val; + } + } + } + return node; + } +} + +const visitor = ({ + Visitor: Visitor, + Transformer: Transformer +}); + +/* eslint object-shorthand: "off", + comma-dangle: "off", + no-labels: "off" */ + +this.EXPORTED_SYMBOLS = [ + ...Object.keys({ + FluentParser, + FluentSerializer, + }), + ...Object.keys(ast), + ...Object.keys(visitor), +]; diff --git a/intl/l10n/L10nRegistry.cpp b/intl/l10n/L10nRegistry.cpp new file mode 100644 index 0000000000..faf075dbbb --- /dev/null +++ b/intl/l10n/L10nRegistry.cpp @@ -0,0 +1,442 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "L10nRegistry.h" +#include "mozilla/RefPtr.h" +#include "mozilla/URLPreloader.h" +#include "nsIChannel.h" +#include "nsILoadInfo.h" +#include "nsNetUtil.h" +#include "nsString.h" +#include "nsContentUtils.h" +#include "FluentResource.h" +#include "FileSource.h" +#include "nsICategoryManager.h" +#include "mozilla/SimpleEnumerator.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/PContent.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/Preferences.h" + +using namespace mozilla; +using namespace mozilla::dom; + +namespace mozilla::intl { + +/* FluentBundleIterator */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentBundleIterator, mGlobal) + +FluentBundleIterator::FluentBundleIterator( + nsIGlobalObject* aGlobal, UniquePtr<ffi::GeckoFluentBundleIterator> aRaw) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +JSObject* FluentBundleIterator::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return FluentBundleIterator_Binding::Wrap(aCx, this, aGivenProto); +} + +void FluentBundleIterator::Next(FluentBundleIteratorResult& aResult) { + UniquePtr<ffi::FluentBundleRc> raw( + ffi::fluent_bundle_iterator_next(mRaw.get())); + if (!raw) { + aResult.mDone = true; + return; + } + aResult.mDone = false; + aResult.mValue = new FluentBundle(mGlobal, std::move(raw)); +} + +already_AddRefed<FluentBundleIterator> FluentBundleIterator::Values() { + return do_AddRef(this); +} + +/* FluentBundleAsyncIterator */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentBundleAsyncIterator, mGlobal) + +FluentBundleAsyncIterator::FluentBundleAsyncIterator( + nsIGlobalObject* aGlobal, + UniquePtr<ffi::GeckoFluentBundleAsyncIteratorWrapper> aRaw) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +JSObject* FluentBundleAsyncIterator::WrapObject( + JSContext* aCx, JS::Handle<JSObject*> aGivenProto) { + return FluentBundleAsyncIterator_Binding::Wrap(aCx, this, aGivenProto); +} + +already_AddRefed<Promise> FluentBundleAsyncIterator::Next(ErrorResult& aError) { + RefPtr<Promise> promise = Promise::Create(mGlobal, aError); + if (aError.Failed()) { + return nullptr; + } + + ffi::fluent_bundle_async_iterator_next( + mRaw.get(), promise, + // callback function which will be invoked by the rust code, passing the + // promise back in. + [](auto* aPromise, ffi::FluentBundleRc* aBundle) { + Promise* promise = const_cast<Promise*>(aPromise); + + FluentBundleIteratorResult res; + + if (aBundle) { + // The Rust caller will transfer the ownership to us. + UniquePtr<ffi::FluentBundleRc> b(aBundle); + nsIGlobalObject* global = promise->GetGlobalObject(); + res.mValue = new FluentBundle(global, std::move(b)); + res.mDone = false; + } else { + res.mDone = true; + } + promise->MaybeResolve(res); + }); + + return promise.forget(); +} + +already_AddRefed<FluentBundleAsyncIterator> +FluentBundleAsyncIterator::Values() { + return do_AddRef(this); +} + +/* L10nRegistry */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(L10nRegistry, mGlobal) + +L10nRegistry::L10nRegistry(nsIGlobalObject* aGlobal, bool aUseIsolating) + : mGlobal(aGlobal), + mRaw(dont_AddRef(ffi::l10nregistry_new(aUseIsolating))) {} + +L10nRegistry::L10nRegistry(nsIGlobalObject* aGlobal, + RefPtr<const ffi::GeckoL10nRegistry> aRaw) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +/* static */ +already_AddRefed<L10nRegistry> L10nRegistry::Constructor( + const GlobalObject& aGlobal, const L10nRegistryOptions& aOptions) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + return MakeAndAddRef<L10nRegistry>(global, + aOptions.mBundleOptions.mUseIsolating); +} + +/* static */ +already_AddRefed<L10nRegistry> L10nRegistry::GetInstance( + const GlobalObject& aGlobal) { + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + return MakeAndAddRef<L10nRegistry>( + global, dont_AddRef(ffi::l10nregistry_instance_get())); +} + +JSObject* L10nRegistry::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return L10nRegistry_Binding::Wrap(aCx, this, aGivenProto); +} + +void L10nRegistry::GetAvailableLocales(nsTArray<nsCString>& aRetVal) { + ffi::l10nregistry_get_available_locales(mRaw.get(), &aRetVal); +} + +void L10nRegistry::RegisterSources( + const Sequence<OwningNonNull<L10nFileSource>>& aSources) { + nsTArray<const ffi::FileSource*> sources(aSources.Length()); + for (const auto& source : aSources) { + sources.AppendElement(source->Raw()); + } + + ffi::l10nregistry_register_sources(mRaw.get(), &sources); +} + +void L10nRegistry::UpdateSources( + const Sequence<OwningNonNull<L10nFileSource>>& aSources) { + nsTArray<const ffi::FileSource*> sources(aSources.Length()); + for (const auto& source : aSources) { + sources.AppendElement(source->Raw()); + } + + ffi::l10nregistry_update_sources(mRaw.get(), &sources); +} + +void L10nRegistry::RemoveSources(const Sequence<nsCString>& aSources) { + ffi::l10nregistry_remove_sources(mRaw.get(), aSources.Elements(), + aSources.Length()); +} + +bool L10nRegistry::HasSource(const nsACString& aName, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + + bool result = ffi::l10nregistry_has_source(mRaw.get(), &aName, &status); + PopulateError(aRv, status); + return result; +} + +already_AddRefed<L10nFileSource> L10nRegistry::GetSource( + const nsACString& aName, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + + RefPtr<const ffi::FileSource> raw( + dont_AddRef(ffi::l10nregistry_get_source(mRaw.get(), &aName, &status))); + if (PopulateError(aRv, status)) { + return nullptr; + } + + return MakeAndAddRef<L10nFileSource>(std::move(raw)); +} + +void L10nRegistry::GetSourceNames(nsTArray<nsCString>& aRetVal) { + ffi::l10nregistry_get_source_names(mRaw.get(), &aRetVal); +} + +void L10nRegistry::ClearSources() { + ffi::l10nregistry_clear_sources(mRaw.get()); +} + +/* static */ +ffi::GeckoResourceId L10nRegistry::ResourceIdToFFI( + const nsCString& aResourceId) { + return ffi::GeckoResourceId{ + aResourceId, + ffi::GeckoResourceType::Required, + }; +} + +/* static */ +ffi::GeckoResourceId L10nRegistry::ResourceIdToFFI( + const dom::OwningUTF8StringOrResourceId& aResourceId) { + if (aResourceId.IsUTF8String()) { + return ffi::GeckoResourceId{ + aResourceId.GetAsUTF8String(), + ffi::GeckoResourceType::Required, + }; + } + return ffi::GeckoResourceId{ + aResourceId.GetAsResourceId().mPath, + aResourceId.GetAsResourceId().mOptional + ? ffi::GeckoResourceType::Optional + : ffi::GeckoResourceType::Required, + }; +} + +/* static */ +nsTArray<ffi::GeckoResourceId> L10nRegistry::ResourceIdsToFFI( + const nsTArray<nsCString>& aResourceIds) { + nsTArray<ffi::GeckoResourceId> ffiResourceIds; + for (const auto& resourceId : aResourceIds) { + ffiResourceIds.EmplaceBack(ResourceIdToFFI(resourceId)); + } + return ffiResourceIds; +} + +/* static */ +nsTArray<ffi::GeckoResourceId> L10nRegistry::ResourceIdsToFFI( + const nsTArray<dom::OwningUTF8StringOrResourceId>& aResourceIds) { + nsTArray<ffi::GeckoResourceId> ffiResourceIds; + for (const auto& resourceId : aResourceIds) { + ffiResourceIds.EmplaceBack(ResourceIdToFFI(resourceId)); + } + return ffiResourceIds; +} + +already_AddRefed<FluentBundleIterator> L10nRegistry::GenerateBundlesSync( + const nsTArray<nsCString>& aLocales, + const nsTArray<ffi::GeckoResourceId>& aResourceIds, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + UniquePtr<ffi::GeckoFluentBundleIterator> iter( + ffi::l10nregistry_generate_bundles_sync( + mRaw, aLocales.Elements(), aLocales.Length(), aResourceIds.Elements(), + aResourceIds.Length(), &status)); + + if (PopulateError(aRv, status) || !iter) { + return nullptr; + } + + return do_AddRef(new FluentBundleIterator(mGlobal, std::move(iter))); +} + +already_AddRefed<FluentBundleIterator> L10nRegistry::GenerateBundlesSync( + const dom::Sequence<nsCString>& aLocales, + const dom::Sequence<dom::OwningUTF8StringOrResourceId>& aResourceIds, + ErrorResult& aRv) { + auto ffiResourceIds{ResourceIdsToFFI(aResourceIds)}; + return GenerateBundlesSync(aLocales, ffiResourceIds, aRv); +} + +already_AddRefed<FluentBundleAsyncIterator> L10nRegistry::GenerateBundles( + const nsTArray<nsCString>& aLocales, + const nsTArray<ffi::GeckoResourceId>& aResourceIds, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + UniquePtr<ffi::GeckoFluentBundleAsyncIteratorWrapper> iter( + ffi::l10nregistry_generate_bundles( + mRaw, aLocales.Elements(), aLocales.Length(), aResourceIds.Elements(), + aResourceIds.Length(), &status)); + if (PopulateError(aRv, status) || !iter) { + return nullptr; + } + + return do_AddRef(new FluentBundleAsyncIterator(mGlobal, std::move(iter))); +} + +already_AddRefed<FluentBundleAsyncIterator> L10nRegistry::GenerateBundles( + const dom::Sequence<nsCString>& aLocales, + const dom::Sequence<dom::OwningUTF8StringOrResourceId>& aResourceIds, + ErrorResult& aRv) { + nsTArray<ffi::GeckoResourceId> resourceIds; + for (const auto& resourceId : aResourceIds) { + resourceIds.EmplaceBack(ResourceIdToFFI(resourceId)); + } + return GenerateBundles(aLocales, resourceIds, aRv); +} + +/* static */ +void L10nRegistry::GetParentProcessFileSourceDescriptors( + nsTArray<L10nFileSourceDescriptor>& aRetVal) { + MOZ_ASSERT(XRE_IsParentProcess()); + nsTArray<ffi::L10nFileSourceDescriptor> sources; + ffi::l10nregistry_get_parent_process_sources(&sources); + for (const auto& source : sources) { + auto descriptor = aRetVal.AppendElement(); + descriptor->name() = source.name; + descriptor->metasource() = source.metasource; + descriptor->locales().AppendElements(std::move(source.locales)); + descriptor->prePath() = source.pre_path; + descriptor->index().AppendElements(std::move(source.index)); + } +} + +/* static */ +void L10nRegistry::RegisterFileSourcesFromParentProcess( + const nsTArray<L10nFileSourceDescriptor>& aDescriptors) { + // This means that in content processes the L10nRegistry + // service instance is created eagerly, not lazily. + // It is necessary so that the instance can store the sources + // provided in the IPC init, which, in turn, is necessary + // for the service to be avialable for sync bundle generation. + // + // L10nRegistry is lightweight and performs no operations, so + // we believe this behavior to be acceptable. + MOZ_ASSERT(XRE_IsContentProcess()); + nsTArray<ffi::L10nFileSourceDescriptor> sources; + for (const auto& desc : aDescriptors) { + auto source = sources.AppendElement(); + source->name = desc.name(); + source->metasource = desc.metasource(); + source->locales.AppendElements(desc.locales()); + source->pre_path = desc.prePath(); + source->index.AppendElements(desc.index()); + } + ffi::l10nregistry_register_parent_process_sources(&sources); +} + +/* static */ +nsresult L10nRegistry::Load(const nsACString& aPath, + nsIStreamLoaderObserver* aObserver) { + nsCOMPtr<nsIURI> uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), aPath); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(uri, NS_ERROR_INVALID_ARG); + + RefPtr<nsIStreamLoader> loader; + rv = NS_NewStreamLoader( + getter_AddRefs(loader), uri, aObserver, + nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + nsIContentPolicy::TYPE_OTHER); + + return rv; +} + +/* static */ +nsresult L10nRegistry::LoadSync(const nsACString& aPath, void** aData, + uint64_t* aSize) { + nsCOMPtr<nsIURI> uri; + + nsresult rv = NS_NewURI(getter_AddRefs(uri), aPath); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ENSURE_TRUE(uri, NS_ERROR_INVALID_ARG); + + auto result = URLPreloader::ReadURI(uri); + if (result.isOk()) { + auto uri = result.unwrap(); + *aData = ToNewCString(uri); + *aSize = uri.Length(); + return NS_OK; + } + + auto err = result.unwrapErr(); + if (err != NS_ERROR_INVALID_ARG && err != NS_ERROR_NOT_INITIALIZED) { + return err; + } + + nsCOMPtr<nsIChannel> channel; + rv = NS_NewChannel( + getter_AddRefs(channel), uri, nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + nsIContentPolicy::TYPE_OTHER, nullptr, /* nsICookieJarSettings */ + nullptr, /* aPerformanceStorage */ + nullptr, /* aLoadGroup */ + nullptr, /* aCallbacks */ + nsIRequest::LOAD_NORMAL); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIInputStream> input; + rv = channel->Open(getter_AddRefs(input)); + NS_ENSURE_SUCCESS(rv, NS_ERROR_INVALID_ARG); + + return NS_ReadInputStreamToBuffer(input, aData, -1, aSize); +} + +/* static */ +bool L10nRegistry::PopulateError(ErrorResult& aError, + ffi::L10nRegistryStatus& aStatus) { + switch (aStatus) { + case ffi::L10nRegistryStatus::InvalidLocaleCode: + aError.ThrowTypeError("Invalid locale code"); + return true; + case ffi::L10nRegistryStatus::EmptyName: + aError.ThrowTypeError("Name cannot be empty."); + return true; + + case ffi::L10nRegistryStatus::None: + return false; + } + MOZ_ASSERT_UNREACHABLE("Unknown status"); + return false; +} + +extern "C" { +nsresult L10nRegistryLoad(const nsACString* aPath, + const nsIStreamLoaderObserver* aObserver) { + if (!aPath || !aObserver) { + return NS_ERROR_INVALID_ARG; + } + + return mozilla::intl::L10nRegistry::Load( + *aPath, const_cast<nsIStreamLoaderObserver*>(aObserver)); +} + +nsresult L10nRegistryLoadSync(const nsACString* aPath, void** aData, + uint64_t* aSize) { + if (!aPath || !aData || !aSize) { + return NS_ERROR_INVALID_ARG; + } + + return mozilla::intl::L10nRegistry::LoadSync(*aPath, aData, aSize); +} + +void L10nRegistrySendUpdateL10nFileSources() { + MOZ_ASSERT(XRE_IsParentProcess()); + nsTArray<L10nFileSourceDescriptor> sources; + L10nRegistry::GetParentProcessFileSourceDescriptors(sources); + + nsTArray<ContentParent*> parents; + ContentParent::GetAll(parents); + for (ContentParent* parent : parents) { + Unused << parent->SendUpdateL10nFileSources(sources); + } +} + +} // extern "C" + +} // namespace mozilla::intl diff --git a/intl/l10n/L10nRegistry.h b/intl/l10n/L10nRegistry.h new file mode 100644 index 0000000000..91047ec99c --- /dev/null +++ b/intl/l10n/L10nRegistry.h @@ -0,0 +1,152 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_L10nRegistry_h +#define mozilla_intl_l10n_L10nRegistry_h + +#include "nsIStreamLoader.h" +#include "nsWrapperCache.h" +#include "nsCycleCollectionParticipant.h" +#include "mozilla/dom/L10nRegistryBinding.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/intl/FluentBindings.h" +#include "mozilla/intl/RegistryBindings.h" + +class nsIGlobalObject; + +namespace mozilla::dom { +class L10nFileSourceDescriptor; +} + +namespace mozilla::intl { + +class FluentBundleAsyncIterator final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentBundleAsyncIterator) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentBundleAsyncIterator) + + FluentBundleAsyncIterator( + nsIGlobalObject* aGlobal, + UniquePtr<ffi::GeckoFluentBundleAsyncIteratorWrapper> aRaw); + + virtual JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + // WebIDL + already_AddRefed<dom::Promise> Next(ErrorResult& aError); + already_AddRefed<FluentBundleAsyncIterator> Values(); + + protected: + ~FluentBundleAsyncIterator() = default; + nsCOMPtr<nsIGlobalObject> mGlobal; + UniquePtr<ffi::GeckoFluentBundleAsyncIteratorWrapper> mRaw; +}; + +class FluentBundleIterator final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentBundleIterator) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentBundleIterator) + + FluentBundleIterator(nsIGlobalObject* aGlobal, + UniquePtr<ffi::GeckoFluentBundleIterator> aRaw); + + virtual JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + // WebIDL + void Next(dom::FluentBundleIteratorResult& aResult); + already_AddRefed<FluentBundleIterator> Values(); + + protected: + ~FluentBundleIterator() = default; + nsCOMPtr<nsIGlobalObject> mGlobal; + UniquePtr<ffi::GeckoFluentBundleIterator> mRaw; +}; + +class L10nRegistry final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(L10nRegistry) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(L10nRegistry) + + L10nRegistry(nsIGlobalObject* aGlobal, bool aUseIsolating); + + L10nRegistry(nsIGlobalObject* aGlobal, + RefPtr<const ffi::GeckoL10nRegistry> aRaw); + + static already_AddRefed<L10nRegistry> Constructor( + const dom::GlobalObject& aGlobal, + const dom::L10nRegistryOptions& aOptions); + + static already_AddRefed<L10nRegistry> GetInstance( + const dom::GlobalObject& aGlobal); + + static void GetParentProcessFileSourceDescriptors( + nsTArray<dom::L10nFileSourceDescriptor>& aRetVal); + static void RegisterFileSourcesFromParentProcess( + const nsTArray<dom::L10nFileSourceDescriptor>& aDescriptors); + + static nsresult Load(const nsACString& aPath, + nsIStreamLoaderObserver* aObserver); + static nsresult LoadSync(const nsACString& aPath, void** aData, + uint64_t* aSize); + + static ffi::GeckoResourceId ResourceIdToFFI(const nsCString& aResourceId); + static ffi::GeckoResourceId ResourceIdToFFI( + const dom::OwningUTF8StringOrResourceId& aResourceId); + static nsTArray<ffi::GeckoResourceId> ResourceIdsToFFI( + const nsTArray<nsCString>& aResourceIds); + static nsTArray<ffi::GeckoResourceId> ResourceIdsToFFI( + const nsTArray<dom::OwningUTF8StringOrResourceId>& aResourceIds); + + void GetAvailableLocales(nsTArray<nsCString>& aRetVal); + + void RegisterSources( + const dom::Sequence<OwningNonNull<L10nFileSource>>& aSources); + void UpdateSources( + const dom::Sequence<OwningNonNull<L10nFileSource>>& aSources); + void RemoveSources(const dom::Sequence<nsCString>& aSources); + bool HasSource(const nsACString& aName, ErrorResult& aRv); + already_AddRefed<L10nFileSource> GetSource(const nsACString& aName, + ErrorResult& aRv); + void GetSourceNames(nsTArray<nsCString>& aRetVal); + void ClearSources(); + + already_AddRefed<FluentBundleIterator> GenerateBundlesSync( + const nsTArray<nsCString>& aLocales, + const nsTArray<ffi::GeckoResourceId>& aResourceIds, ErrorResult& aRv); + already_AddRefed<FluentBundleIterator> GenerateBundlesSync( + const dom::Sequence<nsCString>& aLocales, + const dom::Sequence<dom::OwningUTF8StringOrResourceId>& aResourceIds, + ErrorResult& aRv); + + already_AddRefed<FluentBundleAsyncIterator> GenerateBundles( + const nsTArray<nsCString>& aLocales, + const nsTArray<ffi::GeckoResourceId>& aResourceIds, ErrorResult& aRv); + already_AddRefed<FluentBundleAsyncIterator> GenerateBundles( + const dom::Sequence<nsCString>& aLocales, + const dom::Sequence<dom::OwningUTF8StringOrResourceId>& aResourceIds, + ErrorResult& aRv); + + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + + const ffi::GeckoL10nRegistry* Raw() const { return mRaw; } + + protected: + virtual ~L10nRegistry() = default; + nsCOMPtr<nsIGlobalObject> mGlobal; + const RefPtr<const ffi::GeckoL10nRegistry> mRaw; + static bool PopulateError(ErrorResult& aError, + ffi::L10nRegistryStatus& aStatus); +}; + +} // namespace mozilla::intl + +#endif diff --git a/intl/l10n/Localization.cpp b/intl/l10n/Localization.cpp new file mode 100644 index 0000000000..0d6f1f9882 --- /dev/null +++ b/intl/l10n/Localization.cpp @@ -0,0 +1,522 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Localization.h" +#include "nsIObserverService.h" +#include "mozilla/BasePrincipal.h" +#include "mozilla/Preferences.h" +#include "mozilla/Services.h" +#include "mozilla/dom/PromiseNativeHandler.h" + +#define INTL_APP_LOCALES_CHANGED "intl:app-locales-changed" +#define L10N_PSEUDO_PREF "intl.l10n.pseudo" + +using namespace mozilla; +using namespace mozilla::dom; +using namespace mozilla::intl; + +static const char* kObservedPrefs[] = {L10N_PSEUDO_PREF, nullptr}; + +static nsTArray<ffi::L10nKey> ConvertFromL10nKeys( + const Sequence<OwningUTF8StringOrL10nIdArgs>& aKeys) { + nsTArray<ffi::L10nKey> l10nKeys(aKeys.Length()); + + for (const auto& entry : aKeys) { + if (entry.IsUTF8String()) { + const auto& id = entry.GetAsUTF8String(); + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &id; + } else { + const auto& e = entry.GetAsL10nIdArgs(); + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &e.mId; + if (!e.mArgs.IsNull()) { + FluentBundle::ConvertArgs(e.mArgs.Value(), key->args); + } + } + } + + return l10nKeys; +} + +[[nodiscard]] static bool ConvertToAttributeNameValue( + const nsTArray<ffi::L10nAttribute>& aAttributes, + FallibleTArray<AttributeNameValue>& aValues) { + if (!aValues.SetCapacity(aAttributes.Length(), fallible)) { + return false; + } + for (const auto& attr : aAttributes) { + auto* cvtAttr = aValues.AppendElement(fallible); + MOZ_ASSERT(cvtAttr, "SetCapacity didn't set enough capacity somehow?"); + cvtAttr->mName = attr.name; + cvtAttr->mValue = attr.value; + } + return true; +} + +[[nodiscard]] static bool ConvertToL10nMessages( + const nsTArray<ffi::OptionalL10nMessage>& aMessages, + nsTArray<Nullable<L10nMessage>>& aOut) { + if (!aOut.SetCapacity(aMessages.Length(), fallible)) { + return false; + } + + for (const auto& entry : aMessages) { + Nullable<L10nMessage>* msg = aOut.AppendElement(fallible); + MOZ_ASSERT(msg, "SetCapacity didn't set enough capacity somehow?"); + + if (!entry.is_present) { + continue; + } + + L10nMessage& m = msg->SetValue(); + if (!entry.message.value.IsVoid()) { + m.mValue = entry.message.value; + } + if (!entry.message.attributes.IsEmpty()) { + auto& value = m.mAttributes.SetValue(); + if (!ConvertToAttributeNameValue(entry.message.attributes, value)) { + return false; + } + } + } + + return true; +} + +NS_IMPL_CYCLE_COLLECTING_ADDREF(Localization) +NS_IMPL_CYCLE_COLLECTING_RELEASE(Localization) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(Localization) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsIObserver) + NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIObserver) +NS_INTERFACE_MAP_END +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_WEAK(Localization, mGlobal) + +/* static */ +already_AddRefed<Localization> Localization::Create( + const nsTArray<nsCString>& aResourceIds, bool aIsSync) { + return MakeAndAddRef<Localization>(aResourceIds, aIsSync); +} + +/* static */ +already_AddRefed<Localization> Localization::Create( + const nsTArray<ffi::GeckoResourceId>& aResourceIds, bool aIsSync) { + return MakeAndAddRef<Localization>(aResourceIds, aIsSync); +} + +Localization::Localization(const nsTArray<nsCString>& aResIds, bool aIsSync) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResIds)}; + ffi::localization_new(&ffiResourceIds, aIsSync, nullptr, + getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(const nsTArray<ffi::GeckoResourceId>& aResIds, + bool aIsSync) { + ffi::localization_new(&aResIds, aIsSync, nullptr, getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(nsIGlobalObject* aGlobal, + const nsTArray<nsCString>& aResIds, bool aIsSync) + : mGlobal(aGlobal) { + nsTArray<ffi::GeckoResourceId> resourceIds{ + L10nRegistry::ResourceIdsToFFI(aResIds)}; + ffi::localization_new(&resourceIds, aIsSync, nullptr, getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(nsIGlobalObject* aGlobal, bool aIsSync) + : mGlobal(aGlobal) { + nsTArray<ffi::GeckoResourceId> resIds; + ffi::localization_new(&resIds, aIsSync, nullptr, getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(nsIGlobalObject* aGlobal, bool aIsSync, + const ffi::LocalizationRc* aRaw) + : mGlobal(aGlobal), mRaw(aRaw) { + RegisterObservers(); +} + +already_AddRefed<Localization> Localization::Constructor( + const GlobalObject& aGlobal, + const Sequence<OwningUTF8StringOrResourceId>& aResourceIds, bool aIsSync, + const Optional<NonNull<L10nRegistry>>& aRegistry, + const Optional<Sequence<nsCString>>& aLocales, ErrorResult& aRv) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResourceIds)}; + Maybe<nsTArray<nsCString>> locales; + + if (aLocales.WasPassed()) { + locales.emplace(); + locales->SetCapacity(aLocales.Value().Length()); + for (const auto& locale : aLocales.Value()) { + locales->AppendElement(locale); + } + } + + RefPtr<const ffi::LocalizationRc> raw; + + bool result = ffi::localization_new_with_locales( + &ffiResourceIds, aIsSync, + aRegistry.WasPassed() ? aRegistry.Value().Raw() : nullptr, + locales.ptrOr(nullptr), getter_AddRefs(raw)); + + if (!result) { + aRv.ThrowInvalidStateError( + "Failed to create the Localization. Check the locales arguments."); + return nullptr; + } + + nsCOMPtr<nsIGlobalObject> global = do_QueryInterface(aGlobal.GetAsSupports()); + + return do_AddRef(new Localization(global, aIsSync, raw)); +} + +JSObject* Localization::WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) { + return Localization_Binding::Wrap(aCx, this, aGivenProto); +} + +Localization::~Localization() = default; + +NS_IMETHODIMP +Localization::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + if (!strcmp(aTopic, INTL_APP_LOCALES_CHANGED)) { + OnChange(); + } else { + MOZ_ASSERT(!strcmp("nsPref:changed", aTopic)); + nsDependentString pref(aData); + if (pref.EqualsLiteral(L10N_PSEUDO_PREF)) { + OnChange(); + } + } + + return NS_OK; +} + +void Localization::RegisterObservers() { + DebugOnly<nsresult> rv = Preferences::AddWeakObservers(this, kObservedPrefs); + MOZ_ASSERT(NS_SUCCEEDED(rv), "Adding observers failed."); + + nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService(); + if (obs) { + obs->AddObserver(this, INTL_APP_LOCALES_CHANGED, true); + } +} + +void Localization::OnChange() { ffi::localization_on_change(mRaw.get()); } + +void Localization::AddResourceId(const ffi::GeckoResourceId& aResourceId) { + ffi::localization_add_res_id(mRaw.get(), &aResourceId); +} +void Localization::AddResourceId(const nsCString& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + AddResourceId(ffiResourceId); +} +void Localization::AddResourceId( + const dom::OwningUTF8StringOrResourceId& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + AddResourceId(ffiResourceId); +} + +uint32_t Localization::RemoveResourceId( + const ffi::GeckoResourceId& aResourceId) { + return ffi::localization_remove_res_id(mRaw.get(), &aResourceId); +} +uint32_t Localization::RemoveResourceId(const nsCString& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + return RemoveResourceId(ffiResourceId); +} +uint32_t Localization::RemoveResourceId( + const dom::OwningUTF8StringOrResourceId& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + return RemoveResourceId(ffiResourceId); +} + +void Localization::AddResourceIds( + const nsTArray<dom::OwningUTF8StringOrResourceId>& aResourceIds) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResourceIds)}; + ffi::localization_add_res_ids(mRaw.get(), &ffiResourceIds); +} + +uint32_t Localization::RemoveResourceIds( + const nsTArray<dom::OwningUTF8StringOrResourceId>& aResourceIds) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResourceIds)}; + return ffi::localization_remove_res_ids(mRaw.get(), &ffiResourceIds); +} + +already_AddRefed<Promise> Localization::FormatValue( + const nsACString& aId, const Optional<L10nArgs>& aArgs, ErrorResult& aRv) { + nsTArray<ffi::L10nArg> l10nArgs; + nsTArray<nsCString> errors; + + if (aArgs.WasPassed()) { + const L10nArgs& args = aArgs.Value(); + FluentBundle::ConvertArgs(args, l10nArgs); + } + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + + ffi::localization_format_value( + mRaw.get(), &aId, &l10nArgs, promise, + [](const Promise* aPromise, const nsACString* aValue, + const nsTArray<nsCString>* aErrors) { + Promise* promise = const_cast<Promise*>(aPromise); + + ErrorResult rv; + if (MaybeReportErrorsToGecko(*aErrors, rv, + promise->GetParentObject())) { + promise->MaybeReject(std::move(rv)); + } else { + promise->MaybeResolve(aValue); + } + }); + + return MaybeWrapPromise(promise); +} + +already_AddRefed<Promise> Localization::FormatValues( + const Sequence<OwningUTF8StringOrL10nIdArgs>& aKeys, ErrorResult& aRv) { + nsTArray<ffi::L10nKey> l10nKeys = ConvertFromL10nKeys(aKeys); + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (aRv.Failed()) { + return nullptr; + } + + ffi::localization_format_values( + mRaw.get(), &l10nKeys, promise, + // callback function which will be invoked by the rust code, passing the + // promise back in. + [](const Promise* aPromise, const nsTArray<nsCString>* aValues, + const nsTArray<nsCString>* aErrors) { + Promise* promise = const_cast<Promise*>(aPromise); + + ErrorResult rv; + if (MaybeReportErrorsToGecko(*aErrors, rv, + promise->GetParentObject())) { + promise->MaybeReject(std::move(rv)); + } else { + promise->MaybeResolve(*aValues); + } + }); + + return MaybeWrapPromise(promise); +} + +already_AddRefed<Promise> Localization::FormatMessages( + const Sequence<OwningUTF8StringOrL10nIdArgs>& aKeys, ErrorResult& aRv) { + auto l10nKeys = ConvertFromL10nKeys(aKeys); + + RefPtr<Promise> promise = Promise::Create(mGlobal, aRv); + if (aRv.Failed()) { + return nullptr; + } + + ffi::localization_format_messages( + mRaw.get(), &l10nKeys, promise, + // callback function which will be invoked by the rust code, passing the + // promise back in. + [](const Promise* aPromise, + const nsTArray<ffi::OptionalL10nMessage>* aRaw, + const nsTArray<nsCString>* aErrors) { + Promise* promise = const_cast<Promise*>(aPromise); + + ErrorResult rv; + if (MaybeReportErrorsToGecko(*aErrors, rv, + promise->GetParentObject())) { + promise->MaybeReject(std::move(rv)); + } else { + nsTArray<Nullable<L10nMessage>> messages; + if (!ConvertToL10nMessages(*aRaw, messages)) { + promise->MaybeReject(NS_ERROR_OUT_OF_MEMORY); + } else { + promise->MaybeResolve(std::move(messages)); + } + } + }); + + return MaybeWrapPromise(promise); +} + +void Localization::FormatValueSync(const nsACString& aId, + const Optional<L10nArgs>& aArgs, + nsACString& aRetVal, ErrorResult& aRv) { + nsTArray<ffi::L10nArg> l10nArgs; + nsTArray<nsCString> errors; + + if (aArgs.WasPassed()) { + const L10nArgs& args = aArgs.Value(); + FluentBundle::ConvertArgs(args, l10nArgs); + } + + bool rv = ffi::localization_format_value_sync(mRaw.get(), &aId, &l10nArgs, + &aRetVal, &errors); + + if (rv) { + MaybeReportErrorsToGecko(errors, aRv, GetParentObject()); + } else { + aRv.ThrowInvalidStateError( + "Can't use formatValueSync when state is async."); + } +} + +void Localization::FormatValuesSync( + const Sequence<OwningUTF8StringOrL10nIdArgs>& aKeys, + nsTArray<nsCString>& aRetVal, ErrorResult& aRv) { + nsTArray<ffi::L10nKey> l10nKeys(aKeys.Length()); + nsTArray<nsCString> errors; + + for (const auto& entry : aKeys) { + if (entry.IsUTF8String()) { + const auto& id = entry.GetAsUTF8String(); + nsTArray<ffi::L10nArg> l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &id; + } else { + const auto& e = entry.GetAsL10nIdArgs(); + nsTArray<ffi::L10nArg> l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &e.mId; + if (!e.mArgs.IsNull()) { + FluentBundle::ConvertArgs(e.mArgs.Value(), key->args); + } + } + } + + bool rv = ffi::localization_format_values_sync(mRaw.get(), &l10nKeys, + &aRetVal, &errors); + + if (rv) { + MaybeReportErrorsToGecko(errors, aRv, GetParentObject()); + } else { + aRv.ThrowInvalidStateError( + "Can't use formatValuesSync when state is async."); + } +} + +void Localization::FormatMessagesSync( + const Sequence<OwningUTF8StringOrL10nIdArgs>& aKeys, + nsTArray<Nullable<L10nMessage>>& aRetVal, ErrorResult& aRv) { + nsTArray<ffi::L10nKey> l10nKeys(aKeys.Length()); + nsTArray<nsCString> errors; + + for (const auto& entry : aKeys) { + if (entry.IsUTF8String()) { + const auto& id = entry.GetAsUTF8String(); + nsTArray<ffi::L10nArg> l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &id; + } else { + const auto& e = entry.GetAsL10nIdArgs(); + nsTArray<ffi::L10nArg> l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &e.mId; + if (!e.mArgs.IsNull()) { + FluentBundle::ConvertArgs(e.mArgs.Value(), key->args); + } + } + } + + nsTArray<ffi::OptionalL10nMessage> result(l10nKeys.Length()); + + bool rv = ffi::localization_format_messages_sync(mRaw.get(), &l10nKeys, + &result, &errors); + + if (!rv) { + return aRv.ThrowInvalidStateError( + "Can't use formatMessagesSync when state is async."); + } + MaybeReportErrorsToGecko(errors, aRv, GetParentObject()); + if (aRv.Failed()) { + return; + } + if (!ConvertToL10nMessages(result, aRetVal)) { + return aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + } +} + +void Localization::SetAsync() { ffi::localization_set_async(mRaw.get()); } +bool Localization::IsSync() { return ffi::localization_is_sync(mRaw.get()); } + +/** + * PromiseResolver is a PromiseNativeHandler used + * by MaybeWrapPromise method. + */ +class PromiseResolver final : public PromiseNativeHandler { + public: + NS_DECL_ISUPPORTS + + explicit PromiseResolver(Promise* aPromise) : mPromise(aPromise) {} + void ResolvedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override; + void RejectedCallback(JSContext* aCx, JS::Handle<JS::Value> aValue, + ErrorResult& aRv) override; + + protected: + virtual ~PromiseResolver(); + + RefPtr<Promise> mPromise; +}; + +NS_INTERFACE_MAP_BEGIN(PromiseResolver) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +NS_IMPL_ADDREF(PromiseResolver) +NS_IMPL_RELEASE(PromiseResolver) + +void PromiseResolver::ResolvedCallback(JSContext* aCx, + JS::Handle<JS::Value> aValue, + ErrorResult& aRv) { + mPromise->MaybeResolveWithClone(aCx, aValue); +} + +void PromiseResolver::RejectedCallback(JSContext* aCx, + JS::Handle<JS::Value> aValue, + ErrorResult& aRv) { + mPromise->MaybeRejectWithClone(aCx, aValue); +} + +PromiseResolver::~PromiseResolver() { mPromise = nullptr; } + +/** + * MaybeWrapPromise is a helper method used by Localization + * API methods to clone the value returned by a promise + * into a new context. + * + * This allows for a promise from a privileged context + * to be returned into an unprivileged document. + * + * This method is only used for promises that carry values. + */ +already_AddRefed<Promise> Localization::MaybeWrapPromise( + Promise* aInnerPromise) { + MOZ_ASSERT(aInnerPromise->State() == Promise::PromiseState::Pending); + // For system principal we don't need to wrap the + // result promise at all. + nsIPrincipal* principal = mGlobal->PrincipalOrNull(); + if (principal && principal->IsSystemPrincipal()) { + return do_AddRef(aInnerPromise); + } + + IgnoredErrorResult result; + RefPtr<Promise> docPromise = Promise::Create(mGlobal, result); + if (NS_WARN_IF(result.Failed())) { + return nullptr; + } + + auto resolver = MakeRefPtr<PromiseResolver>(docPromise); + aInnerPromise->AppendNativeHandler(resolver); + return docPromise.forget(); +} diff --git a/intl/l10n/Localization.h b/intl/l10n/Localization.h new file mode 100644 index 0000000000..54c78f68db --- /dev/null +++ b/intl/l10n/Localization.h @@ -0,0 +1,165 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_Localization_h +#define mozilla_intl_l10n_Localization_h + +#include "nsCycleCollectionParticipant.h" +#include "nsIObserver.h" +#include "nsWeakReference.h" +#include "nsWrapperCache.h" +#include "nsWeakReference.h" +#include "nsIScriptError.h" +#include "nsContentUtils.h" +#include "nsPIDOMWindow.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/LocalizationBinding.h" +#include "mozilla/intl/LocalizationBindings.h" +#include "mozilla/intl/L10nRegistry.h" + +namespace mozilla { +namespace intl { + +// The state where the application contains incomplete localization resources +// is much more common than for other types of core resources. +// +// In result, our localization is designed to handle missing resources +// gracefully, and we need a more fine-tuned way to communicate those problems +// to developers. +// +// In particular, we want developers and early adopters to be able to reason +// about missing translations, without bothering end user in production, where +// the user cannot react to that. +// +// We currently differentiate between nightly/dev-edition builds or automation +// where we report the errors, and beta/release, where we silence them. +// +// A side effect of the conditional model of strict vs loose error handling is +// that we don't have a way to write integration tests for behavior we expect +// out of production environment. See bug 1741430. +[[maybe_unused]] static bool MaybeReportErrorsToGecko( + const nsTArray<nsCString>& aErrors, ErrorResult& aRv, + nsIGlobalObject* aGlobal) { + if (!aErrors.IsEmpty()) { + if (xpc::IsInAutomation()) { + aRv.ThrowInvalidStateError(aErrors.ElementAt(0)); + return true; + } + +#if defined(NIGHTLY_BUILD) || defined(MOZ_DEV_EDITION) || defined(DEBUG) + dom::Document* doc = nullptr; + if (aGlobal) { + nsPIDOMWindowInner* innerWindow = aGlobal->AsInnerWindow(); + if (innerWindow) { + doc = innerWindow->GetExtantDoc(); + } + } + + for (const auto& error : aErrors) { + nsContentUtils::ReportToConsoleNonLocalized(NS_ConvertUTF8toUTF16(error), + nsIScriptError::warningFlag, + "l10n"_ns, doc); + printf_stderr("%s\n", error.get()); + } +#endif + } + + return false; +} + +class Localization : public nsIObserver, + public nsWrapperCache, + public nsSupportsWeakReference { + template <typename T, typename... Args> + friend already_AddRefed<T> mozilla::MakeAndAddRef(Args&&... aArgs); + + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS_AMBIGUOUS(Localization, + nsIObserver) + NS_DECL_NSIOBSERVER + + static already_AddRefed<Localization> Constructor( + const dom::GlobalObject& aGlobal, + const dom::Sequence<dom::OwningUTF8StringOrResourceId>& aResourceIds, + bool aIsSync, const dom::Optional<dom::NonNull<L10nRegistry>>& aRegistry, + const dom::Optional<dom::Sequence<nsCString>>& aLocales, + ErrorResult& aRv); + static already_AddRefed<Localization> Create( + const nsTArray<nsCString>& aResourceIds, bool aIsSync); + static already_AddRefed<Localization> Create( + const nsTArray<ffi::GeckoResourceId>& aResourceIds, bool aIsSync); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle<JSObject*> aGivenProto) override; + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + void SetIsSync(bool aIsSync); + + already_AddRefed<dom::Promise> FormatValue( + const nsACString& aId, const dom::Optional<L10nArgs>& aArgs, + ErrorResult& aRv); + + already_AddRefed<dom::Promise> FormatValues( + const dom::Sequence<dom::OwningUTF8StringOrL10nIdArgs>& aKeys, + ErrorResult& aRv); + + already_AddRefed<dom::Promise> FormatMessages( + const dom::Sequence<dom::OwningUTF8StringOrL10nIdArgs>& aKeys, + ErrorResult& aRv); + + void FormatValueSync(const nsACString& aId, + const dom::Optional<L10nArgs>& aArgs, + nsACString& aRetVal, ErrorResult& aRv); + void FormatValuesSync( + const dom::Sequence<dom::OwningUTF8StringOrL10nIdArgs>& aKeys, + nsTArray<nsCString>& aRetVal, ErrorResult& aRv); + void FormatMessagesSync( + const dom::Sequence<dom::OwningUTF8StringOrL10nIdArgs>& aKeys, + nsTArray<dom::Nullable<dom::L10nMessage>>& aRetVal, ErrorResult& aRv); + + void AddResourceId(const ffi::GeckoResourceId& aResourceId); + void AddResourceId(const nsCString& aResourceId); + void AddResourceId(const dom::OwningUTF8StringOrResourceId& aResourceId); + uint32_t RemoveResourceId(const ffi::GeckoResourceId& aResourceId); + uint32_t RemoveResourceId(const nsCString& aResourceId); + uint32_t RemoveResourceId( + const dom::OwningUTF8StringOrResourceId& aResourceId); + void AddResourceIds( + const nsTArray<dom::OwningUTF8StringOrResourceId>& aResourceIds); + uint32_t RemoveResourceIds( + const nsTArray<dom::OwningUTF8StringOrResourceId>& aResourceIds); + + void SetAsync(); + bool IsSync(); + + protected: + Localization(const nsTArray<nsCString>& aResIds, bool aIsSync); + Localization(const nsTArray<ffi::GeckoResourceId>& aResIds, bool aIsSync); + Localization(nsIGlobalObject* aGlobal, bool aIsSync); + + Localization(nsIGlobalObject* aGlobal, const nsTArray<nsCString>& aResIds, + bool aIsSync); + + Localization(nsIGlobalObject* aGlobal, bool aIsSync, + const ffi::LocalizationRc* aRaw); + + virtual ~Localization(); + + void RegisterObservers(); + virtual void OnChange(); + already_AddRefed<dom::Promise> MaybeWrapPromise(dom::Promise* aInnerPromise); + + nsCOMPtr<nsIGlobalObject> mGlobal; + RefPtr<const ffi::LocalizationRc> mRaw; +}; + +} // namespace intl +} // namespace mozilla + +#endif diff --git a/intl/l10n/LocalizationBindings.h b/intl/l10n/LocalizationBindings.h new file mode 100644 index 0000000000..fbf4db2db0 --- /dev/null +++ b/intl/l10n/LocalizationBindings.h @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_LocalizationBindings_h +#define mozilla_intl_l10n_LocalizationBindings_h + +#include "mozilla/intl/localization_ffi_generated.h" + +#include "mozilla/RefPtr.h" + +namespace mozilla { + +template <> +struct RefPtrTraits<intl::ffi::LocalizationRc> { + static void AddRef(const intl::ffi::LocalizationRc* aPtr) { + intl::ffi::localization_addref(aPtr); + } + static void Release(const intl::ffi::LocalizationRc* aPtr) { + intl::ffi::localization_release(aPtr); + } +}; + +} // namespace mozilla + +#endif diff --git a/intl/l10n/README b/intl/l10n/README new file mode 100644 index 0000000000..d507db0522 --- /dev/null +++ b/intl/l10n/README @@ -0,0 +1,8 @@ +The content of this directory is partially sourced from the fluent.js project. + +The following files are affected: + - FluentSyntax.jsm + +At the moment, the tool used to produce those files in fluent.js repository, doesn't +fully align with how the code is structured here, so we perform a manual adjustments +mostly around header and footer. diff --git a/intl/l10n/RegistryBindings.h b/intl/l10n/RegistryBindings.h new file mode 100644 index 0000000000..916082e5cf --- /dev/null +++ b/intl/l10n/RegistryBindings.h @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_RegistryBindings_h +#define mozilla_intl_l10n_RegistryBindings_h + +#include "mozilla/intl/l10nregistry_ffi_generated.h" + +#include "mozilla/RefPtr.h" + +namespace mozilla { + +template <> +struct RefPtrTraits<intl::ffi::FileSource> { + static void AddRef(const intl::ffi::FileSource* aPtr) { + intl::ffi::l10nfilesource_addref(aPtr); + } + static void Release(const intl::ffi::FileSource* aPtr) { + intl::ffi::l10nfilesource_release(aPtr); + } +}; + +template <> +class DefaultDelete<intl::ffi::GeckoFluentBundleIterator> { + public: + void operator()(intl::ffi::GeckoFluentBundleIterator* aPtr) const { + fluent_bundle_iterator_destroy(aPtr); + } +}; + +template <> +class DefaultDelete<intl::ffi::GeckoFluentBundleAsyncIteratorWrapper> { + public: + void operator()( + intl::ffi::GeckoFluentBundleAsyncIteratorWrapper* aPtr) const { + fluent_bundle_async_iterator_destroy(aPtr); + } +}; + +template <> +struct RefPtrTraits<intl::ffi::GeckoL10nRegistry> { + static void AddRef(const intl::ffi::GeckoL10nRegistry* aPtr) { + intl::ffi::l10nregistry_addref(aPtr); + } + static void Release(const intl::ffi::GeckoL10nRegistry* aPtr) { + intl::ffi::l10nregistry_release(aPtr); + } +}; + +} // namespace mozilla + +#endif diff --git a/intl/l10n/docs/crosschannel/commits.rst b/intl/l10n/docs/crosschannel/commits.rst new file mode 100644 index 0000000000..955baf734f --- /dev/null +++ b/intl/l10n/docs/crosschannel/commits.rst @@ -0,0 +1,33 @@ +Commits and Metadata +==================== + +When creating the commit for a particular revision, we need to find the +revisions on the other branches of cross-channel to unify the created +content with. + +To do so, the cross-channel algorithm keeps track of metadata associated with +a revision in the target repository. This metadata is stored in the commit +message: + +.. code-block:: bash + + X-Channel-Repo: mozilla-central + X-Channel-Converted-Revision: af4a1de0a11cb3afbb7e50bcdd0919f56c23959a + X-Channel-Repo: releases/mozilla-beta + X-Channel-Revision: 65fb3f6bce94f8696e1571c2d48104dbdc0b31e2 + X-Channel-Repo: releases/mozilla-release + X-Channel-Revision: 1c5bf69f887359645f1c3df4de0d0e3caf957e59 + X-Channel-Repo: releases/mozilla-esr68 + X-Channel-Revision: 4cbbc30e1ebc3254ec74dc041aff128c81220507 + +This metadata is appended to the original commit message when committing. +For each branch in the cross-channel configuration we have the name and +a revision. The revision that's currently converted is explicitly highlighted +by the ``-Converted-`` marker. On hg.mozilla.org, those revisions are also +marked up as links, so one can navigate from the converted changeset to the +original patch. + +When starting the update for an incremental graph from the previous section, +the metadata is read from the target repository, and the data for the +currently converted branch is updated for each commit. Each revision in +this metadata then goes into the algorithm to create the unified content. diff --git a/intl/l10n/docs/crosschannel/content.rst b/intl/l10n/docs/crosschannel/content.rst new file mode 100644 index 0000000000..01f5e1ab66 --- /dev/null +++ b/intl/l10n/docs/crosschannel/content.rst @@ -0,0 +1,129 @@ +===================== +Cross-channel Content +===================== + +When creating the actual content, there's a number of questions to answer. + +#. Where to take content from? +#. Which content to take? +#. Where to put the content? +#. What to put into each file? + +Content Sources +--------------- + +The content of each revision in ``gecko-strings`` corresponds to a given +revision in each original repository. For example, we could have + ++------------------+--------------+ +| Repository | Revision | ++==================+==============+ +| mozilla-central | 4c92802939c1 | ++------------------+--------------+ +| mozilla-beta | ace4081e8200 | ++------------------+--------------+ +| mozilla-release | 2cf08fbb92b2 | ++------------------+--------------+ +| mozilla-esr68 | 2cf9e0c91d51 | ++------------------+--------------+ +| comm-central | 3f3fc2c0d804 | ++------------------+--------------+ +| comm-beta | f95a6f4408a3 | ++------------------+--------------+ +| comm-release | dc2694f035fa | ++------------------+--------------+ +| comm-esr68 | d05d4d87d25c | ++------------------+--------------+ + +The assumption is that there's no content that's shared between ``mozilla-*`` and +``comm-*``, so we can just convert one repository and its branches at a time. + +Covered Content +--------------- + +Which content is included in ``gecko-strings`` is +controlled by the project configurations of each product, on each branch. +Currently, those are :file:`browser/locales/l10n.toml` and +:file:`mobile/android/locales/l10n.toml` in ``mozilla-central``. + +Created Content Structure +------------------------- + +The created content is laid out in the directory in the same structure as +the files in ``l10n-central``. The localizable files end up like this: + +.. code-block:: + + browser/ + browser/ + browser.ftl + chrome/ + browser.properties + toolkit/ + toolkit/ + about/aboutAbout.ftl + +This matches the file locations in ``mozilla-central`` with the +:file:`locales/en-US` part dropped. + +The project configuration files are also converted and added to the +created file structure. As they're commonly in the :file:`locales` folder +which we strip, they're added to the dedicated :file:`_configs` folder. + +.. code-block:: bash + + $ ls _configs + browser.toml devtools-client.toml devtools-shared.toml + mobile-android.toml toolkit.toml + + +L10n File Contents +------------------ + +Let's assume we have a file to localize in several revisions with different +content. + +== ======= ==== ======= +ID central beta release +== ======= ==== ======= +a one one one +b two two +c three +d four old old +== ======= ==== ======= + +The algorithm then creates content, taking localizable values from the left-most +branch, where *central* overrides *beta*, and *beta* overrides *release*. This +creates content as follows: + +== ======= +ID content +== ======= +a one +b two +c three +d four +== ======= + +If a file doesn't exist in one of the revisions, that revision is dropped +from the content generation for this particular file. + +.. note:: + + The example of the forth string here highlights the impact that changing + an existing string has. We ship one translation of *four* to central, + beta, and release. That's only a good idea if it doesn't matter which of the + two versions of the English copy got translated. + +Project configurations +---------------------- + +The TOML files for project configuration are processed, but not unified +across branches at this point. + +.. note:: + + The content of the ``-central`` branch determines what's localized + from ``gecko-strings``. Thus that TOML file needs to include all + directories across all branches for now. Removing entries requires + that the content is obsolete on all branches in cross-channel. diff --git a/intl/l10n/docs/crosschannel/index.rst b/intl/l10n/docs/crosschannel/index.rst new file mode 100644 index 0000000000..faa28d6157 --- /dev/null +++ b/intl/l10n/docs/crosschannel/index.rst @@ -0,0 +1,88 @@ +============= +Cross-channel +============= + +Firefox is localized with a process nick-named *cross-channel*. This document +explains both the general idea as well as some technical details of that +process. The gist of it is this: + + We use one localization for all release channels. + +There's a number of upsides to that: + +* Localizers maintain a single source of truth. Localizers can work on Nightly, + while updating Beta, Developer Edition or even Release and ESR. +* Localizers can work on strings at their timing. +* Uplifting string changes has less of an impact on the localization toolchain, + and their impact can be evaluated case by case. + +So the problem at hand is to have one localization source +and use that to build 5 different versions of Firefox. The goal is for that +localization to be as complete as possible for each version. While we do +allow for partial localizations, we don't want to enforce partial translations +on any version. + +The process to tackle these follows these steps: + +* Create resource to localize, ``gecko-strings``. + + * Review updates to that resource in *quarantine*. + * Expose a known good state of that resource to localizers. + +* The actual localization work happens in Pontoon. +* Write localizations back to ``l10n-central``. +* Get localizations into the builds. + +.. digraph:: full_tree + + graph [ rankdir=LR ]; + "m-c" -> "quarantine"; + "m-b" -> "quarantine"; + "m-r" -> "quarantine"; + "c-c" -> "quarantine"; + "c-b" -> "quarantine"; + "c-r" -> "quarantine"; + "quarantine" -> "gecko-strings"; + "gecko-strings" -> "Pontoon"; + "Pontoon" -> "l10n-central"; + "l10n-central" -> "Nightly"; + "l10n-central" -> "Beta"; + "l10n-central" -> "Firefox"; + "l10n-central" -> "Daily"; + "l10n-central" -> "Thunderbird"; + { + rank=same; + "quarantine"; + "gecko-strings"; + } + +.. note:: + + The concept behind the quarantine in the process above is to + protect localizers from churn on strings that have technical + problems. Examples like that could be missing localization notes + or copy that should be improved. + +The resource to localize is a Mercurial repository, unifying +all strings to localize for all covered products and branches. Each revision +of this repository holds all the strings for a particular point in time. + +There's three aspects that we'll want to unify here. + +#. Create a version history that allows the localization team + to learn where strings in the generated repository are coming from. +#. Unify the content across different branches for a single app. +#. Unify different apps, coming from different repositories. + +The last item is the easiest, as ``mozilla-*`` and ``comm-*`` don't share +code or history. Thus, they're converted individually to disjunct directories +and files in the target repository, and the Mercurial history of each is interleaved +in the target history. When parents are needed for one repository, they're +rebased over the commits for the other. + +.. toctree:: + :maxdepth: 1 + + commits + content + repositories diff --git a/intl/l10n/docs/crosschannel/repositories.rst b/intl/l10n/docs/crosschannel/repositories.rst new file mode 100644 index 0000000000..8461b32fbd --- /dev/null +++ b/intl/l10n/docs/crosschannel/repositories.rst @@ -0,0 +1,14 @@ +gecko-strings and Quarantine +============================ + +The actual generation is currently done via `taskcluster cron <https://treeherder.mozilla.org/jobs?repo=mozilla-central&searchStr=cross-channel>`_. +The state that is good to use by localizers at large is published at +https://hg.mozilla.org/l10n/gecko-strings/. + +The L10n team is doing a :ref:`review step <exposure-in-gecko-strings>` before publishing the strings, and while +that is ongoing, the intermediate state is published to +https://hg.mozilla.org/l10n/gecko-strings-quarantine/. + +The code is in https://hg.mozilla.org/mozilla-central/file/tip/python/l10n/mozxchannel/, +supported as a mach subcommand in https://hg.mozilla.org/mozilla-central/file/tip/tools/compare-locales/mach_commands.py, +as a taskcluster kind in https://hg.mozilla.org/mozilla-central/file/tip/taskcluster/ci/l10n-cross-channel, and scheduled in cron in https://hg.mozilla.org/mozilla-central/file/tip/.cron.yml. diff --git a/intl/l10n/docs/fluent/index.rst b/intl/l10n/docs/fluent/index.rst new file mode 100644 index 0000000000..84103db5e4 --- /dev/null +++ b/intl/l10n/docs/fluent/index.rst @@ -0,0 +1,25 @@ +====== +Fluent +====== + +`Fluent`_ is a localization system developed by Mozilla, which aims to replace +all existing localization models currently used at Mozilla. + +In case of Firefox it directly supersedes DTD and StringBundle systems, providing +a large number of features and improvements over both of them, for developers +and localizers. + +.. toctree:: + :maxdepth: 2 + + tutorial + review + +Other resources: + + * `Fluent Syntax Guide <http://projectfluent.org/fluent/guide/>`_ + * `Fluent Wiki <https://github.com/projectfluent/fluent/wiki>`_ + * `Fluent.js Wiki <https://github.com/projectfluent/fluent.js/wiki>`_ + * `Fluent DOM L10n Tutorial <https://projectfluent.org/dom-l10n-documentation/>`_ + +.. _Fluent: http://projectfluent.org/ diff --git a/intl/l10n/docs/fluent/review.rst b/intl/l10n/docs/fluent/review.rst new file mode 100644 index 0000000000..83d65ebed9 --- /dev/null +++ b/intl/l10n/docs/fluent/review.rst @@ -0,0 +1,303 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +=============================== +Guidelines for Fluent Reviewers +=============================== + +This document is intended as a guideline for developers and reviewers when +working with FTL (Fluent) files. As such, it’s not meant to replace the +`existing extensive documentation`__ about Fluent. + +__ ./tutorial.html + +`Herald`_ is used to set the group `fluent-reviewers`_ as blocking reviewer for +any patch modifying FTL files committed to Phabricator. The person from this +group performing the review will have to manually set other reviewers as +blocking, if the original developer didn’t originally do it. + + +.. hint:: + + In case of doubt, you should always reach out to the l10n team for + clarifications. + + +Message Identifiers +=================== + +While in Fluent it’s possible to use both lowercase and uppercase characters in +message identifiers, the naming convention in Gecko is to use lowercase and +hyphens (*kebab-case*), avoiding CamelCase and underscores. For example, +:js:`allow-button` should be preferred to :js:`allow_button` or +:js:`allowButton`, unless there are technically constraints – like identifiers +generated at run-time from external sources – that make this impractical. + +When importing multiple FTL files, all messages share the same scope in the +Fluent bundle. For that reason, it’s suggested to add scope to the message +identifier itself: using :js:`cancel` as an identifier increases the chances of +having a conflict, :js:`save-dialog-cancel-button` would make it less likely. + +Message identifiers are also used as the ultimate fall back in case of run-time +errors. Having a descriptive message ID would make such fall back more useful +for the user. + +Comments +======== + +When a message includes placeables (variables), there should always be a +comment explaining the format of the variable, and what kind of content it will +be replaced with. This is the format suggested for such comments: + + +.. code-block:: fluent + + # This string is used on a new line below the add-on name + # Variables: + # $name (String) - Add-on author name + cfr-doorhanger-extension-author = by { $name } + + +By default, a comment is bound to the message immediately following it. Fluent +supports both `file-level and group-level comments`__. Be aware that a group +comment will apply to all messages following that comment until the end of the +file. If that shouldn’t be the case, you’ll need to “reset” the group comment, +by adding an empty one (:js:`##`), or moving the section of messages at the end +of the file. + +__ https://projectfluent.org/fluent/guide/comments.html + +Comments are fundamental for localizers, since they don’t see the file as a +whole, or changes as a fragment of a larger patch. Their work happens on a +message at a time, and the context is only provided by comments. + +License headers are standalone comments, that is, a single :js:`#` as prefix, +and the comment is followed by at least one empty line. + +Changes to Existing Messages +============================ + +You must update the message identifier if: + +- The meaning of the sentence has changed. +- You’re changing the morphology of the message, by adding or removing attributes. + +Messages are identified in the entire localization toolchain by their ID. For +this reason, there’s no need to change attribute names. + +If your changes are relevant only for English — for example, to correct a +typographical error or to make letter case consistent — then there is generally +no need to update the message identifier. + +There is a grey area between needing a new ID or not. In some cases, it will be +necessary to look at all the existing translations to determine if a new ID +would be beneficial. You should always reach out to the l10n team in case of +doubt. + +Changing the message ID will invalidate the existing translation, the new +message will be reported as missing in all tools, and localizers will have to +retranslate it. This is the only reliable method to ensure that localizers +update existing localizations, and run-time stop using obsolete translations. + +You must also update all instances where that message identifier is used in the +source code, including localization comments. + +Non-text Elements in Messages +============================= + +When a message includes non text-elements – like anchors or images – make sure +that they have a :js:`data-l10n-name` associated to them. Additional +attributes, like the URL for an anchor or CSS classes, should not be exposed +for localization in the FTL file. More details can be found in `this page`__ +dedicated to DOM overlays. + +__ https://github.com/projectfluent/fluent.js/wiki/DOM-Overlays#text-level-elements + +This information is not relevant if your code is using `fluent-react`_, where +DOM overlays `work differently`__. + +__ https://github.com/projectfluent/fluent.js/wiki/React-Overlays + +Message References +================== + +Consider the following example: + + +.. code-block:: fluent + + newtab-search-box-search-the-web-text = Search the Web + newtab-search-box-search-the-web-input = + .placeholder = { newtab-search-box-search-the-web-text } + .title = { newtab-search-box-search-the-web-text } + + +This might seem to reduce the work for localizers, but it actually doesn’t +help: + +- A change to the referenced message (:js:`newtab-search-box-search-the-web-text`) + would require a new ID also for all messages referencing it. +- Translation memory can help with matching text, not with message references. + +On the other hand, this approach is helpful if, for example, you want to +reference another element of the UI in your message: + + +.. code-block:: fluent + + help-button = Help + help-explanation = Click the { help-button} to access support + + +This enforces consistency and, if :js:`help-button` changes, all other messages +will need to be updated anyway. + +Terms +===== + +Fluent supports a specific type of message, called `term`_. Terms are similar +to regular messages but they can only be used as references in other messages. +They are best used to define vocabulary and glossary items which can be used +consistently across the localization of the entire product. + +Terms are typically used for brand names, like :js:`Firefox` or :js:`Mozilla`: +it allows to have them in one easily identifiable place, and raise warnings +when a localization is not using them. It helps enforcing consistency and brand +protection. If you simply need to reference a message from another message, you +don’t need a term: cross references between messages are allowed, but they +should not be abused, as already described. + +Variants and plurals +==================== + +Consider the following example: + + +.. code-block:: fluent + + items-selected = + { $num -> + [0] Select items. + [one] One item selected. + *[other] { $num } items selected. + } + + +In this example, there’s no guarantee that all localizations will have this +variant covered, since variants are private by design. The correct approach for +the example would be to have a separate message for the :js:`0` case: + + +.. code-block:: fluent + + # Separate messages which serve different purposes. + items-select = Select items + # The default variant works for all values of the selector. + items-selected = + { $num -> + [one] One item selected. + *[other] { $num } items selected. + } + + +As a rule of thumb: + +- Use variants only if the default variant makes sense for all possible values + of the selector. +- The code shouldn’t depend on the availability of a specific variant. + +More examples about selector and variant abuses can be found in `this wiki`__. + +__ https://github.com/projectfluent/fluent/wiki/Good-Practices-for-Developers#prefer-separate-messages-over-variants-for-ui-logic + +In general, also avoid putting a selector in the middle of a sentence, like in +the example below: + + +.. code-block:: fluent + + items-selected = + { $num -> + [one] One item. + *[other] { $num } items + } selected. + + +:js:`1` should only be used in case you want to cover the literal number. If +it’s a standard plural, you should use the :js:`one` category for singular. +Also make sure to always pass the variable to these messages as a number, not +as a string. + +Access Keys +=========== + +The following is a simple potential example of an access key: + +.. code-block:: fluent + + example-menu-item = + .label = Menu Item + .accesskey = M + +Access keys are used in menus in order to help provide easy keyboard shortcut access. They +are useful for both power users, and for users who have accessibility needs. It is +helpful to first read the `Access keys`__ guide in the Windows Developer documentation, +as it outlines the best practices for Windows applications. + +__ https://docs.microsoft.com/en-us/windows/uwp/design/input/access-keys + +There are some differences between operating systems. Linux mostly follows the same +practices as Windows. However, macOS in general does not have good support for accesskeys, +especially in menus. + +When choosing an access key, it's important that it's unique relative to the current level +of UI. It's preferable to avoid letters with descending parts, such as :code:`g`, +:code:`j`, :code:`p`, and :code:`q` as these will not be underlined nicely in Windows or +Linux. Other problematic characters are ones which are narrow, such as :code:`l`, +:code:`i` and :code:`I`. The underline may not be as visible as other letters in +sans-serif fonts. + +Linter +====== + +:bash:`mach lint` includes a :ref:`l10n linter <L10n>`, called :bash:`moz-l10n-lint`. It +can be run locally by developers but also runs on Treeherder: in the Build +Status section of the diff on Phabricator, open the Treeherder Jobs link and +look for the :js:`l1nt` job. + +Besides displaying errors and warnings due to syntax errors, it’s particularly +important because it also checks for message changes without new IDs, and +conflicts with the cross-channel repository used to ship localized versions of +Firefox. + + +.. warning:: + + Currently, there’s an `issue`__ preventing warnings to be displayed in + Phabricator. Checks can be run locally using :bash:`./mach lint -l l10n -W`. + + __ https://github.com/mozilla/code-review/issues/32 + + +Migrating Strings From Legacy or Fluent Files +============================================= + +If a patch is moving legacy strings (.properties, .DTD) to Fluent, it should +also include a recipe to migrate existing strings to FTL messages. The same is +applicable if a patch moves existing Fluent messages to a different file, or +changes the morphology of existing messages without actual changes to the +content. + +Documentation on how to write and test migration recipes is available in `this +page`__. + +__ ./fluent_migrations.html + + +.. _Herald: https://phabricator.services.mozilla.com/herald/ +.. _fluent-reviewers: https://phabricator.services.mozilla.com/tag/fluent-reviewers/ +.. _fluent-react: https://github.com/projectfluent/fluent.js/wiki/React-Bindings +.. _term: https://projectfluent.org/fluent/guide/terms.html diff --git a/intl/l10n/docs/fluent/tutorial.rst b/intl/l10n/docs/fluent/tutorial.rst new file mode 100644 index 0000000000..1312efd448 --- /dev/null +++ b/intl/l10n/docs/fluent/tutorial.rst @@ -0,0 +1,750 @@ +.. role:: html(code) + :language: html + +.. role:: js(code) + :language: javascript + +============================= +Fluent for Firefox Developers +============================= + + +This tutorial is intended for Firefox engineers already familiar with the previous +localization systems offered by Gecko - `DTD`_ and `StringBundle`_ - and assumes +prior experience with those systems. + +For a more hands-on tutorial of understanding Fluent from the ground up, try +following the `Fluent DOMLocalization Tutorial`__, which provides some background on +how Fluent works and walks you through creating a basic web project from scratch that +uses Fluent for localization. + +__ https://projectfluent.org/dom-l10n-documentation/ + +Using Fluent in Gecko +===================== + +`Fluent`_ is a modern localization system introduced into +the Gecko platform with a focus on quality, performance, maintenance and completeness. + +The legacy DTD system is deprecated, and Fluent should be used where possible. + +Getting a Review +---------------- + +If you work on any patch that touches FTL files, you'll need to get a review +from `fluent-reviewers`__. There's a Herald hook that automatically sets +that group as a blocking reviewer. + +__ https://phabricator.services.mozilla.com/tag/fluent-reviewers/ + +Guidelines for the review process are available `here`__. + +__ ./fluent_review.html + +To lighten the burden on reviewers, please take a moment to review some +best practices before submitting your patch for review. + +- `ProjectFluent Good Practices for Developers`_ +- `Mozilla Localization Best Practices For Developers`_ + +.. _ProjectFluent Good Practices for Developers: https://github.com/projectfluent/fluent/wiki/Good-Practices-for-Developers +.. _Mozilla Localization Best Practices For Developers: https://mozilla-l10n.github.io/documentation/localization/dev_best_practices.html + +Major Benefits +============== + +Fluent `ties tightly`__ into the domain of internationalization +through `Unicode`_, `CLDR`_ and `ICU`_. + +__ https://github.com/projectfluent/fluent/wiki/Fluent-and-Standards + +More specifically, the most observable benefits for each group of consumers are + + +Developers +---------- + + - Support for XUL, XHTML, HTML, Web Components, React, JS, Python and Rust + - Strings are available in a single, unified localization context available for both DOM and runtime code + - Full internationalization (i18n) support: date and time formatting, number formatting, plurals, genders etc. + - Strong focus on `declarative API via DOM attributes`__ + - Extensible with custom formatters, Mozilla-specific APIs etc. + - `Separation of concerns`__: localization details, and the added complexity of some languages, don't leak onto the source code and are no concern for developers + - Compound messages link a single translation unit to a single UI element + - `DOM Overlays`__ allow for localization of DOM fragments + - Simplified build system model + - No need for pre-processing instructions + - Support for pseudolocalization + +__ https://github.com/projectfluent/fluent/wiki/Get-Started +__ https://github.com/projectfluent/fluent/wiki/Design-Principles +__ https://github.com/projectfluent/fluent.js/wiki/DOM-Overlays + + +Product Quality +------------------ + + - A robust, multilevel, `error fallback system`__ prevents XML errors and runtime errors + - Simplified l10n API reduces the amount of l10n specific code and resulting bugs + - Runtime localization allows for dynamic language changes and updates over-the-air + - DOM Overlays increase localization security + +__ https://github.com/projectfluent/fluent/wiki/Error-Handling + + +Fluent Translation List - FTL +============================= + +Fluent introduces a file format designed specifically for easy readability +and the localization features offered by the system. + +At first glance the format is a simple key-value store. It may look like this: + +.. code-block:: fluent + + home-page-header = Home Page + + # The label of a button opening a new tab + new-tab-open = Open New Tab + +But the FTL file format is significantly more powerful and the additional features +quickly add up. In order to familiarize yourself with the basic features, +consider reading through the `Fluent Syntax Guide`_ to understand +a more complex example like: + +.. code-block:: fluent + + ### These messages correspond to security and privacy user interface. + ### + ### Please choose simple and non-threatening language when localizing + ### to help user feel in control when interacting with the UI. + + ## General Section + + -brand-short-name = Firefox + .gender = masculine + + pref-pane = + .title = + { PLATFORM() -> + [windows] Options + *[other] Preferences + } + .accesskey = C + + # Variables: + # $tabCount (Number) - number of container tabs to be closed + containers-disable-alert-ok-button = + { $tabCount -> + [one] Close { $tabCount } Container Tab + *[other] Close { $tabCount } Container Tabs + } + + update-application-info = + You are using { -brand-short-name } Version: { $version }. + Please read the <a>privacy policy</a>. + +The above, of course, is a particular selection of complex strings intended to exemplify +the new features and concepts introduced by Fluent. + +.. important:: + + While in Fluent it’s possible to use both lowercase and uppercase characters in message + identifiers, the naming convention in Gecko is to use lowercase and hyphens, avoiding + CamelCase and underscores. For example, `allow-button` should be preferred to + `allow_button` or `allowButton`, unless there are technically constraints – like + identifiers generated at run-time from external sources – that make this impractical. + +In order to ensure the quality of the output, a lot of checks and tooling +is part of the build system. +`Pontoon`_, the main localization tool used to translate Firefox, also supports +Fluent and its features to help localizers in their work. + + +.. _fluent-tutorial-social-contract: + +Social Contract +=============== + +Fluent uses the concept of a `social contract` between developer and localizers. +This contract is established by the selection of a unique identifier, called :js:`l10n-id`, +which carries a promise of being used in a particular place to carry a particular meaning. + +The use of unique identifiers is shared with legacy localization systems in +Firefox. + +.. important:: + + An important part of the contract is that the developer commits to treat the + localization output as `opaque`. That means that no concatenations, replacements + or splitting should happen after the translation is completed to generate the + desired output. + +In return, localizers enter the social contract by promising to provide an accurate +and clean translation of the messages that match the request. + +In Fluent, the developer is not to be bothered with inner logic and complexity that the +localization will use to construct the response. Whether `declensions`__ or other +variant selection techniques are used is up to a localizer and their particular translation. +From the developer perspective, Fluent returns a final string to be presented to +the user, with no l10n logic required in the running code. + +__ https://en.wikipedia.org/wiki/Declension + + +Markup Localization +=================== + +To localize an element in Fluent, the developer adds a new message to +an FTL file and then has to associate an :js:`l10n-id` with the element +by defining a :js:`data-l10n-id` attribute: + +.. code-block:: html + + <h1 data-l10n-id="home-page-header" /> + + <button data-l10n-id="pref-pane" /> + +Fluent will take care of the rest, populating the element with the message value +in its content and all localizable attributes if defined. + +The developer provides only a single message to localize the whole element, +including the value and selected attributes. + +The value can be a whole fragment of DOM: + +.. code-block:: html + + <p data-l10n-id="update-application-info" data-l10n-args='{"version": "60.0"}'> + <a data-l10n-name="privacy-url" href="http://www.mozilla.org/privacy" /> + </p> + +.. code-block:: fluent + + -brand-short-name = Firefox + update-application-info = + You are using { -brand-short-name } Version: { $version }. + Please read the <a data-l10n-name="privacy-url">privacy policy</a>. + + +Fluent will overlay the translation onto the source fragment preserving attributes like +:code:`class` and :code:`href` from the source and adding translations for the elements +inside. The resulting localized content will look like this: + +.. code-block:: + + <p data-l10n-id="update-application-info" data-l10n-args='{"version": "60.0"}'"> + You are using Firefox Version: 60.0. + Please read the <a href="http://www.mozilla.org/privacy">privacy policy</a>. + </p> + + +This operation is sanitized, and Fluent takes care of selecting which elements and +attributes can be safely provided by the localization. +The list of allowed elements and attributes is `maintained by the W3C`__, and if +the developer needs to allow for localization of additional attributes, they can +allow them using :code:`data-l10n-attrs` list: + +.. code-block:: html + + <label data-l10n-id="search-input" data-l10n-attrs="style" /> + +The above example adds an attribute :code:`style` to be allowed on this +particular :code:`label` element. + + +External Arguments +------------------ + +Notice in the previous example the attribute :code:`data-l10n-args`, which is +a JSON object storing variables exposed by the developer to the localizer. + +This is the main channel for the developer to provide additional variables +to be used in the localization. + +Arguments are rarely needed for situations where it’s currently possible to use +DTD, since such variables would need to be computed from the code at runtime. +It's worth noting that, when the :code:`l10n-args` are set in +the runtime code, they are in fact encoded as JSON and stored together with +:code:`l10n-id` as an attribute of the element. + +__ https://www.w3.org/TR/2011/WD-html5-20110525/text-level-semantics.html + + +Runtime Localization +==================== + +In almost every case the JS runtime code will operate on a particular document, either +XUL, XHTML or HTML. + +If the document has its markup already localized, then Fluent exposes a new +attribute on the :js:`document` element - :js:`document.l10n`. + +This property is an object of type :js:`DOMLocalization` which maintains the main +localization context for this document and exposes it to runtime code as well. + +With a focus on `declarative localization`__, the primary method of localization is +to alter the localization attributes in the DOM. Fluent provides a method to facilitate this: + +.. code-block:: javascript + + document.l10n.setAttributes(element, "new-panel-header"); + +This will set the :code:`data-l10n-id` on the element and translate it before the next +animation frame. + +This API can be used to set both the ID and the arguments at the same time. + +.. code-block:: javascript + + document.l10n.setAttributes(element, "containers-disable-alert-ok-button", { + tabCount: 5 + }); + +If only the arguments need to be updated, then it's possible to use the :code:`setArgs` +method. + +.. code-block:: javascript + + document.l10n.setArgs(element, { + tabCount: 5 + }); + +On debug builds if the Fluent arguments are not provided, then Firefox will crash. This +is done so that these errors are caught in CI. On rare occasions it may be necessary +to work around this crash by providing a blank string as an argument value. + +__ https://github.com/projectfluent/fluent/wiki/Good-Practices-for-Developers + + +Non-Markup Localization +----------------------- + +In rare cases, when the runtime code needs to retrieve the translation and not +apply it onto the DOM, Fluent provides an API to retrieve it: + +.. code-block:: javascript + + let [ msg ] = await document.l10n.formatValues([ + {id: "remove-containers-description"} + ]); + + alert(msg); + +This model is heavily discouraged and should be used only in cases where the +DOM annotation is not possible. + +.. note:: + + This API is available as asynchronous. In case of Firefox, + the only non-DOM localizable calls are used where the output goes to + a third-party like Bluetooth, Notifications etc. + All those cases should already be asynchronous. If you can't avoid synchronous + access, you can use ``mozILocalization.formatMessagesSync`` with synchronous IO. + + +Internationalization +==================== + +The majority of internationalization issues are implicitly handled by Fluent without +any additional requirement. Full Unicode support, `bidirectionality`__, and +correct number formatting work without any action required from either +developer or localizer. + +__ https://github.com/projectfluent/fluent/wiki/BiDi-in-Fluent + +.. code-block:: javascript + + document.l10n.setAttributes(element, "welcome-message", { + userName: "اليسع", + count: 5 + }); + +A message like this localized to American English will correctly wrap the user +name in directionality marks, allowing the layout engine to determine how to +display the bidirectional text. + +On the other hand, the same message localized to Arabic will use the Eastern Arabic +numeral for number "5". + + +Plural Rules +------------ + +The most common localization feature is the ability to provide different variants +of the same string depending on plural categories. Fluent ties into the Unicode CLDR +standard called `Plural Rules`_. + +In order to allow localizers to use it, all the developer has to do is to pass +an external argument number: + +.. code-block:: javascript + + document.l10n.setAttributes(element, "unread-warning", { unreadCount: 5 }); + +Localizers can use the argument to build a multi variant message if their +language requires that: + +.. code-block:: fluent + + unread-warning = + { $unreadCount -> + [one] You have { $unreadCount } unread message + *[other] You have { $unreadCount } unread messages + } + +If the variant selection is performed based on a number, Fluent matches that +number against literal numbers as well as its `plural category`__. + +If the given translation doesn't need pluralization for the string (for example +Japanese often will not), the localizer can replace it with: + +.. code-block:: fluent + + unread-warning = You have { $unreadCount } unread messages + +and the message will preserve the social contract. + +One additional feature is that the localizer can further improve the message by +specifying variants for particular values: + +.. code-block:: fluent + + unread-warning = + { $unreadCount -> + [0] You have no unread messages + [1] You have one unread message + *[other] You have { $unreadCount } unread messages + } + +The advantage here is that per-locale choices don't leak onto the source code +and the developer is not affected. + + +.. note:: + + There is an important distinction between a variant keyed on plural category + `one` and digit `1`. Although in English the two are synonymous, in other + languages category `one` may be used for other numbers. + For example in `Bosnian`__, category `one` is used for numbers like `1`, `21`, `31` + and so on, and also for fractional numbers like `0.1`. + +__ https://unicode.org/cldr/charts/latest/supplemental/language_plural_rules.html +__ https://unicode.org/cldr/charts/latest/supplemental/language_plural_rules.html#bs + +Partially-formatted variables +----------------------------- + +When it comes to formatting data, Fluent allows the developer to provide +a set of parameters for the formatter, and the localizer can fine tune some of them. +This technique is called `partially-formatted variables`__. + +For example, when formatting a date, the developer can just pass a JS :js:`Date` object, +but its default formatting will be pretty expressive. In most cases, the developer +may want to use some of the :js:`Intl.DateTimeFormat` options to select the default +representation of the date in string: + +.. code-block:: javascript + + document.l10n.formatValue("welcome-message", { + startDate: FluentDateTime(new Date(), { + year: "numeric", + month: "long", + day: "numeric" + }) + }); + +.. code-block:: fluent + + welcome-message = Your session will start date: { $startDate } + +In most cases, that will be enough and the date would get formatted in the current +Firefox as `February 28, 2018`. + +But if in some other locale the string would get too long, the localizer can fine +tune the options as well: + +.. code-block:: fluent + + welcome-message = Początek Twojej sesji: { DATETIME($startDate, month: "short") } + +This will adjust the length of the month token in the message to short and get formatted +in Polish as `28 lut 2018`. + +At the moment Fluent supports two formatters that match JS Intl API counterparts: + + * **NUMBER**: `Intl.NumberFormat`__ + * **DATETIME**: `Intl.DateTimeFormat`__ + +With time more formatters will be added. Also, this feature is not exposed +to ``setAttributes`` at this point, as that serializes to JSON. + +__ https://projectfluent.org/fluent/guide/functions.html#partially-formatted-variables +__ https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/NumberFormat +__ https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/DateTimeFormat + +Registering New L10n Files +========================== + +Fluent uses a wildcard statement, packaging all localization resources into +their component's `/localization/` directory. + +That means that, if a new file is added to a component of Firefox already +covered by Fluent like `browser`, it's enough to add the new file to the +repository in a path like `browser/locales/en-US/browser/component/file.ftl`, and +the toolchain will package it into `browser/localization/browser/component/file.ftl`. + +At runtime Firefox uses a special registry for all localization data. It will +register the browser's `/localization/` directory and make all files inside it +available to be referenced. + +To make the document localized using Fluent, all the developer has to do is add +localizable resources for Fluent API to use: + +.. code-block:: html + + <link rel="localization" href="branding/brand.ftl"/> + <link rel="localization" href="browser/preferences/preferences.ftl"/> + +The URI provided to the :html:`<link/>` element are relative paths within the localization +system. + + +Custom Localizations +==================== + +The above method creates a single localization context per document. +In almost all scenarios that's sufficient. + +In rare edge cases where the developer needs to fetch additional resources, or +the same resources in another language, it is possible to create additional +Localization object manually using the `Localization` class: + +.. code-block:: javascript + + const myL10n = new Localization([ + "branding/brand.ftl", + "browser/preferences/preferences.ftl" + ]); + + + let [isDefaultMsg, isNotDefaultMsg] = + await myL10n.formatValues({id: "is-default"}, {id: "is-not-default"}); + + +.. admonition:: Example + + An example of a use case is the Preferences UI in Firefox, which uses the + main context to localize the UI but also to build a search index. + + It is common to build such search index both in a current language and additionally + in English, since a lot of documentation and online help exist only in English. + + A developer may create manually a new context with the same resources as the main one, + but hardcode it to `en-US` and then build the search index using both contexts. + + +By default, all `Localization` contexts are asynchronous. It is possible to create a synchronous +one by passing an `sync = false` argument to the constructor, or calling the `SetIsSync(bool)` method +on the class. + + +.. code-block:: javascript + + const myL10n = new Localization([ + "branding/brand.ftl", + "browser/preferences/preferences.ftl" + ], false); + + + let [isDefaultMsg, isNotDefaultMsg] = + myL10n.formatValuesSync({id: "is-default"}, {id: "is-not-default"}); + + +Synchronous contexts should be always avoided as they require synchronous I/O. If you think your use case +requires a synchronous localization context, please consult Gecko, Performance and L10n Drivers teams. + + +Designing Localizable APIs +========================== + +When designing localizable APIs, the most important rule is to resolve localization as +late as possible. That means that instead of resolving strings somewhere deep in the +codebase and then passing them on, or even caching, it is highly recommended to pass +around :code:`l10n-id` or :code:`[l10n-id, l10n-args]` pairs until the top-most code +resolves them or applies them onto the DOM element. + + +Testing +======= + +When writing tests that involve both I18n and L10n, the general rule is that +result strings are opaque. That means that the developer should not assume any particular +value and should never test against it. + +In case of raw i18n the :js:`resolvedOptions` method on all :js:`Intl.*` formatters +makes it relatively easy. In case of localization, the recommended way is to test that +the code sets the right :code:`l10n-id`/:code:`l10n-args` attributes like this: + +.. code-block:: javascript + + testedFunction(); + + const l10nAttrs = document.l10n.getAttributes(element); + + deepEquals(l10nAttrs, { + id: "my-expected-id", + args: { + unreadCount: 5 + } + }); + +If the code really has to test for particular values in the localized UI, it is +always better to scan for a variable: + +.. code-block:: javascript + + testedFunction(); + + equals(element.textContent.contains("John")); + +.. important:: + + Testing against whole values is brittle and will break when we insert Unicode + bidirectionality marks into the result string or adapt the output in other ways. + + +Manually Testing UI with Pseudolocalization +=========================================== + +When working with a Fluent-backed UI, the developer gets a new tool to test their UI +against several classes of problems. + +Pseudolocalization is a mechanism which transforms messages on the fly, using +specific logic to help emulate how the UI will look once it gets localized. + +The three classes of potential problems that this can help with are: + + - Hardcoded strings. + + Turning on pseudolocalization should expose any strings that were left + hardcoded in the source, since they won't get transformed. + + + - UI space not adapting to longer text. + + Many languages use longer strings than English. For example, German strings + may be 30% longer (or more). Turning on pseudolocalization is a quick way to + test how the layout handles such locales. Strings that don't fit the space + available are truncated and pseudolocalization can also help with detecting them. + + + - Bidi adaptation. + + For many developers, testing the UI in right-to-left mode is hard. + Pseudolocalization shows how a right-to-left locale will look like. + +To turn on pseudolocalization, open the :doc:`Browser Toolbox <../../devtools-user/browser_toolbox/index>`, +click the three dot menu in the top right corner, and choose one of the following: + + - **Enable “accented” locale** - [Ȧȧƈƈḗḗƞŧḗḗḓ Ḗḗƞɠŀīīşħ] + + This strategy replaces all Latin characters with their accented equivalents, + and duplicates some vowels to create roughly 30% longer strings. Strings are + wrapped in markers (square brackets), which help with detecting truncation. + + This option sets the :js:`intl.l10n.pseudo` pref to :js:`accented`. + + + - **Enable bidi locale** - ɥsıʅƃuƎ ıpıԐ + + This strategy replaces all Latin characters with their 180 degree rotated versions + and enforces right to left text flow using Unicode UAX#9 `Explicit Directional Embeddings`__. + In this mode, the UI directionality will also be set to right-to-left. + + This option sets the :js:`intl.l10n.pseudo` pref to :js:`bidi`. + +__ https://www.unicode.org/reports/tr9/#Explicit_Directional_Embeddings + +Inner Structure of Fluent +========================= + +The inner structure of Fluent in Gecko is out of scope of this tutorial, but +since the class and file names may show up during debugging or profiling, +below is a list of major components, each with a corresponding file in `/intl/l10n` +modules in Gecko. + +For more hands-on experience with some of the concepts below, try +following the `Fluent DOMLocalization Tutorial`__, which provides some +background on how Fluent works and walks you through creating a basic +web project from scratch that uses Fluent for localization. + +__ https://projectfluent.org/dom-l10n-documentation/overview.html + +FluentBundle +-------------- + +FluentBundle is the lowest level API. It's fully synchronous, contains a parser for the +FTL file format and a resolver for the logic. It is not meant to be used by +consumers directly. + +In the future we intend to offer this layer for standardization and it may become +part of the :js:`mozIntl.*` or even :js:`Intl.*` API sets. + +That part of the codebase is also the first that we'll be looking to port to Rust. + + +Localization +------------ + +Localization is a higher level API which uses :js:`FluentBundle` internally but +provides a full layer of compound message formatting and robust error fall-backing. + +It is intended for use in runtime code and contains all fundamental localization +methods. + + +DOMLocalization +--------------- + +DOMLocalization extends :js:`Localization` with functionality to operate on HTML, XUL +and the DOM directly including DOM Overlays and Mutation Observers. + +DocumentL10n +------------ + +DocumentL10n implements the DocumentL10n WebIDL API and allows Document to +communicate with DOMLocalization. + +Events +^^^^^^ + +DOM translation is asynchronous (e.g., setting a `data-l10n-id` attribute won't +immediately reflect the localized content in the DOM). + +We expose a :js:`Document.hasPendingL10nMutations` member that reflects whether +there are any async operations pending. When they are finished, the +`L10nMutationsFinished` event is fired on the document, so that chrome code can +be certain all the async operations are done. + +L10nRegistry +------------ + +L10nRegistry is our resource management service. It +maintains the state of resources packaged into the build and language packs, +providing an asynchronous iterator of :js:`FluentBundle` objects for a given locale set +and resources that the :js:`Localization` class uses. + + +.. _Fluent: https://projectfluent.org/ +.. _DTD: https://developer.mozilla.org/en-US/docs/Mozilla/Tech/XUL/Tutorial/Localization +.. _StringBundle: https://developer.mozilla.org/en-US/docs/Mozilla/Tech/XUL/Tutorial/Property_Files +.. _Firefox Preferences: https://bugzilla.mozilla.org/show_bug.cgi?id=1415730 +.. _Unprivileged Contexts: https://bugzilla.mozilla.org/show_bug.cgi?id=1407418 +.. _System Add-ons: https://bugzilla.mozilla.org/show_bug.cgi?id=1425104 +.. _CLDR: http://cldr.unicode.org/ +.. _ICU: http://site.icu-project.org/ +.. _Unicode: https://www.unicode.org/ +.. _Fluent Syntax Guide: https://projectfluent.org/fluent/guide/ +.. _Pontoon: https://pontoon.mozilla.org/ +.. _Plural Rules: http://cldr.unicode.org/index/cldr-spec/plural-rules diff --git a/intl/l10n/docs/glossary.rst b/intl/l10n/docs/glossary.rst new file mode 100644 index 0000000000..f8df34e819 --- /dev/null +++ b/intl/l10n/docs/glossary.rst @@ -0,0 +1,22 @@ +L10N Glossary +============= + +.. glossary:: + :sorted: + + Localization + The process of creating content in a native language, including + translation, but also customizations like Search. + + Localizability + Enabling a piece of software to be localized. This is mostly + externalizing English strings, and writing build support to + pick up localized search engines etc. + + L10n + *Numeronym* for Localization, *L*, 10 chars, *n* + + l10n-merge + nick-name for the process of merging ``en-US`` and a particular + localization into one joint artifact without any missing strings, and + without technical errors, as far as possible. diff --git a/intl/l10n/docs/index.rst b/intl/l10n/docs/index.rst new file mode 100644 index 0000000000..cbd9c3e796 --- /dev/null +++ b/intl/l10n/docs/index.rst @@ -0,0 +1,26 @@ +============ +Localization +============ + +Localization – sometimes written as l10n, where 10 is the number of letters between `l` and `n` – +is an aspect of internationalization focused on adapting software to +different cultural and regional needs. + +The boundary between internationalization and localization is fuzzy. At Mozilla +we refer to localization when we talk about adapting the user interface +and messages, while internationalization handles operations on raw data. + +.. note:: + + Localization is a broader term than translation because it involves extensive research + into the target culture, and in result touches not only text and UI translation but also + cultural adaptation of icons, communication styles, colors, and UX. + +.. toctree:: + :maxdepth: 2 + + overview + fluent/index + migrations/index + crosschannel/index + glossary diff --git a/intl/l10n/docs/migrations/fluent.rst b/intl/l10n/docs/migrations/fluent.rst new file mode 100644 index 0000000000..bc14293ed7 --- /dev/null +++ b/intl/l10n/docs/migrations/fluent.rst @@ -0,0 +1,153 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +.. role:: python(code) + :language: python + + +=========================== +Fluent to Fluent Migrations +=========================== + +When migrating existing Fluent messages, +it's possible to copy a source directly with :python:`COPY_PATTERN`, +or to apply string replacements and other changes +by extending the :python:`TransformPattern` visitor class. + +These transforms work with individual Fluent patterns, +i.e. the body of a Fluent message or one of its attributes. + +Copying Fluent Patterns +----------------------- + +Consider for example a patch modifying an existing message to move the original +value to a :js:`alt` attribute. + +Original message: + + +.. code-block:: fluent + + about-logins-icon = Warning icon + .title = Breached website + + +New message: + + +.. code-block:: fluent + + about-logins-breach-icon = + .alt = Warning icon + .title = Breached website + + +This type of changes requires a new message identifier, which in turn causes +existing translations to be lost. It’s possible to migrate the existing +translated content with: + + +.. code-block:: python + + from fluent.migrate import COPY_PATTERN + + ctx.add_transforms( + "browser/browser/aboutLogins.ftl", + "browser/browser/aboutLogins.ftl", + transforms_from( + """ + about-logins-breach-icon = + .alt = {COPY_PATTERN(from_path, "about-logins-icon")} + .title = {COPY_PATTERN(from_path, "about-logins-icon.title")} + """,from_path="browser/browser/aboutLogins.ftl"), + ) + + +In this specific case, the destination and source files are the same. The dot +notation is used to access attributes: :js:`about-logins-icon.title` matches +the :js:`title` attribute of the message with identifier +:js:`about-logins-icon`, while :js:`about-logins-icon` alone matches the value +of the message. + + +.. warning:: + + The second argument of :python:`COPY_PATTERN` and :python:`TransformPattern` + identifies a pattern, so using the message identifier will not + migrate the message as a whole, with all its attributes, only its value. + +Transforming Fluent Patterns +---------------------------- + +To apply changes to Fluent messages, you may extend the +:python:`TransformPattern` class to create your transformation. +This is a powerful general-purpose tool, of which :python:`COPY_PATTERN` is the +simplest extension that applies no transformation to the source. + +Consider for example a patch copying an existing message to strip out its HTML +content to use as an ARIA value. + +Original message: + + +.. code-block:: fluent + + videocontrols-label = + { $position }<span data-l10n-name="duration"> / { $duration }</span> + + +New message: + + +.. code-block:: fluent + + videocontrols-scrubber = + .aria-valuetext = { $position } / { $duration } + + +A migration may be applied to create this new message with: + + +.. code-block:: python + + from fluent.migrate.transforms import TransformPattern + import fluent.syntax.ast as FTL + + class STRIP_SPAN(TransformPattern): + def visit_TextElement(self, node): + node.value = re.sub("</?span[^>]*>", "", node.value) + return node + + def migrate(ctx): + path = "toolkit/toolkit/global/videocontrols.ftl" + ctx.add_transforms( + path, + path, + [ + FTL.Message( + id=FTL.Identifier("videocontrols-scrubber"), + attributes=[ + FTL.Attribute( + id=FTL.Identifier("aria-valuetext"), + value=STRIP_SPAN(path, "videocontrols-label"), + ), + ], + ), + ], + ) + + +Note that a custom extension such as :python:`STRIP_SPAN` is not supported by +the :python:`transforms_from` utility, so the list of transforms needs to be +defined explicitly. + +Internally, :python:`TransformPattern` extends the `fluent.syntax`__ +:python:`Transformer`, which defines the :python:`FTL` AST used here. +As a specific convenience, pattern element visitors such as +:python:`visit_TextElement` are allowed to return a :python:`FTL.Pattern` +to replace themselves with more than one node. + +__ https://projectfluent.org/python-fluent/fluent.syntax/stable/ diff --git a/intl/l10n/docs/migrations/index.rst b/intl/l10n/docs/migrations/index.rst new file mode 100644 index 0000000000..e9ed12aa22 --- /dev/null +++ b/intl/l10n/docs/migrations/index.rst @@ -0,0 +1,53 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +.. role:: python(code) + :language: python + +============================================= +Migrating Strings From Legacy or Fluent Files +============================================= + +Firefox is a project localized in over 100 languages. As the code for existing +features moves away from the old localization systems and starts using +`Fluent`_, we need to ensure that we don’t lose existing translations, which +would have the adverse effect of forcing contributors to localize hundreds of +strings from scratch. + +`Fluent Migration`_ is a Python library designed to solve this specific problem: +it allows to migrate translations from `.properties` and other legacy file formats, +not only moving strings and transforming them as needed to adapt to the `FTL` +syntax, but also replicating "blame" for each string in VCS. + +The library also includes basic support for migrating existing Fluent messages +without interpolations (e.g. variable replacements). The typical use cases +would be messages moving as-is to a different file, or changes to the +morphology of existing messages (e.g move content from an attribute to the +value of the message). + +.. toctree:: + :maxdepth: 2 + + overview + legacy + fluent + testing + localizations + +How to Get Help +=============== + +Writing migration recipes can be challenging for non trivial cases, and it can +require extensive l10n knowledge to avoid localizability issues. + +Don’t hesitate to reach out to the l10n-drivers for feedback, help to test or +write the migration recipes: + + - Francesco Lodolo (:flod) + - Eemeli Aro (:eemeli) + +.. _Fluent: http://projectfluent.org/ +.. _Fluent Migration: https://hg.mozilla.org/l10n/fluent-migration/ diff --git a/intl/l10n/docs/migrations/legacy.rst b/intl/l10n/docs/migrations/legacy.rst new file mode 100644 index 0000000000..b1287ba4d4 --- /dev/null +++ b/intl/l10n/docs/migrations/legacy.rst @@ -0,0 +1,642 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +.. role:: python(code) + :language: python + +======================== +Migrating Legacy Formats +======================== + +Migrating from legacy formats (.dtd, .properties) is different from migrating +Fluent to Fluent. When migrating legacy code paths, you'll need to adjust the +Fluent strings for the quirks Mozilla uses in the legacy code paths. You'll +find a number of specialized functionalities here. + +Legacy Migration Tools +---------------------- + +To assist with legacy format migrations, some scripting tools are provided: + + - `XUL+DTD to Fluent`_ + - `.properties to Fluent`_ + +When creating a migration, one or both of these tools may provide a good +starting point for manual work by automating at least a part of the migration, +including recipe generation and refactoring the calling code. + +.. _XUL+DTD to Fluent: https://github.com/zbraniecki/convert_xul_to_fluent +.. _.properties to Fluent: https://github.com/mozilla/properties-to-ftl + +Basic Migration +--------------- + +Let’s consider a basic example: one string needs to be migrated, without +any further change, from a DTD file to Fluent. + +The legacy string is stored in :bash:`toolkit/locales/en-US/chrome/global/findbar.dtd`: + + +.. code-block:: dtd + + <!ENTITY next.tooltip "Find the next occurrence of the phrase"> + + +The new Fluent string is stored in :bash:`toolkit/locales/en-US/toolkit/main-window/findbar.ftl`: + + +.. code-block:: properties + + findbar-next = + .tooltiptext = Find the next occurrence of the phrase + + +This is how the migration recipe looks: + + +.. code-block:: python + + # Any copyright is dedicated to the Public Domain. + # http://creativecommons.org/publicdomain/zero/1.0/ + + from __future__ import absolute_import + import fluent.syntax.ast as FTL + from fluent.migrate.helpers import transforms_from + + def migrate(ctx): + """Bug 1411707 - Migrate the findbar XBL binding to a Custom Element, part {index}.""" + + ctx.add_transforms( + "toolkit/toolkit/main-window/findbar.ftl", + "toolkit/toolkit/main-window/findbar.ftl", + transforms_from( + """ + findbar-next = + .tooltiptext = { COPY(from_path, "next.tooltip") } + """, from_path="toolkit/chrome/global/findbar.dtd")) + + +The first important thing to notice is that the migration recipe needs file +paths relative to a localization repository, losing :bash:`locales/en-US/`: + + - :bash:`toolkit/locales/en-US/chrome/global/findbar.dtd` becomes + :bash:`toolkit/chrome/global/findbar.dtd`. + - :bash:`toolkit/locales/en-US/toolkit/main-window/findbar.ftl` becomes + :bash:`toolkit/toolkit/main-window/findbar.ftl`. + +The :python:`context.add_transforms` function takes 3 arguments: + + - Path to the target l10n file. + - Path to the reference (en-US) file. + - An array of Transforms. Transforms are AST nodes which describe how legacy + translations should be migrated. + +.. note:: + + For migrations of Firefox localizations, the target and reference path + are the same. This isn't true for all projects that use Fluent, so both + arguments are required. + +In this case there is only one Transform that migrates the string with ID +:js:`next.tooltip` from :bash:`toolkit/chrome/global/findbar.dtd`, and injects +it in the FTL fragment. The :python:`COPY` Transform allows to copy the string +from an existing file as is, while :python:`from_path` is used to avoid +repeating the same path multiple times, making the recipe more readable. Without +:python:`from_path`, this could be written as: + + +.. code-block:: python + + ctx.add_transforms( + "toolkit/toolkit/main-window/findbar.ftl", + "toolkit/toolkit/main-window/findbar.ftl", + transforms_from( + """ + findbar-next = + .tooltiptext = { COPY("toolkit/chrome/global/findbar.dtd", "next.tooltip") } + """)) + + +This method of writing migration recipes allows to take the original FTL +strings, and simply replace the value of each message with a :python:`COPY` +Transform. :python:`transforms_from` takes care of converting the FTL syntax +into an array of Transforms describing how the legacy translations should be +migrated. This manner of defining migrations is only suitable to simple strings +where a copy operation is sufficient. For more complex use-cases which require +some additional logic in Python, it’s necessary to resort to the raw AST. + + +The example above is equivalent to the following syntax, which exposes +the underlying AST structure: + + +.. code-block:: python + + ctx.add_transforms( + "toolkit/toolkit/main-window/findbar.ftl", + "toolkit/toolkit/main-window/findbar.ftl", + [ + FTL.Message( + id=FTL.Identifier("findbar-next"), + attributes=[ + FTL.Attribute( + id=FTL.Identifier("tooltiptext"), + value=COPY( + "toolkit/chrome/global/findbar.dtd", + "next.tooltip" + ) + ) + ] + ) + ] + ) + +This creates a :python:`Message`, taking the value from the legacy string +:js:`findbar-next`. A message can have an array of attributes, each with an ID +and a value: in this case there is only one attribute, with ID :js:`tooltiptext` +and :js:`value` copied from the legacy string. + +Notice how both the ID of the message and the ID of the attribute are +defined as an :python:`FTL.Identifier`, not simply as a string. + + +.. tip:: + + It’s possible to concatenate arrays of Transforms defined manually, like in + the last example, with those coming from :python:`transforms_from`, by using + the :python:`+` operator. Alternatively, it’s possible to use multiple + :python:`add_transforms`. + + The order of Transforms provided in the recipe is not relevant, the reference + file is used for ordering messages. + + +Replacing Content in Legacy Strings +----------------------------------- + +While :python:`COPY` allows to copy a legacy string as is, :python:`REPLACE` +(from `fluent.migrate`) allows to replace content while performing the +migration. This is necessary, for example, when migrating strings that include +placeholders or entities that need to be replaced to adapt to Fluent syntax. + +Consider for example the following string: + + +.. code-block:: DTD + + <!ENTITY aboutSupport.featuresTitle "&brandShortName; Features"> + + +Which needs to be migrated to: + + +.. code-block:: fluent + + features-title = { -brand-short-name } Features + + +The entity :js:`&brandShortName;` needs to be replaced with a term reference: + + +.. code-block:: python + + FTL.Message( + id=FTL.Identifier("features-title"), + value=REPLACE( + "toolkit/chrome/global/aboutSupport.dtd", + "aboutSupport.featuresTitle", + { + "&brandShortName;": TERM_REFERENCE("brand-short-name"), + }, + ) + ), + + +This creates an :python:`FTL.Message`, taking the value from the legacy string +:js:`aboutSupport.featuresTitle`, but replacing the specified text with a +Fluent term reference. + +.. note:: + :python:`REPLACE` replaces all occurrences of the specified text. + + +It’s also possible to replace content with a specific text: in that case, it +needs to be defined as a :python:`TextElement`. For example, to replace +:js:`example.com` with HTML markup: + + +.. code-block:: python + + value=REPLACE( + "browser/chrome/browser/preferences/preferences.properties", + "searchResults.sorryMessageWin", + { + "example.com": FTL.TextElement('<span data-l10n-name="example"></span>') + } + ) + + +The situation is more complex when a migration recipe needs to replace +:js:`printf` arguments like :js:`%S`. In fact, the format used for localized +and source strings doesn’t need to match, and the two following strings using +unordered and ordered argument are perfectly equivalent: + + +.. code-block:: properties + + btn-quit = Quit %S + btn-quit = Quit %1$S + + +In this scenario, replacing :js:`%S` would work on the first version, but not +on the second, and there’s no guarantee that the localized string uses the +same format as the source string. + +Consider also the following string that uses :js:`%S` for two different +variables, implicitly relying on the order in which the arguments appear: + + +.. code-block:: properties + + updateFullName = %S (%S) + + +And the target Fluent string: + + +.. code-block:: fluent + + update-full-name = { $name } ({ $buildID }) + + +As indicated, :python:`REPLACE` would replace all occurrences of :js:`%S`, so +only one variable could be set. The string needs to be normalized and treated +like: + + +.. code-block:: properties + + updateFullName = %1$S (%2$S) + + +This can be obtained by calling :python:`REPLACE` with +:python:`normalize_printf=True`: + + +.. code-block:: python + + FTL.Message( + id=FTL.Identifier("update-full-name"), + value=REPLACE( + "toolkit/chrome/mozapps/update/updates.properties", + "updateFullName", + { + "%1$S": VARIABLE_REFERENCE("name"), + "%2$S": VARIABLE_REFERENCE("buildID"), + }, + normalize_printf=True + ) + ) + + +.. attention:: + + To avoid any issues :python:`normalize_printf=True` should always be used when + replacing :js:`printf` arguments. This is the default behaviour when working + with .properties files. + +.. note:: + + :python:`VARIABLE_REFERENCE`, :python:`MESSAGE_REFERENCE`, and + :python:`TERM_REFERENCE` are helper Transforms which can be used to save + keystrokes in common cases where using the raw AST is too verbose. + + :python:`VARIABLE_REFERENCE` is used to create a reference to a variable, e.g. + :js:`{ $variable }`. + + :python:`MESSAGE_REFERENCE` is used to create a reference to another message, + e.g. :js:`{ another-string }`. + + :python:`TERM_REFERENCE` is used to create a reference to a `term`__, + e.g. :js:`{ -brand-short-name }`. + + Both Transforms need to be imported at the beginning of the recipe, e.g. + :python:`from fluent.migrate.helpers import VARIABLE_REFERENCE` + + __ https://projectfluent.org/fluent/guide/terms.html + + +Trimming Unnecessary Whitespaces in Translations +------------------------------------------------ + +.. note:: + + This section was updated in May 2020 to reflect the change to the default + behavior: legacy translations are now trimmed, unless the :python:`trim` + parameter is set explicitly. + +It’s not uncommon to have strings with unnecessary leading or trailing spaces +in legacy translations. These are not meaningful, don’t have practical results +on the way the string is displayed in products, and are added mostly for +formatting reasons. For example, consider this DTD string: + + +.. code-block:: DTD + + <!ENTITY aboutAbout.note "This is a list of “about” pages for your convenience.<br/> + Some of them might be confusing. Some are for diagnostic purposes only.<br/> + And some are omitted because they require query strings."> + + +By default, the :python:`COPY`, :python:`REPLACE`, and :python:`PLURALS` +transforms will strip the leading and trailing whitespace from each line of the +translation, as well as the empty leading and trailing lines. The above string +will be migrated as the following Fluent message, despite copious indentation +on the second and the third line in the original: + + +.. code-block:: fluent + + about-about-note = + This is a list of “about” pages for your convenience.<br/> + Some of them might be confusing. Some are for diagnostic purposes only.<br/> + And some are omitted because they require query strings. + + +To disable the default trimming behavior, set :python:`trim:"False"` or +:python:`trim=False`, depending on the context: + + +.. code-block:: python + + transforms_from( + """ + about-about-note = { COPY("toolkit/chrome/global/aboutAbout.dtd", "aboutAbout.note", trim:"False") } + """) + + FTL.Message( + id=FTL.Identifier("discover-description"), + value=REPLACE( + "toolkit/chrome/mozapps/extensions/extensions.dtd", + "discover.description2", + { + "&brandShortName;": TERM_REFERENCE("-brand-short-name") + }, + trim=False + ) + ), + + +Concatenating Strings +--------------------- + +It's best practice to only expose complete phrases to localization, and to avoid +stitching localized strings together in code. With `DTD` and `properties`, +there were few options. So when migrating to Fluent, you'll find +it quite common to concatenate multiple strings coming from `DTD` and +`properties`, for example to create sentences with HTML markup. It’s possible to +concatenate strings and text elements in a migration recipe using the +:python:`CONCAT` Transform. + +Note that in case of simple migrations using :python:`transforms_from`, the +concatenation is carried out implicitly by using the Fluent syntax interleaved +with :python:`COPY()` transform calls to define the migration recipe. + +Consider the following example: + + +.. code-block:: properties + + # %S is replaced by a link, using searchResults.needHelpSupportLink as text + searchResults.needHelp = Need help? Visit %S + + # %S is replaced by "Firefox" + searchResults.needHelpSupportLink = %S Support + + +In Fluent: + + +.. code-block:: fluent + + search-results-need-help-support-link = Need help? Visit <a data-l10n-name="url">{ -brand-short-name } Support</a> + + +This is quite a complex migration: it requires to take 2 legacy strings, and +concatenate their values with HTML markup. Here’s how the Transform is defined: + + +.. code-block:: python + + FTL.Message( + id=FTL.Identifier("search-results-help-link"), + value=REPLACE( + "browser/chrome/browser/preferences/preferences.properties", + "searchResults.needHelp", + { + "%S": CONCAT( + FTL.TextElement('<a data-l10n-name="url">'), + REPLACE( + "browser/chrome/browser/preferences/preferences.properties", + "searchResults.needHelpSupportLink", + { + "%1$S": TERM_REFERENCE("brand-short-name"), + }, + normalize_printf=True + ), + FTL.TextElement("</a>") + ) + } + ) + ), + + +:js:`%S` in :js:`searchResults.needHelpSupportLink` is replaced by a reference +to the term :js:`-brand-short-name`, migrating from :js:`%S Support` to :js:`{ +-brand-short-name } Support`. The result of this operation is then inserted +between two text elements to create the anchor markup. The resulting text is +finally used to replace :js:`%S` in :js:`searchResults.needHelp`, and used as +value for the FTL message. + + +.. important:: + + When concatenating existing strings, avoid introducing changes to the original + text, for example adding spaces or punctuation. Each language has its own + rules, and this might result in poor migrated strings. In case of doubt, + always ask for feedback. + + +When more than 1 element is passed in to concatenate, :python:`CONCAT` +disables whitespace trimming described in the section above on all legacy +Transforms passed into it: :python:`COPY`, :python:`REPLACE`, and +:python:`PLURALS`, unless the :python:`trim` parameters has been set +explicitly on them. This helps ensure that spaces around segments are not +lost during the concatenation. + +When only a single element is passed into :python:`CONCAT`, however, the +trimming behavior is not altered, and follows the rules described in the +previous section. This is meant to make :python:`CONCAT(COPY())` equivalent +to a bare :python:`COPY()`. + + +Plural Strings +-------------- + +Migrating plural strings from `.properties` files usually involves two +Transforms from :python:`fluent.migrate.transforms`: the +:python:`REPLACE_IN_TEXT` Transform takes TextElements as input, making it +possible to pass it as the foreach function of the :python:`PLURALS` Transform. + +Consider the following legacy string: + + +.. code-block:: properties + + # LOCALIZATION NOTE (disableContainersOkButton): Semi-colon list of plural forms. + # See: http://developer.mozilla.org/en/docs/Localization_and_Plurals + # #1 is the number of container tabs + disableContainersOkButton = Close #1 Container Tab;Close #1 Container Tabs + + +In Fluent: + + +.. code-block:: fluent + + containers-disable-alert-ok-button = + { $tabCount -> + [one] Close { $tabCount } Container Tab + *[other] Close { $tabCount } Container Tabs + } + + +This is how the Transform for this string is defined: + + +.. code-block:: python + + FTL.Message( + id=FTL.Identifier("containers-disable-alert-ok-button"), + value=PLURALS( + "browser/chrome/browser/preferences/preferences.properties", + "disableContainersOkButton", + VARIABLE_REFERENCE("tabCount"), + lambda text: REPLACE_IN_TEXT( + text, + { + "#1": VARIABLE_REFERENCE("tabCount") + } + ) + ) + ) + + +The `PLURALS` Transform will take care of creating the correct number of plural +categories for each language. Notice how `#1` is replaced for each of these +variants with :js:`{ $tabCount }`, using :python:`REPLACE_IN_TEXT` and +:python:`VARIABLE_REFERENCE("tabCount")`. + +In this case it’s not possible to use :python:`REPLACE` because it takes a file +path and a message ID as arguments, whereas here the recipe needs to operate on +regular text. The replacement is performed on each plural form of the original +string, where plural forms are separated by a semicolon. + +Explicit Variants +----------------- + +Explicitly creating variants of a string is useful for platform-dependent +terminology, but also in cases where you want a one-vs-many split of a string. +It’s always possible to migrate strings by manually creating the underlying AST +structure. Consider the following complex Fluent string: + + +.. code-block:: fluent + + use-current-pages = + .label = + { $tabCount -> + [1] Use Current Page + *[other] Use Current Pages + } + .accesskey = C + + +The migration for this string is quite complex: the :js:`label` attribute is +created from 2 different legacy strings, and it’s not a proper plural form. +Notice how the first string is associated to the :js:`1` case, not the :js:`one` +category used in plural forms. For these reasons, it’s not possible to use +:python:`PLURALS`, the Transform needs to be crafted recreating the AST. + + +.. code-block:: python + + + FTL.Message( + id=FTL.Identifier("use-current-pages"), + attributes=[ + FTL.Attribute( + id=FTL.Identifier("label"), + value=FTL.Pattern( + elements=[ + FTL.Placeable( + expression=FTL.SelectExpression( + selector=VARIABLE_REFERENCE("tabCount"), + variants=[ + FTL.Variant( + key=FTL.NumberLiteral("1"), + default=False, + value=COPY( + "browser/chrome/browser/preferences/main.dtd", + "useCurrentPage.label", + ) + ), + FTL.Variant( + key=FTL.Identifier("other"), + default=True, + value=COPY( + "browser/chrome/browser/preferences/main.dtd", + "useMultiple.label", + ) + ) + ] + ) + ) + ] + ) + ), + FTL.Attribute( + id=FTL.Identifier("accesskey"), + value=COPY( + "browser/chrome/browser/preferences/main.dtd", + "useCurrentPage.accesskey", + ) + ), + ], + ), + + +This Transform uses several concepts already described in this document. Notable +is the :python:`SelectExpression` inside a :python:`Placeable`, with an array +of :python:`Variant` objects. Exactly one of those variants needs to have +``default=True``. + +This example can still use :py:func:`transforms_from()``, since existing strings +are copied without interpolation. + +.. code-block:: python + + transforms_from( + """ + use-current-pages = + .label = + { $tabCount -> + [1] { COPY(main_dtd, "useCurrentPage.label") } + *[other] { COPY(main_dtd, "useMultiple.label") } + } + .accesskey = { COPY(main_dtd, "useCurrentPage.accesskey") } + """, main_dtd="browser/chrome/browser/preferences/main.dtd" + ) diff --git a/intl/l10n/docs/migrations/localizations.rst b/intl/l10n/docs/migrations/localizations.rst new file mode 100644 index 0000000000..0861d6e52b --- /dev/null +++ b/intl/l10n/docs/migrations/localizations.rst @@ -0,0 +1,42 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +.. role:: python(code) + :language: python + +=========================================== +How Migrations Are Run on l10n Repositories +=========================================== + +Once a patch including new FTL strings and a migration recipe lands in +mozilla-central, l10n-drivers will perform a series of actions to migrate +strings in all 100+ localization repositories: + + - New Fluent strings land in `mozilla-central`, together with a migration + recipe. + - New strings are added to `gecko-strings-quarantine`_, a unified repository + including strings for all shipping versions of Firefox, and used as a buffer + before exposing strings to localizers. + - Migration recipes are run against all l10n repositories, migrating strings + from old to new files, and storing them in VCS. + - New en-US strings are pushed to the official `gecko-strings`_ repository + used by localization tools, and exposed to all localizers. + +Migration recipes could be run again within a release cycle, in order to migrate +translations for legacy strings added after the first run. They’re usually +removed from `mozilla-central` within 2 cycles, e.g. a migration recipe created +for Firefox 59 would be removed when Firefox 61 is available in Nightly. + + +.. tip:: + + A script to run migrations on all l10n repositories is available in `this + repository`__, automating part of the steps described for manual testing, and + it could be adapted to local testing. + + __ https://github.com/flodolo/fluent-migrations +.. _gecko-strings-quarantine: https://hg.mozilla.org/l10n/gecko-strings-quarantine/ +.. _gecko-strings: https://hg.mozilla.org/l10n/gecko-strings diff --git a/intl/l10n/docs/migrations/overview.rst b/intl/l10n/docs/migrations/overview.rst new file mode 100644 index 0000000000..dc9c128fb9 --- /dev/null +++ b/intl/l10n/docs/migrations/overview.rst @@ -0,0 +1,136 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +.. role:: python(code) + :language: python + +===================================== +Migration Recipes and Their Lifecycle +===================================== + +The actual migrations are performed running Python modules called **migration +recipes**, which contain directives on how to migrate strings, which files are +involved, transformations to apply, etc. These recipes are stored in +`mozilla-central`__. + +__ https://hg.mozilla.org/mozilla-central/file/default/python/l10n/fluent_migrations + +When part of Firefox’s UI is migrated to Fluent, a migration recipe should be +attached to the same patch that adds new strings to `.ftl` files. + +Migration recipes can quickly become obsolete, because the referenced strings +and files are removed from repositories as part of ongoing development. +For these reasons, l10n-drivers periodically clean up the `fluent_migrations` +folder in mozilla-central, keeping only recipes for 2 +shipping versions (Nightly and Beta). + + +.. hint:: + + As a developer you don’t need to bother about updating migration recipes + already in `mozilla-central`: if a new patch removes a string or file that is + used in a migration recipe, simply ignore it, since the entire recipe will be + removed within a couple of cycles. + + +How to Write Migration Recipes +============================== + +The migration recipe’s filename should start with a reference to the associated +bug number, and include a brief description of the bug, e.g. +:bash:`bug_1451992_preferences_applicationManager.py` is the migration recipe +used to migrate the Application Manager window in preferences. It’s also +possible to look at existing recipes in `mozilla-central`__ for inspiration. + +__ https://hg.mozilla.org/mozilla-central/file/default/python/l10n/fluent_migrations + + +General Recipe Structure +======================== + +A migration recipe is a Python module, implementing the :py:func:`migrate` +function, which takes a :py:class:`MigrationContext` as input. The API provided +by the context is + +.. code-block:: python + + class MigrationContext: + def add_transforms(self, target, reference, transforms): + """Define transforms for target using reference as template. + + `target` is a path of the destination FTL file relative to the + localization directory. `reference` is a path to the template FTL + file relative to the reference directory. + + Each transform is an extended FTL node with `Transform` nodes as some + values. + + For transforms that merely copy legacy messages or Fluent patterns, + using `fluent.migrate.helpers.transforms_from` is recommended. + """ + +The skeleton of a migration recipe just implements the :py:func:`migrate` +function calling into :py:func:`ctx.add_transforms`, and looks like + +.. code-block:: python + + # coding=utf8 + + # Any copyright is dedicated to the Public Domain. + # http://creativecommons.org/publicdomain/zero/1.0/ + + from __future__ import absolute_import + + + def migrate(ctx): + """Bug 1552333 - Migrate feature to Fluent, part {index}""" + target = 'browser/browser/feature.ftl' + reference = 'browser/browser/feature.ftl' + ctx.add_transforms( + target, + reference, + [], # Actual transforms go here. + ) + +One can call into :py:func:`ctx.add_transforms` multiple times. In particular, one +can create migrated content in multiple files as part of a single migration +recipe by calling :py:func:`ctx.add_transforms` with different target-reference +pairs. + +The *docstring* for this function will be used +as a commit message in VCS, that’s why it’s important to make sure the bug +reference is correct, and to keep the `part {index}` section: multiple strings +could have multiple authors, and would be migrated in distinct commits (part 1, +part 2, etc.). + +Transforms +========== + +The work of the migrations is done by the transforms that are passed as +last argument to :py:func:`ctx.add_transforms`. They're instances of either Fluent +:py:class:`fluent.syntax.ast.Message` or :py:class:`Term`, and their content +can depend on existing translation sources. The skeleton of a Message looks like + +.. code-block:: python + + FTL.Message( + id=FTL.Identifier( + name="msg", + ), + value=FTL.Pattern( + elements=[ + FTL.TextElement( + value="A string", + ), + ], + ), + ) + +When migrating existing legacy translations, you'll replace an +``FTL.TextElement`` with a ``COPY(legacy_path, "old_id")``, or one of its +variations we detail :doc:`next <legacy>`. When migrating existing Fluent +translations, an ``FTL.Pattern`` is replaced with a +``COPY_PATTERN(old_path, "old-id")``. diff --git a/intl/l10n/docs/migrations/testing.rst b/intl/l10n/docs/migrations/testing.rst new file mode 100644 index 0000000000..aa9b9747f6 --- /dev/null +++ b/intl/l10n/docs/migrations/testing.rst @@ -0,0 +1,58 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +.. role:: python(code) + :language: python + +============================= +How to Test Migration Recipes +============================= + +To test migration recipes, use the following mach command: + +.. code-block:: bash + + ./mach fluent-migration-test python/l10n/fluent_migrations/bug_1485002_newtab.py + +This will analyze your migration recipe to check that the :python:`migrate` +function exists, and interacts correctly with the migration context. Once that +passes, it clones :bash:`gecko-strings` into :bash:`$OBJDIR/python/l10n`, creates a +reference localization by adding your local Fluent strings to the ones in +:bash:`gecko-strings`. It then runs the migration recipe, both as dry run and +as actual migration. Finally it analyzes the commits, and checks if any +migrations were actually run and the bug number in the commit message matches +the migration name. + +It will also show the diff between the migrated files and the reference, ignoring +blank lines. + +You can inspect the generated repository further by looking at + +.. code-block:: bash + + ls $OBJDIR/python/l10n/bug_1485002_newtab/en-US + +Caveats +------- + +Be aware of hard-coded English context in migration. Consider for example: + + +.. code-block:: python + + ctx.add_transforms( + "browser/browser/preferences/siteDataSettings.ftl", + "browser/browser/preferences/siteDataSettings.ftl", + transforms_from( + """ + site-usage-persistent = { site-usage-pattern } (Persistent) + """) + ) + + +This Transform will pass a manual comparison, since the two files are identical, +but will result in :js:`(Persistent)` being hard-coded in English for all +languages. diff --git a/intl/l10n/docs/overview.rst b/intl/l10n/docs/overview.rst new file mode 100644 index 0000000000..1f1d90c105 --- /dev/null +++ b/intl/l10n/docs/overview.rst @@ -0,0 +1,199 @@ +.. role:: js(code) + :language: javascript + +============ +Localization +============ + +Localization at Mozilla +======================= + +At Mozilla localizations are managed by locale communities around the world, who +are responsible for maintaining high quality linguistic and cultural adaptation +of Mozilla software into over 100 locales. + +The exact process of localization management differs from project to project, but +in the case of Gecko applications, the localization is primarily done via a web localization +system called `Pontoon`_ and stored in HG repositories under +`hg.mozilla.org/l10n-central`_. + +Developers are expected to keep their code localizable using localization +and internationalization systems, and also serve as localizers into the `en-US` locale +which is used as the `source` locale. + +In between the developers and localizers, there's a sophisticated ecosystem of tools, +tests, automation, validators and other checks on one hand, and management, release, +community and quality processes facilitated by the `L10n Drivers Team`_, on the other. + +Content vs. UI +============== + +The two main categories in localization are content localization vs UI localization. + +The former is usually involved when dealing with large blocks of text such as +documentation, help articles, marketing material and legal documents. + +The latter is the primary type when handling user interfaces for applications such +as Firefox. + +This article will focus on UI localization. + +Lifecycle & Workflow +==================== + +1) New feature +-------------- + +The typical life cycle of a localizable UI starts with a UX/UI or new feature need which +should be accompanied by the UX mockups involving so called `copy` - the original +content to be used in the new piece of UI. + +2) UX mockup + copy review +-------------------------- + +The UX mockup with copy is the first step that should be reviewed by the L10n Drivers Team. +Their aim is to identify potential cultural and localization challenges that may arise +later and ensure that the UI is ready for localization on a linguistic, cultural, +and technical level. + +3) Patch l10n review +-------------------- + +Once that is completed, the next stage is for front-end engineers to create patches +which implement the new UI. Those patches should already contain the `copy` and +place the strings in the localization resources for the source locale (`en-US`). + +The developer uses the localization API by selecting a special identifier we call +`L10n ID` and optionally a list of variables that will be passed to the translation. + +We call this "a social contract" which binds the l10n-id/args combination to a particular +source translation to use in the UI. + +The localizer expects the developer to maintain the contract by ensuring that the +translation will be used in the given location, and will correspond to the +source translation. If that contract is to be changed, the developer will be expected +to update it. More on that in part `6) String Updates`. + +The next review comes from either L10n Drivers, or experienced front end engineers +familiar with the internationalization and localization systems, making sure that +the patches properly use the right APIs and the code is ready to be landed +into `mozilla-central`. + +.. _exposure-in-gecko-strings: + +4) Exposure in `gecko-strings` +------------------------------ + +Once the patch lands in `mozilla-central`, L10n Drivers will take a final look at +the localizability of the introduced strings. In case of issues, developers might +be asked to land a follow up, or the patch could be backed out with the help of sheriffs. + +Every few days, strings are exported into a repository called `gecko-strings-quarantine`, +a unified repository that includes strings for all shipping versions of Firefox +(nightly, beta, release). This repository is used as a buffer to avoid exposing potential +issues to over 100 locales. + +As a last step, strings are pushed into `gecko-strings`, another unified repository that +is exposed to localization tools, like Pontoon, and build automation. + +5) Localization +--------------- + +From that moment localizers will work on providing translations for the new feature +either while the new strings are only in Nightly or after they are merged to Beta. +The goal is to have as much of the UI ready in as many locales as early as possible, +but the process is continuous and we're capable of releasing Firefox with incomplete +translations falling back on a backup locale in case of a missing string. + +While Nightly products use the latest version of localization available in repositories, +the L10n Drivers team is responsible for reviewing and signing off versions of each +localization shipping in Beta and Release versions of Gecko products. + +6) String updates +----------------- + +Later in the software life cycle some strings might need to be changed or removed. +As a general rule, once the strings lands in `mozilla-central`, any further update +to existing strings will need to follow these guidelines, independently from how much +time has passed since previous changes. + +If it's just a string removal, all the engineer has to do is to remove it from the UI +and from the localization resource file in `mozilla-central`. + +If it's an update, we currently have two "levels" of change severity: + +1) If the change is minor, like fixing a spelling error or case, the developer should update +the `en-US` translation without changing the l10n-id. + +2) If the change is anyhow affecting the meaning or tone of the message, the developer +is requested to update the l10n string ID. + +The latter is considered a change in the social contract between the developer and +the localizer and an update to the ID is expected. + +In case of `Fluent`_, any changes to the structure of the message such as adding/removing +attributes also requires an update of the ID. + +The new ID will be recognized by the l10n tooling as untranslated, and the old one +as obsolete. This will give the localizers an opportunity to find and update the +translation, while the old string will be removed from the build process. + +There is a gray area between the two severity levels. In case of doubt, don’t hesitate +to request feedback of review from L10n Drivers to avoid issues once the strings land. + +Selecting L10n Identifier +========================= + +Choosing an identifier for a localization message is tricky. It may seem similar +to picking a variable name, but in reality, it's much closer to designing a public +API. + +An l10n identifier, once defined, is then getting associated to a translated +message in every one of 100+ locales and it becomes very costly to attempt to +migrate that string in all locales to a different identifier. + +Additionally, in Fluent an identifier is used as a last resort string to be displayed in +an error scenario when formatting the message fails, which makes selecting +**meaningful** identifiers particularly valuable. + +Lastly, l10n resources get mixed and matched into localization contexts where +it becomes important to avoid identifier collision from two strings coming +from two different files. + +For all those reasons, a longer identifier such as :js:`privacy-exceptions-button-ok` is +preferred over short identifiers like :js:`ok` or :js:`ok-button`. + +Localization Systems +==================== + +Gecko has two main localization systems: Fluent and StringBundle, a legacy system. + +Fluent +------ + +Fluent is a modern localization system designed by Mozilla to address the challenges +and limitations of older systems. + +It's well suited for modern web development cycle, provides a number of localization +features including good internationalization model and strong bidirectionality support. + + +To learn more about Fluent, follow the `Fluent for Firefox Developers`_ guide. + +StringBundle +------------ + +StringBundle is a runtime API used primarily for localization of C++ code. +The messages are stored in `.properties` files and loaded using the StringBundle API +and then retrieved from there via imperative calls. + +The system provides external arguments which can be placed into the string, and +supports basic plural categories via a proprietary API `PluralForm.sys.mjs`. + +Adding new StringBundle messages should only be done after serious consideration, +and in particular any new use of PluralForm messages should be avoided. + +.. _Pontoon: https://pontoon.mozilla.org/ +.. _hg.mozilla.org/l10n-central: https://hg.mozilla.org/l10n-central/ +.. _L10n Drivers Team: https://wiki.mozilla.org/L10n:Mozilla_Team +.. _Fluent For Firefox Developers: ./fluent/tutorial.html diff --git a/intl/l10n/moz.build b/intl/l10n/moz.build new file mode 100644 index 0000000000..8482824e92 --- /dev/null +++ b/intl/l10n/moz.build @@ -0,0 +1,61 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +EXPORTS.mozilla.intl += [ + "FileSource.h", + "FluentBindings.h", + "FluentBundle.h", + "FluentResource.h", + "L10nRegistry.h", + "Localization.h", + "LocalizationBindings.h", + "RegistryBindings.h", +] + +UNIFIED_SOURCES += [ + "FileSource.cpp", + "FluentBundle.cpp", + "FluentResource.cpp", + "L10nRegistry.cpp", + "Localization.cpp", +] + +TESTING_JS_MODULES += [ + "FluentSyntax.jsm", +] + +TEST_DIRS += [ + "rust/gtest", + "test/gtest", +] + +include("/ipc/chromium/chromium-config.mozbuild") + +USE_LIBS += ["intlcomponents"] + +if CONFIG["COMPILE_ENVIRONMENT"]: + CbindgenHeader("fluent_ffi_generated.h", inputs=["/intl/l10n/rust/fluent-ffi"]) + + CbindgenHeader( + "l10nregistry_ffi_generated.h", inputs=["/intl/l10n/rust/l10nregistry-ffi"] + ) + + CbindgenHeader( + "localization_ffi_generated.h", inputs=["/intl/l10n/rust/localization-ffi"] + ) + + EXPORTS.mozilla.intl += [ + "!fluent_ffi_generated.h", + "!l10nregistry_ffi_generated.h", + "!localization_ffi_generated.h", + ] + +XPCSHELL_TESTS_MANIFESTS += ["test/xpcshell.ini"] +MOCHITEST_CHROME_MANIFESTS += ["test/mochitest/chrome.ini"] + +SPHINX_TREES["/l10n"] = "docs" + +FINAL_LIBRARY = "xul" diff --git a/intl/l10n/rust/fluent-ffi/Cargo.toml b/intl/l10n/rust/fluent-ffi/Cargo.toml new file mode 100644 index 0000000000..dad9498efd --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "fluent-ffi" +version = "0.1.0" +authors = ["Zibi Braniecki <zibi@braniecki.net>"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +fluent = { version = "0.16.0", features = ["fluent-pseudo"] } +fluent-pseudo = "0.3.1" +intl-memoizer = "0.5.1" +unic-langid = "0.9" +nsstring = { path = "../../../../xpcom/rust/nsstring" } +cstr = "0.2" +xpcom = { path = "../../../../xpcom/rust/xpcom" } +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } diff --git a/intl/l10n/rust/fluent-ffi/cbindgen.toml b/intl/l10n/rust/fluent-ffi/cbindgen.toml new file mode 100644 index 0000000000..5384a81b0a --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/cbindgen.toml @@ -0,0 +1,24 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef mozilla_intl_l10n_FluentBindings_h +#error "Don't include this file directly, instead include FluentBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +namespaces = ["mozilla", "intl", "ffi"] + +[parse] +parse_deps = true +include = ["fluent", "fluent-bundle", "intl-memoizer"] + +[enum] +derive_helper_methods = true + +[export.rename] +"ThinVec" = "nsTArray" diff --git a/intl/l10n/rust/fluent-ffi/src/builtins.rs b/intl/l10n/rust/fluent-ffi/src/builtins.rs new file mode 100644 index 0000000000..c7ffe8c3ee --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/builtins.rs @@ -0,0 +1,389 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::ffi; +use fluent::types::{FluentNumberOptions, FluentType, FluentValue}; +use fluent::FluentArgs; +use intl_memoizer::IntlLangMemoizer; +use intl_memoizer::Memoizable; +use nsstring::nsCString; +use std::borrow::Cow; +use std::ptr::NonNull; +use unic_langid::LanguageIdentifier; + +pub struct NumberFormat { + raw: Option<NonNull<ffi::RawNumberFormatter>>, +} + +/** + * According to http://userguide.icu-project.org/design, as long as we constrain + * ourselves to const APIs ICU is const-correct. + */ +unsafe impl Send for NumberFormat {} +unsafe impl Sync for NumberFormat {} + +impl NumberFormat { + pub fn new(locale: LanguageIdentifier, options: &FluentNumberOptions) -> Self { + let loc: String = locale.to_string(); + Self { + raw: unsafe { + NonNull::new(ffi::FluentBuiltInNumberFormatterCreate( + &loc.into(), + &options.into(), + )) + }, + } + } + + pub fn format(&self, input: f64) -> String { + if let Some(raw) = self.raw { + unsafe { + let mut byte_count = 0; + let mut capacity = 0; + let buffer = ffi::FluentBuiltInNumberFormatterFormat( + raw.as_ptr(), + input, + &mut byte_count, + &mut capacity, + ); + if buffer.is_null() { + return String::new(); + } + String::from_raw_parts(buffer, byte_count, capacity) + } + } else { + String::new() + } + } +} + +impl Drop for NumberFormat { + fn drop(&mut self) { + if let Some(raw) = self.raw { + unsafe { ffi::FluentBuiltInNumberFormatterDestroy(raw.as_ptr()) }; + } + } +} + +impl Memoizable for NumberFormat { + type Args = (FluentNumberOptions,); + type Error = &'static str; + fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> { + Ok(NumberFormat::new(lang, &args.0)) + } +} + +#[repr(C)] +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub enum FluentDateTimeStyle { + Full, + Long, + Medium, + Short, + None, +} + +impl Default for FluentDateTimeStyle { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeStyle { + fn from(input: &str) -> Self { + match input { + "full" => Self::Full, + "long" => Self::Long, + "medium" => Self::Medium, + "short" => Self::Short, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeHourCycle { + H24, + H23, + H12, + H11, + None, +} + +impl Default for FluentDateTimeHourCycle { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeHourCycle { + fn from(input: &str) -> Self { + match input { + "h24" => Self::H24, + "h23" => Self::H23, + "h12" => Self::H12, + "h11" => Self::H11, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeTextComponent { + Long, + Short, + Narrow, + None, +} + +impl Default for FluentDateTimeTextComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeTextComponent { + fn from(input: &str) -> Self { + match input { + "long" => Self::Long, + "short" => Self::Short, + "narrow" => Self::Narrow, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeNumericComponent { + Numeric, + TwoDigit, + None, +} + +impl Default for FluentDateTimeNumericComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeNumericComponent { + fn from(input: &str) -> Self { + match input { + "numeric" => Self::Numeric, + "2-digit" => Self::TwoDigit, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeMonthComponent { + Numeric, + TwoDigit, + Long, + Short, + Narrow, + None, +} + +impl Default for FluentDateTimeMonthComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeMonthComponent { + fn from(input: &str) -> Self { + match input { + "numeric" => Self::Numeric, + "2-digit" => Self::TwoDigit, + "long" => Self::Long, + "short" => Self::Short, + "narrow" => Self::Narrow, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub enum FluentDateTimeTimeZoneNameComponent { + Long, + Short, + None, +} + +impl Default for FluentDateTimeTimeZoneNameComponent { + fn default() -> Self { + Self::None + } +} + +impl From<&str> for FluentDateTimeTimeZoneNameComponent { + fn from(input: &str) -> Self { + match input { + "long" => Self::Long, + "short" => Self::Short, + _ => Self::None, + } + } +} + +#[repr(C)] +#[derive(Default, Debug, Clone, Hash, PartialEq, Eq)] +pub struct FluentDateTimeOptions { + pub date_style: FluentDateTimeStyle, + pub time_style: FluentDateTimeStyle, + pub hour_cycle: FluentDateTimeHourCycle, + pub weekday: FluentDateTimeTextComponent, + pub era: FluentDateTimeTextComponent, + pub year: FluentDateTimeNumericComponent, + pub month: FluentDateTimeMonthComponent, + pub day: FluentDateTimeNumericComponent, + pub hour: FluentDateTimeNumericComponent, + pub minute: FluentDateTimeNumericComponent, + pub second: FluentDateTimeNumericComponent, + pub time_zone_name: FluentDateTimeTimeZoneNameComponent, +} + +impl FluentDateTimeOptions { + pub fn merge(&mut self, opts: &FluentArgs) { + for (key, value) in opts.iter() { + match (key, value) { + ("dateStyle", FluentValue::String(n)) => { + self.date_style = n.as_ref().into(); + } + ("timeStyle", FluentValue::String(n)) => { + self.time_style = n.as_ref().into(); + } + ("hourCycle", FluentValue::String(n)) => { + self.hour_cycle = n.as_ref().into(); + } + ("weekday", FluentValue::String(n)) => { + self.weekday = n.as_ref().into(); + } + ("era", FluentValue::String(n)) => { + self.era = n.as_ref().into(); + } + ("year", FluentValue::String(n)) => { + self.year = n.as_ref().into(); + } + ("month", FluentValue::String(n)) => { + self.month = n.as_ref().into(); + } + ("day", FluentValue::String(n)) => { + self.day = n.as_ref().into(); + } + ("hour", FluentValue::String(n)) => { + self.hour = n.as_ref().into(); + } + ("minute", FluentValue::String(n)) => { + self.minute = n.as_ref().into(); + } + ("second", FluentValue::String(n)) => { + self.second = n.as_ref().into(); + } + ("timeZoneName", FluentValue::String(n)) => { + self.time_zone_name = n.as_ref().into(); + } + _ => {} + } + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub struct FluentDateTime { + epoch: f64, + options: FluentDateTimeOptions, +} + +impl FluentType for FluentDateTime { + fn duplicate(&self) -> Box<dyn FluentType + Send> { + Box::new(self.clone()) + } + fn as_string(&self, intls: &IntlLangMemoizer) -> Cow<'static, str> { + let result = intls + .with_try_get::<DateTimeFormat, _, _>((self.options.clone(),), |dtf| { + dtf.format(self.epoch) + }) + .expect("Failed to retrieve a DateTimeFormat instance."); + result.into() + } + fn as_string_threadsafe( + &self, + _: &intl_memoizer::concurrent::IntlLangMemoizer, + ) -> Cow<'static, str> { + unimplemented!() + } +} + +impl std::fmt::Display for FluentDateTime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "DATETIME: {}", self.epoch) + } +} + +impl FluentDateTime { + pub fn new(epoch: f64, options: FluentDateTimeOptions) -> Self { + Self { epoch, options } + } +} + +pub struct DateTimeFormat { + raw: Option<NonNull<ffi::RawDateTimeFormatter>>, +} + +/** + * According to http://userguide.icu-project.org/design, as long as we constrain + * ourselves to const APIs ICU is const-correct. + */ +unsafe impl Send for DateTimeFormat {} +unsafe impl Sync for DateTimeFormat {} + +impl DateTimeFormat { + pub fn new(locale: LanguageIdentifier, options: FluentDateTimeOptions) -> Self { + // ICU needs null-termination here, otherwise we could use nsCStr. + let loc: nsCString = locale.to_string().into(); + Self { + raw: unsafe { NonNull::new(ffi::FluentBuiltInDateTimeFormatterCreate(&loc, options)) }, + } + } + + pub fn format(&self, input: f64) -> String { + if let Some(raw) = self.raw { + unsafe { + let mut byte_count = 0; + let buffer = + ffi::FluentBuiltInDateTimeFormatterFormat(raw.as_ptr(), input, &mut byte_count); + if buffer.is_null() { + return String::new(); + } + String::from_raw_parts(buffer, byte_count as usize, byte_count as usize) + } + } else { + String::new() + } + } +} + +impl Drop for DateTimeFormat { + fn drop(&mut self) { + if let Some(raw) = self.raw { + unsafe { ffi::FluentBuiltInDateTimeFormatterDestroy(raw.as_ptr()) }; + } + } +} + +impl Memoizable for DateTimeFormat { + type Args = (FluentDateTimeOptions,); + type Error = &'static str; + fn construct(lang: LanguageIdentifier, args: Self::Args) -> Result<Self, Self::Error> { + Ok(DateTimeFormat::new(lang, args.0)) + } +} diff --git a/intl/l10n/rust/fluent-ffi/src/bundle.rs b/intl/l10n/rust/fluent-ffi/src/bundle.rs new file mode 100644 index 0000000000..21bf0d52e9 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/bundle.rs @@ -0,0 +1,331 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::builtins::{FluentDateTime, FluentDateTimeOptions, NumberFormat}; +use cstr::cstr; +pub use fluent::{FluentArgs, FluentBundle, FluentError, FluentResource, FluentValue}; +use fluent_pseudo::transform_dom; +pub use intl_memoizer::IntlLangMemoizer; +use nsstring::{nsACString, nsCString}; +use std::borrow::Cow; +use std::ffi::CStr; +use std::mem; +use std::rc::Rc; +use thin_vec::ThinVec; +use unic_langid::LanguageIdentifier; +use xpcom::interfaces::nsIPrefBranch; + +pub type FluentBundleRc = FluentBundle<Rc<FluentResource>>; + +#[derive(Debug)] +#[repr(C, u8)] +pub enum FluentArgument<'s> { + Double_(f64), + String(&'s nsACString), +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nArg<'s> { + pub id: &'s nsACString, + pub value: FluentArgument<'s>, +} + +fn transform_accented(s: &str) -> Cow<str> { + transform_dom(s, false, true, true) +} + +fn transform_bidi(s: &str) -> Cow<str> { + transform_dom(s, false, false, false) +} + +fn format_numbers(num: &FluentValue, intls: &IntlLangMemoizer) -> Option<String> { + match num { + FluentValue::Number(n) => { + let result = intls + .with_try_get::<NumberFormat, _, _>((n.options.clone(),), |nf| nf.format(n.value)) + .expect("Failed to retrieve a NumberFormat instance."); + Some(result) + } + _ => None, + } +} + +fn get_string_pref(name: &CStr) -> Option<nsCString> { + let mut value = nsCString::new(); + let prefs_service = + xpcom::get_service::<nsIPrefBranch>(cstr!("@mozilla.org/preferences-service;1"))?; + unsafe { + prefs_service + .GetCharPref(name.as_ptr(), &mut *value) + .to_result() + .ok()?; + } + Some(value) +} + +fn get_bool_pref(name: &CStr) -> Option<bool> { + let mut value = false; + let prefs_service = + xpcom::get_service::<nsIPrefBranch>(cstr!("@mozilla.org/preferences-service;1"))?; + unsafe { + prefs_service + .GetBoolPref(name.as_ptr(), &mut value) + .to_result() + .ok()?; + } + Some(value) +} + +pub fn adapt_bundle_for_gecko(bundle: &mut FluentBundleRc, pseudo_strategy: Option<&nsACString>) { + bundle.set_formatter(Some(format_numbers)); + + bundle + .add_function("PLATFORM", |_args, _named_args| { + if cfg!(target_os = "linux") { + "linux".into() + } else if cfg!(target_os = "windows") { + "windows".into() + } else if cfg!(target_os = "macos") { + "macos".into() + } else if cfg!(target_os = "android") { + "android".into() + } else { + "other".into() + } + }) + .expect("Failed to add a function to the bundle."); + bundle + .add_function("NUMBER", |args, named| { + if let Some(FluentValue::Number(n)) = args.get(0) { + let mut num = n.clone(); + num.options.merge(named); + FluentValue::Number(num) + } else { + FluentValue::None + } + }) + .expect("Failed to add a function to the bundle."); + bundle + .add_function("DATETIME", |args, named| { + if let Some(FluentValue::Number(n)) = args.get(0) { + let mut options = FluentDateTimeOptions::default(); + options.merge(&named); + FluentValue::Custom(Box::new(FluentDateTime::new(n.value, options))) + } else { + FluentValue::None + } + }) + .expect("Failed to add a function to the bundle."); + + enum PseudoStrategy { + Accented, + Bidi, + None, + } + // This is quirky because we can't coerce Option<&nsACString> and Option<nsCString> + // into bytes easily without allocating. + let strategy_kind = match pseudo_strategy.map(|s| &s[..]) { + Some(b"accented") => PseudoStrategy::Accented, + Some(b"bidi") => PseudoStrategy::Bidi, + _ => { + if let Some(pseudo_strategy) = get_string_pref(cstr!("intl.l10n.pseudo")) { + match &pseudo_strategy[..] { + b"accented" => PseudoStrategy::Accented, + b"bidi" => PseudoStrategy::Bidi, + _ => PseudoStrategy::None, + } + } else { + PseudoStrategy::None + } + } + }; + match strategy_kind { + PseudoStrategy::Accented => bundle.set_transform(Some(transform_accented)), + PseudoStrategy::Bidi => bundle.set_transform(Some(transform_bidi)), + PseudoStrategy::None => bundle.set_transform(None), + } + + // Temporarily disable bidi isolation due to Microsoft not supporting FSI/PDI. + // See bug 1439018 for details. + let default_use_isolating = false; + let use_isolating = + get_bool_pref(cstr!("intl.l10n.enable-bidi-marks")).unwrap_or(default_use_isolating); + bundle.set_use_isolating(use_isolating); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_new_single( + locale: &nsACString, + use_isolating: bool, + pseudo_strategy: &nsACString, +) -> *mut FluentBundleRc { + let id = match locale.to_utf8().parse::<LanguageIdentifier>() { + Ok(id) => id, + Err(..) => return std::ptr::null_mut(), + }; + + Box::into_raw(fluent_bundle_new_internal( + &[id], + use_isolating, + pseudo_strategy, + )) +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_new( + locales: *const nsCString, + locale_count: usize, + use_isolating: bool, + pseudo_strategy: &nsACString, +) -> *mut FluentBundleRc { + let mut langids = Vec::with_capacity(locale_count); + let locales = std::slice::from_raw_parts(locales, locale_count); + for locale in locales { + let id = match locale.to_utf8().parse::<LanguageIdentifier>() { + Ok(id) => id, + Err(..) => return std::ptr::null_mut(), + }; + langids.push(id); + } + + Box::into_raw(fluent_bundle_new_internal( + &langids, + use_isolating, + pseudo_strategy, + )) +} + +fn fluent_bundle_new_internal( + langids: &[LanguageIdentifier], + use_isolating: bool, + pseudo_strategy: &nsACString, +) -> Box<FluentBundleRc> { + let mut bundle = FluentBundle::new(langids.to_vec()); + bundle.set_use_isolating(use_isolating); + + bundle.set_formatter(Some(format_numbers)); + + adapt_bundle_for_gecko(&mut bundle, Some(pseudo_strategy)); + + Box::new(bundle) +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_get_locales( + bundle: &FluentBundleRc, + result: &mut ThinVec<nsCString>, +) { + for locale in &bundle.locales { + result.push(locale.to_string().as_str().into()); + } +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_destroy(bundle: *mut FluentBundleRc) { + let _ = Box::from_raw(bundle); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_has_message(bundle: &FluentBundleRc, id: &nsACString) -> bool { + bundle.has_message(id.to_string().as_str()) +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_get_message( + bundle: &FluentBundleRc, + id: &nsACString, + has_value: &mut bool, + attrs: &mut ThinVec<nsCString>, +) -> bool { + match bundle.get_message(&id.to_utf8()) { + Some(message) => { + attrs.reserve(message.attributes().count()); + *has_value = message.value().is_some(); + for attr in message.attributes() { + attrs.push(attr.id().into()); + } + true + } + None => { + *has_value = false; + false + } + } +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_format_pattern( + bundle: &FluentBundleRc, + id: &nsACString, + attr: &nsACString, + args: &ThinVec<L10nArg>, + ret_val: &mut nsACString, + ret_errors: &mut ThinVec<nsCString>, +) -> bool { + let args = convert_args(&args); + + let message = match bundle.get_message(&id.to_utf8()) { + Some(message) => message, + None => return false, + }; + + let pattern = if !attr.is_empty() { + match message.get_attribute(&attr.to_utf8()) { + Some(attr) => attr.value(), + None => return false, + } + } else { + match message.value() { + Some(value) => value, + None => return false, + } + }; + + let mut errors = vec![]; + bundle + .write_pattern(ret_val, pattern, args.as_ref(), &mut errors) + .expect("Failed to write to a nsCString."); + append_fluent_errors_to_ret_errors(ret_errors, &errors); + true +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_add_resource( + bundle: &mut FluentBundleRc, + r: *const FluentResource, + allow_overrides: bool, + ret_errors: &mut ThinVec<nsCString>, +) { + // we don't own the resource + let r = mem::ManuallyDrop::new(Rc::from_raw(r)); + + if allow_overrides { + bundle.add_resource_overriding(Rc::clone(&r)); + } else if let Err(errors) = bundle.add_resource(Rc::clone(&r)) { + append_fluent_errors_to_ret_errors(ret_errors, &errors); + } +} + +pub fn convert_args<'s>(args: &[L10nArg<'s>]) -> Option<FluentArgs<'s>> { + if args.is_empty() { + return None; + } + + let mut result = FluentArgs::with_capacity(args.len()); + for arg in args { + let val = match arg.value { + FluentArgument::Double_(d) => FluentValue::from(d), + FluentArgument::String(s) => FluentValue::from(s.to_utf8()), + }; + result.set(arg.id.to_string(), val); + } + Some(result) +} + +fn append_fluent_errors_to_ret_errors(ret_errors: &mut ThinVec<nsCString>, errors: &[FluentError]) { + for error in errors { + ret_errors.push(error.to_string().into()); + } +} diff --git a/intl/l10n/rust/fluent-ffi/src/ffi.rs b/intl/l10n/rust/fluent-ffi/src/ffi.rs new file mode 100644 index 0000000000..a264ad11b7 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/ffi.rs @@ -0,0 +1,154 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::builtins::FluentDateTimeOptions; +use fluent::types::FluentNumberCurrencyDisplayStyle; +use fluent::types::FluentNumberOptions; +use fluent::types::FluentNumberStyle; +use nsstring::nsCString; + +pub enum RawNumberFormatter {} + +#[repr(C)] +pub enum FluentNumberStyleRaw { + Decimal, + Currency, + Percent, +} + +impl From<FluentNumberStyle> for FluentNumberStyleRaw { + fn from(input: FluentNumberStyle) -> Self { + match input { + FluentNumberStyle::Decimal => Self::Decimal, + FluentNumberStyle::Currency => Self::Currency, + FluentNumberStyle::Percent => Self::Percent, + } + } +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub enum FluentNumberCurrencyDisplayStyleRaw { + Symbol, + Code, + Name, +} + +impl From<FluentNumberCurrencyDisplayStyle> for FluentNumberCurrencyDisplayStyleRaw { + fn from(input: FluentNumberCurrencyDisplayStyle) -> Self { + match input { + FluentNumberCurrencyDisplayStyle::Symbol => Self::Symbol, + FluentNumberCurrencyDisplayStyle::Code => Self::Code, + FluentNumberCurrencyDisplayStyle::Name => Self::Name, + } + } +} + +#[repr(C)] +pub struct FluentNumberOptionsRaw { + pub style: FluentNumberStyleRaw, + pub currency: nsCString, + pub currency_display: FluentNumberCurrencyDisplayStyleRaw, + pub use_grouping: bool, + pub minimum_integer_digits: usize, + pub minimum_fraction_digits: usize, + pub maximum_fraction_digits: usize, + pub minimum_significant_digits: isize, + pub maximum_significant_digits: isize, +} + +fn get_number_option(val: Option<usize>, min: usize, max: usize, default: usize) -> usize { + if let Some(val) = val { + if val >= min && val <= max { + val + } else { + default + } + } else { + default + } +} + +impl From<&FluentNumberOptions> for FluentNumberOptionsRaw { + fn from(input: &FluentNumberOptions) -> Self { + let currency: nsCString = if let Some(ref currency) = input.currency { + currency.into() + } else { + nsCString::new() + }; + + //XXX: This should be fetched from currency table. + let currency_digits = 2; + + // Keep it aligned with ECMA402 NumberFormat logic. + let minfd_default = if input.style == FluentNumberStyle::Currency { + currency_digits + } else { + 0 + }; + let maxfd_default = match input.style { + FluentNumberStyle::Decimal => 3, + FluentNumberStyle::Currency => currency_digits, + FluentNumberStyle::Percent => 0, + }; + let minid = get_number_option(input.minimum_integer_digits, 1, 21, 1); + let minfd = get_number_option(input.minimum_fraction_digits, 0, 20, minfd_default); + let maxfd_actual_default = std::cmp::max(minfd, maxfd_default); + let maxfd = get_number_option( + input.maximum_fraction_digits, + minfd, + 20, + maxfd_actual_default, + ); + + let (minsd, maxsd) = if input.minimum_significant_digits.is_some() + || input.maximum_significant_digits.is_some() + { + let minsd = get_number_option(input.minimum_significant_digits, 1, 21, 1); + let maxsd = get_number_option(input.maximum_significant_digits, minsd, 21, 21); + (minsd as isize, maxsd as isize) + } else { + (-1, -1) + }; + + Self { + style: input.style.into(), + currency, + currency_display: input.currency_display.into(), + use_grouping: input.use_grouping, + minimum_integer_digits: minid, + minimum_fraction_digits: minfd, + maximum_fraction_digits: maxfd, + minimum_significant_digits: minsd, + maximum_significant_digits: maxsd, + } + } +} + +pub enum RawDateTimeFormatter {} + +extern "C" { + pub fn FluentBuiltInNumberFormatterCreate( + locale: &nsCString, + options: &FluentNumberOptionsRaw, + ) -> *mut RawNumberFormatter; + pub fn FluentBuiltInNumberFormatterFormat( + formatter: *const RawNumberFormatter, + input: f64, + out_count: &mut usize, + out_capacity: &mut usize, + ) -> *mut u8; + pub fn FluentBuiltInNumberFormatterDestroy(formatter: *mut RawNumberFormatter); + + pub fn FluentBuiltInDateTimeFormatterCreate( + locale: &nsCString, + options: FluentDateTimeOptions, + ) -> *mut RawDateTimeFormatter; + pub fn FluentBuiltInDateTimeFormatterFormat( + formatter: *const RawDateTimeFormatter, + input: f64, + out_count: &mut u32, + ) -> *mut u8; + pub fn FluentBuiltInDateTimeFormatterDestroy(formatter: *mut RawDateTimeFormatter); +} diff --git a/intl/l10n/rust/fluent-ffi/src/lib.rs b/intl/l10n/rust/fluent-ffi/src/lib.rs new file mode 100644 index 0000000000..6cf56c2e4a --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/lib.rs @@ -0,0 +1,11 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +mod builtins; +mod bundle; +mod ffi; +mod resource; + +pub use bundle::*; +pub use resource::*; diff --git a/intl/l10n/rust/fluent-ffi/src/resource.rs b/intl/l10n/rust/fluent-ffi/src/resource.rs new file mode 100644 index 0000000000..dc011b9462 --- /dev/null +++ b/intl/l10n/rust/fluent-ffi/src/resource.rs @@ -0,0 +1,39 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub use fluent::FluentResource; +use nsstring::nsACString; +use std::{ + mem::{self, ManuallyDrop}, + rc::Rc, +}; + +#[no_mangle] +pub extern "C" fn fluent_resource_new( + name: &nsACString, + has_errors: &mut bool, +) -> *const FluentResource { + let res = match FluentResource::try_new(name.to_string()) { + Ok(res) => { + *has_errors = false; + res + } + Err((res, _)) => { + *has_errors = true; + res + } + }; + Rc::into_raw(Rc::new(res)) +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_resource_addref(res: *const FluentResource) { + let raw = ManuallyDrop::new(Rc::from_raw(res)); + mem::forget(Rc::clone(&raw)); +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_resource_release(res: *const FluentResource) { + let _ = Rc::from_raw(res); +} diff --git a/intl/l10n/rust/gtest/Cargo.toml b/intl/l10n/rust/gtest/Cargo.toml new file mode 100644 index 0000000000..ebc532f599 --- /dev/null +++ b/intl/l10n/rust/gtest/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "l10nregistry-ffi-gtest" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +license = "MPL-2.0" +description = "Tests for rust bindings to l10nRegistry" +edition = "2018" + +[dependencies] +l10nregistry-ffi = { path = "../l10nregistry-ffi" } +moz_task = { path = "../../../../xpcom/rust/moz_task" } + +[lib] +path = "test.rs" diff --git a/intl/l10n/rust/gtest/Test.cpp b/intl/l10n/rust/gtest/Test.cpp new file mode 100644 index 0000000000..98e7a8b5c6 --- /dev/null +++ b/intl/l10n/rust/gtest/Test.cpp @@ -0,0 +1,23 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" + +extern "C" void Rust_L10NLoadAsync(bool* aItWorked); + +TEST(RustL10N, LoadAsync) +{ + bool itWorked = false; + Rust_L10NLoadAsync(&itWorked); + EXPECT_TRUE(itWorked); +} + +extern "C" void Rust_L10NLoadSync(bool* aItWorked); + +TEST(RustL10N, LoadSync) +{ + bool itWorked = false; + Rust_L10NLoadSync(&itWorked); + EXPECT_TRUE(itWorked); +} diff --git a/intl/l10n/rust/gtest/moz.build b/intl/l10n/rust/gtest/moz.build new file mode 100644 index 0000000000..7c73e04fc8 --- /dev/null +++ b/intl/l10n/rust/gtest/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "Test.cpp", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/intl/l10n/rust/gtest/test.rs b/intl/l10n/rust/gtest/test.rs new file mode 100644 index 0000000000..3e993f4789 --- /dev/null +++ b/intl/l10n/rust/gtest/test.rs @@ -0,0 +1,58 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use l10nregistry_ffi::load::{load_async, load_sync}; +use moz_task; +use std::borrow::Borrow; + +// We want to test a file that ships in every platform configuration, so we take +// something from `toolkit/`. But we don't want to depend on the specifics of +// the text, or the packaging of that text, since those can change. It would be +// best to register an untranslated `.ftl` for this test, but that's difficult. +// Second best is to ship an untranslated `.ftl`, but that is not well-supported +// by existing processes either. So we settle for depending on the form of +// specific identifiers, whose names will appear in future searches, while +// depending on the specific messages or the file packaging. +fn assert_about_about_correct<T: Borrow<[u8]>> (res: T) { + assert!(res.borrow().len() > 0); + + // `windows` is a convenient, if inefficient, way to look for a subslice. + let needle = b"about-about-title"; + assert!(res.borrow().windows(needle.len()).position(|window| window == needle).is_some()); + + let needle = b"about-about-note"; + assert!(res.borrow().windows(needle.len()).position(|window| window == needle).is_some()); +} + +#[no_mangle] +pub extern "C" fn Rust_L10NLoadAsync(it_worked: *mut bool) { + let future = async move { + match load_async("resource://gre/localization/en-US/toolkit/about/aboutAbout.ftl").await { + Ok(res) => { + assert_about_about_correct(res); + unsafe { + *it_worked = true; + } + } + Err(err) => panic!("{:?}", err), + } + }; + + unsafe { + moz_task::gtest_only::spin_event_loop_until("Rust_L10NLoadAsync", future).unwrap(); + } +} + +#[no_mangle] +pub extern "C" fn Rust_L10NLoadSync(it_worked: *mut bool) { + match load_sync("resource://gre/localization/en-US/toolkit/about/aboutAbout.ftl") { + Ok(res) => { + assert_about_about_correct(res); + unsafe { + *it_worked = true; + } + } + Err(err) => panic!("{:?}", err), + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/Cargo.toml b/intl/l10n/rust/l10nregistry-ffi/Cargo.toml new file mode 100644 index 0000000000..8d5d575074 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "l10nregistry-ffi" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +futures-channel = "0.3" +futures = "0.3" +libc = "0.2" +cstr = "0.2" +log = "0.4" +nserror = { path = "../../../../xpcom/rust/nserror" } +nsstring = { path = "../../../../xpcom/rust/nsstring" } +l10nregistry = { path = "../l10nregistry-rs" } +fluent = { version = "0.16.0", features = ["fluent-pseudo"] } +unic-langid = "0.9" +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } +async-trait = "0.1" +moz_task = { path = "../../../../xpcom/rust/moz_task" } +xpcom = { path = "../../../../xpcom/rust/xpcom" } +fluent-ffi = { path = "../fluent-ffi" } +fluent-fallback = "0.7.0" diff --git a/intl/l10n/rust/l10nregistry-ffi/cbindgen.toml b/intl/l10n/rust/l10nregistry-ffi/cbindgen.toml new file mode 100644 index 0000000000..9e7127eeaa --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/cbindgen.toml @@ -0,0 +1,26 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef mozilla_intl_l10n_RegistryBindings_h +#error "Don't include this file directly, instead include RegistryBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +namespaces = ["mozilla", "intl", "ffi"] +includes = ["nsIStreamLoader.h"] + +[parse] +parse_deps = true +include = ["fluent-bundle", "fluent-fallback", "l10nregistry"] + +[enum] +derive_helper_methods = true + +[export.rename] +"ThinVec" = "nsTArray" +"Promise" = "dom::Promise" diff --git a/intl/l10n/rust/l10nregistry-ffi/src/env.rs b/intl/l10n/rust/l10nregistry-ffi/src/env.rs new file mode 100644 index 0000000000..7a77af2176 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/env.rs @@ -0,0 +1,132 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use crate::xpcom_utils::get_app_locales; +use cstr::cstr; +use fluent_fallback::env::LocalesProvider; +use l10nregistry::{ + env::ErrorReporter, + errors::{L10nRegistryError, L10nRegistrySetupError}, +}; +use log::warn; +use nserror::{nsresult, NS_ERROR_NOT_AVAILABLE}; +use nsstring::{nsCStr, nsString}; +use std::fmt::{self, Write}; +use unic_langid::LanguageIdentifier; +use xpcom::interfaces; + +#[derive(Clone)] +pub struct GeckoEnvironment { + custom_locales: Option<Vec<LanguageIdentifier>>, +} + +impl GeckoEnvironment { + pub fn new(custom_locales: Option<Vec<LanguageIdentifier>>) -> Self { + Self { custom_locales } + } + + pub fn report_l10nregistry_setup_error(error: &L10nRegistrySetupError) { + warn!("L10nRegistry setup error: {}", error); + let result = log_simple_console_error( + &error.to_string(), + false, + true, + None, + (0, 0), + interfaces::nsIScriptError::errorFlag as u32, + ); + if let Err(err) = result { + warn!("Error while reporting an error: {}", err); + } + } +} + +impl ErrorReporter for GeckoEnvironment { + fn report_errors(&self, errors: Vec<L10nRegistryError>) { + for error in errors { + warn!("L10nRegistry error: {}", error); + let result = match error { + L10nRegistryError::FluentError { + resource_id, + loc, + error, + } => log_simple_console_error( + &error.to_string(), + false, + true, + Some(nsString::from(&resource_id.value)), + loc.map_or((0, 0), |(l, c)| (l as u32, c as u32)), + interfaces::nsIScriptError::errorFlag as u32, + ), + L10nRegistryError::MissingResource { .. } => log_simple_console_error( + &error.to_string(), + false, + true, + None, + (0, 0), + interfaces::nsIScriptError::warningFlag as u32, + ), + }; + if let Err(err) = result { + warn!("Error while reporting an error: {}", err); + } + } + } +} + +impl LocalesProvider for GeckoEnvironment { + type Iter = std::vec::IntoIter<unic_langid::LanguageIdentifier>; + fn locales(&self) -> Self::Iter { + if let Some(custom_locales) = &self.custom_locales { + custom_locales.clone().into_iter() + } else { + let result = get_app_locales() + .expect("Failed to retrieve app locales") + .into_iter() + .map(|s| LanguageIdentifier::from_bytes(&s).expect("Failed to parse a locale")) + .collect::<Vec<_>>(); + result.into_iter() + } + } +} + +fn log_simple_console_error( + error: &impl fmt::Display, + from_private_window: bool, + from_chrome_context: bool, + path: Option<nsString>, + pos: (u32, u32), + error_flags: u32, +) -> Result<(), nsresult> { + // Format whatever error argument into a wide string with `Display`. + let mut error_str = nsString::new(); + write!(&mut error_str, "{}", error).expect("nsString has an infallible Write impl"); + + // Get the relevant services, and create the script error object. + let console_service = + xpcom::get_service::<interfaces::nsIConsoleService>(cstr!("@mozilla.org/consoleservice;1")) + .ok_or(NS_ERROR_NOT_AVAILABLE)?; + let script_error = + xpcom::create_instance::<interfaces::nsIScriptError>(cstr!("@mozilla.org/scripterror;1")) + .ok_or(NS_ERROR_NOT_AVAILABLE)?; + let category = nsCStr::from("l10n"); + unsafe { + script_error + .Init( + &*error_str, + &*path.unwrap_or(nsString::new()), /* aSourceName */ + &*nsString::new(), /* aSourceLine */ + pos.0, /* aLineNumber */ + pos.1, /* aColNumber */ + error_flags, + &*category, + from_private_window, + from_chrome_context, + ) + .to_result()?; + + console_service.LogMessage(&**script_error).to_result()?; + } + Ok(()) +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs b/intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs new file mode 100644 index 0000000000..aba8a81470 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs @@ -0,0 +1,70 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use l10nregistry::source::{FileFetcher, ResourceId}; + +use std::{borrow::Cow, io}; + +pub struct GeckoFileFetcher; + +fn try_string_from_box_u8(input: Box<[u8]>) -> io::Result<String> { + String::from_utf8(input.into()) + .map_err(|err| io::Error::new(io::ErrorKind::InvalidData, err.utf8_error())) +} + +// For historical reasons we maintain a locale in Firefox with a codename `ja-JP-mac`. +// This string is an invalid BCP-47 language tag, so we don't store it in Gecko, which uses +// valid BCP-47 tags only, but rather keep that quirk local to Gecko L10nRegistry file fetcher. +// +// Here, if we encounter `ja-JP-macos` (valid BCP-47), we swap it for `ja-JP-mac`. +// +// See bug 1726586 for details, and source::get_locale_from_gecko. +fn get_path_for_gecko<'s>(input: &'s str) -> Cow<'s, str> { + if input.contains("ja-JP-macos") { + input.replace("ja-JP-macos", "ja-JP-mac").into() + } else { + input.into() + } +} + +#[async_trait::async_trait(?Send)] +impl FileFetcher for GeckoFileFetcher { + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String> { + let path = get_path_for_gecko(&resource_id.value); + crate::load::load_sync(path).and_then(try_string_from_box_u8) + } + + async fn fetch(&self, resource_id: &ResourceId) -> io::Result<String> { + let path = get_path_for_gecko(&resource_id.value); + crate::load::load_async(path) + .await + .and_then(try_string_from_box_u8) + } +} + +pub struct MockFileFetcher { + fs: Vec<(String, String)>, +} + +impl MockFileFetcher { + pub fn new(fs: Vec<(String, String)>) -> Self { + Self { fs } + } +} + +#[async_trait::async_trait(?Send)] +impl FileFetcher for MockFileFetcher { + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String> { + for (p, source) in &self.fs { + if p == &resource_id.value { + return Ok(source.clone()); + } + } + Err(io::Error::new(io::ErrorKind::NotFound, "File not found")) + } + + async fn fetch(&self, resource_id: &ResourceId) -> io::Result<String> { + self.fetch_sync(resource_id) + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/lib.rs b/intl/l10n/rust/l10nregistry-ffi/src/lib.rs new file mode 100644 index 0000000000..843860abf9 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/lib.rs @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +pub mod env; +mod fetcher; +pub mod load; +pub mod registry; +mod source; +mod xpcom_utils; diff --git a/intl/l10n/rust/l10nregistry-ffi/src/load.rs b/intl/l10n/rust/l10nregistry-ffi/src/load.rs new file mode 100644 index 0000000000..04a041246f --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/load.rs @@ -0,0 +1,113 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use futures_channel::oneshot; +use nserror::{nsresult, NS_OK, NS_SUCCESS_ADOPTED_DATA}; +use nsstring::{nsACString, nsCStringLike}; +use std::{ + cell::Cell, + ffi::c_void, + io::{self, Error, ErrorKind}, + ptr, +}; +use xpcom::{ + interfaces::{nsIStreamLoader, nsIStreamLoaderObserver, nsISupports}, + xpcom, +}; + +unsafe fn boxed_slice_from_raw(ptr: *mut u8, len: usize) -> Box<[u8]> { + if ptr.is_null() { + // It is undefined behaviour to create a `Box<[u8]>` with a null pointer, + // so avoid that case. + assert_eq!(len, 0); + Box::new([]) + } else { + Box::from_raw(ptr::slice_from_raw_parts_mut(ptr, len)) + } +} + +#[xpcom(implement(nsIStreamLoaderObserver), nonatomic)] +struct StreamLoaderObserver { + sender: Cell<Option<oneshot::Sender<Result<Box<[u8]>, nsresult>>>>, +} + +impl StreamLoaderObserver { + #[allow(non_snake_case)] + unsafe fn OnStreamComplete( + &self, + _loader: *const nsIStreamLoader, + _ctxt: *const nsISupports, + status: nsresult, + result_length: u32, + result: *const u8, + ) -> nsresult { + let sender = match self.sender.take() { + Some(sender) => sender, + None => return NS_OK, + }; + + if status.failed() { + sender.send(Err(status)).expect("Failed to send data"); + return NS_OK; + } + + // safety: take ownership of the data passed in. This is OK because we + // have configured Rust and C++ to use the same allocator, and our + // caller won't free the `result` pointer if we return + // NS_SUCCESS_ADOPTED_DATA. + sender + .send(Ok(boxed_slice_from_raw( + result as *mut u8, + result_length as usize, + ))) + .expect("Failed to send data"); + NS_SUCCESS_ADOPTED_DATA + } +} + +extern "C" { + fn L10nRegistryLoad( + path: *const nsACString, + observer: *const nsIStreamLoaderObserver, + ) -> nsresult; + + fn L10nRegistryLoadSync( + aPath: *const nsACString, + aData: *mut *mut c_void, + aSize: *mut u64, + ) -> nsresult; +} + +pub async fn load_async(path: impl nsCStringLike) -> io::Result<Box<[u8]>> { + let (sender, receiver) = oneshot::channel::<Result<Box<[u8]>, nsresult>>(); + let observer = StreamLoaderObserver::allocate(InitStreamLoaderObserver { + sender: Cell::new(Some(sender)), + }); + unsafe { + L10nRegistryLoad(&*path.adapt(), observer.coerce()) + .to_result() + .map_err(|err| Error::new(ErrorKind::Other, err))?; + } + receiver + .await + .expect("Failed to receive from observer.") + .map_err(|err| Error::new(ErrorKind::Other, err)) +} + +pub fn load_sync(path: impl nsCStringLike) -> io::Result<Box<[u8]>> { + let mut data_ptr: *mut c_void = ptr::null_mut(); + let mut data_length: u64 = 0; + unsafe { + L10nRegistryLoadSync(&*path.adapt(), &mut data_ptr, &mut data_length) + .to_result() + .map_err(|err| Error::new(ErrorKind::Other, err))?; + + // The call succeeded, meaning `data_ptr` and `size` have been filled in with owning pointers to actual data payloads (or null). + // If we get a null, return a successful read of the empty file. + Ok(boxed_slice_from_raw( + data_ptr as *mut u8, + data_length as usize, + )) + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/registry.rs b/intl/l10n/rust/l10nregistry-ffi/src/registry.rs new file mode 100644 index 0000000000..846f12273c --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/registry.rs @@ -0,0 +1,519 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use fluent_ffi::{adapt_bundle_for_gecko, FluentBundleRc}; +use nsstring::{nsACString, nsCString}; +use std::mem; +use std::rc::Rc; +use thin_vec::ThinVec; + +use crate::{env::GeckoEnvironment, fetcher::GeckoFileFetcher, xpcom_utils::is_parent_process}; +use fluent_fallback::{generator::BundleGenerator, types::ResourceType}; +use futures_channel::mpsc::{unbounded, UnboundedSender}; +pub use l10nregistry::{ + errors::L10nRegistrySetupError, + registry::{BundleAdapter, GenerateBundles, GenerateBundlesSync, L10nRegistry}, + source::{FileSource, ResourceId, ToResourceId}, +}; +use unic_langid::LanguageIdentifier; +use xpcom::RefPtr; + +#[derive(Clone)] +pub struct GeckoBundleAdapter { + use_isolating: bool, +} + +impl Default for GeckoBundleAdapter { + fn default() -> Self { + Self { + use_isolating: true, + } + } +} + +impl BundleAdapter for GeckoBundleAdapter { + fn adapt_bundle(&self, bundle: &mut l10nregistry::fluent::FluentBundle) { + bundle.set_use_isolating(self.use_isolating); + adapt_bundle_for_gecko(bundle, None); + } +} + +thread_local!(static L10N_REGISTRY: Rc<GeckoL10nRegistry> = { + let sources = if is_parent_process() { + let packaged_locales = get_packaged_locales(); + let entries = get_l10n_registry_category_entries(); + + Some(entries + .into_iter() + .map(|entry| { + FileSource::new( + entry.entry.to_string(), + Some("app".to_string()), + packaged_locales.clone(), + entry.value.to_string(), + Default::default(), + GeckoFileFetcher, + ) + }) + .collect()) + + } else { + None + }; + + create_l10n_registry(sources) +}); + +pub type GeckoL10nRegistry = L10nRegistry<GeckoEnvironment, GeckoBundleAdapter>; +pub type GeckoFluentBundleIterator = GenerateBundlesSync<GeckoEnvironment, GeckoBundleAdapter>; + +trait GeckoReportError<V, E> { + fn report_error(self) -> Result<V, E>; +} + +impl<V> GeckoReportError<V, L10nRegistrySetupError> for Result<V, L10nRegistrySetupError> { + fn report_error(self) -> Self { + if let Err(ref err) = self { + GeckoEnvironment::report_l10nregistry_setup_error(err); + } + self + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nFileSourceDescriptor { + name: nsCString, + metasource: nsCString, + locales: ThinVec<nsCString>, + pre_path: nsCString, + index: ThinVec<nsCString>, +} + +fn get_l10n_registry_category_entries() -> Vec<crate::xpcom_utils::CategoryEntry> { + crate::xpcom_utils::get_category_entries(&nsCString::from("l10n-registry")).unwrap_or_default() +} + +fn get_packaged_locales() -> Vec<LanguageIdentifier> { + crate::xpcom_utils::get_packaged_locales() + .map(|locales| { + locales + .into_iter() + .map(|s| s.to_utf8().parse().expect("Failed to parse locale.")) + .collect() + }) + .unwrap_or_default() +} + +fn create_l10n_registry(sources: Option<Vec<FileSource>>) -> Rc<GeckoL10nRegistry> { + let env = GeckoEnvironment::new(None); + let mut reg = L10nRegistry::with_provider(env); + + reg.set_bundle_adapter(GeckoBundleAdapter::default()) + .expect("Failed to set bundle adaptation closure."); + + if let Some(sources) = sources { + reg.register_sources(sources) + .expect("Failed to register sources."); + } + Rc::new(reg) +} + +pub fn set_l10n_registry(new_sources: &ThinVec<L10nFileSourceDescriptor>) { + L10N_REGISTRY.with(|reg| { + let new_source_names: Vec<_> = new_sources + .iter() + .map(|d| d.name.to_utf8().to_string()) + .collect(); + let old_sources = reg.get_source_names().unwrap(); + + let mut sources_to_be_removed = vec![]; + for name in &old_sources { + if !new_source_names.contains(&name) { + sources_to_be_removed.push(name); + } + } + reg.remove_sources(sources_to_be_removed).unwrap(); + + let mut add_sources = vec![]; + for desc in new_sources { + if !old_sources.contains(&desc.name.to_string()) { + add_sources.push(FileSource::new( + desc.name.to_string(), + Some(desc.metasource.to_string()), + desc.locales + .iter() + .map(|s| s.to_utf8().parse().unwrap()) + .collect(), + desc.pre_path.to_string(), + Default::default(), + GeckoFileFetcher, + )); + } + } + reg.register_sources(add_sources).unwrap(); + }); +} + +pub fn get_l10n_registry() -> Rc<GeckoL10nRegistry> { + L10N_REGISTRY.with(|reg| reg.clone()) +} + +#[repr(C)] +#[derive(Clone, Copy)] +pub enum GeckoResourceType { + Optional, + Required, +} + +#[repr(C)] +pub struct GeckoResourceId { + value: nsCString, + resource_type: GeckoResourceType, +} + +impl From<&GeckoResourceId> for ResourceId { + fn from(resource_id: &GeckoResourceId) -> Self { + resource_id + .value + .to_string() + .to_resource_id(match resource_id.resource_type { + GeckoResourceType::Optional => ResourceType::Optional, + GeckoResourceType::Required => ResourceType::Required, + }) + } +} + +#[repr(C)] +pub enum L10nRegistryStatus { + None, + EmptyName, + InvalidLocaleCode, +} + +#[no_mangle] +pub extern "C" fn l10nregistry_new(use_isolating: bool) -> *const GeckoL10nRegistry { + let env = GeckoEnvironment::new(None); + let mut reg = L10nRegistry::with_provider(env); + let _ = reg + .set_bundle_adapter(GeckoBundleAdapter { use_isolating }) + .report_error(); + Rc::into_raw(Rc::new(reg)) +} + +#[no_mangle] +pub extern "C" fn l10nregistry_instance_get() -> *const GeckoL10nRegistry { + let reg = get_l10n_registry(); + Rc::into_raw(reg) +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_get_parent_process_sources( + sources: &mut ThinVec<L10nFileSourceDescriptor>, +) { + debug_assert!( + is_parent_process(), + "This should be called only in parent process." + ); + + // If at the point when the first content process is being initialized, the parent + // process `L10nRegistryService` has not been initialized yet, this will trigger it. + // + // This is architecturally imperfect, but acceptable for simplicity reasons because + // `L10nRegistry` instance is cheap and mainly servers as a store of state. + let reg = get_l10n_registry(); + for name in reg.get_source_names().unwrap() { + let source = reg.file_source_by_name(&name).unwrap().unwrap(); + let descriptor = L10nFileSourceDescriptor { + name: source.name.as_str().into(), + metasource: source.metasource.as_str().into(), + locales: source + .locales() + .iter() + .map(|l| l.to_string().into()) + .collect(), + pre_path: source.pre_path.as_str().into(), + index: source + .get_index() + .map(|index| index.into_iter().map(|s| s.into()).collect()) + .unwrap_or_default(), + }; + sources.push(descriptor); + } +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_register_parent_process_sources( + sources: &ThinVec<L10nFileSourceDescriptor>, +) { + debug_assert!( + !is_parent_process(), + "This should be called only in content process." + ); + set_l10n_registry(sources); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_addref(reg: *const GeckoL10nRegistry) { + let raw = Rc::from_raw(reg); + mem::forget(Rc::clone(&raw)); + mem::forget(raw); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_release(reg: *const GeckoL10nRegistry) { + let _ = Rc::from_raw(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_get_available_locales( + reg: &GeckoL10nRegistry, + result: &mut ThinVec<nsCString>, +) { + if let Ok(locales) = reg.get_available_locales().report_error() { + result.extend(locales.into_iter().map(|locale| locale.to_string().into())); + } +} + +fn broadcast_settings_if_parent(reg: &GeckoL10nRegistry) { + if !is_parent_process() { + return; + } + + L10N_REGISTRY.with(|reg_service| { + if std::ptr::eq(Rc::as_ptr(reg_service), reg) { + let locales = reg + .get_available_locales() + .unwrap() + .iter() + .map(|loc| loc.to_string().into()) + .collect(); + + unsafe { + crate::xpcom_utils::set_available_locales(&locales); + L10nRegistrySendUpdateL10nFileSources(); + } + } + }); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_register_sources( + reg: &GeckoL10nRegistry, + sources: &ThinVec<&FileSource>, +) { + let _ = reg + .register_sources(sources.iter().map(|&s| s.clone()).collect()) + .report_error(); + + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_update_sources( + reg: &GeckoL10nRegistry, + sources: &mut ThinVec<&FileSource>, +) { + let _ = reg + .update_sources(sources.iter().map(|&s| s.clone()).collect()) + .report_error(); + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_remove_sources( + reg: &GeckoL10nRegistry, + sources_elements: *const nsCString, + sources_length: usize, +) { + if sources_elements.is_null() { + return; + } + + let sources = std::slice::from_raw_parts(sources_elements, sources_length); + let _ = reg.remove_sources(sources.to_vec()).report_error(); + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_has_source( + reg: &GeckoL10nRegistry, + name: &nsACString, + status: &mut L10nRegistryStatus, +) -> bool { + if name.is_empty() { + *status = L10nRegistryStatus::EmptyName; + return false; + } + *status = L10nRegistryStatus::None; + reg.has_source(&name.to_utf8()) + .report_error() + .unwrap_or(false) +} + +#[no_mangle] +pub extern "C" fn l10nregistry_get_source( + reg: &GeckoL10nRegistry, + name: &nsACString, + status: &mut L10nRegistryStatus, +) -> *mut FileSource { + if name.is_empty() { + *status = L10nRegistryStatus::EmptyName; + return std::ptr::null_mut(); + } + + *status = L10nRegistryStatus::None; + + if let Ok(Some(source)) = reg.file_source_by_name(&name.to_utf8()).report_error() { + Box::into_raw(Box::new(source)) + } else { + std::ptr::null_mut() + } +} + +#[no_mangle] +pub extern "C" fn l10nregistry_clear_sources(reg: &GeckoL10nRegistry) { + let _ = reg.clear_sources().report_error(); + + broadcast_settings_if_parent(reg); +} + +#[no_mangle] +pub extern "C" fn l10nregistry_get_source_names( + reg: &GeckoL10nRegistry, + result: &mut ThinVec<nsCString>, +) { + if let Ok(names) = reg.get_source_names().report_error() { + result.extend(names.into_iter().map(|name| nsCString::from(name))); + } +} + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_generate_bundles_sync( + reg: &GeckoL10nRegistry, + locales_elements: *const nsCString, + locales_length: usize, + res_ids_elements: *const GeckoResourceId, + res_ids_length: usize, + status: &mut L10nRegistryStatus, +) -> *mut GeckoFluentBundleIterator { + let locales = std::slice::from_raw_parts(locales_elements, locales_length); + let res_ids = std::slice::from_raw_parts(res_ids_elements, res_ids_length) + .into_iter() + .map(ResourceId::from) + .collect(); + let locales: Result<Vec<LanguageIdentifier>, _> = + locales.into_iter().map(|s| s.to_utf8().parse()).collect(); + + match locales { + Ok(locales) => { + *status = L10nRegistryStatus::None; + let iter = reg.bundles_iter(locales.into_iter(), res_ids); + Box::into_raw(Box::new(iter)) + } + Err(_) => { + *status = L10nRegistryStatus::InvalidLocaleCode; + std::ptr::null_mut() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_iterator_destroy(iter: *mut GeckoFluentBundleIterator) { + let _ = Box::from_raw(iter); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_iterator_next( + iter: &mut GeckoFluentBundleIterator, +) -> *mut FluentBundleRc { + if let Some(Ok(result)) = iter.next() { + Box::into_raw(Box::new(result)) + } else { + std::ptr::null_mut() + } +} + +pub struct NextRequest { + promise: RefPtr<xpcom::Promise>, + // Ownership is transferred here. + callback: unsafe extern "C" fn(&xpcom::Promise, *mut FluentBundleRc), +} + +pub struct GeckoFluentBundleAsyncIteratorWrapper(UnboundedSender<NextRequest>); + +#[no_mangle] +pub unsafe extern "C" fn l10nregistry_generate_bundles( + reg: &GeckoL10nRegistry, + locales_elements: *const nsCString, + locales_length: usize, + res_ids_elements: *const GeckoResourceId, + res_ids_length: usize, + status: &mut L10nRegistryStatus, +) -> *mut GeckoFluentBundleAsyncIteratorWrapper { + let locales = std::slice::from_raw_parts(locales_elements, locales_length); + let res_ids = std::slice::from_raw_parts(res_ids_elements, res_ids_length) + .into_iter() + .map(ResourceId::from) + .collect(); + let locales: Result<Vec<LanguageIdentifier>, _> = + locales.into_iter().map(|s| s.to_utf8().parse()).collect(); + + match locales { + Ok(locales) => { + *status = L10nRegistryStatus::None; + let mut iter = reg.bundles_stream(locales.into_iter(), res_ids); + + // Immediately spawn the task which will handle the async calls, and use an `UnboundedSender` + // to send callbacks for specific `next()` calls to it. + let (sender, mut receiver) = unbounded::<NextRequest>(); + moz_task::spawn_local("l10nregistry_generate_bundles", async move { + use futures::StreamExt; + while let Some(req) = receiver.next().await { + let result = match iter.next().await { + Some(Ok(result)) => Box::into_raw(Box::new(result)), + _ => std::ptr::null_mut(), + }; + (req.callback)(&req.promise, result); + } + }) + .detach(); + let iter = GeckoFluentBundleAsyncIteratorWrapper(sender); + Box::into_raw(Box::new(iter)) + } + Err(_) => { + *status = L10nRegistryStatus::InvalidLocaleCode; + std::ptr::null_mut() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn fluent_bundle_async_iterator_destroy( + iter: *mut GeckoFluentBundleAsyncIteratorWrapper, +) { + let _ = Box::from_raw(iter); +} + +#[no_mangle] +pub extern "C" fn fluent_bundle_async_iterator_next( + iter: &GeckoFluentBundleAsyncIteratorWrapper, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, *mut FluentBundleRc), +) { + if iter + .0 + .unbounded_send(NextRequest { + promise: RefPtr::new(promise), + callback, + }) + .is_err() + { + callback(promise, std::ptr::null_mut()); + } +} + +extern "C" { + pub fn L10nRegistrySendUpdateL10nFileSources(); +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/source.rs b/intl/l10n/rust/l10nregistry-ffi/src/source.rs new file mode 100644 index 0000000000..7003d8da97 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/source.rs @@ -0,0 +1,359 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use super::fetcher::{GeckoFileFetcher, MockFileFetcher}; +use crate::env::GeckoEnvironment; + +use fluent::FluentResource; +use l10nregistry::source::{FileSource, FileSourceOptions, ResourceOption, ResourceStatus, RcResource}; + +use nsstring::{nsACString, nsCString}; +use thin_vec::ThinVec; +use unic_langid::LanguageIdentifier; + +use std::{borrow::Cow, mem, rc::Rc}; +use xpcom::RefPtr; + +#[repr(C)] +pub enum L10nFileSourceStatus { + None, + EmptyName, + EmptyPrePath, + EmptyResId, + InvalidLocaleCode, +} + +// For historical reasons we maintain a locale in Firefox with a codename `ja-JP-mac`. +// This string is an invalid BCP-47 language tag, so we don't store it in Gecko, which uses +// valid BCP-47 tags only, but rather keep that quirk local to Gecko L10nRegistry file fetcher. +// +// Here, if we encounter `ja-JP-mac` (invalid BCP-47), we swap it for a valid equivalent: `ja-JP-macos`. +// +// See bug 1726586 for details and fetcher::get_locale_for_gecko. +fn get_locale_from_gecko<'s>(input: Cow<'s, str>) -> Cow<'s, str> { + if input == "ja-JP-mac" { + "ja-JP-macos".into() + } else { + input + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_new( + name: &nsACString, + metasource: &nsACString, + locales: &ThinVec<nsCString>, + pre_path: &nsACString, + allow_override: bool, + status: &mut L10nFileSourceStatus, +) -> *const FileSource { + if name.is_empty() { + *status = L10nFileSourceStatus::EmptyName; + return std::ptr::null(); + } + if pre_path.is_empty() { + *status = L10nFileSourceStatus::EmptyPrePath; + return std::ptr::null(); + } + + let locales: Result<Vec<LanguageIdentifier>, _> = locales + .iter() + .map(|l| get_locale_from_gecko(l.to_utf8()).parse()) + .collect(); + + let locales = match locales { + Ok(locales) => locales, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + let mut source = FileSource::new( + name.to_string(), + Some(metasource.to_string()), + locales, + pre_path.to_string(), + FileSourceOptions { allow_override }, + GeckoFileFetcher, + ); + source.set_reporter(GeckoEnvironment::new(None)); + + *status = L10nFileSourceStatus::None; + Rc::into_raw(Rc::new(source)) +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_new_with_index( + name: &nsACString, + metasource: &nsACString, + locales: &ThinVec<nsCString>, + pre_path: &nsACString, + index_elements: *const nsCString, + index_length: usize, + allow_override: bool, + status: &mut L10nFileSourceStatus, +) -> *const FileSource { + if name.is_empty() { + *status = L10nFileSourceStatus::EmptyName; + return std::ptr::null(); + } + if pre_path.is_empty() { + *status = L10nFileSourceStatus::EmptyPrePath; + return std::ptr::null(); + } + + let locales: Result<Vec<LanguageIdentifier>, _> = locales + .iter() + .map(|l| get_locale_from_gecko(l.to_utf8()).parse()) + .collect(); + + let index = if index_length > 0 { + assert!(!index_elements.is_null()); + std::slice::from_raw_parts(index_elements, index_length) + } else { + &[] + } + .into_iter() + .map(|s| s.to_string()) + .collect(); + + let locales = match locales { + Ok(locales) => locales, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + let mut source = FileSource::new_with_index( + name.to_string(), + Some(metasource.to_string()), + locales, + pre_path.to_string(), + FileSourceOptions { allow_override }, + GeckoFileFetcher, + index, + ); + source.set_reporter(GeckoEnvironment::new(None)); + + *status = L10nFileSourceStatus::None; + Rc::into_raw(Rc::new(source)) +} + +#[repr(C)] +pub struct L10nFileSourceMockFile { + path: nsCString, + source: nsCString, +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_new_mock( + name: &nsACString, + metasource: &nsACString, + locales: &ThinVec<nsCString>, + pre_path: &nsACString, + fs: &ThinVec<L10nFileSourceMockFile>, + status: &mut L10nFileSourceStatus, +) -> *const FileSource { + if name.is_empty() { + *status = L10nFileSourceStatus::EmptyName; + return std::ptr::null(); + } + if pre_path.is_empty() { + *status = L10nFileSourceStatus::EmptyPrePath; + return std::ptr::null(); + } + + let locales: Result<Vec<LanguageIdentifier>, _> = locales + .iter() + .map(|l| get_locale_from_gecko(l.to_utf8()).parse()) + .collect(); + + let locales = match locales { + Ok(locales) => locales, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + let fs = fs + .iter() + .map(|mock| (mock.path.to_string(), mock.source.to_string())) + .collect(); + let fetcher = MockFileFetcher::new(fs); + let mut source = FileSource::new( + name.to_string(), + Some(metasource.to_string()), + locales, + pre_path.to_string(), + Default::default(), + fetcher, + ); + source.set_reporter(GeckoEnvironment::new(None)); + + *status = L10nFileSourceStatus::None; + Rc::into_raw(Rc::new(source)) +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_addref(source: *const FileSource) { + let raw = Rc::from_raw(source); + mem::forget(Rc::clone(&raw)); + mem::forget(raw); +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_release(source: *const FileSource) { + let _ = Rc::from_raw(source); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_name(source: &FileSource, ret_val: &mut nsACString) { + ret_val.assign(&source.name); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_metasource(source: &FileSource, ret_val: &mut nsACString) { + ret_val.assign(&source.metasource); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_locales( + source: &FileSource, + ret_val: &mut ThinVec<nsCString>, +) { + for locale in source.locales() { + ret_val.push(locale.to_string().into()); + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_prepath(source: &FileSource, ret_val: &mut nsACString) { + ret_val.assign(&source.pre_path); +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_get_index( + source: &FileSource, + ret_val: &mut ThinVec<nsCString>, +) -> bool { + if let Some(index) = source.get_index() { + for entry in index { + ret_val.push(entry.to_string().into()); + } + true + } else { + false + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_has_file( + source: &FileSource, + locale: &nsACString, + path: &nsACString, + status: &mut L10nFileSourceStatus, + present: &mut bool, +) -> bool { + if path.is_empty() { + *status = L10nFileSourceStatus::EmptyResId; + return false; + } + + let locale = match locale.to_utf8().parse() { + Ok(locale) => locale, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return false; + } + }; + + *status = L10nFileSourceStatus::None; + // To work around Option<bool> we return bool for the option, + // and the `present` argument is the value of it. + if let Some(val) = source.has_file(&locale, &path.to_utf8().into()) { + *present = val; + true + } else { + false + } +} + +#[no_mangle] +pub extern "C" fn l10nfilesource_fetch_file_sync( + source: &FileSource, + locale: &nsACString, + path: &nsACString, + status: &mut L10nFileSourceStatus, +) -> *const FluentResource { + if path.is_empty() { + *status = L10nFileSourceStatus::EmptyResId; + return std::ptr::null(); + } + + let locale = match locale.to_utf8().parse() { + Ok(locale) => locale, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return std::ptr::null(); + } + }; + + *status = L10nFileSourceStatus::None; + //XXX: Bug 1723191 - if we encounter a request for sync load while async load is in progress + // we will discard the async load and force the sync load instead. + // There may be a better option but we haven't had time to explore it. + if let ResourceOption::Some(res) = + source.fetch_file_sync(&locale, &path.to_utf8().into(), /* overload */ true) + { + Rc::into_raw(res) + } else { + std::ptr::null() + } +} + +#[no_mangle] +pub unsafe extern "C" fn l10nfilesource_fetch_file( + source: &FileSource, + locale: &nsACString, + path: &nsACString, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, Option<&FluentResource>), + status: &mut L10nFileSourceStatus, +) { + if path.is_empty() { + *status = L10nFileSourceStatus::EmptyResId; + return; + } + + let locale = match locale.to_utf8().parse() { + Ok(locale) => locale, + Err(..) => { + *status = L10nFileSourceStatus::InvalidLocaleCode; + return; + } + }; + + *status = L10nFileSourceStatus::None; + + let path = path.to_utf8().into(); + + match source.fetch_file(&locale, &path) { + ResourceStatus::MissingOptional => callback(promise, None), + ResourceStatus::MissingRequired => callback(promise, None), + ResourceStatus::Loaded(res) => callback(promise, Some(&res)), + res @ ResourceStatus::Loading(_) => { + let strong_promise = RefPtr::new(promise); + moz_task::spawn_local("l10nfilesource_fetch_file", async move { + callback( + &strong_promise, + Option::<RcResource>::from(res.await).as_ref().map(|r| &**r), + ); + }) + .detach(); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs b/intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs new file mode 100644 index 0000000000..7c83088075 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs @@ -0,0 +1,129 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +use cstr::cstr; +use nsstring::{nsACString, nsCString}; +use std::marker::PhantomData; +use thin_vec::ThinVec; +use xpcom::{ + get_service, getter_addrefs, + interfaces::{ + mozILocaleService, nsICategoryEntry, nsICategoryManager, nsISimpleEnumerator, nsIXULRuntime, + }, + RefPtr, XpCom, +}; + +pub struct IterSimpleEnumerator<T> { + enumerator: RefPtr<nsISimpleEnumerator>, + phantom: PhantomData<T>, +} + +impl<T: XpCom> IterSimpleEnumerator<T> { + /// Convert a `nsISimpleEnumerator` into a rust `Iterator` type. + pub fn new(enumerator: RefPtr<nsISimpleEnumerator>) -> Self { + IterSimpleEnumerator { + enumerator, + phantom: PhantomData, + } + } +} + +impl<T: XpCom + 'static> Iterator for IterSimpleEnumerator<T> { + type Item = RefPtr<T>; + + fn next(&mut self) -> Option<Self::Item> { + let mut more = false; + unsafe { + self.enumerator + .HasMoreElements(&mut more) + .to_result() + .ok()? + } + if !more { + return None; + } + + let element = getter_addrefs(|p| unsafe { self.enumerator.GetNext(p) }).ok()?; + element.query_interface::<T>() + } +} + +fn process_type() -> u32 { + if let Ok(appinfo) = xpcom::components::XULRuntime::service::<nsIXULRuntime>() { + let mut process_type = nsIXULRuntime::PROCESS_TYPE_DEFAULT; + if unsafe { appinfo.GetProcessType(&mut process_type).succeeded() } { + return process_type; + } + } + nsIXULRuntime::PROCESS_TYPE_DEFAULT +} + +pub fn is_parent_process() -> bool { + process_type() == nsIXULRuntime::PROCESS_TYPE_DEFAULT +} + +pub fn get_packaged_locales() -> Option<ThinVec<nsCString>> { + let locale_service = + get_service::<mozILocaleService>(cstr!("@mozilla.org/intl/localeservice;1"))?; + let mut locales = ThinVec::new(); + unsafe { + locale_service + .GetPackagedLocales(&mut locales) + .to_result() + .ok()?; + } + Some(locales) +} + +pub fn get_app_locales() -> Option<ThinVec<nsCString>> { + let locale_service = + get_service::<mozILocaleService>(cstr!("@mozilla.org/intl/localeservice;1"))?; + let mut locales = ThinVec::new(); + unsafe { + locale_service + .GetAppLocalesAsBCP47(&mut locales) + .to_result() + .ok()?; + } + Some(locales) +} + +pub fn set_available_locales(locales: &ThinVec<nsCString>) { + let locale_service = + get_service::<mozILocaleService>(cstr!("@mozilla.org/intl/localeservice;1")) + .expect("Failed to get a service."); + unsafe { + locale_service + .SetAvailableLocales(locales) + .to_result() + .expect("Failed to set locales."); + } +} + +pub struct CategoryEntry { + pub entry: nsCString, + pub value: nsCString, +} + +pub fn get_category_entries(category: &nsACString) -> Option<Vec<CategoryEntry>> { + let category_manager = + get_service::<nsICategoryManager>(cstr!("@mozilla.org/categorymanager;1"))?; + + let enumerator = + getter_addrefs(|p| unsafe { category_manager.EnumerateCategory(category, p) }).ok()?; + + Some( + IterSimpleEnumerator::<nsICategoryEntry>::new(enumerator) + .map(|ientry| { + let mut entry = nsCString::new(); + let mut value = nsCString::new(); + unsafe { + let _ = ientry.GetEntry(&mut *entry); + let _ = ientry.GetValue(&mut *value); + } + CategoryEntry { entry, value } + }) + .collect(), + ) +} diff --git a/intl/l10n/rust/l10nregistry-rs/.gitignore b/intl/l10n/rust/l10nregistry-rs/.gitignore new file mode 100644 index 0000000000..96ef6c0b94 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/intl/l10n/rust/l10nregistry-rs/Cargo.toml b/intl/l10n/rust/l10nregistry-rs/Cargo.toml new file mode 100644 index 0000000000..6a567d1dcd --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/Cargo.toml @@ -0,0 +1,87 @@ +[package] +name = "l10nregistry" +version = "0.3.0" +authors = ["Zibi Braniecki <gandalf@mozilla.com>"] +license = "Apache-2.0/MIT" +edition = "2018" + +[dependencies] +async-trait = "0.1" +fluent-bundle = "0.15.2" +fluent-fallback = "0.7.0" +fluent-testing = { version = "0.0.3", optional = true, features = ["sync", "async"] } +futures = "0.3" +pin-project-lite = "0.2" +replace_with = "0.1" +rustc-hash = "1" +tokio = { version = "1.0", optional = true, features = ["rt-multi-thread", "macros"] } +unic-langid = "0.9" + +[dev-dependencies] +unic-langid = { version = "0.9", features = ["macros"] } +# Serial_test is only used in a test that is behind the test-fluent feature +# and this not run on gecko CI. However it has a dependency on syn 1 that is +# best avoided as we transition to syn 2, and while newer versions of +# serial_test updated to syn 2, they also bring a large dashmap dependency. +#serial_test = "0.6" +# The mozilla-central vendoring process unfortunately always pulls in +# dev-dependencies. The criterion package pulls in duplicated versions of +# other crates as well as large data files that the vendoring process is +# unhappy with. This can be uncommented to run benchmarks, but we should +# not vendor it into the tree in its current state. +#criterion = "0.3" + +[features] +default = [] +tokio-io = ["tokio"] +test-fluent = [] + +[[bench]] +name = "preferences" +harness = false +required-features = ["tokio", "test-fluent"] + +[[bench]] +name = "localization" +harness = false +required-features = ["tokio", "test-fluent"] + +[[bench]] +name = "source" +harness = false +required-features = ["tokio", "test-fluent"] + +[[bench]] +name = "solver" +harness = false +required-features = ["tokio", "test-fluent"] + +[[bench]] +name = "registry" +harness = false +required-features = ["tokio", "test-fluent"] + +[[test]] +name = "source" +path = "tests/source.rs" +required-features = ["tokio", "test-fluent"] + +[[test]] +name = "registry" +path = "tests/registry.rs" +required-features = ["tokio", "test-fluent"] + +[[test]] +name = "localization" +path = "tests/localization.rs" +required-features = ["tokio", "test-fluent"] + +[[test]] +name = "scenarios_sync" +path = "tests/scenarios_sync.rs" +required-features = ["test-fluent"] + +[[test]] +name = "scenarios_async" +path = "tests/scenarios_async.rs" +required-features = ["tokio", "test-fluent"] diff --git a/intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE b/intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE new file mode 100644 index 0000000000..35582f166b --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Mozilla + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/intl/l10n/rust/l10nregistry-rs/LICENSE-MIT b/intl/l10n/rust/l10nregistry-rs/LICENSE-MIT new file mode 100644 index 0000000000..5655fa311c --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/LICENSE-MIT @@ -0,0 +1,19 @@ +Copyright 2017 Mozilla + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/intl/l10n/rust/l10nregistry-rs/README.md b/intl/l10n/rust/l10nregistry-rs/README.md new file mode 100644 index 0000000000..873555df89 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/README.md @@ -0,0 +1,17 @@ +# l10nregistry-rs + +The `L10nRegistry` is responsible for taking `FileSources` across the app, and turning them into bundles. It is hooked into the `L10nRegistry` global available from privileged JavaScript. See the [L10nRegistry.webidl](https://searchfox.org/mozilla-central/source/dom/chrome-webidl/L10nRegistry.webidl#100) for detailed information about this API, and `intl/l10n/test/test_l10nregistry.js` for integration tests with examples of how it can be used. + +## Testing + +Tests can be run directly from this directory via: + +``` +cargo test --all-features +``` + +Benchmarks are also available. First uncomment the `criterion` dependency in the `Cargo.toml` and then run. + +``` +cargo test bench --all-features +``` diff --git a/intl/l10n/rust/l10nregistry-rs/benches/localization.rs b/intl/l10n/rust/l10nregistry-rs/benches/localization.rs new file mode 100644 index 0000000000..696cc244ed --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/benches/localization.rs @@ -0,0 +1,70 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use fluent_bundle::FluentArgs; +use fluent_fallback::{types::L10nKey, Localization}; +use fluent_testing::get_scenarios; +use l10nregistry::testing::TestFileFetcher; + +fn preferences_bench(c: &mut Criterion) { + let fetcher = TestFileFetcher::new(); + + let mut group = c.benchmark_group("localization/scenarios"); + + for scenario in get_scenarios() { + let res_ids = scenario.res_ids.clone(); + let l10n_keys: Vec<(String, Option<FluentArgs>)> = scenario + .queries + .iter() + .map(|q| { + ( + q.input.id.clone(), + q.input.args.as_ref().map(|args| { + let mut result = FluentArgs::new(); + for arg in args.as_slice() { + result.set(arg.id.clone(), arg.value.clone()); + } + result + }), + ) + }) + .collect(); + + group.bench_function(format!("{}/format_value_sync", scenario.name), |b| { + b.iter(|| { + let (env, reg) = fetcher.get_registry_and_environment(&scenario); + let mut errors = vec![]; + + let loc = Localization::with_env(res_ids.clone(), true, env.clone(), reg.clone()); + let bundles = loc.bundles(); + + for key in l10n_keys.iter() { + bundles.format_value_sync(&key.0, key.1.as_ref(), &mut errors); + } + }) + }); + + let keys: Vec<L10nKey> = l10n_keys + .into_iter() + .map(|key| L10nKey { + id: key.0.into(), + args: key.1, + }) + .collect(); + group.bench_function(format!("{}/format_messages_sync", scenario.name), |b| { + b.iter(|| { + let (env, reg) = fetcher.get_registry_and_environment(&scenario); + let mut errors = vec![]; + let loc = Localization::with_env(res_ids.clone(), true, env.clone(), reg.clone()); + let bundles = loc.bundles(); + bundles.format_messages_sync(&keys, &mut errors); + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, preferences_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-rs/benches/preferences.rs b/intl/l10n/rust/l10nregistry-rs/benches/preferences.rs new file mode 100644 index 0000000000..e4405c6537 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/benches/preferences.rs @@ -0,0 +1,65 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use fluent_testing::get_scenarios; +use l10nregistry::testing::TestFileFetcher; + +use unic_langid::LanguageIdentifier; + +fn preferences_bench(c: &mut Criterion) { + let fetcher = TestFileFetcher::new(); + + let mut group = c.benchmark_group("registry/scenarios"); + + for scenario in get_scenarios() { + let res_ids = scenario.res_ids.clone(); + + let locales: Vec<LanguageIdentifier> = scenario + .locales + .iter() + .map(|l| l.parse().unwrap()) + .collect(); + + group.bench_function(format!("{}/sync/first_bundle", scenario.name), |b| { + b.iter(|| { + let reg = fetcher.get_registry(&scenario); + let mut bundles = + reg.generate_bundles_sync(locales.clone().into_iter(), res_ids.clone()); + for _ in 0..locales.len() { + if bundles.next().is_some() { + break; + } + } + }) + }); + + #[cfg(feature = "tokio")] + { + use futures::stream::StreamExt; + + let rt = tokio::runtime::Runtime::new().unwrap(); + + group.bench_function(&format!("{}/async/first_bundle", scenario.name), |b| { + b.iter(|| { + rt.block_on(async { + let reg = fetcher.get_registry(&scenario); + + let mut bundles = + reg.generate_bundles(locales.clone().into_iter(), res_ids.clone()); + for _ in 0..locales.len() { + if bundles.next().await.is_some() { + break; + } + } + }); + }) + }); + } + } + + group.finish(); +} + +criterion_group!(benches, preferences_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-rs/benches/registry.rs b/intl/l10n/rust/l10nregistry-rs/benches/registry.rs new file mode 100644 index 0000000000..959b510943 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/benches/registry.rs @@ -0,0 +1,133 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use futures::stream::StreamExt; +use l10nregistry::source::ResourceId; +use l10nregistry::testing::{FileSource, RegistrySetup, TestFileFetcher}; +use unic_langid::LanguageIdentifier; + +fn get_paths() -> Vec<ResourceId> { + let paths: Vec<&'static str> = vec![ + "branding/brand.ftl", + "browser/sanitize.ftl", + "browser/preferences/blocklists.ftl", + "browser/preferences/colors.ftl", + "browser/preferences/selectBookmark.ftl", + "browser/preferences/connection.ftl", + "browser/preferences/addEngine.ftl", + "browser/preferences/siteDataSettings.ftl", + "browser/preferences/fonts.ftl", + "browser/preferences/languages.ftl", + "browser/preferences/preferences.ftl", + "security/certificates/certManager.ftl", + "security/certificates/deviceManager.ftl", + "toolkit/global/textActions.ftl", + "toolkit/printing/printUI.ftl", + "toolkit/updates/history.ftl", + "toolkit/featuregates/features.ftl", + ]; + + paths.into_iter().map(ResourceId::from).collect() +} + +fn registry_bench(c: &mut Criterion) { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let mut group = c.benchmark_group("non-metasource"); + + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + group.bench_function(&format!("serial",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), get_paths()); + while let Some(_) = i.next() {} + }) + }); + + let rt = tokio::runtime::Runtime::new().unwrap(); + group.bench_function(&format!("parallel",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles(lang_ids.into_iter(), get_paths()); + rt.block_on(async { while let Some(_) = i.next().await {} }); + }) + }); + + group.finish(); +} + +fn registry_metasource_bench(c: &mut Criterion) { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let mut group = c.benchmark_group("metasource"); + + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + Some("app"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("app"), + vec![en_us.clone()], + "browser/{locale}/", + ), + FileSource::new( + "toolkit", + Some("langpack"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("langpack"), + vec![en_us.clone()], + "browser/{locale}/", + ), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + group.bench_function(&format!("serial",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), get_paths()); + while let Some(_) = i.next() {} + }) + }); + + let rt = tokio::runtime::Runtime::new().unwrap(); + group.bench_function(&format!("parallel",), |b| { + b.iter(|| { + let lang_ids = vec![en_us.clone()]; + let mut i = reg.generate_bundles(lang_ids.into_iter(), get_paths()); + rt.block_on(async { while let Some(_) = i.next().await {} }); + }) + }); + + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(10); + targets = registry_bench, registry_metasource_bench +); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-rs/benches/solver.rs b/intl/l10n/rust/l10nregistry-rs/benches/solver.rs new file mode 100644 index 0000000000..25906fa7c0 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/benches/solver.rs @@ -0,0 +1,120 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use futures::stream::Collect; +use futures::stream::FuturesOrdered; +use futures::StreamExt; +use l10nregistry::solver::testing::get_scenarios; +use l10nregistry::solver::{AsyncTester, ParallelProblemSolver, SerialProblemSolver, SyncTester}; +use std::future::Future; +use std::pin::Pin; +use std::task::{Context, Poll}; + +pub struct MockTester { + values: Vec<Vec<bool>>, +} + +impl SyncTester for MockTester { + fn test_sync(&self, res_idx: usize, source_idx: usize) -> bool { + self.values[res_idx][source_idx] + } +} + +pub struct SingleTestResult(bool); + +impl Future for SingleTestResult { + type Output = bool; + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<Self::Output> { + self.0.into() + } +} + +pub type ResourceSetStream = Collect<FuturesOrdered<SingleTestResult>, Vec<bool>>; +pub struct TestResult(ResourceSetStream); + +impl std::marker::Unpin for TestResult {} + +impl Future for TestResult { + type Output = Vec<bool>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + let pinned = Pin::new(&mut self.0); + pinned.poll(cx) + } +} + +impl AsyncTester for MockTester { + type Result = TestResult; + + fn test_async(&self, query: Vec<(usize, usize)>) -> Self::Result { + let futures = query + .into_iter() + .map(|(res_idx, source_idx)| SingleTestResult(self.test_sync(res_idx, source_idx))) + .collect::<Vec<_>>(); + TestResult(futures.into_iter().collect::<FuturesOrdered<_>>().collect()) + } +} + +struct TestStream<'t> { + solver: ParallelProblemSolver<MockTester>, + tester: &'t MockTester, +} + +impl<'t> TestStream<'t> { + pub fn new(solver: ParallelProblemSolver<MockTester>, tester: &'t MockTester) -> Self { + Self { solver, tester } + } +} + +impl<'t> futures::stream::Stream for TestStream<'t> { + type Item = Vec<usize>; + + fn poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + ) -> std::task::Poll<Option<Self::Item>> { + let tester = self.tester; + let solver = &mut self.solver; + let pinned = std::pin::Pin::new(solver); + pinned + .try_poll_next(cx, tester, false) + .map(|v| v.ok().flatten()) + } +} + +fn solver_bench(c: &mut Criterion) { + let scenarios = get_scenarios(); + + let mut group = c.benchmark_group("solver"); + + for scenario in scenarios { + let tester = MockTester { + values: scenario.values.clone(), + }; + + group.bench_function(&format!("serial/{}", &scenario.name), |b| { + b.iter(|| { + let mut gen = SerialProblemSolver::new(scenario.width, scenario.depth); + while let Ok(Some(_)) = gen.try_next(&tester, false) {} + }) + }); + + { + let rt = tokio::runtime::Runtime::new().unwrap(); + + group.bench_function(&format!("parallel/{}", &scenario.name), |b| { + b.iter(|| { + let gen = ParallelProblemSolver::new(scenario.width, scenario.depth); + let mut t = TestStream::new(gen, &tester); + rt.block_on(async { while let Some(_) = t.next().await {} }); + }) + }); + } + } + group.finish(); +} + +criterion_group!(benches, solver_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-rs/benches/source.rs b/intl/l10n/rust/l10nregistry-rs/benches/source.rs new file mode 100644 index 0000000000..35668781da --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/benches/source.rs @@ -0,0 +1,60 @@ +use criterion::criterion_group; +use criterion::criterion_main; +use criterion::Criterion; + +use fluent_testing::get_scenarios; +use l10nregistry::testing::TestFileFetcher; + +use unic_langid::LanguageIdentifier; + +fn get_locales<S>(input: &[S]) -> Vec<LanguageIdentifier> +where + S: AsRef<str>, +{ + input.iter().map(|s| s.as_ref().parse().unwrap()).collect() +} + +fn source_bench(c: &mut Criterion) { + let fetcher = TestFileFetcher::new(); + + let mut group = c.benchmark_group("source/scenarios"); + + for scenario in get_scenarios() { + let res_ids = scenario.res_ids.clone(); + + let locales: Vec<LanguageIdentifier> = get_locales(&scenario.locales); + + let sources: Vec<_> = scenario + .file_sources + .iter() + .map(|s| { + fetcher.get_test_file_source(&s.name, None, get_locales(&s.locales), &s.path_scheme) + }) + .collect(); + + group.bench_function(format!("{}/has_file", scenario.name), |b| { + b.iter(|| { + for source in &sources { + for res_id in &res_ids { + source.has_file(&locales[0], &res_id); + } + } + }) + }); + + group.bench_function(format!("{}/sync/fetch_file_sync", scenario.name), |b| { + b.iter(|| { + for source in &sources { + for res_id in &res_ids { + source.fetch_file_sync(&locales[0], &res_id, false); + } + } + }) + }); + } + + group.finish(); +} + +criterion_group!(benches, source_bench); +criterion_main!(benches); diff --git a/intl/l10n/rust/l10nregistry-rs/src/env.rs b/intl/l10n/rust/l10nregistry-rs/src/env.rs new file mode 100644 index 0000000000..7cd1ff30f4 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/env.rs @@ -0,0 +1,5 @@ +use crate::errors::L10nRegistryError; + +pub trait ErrorReporter { + fn report_errors(&self, errors: Vec<L10nRegistryError>); +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/errors.rs b/intl/l10n/rust/l10nregistry-rs/src/errors.rs new file mode 100644 index 0000000000..d58f02ea8e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/errors.rs @@ -0,0 +1,74 @@ +use fluent_bundle::FluentError; +use fluent_fallback::types::ResourceId; +use std::error::Error; +use unic_langid::LanguageIdentifier; + +#[derive(Debug, Clone, PartialEq)] +pub enum L10nRegistryError { + FluentError { + resource_id: ResourceId, + loc: Option<(usize, usize)>, + error: FluentError, + }, + MissingResource { + locale: LanguageIdentifier, + resource_id: ResourceId, + }, +} + +impl std::fmt::Display for L10nRegistryError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::MissingResource { + locale, + resource_id, + } => { + write!( + f, + "Missing resource in locale {}: {}", + locale, resource_id.value + ) + } + Self::FluentError { + resource_id, + loc, + error, + } => { + if let Some(loc) = loc { + write!( + f, + "Fluent Error in {}[line: {}, col: {}]: {}", + resource_id.value, loc.0, loc.1, error + ) + } else { + write!(f, "Fluent Error in {}: {}", resource_id.value, error) + } + } + } + } +} + +impl Error for L10nRegistryError {} + +#[derive(Debug, Clone, PartialEq)] +pub enum L10nRegistrySetupError { + RegistryLocked, + DuplicatedSource { name: String }, + MissingSource { name: String }, +} + +impl std::fmt::Display for L10nRegistrySetupError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::RegistryLocked => write!(f, "Can't modify a registry when locked."), + Self::DuplicatedSource { name } => { + write!(f, "Source with a name {} is already registered.", &name) + } + Self::MissingSource { name } => { + write!(f, "Cannot find a source with a name {}.", &name) + } + } + } +} + +impl Error for L10nRegistrySetupError {} diff --git a/intl/l10n/rust/l10nregistry-rs/src/fluent.rs b/intl/l10n/rust/l10nregistry-rs/src/fluent.rs new file mode 100644 index 0000000000..b6ac2a12ab --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/fluent.rs @@ -0,0 +1,5 @@ +use fluent_bundle::FluentBundle as FluentBundleBase; +pub use fluent_bundle::{FluentError, FluentResource}; +use std::rc::Rc; + +pub type FluentBundle = FluentBundleBase<Rc<FluentResource>>; diff --git a/intl/l10n/rust/l10nregistry-rs/src/lib.rs b/intl/l10n/rust/l10nregistry-rs/src/lib.rs new file mode 100644 index 0000000000..cbd72c09ce --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/lib.rs @@ -0,0 +1,8 @@ +pub mod env; +pub mod errors; +pub mod fluent; +pub mod registry; +pub mod solver; +pub mod source; +#[cfg(feature = "test-fluent")] +pub mod testing; diff --git a/intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs b/intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs new file mode 100644 index 0000000000..bfcff941b5 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs @@ -0,0 +1,294 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +use crate::{ + env::ErrorReporter, + errors::{L10nRegistryError, L10nRegistrySetupError}, + fluent::{FluentBundle, FluentError}, + registry::{BundleAdapter, L10nRegistry, MetaSources}, + solver::{AsyncTester, ParallelProblemSolver}, + source::{ResourceOption, ResourceStatus}, +}; + +use fluent_fallback::{generator::BundleStream, types::ResourceId}; +use futures::{ + stream::{Collect, FuturesOrdered}, + Stream, StreamExt, +}; +use std::future::Future; +use unic_langid::LanguageIdentifier; + +impl<P, B> L10nRegistry<P, B> +where + P: Clone, + B: Clone, +{ + /// This method is useful for testing various configurations. + #[cfg(feature = "test-fluent")] + pub fn generate_bundles_for_lang( + &self, + langid: LanguageIdentifier, + resource_ids: Vec<ResourceId>, + ) -> Result<GenerateBundles<P, B>, L10nRegistrySetupError> { + let lang_ids = vec![langid]; + + Ok(GenerateBundles::new( + self.clone(), + lang_ids.into_iter(), + resource_ids, + // Cheaply create an immutable shallow copy of the [MetaSources]. + self.try_borrow_metasources()?.clone(), + )) + } + + // Asynchronously generate the bundles. + pub fn generate_bundles( + &self, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + ) -> Result<GenerateBundles<P, B>, L10nRegistrySetupError> { + Ok(GenerateBundles::new( + self.clone(), + locales, + resource_ids, + // Cheaply create an immutable shallow copy of the [MetaSources]. + self.try_borrow_metasources()?.clone(), + )) + } +} + +/// This enum contains the various states the [GenerateBundles] can be in during the +/// asynchronous generation step. +enum State<P, B> { + Empty, + Locale(LanguageIdentifier), + Solver { + locale: LanguageIdentifier, + solver: ParallelProblemSolver<GenerateBundles<P, B>>, + }, +} + +impl<P, B> Default for State<P, B> { + fn default() -> Self { + Self::Empty + } +} + +impl<P, B> State<P, B> { + fn get_locale(&self) -> &LanguageIdentifier { + match self { + Self::Locale(locale) => locale, + Self::Solver { locale, .. } => locale, + Self::Empty => unreachable!("Attempting to get a locale for an empty state."), + } + } + + fn take_solver(&mut self) -> ParallelProblemSolver<GenerateBundles<P, B>> { + replace_with::replace_with_or_default_and_return(self, |self_| match self_ { + Self::Solver { locale, solver } => (solver, Self::Locale(locale)), + _ => unreachable!("Attempting to take a solver in an invalid state."), + }) + } + + fn put_back_solver(&mut self, solver: ParallelProblemSolver<GenerateBundles<P, B>>) { + replace_with::replace_with_or_default(self, |self_| match self_ { + Self::Locale(locale) => Self::Solver { locale, solver }, + _ => unreachable!("Attempting to put back a solver in an invalid state."), + }) + } +} + +pub struct GenerateBundles<P, B> { + /// Do not access the metasources in the registry, as they may be mutated between + /// async iterations. + reg: L10nRegistry<P, B>, + /// This is an immutable shallow copy of the MetaSources that should not be mutated + /// during the iteration process. This ensures that the iterator will still be + /// valid if the L10nRegistry is mutated while iterating through the sources. + metasources: MetaSources, + locales: std::vec::IntoIter<LanguageIdentifier>, + current_metasource: usize, + resource_ids: Vec<ResourceId>, + state: State<P, B>, +} + +impl<P, B> GenerateBundles<P, B> { + fn new( + reg: L10nRegistry<P, B>, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + metasources: MetaSources, + ) -> Self { + Self { + reg, + metasources, + locales, + current_metasource: 0, + resource_ids, + state: State::Empty, + } + } +} + +pub type ResourceSetStream = Collect<FuturesOrdered<ResourceStatus>, Vec<ResourceOption>>; +pub struct TestResult(ResourceSetStream); +impl std::marker::Unpin for TestResult {} + +impl Future for TestResult { + type Output = Vec<bool>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> { + let pinned = Pin::new(&mut self.0); + pinned + .poll(cx) + .map(|set| set.iter().map(|c| !c.is_required_and_missing()).collect()) + } +} + +impl<'l, P, B> AsyncTester for GenerateBundles<P, B> { + type Result = TestResult; + + fn test_async(&self, query: Vec<(usize, usize)>) -> Self::Result { + let locale = self.state.get_locale(); + + let stream = query + .iter() + .map(|(res_idx, source_idx)| { + let resource_id = &self.resource_ids[*res_idx]; + self.metasources + .filesource(self.current_metasource, *source_idx) + .fetch_file(locale, resource_id) + }) + .collect::<FuturesOrdered<_>>(); + TestResult(stream.collect::<_>()) + } +} + +#[async_trait::async_trait(?Send)] +impl<P, B> BundleStream for GenerateBundles<P, B> { + async fn prefetch_async(&mut self) { + todo!(); + } +} + +/// Generate [FluentBundles](FluentBundle) asynchronously. +impl<P, B> Stream for GenerateBundles<P, B> +where + P: ErrorReporter, + B: BundleAdapter, +{ + type Item = Result<FluentBundle, (FluentBundle, Vec<FluentError>)>; + + /// Asynchronously try and get a solver, and then with the solver generate a bundle. + /// If the solver is not ready yet, then this function will return as `Pending`, and + /// the Future runner will need to re-enter at a later point to try again. + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> { + if self.metasources.is_empty() { + // There are no metasources available, so no bundles can be generated. + return None.into(); + } + loop { + if let State::Solver { .. } = self.state { + // A solver has already been set up, continue iterating through the + // resources and generating a bundle. + + // Pin the solver so that the async try_poll_next can be called. + let mut solver = self.state.take_solver(); + let pinned_solver = Pin::new(&mut solver); + + if let std::task::Poll::Ready(solver_result) = + pinned_solver.try_poll_next(cx, &self, false) + { + // The solver is ready, but may not have generated an ordering. + + if let Ok(Some(order)) = solver_result { + // The solver resolved an ordering, and a bundle may be able + // to be generated. + + let bundle = self.metasources.bundle_from_order( + self.current_metasource, + self.state.get_locale().clone(), + &order, + &self.resource_ids, + &self.reg.shared.provider, + self.reg.shared.bundle_adapter.as_ref(), + ); + + self.state.put_back_solver(solver); + + if bundle.is_some() { + // The bundle was successfully generated. + return bundle.into(); + } + + // No bundle was generated, continue on. + continue; + } + + // There is no bundle ordering available. + + if self.current_metasource > 0 { + // There are more metasources, create a new solver and try the + // next metasource. If there is an error in the solver_result + // ignore it for now, since there are more metasources. + self.current_metasource -= 1; + let solver = ParallelProblemSolver::new( + self.resource_ids.len(), + self.metasources.get(self.current_metasource).len(), + ); + self.state = State::Solver { + locale: self.state.get_locale().clone(), + solver, + }; + continue; + } + + if let Err(idx) = solver_result { + // Since there are no more metasources, and there is an error, + // report it instead of ignoring it. + self.reg.shared.provider.report_errors(vec![ + L10nRegistryError::MissingResource { + locale: self.state.get_locale().clone(), + resource_id: self.resource_ids[idx].clone(), + }, + ]); + } + + // There are no more metasources. + self.state = State::Empty; + continue; + } + + // The solver is not ready yet, so exit out of this async task + // and mark it as pending. It can be tried again later. + self.state.put_back_solver(solver); + return std::task::Poll::Pending; + } + + // There are no more metasources to search. + + // Try the next locale. + if let Some(locale) = self.locales.next() { + // Restart at the end of the metasources for this locale, and iterate + // backwards. + let last_metasource_idx = self.metasources.len() - 1; + self.current_metasource = last_metasource_idx; + + let solver = ParallelProblemSolver::new( + self.resource_ids.len(), + self.metasources.get(self.current_metasource).len(), + ); + self.state = State::Solver { locale, solver }; + + // Continue iterating on the next solver. + continue; + } + + // There are no more locales or metasources to search. This iterator + // is done. + return None.into(); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs new file mode 100644 index 0000000000..c342aa55aa --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs @@ -0,0 +1,363 @@ +mod asynchronous; +mod synchronous; + +use crate::{ + env::ErrorReporter, + errors::L10nRegistrySetupError, + fluent::FluentBundle, + source::{FileSource, ResourceId}, +}; +use fluent_bundle::FluentResource; +use fluent_fallback::generator::BundleGenerator; +use rustc_hash::FxHashSet; +use std::{ + cell::{Ref, RefCell, RefMut}, + collections::HashSet, + rc::Rc, +}; +use unic_langid::LanguageIdentifier; + +pub use asynchronous::GenerateBundles; +pub use synchronous::GenerateBundlesSync; + +pub type FluentResourceSet = Vec<Rc<FluentResource>>; + +/// The shared information that makes up the configuration the L10nRegistry. It is +/// broken out into a separate struct so that it can be shared via an Rc pointer. +#[derive(Default)] +struct Shared<P, B> { + metasources: RefCell<MetaSources>, + provider: P, + bundle_adapter: Option<B>, +} + +/// [FileSources](FileSource) represent a single directory location to look for .ftl +/// files. These are Stored in a [Vec]. For instance, in a built version of Firefox with +/// the en-US locale, each [FileSource] may represent a different folder with many +/// different files. +/// +/// Firefox supports other *meta sources* for localization files in the form of language +/// packs which can be downloaded from the addon store. These language packs then would +/// be a separate metasource than the app' language. This [MetaSources] adds another [Vec] +/// over the [Vec] of [FileSources](FileSource) in order to provide a unified way to +/// iterate over all possible [FileSource] locations to finally obtain the final bundle. +/// +/// This structure uses an [Rc] to point to the [FileSource] so that a shallow copy +/// of these [FileSources](FileSource) can be obtained for iteration. This makes +/// it quick to copy the list of [MetaSources] for iteration, and guards against +/// invalidating that async nature of iteration when the underlying data mutates. +/// +/// Note that the async iteration of bundles is still only happening in one thread, +/// and is not multi-threaded. The processing is just split over time. +/// +/// The [MetaSources] are ultimately owned by the [Shared] in a [RefCell] so that the +/// source of truth can be mutated, and shallow copies of the [MetaSources] used for +/// iteration will be unaffected. +/// +/// Deriving [Clone] here is a relatively cheap operation, since the [Rc] will be cloned, +/// and point to the original [FileSource]. +#[derive(Default, Clone)] +pub struct MetaSources(Vec<Vec<Rc<FileSource>>>); + +impl MetaSources { + /// Iterate over all FileSources in all MetaSources. + pub fn filesources(&self) -> impl Iterator<Item = &Rc<FileSource>> { + self.0.iter().flatten() + } + + /// Iterate over all FileSources in all MetaSources. + pub fn iter_mut(&mut self) -> impl Iterator<Item = &mut Vec<Rc<FileSource>>> { + self.0.iter_mut() + } + + /// The number of metasources. + pub fn len(&self) -> usize { + self.0.len() + } + + /// If there are no metasources. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Clears out all metasources. + pub fn clear(&mut self) { + self.0.clear(); + } + + /// Clears out only empty metasources. + pub fn clear_empty_metasources(&mut self) { + self.0.retain(|metasource| !metasource.is_empty()); + } + + /// Adds a [FileSource] to its appropriate metasource. + pub fn add_filesource(&mut self, new_source: FileSource) { + if let Some(metasource) = self + .0 + .iter_mut() + .find(|source| source[0].metasource == new_source.metasource) + { + // A metasource was found, add to the existing one. + metasource.push(Rc::new(new_source)); + } else { + // Create a new metasource. + self.0.push(vec![Rc::new(new_source)]); + } + } + + /// Adds a [FileSource] to its appropriate metasource. + pub fn update_filesource(&mut self, new_source: &FileSource) -> bool { + if let Some(metasource) = self + .0 + .iter_mut() + .find(|source| source[0].metasource == new_source.metasource) + { + if let Some(idx) = metasource.iter().position(|source| **source == *new_source) { + *metasource.get_mut(idx).unwrap() = Rc::new(new_source.clone()); + return true; + } + } + false + } + + /// Get a metasource by index, but provide a nice error message if the index + /// is out of bounds. + pub fn get(&self, metasource_idx: usize) -> &Vec<Rc<FileSource>> { + if let Some(metasource) = self.0.get(metasource_idx) { + return &metasource; + } + panic!( + "Metasource index of {} is out of range of the list of {} meta sources.", + metasource_idx, + self.0.len() + ); + } + + /// Get a [FileSource] from a metasource, but provide a nice error message if the + /// index is out of bounds. + pub fn filesource(&self, metasource_idx: usize, filesource_idx: usize) -> &FileSource { + let metasource = self.get(metasource_idx); + let reversed_idx = metasource.len() - 1 - filesource_idx; + if let Some(file_source) = metasource.get(reversed_idx) { + return file_source; + } + panic!( + "File source index of {} is out of range of the list of {} file sources.", + filesource_idx, + metasource.len() + ); + } + + /// Get a [FileSource] by name from a metasource. This is useful for testing. + #[cfg(feature = "test-fluent")] + pub fn file_source_by_name(&self, metasource_idx: usize, name: &str) -> Option<&FileSource> { + use std::borrow::Borrow; + self.get(metasource_idx) + .iter() + .find(|&source| source.name == name) + .map(|source| source.borrow()) + } + + /// Get an iterator for the [FileSources](FileSource) that match the [LanguageIdentifier] + /// and [ResourceId]. + #[cfg(feature = "test-fluent")] + pub fn get_sources_for_resource<'l>( + &'l self, + metasource_idx: usize, + langid: &'l LanguageIdentifier, + resource_id: &'l ResourceId, + ) -> impl Iterator<Item = &FileSource> { + use std::borrow::Borrow; + self.get(metasource_idx) + .iter() + .filter(move |source| source.has_file(langid, resource_id) != Some(false)) + .map(|source| source.borrow()) + } +} + +/// The [BundleAdapter] can adapt the bundle to the environment with such actions as +/// setting the platform, and hooking up functions such as Fluent's DATETIME and +/// NUMBER formatting functions. +pub trait BundleAdapter { + fn adapt_bundle(&self, bundle: &mut FluentBundle); +} + +/// The L10nRegistry is the main struct for owning the registry information. +/// +/// `P` - A provider +/// `B` - A bundle adapter +#[derive(Clone)] +pub struct L10nRegistry<P, B> { + shared: Rc<Shared<P, B>>, +} + +impl<P, B> L10nRegistry<P, B> { + /// Create a new [L10nRegistry] from a provider. + pub fn with_provider(provider: P) -> Self { + Self { + shared: Rc::new(Shared { + metasources: Default::default(), + provider, + bundle_adapter: None, + }), + } + } + + /// Set the bundle adapter. See [BundleAdapter] for more information. + pub fn set_bundle_adapter(&mut self, bundle_adapter: B) -> Result<(), L10nRegistrySetupError> + where + B: BundleAdapter, + { + let shared = Rc::get_mut(&mut self.shared).ok_or(L10nRegistrySetupError::RegistryLocked)?; + shared.bundle_adapter = Some(bundle_adapter); + Ok(()) + } + + pub fn try_borrow_metasources(&self) -> Result<Ref<MetaSources>, L10nRegistrySetupError> { + self.shared + .metasources + .try_borrow() + .map_err(|_| L10nRegistrySetupError::RegistryLocked) + } + + pub fn try_borrow_metasources_mut( + &self, + ) -> Result<RefMut<MetaSources>, L10nRegistrySetupError> { + self.shared + .metasources + .try_borrow_mut() + .map_err(|_| L10nRegistrySetupError::RegistryLocked) + } + + /// Adds a new [FileSource] to the registry and to its appropriate metasource. If the + /// metasource for this [FileSource] does not exist, then it is created. + pub fn register_sources( + &self, + new_sources: Vec<FileSource>, + ) -> Result<(), L10nRegistrySetupError> { + for new_source in new_sources { + self.try_borrow_metasources_mut()? + .add_filesource(new_source); + } + Ok(()) + } + + /// Update the information about sources already stored in the registry. Each + /// [FileSource] provided must exist, or else a [L10nRegistrySetupError] will + /// be returned. + pub fn update_sources( + &self, + new_sources: Vec<FileSource>, + ) -> Result<(), L10nRegistrySetupError> { + for new_source in new_sources { + if !self + .try_borrow_metasources_mut()? + .update_filesource(&new_source) + { + return Err(L10nRegistrySetupError::MissingSource { + name: new_source.name, + }); + } + } + Ok(()) + } + + /// Remove the provided sources. If a metasource becomes empty after this operation, + /// the metasource is also removed. + pub fn remove_sources<S>(&self, del_sources: Vec<S>) -> Result<(), L10nRegistrySetupError> + where + S: ToString, + { + let del_sources: Vec<String> = del_sources.into_iter().map(|s| s.to_string()).collect(); + + for metasource in self.try_borrow_metasources_mut()?.iter_mut() { + metasource.retain(|source| !del_sources.contains(&source.name)); + } + + self.try_borrow_metasources_mut()?.clear_empty_metasources(); + + Ok(()) + } + + /// Clears out all metasources and sources. + pub fn clear_sources(&self) -> Result<(), L10nRegistrySetupError> { + self.try_borrow_metasources_mut()?.clear(); + Ok(()) + } + + /// Flattens out all metasources and returns the complete list of source names. + pub fn get_source_names(&self) -> Result<Vec<String>, L10nRegistrySetupError> { + Ok(self + .try_borrow_metasources()? + .filesources() + .map(|s| s.name.clone()) + .collect()) + } + + /// Checks if any metasources has a source, by the name. + pub fn has_source(&self, name: &str) -> Result<bool, L10nRegistrySetupError> { + Ok(self + .try_borrow_metasources()? + .filesources() + .any(|source| source.name == name)) + } + + /// Get a [FileSource] by name by searching through all meta sources. + pub fn file_source_by_name( + &self, + name: &str, + ) -> Result<Option<FileSource>, L10nRegistrySetupError> { + Ok(self + .try_borrow_metasources()? + .filesources() + .find(|source| source.name == name) + .map(|source| (**source).clone())) + } + + /// Returns a unique list of locale names from all sources. + pub fn get_available_locales(&self) -> Result<Vec<LanguageIdentifier>, L10nRegistrySetupError> { + let mut result = HashSet::new(); + let metasources = self.try_borrow_metasources()?; + for source in metasources.filesources() { + for locale in source.locales() { + result.insert(locale); + } + } + Ok(result.into_iter().map(|l| l.to_owned()).collect()) + } +} + +/// Defines how to generate bundles synchronously and asynchronously. +impl<P, B> BundleGenerator for L10nRegistry<P, B> +where + P: ErrorReporter + Clone, + B: BundleAdapter + Clone, +{ + type Resource = Rc<FluentResource>; + type Iter = GenerateBundlesSync<P, B>; + type Stream = GenerateBundles<P, B>; + type LocalesIter = std::vec::IntoIter<LanguageIdentifier>; + + /// The synchronous version of the bundle generator. This is hooked into Gecko + /// code via the `l10nregistry_generate_bundles_sync` function. + fn bundles_iter( + &self, + locales: Self::LocalesIter, + resource_ids: FxHashSet<ResourceId>, + ) -> Self::Iter { + let resource_ids = resource_ids.into_iter().collect(); + self.generate_bundles_sync(locales, resource_ids) + } + + /// The asynchronous version of the bundle generator. This is hooked into Gecko + /// code via the `l10nregistry_generate_bundles` function. + fn bundles_stream( + &self, + locales: Self::LocalesIter, + resource_ids: FxHashSet<ResourceId>, + ) -> Self::Stream { + let resource_ids = resource_ids.into_iter().collect(); + self.generate_bundles(locales, resource_ids) + .expect("Unable to get the MetaSources.") + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs b/intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs new file mode 100644 index 0000000000..097ca68eee --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs @@ -0,0 +1,307 @@ +use super::{BundleAdapter, L10nRegistry, MetaSources}; +use crate::env::ErrorReporter; +use crate::errors::L10nRegistryError; +use crate::fluent::{FluentBundle, FluentError}; +use crate::solver::{SerialProblemSolver, SyncTester}; +use crate::source::ResourceOption; +use fluent_fallback::{generator::BundleIterator, types::ResourceId}; +use unic_langid::LanguageIdentifier; + +impl MetaSources { + pub(crate) fn bundle_from_order<P, B>( + &self, + metasource: usize, + locale: LanguageIdentifier, + source_order: &[usize], + resource_ids: &[ResourceId], + error_reporter: &P, + bundle_adapter: Option<&B>, + ) -> Option<Result<FluentBundle, (FluentBundle, Vec<FluentError>)>> + where + P: ErrorReporter, + B: BundleAdapter, + { + let mut bundle = FluentBundle::new(vec![locale.clone()]); + + if let Some(bundle_adapter) = bundle_adapter { + bundle_adapter.adapt_bundle(&mut bundle); + } + + let mut errors = vec![]; + + for (&source_idx, resource_id) in source_order.iter().zip(resource_ids.iter()) { + let source = self.filesource(metasource, source_idx); + if let ResourceOption::Some(res) = + source.fetch_file_sync(&locale, resource_id, /* overload */ true) + { + if source.options.allow_override { + bundle.add_resource_overriding(res); + } else if let Err(err) = bundle.add_resource(res) { + errors.extend(err.into_iter().map(|error| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: None, + error, + })); + } + } else if resource_id.is_required() { + return None; + } + } + + if !errors.is_empty() { + error_reporter.report_errors(errors); + } + Some(Ok(bundle)) + } +} + +impl<P, B> L10nRegistry<P, B> +where + P: Clone, + B: Clone, +{ + /// A test-only function for easily generating bundles for a single langid. + #[cfg(feature = "test-fluent")] + pub fn generate_bundles_for_lang_sync( + &self, + langid: LanguageIdentifier, + resource_ids: Vec<ResourceId>, + ) -> GenerateBundlesSync<P, B> { + let lang_ids = vec![langid]; + + GenerateBundlesSync::new(self.clone(), lang_ids.into_iter(), resource_ids) + } + + /// Wiring for hooking up the synchronous bundle generation to the + /// [BundleGenerator] trait. + pub fn generate_bundles_sync( + &self, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + ) -> GenerateBundlesSync<P, B> { + GenerateBundlesSync::new(self.clone(), locales, resource_ids) + } +} + +enum State { + Empty, + Locale(LanguageIdentifier), + Solver { + locale: LanguageIdentifier, + solver: SerialProblemSolver, + }, +} + +impl Default for State { + fn default() -> Self { + Self::Empty + } +} + +impl State { + fn get_locale(&self) -> &LanguageIdentifier { + match self { + Self::Locale(locale) => locale, + Self::Solver { locale, .. } => locale, + Self::Empty => unreachable!("Attempting to get a locale for an empty state."), + } + } + + fn take_solver(&mut self) -> SerialProblemSolver { + replace_with::replace_with_or_default_and_return(self, |self_| match self_ { + Self::Solver { locale, solver } => (solver, Self::Locale(locale)), + _ => unreachable!("Attempting to take a solver in an invalid state."), + }) + } + + fn put_back_solver(&mut self, solver: SerialProblemSolver) { + replace_with::replace_with_or_default(self, |self_| match self_ { + Self::Locale(locale) => Self::Solver { locale, solver }, + _ => unreachable!("Attempting to put back a solver in an invalid state."), + }) + } +} + +pub struct GenerateBundlesSync<P, B> { + reg: L10nRegistry<P, B>, + locales: std::vec::IntoIter<LanguageIdentifier>, + current_metasource: usize, + resource_ids: Vec<ResourceId>, + state: State, +} + +impl<P, B> GenerateBundlesSync<P, B> { + fn new( + reg: L10nRegistry<P, B>, + locales: std::vec::IntoIter<LanguageIdentifier>, + resource_ids: Vec<ResourceId>, + ) -> Self { + Self { + reg, + locales, + current_metasource: 0, + resource_ids, + state: State::Empty, + } + } +} + +impl<P, B> SyncTester for GenerateBundlesSync<P, B> { + fn test_sync(&self, res_idx: usize, source_idx: usize) -> bool { + let locale = self.state.get_locale(); + let resource_id = &self.resource_ids[res_idx]; + !self + .reg + .try_borrow_metasources() + .expect("Unable to get the MetaSources.") + .filesource(self.current_metasource, source_idx) + .fetch_file_sync(locale, resource_id, /* overload */ true) + .is_required_and_missing() + } +} + +impl<P, B> BundleIterator for GenerateBundlesSync<P, B> +where + P: ErrorReporter, +{ + fn prefetch_sync(&mut self) { + if let State::Solver { .. } = self.state { + let mut solver = self.state.take_solver(); + if let Err(idx) = solver.try_next(self, true) { + self.reg + .shared + .provider + .report_errors(vec![L10nRegistryError::MissingResource { + locale: self.state.get_locale().clone(), + resource_id: self.resource_ids[idx].clone(), + }]); + } + self.state.put_back_solver(solver); + return; + } + + if let Some(locale) = self.locales.next() { + let mut solver = SerialProblemSolver::new( + self.resource_ids.len(), + self.reg + .try_borrow_metasources() + .expect("Unable to get the MetaSources.") + .get(self.current_metasource) + .len(), + ); + self.state = State::Locale(locale.clone()); + if let Err(idx) = solver.try_next(self, true) { + self.reg + .shared + .provider + .report_errors(vec![L10nRegistryError::MissingResource { + locale, + resource_id: self.resource_ids[idx].clone(), + }]); + } + self.state.put_back_solver(solver); + } + } +} + +impl<P, B> Iterator for GenerateBundlesSync<P, B> +where + P: ErrorReporter, + B: BundleAdapter, +{ + type Item = Result<FluentBundle, (FluentBundle, Vec<FluentError>)>; + + /// Synchronously generate a bundle based on a solver. + fn next(&mut self) -> Option<Self::Item> { + let metasources = self + .reg + .try_borrow_metasources() + .expect("Unable to get the MetaSources."); + + if metasources.is_empty() { + // There are no metasources available, so no bundles can be generated. + return None; + } + + loop { + if let State::Solver { .. } = self.state { + // A solver has already been set up, continue iterating through the + // resources and generating a bundle. + let mut solver = self.state.take_solver(); + let solver_result = solver.try_next(self, false); + + if let Ok(Some(order)) = solver_result { + // The solver resolved an ordering, and a bundle may be able + // to be generated. + + let bundle = metasources.bundle_from_order( + self.current_metasource, + self.state.get_locale().clone(), + &order, + &self.resource_ids, + &self.reg.shared.provider, + self.reg.shared.bundle_adapter.as_ref(), + ); + + self.state.put_back_solver(solver); + + if bundle.is_some() { + // The bundle was successfully generated. + return bundle; + } + + // No bundle was generated, continue on. + continue; + } + + // There is no bundle ordering available. + + if self.current_metasource > 0 { + // There are more metasources, create a new solver and try the + // next metasource. If there is an error in the solver_result + // ignore it for now, since there are more metasources. + self.current_metasource -= 1; + let solver = SerialProblemSolver::new( + self.resource_ids.len(), + metasources.get(self.current_metasource).len(), + ); + self.state = State::Solver { + locale: self.state.get_locale().clone(), + solver, + }; + continue; + } + + if let Err(idx) = solver_result { + // Since there are no more metasources, and there is an error, + // report it instead of ignoring it. + self.reg.shared.provider.report_errors(vec![ + L10nRegistryError::MissingResource { + locale: self.state.get_locale().clone(), + resource_id: self.resource_ids[idx].clone(), + }, + ]); + } + + self.state = State::Empty; + continue; + } + + // Try the next locale, or break out of the loop if there are none left. + let locale = self.locales.next()?; + + // Restart at the end of the metasources for this locale, and iterate + // backwards. + let last_metasource_idx = metasources.len() - 1; + self.current_metasource = last_metasource_idx; + + let solver = SerialProblemSolver::new( + self.resource_ids.len(), + metasources.get(self.current_metasource).len(), + ); + + // Continue iterating on the next solver. + self.state = State::Solver { locale, solver }; + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/README.md b/intl/l10n/rust/l10nregistry-rs/src/solver/README.md new file mode 100644 index 0000000000..acd56b52b4 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/README.md @@ -0,0 +1,239 @@ + +Source Order Problem Solver +====================== + +This module contains an algorithm used to power the `FluentBundle` generator in `L10nRegistry`. + +The main concept behind it is a problem solver which takes a list of resources and a list of sources and computes all possible iterations of valid combinations of source orders that allow for creation of `FluentBundle` with the requested resources. + +The algorithm is notoriously hard to read, write, and modify, which prompts this documentation to be extensive and provide an example with diagram presentations to aid the reader. + +# Example +For the purpose of a graphical illustration of the example, we will evaluate a scenario with two sources and three resources. + +The sources and resource identifiers will be named in concise way (*1* or *A*) to simplify diagrams, while a more tangible names derived from real-world examples in Firefox use-case will be listed in their initial definition. + +### Sources +A source can be a packaged directory, and a language pack, or any other directory, zip file, or remote source which contains localization resource files. +In the example, we have two sources: +* Source 1 named ***0*** (e.g. `browser`) +* Source 2 named ***1*** (e.g. `toolkit`) + +### Resources +A resource is a single Fluent Translation List file. `FluentBundle` is a combination of such resources used together to resolve translations. This algorithm operates on lists of resource identifiers which represent relative paths within the source. +In the example we have three resources: +* Resource 1 named ***A*** (e.g. `branding/brand.ftl`) +* Resource 2 named ***B*** (e.g. `errors/common.ftl`) +* Resource 3 named ***C*** (e.g. `menu/list.ftl`) + +## Task +The task in this example is to generate all possible iterations of the three resources from the given two sources. Since I/O is expensive, and in most production scenarios all necessary translations are available in the first set, the iterator is used to lazily fallback on the alternative sets only in case of missing translations. + +If all resources are available in both sources, the iterator should produce the following results: +1. `[A0, B0, C0]` +2. `[A0, B0, C1]` +3. `[A0, B1, C0]` +4. `[A0, B1, C1]` +5. `[A1, B0, C0]` +6. `[A1, B0, C1]` +7. `[A1, B1, C0]` +8. `[A1, B1, C1]` + +Since the resources are defined by their column, we can store the resources as `[A, B, C]` separately and simplify the notation to just: +1. `[0, 0, 0]` +2. `[0, 0, 1]` +3. `[0, 1, 0]` +4. `[0, 1, 1]` +5. `[1, 0, 0]` +6. `[1, 0, 1]` +7. `[1, 1, 0]` +8. `[1, 1, 1]` + +This notation will be used from now on. + +## State + +For the in-detail diagrams on the algorithm, we'll use another way to look at the iterator - by evaluating it state. At every point of the algorithm, there is a *partial solution* which may lead to a *complete solution*. It is encoded as: + +```rust +struct Solution { + candidate: Vec<usize>, + idx: usize, +} +``` + +and which starting point can be visualized as: + +```text + ▼ +┌┲━┱┬───┬───┐ +│┃0┃│ │ │ +└╂─╂┴───┴───┘ + ┃ ┃ + ┗━┛ +``` +###### Diagrams generated with use of http://marklodato.github.io/js-boxdrawing/ + +where the horizontal block is a candidate, vertical block is a set of sources possible for each resource, and the arrow represents the index of a resource the iterator is currently evaluating. + +With those tools introduced, we can now guide the reader through how the algorithm works. +But before we do that, it is important to justify writing a custom algorithm in place of existing generic solutions, and explain the two testing strategies which heavily impact the algorithm. + +# Existing libraries +Intuitively, the starting point to exploration of the problem scope would be to look at it as some variation of the [Cartesian Product](https://en.wikipedia.org/wiki/Cartesian_product) iterator. + +#### Python + +In Python, `itertools` package provides a function [`itertools::product`](https://docs.python.org/3/library/itertools.html#itertools.product) which can be used to generate such iterator: +```python +import itertools + +for set in itertools.product(range(2), repeat=3): + print(set) +``` + +#### Rust + +In Rust, crate [`itertools`](https://crates.io/crates/itertools) provides, [`multi_cartesian_product`](https://docs.rs/itertools/0.9.0/itertools/trait.Itertools.html#method.multi_cartesian_product) which can be used like this: +```rust +use itertools::Itertools; + +let multi_prod = (0..3).map(|i| 0..2) + .multi_cartesian_product(); + +for set in multi_prod { + println!("{:?}", set); +} +``` +([playground](https://play.rust-lang.org/?version=stable&mode=debug&edition=2018&gist=6ef231f6b011b234babb0aa3e68b78ab)) + +#### Reasons for a custom algorithm + +Unfortunately, the computational complexity of generating all possible sets is growing exponentially, both in the cost of CPU and memory use. +On a high-end laptop, computing the sets for all possible variations of the example above generates *8* sets and takes only *700 nanoseconds*, but computing the same for four sources and 16 resources (a scenario theoretically possible in Firefox with one language pack and Preferences UI for example) generates over *4 billion* sets and takes over *2 minutes*. + +Since one part of static cost is the I/O, the application of a [Memoization](https://en.wikipedia.org/wiki/Memoization) technique allows us to minimize the cost of constructing, storing and retrieving sets. + +Second important observation is that in most scenarios any resource exists in only some of the sources, and ability to bail out from a branch of candidates that cannot lead to a solution yields significantly fewer permutations in result. + +## Optimizations + +The algorithm used here is highly efficient. For the conservative scenario listed above, where 4 sources and 15 resources are all present in every source, the total time on the reference hardware is cut from *2 minutes* to *24 seconds*, while generating the same *4 billion* sets for a **5x** performance improvement. + +### Streaming Iterator +Unline regular iterator, a streaming iterator allows a borrowed reference to be returned, which in this case, where the solver yields a read-only "view" of a solution, allows us to avoid having to clone it. + +### Cache +Memory is much less of a problem for the algorithm than CPU usage, so the solver uses a matrix of source/resource `Option` to memoize visited cells. This allows for each source/resource combination to be tested only once after which all future tests can be skipped. + +### Backtracking +This optimization allows to benefit from the recognition of the fact that most resources are only available in some sources. +Instead of generating all possible sets and then ignoring ones which are incomplete, it allows the algorithm to [backtrack](https://en.wikipedia.org/wiki/Backtracking) from partial candidates that cannot lead to a complete solution. + +That technique is very powerful in the `L10nRegistry` use case and in many scenarios leads to 10-100x speed ups even in cases where all sets have to be generated. + +# Serial vs Parallel Testing +At the core of the solver is a *tester* component which is responsible for eagerly evaluating candidates to allow for early bailouts from partial solutions which cannot lead to a complete solution. + +This can be performed in one of two ways: + +### Serial + The algorithm is synchronous and each extension of the candidate is evaluated serially, one by one, allowing the for *backtracking* as soon as a given extension of a partial solution is confirmed to not lead to a complete solution. + +Bringing back the initial state of the solver: + +```text + ▼ +┌┲━┱┬───┬───┐ +│┃0┃│ │ │ +└╂─╂┴───┴───┘ + ┃ ┃ + ┗━┛ +``` + +The tester will evaluate whether the first resource **A** is available in the first source **0**. The testing will be performed synchronously, and the result will inform the algorithm on whether the candidate may lead to a complete solution, or this branch should be bailed out from, and the next candidate must be tried. + +#### Success case + +If the test returns a success, the extensions of the candidate is generated: +```text + ▼ +┌┲━┱┬┲━┱┬───┐ +│┃0┃│┃0┃│ │ +└╂─╂┴╂─╂┴───┘ + ┃ ┃ ┃ ┃ + ┗━┛ ┗━┛ +``` + +When a candidate is complete, in other words, when the last cell of a candidate has been tested and did not lead to a backtrack, we know that the candidate is a solution to the problem, and we can yield it from the iterator. + +#### Failure case + +If the test returns a failure, the next step is to evaluate alternative source for the same resource. Let's assume that *Source 0* had *Resource A* but it does not have *Resource B*. In such case, the algorithm will increment the second cell's source index: + +```text + ▼ + ┏━┓ + ┃0┃ +┌┲━┱┬╂─╂┬───┐ +│┃0┃│┃1┃│ │ +└╂─╂┴┺━┹┴───┘ + ┃ ┃ + ┗━┛ + ``` + +and that will potentially lead to a partial solution `[0, 1, ]` to be stored for the next iteration. + +If the test fails and no more sources can be generated, the algorithm will *backtrack* from the current cell looking for a cell with the **highest** index prior to the cell that was being evaluated which is not yet on the last source. If such cell is found, the results of all cells **to the right** of the newfound cell are **erased** and the next branch can be evaluated. + +If no such cell can be found, that means that the iterator is complete. + +### Parallel + +If the testing can be performed in parallel, like an asynchronous I/O, the above *serial* solution is sub-optimal as it misses on the benefit of testing multiple cells at once. + +In such a scenario, the algorithm will construct a candidate that *can* be valid (bailing only from candidates that have been already memoized as unavailable), and then test all of the untested cells in that candidate at once. + +```text + ▼ +┌┲━┱┬┲━┱┬┲━┱┐ +│┃0┃│┃0┃│┃0┃│ +└╂─╂┴╂─╂┴╂─╂┘ + ┃ ┃ ┃ ┃ ┃ ┃ + ┗━┛ ┗━┛ ┗━┛ +``` + +When the parallel execution returns, the algorithm memoizes all new cell results and tests if the candidate is now a valid complete solution. + +#### Success case + +If the result a set of successes, the candidate is returned as a solution, and the algorithm proceeds to the same operation as if it was a failure. + +#### Failure case +If the result contains failures, the iterator will now backtrack to find the closest lower or equal cell to the current index which can be advanced to the next source. +In the example state above, the current cell can be advanced to *source 1* and then just a set of `[None, None, 1]` is to be evaluated by the tester (since we know that *A0* and *B0* are valid). + +If that is successful, the `[0, 0, 1]` set is a complete solution and is yielded. + +Then, if the iterator is resumed, the next state to be tested is: + +```text + ▼ + ┏━┓ + ┃0┃ +┌┲━┱┬╂─╂┬┲━┱┐ +│┃0┃│┃1┃│┃0┃│ +└╂─╂┴┺━┹┴╂─╂┘ + ┃ ┃ ┃ ┃ + ┗━┛ ┗━┛ +``` + +since cell *2* was at the highest index, cell *1* is the highest lower than *2* that was not at the highest source index position. That cell is advanced, and all cells after it are *pruned* (in this case, cell *2* is the only one). Then, the memoization kicks in, and since *A0* and *C0* are already cached as valid, the tester receives just `[None, 1, None]` to be tested and the algorithm continues. + +# Summary + +The algorithm explained above is tailored to the problem domain of `L10nRegistry` and is designed to be further extended in the future. + +It is important to maintain this guide up to date as any changes to the algorithm are to be made. + +Good luck. diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs new file mode 100644 index 0000000000..f14fbfe641 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs @@ -0,0 +1,122 @@ +mod parallel; +mod serial; +pub mod testing; + +pub use parallel::{AsyncTester, ParallelProblemSolver}; +pub use serial::{SerialProblemSolver, SyncTester}; + +pub struct ProblemSolver { + width: usize, + depth: usize, + + cache: Vec<Vec<Option<bool>>>, + + solution: Vec<usize>, + idx: usize, + + dirty: bool, +} + +impl ProblemSolver { + pub fn new(width: usize, depth: usize) -> Self { + Self { + width, + depth, + cache: vec![vec![None; depth]; width], + + solution: vec![0; width], + idx: 0, + + dirty: false, + } + } +} + +impl ProblemSolver { + pub fn bail(&mut self) -> bool { + if self.try_advance_source() { + true + } else { + self.try_backtrack() + } + } + + pub fn has_missing_cell(&self) -> Option<usize> { + for res_idx in 0..self.width { + if self.cache[res_idx].iter().all(|c| *c == Some(false)) { + return Some(res_idx); + } + } + None + } + + fn is_cell_missing(&self, res_idx: usize, source_idx: usize) -> bool { + if let Some(false) = self.cache[res_idx][source_idx] { + return true; + } + false + } + + fn is_current_cell_missing(&self) -> bool { + let res_idx = self.idx; + let source_idx = self.solution[res_idx]; + let cell = &self.cache[res_idx][source_idx]; + if let Some(false) = cell { + return true; + } + false + } + + pub fn try_advance_resource(&mut self) -> bool { + if self.idx >= self.width - 1 { + false + } else { + self.idx += 1; + while self.is_current_cell_missing() { + if !self.try_advance_source() { + return false; + } + } + true + } + } + + pub fn try_advance_source(&mut self) -> bool { + while self.solution[self.idx] < self.depth - 1 { + self.solution[self.idx] += 1; + if !self.is_current_cell_missing() { + return true; + } + } + false + } + + pub fn try_backtrack(&mut self) -> bool { + while self.solution[self.idx] == self.depth - 1 { + if self.idx == 0 { + return false; + } + self.idx -= 1; + } + self.solution[self.idx] += 1; + self.prune() + } + + pub fn prune(&mut self) -> bool { + for i in self.idx + 1..self.width { + let mut source_idx = 0; + while self.is_cell_missing(i, source_idx) { + if source_idx >= self.depth - 1 { + return false; + } + source_idx += 1; + } + self.solution[i] = source_idx; + } + true + } + + pub fn is_complete(&self) -> bool { + self.idx == self.width - 1 + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs new file mode 100644 index 0000000000..320ad65c89 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs @@ -0,0 +1,175 @@ +use super::ProblemSolver; +use std::ops::{Deref, DerefMut}; + +use futures::ready; +use std::future::Future; +use std::pin::Pin; + +pub trait AsyncTester { + type Result: Future<Output = Vec<bool>>; + + fn test_async(&self, query: Vec<(usize, usize)>) -> Self::Result; +} + +pub struct ParallelProblemSolver<T> +where + T: AsyncTester, +{ + solver: ProblemSolver, + current_test: Option<(T::Result, Vec<usize>)>, +} + +impl<T: AsyncTester> Deref for ParallelProblemSolver<T> { + type Target = ProblemSolver; + + fn deref(&self) -> &Self::Target { + &self.solver + } +} + +impl<T: AsyncTester> DerefMut for ParallelProblemSolver<T> { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.solver + } +} + +impl<T: AsyncTester> ParallelProblemSolver<T> { + pub fn new(width: usize, depth: usize) -> Self { + Self { + solver: ProblemSolver::new(width, depth), + current_test: None, + } + } +} + +type TestQuery = (Vec<(usize, usize)>, Vec<usize>); + +impl<T: AsyncTester> ParallelProblemSolver<T> { + pub fn try_generate_complete_candidate(&mut self) -> bool { + while !self.is_complete() { + while self.is_current_cell_missing() { + if !self.try_advance_source() { + return false; + } + } + if !self.try_advance_resource() { + return false; + } + } + true + } + + fn try_generate_test_query(&mut self) -> Result<TestQuery, usize> { + let mut test_cells = vec![]; + let query = self + .solution + .iter() + .enumerate() + .filter_map(|(res_idx, source_idx)| { + let cell = self.cache[res_idx][*source_idx]; + match cell { + None => { + test_cells.push(res_idx); + Some(Ok((res_idx, *source_idx))) + } + Some(false) => Some(Err(res_idx)), + Some(true) => None, + } + }) + .collect::<Result<_, _>>()?; + Ok((query, test_cells)) + } + + fn apply_test_result( + &mut self, + resources: Vec<bool>, + testing_cells: Vec<usize>, + ) -> Result<(), usize> { + let mut first_missing = None; + for (result, res_idx) in resources.into_iter().zip(testing_cells) { + let source_idx = self.solution[res_idx]; + self.cache[res_idx][source_idx] = Some(result); + if !result && first_missing.is_none() { + first_missing = Some(res_idx); + } + } + if let Some(idx) = first_missing { + Err(idx) + } else { + Ok(()) + } + } + + pub fn try_poll_next( + mut self: std::pin::Pin<&mut Self>, + cx: &mut std::task::Context<'_>, + tester: &T, + prefetch: bool, + ) -> std::task::Poll<Result<Option<Vec<usize>>, usize>> + where + <T as AsyncTester>::Result: Unpin, + { + if self.width == 0 || self.depth == 0 { + return Ok(None).into(); + } + + 'outer: loop { + if let Some((test, testing_cells)) = &mut self.current_test { + let pinned = Pin::new(test); + let set = ready!(pinned.poll(cx)); + let testing_cells = testing_cells.clone(); + + if let Err(res_idx) = self.apply_test_result(set, testing_cells) { + self.idx = res_idx; + self.prune(); + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx).into(); + } else { + return Ok(None).into(); + } + } + self.current_test = None; + continue 'outer; + } else { + self.current_test = None; + if !prefetch { + self.dirty = true; + } + return Ok(Some(self.solution.clone())).into(); + } + } else { + if self.dirty { + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx).into(); + } else { + return Ok(None).into(); + } + } + self.dirty = false; + } + while self.try_generate_complete_candidate() { + match self.try_generate_test_query() { + Ok((query, testing_cells)) => { + self.current_test = Some((tester.test_async(query), testing_cells)); + continue 'outer; + } + Err(res_idx) => { + self.idx = res_idx; + self.prune(); + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx).into(); + } else { + return Ok(None).into(); + } + } + } + } + } + return Ok(None).into(); + } + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs new file mode 100644 index 0000000000..9368c12c9e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs @@ -0,0 +1,76 @@ +use super::ProblemSolver; +use std::ops::{Deref, DerefMut}; + +pub trait SyncTester { + fn test_sync(&self, res_idx: usize, source_idx: usize) -> bool; +} + +pub struct SerialProblemSolver(ProblemSolver); + +impl Deref for SerialProblemSolver { + type Target = ProblemSolver; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for SerialProblemSolver { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl SerialProblemSolver { + pub fn new(width: usize, depth: usize) -> Self { + Self(ProblemSolver::new(width, depth)) + } +} + +impl SerialProblemSolver { + fn test_current_cell<T>(&mut self, tester: &T) -> bool + where + T: SyncTester, + { + let res_idx = self.idx; + let source_idx = self.solution[res_idx]; + let cell = &mut self.cache[res_idx][source_idx]; + *cell.get_or_insert_with(|| tester.test_sync(res_idx, source_idx)) + } + + pub fn try_next<T>(&mut self, tester: &T, prefetch: bool) -> Result<Option<&[usize]>, usize> + where + T: SyncTester, + { + if self.width == 0 || self.depth == 0 { + return Ok(None); + } + if self.dirty { + if !self.bail() { + return Ok(None); + } + self.dirty = false; + } + loop { + if !self.test_current_cell(tester) { + if !self.bail() { + if let Some(res_idx) = self.has_missing_cell() { + return Err(res_idx); + } else { + return Ok(None); + } + } + continue; + } + if self.is_complete() { + if !prefetch { + self.dirty = true; + } + return Ok(Some(&self.solution)); + } + if !self.try_advance_resource() { + return Ok(None); + } + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/testing/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/testing/mod.rs new file mode 100644 index 0000000000..68f566250e --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/testing/mod.rs @@ -0,0 +1,38 @@ +mod scenarios; + +pub use scenarios::get_scenarios; + +/// Define a testing scenario. +pub struct Scenario { + /// Name of the scenario. + pub name: String, + /// Number of resources. + pub width: usize, + /// Number of sources. + pub depth: usize, + /// Vector of resources, containing a vector of sources, with true indicating + /// whether the resource is present in that source. + pub values: Vec<Vec<bool>>, + /// Vector of solutions, each containing a vector of resources, with the index + /// indicating from which source the resource is chosen. + /// TODO(issue#17): This field is currently unused! + pub solutions: Vec<Vec<usize>>, +} + +impl Scenario { + pub fn new<S: ToString>( + name: S, + width: usize, + depth: usize, + values: Vec<Vec<bool>>, + solutions: Vec<Vec<usize>>, + ) -> Self { + Self { + name: name.to_string(), + width, + depth, + values, + solutions, + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/solver/testing/scenarios.rs b/intl/l10n/rust/l10nregistry-rs/src/solver/testing/scenarios.rs new file mode 100644 index 0000000000..8addec979b --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/solver/testing/scenarios.rs @@ -0,0 +1,151 @@ +use super::*; + +pub fn get_scenarios() -> Vec<Scenario> { + vec![ + Scenario::new("no-sources", 1, 0, vec![], vec![]), + Scenario::new("no-resources", 1, 0, vec![vec![true]], vec![]), + Scenario::new("no-keys", 0, 1, vec![], vec![]), + Scenario::new( + "one-res-two-sources", + 1, + 2, + vec![vec![true, true]], + vec![vec![0], vec![1]], + ), + Scenario::new( + "two-res-two-sources", + 2, + 2, + vec![vec![false, true], vec![true, false]], + vec![vec![1, 0]], + ), + Scenario::new( + "small", + 3, + 2, + vec![vec![true, true], vec![true, true], vec![true, true]], + vec![ + vec![0, 0, 0], + vec![0, 0, 1], + vec![0, 1, 0], + vec![0, 1, 1], + vec![1, 0, 0], + vec![1, 0, 1], + vec![1, 1, 0], + vec![1, 1, 1], + ], + ), + Scenario::new( + "incomplete", + 3, + 2, + vec![vec![true, false], vec![false, true], vec![true, true]], + vec![vec![0, 1, 0], vec![0, 1, 1]], + ), + Scenario::new( + "preferences", + 19, + 2, + vec![ + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![true, false], + vec![false, true], + vec![false, true], + vec![false, true], + ], + vec![vec![ + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, + ]], + ), + Scenario::new( + "langpack", + 3, + 4, + vec![ + vec![true, true, true, true], + vec![true, true, true, true], + vec![true, true, true, true], + ], + vec![ + vec![0, 0, 0], + vec![0, 0, 1], + vec![0, 0, 2], + vec![0, 0, 3], + vec![0, 1, 0], + vec![0, 1, 1], + vec![0, 1, 2], + vec![0, 1, 3], + vec![0, 2, 0], + vec![0, 2, 1], + vec![0, 2, 2], + vec![0, 2, 3], + vec![0, 3, 0], + vec![0, 3, 1], + vec![0, 3, 2], + vec![0, 3, 3], + vec![1, 0, 0], + vec![1, 0, 1], + vec![1, 0, 2], + vec![1, 0, 3], + vec![1, 1, 0], + vec![1, 1, 1], + vec![1, 1, 2], + vec![1, 1, 3], + vec![1, 2, 0], + vec![1, 2, 1], + vec![1, 2, 2], + vec![1, 2, 3], + vec![1, 3, 0], + vec![1, 3, 1], + vec![1, 3, 2], + vec![1, 3, 3], + vec![2, 0, 0], + vec![2, 0, 1], + vec![2, 0, 2], + vec![2, 0, 3], + vec![2, 1, 0], + vec![2, 1, 1], + vec![2, 1, 2], + vec![2, 1, 3], + vec![2, 2, 0], + vec![2, 2, 1], + vec![2, 2, 2], + vec![2, 2, 3], + vec![2, 3, 0], + vec![2, 3, 1], + vec![2, 3, 2], + vec![2, 3, 3], + vec![3, 0, 0], + vec![3, 0, 1], + vec![3, 0, 2], + vec![3, 0, 3], + vec![3, 1, 0], + vec![3, 1, 1], + vec![3, 1, 2], + vec![3, 1, 3], + vec![3, 2, 0], + vec![3, 2, 1], + vec![3, 2, 2], + vec![3, 2, 3], + vec![3, 3, 0], + vec![3, 3, 1], + vec![3, 3, 2], + vec![3, 3, 3], + ], + ), + ] +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs new file mode 100644 index 0000000000..3a022990a6 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs @@ -0,0 +1,30 @@ +use async_trait::async_trait; +use fluent_fallback::types::ResourceId; +use std::io; + +/// The users of [`FileSource`] implement this trait to provide loading of +/// resources, returning the contents of a resource as a +/// `String`. [`FileSource`] handles the conversion from string representation +/// into `FluentResource`. +/// +/// [`FileSource`]: source/struct.FileSource.html +#[async_trait(?Send)] +pub trait FileFetcher { + /// Return the `String` representation for `path`. This version is + /// blocking. + /// + /// See [`fetch`](#tymethod.fetch). + fn fetch_sync(&self, resource_id: &ResourceId) -> io::Result<String>; + + /// Return the `String` representation for `path`. + /// + /// On success, returns `Poll::Ready(Ok(..))`. + /// + /// If no resource is available to be fetched, the method returns + /// `Poll::Pending` and arranges for the current task (via + /// `cx.waker().wake_by_ref()`) to receive a notification when the resource + /// is available. + /// + /// See [`fetch_sync`](#tymethod.fetch_sync) + async fn fetch(&self, path: &ResourceId) -> io::Result<String>; +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs new file mode 100644 index 0000000000..1c72065b38 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/source/mod.rs @@ -0,0 +1,558 @@ +mod fetcher; +pub use fetcher::FileFetcher; +pub use fluent_fallback::types::{ResourceId, ToResourceId}; + +use crate::env::ErrorReporter; +use crate::errors::L10nRegistryError; +use crate::fluent::FluentResource; + +use std::{ + borrow::Borrow, + cell::RefCell, + fmt, + hash::{Hash, Hasher}, + pin::Pin, + rc::Rc, + task::Poll, +}; + +use futures::{future::Shared, Future, FutureExt}; +use rustc_hash::FxHashMap; +use unic_langid::LanguageIdentifier; + +pub type RcResource = Rc<FluentResource>; + +/// An option type whose None variant is either optional or required. +/// +/// This behaves similarly to the standard-library [`Option`] type +/// except that there are two [`None`]-like variants: +/// [`ResourceOption::MissingOptional`] and [`ResourceOption::MissingRequired`]. +#[derive(Clone, Debug)] +pub enum ResourceOption { + /// An available resource. + Some(RcResource), + /// A missing optional resource. + MissingOptional, + /// A missing required resource. + MissingRequired, +} + +impl ResourceOption { + /// Creates a resource option that is either [`ResourceOption::MissingRequired`] + /// or [`ResourceOption::MissingOptional`] based on whether the given [`ResourceId`] + /// is required or optional. + pub fn missing_resource(resource_id: &ResourceId) -> Self { + if resource_id.is_required() { + Self::MissingRequired + } else { + Self::MissingOptional + } + } + + /// Returns [`true`] if this option contains a recource, otherwise [`false`]. + pub fn is_some(&self) -> bool { + matches!(self, Self::Some(_)) + } + + /// Resource [`true`] if this option is missing a resource of any type, otherwise [`false`]. + pub fn is_none(&self) -> bool { + matches!(self, Self::MissingOptional | Self::MissingRequired) + } + + /// Returns [`true`] if this option is missing a required resource, otherwise [`false`]. + pub fn is_required_and_missing(&self) -> bool { + matches!(self, Self::MissingRequired) + } +} + +impl From<ResourceOption> for Option<RcResource> { + fn from(other: ResourceOption) -> Self { + match other { + ResourceOption::Some(id) => Some(id), + _ => None, + } + } +} + +pub type ResourceFuture = Shared<Pin<Box<dyn Future<Output = ResourceOption>>>>; + +#[derive(Debug, Clone)] +pub enum ResourceStatus { + /// The resource is missing. Don't bother trying to fetch. + MissingRequired, + MissingOptional, + /// The resource is loading and future will deliver the result. + Loading(ResourceFuture), + /// The resource is loaded and parsed. + Loaded(RcResource), +} + +impl From<ResourceOption> for ResourceStatus { + fn from(input: ResourceOption) -> Self { + match input { + ResourceOption::Some(res) => Self::Loaded(res), + ResourceOption::MissingOptional => Self::MissingOptional, + ResourceOption::MissingRequired => Self::MissingRequired, + } + } +} + +impl Future for ResourceStatus { + type Output = ResourceOption; + + fn poll(mut self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> { + use ResourceStatus::*; + + let this = &mut *self; + + match this { + MissingRequired => ResourceOption::MissingRequired.into(), + MissingOptional => ResourceOption::MissingOptional.into(), + Loaded(res) => ResourceOption::Some(res.clone()).into(), + Loading(res) => Pin::new(res).poll(cx), + } + } +} + +/// `FileSource` provides a generic fetching and caching of fluent resources. +/// The user of `FileSource` provides a [`FileFetcher`](trait.FileFetcher.html) +/// implementation and `FileSource` takes care of the rest. +#[derive(Clone)] +pub struct FileSource { + /// Name of the FileSource, e.g. "browser" + pub name: String, + /// Pre-formatted path for the FileSource, e.g. "/browser/data/locale/{locale}/" + pub pre_path: String, + /// Metasource name for the FileSource, e.g. "app", "langpack" + /// Only sources from the same metasource are passed into the solver. + pub metasource: String, + /// The locales for which data is present in the FileSource, e.g. ["en-US", "pl"] + locales: Vec<LanguageIdentifier>, + shared: Rc<Inner>, + index: Option<Vec<String>>, + pub options: FileSourceOptions, +} + +struct Inner { + fetcher: Box<dyn FileFetcher>, + error_reporter: Option<RefCell<Box<dyn ErrorReporter>>>, + entries: RefCell<FxHashMap<String, ResourceStatus>>, +} + +impl fmt::Display for FileSource { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.name) + } +} + +impl PartialEq<FileSource> for FileSource { + fn eq(&self, other: &Self) -> bool { + self.name == other.name && self.metasource == other.metasource + } +} + +impl Eq for FileSource {} + +impl Hash for FileSource { + fn hash<H: Hasher>(&self, state: &mut H) { + self.name.hash(state) + } +} + +#[derive(PartialEq, Clone, Debug)] +pub struct FileSourceOptions { + pub allow_override: bool, +} + +impl Default for FileSourceOptions { + fn default() -> Self { + Self { + allow_override: false, + } + } +} + +impl FileSource { + /// Create a `FileSource` using the provided [`FileFetcher`](../trait.FileFetcher.html). + pub fn new( + name: String, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + pre_path: String, + options: FileSourceOptions, + fetcher: impl FileFetcher + 'static, + ) -> Self { + FileSource { + name, + metasource: metasource.unwrap_or_default(), + pre_path, + locales, + index: None, + shared: Rc::new(Inner { + entries: RefCell::new(FxHashMap::default()), + fetcher: Box::new(fetcher), + error_reporter: None, + }), + options, + } + } + + pub fn new_with_index( + name: String, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + pre_path: String, + options: FileSourceOptions, + fetcher: impl FileFetcher + 'static, + index: Vec<String>, + ) -> Self { + FileSource { + name, + metasource: metasource.unwrap_or_default(), + pre_path, + locales, + index: Some(index), + shared: Rc::new(Inner { + entries: RefCell::new(FxHashMap::default()), + fetcher: Box::new(fetcher), + error_reporter: None, + }), + options, + } + } + + pub fn set_reporter(&mut self, reporter: impl ErrorReporter + 'static) { + let mut shared = Rc::get_mut(&mut self.shared).unwrap(); + shared.error_reporter = Some(RefCell::new(Box::new(reporter))); + } +} + +fn calculate_pos_in_source(source: &str, idx: usize) -> (usize, usize) { + let mut ptr = 0; + let mut result = (1, 1); + for line in source.lines() { + let bytes = line.as_bytes().len(); + if ptr + bytes < idx { + ptr += bytes + 1; + result.0 += 1; + } else { + result.1 = idx - ptr + 1; + break; + } + } + result +} + +impl FileSource { + fn get_path(&self, locale: &LanguageIdentifier, resource_id: &ResourceId) -> String { + format!( + "{}{}", + self.pre_path.replace("{locale}", &locale.to_string()), + resource_id.value, + ) + } + + fn fetch_sync(&self, resource_id: &ResourceId) -> ResourceOption { + self.shared + .fetcher + .fetch_sync(resource_id) + .ok() + .map(|source| match FluentResource::try_new(source) { + Ok(res) => ResourceOption::Some(Rc::new(res)), + Err((res, errors)) => { + if let Some(reporter) = &self.shared.error_reporter { + reporter.borrow().report_errors( + errors + .into_iter() + .map(|e| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), + error: e.into(), + }) + .collect(), + ); + } + ResourceOption::Some(Rc::new(res)) + } + }) + .unwrap_or_else(|| ResourceOption::missing_resource(resource_id)) + } + + /// Attempt to synchronously fetch resource for the combination of `locale` + /// and `path`. Returns `Some(ResourceResult)` if the resource is available, + /// else `None`. + pub fn fetch_file_sync( + &self, + locale: &LanguageIdentifier, + resource_id: &ResourceId, + overload: bool, + ) -> ResourceOption { + use ResourceStatus::*; + + if self.has_file(locale, resource_id) == Some(false) { + return ResourceOption::missing_resource(resource_id); + } + + let full_path_id = self + .get_path(locale, resource_id) + .to_resource_id(resource_id.resource_type); + + let res = self.shared.lookup_resource(full_path_id.clone(), || { + self.fetch_sync(&full_path_id).into() + }); + + match res { + MissingRequired => ResourceOption::MissingRequired, + MissingOptional => ResourceOption::MissingOptional, + Loaded(res) => ResourceOption::Some(res), + Loading(..) if overload => { + // A sync load has been requested for the same resource that has + // a pending async load in progress. How do we handle this? + // + // Ideally, we would sync load and resolve all the pending + // futures with the result. With the current Futures and + // combinators, it's unclear how to proceed. One potential + // solution is to store a oneshot::Sender and + // Shared<oneshot::Receiver>. When the async loading future + // resolves it would check that the state is still `Loading`, + // and if so, send the result. The sync load would do the same + // send on the oneshot::Sender. + // + // For now, we warn and return the resource, paying the cost of + // duplication of the resource. + self.fetch_sync(&full_path_id) + } + Loading(..) => { + panic!("[l10nregistry] Attempting to synchronously load file {} while it's being loaded asynchronously.", &full_path_id.value); + } + } + } + + /// Attempt to fetch resource for the combination of `locale` and `path`. + /// Returns [`ResourceStatus`](enum.ResourceStatus.html) which is + /// a `Future` that can be polled. + pub fn fetch_file( + &self, + locale: &LanguageIdentifier, + resource_id: &ResourceId, + ) -> ResourceStatus { + use ResourceStatus::*; + + if self.has_file(locale, resource_id) == Some(false) { + return ResourceOption::missing_resource(resource_id).into(); + } + + let full_path_id = self + .get_path(locale, resource_id) + .to_resource_id(resource_id.resource_type); + + self.shared.lookup_resource(full_path_id.clone(), || { + let shared = self.shared.clone(); + Loading(read_resource(full_path_id, shared).boxed_local().shared()) + }) + } + + /// Determine if the `FileSource` has a loaded resource for the combination + /// of `locale` and `path`. Returns `Some(true)` if the file is loaded, else + /// `Some(false)`. `None` is returned if there is an outstanding async fetch + /// pending and the status is yet to be determined. + pub fn has_file<L: Borrow<LanguageIdentifier>>( + &self, + locale: L, + path: &ResourceId, + ) -> Option<bool> { + let locale = locale.borrow(); + if !self.locales.contains(locale) { + Some(false) + } else { + let full_path = self.get_path(locale, path); + if let Some(index) = &self.index { + return Some(index.iter().any(|p| p == &full_path)); + } + self.shared.has_file(&full_path) + } + } + + pub fn locales(&self) -> &[LanguageIdentifier] { + &self.locales + } + + pub fn get_index(&self) -> Option<&Vec<String>> { + self.index.as_ref() + } +} + +impl std::fmt::Debug for FileSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> fmt::Result { + if let Some(index) = &self.index { + f.debug_struct("FileSource") + .field("name", &self.name) + .field("metasource", &self.metasource) + .field("locales", &self.locales) + .field("pre_path", &self.pre_path) + .field("index", index) + .finish() + } else { + f.debug_struct("FileSource") + .field("name", &self.name) + .field("metasource", &self.metasource) + .field("locales", &self.locales) + .field("pre_path", &self.pre_path) + .finish() + } + } +} + +impl Inner { + fn lookup_resource<F>(&self, resource_id: ResourceId, f: F) -> ResourceStatus + where + F: FnOnce() -> ResourceStatus, + { + let mut lock = self.entries.borrow_mut(); + lock.entry(resource_id.value).or_insert_with(|| f()).clone() + } + + fn update_resource(&self, resource_id: ResourceId, resource: ResourceOption) -> ResourceOption { + let mut lock = self.entries.borrow_mut(); + let entry = lock.get_mut(&resource_id.value); + match entry { + Some(entry) => *entry = resource.clone().into(), + _ => panic!("Expected "), + } + resource + } + + pub fn has_file(&self, full_path: &str) -> Option<bool> { + match self.entries.borrow().get(full_path) { + Some(ResourceStatus::MissingRequired) => Some(false), + Some(ResourceStatus::MissingOptional) => Some(false), + Some(ResourceStatus::Loaded(_)) => Some(true), + Some(ResourceStatus::Loading(_)) | None => None, + } + } +} + +async fn read_resource(resource_id: ResourceId, shared: Rc<Inner>) -> ResourceOption { + let resource = shared + .fetcher + .fetch(&resource_id) + .await + .ok() + .map(|source| match FluentResource::try_new(source) { + Ok(res) => ResourceOption::Some(Rc::new(res)), + Err((res, errors)) => { + if let Some(reporter) = &shared.error_reporter.borrow() { + reporter.borrow().report_errors( + errors + .into_iter() + .map(|e| L10nRegistryError::FluentError { + resource_id: resource_id.clone(), + loc: Some(calculate_pos_in_source(res.source(), e.pos.start)), + error: e.into(), + }) + .collect(), + ); + } + ResourceOption::Some(Rc::new(res)) + } + }) + .unwrap_or_else(|| ResourceOption::missing_resource(&resource_id)); + // insert the resource into the cache + shared.update_resource(resource_id, resource) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn calculate_source_pos() { + let source = r#" +key = Value + +key2 = Value 2 +"# + .trim(); + let result = calculate_pos_in_source(source, 0); + assert_eq!(result, (1, 1)); + + let result = calculate_pos_in_source(source, 1); + assert_eq!(result, (1, 2)); + + let result = calculate_pos_in_source(source, 12); + assert_eq!(result, (2, 1)); + + let result = calculate_pos_in_source(source, 13); + assert_eq!(result, (3, 1)); + } +} + +#[cfg(test)] +#[cfg(all(feature = "tokio", feature = "test-fluent"))] +mod tests_tokio { + use super::*; + use crate::testing::TestFileFetcher; + + static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl"; + static FTL_RESOURCE_MISSING: &str = "missing.ftl"; + + #[tokio::test] + async fn file_source_fetch() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + } + + #[tokio::test] + async fn file_source_fetch_missing() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await; + assert!(file.is_none()); + } + + #[tokio::test] + async fn file_source_already_loaded() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + } + + #[tokio::test] + async fn file_source_concurrent() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file1 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + let file2 = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + assert!(file1.await.is_some()); + assert!(file2.await.is_some()); + } + + #[test] + fn file_source_sync_after_async_fail() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let _ = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()); + let file2 = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), true); + assert!(file2.is_some()); + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/src/testing.rs b/intl/l10n/rust/l10nregistry-rs/src/testing.rs new file mode 100644 index 0000000000..f1ff47e2db --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/src/testing.rs @@ -0,0 +1,322 @@ +use crate::env::ErrorReporter; +use crate::errors::L10nRegistryError; +use crate::fluent::FluentBundle; +use crate::registry::BundleAdapter; +use crate::registry::L10nRegistry; +use crate::source::FileFetcher; +use async_trait::async_trait; +use fluent_fallback::{env::LocalesProvider, types::ResourceId}; +use fluent_testing::MockFileSystem; +use std::cell::RefCell; +use std::rc::Rc; +use unic_langid::LanguageIdentifier; + +pub struct RegistrySetup { + pub name: String, + pub file_sources: Vec<FileSource>, + pub locales: Vec<LanguageIdentifier>, +} + +pub struct FileSource { + pub name: String, + pub metasource: String, + pub locales: Vec<LanguageIdentifier>, + pub path_scheme: String, +} + +#[derive(Clone)] +pub struct MockBundleAdapter; + +impl BundleAdapter for MockBundleAdapter { + fn adapt_bundle(&self, _bundle: &mut FluentBundle) {} +} + +impl FileSource { + pub fn new<S>( + name: S, + metasource: Option<S>, + locales: Vec<LanguageIdentifier>, + path_scheme: S, + ) -> Self + where + S: ToString, + { + let metasource = match metasource { + Some(s) => s.to_string(), + None => String::default(), + }; + + Self { + name: name.to_string(), + metasource, + locales, + path_scheme: path_scheme.to_string(), + } + } +} + +impl RegistrySetup { + pub fn new( + name: &str, + file_sources: Vec<FileSource>, + locales: Vec<LanguageIdentifier>, + ) -> Self { + Self { + name: name.to_string(), + file_sources, + locales, + } + } +} + +impl From<fluent_testing::scenarios::structs::Scenario> for RegistrySetup { + fn from(s: fluent_testing::scenarios::structs::Scenario) -> Self { + Self { + name: s.name, + file_sources: s + .file_sources + .into_iter() + .map(|source| { + FileSource::new( + source.name, + None, + source + .locales + .into_iter() + .map(|l| l.parse().unwrap()) + .collect(), + source.path_scheme, + ) + }) + .collect(), + locales: s + .locales + .into_iter() + .map(|loc| loc.parse().unwrap()) + .collect(), + } + } +} + +impl From<&fluent_testing::scenarios::structs::Scenario> for RegistrySetup { + fn from(s: &fluent_testing::scenarios::structs::Scenario) -> Self { + Self { + name: s.name.clone(), + file_sources: s + .file_sources + .iter() + .map(|source| { + FileSource::new( + source.name.clone(), + None, + source.locales.iter().map(|l| l.parse().unwrap()).collect(), + source.path_scheme.clone(), + ) + }) + .collect(), + locales: s.locales.iter().map(|loc| loc.parse().unwrap()).collect(), + } + } +} + +#[derive(Default)] +struct InnerFileFetcher { + fs: MockFileSystem, +} + +#[derive(Clone)] +pub struct TestFileFetcher { + inner: Rc<InnerFileFetcher>, +} + +impl TestFileFetcher { + pub fn new() -> Self { + Self { + inner: Rc::new(InnerFileFetcher::default()), + } + } + + pub fn get_test_file_source( + &self, + name: &str, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + path: &str, + ) -> crate::source::FileSource { + crate::source::FileSource::new( + name.to_string(), + metasource, + locales, + path.to_string(), + Default::default(), + self.clone(), + ) + } + + pub fn get_test_file_source_with_index( + &self, + name: &str, + metasource: Option<String>, + locales: Vec<LanguageIdentifier>, + path: &str, + index: Vec<&str>, + ) -> crate::source::FileSource { + crate::source::FileSource::new_with_index( + name.to_string(), + metasource, + locales, + path.to_string(), + Default::default(), + self.clone(), + index.into_iter().map(|s| s.to_string()).collect(), + ) + } + + pub fn get_registry<S>(&self, setup: S) -> L10nRegistry<TestEnvironment, MockBundleAdapter> + where + S: Into<RegistrySetup>, + { + self.get_registry_and_environment(setup).1 + } + + pub fn get_registry_and_environment<S>( + &self, + setup: S, + ) -> ( + TestEnvironment, + L10nRegistry<TestEnvironment, MockBundleAdapter>, + ) + where + S: Into<RegistrySetup>, + { + let setup: RegistrySetup = setup.into(); + let provider = TestEnvironment::new(setup.locales); + + let reg = L10nRegistry::with_provider(provider.clone()); + let sources = setup + .file_sources + .into_iter() + .map(|source| { + let mut s = self.get_test_file_source( + &source.name, + Some(source.metasource), + source.locales, + &source.path_scheme, + ); + s.set_reporter(provider.clone()); + s + }) + .collect(); + reg.register_sources(sources).unwrap(); + (provider, reg) + } + + pub fn get_registry_and_environment_with_adapter<S, B>( + &self, + setup: S, + bundle_adapter: B, + ) -> (TestEnvironment, L10nRegistry<TestEnvironment, B>) + where + S: Into<RegistrySetup>, + B: BundleAdapter, + { + let setup: RegistrySetup = setup.into(); + let provider = TestEnvironment::new(setup.locales); + + let mut reg = L10nRegistry::with_provider(provider.clone()); + let sources = setup + .file_sources + .into_iter() + .map(|source| { + let mut s = self.get_test_file_source( + &source.name, + None, + source.locales, + &source.path_scheme, + ); + s.set_reporter(provider.clone()); + s + }) + .collect(); + reg.register_sources(sources).unwrap(); + reg.set_bundle_adapter(bundle_adapter) + .expect("Failed to set bundle adapter."); + (provider, reg) + } +} + +#[async_trait(?Send)] +impl FileFetcher for TestFileFetcher { + fn fetch_sync(&self, resource_id: &ResourceId) -> std::io::Result<String> { + self.inner.fs.get_test_file_sync(&resource_id.value) + } + + async fn fetch(&self, resource_id: &ResourceId) -> std::io::Result<String> { + self.inner.fs.get_test_file_async(&resource_id.value).await + } +} + +pub enum ErrorStrategy { + Panic, + Report, + Nothing, +} + +pub struct InnerTestEnvironment { + locales: Vec<LanguageIdentifier>, + errors: Vec<L10nRegistryError>, + error_strategy: ErrorStrategy, +} + +#[derive(Clone)] +pub struct TestEnvironment { + inner: Rc<RefCell<InnerTestEnvironment>>, +} + +impl TestEnvironment { + pub fn new(locales: Vec<LanguageIdentifier>) -> Self { + Self { + inner: Rc::new(RefCell::new(InnerTestEnvironment { + locales, + errors: vec![], + error_strategy: ErrorStrategy::Report, + })), + } + } + + pub fn set_locales(&self, locales: Vec<LanguageIdentifier>) { + self.inner.borrow_mut().locales = locales; + } + + pub fn errors(&self) -> Vec<L10nRegistryError> { + self.inner.borrow().errors.clone() + } + + pub fn clear_errors(&self) { + self.inner.borrow_mut().errors.clear() + } +} + +impl LocalesProvider for TestEnvironment { + type Iter = std::vec::IntoIter<LanguageIdentifier>; + + fn locales(&self) -> Self::Iter { + self.inner.borrow().locales.clone().into_iter() + } +} + +impl ErrorReporter for TestEnvironment { + fn report_errors(&self, errors: Vec<L10nRegistryError>) { + match self.inner.borrow().error_strategy { + ErrorStrategy::Panic => { + panic!("Errors: {:#?}", errors); + } + ErrorStrategy::Report => { + #[cfg(test)] // Don't let printing affect benchmarks + eprintln!("Errors: {:#?}", errors); + } + ErrorStrategy::Nothing => {} + } + self.inner.borrow_mut().errors.extend(errors); + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/tests/localization.rs b/intl/l10n/rust/l10nregistry-rs/tests/localization.rs new file mode 100644 index 0000000000..dd6d7d9c9f --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/tests/localization.rs @@ -0,0 +1,201 @@ +use std::borrow::Cow; + +use fluent_fallback::{ + env::LocalesProvider, + types::{L10nKey, ResourceId}, + Localization, +}; +use l10nregistry::testing::{ + FileSource, MockBundleAdapter, RegistrySetup, TestEnvironment, TestFileFetcher, +}; +use serial_test::serial; +use unic_langid::{langid, LanguageIdentifier}; + +type L10nRegistry = l10nregistry::registry::L10nRegistry<TestEnvironment, MockBundleAdapter>; + +static LOCALES: &[LanguageIdentifier] = &[langid!("pl"), langid!("en-US")]; +static mut FILE_FETCHER: Option<TestFileFetcher> = None; +static mut L10N_REGISTRY: Option<L10nRegistry> = None; + +const FTL_RESOURCE: &str = "toolkit/updates/history.ftl"; +const L10N_ID_PL_EN: (&str, Option<&str>) = ("history-title", Some("Historia aktualizacji")); +const L10N_ID_MISSING: (&str, Option<&str>) = ("missing-id", None); +const L10N_ID_ONLY_EN: (&str, Option<&str>) = ( + "history-intro", + Some("The following updates have been installed"), +); + +fn get_file_fetcher() -> &'static TestFileFetcher { + let fetcher: &mut Option<TestFileFetcher> = unsafe { &mut FILE_FETCHER }; + + fetcher.get_or_insert_with(|| TestFileFetcher::new()) +} + +fn get_l10n_registry() -> &'static L10nRegistry { + let reg: &mut Option<L10nRegistry> = unsafe { &mut L10N_REGISTRY }; + + reg.get_or_insert_with(|| { + let fetcher = get_file_fetcher(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + None, + get_app_locales().to_vec(), + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + None, + get_app_locales().to_vec(), + "browser/{locale}/", + ), + ], + get_app_locales().to_vec(), + ); + fetcher.get_registry_and_environment(setup).1 + }) +} + +fn get_app_locales() -> &'static [LanguageIdentifier] { + LOCALES +} + +struct LocalesService; + +impl LocalesProvider for LocalesService { + type Iter = std::vec::IntoIter<LanguageIdentifier>; + + fn locales(&self) -> Self::Iter { + get_app_locales().to_vec().into_iter() + } +} + +fn sync_localization( + reg: &'static L10nRegistry, + res_ids: Vec<ResourceId>, +) -> Localization<L10nRegistry, LocalesService> { + Localization::with_env(res_ids, true, LocalesService, reg.clone()) +} + +fn async_localization( + reg: &'static L10nRegistry, + res_ids: Vec<ResourceId>, +) -> Localization<L10nRegistry, LocalesService> { + Localization::with_env(res_ids, false, LocalesService, reg.clone()) +} + +fn setup_sync_test() -> Localization<L10nRegistry, LocalesService> { + sync_localization(get_l10n_registry(), vec![FTL_RESOURCE.into()]) +} + +fn setup_async_test() -> Localization<L10nRegistry, LocalesService> { + async_localization(get_l10n_registry(), vec![FTL_RESOURCE.into()]) +} + +#[test] +#[serial] +fn localization_format_value_sync() { + let loc = setup_sync_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + for query in &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN] { + let value = bundles + .format_value_sync(query.0, None, &mut errors) + .unwrap(); + assert_eq!(value, query.1.map(|s| Cow::Borrowed(s))); + } + + assert_eq!(errors.len(), 4); +} + +#[test] +#[serial] +fn localization_format_values_sync() { + let loc = setup_sync_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + let ids = &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN]; + let keys = ids + .iter() + .map(|query| L10nKey { + id: query.0.into(), + args: None, + }) + .collect::<Vec<_>>(); + + let values = bundles.format_values_sync(&keys, &mut errors).unwrap(); + + assert_eq!(values.len(), ids.len()); + + for (value, query) in values.iter().zip(ids) { + if let Some(expected) = query.1 { + assert_eq!(*value, Some(Cow::Borrowed(expected))); + } + } + assert_eq!(errors.len(), 4); +} + +#[tokio::test] +#[serial] +async fn localization_format_value_async() { + let loc = setup_async_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + for query in &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN] { + let value = bundles.format_value(query.0, None, &mut errors).await; + if let Some(expected) = query.1 { + assert_eq!(value, Some(Cow::Borrowed(expected))); + } + } +} + +#[tokio::test] +#[serial] +async fn localization_format_values_async() { + let loc = setup_async_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + + let ids = &[L10N_ID_PL_EN, L10N_ID_MISSING, L10N_ID_ONLY_EN]; + let keys = ids + .iter() + .map(|query| L10nKey { + id: query.0.into(), + args: None, + }) + .collect::<Vec<_>>(); + + let values = bundles.format_values(&keys, &mut errors).await; + + assert_eq!(values.len(), ids.len()); + + for (value, query) in values.iter().zip(ids) { + if let Some(expected) = query.1 { + assert_eq!(*value, Some(Cow::Borrowed(expected))); + } + } +} + +#[tokio::test] +#[serial] +async fn localization_upgrade() { + let mut loc = setup_sync_test(); + let bundles = loc.bundles(); + let mut errors = vec![]; + let value = bundles + .format_value_sync(L10N_ID_PL_EN.0, None, &mut errors) + .unwrap(); + assert_eq!(value, L10N_ID_PL_EN.1.map(|s| Cow::Borrowed(s))); + + loc.set_async(); + let bundles = loc.bundles(); + let value = bundles + .format_value(L10N_ID_PL_EN.0, None, &mut errors) + .await; + assert_eq!(value, L10N_ID_PL_EN.1.map(|s| Cow::Borrowed(s))); +} diff --git a/intl/l10n/rust/l10nregistry-rs/tests/registry.rs b/intl/l10n/rust/l10nregistry-rs/tests/registry.rs new file mode 100644 index 0000000000..2a7569b7aa --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/tests/registry.rs @@ -0,0 +1,304 @@ +use l10nregistry::testing::{FileSource, RegistrySetup, TestFileFetcher}; +use unic_langid::LanguageIdentifier; + +static FTL_RESOURCE_TOOLKIT: &str = "toolkit/global/textActions.ftl"; +static FTL_RESOURCE_BROWSER: &str = "branding/brand.ftl"; + +#[test] +fn test_get_sources_for_resource() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + { + let metasources = reg + .try_borrow_metasources() + .expect("Unable to borrow metasources."); + + let toolkit = metasources.file_source_by_name(0, "toolkit").unwrap(); + let browser = metasources.file_source_by_name(0, "browser").unwrap(); + let toolkit_resource_id = FTL_RESOURCE_TOOLKIT.into(); + + let mut i = metasources.get_sources_for_resource(0, &en_us, &toolkit_resource_id); + + assert_eq!(i.next(), Some(toolkit)); + assert_eq!(i.next(), Some(browser)); + assert_eq!(i.next(), None); + + assert!(browser + .fetch_file_sync(&en_us, &FTL_RESOURCE_TOOLKIT.into(), false) + .is_none()); + + let mut i = metasources.get_sources_for_resource(0, &en_us, &toolkit_resource_id); + assert_eq!(i.next(), Some(toolkit)); + assert_eq!(i.next(), None); + + assert!(toolkit + .fetch_file_sync(&en_us, &FTL_RESOURCE_TOOLKIT.into(), false) + .is_some()); + + let mut i = metasources.get_sources_for_resource(0, &en_us, &toolkit_resource_id); + assert_eq!(i.next(), Some(toolkit)); + assert_eq!(i.next(), None); + } +} + +#[test] +fn test_generate_bundles_for_lang_sync() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_for_lang_sync(en_us.clone(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[test] +fn test_generate_bundles_sync() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let lang_ids = vec![en_us]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[tokio::test] +async fn test_generate_bundles_for_lang() { + use futures::stream::StreamExt; + + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg + .generate_bundles_for_lang(en_us, paths) + .expect("Failed to get GenerateBundles."); + + assert!(i.next().await.is_some()); + assert!(i.next().await.is_none()); +} + +#[tokio::test] +async fn test_generate_bundles() { + use futures::stream::StreamExt; + + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let langs = vec![en_us]; + let mut i = reg + .generate_bundles(langs.into_iter(), paths) + .expect("Failed to get GenerateBundles."); + + assert!(i.next().await.is_some()); + assert!(i.next().await.is_none()); +} + +#[test] +fn test_manage_sources() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"), + FileSource::new("browser", None, vec![en_us.clone()], "browser/{locale}/"), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let lang_ids = vec![en_us]; + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_none()); + + reg.clone() + .remove_sources(vec!["toolkit"]) + .expect("Failed to remove a source."); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_none()); + + let paths = vec![FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_some()); + assert!(i.next().is_none()); + + reg.register_sources(vec![fetcher.get_test_file_source( + "toolkit", + None, + lang_ids.clone(), + "browser/{locale}/", + )]) + .expect("Failed to register a source."); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_none()); + + reg.update_sources(vec![fetcher.get_test_file_source( + "toolkit", + None, + lang_ids.clone(), + "toolkit/{locale}/", + )]) + .expect("Failed to update a source."); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let mut i = reg.generate_bundles_sync(lang_ids.clone().into_iter(), paths); + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[test] +fn test_generate_bundles_with_metasources_sync() { + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + Some("app"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("app"), + vec![en_us.clone()], + "browser/{locale}/", + ), + FileSource::new( + "toolkit", + Some("langpack"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("langpack"), + vec![en_us.clone()], + "browser/{locale}/", + ), + ], + vec![en_us.clone()], + ); + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let lang_ids = vec![en_us]; + let mut i = reg.generate_bundles_sync(lang_ids.into_iter(), paths); + + assert!(i.next().is_some()); + assert!(i.next().is_some()); + assert!(i.next().is_none()); +} + +#[tokio::test] +async fn test_generate_bundles_with_metasources() { + use futures::stream::StreamExt; + + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let setup = RegistrySetup::new( + "test", + vec![ + FileSource::new( + "toolkit", + Some("app"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("app"), + vec![en_us.clone()], + "browser/{locale}/", + ), + FileSource::new( + "toolkit", + Some("langpack"), + vec![en_us.clone()], + "toolkit/{locale}/", + ), + FileSource::new( + "browser", + Some("langpack"), + vec![en_us.clone()], + "browser/{locale}/", + ), + ], + vec![en_us.clone()], + ); + + let fetcher = TestFileFetcher::new(); + let (_, reg) = fetcher.get_registry_and_environment(setup); + + let paths = vec![FTL_RESOURCE_TOOLKIT.into(), FTL_RESOURCE_BROWSER.into()]; + let langs = vec![en_us]; + let mut i = reg + .generate_bundles(langs.into_iter(), paths) + .expect("Failed to get GenerateBundles."); + + assert!(i.next().await.is_some()); + assert!(i.next().await.is_some()); + assert!(i.next().await.is_none()); +} diff --git a/intl/l10n/rust/l10nregistry-rs/tests/scenarios_async.rs b/intl/l10n/rust/l10nregistry-rs/tests/scenarios_async.rs new file mode 100644 index 0000000000..839cd4b3d8 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/tests/scenarios_async.rs @@ -0,0 +1,109 @@ +use fluent_bundle::FluentArgs; +use fluent_fallback::Localization; +use fluent_testing::get_scenarios; +use l10nregistry::fluent::FluentBundle; +use l10nregistry::registry::BundleAdapter; +use l10nregistry::testing::{RegistrySetup, TestFileFetcher}; + +#[derive(Clone)] +struct ScenarioBundleAdapter {} + +impl BundleAdapter for ScenarioBundleAdapter { + fn adapt_bundle(&self, bundle: &mut FluentBundle) { + bundle.set_use_isolating(false); + bundle + .add_function("PLATFORM", |_positional, _named| "linux".into()) + .expect("Failed to add a function to the bundle."); + } +} + +#[tokio::test] +async fn scenarios_async() { + use fluent_testing::scenarios::structs::Scenario; + let fetcher = TestFileFetcher::new(); + + let scenarios = get_scenarios(); + + let adapter = ScenarioBundleAdapter {}; + + let cannot_produce_bundle = |scenario: &Scenario| { + scenario + .queries + .iter() + .any(|query| query.exceptional_context.blocks_bundle_generation()) + }; + + for scenario in scenarios { + println!("scenario: {}", scenario.name); + let setup: RegistrySetup = (&scenario).into(); + let (env, reg) = fetcher.get_registry_and_environment_with_adapter(setup, adapter.clone()); + + let loc = Localization::with_env(scenario.res_ids.clone(), false, env.clone(), reg); + let bundles = loc.bundles(); + let no_bundles = cannot_produce_bundle(&scenario); + + let mut errors = vec![]; + + for query in scenario.queries.iter() { + let errors_start_len = errors.len(); + + let args = query.input.args.as_ref().map(|args| { + let mut result = FluentArgs::new(); + for arg in args.as_slice() { + result.set(arg.id.clone(), arg.value.clone()); + } + result + }); + + if let Some(output) = &query.output { + if let Some(value) = &output.value { + let v = bundles + .format_value(&query.input.id, args.as_ref(), &mut errors) + .await; + if no_bundles || query.exceptional_context.causes_failed_value_lookup() { + assert!(v.is_none()); + if no_bundles { + continue; + } + } else { + assert_eq!(v.unwrap(), value.as_str()) + } + } + } + + if query.exceptional_context.causes_reported_format_error() { + assert!( + errors.len() > errors_start_len, + "expected reported errors for query {:#?}", + query + ); + } else { + assert_eq!( + errors.len(), + errors_start_len, + "expected no reported errors for query {:#?}", + query + ); + } + } + + if scenario + .queries + .iter() + .any(|query| query.exceptional_context.missing_required_resource()) + { + assert!( + !env.errors().is_empty(), + "expected errors for scenario {{ {} }}, but found none", + scenario.name + ); + } else { + assert!( + env.errors().is_empty(), + "expected no errors for scenario {{ {} }}, but found {:#?}", + scenario.name, + env.errors() + ); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/tests/scenarios_sync.rs b/intl/l10n/rust/l10nregistry-rs/tests/scenarios_sync.rs new file mode 100644 index 0000000000..280b32e927 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/tests/scenarios_sync.rs @@ -0,0 +1,107 @@ +use fluent_bundle::FluentArgs; +use fluent_fallback::Localization; +use fluent_testing::get_scenarios; +use l10nregistry::fluent::FluentBundle; +use l10nregistry::registry::BundleAdapter; +use l10nregistry::testing::{RegistrySetup, TestFileFetcher}; + +#[derive(Clone)] +struct ScenarioBundleAdapter {} + +impl BundleAdapter for ScenarioBundleAdapter { + fn adapt_bundle(&self, bundle: &mut FluentBundle) { + bundle.set_use_isolating(false); + bundle + .add_function("PLATFORM", |_positional, _named| "linux".into()) + .expect("Failed to add a function to the bundle."); + } +} + +#[test] +fn scenarios_sync() { + use fluent_testing::scenarios::structs::Scenario; + let fetcher = TestFileFetcher::new(); + + let scenarios = get_scenarios(); + + let adapter = ScenarioBundleAdapter {}; + + let cannot_produce_bundle = |scenario: &Scenario| { + scenario + .queries + .iter() + .any(|query| query.exceptional_context.blocks_bundle_generation()) + }; + + for scenario in scenarios { + println!("scenario: {}", scenario.name); + let setup: RegistrySetup = (&scenario).into(); + let (env, reg) = fetcher.get_registry_and_environment_with_adapter(setup, adapter.clone()); + + let loc = Localization::with_env(scenario.res_ids.clone(), true, env.clone(), reg); + let bundles = loc.bundles(); + let no_bundles = cannot_produce_bundle(&scenario); + + let mut errors = vec![]; + + for query in scenario.queries.iter() { + let errors_start_len = errors.len(); + + let args = query.input.args.as_ref().map(|args| { + let mut result = FluentArgs::new(); + for arg in args.as_slice() { + result.set(arg.id.clone(), arg.value.clone()); + } + result + }); + + if let Some(output) = &query.output { + if let Some(value) = &output.value { + let v = bundles.format_value_sync(&query.input.id, args.as_ref(), &mut errors); + if no_bundles || query.exceptional_context.causes_failed_value_lookup() { + assert!(v.is_err() || v.unwrap().is_none()); + if no_bundles { + continue; + } + } else { + assert_eq!(v.unwrap().unwrap(), value.as_str()) + } + } + } + + if query.exceptional_context.causes_reported_format_error() { + assert!( + errors.len() > errors_start_len, + "expected reported errors for query {:#?}", + query + ); + } else { + assert_eq!( + errors.len(), + errors_start_len, + "expected no reported errors for query {:#?}", + query + ); + } + } + + if scenario + .queries + .iter() + .any(|query| query.exceptional_context.missing_required_resource()) + { + assert!( + !env.errors().is_empty(), + "expected errors for scenario {{ {} }}, but found none", + scenario.name + ); + } else { + assert!( + env.errors().is_empty(), + "expected no errors for scenario {{ {} }}, but found {:#?}", + scenario.name, + env.errors() + ); + } + } +} diff --git a/intl/l10n/rust/l10nregistry-rs/tests/source.rs b/intl/l10n/rust/l10nregistry-rs/tests/source.rs new file mode 100644 index 0000000000..a54ff23438 --- /dev/null +++ b/intl/l10n/rust/l10nregistry-rs/tests/source.rs @@ -0,0 +1,305 @@ +use fluent_fallback::types::{ResourceType, ToResourceId}; +use futures::future::join_all; +use l10nregistry::testing::TestFileFetcher; +use unic_langid::LanguageIdentifier; + +static FTL_RESOURCE_PRESENT: &str = "toolkit/global/textActions.ftl"; +static FTL_RESOURCE_MISSING: &str = "missing.ftl"; + +#[test] +fn test_fetch_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false); + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Required), + false, + ); + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional), + false, + ); + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file_sync(&en_us, &FTL_RESOURCE_MISSING.into(), false); + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Required), + false, + ); + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1.fetch_file_sync( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional), + false, + ); + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(!file.is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()).await; + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Required), + ) + .await; + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional), + ) + .await; + assert!(file.is_some()); + assert!(!file.is_none()); + assert!(!file.is_required_and_missing()); + + let file = fs1.fetch_file(&en_us, &FTL_RESOURCE_MISSING.into()).await; + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Required), + ) + .await; + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(file.is_required_and_missing()); + + let file = fs1 + .fetch_file( + &en_us, + &FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional), + ) + .await; + assert!(!file.is_some()); + assert!(file.is_none()); + assert!(!file.is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_sync_2_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert!(fs1 + .fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false) + .is_some()); + assert!(fs1 + .fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()) + .await + .is_some()); + assert!(fs1 + .fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false) + .is_some()); +} + +#[tokio::test] +async fn test_fetch_async_2_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert!(fs1 + .fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()) + .await + .is_some()); + assert!(fs1 + .fetch_file_sync(&en_us, &FTL_RESOURCE_PRESENT.into(), false) + .is_some()); +} + +#[test] +fn test_fetch_has_value_required_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.into(); + let path_missing = FTL_RESOURCE_MISSING.into(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file_sync(&en_us, &path, false).is_some()); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); +} + +#[test] +fn test_fetch_has_value_optional_sync() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional); + let path_missing = FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file_sync(&en_us, &path, false).is_some()); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); +} + +#[tokio::test] +async fn test_fetch_has_value_required_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.into(); + let path_missing = FTL_RESOURCE_MISSING.into(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file(&en_us, &path).await.is_some()); + println!("Completed"); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + + assert!(fs1.fetch_file(&en_us, &path_missing).await.is_none()); + assert!(fs1 + .fetch_file(&en_us, &path_missing) + .await + .is_required_and_missing()); + + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); + + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert!(fs1 + .fetch_file_sync(&en_us, &path_missing, false) + .is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_has_value_optional_async() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT.to_resource_id(ResourceType::Optional); + let path_missing = FTL_RESOURCE_MISSING.to_resource_id(ResourceType::Optional); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + assert_eq!(fs1.has_file(&en_us, &path), None); + assert!(fs1.fetch_file(&en_us, &path).await.is_some()); + println!("Completed"); + assert_eq!(fs1.has_file(&en_us, &path), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing), None); + assert!(fs1.fetch_file(&en_us, &path_missing).await.is_none()); + assert!(!fs1 + .fetch_file(&en_us, &path_missing) + .await + .is_required_and_missing()); + + assert_eq!(fs1.has_file(&en_us, &path_missing), Some(false)); + + assert!(fs1.fetch_file_sync(&en_us, &path_missing, false).is_none()); + assert!(!fs1 + .fetch_file_sync(&en_us, &path_missing, false) + .is_required_and_missing()); +} + +#[tokio::test] +async fn test_fetch_async_consecutive() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + + let fs1 = + fetcher.get_test_file_source("toolkit", None, vec![en_us.clone()], "toolkit/{locale}/"); + + let results = join_all(vec![ + fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()), + fs1.fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()), + ]) + .await; + assert!(results[0].is_some()); + assert!(results[1].is_some()); + + assert!(fs1 + .fetch_file(&en_us, &FTL_RESOURCE_PRESENT.into()) + .await + .is_some()); +} + +#[test] +fn test_indexed() { + let fetcher = TestFileFetcher::new(); + let en_us: LanguageIdentifier = "en-US".parse().unwrap(); + let path = FTL_RESOURCE_PRESENT; + let path_missing = FTL_RESOURCE_MISSING; + + let fs1 = fetcher.get_test_file_source_with_index( + "toolkit", + None, + vec![en_us.clone()], + "toolkit/{locale}/", + vec!["toolkit/en-US/toolkit/global/textActions.ftl"], + ); + + assert_eq!(fs1.has_file(&en_us, &path.into()), Some(true)); + assert!(fs1.fetch_file_sync(&en_us, &path.into(), false).is_some()); + assert_eq!(fs1.has_file(&en_us, &path.into()), Some(true)); + + assert_eq!(fs1.has_file(&en_us, &path_missing.into()), Some(false)); + assert!(fs1 + .fetch_file_sync(&en_us, &path_missing.into(), false) + .is_none()); + assert_eq!(fs1.has_file(&en_us, &path_missing.into()), Some(false)); +} diff --git a/intl/l10n/rust/localization-ffi/Cargo.toml b/intl/l10n/rust/localization-ffi/Cargo.toml new file mode 100644 index 0000000000..59445bdc5f --- /dev/null +++ b/intl/l10n/rust/localization-ffi/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "localization-ffi" +version = "0.1.0" +authors = ["The Mozilla Project Developers"] +edition = "2018" +license = "MPL-2.0" + +[dependencies] +futures-channel = "0.3" +futures = "0.3" +nserror = { path = "../../../../xpcom/rust/nserror" } +nsstring = { path = "../../../../xpcom/rust/nsstring" } +l10nregistry = { path = "../l10nregistry-rs" } +fluent = "0.16.0" +unic-langid = "0.9" +thin-vec = { version = "0.2.1", features = ["gecko-ffi"] } +async-trait = "0.1" +moz_task = { path = "../../../../xpcom/rust/moz_task" } +fluent-ffi = { path = "../fluent-ffi" } +fluent-fallback = "0.7.0" +l10nregistry-ffi = { path = "../l10nregistry-ffi" } +xpcom = { path = "../../../../xpcom/rust/xpcom" } +cstr = "0.2" diff --git a/intl/l10n/rust/localization-ffi/cbindgen.toml b/intl/l10n/rust/localization-ffi/cbindgen.toml new file mode 100644 index 0000000000..584ce6e9cc --- /dev/null +++ b/intl/l10n/rust/localization-ffi/cbindgen.toml @@ -0,0 +1,33 @@ +header = """/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */""" +autogen_warning = """/* DO NOT MODIFY THIS MANUALLY! This file was generated using cbindgen. See RunCbindgen.py */ +#ifndef mozilla_intl_l10n_LocalizationBindings_h +#error "Don't include this file directly, instead include LocalizationBindings.h" +#endif +""" +include_version = true +braces = "SameLine" +line_length = 100 +tab_width = 2 +language = "C++" +namespaces = ["mozilla", "intl", "ffi"] +includes = ["mozilla/intl/RegistryBindings.h"] + +[parse] +parse_deps = true +include = ["fluent-fallback", "l10nregistry-ffi"] + +[enum] +derive_helper_methods = true + +[export.rename] +"ThinVec" = "nsTArray" +"Promise" = "dom::Promise" + +[export] +# These are already exported by l10nregistry-ffi. +exclude = [ + "GeckoResourceId", + "GeckoResourceType", +] diff --git a/intl/l10n/rust/localization-ffi/src/lib.rs b/intl/l10n/rust/localization-ffi/src/lib.rs new file mode 100644 index 0000000000..8896231786 --- /dev/null +++ b/intl/l10n/rust/localization-ffi/src/lib.rs @@ -0,0 +1,625 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ + +use fluent::FluentValue; +use fluent_fallback::{ + types::{ + L10nAttribute as FluentL10nAttribute, L10nKey as FluentL10nKey, + L10nMessage as FluentL10nMessage, ResourceId, + }, + Localization, +}; +use fluent_ffi::{convert_args, FluentArgs, FluentArgument, L10nArg}; +use l10nregistry_ffi::{ + env::GeckoEnvironment, + registry::{get_l10n_registry, GeckoL10nRegistry, GeckoResourceId}, +}; +use nsstring::{nsACString, nsCString}; +use std::os::raw::c_void; +use std::{borrow::Cow, cell::RefCell}; +use thin_vec::ThinVec; +use unic_langid::LanguageIdentifier; +use xpcom::{ + interfaces::{nsIRunnablePriority}, + RefCounted, RefPtr, Refcnt, +}; + +#[derive(Debug)] +#[repr(C)] +pub struct L10nKey<'s> { + id: &'s nsACString, + args: ThinVec<L10nArg<'s>>, +} + +impl<'s> From<&'s L10nKey<'s>> for FluentL10nKey<'static> { + fn from(input: &'s L10nKey<'s>) -> Self { + FluentL10nKey { + id: input.id.to_utf8().to_string().into(), + args: convert_args_to_owned(&input.args), + } + } +} + +// This is a variant of `convert_args` from `fluent-ffi` with a 'static constrain +// put on the resulting `FluentArgs` to make it acceptable into `spqwn_current_thread`. +pub fn convert_args_to_owned(args: &[L10nArg]) -> Option<FluentArgs<'static>> { + if args.is_empty() { + return None; + } + + let mut result = FluentArgs::with_capacity(args.len()); + for arg in args { + let val = match arg.value { + FluentArgument::Double_(d) => FluentValue::from(d), + // We need this to be owned because we pass the result into `spawn_local`. + FluentArgument::String(s) => FluentValue::from(Cow::Owned(s.to_utf8().to_string())), + }; + result.set(arg.id.to_string(), val); + } + Some(result) +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nAttribute { + name: nsCString, + value: nsCString, +} + +impl From<FluentL10nAttribute<'_>> for L10nAttribute { + fn from(attr: FluentL10nAttribute<'_>) -> Self { + Self { + name: nsCString::from(&*attr.name), + value: nsCString::from(&*attr.value), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct L10nMessage { + value: nsCString, + attributes: ThinVec<L10nAttribute>, +} + +impl std::default::Default for L10nMessage { + fn default() -> Self { + Self { + value: nsCString::new(), + attributes: ThinVec::new(), + } + } +} + +#[derive(Debug)] +#[repr(C)] +pub struct OptionalL10nMessage { + is_present: bool, + message: L10nMessage, +} + +impl From<FluentL10nMessage<'_>> for L10nMessage { + fn from(input: FluentL10nMessage) -> Self { + let value = if let Some(value) = input.value { + value.to_string().into() + } else { + let mut s = nsCString::new(); + s.set_is_void(true); + s + }; + Self { + value, + attributes: input.attributes.into_iter().map(Into::into).collect(), + } + } +} + +pub struct LocalizationRc { + inner: RefCell<Localization<GeckoL10nRegistry, GeckoEnvironment>>, + refcnt: Refcnt, +} + +// xpcom::RefPtr support +unsafe impl RefCounted for LocalizationRc { + unsafe fn addref(&self) { + localization_addref(self); + } + unsafe fn release(&self) { + localization_release(self); + } +} + +impl LocalizationRc { + pub fn new( + res_ids: Vec<ResourceId>, + is_sync: bool, + registry: Option<&GeckoL10nRegistry>, + locales: Option<Vec<LanguageIdentifier>>, + ) -> RefPtr<Self> { + let env = GeckoEnvironment::new(locales); + let inner = if let Some(reg) = registry { + Localization::with_env(res_ids, is_sync, env, reg.clone()) + } else { + let reg = (*get_l10n_registry()).clone(); + Localization::with_env(res_ids, is_sync, env, reg) + }; + + let loc = Box::new(LocalizationRc { + inner: RefCell::new(inner), + refcnt: unsafe { Refcnt::new() }, + }); + + unsafe { + RefPtr::from_raw(Box::into_raw(loc)) + .expect("Failed to create RefPtr<LocalizationRc> from Box<LocalizationRc>") + } + } + + pub fn add_resource_id(&self, res_id: ResourceId) { + self.inner.borrow_mut().add_resource_id(res_id); + } + + pub fn add_resource_ids(&self, res_ids: Vec<ResourceId>) { + self.inner.borrow_mut().add_resource_ids(res_ids); + } + + pub fn remove_resource_id(&self, res_id: ResourceId) -> usize { + self.inner.borrow_mut().remove_resource_id(res_id) + } + + pub fn remove_resource_ids(&self, res_ids: Vec<ResourceId>) -> usize { + self.inner.borrow_mut().remove_resource_ids(res_ids) + } + + pub fn set_async(&self) { + if self.is_sync() { + self.inner.borrow_mut().set_async(); + } + } + + pub fn is_sync(&self) -> bool { + self.inner.borrow().is_sync() + } + + pub fn on_change(&self) { + self.inner.borrow_mut().on_change(); + } + + pub fn format_value_sync( + &self, + id: &nsACString, + args: &ThinVec<L10nArg>, + ret_val: &mut nsACString, + ret_err: &mut ThinVec<nsCString>, + ) -> bool { + let mut errors = vec![]; + let args = convert_args(&args); + if let Ok(value) = self.inner.borrow().bundles().format_value_sync( + &id.to_utf8(), + args.as_ref(), + &mut errors, + ) { + if let Some(value) = value { + ret_val.assign(&value); + } else { + ret_val.set_is_void(true); + } + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &[id], |id| id.to_string()); + ret_err.extend(errors.into_iter().map(|err| err.to_string().into())); + true + } else { + false + } + } + + pub fn format_values_sync( + &self, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<nsCString>, + ret_err: &mut ThinVec<nsCString>, + ) -> bool { + ret_val.reserve(keys.len()); + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + let mut errors = vec![]; + if let Ok(values) = self + .inner + .borrow() + .bundles() + .format_values_sync(&keys, &mut errors) + { + for value in values.iter() { + if let Some(value) = value { + ret_val.push(value.as_ref().into()); + } else { + let mut void_string = nsCString::new(); + void_string.set_is_void(true); + ret_val.push(void_string); + } + } + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + ret_err.extend(errors.into_iter().map(|err| err.to_string().into())); + true + } else { + false + } + } + + pub fn format_messages_sync( + &self, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<OptionalL10nMessage>, + ret_err: &mut ThinVec<nsCString>, + ) -> bool { + ret_val.reserve(keys.len()); + let mut errors = vec![]; + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + if let Ok(messages) = self + .inner + .borrow() + .bundles() + .format_messages_sync(&keys, &mut errors) + { + for msg in messages { + ret_val.push(if let Some(msg) = msg { + OptionalL10nMessage { + is_present: true, + message: msg.into(), + } + } else { + OptionalL10nMessage { + is_present: false, + message: L10nMessage::default(), + } + }); + } + assert_eq!(keys.len(), ret_val.len()); + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + ret_err.extend(errors.into_iter().map(|err| err.to_string().into())); + true + } else { + false + } + } + + pub fn format_value( + &self, + id: &nsACString, + args: &ThinVec<L10nArg>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &nsACString, &ThinVec<nsCString>), + ) { + let bundles = self.inner.borrow().bundles().clone(); + + let args = convert_args_to_owned(&args); + + let id = nsCString::from(id); + let strong_promise = RefPtr::new(promise); + + moz_task::TaskBuilder::new("LocalizationRc::format_value", async move { + let mut errors = vec![]; + let value = if let Some(value) = bundles + .format_value(&id.to_utf8(), args.as_ref(), &mut errors) + .await + { + let v: nsCString = value.to_string().into(); + v + } else { + let mut v = nsCString::new(); + v.set_is_void(true); + v + }; + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &[id], |id| id.to_string()); + let errors = errors + .into_iter() + .map(|err| err.to_string().into()) + .collect(); + callback(&strong_promise, &value, &errors); + }) + .priority(nsIRunnablePriority::PRIORITY_RENDER_BLOCKING as u32) + .spawn_local() + .detach(); + } + + pub fn format_values( + &self, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &ThinVec<nsCString>, &ThinVec<nsCString>), + ) { + let bundles = self.inner.borrow().bundles().clone(); + + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + + let strong_promise = RefPtr::new(promise); + + moz_task::TaskBuilder::new("LocalizationRc::format_values", async move { + let mut errors = vec![]; + let ret_val = bundles + .format_values(&keys, &mut errors) + .await + .into_iter() + .map(|value| { + if let Some(value) = value { + nsCString::from(value.as_ref()) + } else { + let mut v = nsCString::new(); + v.set_is_void(true); + v + } + }) + .collect::<ThinVec<_>>(); + + assert_eq!(keys.len(), ret_val.len()); + + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + let errors = errors + .into_iter() + .map(|err| err.to_string().into()) + .collect(); + + callback(&strong_promise, &ret_val, &errors); + }) + .priority(nsIRunnablePriority::PRIORITY_RENDER_BLOCKING as u32) + .spawn_local() + .detach(); + } + + pub fn format_messages( + &self, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn( + &xpcom::Promise, + &ThinVec<OptionalL10nMessage>, + &ThinVec<nsCString>, + ), + ) { + let bundles = self.inner.borrow().bundles().clone(); + + let keys: Vec<FluentL10nKey> = keys.into_iter().map(|k| k.into()).collect(); + + let strong_promise = RefPtr::new(promise); + + moz_task::TaskBuilder::new("LocalizationRc::format_messages", async move { + let mut errors = vec![]; + let ret_val = bundles + .format_messages(&keys, &mut errors) + .await + .into_iter() + .map(|msg| { + if let Some(msg) = msg { + OptionalL10nMessage { + is_present: true, + message: msg.into(), + } + } else { + OptionalL10nMessage { + is_present: false, + message: L10nMessage::default(), + } + } + }) + .collect::<ThinVec<_>>(); + + assert_eq!(keys.len(), ret_val.len()); + + #[cfg(debug_assertions)] + debug_assert_variables_exist(&errors, &keys, |key| key.id.to_string()); + + let errors = errors + .into_iter() + .map(|err| err.to_string().into()) + .collect(); + + callback(&strong_promise, &ret_val, &errors); + }) + .priority(nsIRunnablePriority::PRIORITY_RENDER_BLOCKING as u32) + .spawn_local() + .detach(); + } +} + +#[no_mangle] +pub extern "C" fn localization_parse_locale(input: &nsCString) -> *const c_void { + let l: LanguageIdentifier = input.to_utf8().parse().unwrap(); + Box::into_raw(Box::new(l)) as *const c_void +} + +#[no_mangle] +pub extern "C" fn localization_new( + res_ids: &ThinVec<GeckoResourceId>, + is_sync: bool, + reg: Option<&GeckoL10nRegistry>, + result: &mut *const LocalizationRc, +) { + *result = std::ptr::null(); + let res_ids: Vec<ResourceId> = res_ids.iter().map(ResourceId::from).collect(); + *result = RefPtr::forget_into_raw(LocalizationRc::new(res_ids, is_sync, reg, None)); +} + +#[no_mangle] +pub extern "C" fn localization_new_with_locales( + res_ids: &ThinVec<GeckoResourceId>, + is_sync: bool, + reg: Option<&GeckoL10nRegistry>, + locales: Option<&ThinVec<nsCString>>, + result: &mut *const LocalizationRc, +) -> bool { + *result = std::ptr::null(); + let res_ids: Vec<ResourceId> = res_ids.iter().map(ResourceId::from).collect(); + let locales: Result<Option<Vec<LanguageIdentifier>>, _> = locales + .map(|locales| { + locales + .iter() + .map(|s| LanguageIdentifier::from_bytes(&s)) + .collect() + }) + .transpose(); + + if let Ok(locales) = locales { + *result = RefPtr::forget_into_raw(LocalizationRc::new(res_ids, is_sync, reg, locales)); + true + } else { + false + } +} + +#[no_mangle] +pub unsafe extern "C" fn localization_addref(loc: &LocalizationRc) { + loc.refcnt.inc(); +} + +#[no_mangle] +pub unsafe extern "C" fn localization_release(loc: *const LocalizationRc) { + let rc = (*loc).refcnt.dec(); + if rc == 0 { + std::mem::drop(Box::from_raw(loc as *const _ as *mut LocalizationRc)); + } +} + +#[no_mangle] +pub extern "C" fn localization_add_res_id(loc: &LocalizationRc, res_id: &GeckoResourceId) { + loc.add_resource_id(res_id.into()); +} + +#[no_mangle] +pub extern "C" fn localization_add_res_ids(loc: &LocalizationRc, res_ids: &ThinVec<GeckoResourceId>) { + let res_ids = res_ids.iter().map(ResourceId::from).collect(); + loc.add_resource_ids(res_ids); +} + +#[no_mangle] +pub extern "C" fn localization_remove_res_id(loc: &LocalizationRc, res_id: &GeckoResourceId) -> usize { + loc.remove_resource_id(res_id.into()) +} + +#[no_mangle] +pub extern "C" fn localization_remove_res_ids( + loc: &LocalizationRc, + res_ids: &ThinVec<GeckoResourceId>, +) -> usize { + let res_ids = res_ids.iter().map(ResourceId::from).collect(); + loc.remove_resource_ids(res_ids) +} + +#[no_mangle] +pub extern "C" fn localization_format_value_sync( + loc: &LocalizationRc, + id: &nsACString, + args: &ThinVec<L10nArg>, + ret_val: &mut nsACString, + ret_err: &mut ThinVec<nsCString>, +) -> bool { + loc.format_value_sync(id, args, ret_val, ret_err) +} + +#[no_mangle] +pub extern "C" fn localization_format_values_sync( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<nsCString>, + ret_err: &mut ThinVec<nsCString>, +) -> bool { + loc.format_values_sync(keys, ret_val, ret_err) +} + +#[no_mangle] +pub extern "C" fn localization_format_messages_sync( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + ret_val: &mut ThinVec<OptionalL10nMessage>, + ret_err: &mut ThinVec<nsCString>, +) -> bool { + loc.format_messages_sync(keys, ret_val, ret_err) +} + +#[no_mangle] +pub extern "C" fn localization_format_value( + loc: &LocalizationRc, + id: &nsACString, + args: &ThinVec<L10nArg>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &nsACString, &ThinVec<nsCString>), +) { + loc.format_value(id, args, promise, callback); +} + +#[no_mangle] +pub extern "C" fn localization_format_values( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &ThinVec<nsCString>, &ThinVec<nsCString>), +) { + loc.format_values(keys, promise, callback); +} + +#[no_mangle] +pub extern "C" fn localization_format_messages( + loc: &LocalizationRc, + keys: &ThinVec<L10nKey>, + promise: &xpcom::Promise, + callback: extern "C" fn(&xpcom::Promise, &ThinVec<OptionalL10nMessage>, &ThinVec<nsCString>), +) { + loc.format_messages(keys, promise, callback); +} + +#[no_mangle] +pub extern "C" fn localization_set_async(loc: &LocalizationRc) { + loc.set_async(); +} + +#[no_mangle] +pub extern "C" fn localization_is_sync(loc: &LocalizationRc) -> bool { + loc.is_sync() +} + +#[no_mangle] +pub extern "C" fn localization_on_change(loc: &LocalizationRc) { + loc.on_change(); +} + +#[cfg(debug_assertions)] +fn debug_assert_variables_exist<K, F>( + errors: &[fluent_fallback::LocalizationError], + keys: &[K], + to_string: F, +) where + F: Fn(&K) -> String, +{ + for error in errors { + if let fluent_fallback::LocalizationError::Resolver { errors, .. } = error { + use fluent::{ + resolver::{errors::ReferenceKind, ResolverError}, + FluentError, + }; + for error in errors { + if let FluentError::ResolverError(ResolverError::Reference( + ReferenceKind::Variable { id }, + )) = error + { + // This error needs to be actionable for Firefox engineers to fix + // their Fluent issues. It might be nicer to share the specific + // message, but at this point we don't have that information. + eprintln!( + "Fluent error, the argument \"${}\" was not provided a value.", + id + ); + eprintln!("This error happened while formatting the following messages:"); + for key in keys { + eprintln!(" {:?}", to_string(key)) + } + + // Panic with the slightly more cryptic ResolverError. + panic!("{}", error.to_string()); + } + } + } + } +} diff --git a/intl/l10n/test/gtest/TestLocalization.cpp b/intl/l10n/test/gtest/TestLocalization.cpp new file mode 100644 index 0000000000..804c228c98 --- /dev/null +++ b/intl/l10n/test/gtest/TestLocalization.cpp @@ -0,0 +1,118 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "gtest/gtest.h" +#include "mozilla/intl/Localization.h" + +using namespace mozilla; +using namespace mozilla::dom; +using namespace mozilla::intl; + +TEST(Intl_Localization, FormatValueSyncMissing) +{ + nsTArray<nsCString> resIds = { + "toolkit/global/handlerDialog.ftl"_ns, + }; + RefPtr<Localization> l10n = Localization::Create(resIds, true); + + auto l10nId = "non-existing-l10n-id"_ns; + IgnoredErrorResult rv; + nsAutoCString result; + + l10n->FormatValueSync(l10nId, {}, result, rv); + ASSERT_FALSE(rv.Failed()); + ASSERT_TRUE(result.IsEmpty()); +} + +TEST(Intl_Localization, FormatValueSync) +{ + nsTArray<nsCString> resIds = { + "toolkit/global/handlerDialog.ftl"_ns, + }; + RefPtr<Localization> l10n = Localization::Create(resIds, true); + + auto l10nId = "permission-dialog-unset-description"_ns; + IgnoredErrorResult rv; + nsAutoCString result; + + l10n->FormatValueSync(l10nId, {}, result, rv); + ASSERT_FALSE(rv.Failed()); + ASSERT_FALSE(result.IsEmpty()); +} + +TEST(Intl_Localization, FormatValueSyncWithArgs) +{ + nsTArray<nsCString> resIds = { + "toolkit/global/handlerDialog.ftl"_ns, + }; + RefPtr<Localization> l10n = Localization::Create(resIds, true); + + auto l10nId = "permission-dialog-description"_ns; + + auto l10nArgs = dom::Optional<intl::L10nArgs>(); + l10nArgs.Construct(); + + auto dirArg = l10nArgs.Value().Entries().AppendElement(); + dirArg->mKey = "scheme"_ns; + dirArg->mValue.SetValue().SetAsUTF8String().Assign("Foo"_ns); + + IgnoredErrorResult rv; + nsAutoCString result; + + l10n->FormatValueSync(l10nId, l10nArgs, result, rv); + ASSERT_FALSE(rv.Failed()); + ASSERT_TRUE(result.Find("Foo"_ns) > -1); +} + +TEST(Intl_Localization, FormatMessagesSync) +{ + nsTArray<nsCString> resIds = { + "toolkit/global/handlerDialog.ftl"_ns, + }; + RefPtr<Localization> l10n = Localization::Create(resIds, true); + + dom::Sequence<dom::OwningUTF8StringOrL10nIdArgs> l10nIds; + auto* l10nId = l10nIds.AppendElement(fallible); + ASSERT_TRUE(l10nId); + l10nId->SetAsUTF8String().Assign("permission-dialog-unset-description"_ns); + + IgnoredErrorResult rv; + nsTArray<dom::Nullable<dom::L10nMessage>> result; + + l10n->FormatMessagesSync(l10nIds, result, rv); + ASSERT_FALSE(rv.Failed()); + ASSERT_FALSE(result.IsEmpty()); +} + +TEST(Intl_Localization, FormatMessagesSyncWithArgs) +{ + nsTArray<nsCString> resIds = { + "toolkit/global/handlerDialog.ftl"_ns, + }; + RefPtr<Localization> l10n = Localization::Create(resIds, true); + + dom::Sequence<dom::OwningUTF8StringOrL10nIdArgs> l10nIds; + L10nIdArgs& key0 = l10nIds.AppendElement(fallible)->SetAsL10nIdArgs(); + key0.mId.Assign("permission-dialog-description"_ns); + auto arg = key0.mArgs.SetValue().Entries().AppendElement(); + arg->mKey = "scheme"_ns; + arg->mValue.SetValue().SetAsUTF8String().Assign("Foo"_ns); + + L10nIdArgs& key1 = l10nIds.AppendElement(fallible)->SetAsL10nIdArgs(); + key1.mId.Assign("chooser-window"_ns); + + IgnoredErrorResult rv; + nsTArray<dom::Nullable<dom::L10nMessage>> result; + + l10n->FormatMessagesSync(l10nIds, result, rv); + ASSERT_FALSE(rv.Failed()); + ASSERT_TRUE(result.Length() == 2); + ASSERT_TRUE(result.ElementAt(0).Value().mValue.Find("Foo"_ns) > -1); + + auto fmtAttr = result.ElementAt(1).Value().mAttributes.Value(); + ASSERT_TRUE(fmtAttr.Length() == 2); + ASSERT_FALSE(fmtAttr.ElementAt(0).mName.IsEmpty()); + ASSERT_FALSE(fmtAttr.ElementAt(0).mValue.IsEmpty()); +} diff --git a/intl/l10n/test/gtest/moz.build b/intl/l10n/test/gtest/moz.build new file mode 100644 index 0000000000..6d04077a50 --- /dev/null +++ b/intl/l10n/test/gtest/moz.build @@ -0,0 +1,11 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +UNIFIED_SOURCES += [ + "TestLocalization.cpp", +] + +FINAL_LIBRARY = "xul-gtest" diff --git a/intl/l10n/test/mochitest/chrome.ini b/intl/l10n/test/mochitest/chrome.ini new file mode 100644 index 0000000000..1b18537cc6 --- /dev/null +++ b/intl/l10n/test/mochitest/chrome.ini @@ -0,0 +1,6 @@ +[localization/test_formatValue.html] +skip-if = debug # Intentionally triggers a debug assert for missing Fluent arguments. +[localization/test_formatValues.html] +skip-if = debug # Intentionally triggers a debug assert for missing Fluent arguments. +[localization/test_formatMessages.html] +skip-if = debug # Intentionally triggers a debug assert for missing Fluent arguments. diff --git a/intl/l10n/test/mochitest/localization/test_formatMessages.html b/intl/l10n/test/mochitest/localization/test_formatMessages.html new file mode 100644 index 0000000000..6e5a9f3f41 --- /dev/null +++ b/intl/l10n/test/mochitest/localization/test_formatMessages.html @@ -0,0 +1,101 @@ +<!DOCTYPE HTML> +<html> +<head> + <meta charset="utf-8"> + <title>Test Localization.prototype.formatMessages API</title> + <script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> + <script type="application/javascript"> + "use strict"; + const mockSource = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}/", [ + { + path: "/localization/en-US/mock.ftl", + source: ` +key1 = Value + .title = Title 1 + .accesskey = K +key2 = + .label = This is a label for { $user } +` + } + ]); + let registry = new L10nRegistry({ + bundleOptions: { + useIsolating: false + } + }); + registry.registerSources([mockSource]); + + function getAttributeByName(attributes, name) { + return attributes.find(attr => attr.name === name); + } + + (async () => { + SimpleTest.waitForExplicitFinish(); + + const loc = new Localization( + ['mock.ftl'], + false, + registry, + ); + + { + // Simple mix works. + let msgs = await loc.formatMessages([ + {id: "key1"}, + {id: "key2", args: { user: "Amy"}}, + ]); + { + is(msgs[0].value, "Value"); + let attr0 = getAttributeByName(msgs[0].attributes, "title"); + is(attr0.name, "title"); + is(attr0.value, "Title 1"); + let attr1 = getAttributeByName(msgs[0].attributes, "accesskey"); + is(attr1.name, "accesskey"); + is(attr1.value, "K"); + } + + { + is(msgs[1].value, null); + let attr0 = getAttributeByName(msgs[1].attributes, "label"); + is(attr0.name, "label"); + is(attr0.value, "This is a label for Amy"); + } + } + + { + // Missing arguments cause exception in automation. + try { + let msgs = await loc.formatMessages([ + {id: "key1"}, + {id: "key2"}, + ]); + ok(false, "Missing argument didn't cause an exception."); + } catch (e) { + is(e.message, + "[fluent][resolver] errors in en-US/key2: Resolver error: Unknown variable: $user", + "Missing key causes an exception."); + } + } + + { + // Missing keys cause exception in automation. + try { + let msgs = await loc.formatMessages([ + { id: "key1" }, + { id: "key4" }, + ]); + ok(false, "Missing key didn't cause an exception."); + } catch (e) { + is(e.message, + "[fluent] Missing message in locale en-US: key4", + "Missing key causes an exception."); + } + } + SimpleTest.finish(); + })(); + </script> +</head> +<body> +</body> +</html> diff --git a/intl/l10n/test/mochitest/localization/test_formatValue.html b/intl/l10n/test/mochitest/localization/test_formatValue.html new file mode 100644 index 0000000000..e1ee02fa7a --- /dev/null +++ b/intl/l10n/test/mochitest/localization/test_formatValue.html @@ -0,0 +1,81 @@ +<!DOCTYPE HTML> +<html> +<head> + <meta charset="utf-8"> + <title>Test Localization.prototype.formatValue API</title> + <script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> + <script type="application/javascript"> + "use strict"; + const mockSource = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}/", [ + { + path: "/localization/en-US/mock.ftl", + source: ` +key1 = Value +key2 = Value { $user } +key3 = Value { $count } +` + } + ]); + let registry = new L10nRegistry({ + bundleOptions: { + useIsolating: false + } + }); + registry.registerSources([mockSource]); + + (async () => { + SimpleTest.waitForExplicitFinish(); + + const loc = new Localization( + ['mock.ftl'], + false, + registry, + ); + + { + // Simple value works. + let val = await loc.formatValue("key1"); + is(val, "Value"); + } + + { + // Value with a string argument works. + let val = await loc.formatValue("key2", { user: "John" }); + is(val, "Value John"); + } + + { + // Value with a number argument works. + let val = await loc.formatValue("key3", { count: -3.21 }); + is(val, "Value -3.21"); + } + + { + // Verify that in automation, a missing + // argument causes an exception. + try { + let val = await loc.formatValue("key3"); + ok(false, "Missing argument didn't cause an exception."); + } catch (e) { + is(e.message, + "[fluent][resolver] errors in en-US/key3: Resolver error: Unknown variable: $count", + "Missing key causes an exception."); + } + } + + { + // Incorrect argument type works. + // Due to how WebIDL handles union types, it'll convert + // the argument to a string `[object Object]`. + let val = await loc.formatValue("key2", { user: { name: true } }); + is(val, "Value [object Object]"); + } + + SimpleTest.finish(); + })(); + </script> +</head> +<body> +</body> +</html> diff --git a/intl/l10n/test/mochitest/localization/test_formatValues.html b/intl/l10n/test/mochitest/localization/test_formatValues.html new file mode 100644 index 0000000000..819d418b2e --- /dev/null +++ b/intl/l10n/test/mochitest/localization/test_formatValues.html @@ -0,0 +1,85 @@ +<!DOCTYPE HTML> +<html> +<head> + <meta charset="utf-8"> + <title>Test Localization.prototype.formatValues API</title> + <script src="chrome://mochikit/content/tests/SimpleTest/SimpleTest.js"></script> + <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css"> + <script type="application/javascript"> + "use strict"; + const mockSource = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}/", [ + { + path: "/localization/en-US/mock.ftl", + source: ` +key1 = Value +key2 = Value { $user } +key3 = Value { $count } +` + } + ]); + let registry = new L10nRegistry({ + bundleOptions: { + useIsolating: false + } + }); + registry.registerSources([mockSource]); + + (async () => { + SimpleTest.waitForExplicitFinish(); + + const loc = new Localization( + ['mock.ftl'], + false, + registry, + ); + + { + // Simple mix works. + let vals = await loc.formatValues([ + {id: "key1"}, + {id: "key2", args: { user: "Amy"}}, + {id: "key3", args: { count: -32.12 }}, + ]); + is(vals[0], "Value"); + is(vals[1], "Value Amy"); + is(vals[2], "Value -32.12"); + } + + { + // Missing arguments cause exception in automation. + try { + let vals = await loc.formatValues([ + {id: "key1"}, + {id: "key2"}, + {id: "key3", args: { count: -32.12 }}, + ]); + ok(false, "Missing argument didn't cause an exception."); + } catch (e) { + is(e.message, + "[fluent][resolver] errors in en-US/key2: Resolver error: Unknown variable: $user", + "Missing key causes an exception."); + } + } + + { + // Missing keys cause exception in automation. + try { + let vals = await loc.formatValues([ + { id: "key1" }, + { id: "key4", args: { count: -32.12 } }, + ]); + ok(false, "Missing key didn't cause an exception."); + } catch (e) { + is(e.message, + "[fluent] Missing message in locale en-US: key4", + "Missing key causes an exception."); + } + } + + SimpleTest.finish(); + })(); + </script> +</head> +<body> +</body> +</html> diff --git a/intl/l10n/test/test_datetimeformat.js b/intl/l10n/test/test_datetimeformat.js new file mode 100644 index 0000000000..4c0794424b --- /dev/null +++ b/intl/l10n/test/test_datetimeformat.js @@ -0,0 +1,76 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const FIREFOX_RELEASE_TIMESTAMP = 1032800850000; +const FIREFOX_RELEASE_DATE = new Date(FIREFOX_RELEASE_TIMESTAMP); + +add_task(function test_date_time_format() { + const bundle = new FluentBundle(["en-US"]); + + bundle.addResource( + new FluentResource(` +dt-arg = Just the arg is: {$dateArg} +dt-bare = The bare date is: { DATETIME($dateArg) } +dt-month-year = Months and year are not time-zone dependent here: { DATETIME($dateArg, month: "long") } +dt-bad = This is a bad month: { DATETIME($dateArg, month: "oops") } +# TODO - Bug 1707728: +dt-timezone = The timezone: { DATETIME($dateArg, timezone: "America/New_York") } +dt-unknown = Unknown: { DATETIME($dateArg, unknown: "unknown") } +dt-style = Style formatting: { DATETIME($dateArg, dateStyle: "short", timeStyle: "short") } + `) + ); + + function testMessage(id, dateArg, expectedMessage) { + const message = bundle.formatPattern(bundle.getMessage(id).value, { + dateArg, + }); + + if (typeof expectedMessage === "object") { + // Assume regex. + ok( + expectedMessage.test(message), + `"${message}" matches regex: ${expectedMessage.toString()}` + ); + } else { + // Assume string. + equal(message, expectedMessage); + } + } + + // TODO - Bug 1707728 - Some of these are implemented as regexes since time zones are not + // supported in fluent messages as of yet. They could be simplified if a time zone were + // specified. + testMessage( + "dt-arg", + FIREFOX_RELEASE_DATE, + /^Just the arg is: (Sun|Mon|Tue) Sep \d+ 2002 \d+:\d+:\d+ .* \(.*\)$/ + ); + testMessage( + "dt-bare", + FIREFOX_RELEASE_TIMESTAMP, + /^The bare date is: Sep \d+, 2002, \d+:\d+:\d+ (AM|PM)$/ + ); + testMessage( + "dt-month-year", + FIREFOX_RELEASE_TIMESTAMP, + "Months and year are not time-zone dependent here: September" + ); + testMessage( + "dt-bad", + FIREFOX_RELEASE_TIMESTAMP, + /^This is a bad month: Sep \d+, 2002, \d+:\d+:\d+ (AM|PM)$/ + ); + testMessage( + "dt-unknown", + FIREFOX_RELEASE_TIMESTAMP, + /^Unknown: Sep \d+, 2002, \d+:\d+:\d+ (AM|PM)$/ + ); + testMessage( + "dt-style", + FIREFOX_RELEASE_TIMESTAMP, + /^Style formatting: \d+\/\d+\/\d+, \d+:\d+ (AM|PM)$/ + ); + +// TODO - Bug 1707728 +// testMessage("dt-timezone", ...); +}); diff --git a/intl/l10n/test/test_l10nregistry.js b/intl/l10n/test/test_l10nregistry.js new file mode 100644 index 0000000000..cbbb1e7316 --- /dev/null +++ b/intl/l10n/test/test_l10nregistry.js @@ -0,0 +1,563 @@ +/* Any copyrighequal dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const {setTimeout} = ChromeUtils.importESModule("resource://gre/modules/Timer.sys.mjs"); + +const l10nReg = new L10nRegistry(); + +add_task(function test_methods_presence() { + equal(typeof l10nReg.generateBundles, "function"); + equal(typeof l10nReg.generateBundlesSync, "function"); + equal(typeof l10nReg.getAvailableLocales, "function"); + equal(typeof l10nReg.registerSources, "function"); + equal(typeof l10nReg.removeSources, "function"); + equal(typeof l10nReg.updateSources, "function"); +}); + +/** + * Test that passing empty resourceIds list works. + */ +add_task(async function test_empty_resourceids() { + const fs = []; + + const source = L10nFileSource.createMock("test", "", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const bundles = l10nReg.generateBundles(["en-US"], []); + + const done = (await bundles.next()).done; + + equal(done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * Test that passing empty sources list works. + */ +add_task(async function test_empty_sources() { + const fs = []; + const bundles = l10nReg.generateBundlesSync(["en-US"], fs); + + const done = (await bundles.next()).done; + + equal(done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test tests generation of a proper context for a single + * source scenario + */ +add_task(async function test_methods_calling() { + const fs = [ + { path: "/localization/en-US/browser/menu.ftl", source: "key = Value" } + ]; + const source = L10nFileSource.createMock("test", "", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const bundles = l10nReg.generateBundles(["en-US"], ["/browser/menu.ftl"]); + + const bundle = (await bundles.next()).value; + + equal(bundle.hasMessage("key"), true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that the public methods return expected values + * for the single source scenario + */ +add_task(async function test_has_one_source() { + const fs = [ + {path: "./app/data/locales/en-US/test.ftl", source: "key = value en-US"} + ]; + let oneSource = L10nFileSource.createMock("app", "", ["en-US"], "./app/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource]); + + + // has one source + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("app"), true); + + + // returns a single context + + let bundles = l10nReg.generateBundles(["en-US"], ["test.ftl"]); + let bundle0 = (await bundles.next()).value; + equal(bundle0.hasMessage("key"), true); + + equal((await bundles.next()).done, true); + + + // returns no contexts for missing locale + + bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that public methods return expected values + * for the dual source scenario. + */ +add_task(async function test_has_two_sources() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "key = platform value" }, + { path: "./app/data/locales/pl/test.ftl", source: "key = app value" } + ]; + let oneSource = L10nFileSource.createMock("platform", "", ["en-US"], "./platform/data/locales/{locale}/", fs); + let secondSource = L10nFileSource.createMock("app", "", ["pl"], "./app/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource, secondSource]); + + // has two sources + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("app"), true); + equal(l10nReg.hasSource("platform"), true); + + + // returns correct contexts for en-US + + let bundles = l10nReg.generateBundles(["en-US"], ["test.ftl"]); + let bundle0 = (await bundles.next()).value; + + equal(bundle0.hasMessage("key"), true); + let msg = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg.value), "platform value"); + + equal((await bundles.next()).done, true); + + + // returns correct contexts for [pl, en-US] + + bundles = l10nReg.generateBundles(["pl", "en-US"], ["test.ftl"]); + bundle0 = (await bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "app value"); + + let bundle1 = (await bundles.next()).value; + equal(bundle1.locales[0], "en-US"); + equal(bundle1.hasMessage("key"), true); + let msg1 = bundle1.getMessage("key"); + equal(bundle1.formatPattern(msg1.value), "platform value"); + + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that behavior specific to the L10nFileSource + * works correctly. + * + * In particular it tests that L10nFileSource correctly returns + * missing files as `false` instead of `undefined`. + */ +add_task(function test_indexed() { + let oneSource = new L10nFileSource("langpack-pl", "app", ["pl"], "/data/locales/{locale}/", {}, [ + "/data/locales/pl/test.ftl", + ]); + equal(oneSource.hasFile("pl", "test.ftl"), "present"); + equal(oneSource.hasFile("pl", "missing.ftl"), "missing"); +}); + +/** + * This test checks if the correct order of contexts is used for + * scenarios where a new file source is added on top of the default one. + */ +add_task(async function test_override() { + const fs = [ + { path: "/app/data/locales/pl/test.ftl", source: "key = value" }, + { path: "/data/locales/pl/test.ftl", source: "key = addon value"}, + ]; + let fileSource = L10nFileSource.createMock("app", "", ["pl"], "/app/data/locales/{locale}/", fs); + let oneSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", fs); + l10nReg.registerSources([fileSource, oneSource]); + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("langpack-pl"), true); + + let bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + let bundle0 = (await bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "addon value"); + + let bundle1 = (await bundles.next()).value; + equal(bundle1.locales[0], "pl"); + equal(bundle1.hasMessage("key"), true); + let msg1 = bundle1.getMessage("key"); + equal(bundle1.formatPattern(msg1.value), "value"); + + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that new contexts are returned + * after source update. + */ +add_task(async function test_updating() { + const fs = [ + { path: "/data/locales/pl/test.ftl", source: "key = value" } + ]; + let oneSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource]); + + let bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + let bundle0 = (await bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "value"); + + + const newSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", [ + { path: "/data/locales/pl/test.ftl", source: "key = new value" } + ]); + l10nReg.updateSources([newSource]); + + equal(l10nReg.getSourceNames().length, 1); + bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + bundle0 = (await bundles.next()).value; + msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "new value"); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that generated contexts return correct values + * after sources are being removed. + */ +add_task(async function test_removing() { + const fs = [ + { path: "/app/data/locales/pl/test.ftl", source: "key = value" }, + { path: "/data/locales/pl/test.ftl", source: "key = addon value" }, + ]; + + let fileSource = L10nFileSource.createMock("app", "", ["pl"], "/app/data/locales/{locale}/", fs); + let oneSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", fs); + l10nReg.registerSources([fileSource, oneSource]); + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("langpack-pl"), true); + + let bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + let bundle0 = (await bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "addon value"); + + let bundle1 = (await bundles.next()).value; + equal(bundle1.locales[0], "pl"); + equal(bundle1.hasMessage("key"), true); + let msg1 = bundle1.getMessage("key"); + equal(bundle1.formatPattern(msg1.value), "value"); + + equal((await bundles.next()).done, true); + + // Remove langpack + + l10nReg.removeSources(["langpack-pl"]); + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("langpack-pl"), false); + + bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + bundle0 = (await bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "value"); + + equal((await bundles.next()).done, true); + + // Remove app source + + l10nReg.removeSources(["app"]); + + equal(l10nReg.getSourceNames().length, 0); + + bundles = l10nReg.generateBundles(["pl"], ["test.ftl"]); + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that the logic works correctly when there's a missing + * file in the FileSource scenario. + */ +add_task(async function test_missing_file() { + const fs = [ + { path: "./app/data/locales/en-US/test.ftl", source: "key = value en-US" }, + { path: "./platform/data/locales/en-US/test.ftl", source: "key = value en-US" }, + { path: "./platform/data/locales/en-US/test2.ftl", source: "key2 = value2 en-US" }, + ]; + let oneSource = L10nFileSource.createMock("app", "", ["en-US"], "./app/data/locales/{locale}/", fs); + let twoSource = L10nFileSource.createMock("platform", "", ["en-US"], "./platform/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource, twoSource]); + + // has two sources + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("app"), true); + equal(l10nReg.hasSource("platform"), true); + + + // returns a single context + + let bundles = l10nReg.generateBundles(["en-US"], ["test.ftl", "test2.ftl"]); + + // First permutation: + // [platform, platform] - both present + let bundle1 = (await bundles.next()); + equal(bundle1.value.hasMessage("key"), true); + + // Second permutation skipped: + // [platform, app] - second missing + // Third permutation: + // [app, platform] - both present + let bundle2 = (await bundles.next()); + equal(bundle2.value.hasMessage("key"), true); + + // Fourth permutation skipped: + // [app, app] - second missing + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +add_task(async function test_hasSource() { + equal(l10nReg.hasSource("nonsense"), false, "Non-existing source doesn't exist"); + equal(l10nReg.hasSource("app"), false, "hasSource returns true before registering a source"); + let oneSource = new L10nFileSource("app", "app", ["en-US"], "/{locale}/"); + l10nReg.registerSources([oneSource]); + equal(l10nReg.hasSource("app"), true, "hasSource returns true after registering a source"); + l10nReg.clearSources(); +}); + +/** + * This test verifies that we handle correctly a scenario where a source + * is being removed while the iterator operates. + */ +add_task(async function test_remove_source_mid_iter_cycle() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "key = platform value" }, + { path: "./app/data/locales/pl/test.ftl", source: "key = app value" }, + ]; + let oneSource = L10nFileSource.createMock("platform", "", ["en-US"], "./platform/data/locales/{locale}/", fs); + let secondSource = L10nFileSource.createMock("app", "", ["pl"], "./app/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource, secondSource]); + + let bundles = l10nReg.generateBundles(["en-US", "pl"], ["test.ftl"]); + + let bundle0 = await bundles.next(); + + // The registry has a copy of the file sources, so it will be unaffected. + l10nReg.removeSources(["app"]); + + let bundle1 = await bundles.next(); + + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +add_task(async function test_metasources() { + let fs = [ + { path: "/localization/en-US/browser/menu1.ftl", source: "key1 = Value" }, + { path: "/localization/en-US/browser/menu2.ftl", source: "key2 = Value" }, + { path: "/localization/en-US/browser/menu3.ftl", source: "key3 = Value" }, + { path: "/localization/en-US/browser/menu4.ftl", source: "key4 = Value" }, + { path: "/localization/en-US/browser/menu5.ftl", source: "key5 = Value" }, + { path: "/localization/en-US/browser/menu6.ftl", source: "key6 = Value" }, + { path: "/localization/en-US/browser/menu7.ftl", source: "key7 = Value" }, + { path: "/localization/en-US/browser/menu8.ftl", source: "key8 = Value" }, + ]; + + const browser = L10nFileSource.createMock("browser", "app", ["en-US"], "/localization/{locale}", fs); + const toolkit = L10nFileSource.createMock("toolkit", "app", ["en-US"], "/localization/{locale}", fs); + const browser2 = L10nFileSource.createMock("browser2", "langpack", ["en-US"], "/localization/{locale}", fs); + const toolkit2 = L10nFileSource.createMock("toolkit2", "langpack", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([toolkit, browser, toolkit2, browser2]); + + let res = [ + "/browser/menu1.ftl", + "/browser/menu2.ftl", + "/browser/menu3.ftl", + "/browser/menu4.ftl", + "/browser/menu5.ftl", + "/browser/menu6.ftl", + "/browser/menu7.ftl", + {path: "/browser/menu8.ftl", optional: false}, + ]; + + const bundles = l10nReg.generateBundles(["en-US"], res); + + let nbundles = 0; + while (!(await bundles.next()).done) { + nbundles += 1; + } + + // If metasources are working properly, we'll generate 2^8 = 256 bundles for + // each metasource giving 512 bundles in total. Otherwise, we generate + // 4^8 = 65536 bundles. + equal(nbundles, 512); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that when a required resource is missing for a locale, + * we do not produce a bundle for that locale. + */ +add_task(async function test_missing_required_resource() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "test-key = en-US value" }, + { path: "./platform/data/locales/pl/missing-in-en-US.ftl", source: "missing-key = pl value" }, + { path: "./platform/data/locales/pl/test.ftl", source: "test-key = pl value" }, + ]; + let source = L10nFileSource.createMock("platform", "", ["en-US", "pl"], "./platform/data/locales/{locale}/", fs); + l10nReg.registerSources([source]); + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("platform"), true); + + + // returns correct contexts for [en-US, pl] + + let bundles = l10nReg.generateBundlesSync(["en-US", "pl"], ["test.ftl", "missing-in-en-US.ftl"]); + let bundle0 = (await bundles.next()).value; + + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("test-key"), true); + equal(bundle0.hasMessage("missing-key"), true); + + let msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "pl value"); + + let msg1 = bundle0.getMessage("missing-key"); + equal(bundle0.formatPattern(msg1.value), "pl value"); + + equal((await bundles.next()).done, true); + + + // returns correct contexts for [pl, en-US] + + bundles = l10nReg.generateBundlesSync(["pl", "en-US"], ["test.ftl", {path: "missing-in-en-US.ftl", optional: false}]); + bundle0 = (await bundles.next()).value; + + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("test-key"), true); + + msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "pl value"); + + msg1 = bundle0.getMessage("missing-key"); + equal(bundle0.formatPattern(msg1.value), "pl value"); + + equal((await bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that when an optional resource is missing, we continue + * to produce a bundle for that locale. The bundle will have missing entries + * with regard to the missing optional resource. + */ +add_task(async function test_missing_optional_resource() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "test-key = en-US value" }, + { path: "./platform/data/locales/pl/missing-in-en-US.ftl", source: "missing-key = pl value" }, + { path: "./platform/data/locales/pl/test.ftl", source: "test-key = pl value" }, + ]; + let source = L10nFileSource.createMock("platform", "", ["en-US", "pl"], "./platform/data/locales/{locale}/", fs); + l10nReg.registerSources([source]); + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("platform"), true); + + + // returns correct contexts for [en-US, pl] + + let bundles = l10nReg.generateBundlesSync(["en-US", "pl"], ["test.ftl", { path: "missing-in-en-US.ftl", optional: true }]); + let bundle0 = (await bundles.next()).value; + + equal(bundle0.locales[0], "en-US"); + equal(bundle0.hasMessage("test-key"), true); + equal(bundle0.hasMessage("missing-key"), false); + + let msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "en-US value"); + + equal(bundle0.getMessage("missing-key"), null); + + let bundle1 = (await bundles.next()).value; + + equal(bundle1.locales[0], "pl"); + equal(bundle1.hasMessage("test-key"), true); + equal(bundle1.hasMessage("missing-key"), true); + + msg0 = bundle1.getMessage("test-key"); + equal(bundle1.formatPattern(msg0.value), "pl value"); + + msg1 = bundle1.getMessage("missing-key"); + equal(bundle1.formatPattern(msg1.value), "pl value"); + + equal((await bundles.next()).done, true); + + // returns correct contexts for [pl, en-US] + + bundles = l10nReg.generateBundlesSync(["pl", "en-US"], ["test.ftl", { path: "missing-in-en-US.ftl", optional: true }]); + bundle0 = (await bundles.next()).value; + + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("test-key"), true); + equal(bundle0.hasMessage("missing-key"), true); + + msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "pl value"); + + msg1 = bundle0.getMessage("missing-key"); + equal(bundle0.formatPattern(msg1.value), "pl value"); + + bundle1 = (await bundles.next()).value; + + equal(bundle1.locales[0], "en-US"); + equal(bundle1.hasMessage("test-key"), true); + equal(bundle1.hasMessage("missing-key"), false); + + msg0 = bundle1.getMessage("test-key"); + equal(bundle1.formatPattern(msg0.value), "en-US value"); + + equal(bundle1.getMessage("missing-key"), null); + + // cleanup + l10nReg.clearSources(); +}); diff --git a/intl/l10n/test/test_l10nregistry_fuzzed.js b/intl/l10n/test/test_l10nregistry_fuzzed.js new file mode 100644 index 0000000000..f99c3b61e5 --- /dev/null +++ b/intl/l10n/test/test_l10nregistry_fuzzed.js @@ -0,0 +1,205 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +/** + * This test is a fuzzing test for the L10nRegistry API. It was written to find + * a hard to reproduce bug in the L10nRegistry code. If it fails, place the seed + * from the failing run in the code directly below to make it consistently reproducible. + */ +let seed = Math.floor(Math.random() * 1e9); + +console.log(`Starting a fuzzing run with seed: ${seed}.`); +console.log("To reproduce this test locally, re-run it locally with:"); +console.log(`let seed = ${seed};`); + +/** + * A simple non-robust psuedo-random number generator. + * + * It is implemented using a Lehmer random number generator. + * https://en.wikipedia.org/wiki/16,807 + * + * @returns {number} Ranged [0, 1) + */ +function prng() { + const multiplier = 16807; + const prime = 2147483647; + seed = seed * multiplier % prime + return (seed - 1) / prime +} + +/** + * Generate a name like "mock-dmsxfodrqboljmxdeayt". + * @returns {string} + */ +function generateRandomName() { + let name = 'mock-' + const letters = "abcdefghijklmnopqrstuvwxyz"; + for (let i = 0; i < 20; i++) { + name += letters[Math.floor(prng() * letters.length)]; + } + return name; +} + +/** + * Picks one item from an array. + * + * @param {Array<T>} + * @returns {T} + */ +function pickOne(list) { + return list[Math.floor(prng() * list.length)] +} + +/** + * Picks a random subset from an array. + * + * @param {Array<T>} + * @returns {Array<T>} + */ +function pickN(list, count) { + list = list.slice(); + const result = []; + for (let i = 0; i < count && i < list.length; i++) { + // Pick a random item. + const index = Math.floor(prng() * list.length); + + // Swap item to the end. + const a = list[index]; + const b = list[list.length - 1]; + list[index] = b; + list[list.length - 1] = a + + // Now that the random item is on the end, pop it off and add it to the results. + result.push(list.pop()); + } + + return result +} + +/** + * Generate a random number + * @param {number} min + * @param {number} max + * @returns {number} + */ +function random(min, max) { + const delta = max - min; + return min + delta * prng(); +} + +/** + * Generate a random number generator with a distribution more towards the lower end. + * @param {number} min + * @param {number} max + * @returns {number} + */ +function randomPow(min, max) { + const delta = max - min; + const r = prng() + return min + delta * r * r; +} + +add_task(async function test_fuzzing_sources() { + const iterations = 100; + const maxSources = 10; + + const metasources = ["app", "langpack", ""]; + const availableLocales = ["en", "en-US", "pl", "en-CA", "es-AR", "es-ES"]; + + const l10nReg = new L10nRegistry(); + + for (let i = 0; i < iterations; i++) { + console.log("----------------------------------------------------------------------"); + console.log("Iteration", i); + let sourceCount = randomPow(0, maxSources); + + const mocks = []; + const fs = []; + + const locales = new Set(); + const filenames = new Set(); + + for (let j = 0; j < sourceCount; j++) { + const locale = pickOne(availableLocales); + locales.add(locale); + + let metasource = pickOne(metasources); + if (metasource === "langpack") { + metasource = `${metasource}-${locale}` + } + + const dir = generateRandomName(); + const filename = generateRandomName() + j + ".ftl"; + const path = `${dir}/${locale}/${filename}` + const name = metasource || "app"; + const source = "key = value"; + + filenames.add(filename); + + console.log("Add source", { name, metasource, path, source }); + fs.push({ path, source }); + + mocks.push([ + name, // name + metasource, // metasource, + [locale], // locales, + dir + "/{locale}/", + fs + ]) + } + + l10nReg.registerSources(mocks.map(args => L10nFileSource.createMock(...args))); + + const bundleLocales = pickN([...locales], random(1, 4)); + const bundleFilenames = pickN([...filenames], random(1, 10)); + + console.log("generateBundles", {bundleLocales, bundleFilenames}); + const bundles = l10nReg.generateBundles( + bundleLocales, + bundleFilenames + ); + + function next() { + console.log("Getting next bundle"); + const bundle = bundles.next() + console.log("Next bundle obtained", bundle); + return bundle; + } + + const ops = [ + // Increase the frequency of next being called. + next, + next, + next, + () => { + const newMocks = []; + for (const mock of pickN(mocks, random(0, 3))) { + const newMock = mock.slice(); + newMocks.push(newMock) + } + console.log("l10nReg.updateSources"); + l10nReg.updateSources(newMocks.map(mock => L10nFileSource.createMock(...mock))); + }, + () => { + console.log("l10nReg.clearSources"); + l10nReg.clearSources(); + } + ]; + + console.log("Start the operation loop"); + while (true) { + console.log("Next operation"); + const op = pickOne(ops); + const result = await op(); + if (result?.done) { + // The iterator completed. + break; + } + } + + console.log("Clear sources"); + l10nReg.clearSources(); + } + + ok(true, "The L10nRegistry fuzzing did not crash.") +}); diff --git a/intl/l10n/test/test_l10nregistry_sync.js b/intl/l10n/test/test_l10nregistry_sync.js new file mode 100644 index 0000000000..178a24ff6d --- /dev/null +++ b/intl/l10n/test/test_l10nregistry_sync.js @@ -0,0 +1,536 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const {setTimeout} = ChromeUtils.importESModule("resource://gre/modules/Timer.sys.mjs"); + +const l10nReg = new L10nRegistry(); + +add_task(function test_methods_presence() { + equal(typeof l10nReg.generateBundles, "function"); + equal(typeof l10nReg.generateBundlesSync, "function"); + equal(typeof l10nReg.getAvailableLocales, "function"); + equal(typeof l10nReg.registerSources, "function"); + equal(typeof l10nReg.removeSources, "function"); + equal(typeof l10nReg.updateSources, "function"); +}); + +/** + * Test that passing empty resourceIds list works. + */ +add_task(function test_empty_resourceids() { + const fs = []; + + const source = L10nFileSource.createMock("test", "", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const bundles = l10nReg.generateBundlesSync(["en-US"], []); + + const done = (bundles.next()).done; + + equal(done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * Test that passing empty sources list works. + */ +add_task(function test_empty_sources() { + const fs = []; + const bundles = l10nReg.generateBundlesSync(["en-US"], fs); + + const done = (bundles.next()).done; + + equal(done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test tests generation of a proper context for a single + * source scenario + */ +add_task(function test_methods_calling() { + const fs = [ + { path: "/localization/en-US/browser/menu.ftl", source: "key = Value" } + ]; + const source = L10nFileSource.createMock("test", "", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const bundles = l10nReg.generateBundlesSync(["en-US"], ["/browser/menu.ftl"]); + + const bundle = (bundles.next()).value; + + equal(bundle.hasMessage("key"), true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that the public methods return expected values + * for the single source scenario + */ +add_task(function test_has_one_source() { + const fs = [ + {path: "./app/data/locales/en-US/test.ftl", source: "key = value en-US"} + ]; + let oneSource = L10nFileSource.createMock("app", "", ["en-US"], "./app/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource]); + + + // has one source + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("app"), true); + + + // returns a single context + + let bundles = l10nReg.generateBundlesSync(["en-US"], ["test.ftl"]); + let bundle0 = (bundles.next()).value; + equal(bundle0.hasMessage("key"), true); + + equal((bundles.next()).done, true); + + + // returns no contexts for missing locale + + bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that public methods return expected values + * for the dual source scenario. + */ +add_task(function test_has_two_sources() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "key = platform value" }, + { path: "./app/data/locales/pl/test.ftl", source: "key = app value" } + ]; + let oneSource = L10nFileSource.createMock("platform", "", ["en-US"], "./platform/data/locales/{locale}/", fs); + let secondSource = L10nFileSource.createMock("app", "", ["pl"], "./app/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource, secondSource]); + + // has two sources + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("app"), true); + equal(l10nReg.hasSource("platform"), true); + + + // returns correct contexts for en-US + + let bundles = l10nReg.generateBundlesSync(["en-US"], ["test.ftl"]); + let bundle0 = (bundles.next()).value; + + equal(bundle0.hasMessage("key"), true); + let msg = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg.value), "platform value"); + + equal((bundles.next()).done, true); + + + // returns correct contexts for [pl, en-US] + + bundles = l10nReg.generateBundlesSync(["pl", "en-US"], ["test.ftl"]); + bundle0 = (bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "app value"); + + let bundle1 = (bundles.next()).value; + equal(bundle1.locales[0], "en-US"); + equal(bundle1.hasMessage("key"), true); + let msg1 = bundle1.getMessage("key"); + equal(bundle1.formatPattern(msg1.value), "platform value"); + + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test checks if the correct order of contexts is used for + * scenarios where a new file source is added on top of the default one. + */ +add_task(function test_override() { + const fs = [ + { path: "/app/data/locales/pl/test.ftl", source: "key = value" }, + { path: "/data/locales/pl/test.ftl", source: "key = addon value"}, + ]; + let fileSource = L10nFileSource.createMock("app", "", ["pl"], "/app/data/locales/{locale}/", fs); + let oneSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", fs); + l10nReg.registerSources([fileSource, oneSource]); + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("langpack-pl"), true); + + let bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + let bundle0 = (bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "addon value"); + + let bundle1 = (bundles.next()).value; + equal(bundle1.locales[0], "pl"); + equal(bundle1.hasMessage("key"), true); + let msg1 = bundle1.getMessage("key"); + equal(bundle1.formatPattern(msg1.value), "value"); + + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that new contexts are returned + * after source update. + */ +add_task(function test_updating() { + const fs = [ + { path: "/data/locales/pl/test.ftl", source: "key = value" } + ]; + let oneSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource]); + + let bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + let bundle0 = (bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "value"); + + + const newSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", [ + { path: "/data/locales/pl/test.ftl", source: "key = new value" } + ]); + l10nReg.updateSources([newSource]); + + equal(l10nReg.getSourceNames().length, 1); + bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + bundle0 = (bundles.next()).value; + msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "new value"); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that generated contexts return correct values + * after sources are being removed. + */ +add_task(function test_removing() { + const fs = [ + { path: "/app/data/locales/pl/test.ftl", source: "key = value" }, + { path: "/data/locales/pl/test.ftl", source: "key = addon value" }, + ]; + + let fileSource = L10nFileSource.createMock("app", "", ["pl"], "/app/data/locales/{locale}/", fs); + let oneSource = L10nFileSource.createMock("langpack-pl", "", ["pl"], "/data/locales/{locale}/", fs); + l10nReg.registerSources([fileSource, oneSource]); + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("langpack-pl"), true); + + let bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + let bundle0 = (bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + let msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "addon value"); + + let bundle1 = (bundles.next()).value; + equal(bundle1.locales[0], "pl"); + equal(bundle1.hasMessage("key"), true); + let msg1 = bundle1.getMessage("key"); + equal(bundle1.formatPattern(msg1.value), "value"); + + equal((bundles.next()).done, true); + + // Remove langpack + + l10nReg.removeSources(["langpack-pl"]); + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("langpack-pl"), false); + + bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + bundle0 = (bundles.next()).value; + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("key"), true); + msg0 = bundle0.getMessage("key"); + equal(bundle0.formatPattern(msg0.value), "value"); + + equal((bundles.next()).done, true); + + // Remove app source + + l10nReg.removeSources(["app"]); + + equal(l10nReg.getSourceNames().length, 0); + + bundles = l10nReg.generateBundlesSync(["pl"], ["test.ftl"]); + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that the logic works correctly when there's a missing + * file in the FileSource scenario. + */ +add_task(function test_missing_file() { + const fs = [ + { path: "./app/data/locales/en-US/test.ftl", source: "key = value en-US" }, + { path: "./platform/data/locales/en-US/test.ftl", source: "key = value en-US" }, + { path: "./platform/data/locales/en-US/test2.ftl", source: "key2 = value2 en-US" }, + ]; + let oneSource = L10nFileSource.createMock("app", "", ["en-US"], "./app/data/locales/{locale}/", fs); + let twoSource = L10nFileSource.createMock("platform", "", ["en-US"], "./platform/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource, twoSource]); + + // has two sources + + equal(l10nReg.getSourceNames().length, 2); + equal(l10nReg.hasSource("app"), true); + equal(l10nReg.hasSource("platform"), true); + + + // returns a single context + + let bundles = l10nReg.generateBundlesSync(["en-US"], ["test.ftl", "test2.ftl"]); + + // First permutation: + // [platform, platform] - both present + let bundle1 = (bundles.next()); + equal(bundle1.value.hasMessage("key"), true); + + // Second permutation skipped: + // [platform, app] - second missing + // Third permutation: + // [app, platform] - both present + let bundle2 = (bundles.next()); + equal(bundle2.value.hasMessage("key"), true); + + // Fourth permutation skipped: + // [app, app] - second missing + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that we handle correctly a scenario where a source + * is being removed while the iterator operates. + */ +add_task(function test_remove_source_mid_iter_cycle() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "key = platform value" }, + { path: "./app/data/locales/pl/test.ftl", source: "key = app value" }, + ]; + let oneSource = L10nFileSource.createMock("platform", "", ["en-US"], "./platform/data/locales/{locale}/", fs); + let secondSource = L10nFileSource.createMock("app", "", ["pl"], "./app/data/locales/{locale}/", fs); + l10nReg.registerSources([oneSource, secondSource]); + + let bundles = l10nReg.generateBundlesSync(["en-US", "pl"], ["test.ftl"]); + + let bundle0 = bundles.next(); + + l10nReg.removeSources(["app"]); + + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +add_task(async function test_metasources() { + let fs = [ + { path: "/localization/en-US/browser/menu1.ftl", source: "key1 = Value" }, + { path: "/localization/en-US/browser/menu2.ftl", source: "key2 = Value" }, + { path: "/localization/en-US/browser/menu3.ftl", source: "key3 = Value" }, + { path: "/localization/en-US/browser/menu4.ftl", source: "key4 = Value" }, + { path: "/localization/en-US/browser/menu5.ftl", source: "key5 = Value" }, + { path: "/localization/en-US/browser/menu6.ftl", source: "key6 = Value" }, + { path: "/localization/en-US/browser/menu7.ftl", source: "key7 = Value" }, + { path: "/localization/en-US/browser/menu8.ftl", source: "key8 = Value" }, + ]; + + const browser = L10nFileSource.createMock("browser", "app", ["en-US"], "/localization/{locale}", fs); + const toolkit = L10nFileSource.createMock("toolkit", "app", ["en-US"], "/localization/{locale}", fs); + const browser2 = L10nFileSource.createMock("browser2", "langpack", ["en-US"], "/localization/{locale}", fs); + const toolkit2 = L10nFileSource.createMock("toolkit2", "langpack", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([toolkit, browser, toolkit2, browser2]); + + let res = [ + "/browser/menu1.ftl", + "/browser/menu2.ftl", + "/browser/menu3.ftl", + "/browser/menu4.ftl", + "/browser/menu5.ftl", + "/browser/menu6.ftl", + "/browser/menu7.ftl", + {path: "/browser/menu8.ftl", optional: false}, + ]; + + const bundles = l10nReg.generateBundlesSync(["en-US"], res); + + let nbundles = 0; + while (!bundles.next().done) { + nbundles += 1; + } + + // If metasources are working properly, we'll generate 2^8 = 256 bundles for + // each metasource giving 512 bundles in total. Otherwise, we generate + // 4^8 = 65536 bundles. + equal(nbundles, 512); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that when a required resource is missing for a locale, + * we do not produce a bundle for that locale. + */ +add_task(function test_missing_required_resource() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "test-key = en-US value" }, + { path: "./platform/data/locales/pl/missing-in-en-US.ftl", source: "missing-key = pl value" }, + { path: "./platform/data/locales/pl/test.ftl", source: "test-key = pl value" }, + ]; + let source = L10nFileSource.createMock("platform", "", ["en-US", "pl"], "./platform/data/locales/{locale}/", fs); + l10nReg.registerSources([source]); + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("platform"), true); + + + // returns correct contexts for [en-US, pl] + + let bundles = l10nReg.generateBundlesSync(["en-US", "pl"], ["test.ftl", "missing-in-en-US.ftl"]); + let bundle0 = (bundles.next()).value; + + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("test-key"), true); + equal(bundle0.hasMessage("missing-key"), true); + + let msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "pl value"); + + let msg1 = bundle0.getMessage("missing-key"); + equal(bundle0.formatPattern(msg1.value), "pl value"); + + equal((bundles.next()).done, true); + + + // returns correct contexts for [pl, en-US] + + bundles = l10nReg.generateBundlesSync(["pl", "en-US"], ["test.ftl", {path: "missing-in-en-US.ftl", optional: false}]); + bundle0 = (bundles.next()).value; + + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("test-key"), true); + + msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "pl value"); + + msg1 = bundle0.getMessage("missing-key"); + equal(bundle0.formatPattern(msg1.value), "pl value"); + + equal((bundles.next()).done, true); + + // cleanup + l10nReg.clearSources(); +}); + +/** + * This test verifies that when an optional resource is missing, we continue + * to produce a bundle for that locale. The bundle will have missing entries + * with regard to the missing optional resource. + */ +add_task(function test_missing_optional_resource() { + const fs = [ + { path: "./platform/data/locales/en-US/test.ftl", source: "test-key = en-US value" }, + { path: "./platform/data/locales/pl/missing-in-en-US.ftl", source: "missing-key = pl value" }, + { path: "./platform/data/locales/pl/test.ftl", source: "test-key = pl value" }, + ]; + let source = L10nFileSource.createMock("platform", "", ["en-US", "pl"], "./platform/data/locales/{locale}/", fs); + l10nReg.registerSources([source]); + + equal(l10nReg.getSourceNames().length, 1); + equal(l10nReg.hasSource("platform"), true); + + + // returns correct contexts for [en-US, pl] + + let bundles = l10nReg.generateBundlesSync(["en-US", "pl"], ["test.ftl", { path: "missing-in-en-US.ftl", optional: true }]); + let bundle0 = (bundles.next()).value; + + equal(bundle0.locales[0], "en-US"); + equal(bundle0.hasMessage("test-key"), true); + equal(bundle0.hasMessage("missing-key"), false); + + let msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "en-US value"); + + equal(bundle0.getMessage("missing-key"), null); + + let bundle1 = (bundles.next()).value; + + equal(bundle1.locales[0], "pl"); + equal(bundle1.hasMessage("test-key"), true); + equal(bundle1.hasMessage("missing-key"), true); + + msg0 = bundle1.getMessage("test-key"); + equal(bundle1.formatPattern(msg0.value), "pl value"); + + msg1 = bundle1.getMessage("missing-key"); + equal(bundle1.formatPattern(msg1.value), "pl value"); + + equal((bundles.next()).done, true); + + // returns correct contexts for [pl, en-US] + + bundles = l10nReg.generateBundlesSync(["pl", "en-US"], ["test.ftl", { path: "missing-in-en-US.ftl", optional: true }]); + bundle0 = (bundles.next()).value; + + equal(bundle0.locales[0], "pl"); + equal(bundle0.hasMessage("test-key"), true); + equal(bundle0.hasMessage("missing-key"), true); + + msg0 = bundle0.getMessage("test-key"); + equal(bundle0.formatPattern(msg0.value), "pl value"); + + msg1 = bundle0.getMessage("missing-key"); + equal(bundle0.formatPattern(msg1.value), "pl value"); + + bundle1 = (bundles.next()).value; + + equal(bundle1.locales[0], "en-US"); + equal(bundle1.hasMessage("test-key"), true); + equal(bundle1.hasMessage("missing-key"), false); + + msg0 = bundle1.getMessage("test-key"); + equal(bundle1.formatPattern(msg0.value), "en-US value"); + + equal(bundle1.getMessage("missing-key"), null); + + // cleanup + l10nReg.clearSources(); +}); diff --git a/intl/l10n/test/test_localization.js b/intl/l10n/test/test_localization.js new file mode 100644 index 0000000000..1b89a0549a --- /dev/null +++ b/intl/l10n/test/test_localization.js @@ -0,0 +1,319 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AppConstants } = ChromeUtils.importESModule("resource://gre/modules/AppConstants.sys.mjs"); + +// Disable `xpc::IsInAutomation()` so incomplete locales do not generate +// errors. +Services.prefs.setBoolPref( + "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", + false +); + +add_task(function test_methods_presence() { + strictEqual(typeof Localization.prototype.formatValues, "function"); + strictEqual(typeof Localization.prototype.formatMessages, "function"); + strictEqual(typeof Localization.prototype.formatValue, "function"); +}); + +add_task(async function test_methods_calling() { + const l10nReg = new L10nRegistry(); + const fs = [ + { path: "/localization/de/browser/menu.ftl", source: ` +key-value1 = [de] Value2 +` }, + { path: "/localization/en-US/browser/menu.ftl", source: ` +key-value1 = [en] Value2 +key-value2 = [en] Value3 +key-attr = + .label = [en] Label 3 +` }, + ]; + const originalRequested = Services.locale.requestedLocales; + + const source = L10nFileSource.createMock("test", "app", ["de", "en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const l10n = new Localization([ + "/browser/menu.ftl", + ], false, l10nReg, ["de", "en-US"]); + + { + let values = await l10n.formatValues([ + {id: "key-value1"}, + {id: "key-value2"}, + {id: "key-missing"}, + {id: "key-attr"} + ]); + + strictEqual(values[0], "[de] Value2"); + strictEqual(values[1], "[en] Value3"); + strictEqual(values[2], null); + strictEqual(values[3], null); + } + + { + let values = await l10n.formatValues([ + "key-value1", + "key-value2", + "key-missing", + "key-attr" + ]); + + strictEqual(values[0], "[de] Value2"); + strictEqual(values[1], "[en] Value3"); + strictEqual(values[2], null); + strictEqual(values[3], null); + } + + { + strictEqual(await l10n.formatValue("key-missing"), null); + strictEqual(await l10n.formatValue("key-value1"), "[de] Value2"); + strictEqual(await l10n.formatValue("key-value2"), "[en] Value3"); + strictEqual(await l10n.formatValue("key-attr"), null); + } + + { + let messages = await l10n.formatMessages([ + {id: "key-value1"}, + {id: "key-missing"}, + {id: "key-value2"}, + {id: "key-attr"}, + ]); + + strictEqual(messages[0].value, "[de] Value2"); + strictEqual(messages[1], null); + strictEqual(messages[2].value, "[en] Value3"); + strictEqual(messages[3].value, null); + } +}); + +add_task(async function test_builtins() { + const l10nReg = new L10nRegistry(); + const known_platforms = { + "linux": "linux", + "win": "windows", + "macosx": "macos", + "android": "android", + }; + + const fs = [ + { path: "/localization/en-US/test.ftl", source: ` +key = { PLATFORM() -> + ${ Object.values(known_platforms).map( + name => ` [${ name }] ${ name.toUpperCase() } Value\n`).join("") } + *[other] OTHER Value + }` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const l10n = new Localization([ + "/test.ftl", + ], false, l10nReg, ["en-US"]); + + let values = await l10n.formatValues([{id: "key"}]); + + ok(values[0].includes( + `${ known_platforms[AppConstants.platform].toUpperCase() } Value`)); +}); + +add_task(async function test_add_remove_resourceIds() { + const l10nReg = new L10nRegistry(); + const fs = [ + { path: "/localization/en-US/browser/menu.ftl", source: "key1 = Value1" }, + { path: "/localization/en-US/toolkit/menu.ftl", source: "key2 = Value2" }, + ]; + + + const source = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const l10n = new Localization(["/browser/menu.ftl"], false, l10nReg, ["en-US"]); + + let values = await l10n.formatValues([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], null); + + l10n.addResourceIds(["/toolkit/menu.ftl"]); + + values = await l10n.formatValues([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + values = await l10n.formatValues(["key1", {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + values = await l10n.formatValues([{id: "key1"}, "key2"]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + l10n.removeResourceIds(["/browser/menu.ftl"]); + + values = await l10n.formatValues([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], null); + strictEqual(values[1], "Value2"); +}); + +add_task(async function test_switch_to_async() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/en-US/browser/menu.ftl", source: "key1 = Value1" }, + { path: "/localization/en-US/toolkit/menu.ftl", source: "key2 = Value2" }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const l10n = new Localization(["/browser/menu.ftl"], true, l10nReg, ["en-US"]); + + let values = l10n.formatValuesSync([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], null); + + l10n.setAsync(); + + Assert.throws(() => { + l10n.formatValuesSync([{ id: "key1" }, { id: "key2" }]); + }, /Can't use formatValuesSync when state is async./); + + l10n.addResourceIds(["/toolkit/menu.ftl"]); + + values = await l10n.formatValues([{id: "key1"}, {id: "key2"}]); + let values2 = await l10n.formatValues([{id: "key1"}, {id: "key2"}]); + + deepEqual(values, values2); + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + l10n.removeResourceIds(["/browser/menu.ftl"]); + + values = await l10n.formatValues([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], null); + strictEqual(values[1], "Value2"); +}); + +/** + * This test verifies that when a required resource is missing, + * we fallback entirely to the next locale for all entries. + */ +add_task(async function test_format_from_missing_required_resource() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/de/browser/menu.ftl", source: ` +key-value = [de] Value1 +` }, + { path: "/localization/de/browser/missing-in-en-US.ftl", source: ` +key-missing = [de] MissingValue +` }, + { path: "/localization/en-US/browser/menu.ftl", source: ` +key-value = [en] Value1 +` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["de", "en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + // returns correct contexts for [en-US, de] + + let l10n = new Localization([ + "/browser/menu.ftl", + "/browser/missing-in-en-US.ftl", + ], false, l10nReg, ["en-US", "de"]); + + { + let values = await l10n.formatValues([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[de] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } + + // returns correct contexts for [de, en-US] + + l10n = new Localization([ + "/browser/menu.ftl", + {path: "/browser/missing-in-en-US.ftl", optional: false}, + ], false, l10nReg, ["de", "en-US"]); + + { + let values = await l10n.formatValues([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[de] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } +}); + +/** + * This test verifies that when an optional resource is missing, we continue + * to populate entires from other resources in the same locale, and we only + * fallback entries from the missing optional resource to the next locale. + */ +add_task(async function test_format_from_missing_optional_resource() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/de/browser/menu.ftl", source: ` +key-value = [de] Value1 +` }, + { path: "/localization/de/browser/missing-in-en-US.ftl", source: ` +key-missing = [de] MissingValue +` }, + { path: "/localization/en-US/browser/menu.ftl", source: ` +key-value = [en] Value1 +` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["de", "en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + // returns correct contexts for [en-US, de] + + let l10n = new Localization([ + {path: "/browser/menu.ftl", optional: false}, + {path: "/browser/missing-in-en-US.ftl", optional: true}, + ], false, l10nReg, ["en-US", "de"]); + + { + let values = await l10n.formatValues([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[en] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } + + // returns correct contexts for [de, en-US] + + l10n = new Localization([ + {path: "/browser/menu.ftl", optional: false}, + {path: "/browser/missing-in-en-US.ftl", optional: true}, + ], false, l10nReg, ["de", "en-US"]); + + { + let values = await l10n.formatValues([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[de] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } +}); diff --git a/intl/l10n/test/test_localization_sync.js b/intl/l10n/test/test_localization_sync.js new file mode 100644 index 0000000000..fa2186bc07 --- /dev/null +++ b/intl/l10n/test/test_localization_sync.js @@ -0,0 +1,289 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +const { AppConstants } = ChromeUtils.importESModule("resource://gre/modules/AppConstants.sys.mjs"); + +// Disable `xpc::IsInAutomation()` so incomplete locales do not generate +// errors. +Services.prefs.setBoolPref( + "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", + false +); + +add_task(function test_methods_calling() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/de/browser/menu.ftl", source: ` +key-value1 = [de] Value2 +` }, + { path: "/localization/en-US/browser/menu.ftl", source: ` +key-value1 = [en] Value2 +key-value2 = [en] Value3 +key-attr = + .label = [en] Label 3 +` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["de", "en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + const l10n = new Localization([ + "/browser/menu.ftl", + ], true, l10nReg, ["de", "en-US"]); + + + { + let values = l10n.formatValuesSync([ + {id: "key-value1"}, + {id: "key-value2"}, + {id: "key-missing"}, + {id: "key-attr"} + ]); + + strictEqual(values[0], "[de] Value2"); + strictEqual(values[1], "[en] Value3"); + strictEqual(values[2], null); + strictEqual(values[3], null); + } + + { + let values = l10n.formatValuesSync([ + "key-value1", + "key-value2", + "key-missing", + "key-attr" + ]); + + strictEqual(values[0], "[de] Value2"); + strictEqual(values[1], "[en] Value3"); + strictEqual(values[2], null); + strictEqual(values[3], null); + } + + { + strictEqual(l10n.formatValueSync("key-missing"), null); + strictEqual(l10n.formatValueSync("key-value1"), "[de] Value2"); + strictEqual(l10n.formatValueSync("key-value2"), "[en] Value3"); + strictEqual(l10n.formatValueSync("key-attr"), null); + } + + { + let messages = l10n.formatMessagesSync([ + {id: "key-value1"}, + {id: "key-missing"}, + {id: "key-value2"}, + {id: "key-attr"}, + ]); + + strictEqual(messages[0].value, "[de] Value2"); + strictEqual(messages[1], null); + strictEqual(messages[2].value, "[en] Value3"); + strictEqual(messages[3].value, null); + } +}); + +add_task(function test_builtins() { + const known_platforms = { + "linux": "linux", + "win": "windows", + "macosx": "macos", + "android": "android", + }; + + const fs = [ + { path: "/localization/en-US/test.ftl", source: ` +key = { PLATFORM() -> + ${ Object.values(known_platforms).map( + name => ` [${ name }] ${ name.toUpperCase() } Value\n`).join("") } + *[other] OTHER Value + }` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}", fs); + const l10nReg = new L10nRegistry(); + l10nReg.registerSources([source]); + + const l10n = new Localization([ + "/test.ftl", + ], true, l10nReg, ["en-US"]); + + let values = l10n.formatValuesSync([{id: "key"}]); + + ok(values[0].includes( + `${ known_platforms[AppConstants.platform].toUpperCase() } Value`)); +}); + +add_task(function test_add_remove_resourceIds() { + const fs = [ + { path: "/localization/en-US/browser/menu.ftl", source: "key1 = Value1" }, + { path: "/localization/en-US/toolkit/menu.ftl", source: "key2 = Value2" }, + ]; + const originalRequested = Services.locale.requestedLocales; + + const source = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}", fs); + const l10nReg = new L10nRegistry(); + l10nReg.registerSources([source]); + + const l10n = new Localization(["/browser/menu.ftl"], true, l10nReg, ["en-US"]); + + let values = l10n.formatValuesSync([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], null); + + l10n.addResourceIds(["/toolkit/menu.ftl"]); + + values = l10n.formatValuesSync([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + values = l10n.formatValuesSync(["key1", {id: "key2"}]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + values = l10n.formatValuesSync([{id: "key1"}, "key2"]); + + strictEqual(values[0], "Value1"); + strictEqual(values[1], "Value2"); + + l10n.removeResourceIds(["/browser/menu.ftl"]); + + values = l10n.formatValuesSync([{id: "key1"}, {id: "key2"}]); + + strictEqual(values[0], null); + strictEqual(values[1], "Value2"); +}); + +add_task(function test_calling_sync_methods_in_async_mode_fails() { + const l10n = new Localization(["/browser/menu.ftl"], false); + + Assert.throws(() => { + l10n.formatValuesSync([{ id: "key1" }, { id: "key2" }]); + }, /Can't use formatValuesSync when state is async./); + + Assert.throws(() => { + l10n.formatValueSync("key1"); + }, /Can't use formatValueSync when state is async./); + + Assert.throws(() => { + l10n.formatMessagesSync([{ id: "key1"}]); + }, /Can't use formatMessagesSync when state is async./); +}); + +/** + * This test verifies that when a required resource is missing, + * we fallback entirely to the next locale for all entries. + */ +add_task(function test_format_from_missing_required_resource() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/de/browser/menu.ftl", source: ` +key-value = [de] Value1 +` }, + { path: "/localization/de/browser/missing-in-en-US.ftl", source: ` +key-missing = [de] MissingValue +` }, + { path: "/localization/en-US/browser/menu.ftl", source: ` +key-value = [en] Value1 +` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["de", "en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + // returns correct contexts for [en-US, de] + + let l10n = new Localization([ + "/browser/menu.ftl", + "/browser/missing-in-en-US.ftl", + ], true, l10nReg, ["en-US", "de"]); + + { + let values = l10n.formatValuesSync([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[de] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } + + // returns correct contexts for [de, en-US] + + l10n = new Localization([ + "/browser/menu.ftl", + {path: "/browser/missing-in-en-US.ftl", optional: false}, + ], true, l10nReg, ["de", "en-US"]); + + { + let values = l10n.formatValuesSync([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[de] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } +}); + +/** + * This test verifies that when an optional resource is missing + * we continue to populate entires from other resources in the same locale + * and only fallback entries from the optional resource to the next locale. + */ +add_task(function test_format_from_missing_optional_resource() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/de/browser/menu.ftl", source: ` +key-value = [de] Value1 +` }, + { path: "/localization/de/browser/missing-in-en-US.ftl", source: ` +key-missing = [de] MissingValue +` }, + { path: "/localization/en-US/browser/menu.ftl", source: ` +key-value = [en] Value1 +` }, + ]; + + const source = L10nFileSource.createMock("test", "app", ["de", "en-US"], "/localization/{locale}", fs); + l10nReg.registerSources([source]); + + // returns correct contexts for [en-US, de] + + let l10n = new Localization([ + {path: "/browser/menu.ftl", optional: false}, + {path: "/browser/missing-in-en-US.ftl", optional: true}, + ], true, l10nReg, ["en-US", "de"]); + + { + let values = l10n.formatValuesSync([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[en] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } + + // returns correct contexts for [de, en-US] + + l10n = new Localization([ + {path: "/browser/menu.ftl", optional: false}, + {path: "/browser/missing-in-en-US.ftl", optional: true}, + ], true, l10nReg, ["de", "en-US"]); + + { + let values = l10n.formatValuesSync([ + {id: "key-value"}, + {id: "key-missing"}, + ]); + + strictEqual(values[0], "[de] Value1"); + strictEqual(values[1], "[de] MissingValue"); + } +}); diff --git a/intl/l10n/test/test_messagecontext.js b/intl/l10n/test/test_messagecontext.js new file mode 100644 index 0000000000..7ab17dea90 --- /dev/null +++ b/intl/l10n/test/test_messagecontext.js @@ -0,0 +1,47 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +function run_test() { + test_methods_presence(FluentBundle); + test_methods_calling(FluentBundle, FluentResource); + test_number_options(FluentBundle, FluentResource); + + ok(true); +} + +function test_methods_presence(FluentBundle) { + const bundle = new FluentBundle(["en-US", "pl"]); + equal(typeof bundle.addResource, "function"); + equal(typeof bundle.formatPattern, "function"); +} + +function test_methods_calling(FluentBundle, FluentResource) { + const bundle = new FluentBundle(["en-US", "pl"], { + useIsolating: false, + }); + bundle.addResource(new FluentResource("key = Value")); + + const msg = bundle.getMessage("key"); + equal(bundle.formatPattern(msg.value), "Value"); + + bundle.addResource(new FluentResource("key2 = Hello { $name }")); + + const msg2 = bundle.getMessage("key2"); + equal(bundle.formatPattern(msg2.value, { name: "Amy" }), "Hello Amy"); + ok(true); +} + +function test_number_options(FluentBundle, FluentResource) { + const bundle = new FluentBundle(["en-US", "pl"], { + useIsolating: false, + }); + bundle.addResource(new FluentResource(` +key = { NUMBER(0.53, style: "percent") } { NUMBER(0.12, style: "percent", minimumFractionDigits: 0) } + { NUMBER(-2.5, style: "percent") } { NUMBER(2.91, style: "percent") } { NUMBER("wrong", style: "percent") } +`)); + + const msg = bundle.getMessage("key"); + equal(bundle.formatPattern(msg.value), "53.00% 12%\n-250.0% 291.00% "); + + ok(true); +} diff --git a/intl/l10n/test/test_missing_variables.js b/intl/l10n/test/test_missing_variables.js new file mode 100644 index 0000000000..8c05f7f94f --- /dev/null +++ b/intl/l10n/test/test_missing_variables.js @@ -0,0 +1,35 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + +// Disable `xpc::IsInAutomation()` so that missing variables don't throw +// errors. +Services.prefs.setBoolPref( + "security.turn_off_all_security_so_that_viruses_can_take_over_this_computer", + false +); + +/** + * The following test demonstrates crashing behavior. + */ +add_task(function test_missing_variables() { + const l10nReg = new L10nRegistry(); + + const fs = [ + { path: "/localization/en-US/browser/test.ftl", source: "welcome-message = Welcome { $user }\n" } + ] + const locales = ["en-US"]; + const source = L10nFileSource.createMock("test", "app", locales, "/localization/{locale}", fs); + l10nReg.registerSources([source]); + const l10n = new Localization(["/browser/test.ftl"], true, l10nReg, locales); + + { + const [message] = l10n.formatValuesSync([{ id: "welcome-message", args: { user: "Greg" } }]); + equal(message, "Welcome Greg"); + } + + { + // This will crash in debug builds. + const [message] = l10n.formatValuesSync([{ id: "welcome-message" }]); + equal(message, "Welcome {$user}"); + } +}); diff --git a/intl/l10n/test/test_pseudo.js b/intl/l10n/test/test_pseudo.js new file mode 100644 index 0000000000..23404aab12 --- /dev/null +++ b/intl/l10n/test/test_pseudo.js @@ -0,0 +1,131 @@ +/* Any copyright is dedicated to the Public Domain. + http://creativecommons.org/publicdomain/zero/1.0/ */ + + +const originalValues = { + requested: Services.locale.requestedLocales, +}; + +const l10nReg = new L10nRegistry(); + +function getMockRegistry() { + const mockSource = L10nFileSource.createMock("test", "app", ["en-US"], "/localization/{locale}/", [ + { + path: "/localization/en-US/mock.ftl", + source: ` +key = This is a single message + .tooltip = This is a tooltip + .accesskey = f +` + } + ]); + let registry = new L10nRegistry(); + registry.registerSources([mockSource]); + return registry; +} + +function getAttributeByName(attributes, name) { + return attributes.find(attr => attr.name === name); +} + +/** + * This test verifies that as we switch between + * different pseudo strategies, the Localization object + * follows and formats using the given strategy. + * + * We test values and attributes and make sure that + * a single-character attributes, commonly used for access keys + * don't get transformed. + */ +add_task(async function test_pseudo_works() { + Services.prefs.setStringPref("intl.l10n.pseudo", ""); + + let mockRegistry = getMockRegistry(); + + const l10n = new Localization([ + "mock.ftl", + ], false, mockRegistry); + + { + // 1. Start with no pseudo + + let message = (await l10n.formatMessages([{id: "key"}]))[0]; + + ok(message.value.includes("This is a single message")); + let attr0 = getAttributeByName(message.attributes, "tooltip"); + ok(attr0.value.includes("This is a tooltip")); + let attr1 = getAttributeByName(message.attributes, "accesskey"); + equal(attr1.value, "f"); + } + + { + // 2. Set Accented Pseudo + + Services.prefs.setStringPref("intl.l10n.pseudo", "accented"); + let message = (await l10n.formatMessages([{id: "key"}]))[0]; + + ok(message.value.includes("[Ŧħiş iş aa şiƞɠŀee ḿeeşşaaɠee]")); + let attr0 = getAttributeByName(message.attributes, "tooltip"); + ok(attr0.value.includes("[Ŧħiş iş aa ŧooooŀŧiƥ]")); + let attr1 = getAttributeByName(message.attributes, "accesskey"); + equal(attr1.value, "f"); + } + + { + // 3. Set Bidi Pseudo + + Services.prefs.setStringPref("intl.l10n.pseudo", "bidi"); + let message = (await l10n.formatMessages([{id: "key"}]))[0]; + + ok(message.value.includes("iş a şiƞɠŀe ḿeşşaɠe")); + let attr0 = getAttributeByName(message.attributes, "tooltip"); + ok(attr0.value.includes("Ŧħiş iş a ŧooŀŧiƥ")); + let attr1 = getAttributeByName(message.attributes, "accesskey"); + equal(attr1.value, "f"); + } + + { + // 4. Remove pseudo + + Services.prefs.setStringPref("intl.l10n.pseudo", ""); + let message = (await l10n.formatMessages([{id: "key"}]))[0]; + + ok(message.value.includes("This is a single message")); + let attr0 = getAttributeByName(message.attributes, "tooltip"); + ok(attr0.value.includes("This is a tooltip")); + let attr1 = getAttributeByName(message.attributes, "accesskey"); + equal(attr1.value, "f"); + } + + Services.locale.requestedLocales = originalValues.requested; +}); + +/** + * This test verifies that setting a bogus pseudo locale + * strategy doesn't break anything. + */ +add_task(async function test_unavailable_strategy_works() { + Services.prefs.setStringPref("intl.l10n.pseudo", ""); + + let mockRegistry = getMockRegistry(); + + const l10n = new Localization([ + "mock.ftl", + ], false, mockRegistry); + + { + // 1. Set unavailable pseudo strategy + Services.prefs.setStringPref("intl.l10n.pseudo", "unknown-strategy"); + + let message = (await l10n.formatMessages([{id: "key"}]))[0]; + + ok(message.value.includes("This is a single message")); + let attr0 = getAttributeByName(message.attributes, "tooltip"); + ok(attr0.value.includes("This is a tooltip")); + let attr1 = getAttributeByName(message.attributes, "accesskey"); + equal(attr1.value, "f"); + } + + Services.prefs.setStringPref("intl.l10n.pseudo", ""); + Services.locale.requestedLocales = originalValues.requested; +}); diff --git a/intl/l10n/test/xpcshell.ini b/intl/l10n/test/xpcshell.ini new file mode 100644 index 0000000000..4b5073ebe0 --- /dev/null +++ b/intl/l10n/test/xpcshell.ini @@ -0,0 +1,13 @@ +[DEFAULT] +head = + +[test_datetimeformat.js] +[test_l10nregistry.js] +[test_l10nregistry_fuzzed.js] +[test_l10nregistry_sync.js] +[test_localization.js] +[test_localization_sync.js] +[test_messagecontext.js] +[test_missing_variables.js] +skip-if = debug # Intentionally triggers a debug assert for missing Fluent arguments. +[test_pseudo.js] |