From 36d22d82aa202bb199967e9512281e9a53db42c9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sun, 7 Apr 2024 21:33:14 +0200 Subject: Adding upstream version 115.7.0esr. Signed-off-by: Daniel Baumann --- intl/l10n/FileSource.cpp | 192 ++ intl/l10n/FileSource.h | 73 + intl/l10n/FluentBindings.h | 34 + intl/l10n/FluentBundle.cpp | 506 +++++ intl/l10n/FluentBundle.h | 99 + intl/l10n/FluentResource.cpp | 45 + intl/l10n/FluentResource.h | 47 + intl/l10n/FluentSyntax.jsm | 1957 ++++++++++++++++++++ intl/l10n/L10nRegistry.cpp | 442 +++++ intl/l10n/L10nRegistry.h | 152 ++ intl/l10n/Localization.cpp | 522 ++++++ intl/l10n/Localization.h | 165 ++ intl/l10n/LocalizationBindings.h | 26 + intl/l10n/README | 8 + intl/l10n/RegistryBindings.h | 53 + intl/l10n/docs/crosschannel/commits.rst | 33 + intl/l10n/docs/crosschannel/content.rst | 129 ++ intl/l10n/docs/crosschannel/index.rst | 88 + intl/l10n/docs/crosschannel/repositories.rst | 14 + intl/l10n/docs/fluent/index.rst | 25 + intl/l10n/docs/fluent/review.rst | 303 +++ intl/l10n/docs/fluent/tutorial.rst | 750 ++++++++ intl/l10n/docs/glossary.rst | 22 + intl/l10n/docs/index.rst | 26 + intl/l10n/docs/migrations/fluent.rst | 153 ++ intl/l10n/docs/migrations/index.rst | 53 + intl/l10n/docs/migrations/legacy.rst | 642 +++++++ intl/l10n/docs/migrations/localizations.rst | 42 + intl/l10n/docs/migrations/overview.rst | 136 ++ intl/l10n/docs/migrations/testing.rst | 58 + intl/l10n/docs/overview.rst | 199 ++ intl/l10n/moz.build | 61 + intl/l10n/rust/fluent-ffi/Cargo.toml | 16 + intl/l10n/rust/fluent-ffi/cbindgen.toml | 24 + intl/l10n/rust/fluent-ffi/src/builtins.rs | 389 ++++ intl/l10n/rust/fluent-ffi/src/bundle.rs | 331 ++++ intl/l10n/rust/fluent-ffi/src/ffi.rs | 154 ++ intl/l10n/rust/fluent-ffi/src/lib.rs | 11 + intl/l10n/rust/fluent-ffi/src/resource.rs | 39 + intl/l10n/rust/gtest/Cargo.toml | 14 + intl/l10n/rust/gtest/Test.cpp | 23 + intl/l10n/rust/gtest/moz.build | 11 + intl/l10n/rust/gtest/test.rs | 58 + intl/l10n/rust/l10nregistry-ffi/Cargo.toml | 24 + intl/l10n/rust/l10nregistry-ffi/cbindgen.toml | 26 + intl/l10n/rust/l10nregistry-ffi/src/env.rs | 132 ++ intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs | 70 + intl/l10n/rust/l10nregistry-ffi/src/lib.rs | 10 + intl/l10n/rust/l10nregistry-ffi/src/load.rs | 113 ++ intl/l10n/rust/l10nregistry-ffi/src/registry.rs | 519 ++++++ intl/l10n/rust/l10nregistry-ffi/src/source.rs | 359 ++++ intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs | 129 ++ intl/l10n/rust/l10nregistry-rs/.gitignore | 2 + intl/l10n/rust/l10nregistry-rs/Cargo.toml | 87 + intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE | 201 ++ intl/l10n/rust/l10nregistry-rs/LICENSE-MIT | 19 + intl/l10n/rust/l10nregistry-rs/README.md | 17 + .../rust/l10nregistry-rs/benches/localization.rs | 70 + .../rust/l10nregistry-rs/benches/preferences.rs | 65 + intl/l10n/rust/l10nregistry-rs/benches/registry.rs | 133 ++ intl/l10n/rust/l10nregistry-rs/benches/solver.rs | 120 ++ intl/l10n/rust/l10nregistry-rs/benches/source.rs | 60 + intl/l10n/rust/l10nregistry-rs/src/env.rs | 5 + intl/l10n/rust/l10nregistry-rs/src/errors.rs | 74 + intl/l10n/rust/l10nregistry-rs/src/fluent.rs | 5 + intl/l10n/rust/l10nregistry-rs/src/lib.rs | 8 + .../l10nregistry-rs/src/registry/asynchronous.rs | 294 +++ intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs | 363 ++++ .../l10nregistry-rs/src/registry/synchronous.rs | 307 +++ .../l10n/rust/l10nregistry-rs/src/solver/README.md | 239 +++ intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs | 122 ++ .../rust/l10nregistry-rs/src/solver/parallel.rs | 175 ++ .../l10n/rust/l10nregistry-rs/src/solver/serial.rs | 76 + .../rust/l10nregistry-rs/src/solver/testing/mod.rs | 38 + .../src/solver/testing/scenarios.rs | 151 ++ .../rust/l10nregistry-rs/src/source/fetcher.rs | 30 + intl/l10n/rust/l10nregistry-rs/src/source/mod.rs | 558 ++++++ intl/l10n/rust/l10nregistry-rs/src/testing.rs | 322 ++++ .../rust/l10nregistry-rs/tests/localization.rs | 201 ++ intl/l10n/rust/l10nregistry-rs/tests/registry.rs | 304 +++ .../rust/l10nregistry-rs/tests/scenarios_async.rs | 109 ++ .../rust/l10nregistry-rs/tests/scenarios_sync.rs | 107 ++ intl/l10n/rust/l10nregistry-rs/tests/source.rs | 305 +++ intl/l10n/rust/localization-ffi/Cargo.toml | 23 + intl/l10n/rust/localization-ffi/cbindgen.toml | 33 + intl/l10n/rust/localization-ffi/src/lib.rs | 625 +++++++ intl/l10n/test/gtest/TestLocalization.cpp | 118 ++ intl/l10n/test/gtest/moz.build | 11 + intl/l10n/test/mochitest/chrome.ini | 6 + .../localization/test_formatMessages.html | 101 + .../mochitest/localization/test_formatValue.html | 81 + .../mochitest/localization/test_formatValues.html | 85 + intl/l10n/test/test_datetimeformat.js | 76 + intl/l10n/test/test_l10nregistry.js | 563 ++++++ intl/l10n/test/test_l10nregistry_fuzzed.js | 205 ++ intl/l10n/test/test_l10nregistry_sync.js | 536 ++++++ intl/l10n/test/test_localization.js | 319 ++++ intl/l10n/test/test_localization_sync.js | 289 +++ intl/l10n/test/test_messagecontext.js | 47 + intl/l10n/test/test_missing_variables.js | 35 + intl/l10n/test/test_pseudo.js | 131 ++ intl/l10n/test/xpcshell.ini | 13 + 102 files changed, 17371 insertions(+) create mode 100644 intl/l10n/FileSource.cpp create mode 100644 intl/l10n/FileSource.h create mode 100644 intl/l10n/FluentBindings.h create mode 100644 intl/l10n/FluentBundle.cpp create mode 100644 intl/l10n/FluentBundle.h create mode 100644 intl/l10n/FluentResource.cpp create mode 100644 intl/l10n/FluentResource.h create mode 100644 intl/l10n/FluentSyntax.jsm create mode 100644 intl/l10n/L10nRegistry.cpp create mode 100644 intl/l10n/L10nRegistry.h create mode 100644 intl/l10n/Localization.cpp create mode 100644 intl/l10n/Localization.h create mode 100644 intl/l10n/LocalizationBindings.h create mode 100644 intl/l10n/README create mode 100644 intl/l10n/RegistryBindings.h create mode 100644 intl/l10n/docs/crosschannel/commits.rst create mode 100644 intl/l10n/docs/crosschannel/content.rst create mode 100644 intl/l10n/docs/crosschannel/index.rst create mode 100644 intl/l10n/docs/crosschannel/repositories.rst create mode 100644 intl/l10n/docs/fluent/index.rst create mode 100644 intl/l10n/docs/fluent/review.rst create mode 100644 intl/l10n/docs/fluent/tutorial.rst create mode 100644 intl/l10n/docs/glossary.rst create mode 100644 intl/l10n/docs/index.rst create mode 100644 intl/l10n/docs/migrations/fluent.rst create mode 100644 intl/l10n/docs/migrations/index.rst create mode 100644 intl/l10n/docs/migrations/legacy.rst create mode 100644 intl/l10n/docs/migrations/localizations.rst create mode 100644 intl/l10n/docs/migrations/overview.rst create mode 100644 intl/l10n/docs/migrations/testing.rst create mode 100644 intl/l10n/docs/overview.rst create mode 100644 intl/l10n/moz.build create mode 100644 intl/l10n/rust/fluent-ffi/Cargo.toml create mode 100644 intl/l10n/rust/fluent-ffi/cbindgen.toml create mode 100644 intl/l10n/rust/fluent-ffi/src/builtins.rs create mode 100644 intl/l10n/rust/fluent-ffi/src/bundle.rs create mode 100644 intl/l10n/rust/fluent-ffi/src/ffi.rs create mode 100644 intl/l10n/rust/fluent-ffi/src/lib.rs create mode 100644 intl/l10n/rust/fluent-ffi/src/resource.rs create mode 100644 intl/l10n/rust/gtest/Cargo.toml create mode 100644 intl/l10n/rust/gtest/Test.cpp create mode 100644 intl/l10n/rust/gtest/moz.build create mode 100644 intl/l10n/rust/gtest/test.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/Cargo.toml create mode 100644 intl/l10n/rust/l10nregistry-ffi/cbindgen.toml create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/env.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/fetcher.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/lib.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/load.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/registry.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/source.rs create mode 100644 intl/l10n/rust/l10nregistry-ffi/src/xpcom_utils.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/.gitignore create mode 100644 intl/l10n/rust/l10nregistry-rs/Cargo.toml create mode 100644 intl/l10n/rust/l10nregistry-rs/LICENSE-APACHE create mode 100644 intl/l10n/rust/l10nregistry-rs/LICENSE-MIT create mode 100644 intl/l10n/rust/l10nregistry-rs/README.md create mode 100644 intl/l10n/rust/l10nregistry-rs/benches/localization.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/benches/preferences.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/benches/registry.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/benches/solver.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/benches/source.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/env.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/errors.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/fluent.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/lib.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/registry/asynchronous.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/registry/mod.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/registry/synchronous.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/solver/README.md create mode 100644 intl/l10n/rust/l10nregistry-rs/src/solver/mod.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/solver/parallel.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/solver/serial.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/solver/testing/mod.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/solver/testing/scenarios.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/source/fetcher.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/source/mod.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/src/testing.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/tests/localization.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/tests/registry.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/tests/scenarios_async.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/tests/scenarios_sync.rs create mode 100644 intl/l10n/rust/l10nregistry-rs/tests/source.rs create mode 100644 intl/l10n/rust/localization-ffi/Cargo.toml create mode 100644 intl/l10n/rust/localization-ffi/cbindgen.toml create mode 100644 intl/l10n/rust/localization-ffi/src/lib.rs create mode 100644 intl/l10n/test/gtest/TestLocalization.cpp create mode 100644 intl/l10n/test/gtest/moz.build create mode 100644 intl/l10n/test/mochitest/chrome.ini create mode 100644 intl/l10n/test/mochitest/localization/test_formatMessages.html create mode 100644 intl/l10n/test/mochitest/localization/test_formatValue.html create mode 100644 intl/l10n/test/mochitest/localization/test_formatValues.html create mode 100644 intl/l10n/test/test_datetimeformat.js create mode 100644 intl/l10n/test/test_l10nregistry.js create mode 100644 intl/l10n/test/test_l10nregistry_fuzzed.js create mode 100644 intl/l10n/test/test_l10nregistry_sync.js create mode 100644 intl/l10n/test/test_localization.js create mode 100644 intl/l10n/test/test_localization_sync.js create mode 100644 intl/l10n/test/test_messagecontext.js create mode 100644 intl/l10n/test/test_missing_variables.js create mode 100644 intl/l10n/test/test_pseudo.js create mode 100644 intl/l10n/test/xpcshell.ini (limited to 'intl/l10n') diff --git a/intl/l10n/FileSource.cpp b/intl/l10n/FileSource.cpp new file mode 100644 index 0000000000..2469f597c6 --- /dev/null +++ b/intl/l10n/FileSource.cpp @@ -0,0 +1,192 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FileSource.h" +#include "mozilla/dom/Promise.h" + +using namespace mozilla::dom; + +namespace mozilla::intl { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(L10nFileSource, mGlobal) + +L10nFileSource::L10nFileSource(RefPtr aRaw, + nsIGlobalObject* aGlobal) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +/* static */ +already_AddRefed L10nFileSource::Constructor( + const GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray& aLocales, + const nsACString& aPrePath, const dom::FileSourceOptions& aOptions, + const Optional>& aIndex, ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + + ffi::L10nFileSourceStatus status; + + bool allowOverrides = aOptions.mAddResourceOptions.mAllowOverrides; + + RefPtr raw; + if (aIndex.WasPassed()) { + raw = dont_AddRef(ffi::l10nfilesource_new_with_index( + &aName, &aMetaSource, &aLocales, &aPrePath, aIndex.Value().Elements(), + aIndex.Value().Length(), allowOverrides, &status)); + } else { + raw = dont_AddRef(ffi::l10nfilesource_new( + &aName, &aMetaSource, &aLocales, &aPrePath, allowOverrides, &status)); + } + + if (PopulateError(aRv, status)) { + return nullptr; + } + return MakeAndAddRef(std::move(raw), global); +} + +/* static */ +already_AddRefed L10nFileSource::CreateMock( + const GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray& aLocales, + const nsACString& aPrePath, const nsTArray& aFS, + ErrorResult& aRv) { + nsTArray fs(aFS.Length()); + for (const auto& file : aFS) { + auto f = fs.AppendElement(); + f->path = file.mPath; + f->source = file.mSource; + } + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + + ffi::L10nFileSourceStatus status; + + RefPtr raw(dont_AddRef(ffi::l10nfilesource_new_mock( + &aName, &aMetaSource, &aLocales, &aPrePath, &fs, &status))); + + if (PopulateError(aRv, status)) { + return nullptr; + } + return MakeAndAddRef(std::move(raw), global); +} + +JSObject* L10nFileSource::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return L10nFileSource_Binding::Wrap(aCx, this, aGivenProto); +} + +void L10nFileSource::GetName(nsCString& aRetVal) { + ffi::l10nfilesource_get_name(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetMetaSource(nsCString& aRetVal) { + ffi::l10nfilesource_get_metasource(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetLocales(nsTArray& aRetVal) { + ffi::l10nfilesource_get_locales(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetPrePath(nsCString& aRetVal) { + ffi::l10nfilesource_get_prepath(mRaw.get(), &aRetVal); +} + +void L10nFileSource::GetIndex(Nullable>& aRetVal) { + bool hasIndex = + ffi::l10nfilesource_get_index(mRaw.get(), &aRetVal.SetValue()); + if (!hasIndex) { + aRetVal.SetNull(); + } +} + +L10nFileSourceHasFileStatus L10nFileSource::HasFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv) { + ffi::L10nFileSourceStatus status; + + bool isPresent = false; + bool hasValue = ffi::l10nfilesource_has_file(mRaw.get(), &aLocale, &aPath, + &status, &isPresent); + + if (!PopulateError(aRv, status) && hasValue) { + if (isPresent) { + return L10nFileSourceHasFileStatus::Present; + } + + return L10nFileSourceHasFileStatus::Missing; + } + return L10nFileSourceHasFileStatus::Unknown; +} + +already_AddRefed L10nFileSource::FetchFileSync( + const nsACString& aLocale, const nsACString& aPath, ErrorResult& aRv) { + ffi::L10nFileSourceStatus status; + + RefPtr raw = + dont_AddRef(ffi::l10nfilesource_fetch_file_sync(mRaw.get(), &aLocale, + &aPath, &status)); + + if (!PopulateError(aRv, status) && raw) { + return MakeAndAddRef(mGlobal, raw); + } + + return nullptr; +} + +already_AddRefed L10nFileSource::FetchFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv) { + RefPtr promise = Promise::Create(mGlobal, aRv); + if (aRv.Failed()) { + return nullptr; + } + + ffi::L10nFileSourceStatus status; + + ffi::l10nfilesource_fetch_file( + mRaw.get(), &aLocale, &aPath, promise, + [](const Promise* aPromise, const ffi::FluentResource* aRes) { + Promise* promise = const_cast(aPromise); + + if (aRes) { + nsIGlobalObject* global = promise->GetGlobalObject(); + RefPtr res = new FluentResource(global, aRes); + promise->MaybeResolve(res); + } else { + promise->MaybeResolve(JS::NullHandleValue); + } + }, + &status); + + if (PopulateError(aRv, status)) { + return nullptr; + } + + return promise.forget(); +} + +/* static */ +bool L10nFileSource::PopulateError(ErrorResult& aError, + ffi::L10nFileSourceStatus& aStatus) { + switch (aStatus) { + case ffi::L10nFileSourceStatus::InvalidLocaleCode: + aError.ThrowTypeError("Invalid locale code"); + return true; + case ffi::L10nFileSourceStatus::EmptyName: + aError.ThrowTypeError("Name cannot be empty."); + return true; + case ffi::L10nFileSourceStatus::EmptyPrePath: + aError.ThrowTypeError("prePath cannot be empty."); + return true; + case ffi::L10nFileSourceStatus::EmptyResId: + aError.ThrowTypeError("resId cannot be empty."); + return true; + + case ffi::L10nFileSourceStatus::None: + return false; + } + MOZ_ASSERT_UNREACHABLE("Unknown status"); + return false; +} + +} // namespace mozilla::intl diff --git a/intl/l10n/FileSource.h b/intl/l10n/FileSource.h new file mode 100644 index 0000000000..e719dff74f --- /dev/null +++ b/intl/l10n/FileSource.h @@ -0,0 +1,73 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FileSource_h +#define mozilla_intl_l10n_FileSource_h + +#include "nsWrapperCache.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/L10nRegistryBinding.h" +#include "mozilla/dom/FluentBinding.h" +#include "mozilla/intl/RegistryBindings.h" + +class nsIGlobalObject; + +namespace mozilla::intl { + +class L10nFileSource : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(L10nFileSource) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(L10nFileSource) + + explicit L10nFileSource(RefPtr aRaw, + nsIGlobalObject* aGlobal = nullptr); + + static already_AddRefed Constructor( + const dom::GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray& aLocales, + const nsACString& aPrePath, const dom::FileSourceOptions& aOptions, + const dom::Optional>& aIndex, ErrorResult& aRv); + + static already_AddRefed CreateMock( + const dom::GlobalObject& aGlobal, const nsACString& aName, + const nsACString& aMetaSource, const nsTArray& aLocales, + const nsACString& aPrePath, + const nsTArray& aFS, ErrorResult& aRv); + + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + void GetName(nsCString& aRetVal); + void GetMetaSource(nsCString& aRetVal); + void GetLocales(nsTArray& aRetVal); + void GetPrePath(nsCString& aRetVal); + void GetIndex(dom::Nullable>& aRetVal); + + dom::L10nFileSourceHasFileStatus HasFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv); + already_AddRefed FetchFile(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv); + already_AddRefed FetchFileSync(const nsACString& aLocale, + const nsACString& aPath, + ErrorResult& aRv); + + const ffi::FileSource* Raw() const { return mRaw; } + + protected: + virtual ~L10nFileSource() = default; + nsCOMPtr mGlobal; + const RefPtr mRaw; + static bool PopulateError(ErrorResult& aError, + ffi::L10nFileSourceStatus& aStatus); +}; + +} // namespace mozilla::intl + +#endif diff --git a/intl/l10n/FluentBindings.h b/intl/l10n/FluentBindings.h new file mode 100644 index 0000000000..836929eef0 --- /dev/null +++ b/intl/l10n/FluentBindings.h @@ -0,0 +1,34 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FluentBindings_h +#define mozilla_intl_l10n_FluentBindings_h + +#include "mozilla/intl/fluent_ffi_generated.h" + +#include "mozilla/RefPtr.h" + +namespace mozilla { + +template <> +struct RefPtrTraits { + static void AddRef(const intl::ffi::FluentResource* aPtr) { + intl::ffi::fluent_resource_addref(aPtr); + } + static void Release(const intl::ffi::FluentResource* aPtr) { + intl::ffi::fluent_resource_release(aPtr); + } +}; + +template <> +class DefaultDelete { + public: + void operator()(intl::ffi::FluentBundleRc* aPtr) const { + fluent_bundle_destroy(aPtr); + } +}; + +} // namespace mozilla + +#endif diff --git a/intl/l10n/FluentBundle.cpp b/intl/l10n/FluentBundle.cpp new file mode 100644 index 0000000000..a20fca5564 --- /dev/null +++ b/intl/l10n/FluentBundle.cpp @@ -0,0 +1,506 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "FluentBundle.h" +#include "nsContentUtils.h" +#include "mozilla/dom/ToJSValue.h" +#include "mozilla/dom/UnionTypes.h" +#include "mozilla/intl/NumberFormat.h" +#include "mozilla/intl/DateTimeFormat.h" +#include "mozilla/intl/DateTimePatternGenerator.h" +#include "nsIInputStream.h" +#include "nsStringFwd.h" +#include "nsTArray.h" +#include "js/PropertyAndElement.h" // JS_DefineElement + +using namespace mozilla::dom; + +namespace mozilla { +namespace intl { + +class SizeableUTF8Buffer { + public: + using CharType = char; + + bool reserve(size_t size) { + mBuffer.reset(reinterpret_cast(malloc(size))); + mCapacity = size; + return true; + } + + CharType* data() { return mBuffer.get(); } + + size_t capacity() const { return mCapacity; } + + void written(size_t amount) { mWritten = amount; } + + size_t mWritten = 0; + size_t mCapacity = 0; + + struct FreePolicy { + void operator()(const void* ptr) { free(const_cast(ptr)); } + }; + + UniquePtr mBuffer; +}; + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentPattern, mParent) + +FluentPattern::FluentPattern(nsISupports* aParent, const nsACString& aId) + : mId(aId), mParent(aParent) { + MOZ_COUNT_CTOR(FluentPattern); +} +FluentPattern::FluentPattern(nsISupports* aParent, const nsACString& aId, + const nsACString& aAttrName) + : mId(aId), mAttrName(aAttrName), mParent(aParent) { + MOZ_COUNT_CTOR(FluentPattern); +} + +JSObject* FluentPattern::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return FluentPattern_Binding::Wrap(aCx, this, aGivenProto); +} + +FluentPattern::~FluentPattern() { MOZ_COUNT_DTOR(FluentPattern); }; + +/* FluentBundle */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentBundle, mParent) + +FluentBundle::FluentBundle(nsISupports* aParent, + UniquePtr aRaw) + : mParent(aParent), mRaw(std::move(aRaw)) { + MOZ_COUNT_CTOR(FluentBundle); +} + +already_AddRefed FluentBundle::Constructor( + const dom::GlobalObject& aGlobal, + const UTF8StringOrUTF8StringSequence& aLocales, + const dom::FluentBundleOptions& aOptions, ErrorResult& aRv) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + if (!global) { + aRv.Throw(NS_ERROR_FAILURE); + return nullptr; + } + + bool useIsolating = aOptions.mUseIsolating; + + nsAutoCString pseudoStrategy; + if (aOptions.mPseudoStrategy.WasPassed()) { + pseudoStrategy = aOptions.mPseudoStrategy.Value(); + } + + UniquePtr raw; + + if (aLocales.IsUTF8String()) { + const nsACString& locale = aLocales.GetAsUTF8String(); + raw.reset( + ffi::fluent_bundle_new_single(&locale, useIsolating, &pseudoStrategy)); + } else { + const auto& locales = aLocales.GetAsUTF8StringSequence(); + raw.reset(ffi::fluent_bundle_new(locales.Elements(), locales.Length(), + useIsolating, &pseudoStrategy)); + } + + if (!raw) { + aRv.ThrowInvalidStateError( + "Failed to create the FluentBundle. Check the " + "locales and pseudo strategy arguments."); + return nullptr; + } + + return do_AddRef(new FluentBundle(global, std::move(raw))); +} + +JSObject* FluentBundle::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return FluentBundle_Binding::Wrap(aCx, this, aGivenProto); +} + +FluentBundle::~FluentBundle() { MOZ_COUNT_DTOR(FluentBundle); }; + +void FluentBundle::GetLocales(nsTArray& aLocales) { + fluent_bundle_get_locales(mRaw.get(), &aLocales); +} + +void FluentBundle::AddResource( + FluentResource& aResource, + const dom::FluentBundleAddResourceOptions& aOptions) { + bool allowOverrides = aOptions.mAllowOverrides; + nsTArray errors; + + fluent_bundle_add_resource(mRaw.get(), aResource.Raw(), allowOverrides, + &errors); + + for (auto& err : errors) { + nsContentUtils::LogSimpleConsoleError(NS_ConvertUTF8toUTF16(err), "L10n"_ns, + false, true, + nsIScriptError::warningFlag); + } +} + +bool FluentBundle::HasMessage(const nsACString& aId) { + return fluent_bundle_has_message(mRaw.get(), &aId); +} + +void FluentBundle::GetMessage(const nsACString& aId, + Nullable& aRetVal) { + bool hasValue = false; + nsTArray attributes; + bool exists = + fluent_bundle_get_message(mRaw.get(), &aId, &hasValue, &attributes); + if (exists) { + FluentMessage& msg = aRetVal.SetValue(); + if (hasValue) { + msg.mValue = new FluentPattern(mParent, aId); + } + for (auto& name : attributes) { + auto newEntry = msg.mAttributes.Entries().AppendElement(fallible); + newEntry->mKey = name; + newEntry->mValue = new FluentPattern(mParent, aId, name); + } + } +} + +bool extendJSArrayWithErrors(JSContext* aCx, JS::Handle aErrors, + nsTArray& aInput) { + uint32_t length; + if (NS_WARN_IF(!JS::GetArrayLength(aCx, aErrors, &length))) { + return false; + } + + for (auto& err : aInput) { + JS::Rooted jsval(aCx); + if (!ToJSValue(aCx, NS_ConvertUTF8toUTF16(err), &jsval)) { + return false; + } + if (!JS_DefineElement(aCx, aErrors, length++, jsval, JSPROP_ENUMERATE)) { + return false; + } + } + return true; +} + +/* static */ +void FluentBundle::ConvertArgs(const L10nArgs& aArgs, + nsTArray& aRetVal) { + aRetVal.SetCapacity(aArgs.Entries().Length()); + for (const auto& entry : aArgs.Entries()) { + if (!entry.mValue.IsNull()) { + const auto& value = entry.mValue.Value(); + + if (value.IsUTF8String()) { + aRetVal.AppendElement(ffi::L10nArg{ + &entry.mKey, + ffi::FluentArgument::String(&value.GetAsUTF8String())}); + } else { + aRetVal.AppendElement(ffi::L10nArg{ + &entry.mKey, ffi::FluentArgument::Double_(value.GetAsDouble())}); + } + } + } +} + +void FluentBundle::FormatPattern(JSContext* aCx, const FluentPattern& aPattern, + const Nullable& aArgs, + const Optional>& aErrors, + nsACString& aRetVal, ErrorResult& aRv) { + nsTArray l10nArgs; + + if (!aArgs.IsNull()) { + const L10nArgs& args = aArgs.Value(); + ConvertArgs(args, l10nArgs); + } + + nsTArray errors; + bool succeeded = fluent_bundle_format_pattern(mRaw.get(), &aPattern.mId, + &aPattern.mAttrName, &l10nArgs, + &aRetVal, &errors); + + if (!succeeded) { + return aRv.ThrowInvalidStateError( + "Failed to format the FluentPattern. Likely the " + "pattern could not be retrieved from the bundle."); + } + + if (aErrors.WasPassed()) { + if (!extendJSArrayWithErrors(aCx, aErrors.Value(), errors)) { + aRv.ThrowUnknownError("Failed to add errors to an error array."); + } + } +} + +// FFI + +extern "C" { +ffi::RawNumberFormatter* FluentBuiltInNumberFormatterCreate( + const nsCString* aLocale, const ffi::FluentNumberOptionsRaw* aOptions) { + NumberFormatOptions options; + switch (aOptions->style) { + case ffi::FluentNumberStyleRaw::Decimal: + break; + case ffi::FluentNumberStyleRaw::Currency: { + std::string currency = aOptions->currency.get(); + switch (aOptions->currency_display) { + case ffi::FluentNumberCurrencyDisplayStyleRaw::Symbol: + options.mCurrency = Some(std::make_pair( + currency, NumberFormatOptions::CurrencyDisplay::Symbol)); + break; + case ffi::FluentNumberCurrencyDisplayStyleRaw::Code: + options.mCurrency = Some(std::make_pair( + currency, NumberFormatOptions::CurrencyDisplay::Code)); + break; + case ffi::FluentNumberCurrencyDisplayStyleRaw::Name: + options.mCurrency = Some(std::make_pair( + currency, NumberFormatOptions::CurrencyDisplay::Name)); + break; + default: + MOZ_ASSERT_UNREACHABLE(); + break; + } + } break; + case ffi::FluentNumberStyleRaw::Percent: + options.mPercent = true; + break; + default: + MOZ_ASSERT_UNREACHABLE(); + break; + } + + options.mGrouping = aOptions->use_grouping + ? NumberFormatOptions::Grouping::Auto + : NumberFormatOptions::Grouping::Never; + options.mMinIntegerDigits = Some(aOptions->minimum_integer_digits); + + if (aOptions->minimum_significant_digits >= 0 || + aOptions->maximum_significant_digits >= 0) { + options.mSignificantDigits = + Some(std::make_pair(aOptions->minimum_significant_digits, + aOptions->maximum_significant_digits)); + } else { + options.mFractionDigits = Some(std::make_pair( + aOptions->minimum_fraction_digits, aOptions->maximum_fraction_digits)); + } + + Result, ICUError> result = + NumberFormat::TryCreate(aLocale->get(), options); + + MOZ_ASSERT(result.isOk()); + + if (result.isOk()) { + return reinterpret_cast( + result.unwrap().release()); + } + + return nullptr; +} + +uint8_t* FluentBuiltInNumberFormatterFormat( + const ffi::RawNumberFormatter* aFormatter, double input, size_t* aOutCount, + size_t* aOutCapacity) { + const NumberFormat* nf = reinterpret_cast(aFormatter); + + SizeableUTF8Buffer buffer; + if (nf->format(input, buffer).isOk()) { + *aOutCount = buffer.mWritten; + *aOutCapacity = buffer.mCapacity; + return reinterpret_cast(buffer.mBuffer.release()); + } + + return nullptr; +} + +void FluentBuiltInNumberFormatterDestroy(ffi::RawNumberFormatter* aFormatter) { + delete reinterpret_cast(aFormatter); +} + +/* DateTime */ + +static Maybe GetStyle(ffi::FluentDateTimeStyle aStyle) { + switch (aStyle) { + case ffi::FluentDateTimeStyle::Full: + return Some(DateTimeFormat::Style::Full); + case ffi::FluentDateTimeStyle::Long: + return Some(DateTimeFormat::Style::Long); + case ffi::FluentDateTimeStyle::Medium: + return Some(DateTimeFormat::Style::Medium); + case ffi::FluentDateTimeStyle::Short: + return Some(DateTimeFormat::Style::Short); + case ffi::FluentDateTimeStyle::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe GetText( + ffi::FluentDateTimeTextComponent aText) { + switch (aText) { + case ffi::FluentDateTimeTextComponent::Long: + return Some(DateTimeFormat::Text::Long); + case ffi::FluentDateTimeTextComponent::Short: + return Some(DateTimeFormat::Text::Short); + case ffi::FluentDateTimeTextComponent::Narrow: + return Some(DateTimeFormat::Text::Narrow); + case ffi::FluentDateTimeTextComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe GetMonth( + ffi::FluentDateTimeMonthComponent aMonth) { + switch (aMonth) { + case ffi::FluentDateTimeMonthComponent::Numeric: + return Some(DateTimeFormat::Month::Numeric); + case ffi::FluentDateTimeMonthComponent::TwoDigit: + return Some(DateTimeFormat::Month::TwoDigit); + case ffi::FluentDateTimeMonthComponent::Long: + return Some(DateTimeFormat::Month::Long); + case ffi::FluentDateTimeMonthComponent::Short: + return Some(DateTimeFormat::Month::Short); + case ffi::FluentDateTimeMonthComponent::Narrow: + return Some(DateTimeFormat::Month::Narrow); + case ffi::FluentDateTimeMonthComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe GetNumeric( + ffi::FluentDateTimeNumericComponent aNumeric) { + switch (aNumeric) { + case ffi::FluentDateTimeNumericComponent::Numeric: + return Some(DateTimeFormat::Numeric::Numeric); + case ffi::FluentDateTimeNumericComponent::TwoDigit: + return Some(DateTimeFormat::Numeric::TwoDigit); + case ffi::FluentDateTimeNumericComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe GetTimeZoneName( + ffi::FluentDateTimeTimeZoneNameComponent aTimeZoneName) { + switch (aTimeZoneName) { + case ffi::FluentDateTimeTimeZoneNameComponent::Long: + return Some(DateTimeFormat::TimeZoneName::Long); + case ffi::FluentDateTimeTimeZoneNameComponent::Short: + return Some(DateTimeFormat::TimeZoneName::Short); + case ffi::FluentDateTimeTimeZoneNameComponent::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe GetHourCycle( + ffi::FluentDateTimeHourCycle aHourCycle) { + switch (aHourCycle) { + case ffi::FluentDateTimeHourCycle::H24: + return Some(DateTimeFormat::HourCycle::H24); + case ffi::FluentDateTimeHourCycle::H23: + return Some(DateTimeFormat::HourCycle::H23); + case ffi::FluentDateTimeHourCycle::H12: + return Some(DateTimeFormat::HourCycle::H12); + case ffi::FluentDateTimeHourCycle::H11: + return Some(DateTimeFormat::HourCycle::H11); + case ffi::FluentDateTimeHourCycle::None: + return Nothing(); + } + MOZ_ASSERT_UNREACHABLE(); + return Nothing(); +} + +static Maybe GetComponentsBag( + ffi::FluentDateTimeOptions aOptions) { + if (GetStyle(aOptions.date_style) || GetStyle(aOptions.time_style)) { + return Nothing(); + } + + DateTimeFormat::ComponentsBag components; + components.era = GetText(aOptions.era); + components.year = GetNumeric(aOptions.year); + components.month = GetMonth(aOptions.month); + components.day = GetNumeric(aOptions.day); + components.weekday = GetText(aOptions.weekday); + components.hour = GetNumeric(aOptions.hour); + components.minute = GetNumeric(aOptions.minute); + components.second = GetNumeric(aOptions.second); + components.timeZoneName = GetTimeZoneName(aOptions.time_zone_name); + components.hourCycle = GetHourCycle(aOptions.hour_cycle); + + if (!components.era && !components.year && !components.month && + !components.day && !components.weekday && !components.hour && + !components.minute && !components.second && !components.timeZoneName) { + return Nothing(); + } + + return Some(components); +} + +ffi::RawDateTimeFormatter* FluentBuiltInDateTimeFormatterCreate( + const nsCString* aLocale, ffi::FluentDateTimeOptions aOptions) { + auto genResult = DateTimePatternGenerator::TryCreate(aLocale->get()); + if (genResult.isErr()) { + MOZ_ASSERT_UNREACHABLE("There was an error in DateTimeFormat"); + return nullptr; + } + UniquePtr dateTimePatternGenerator = + genResult.unwrap(); + + if (auto components = GetComponentsBag(aOptions)) { + auto result = DateTimeFormat::TryCreateFromComponents( + Span(*aLocale), *components, dateTimePatternGenerator.get()); + if (result.isErr()) { + MOZ_ASSERT_UNREACHABLE("There was an error in DateTimeFormat"); + return nullptr; + } + + return reinterpret_cast( + result.unwrap().release()); + } + + DateTimeFormat::StyleBag style; + style.date = GetStyle(aOptions.date_style); + style.time = GetStyle(aOptions.time_style); + + auto result = DateTimeFormat::TryCreateFromStyle( + Span(*aLocale), style, dateTimePatternGenerator.get()); + + if (result.isErr()) { + MOZ_ASSERT_UNREACHABLE("There was an error in DateTimeFormat"); + return nullptr; + } + + return reinterpret_cast( + result.unwrap().release()); +} + +uint8_t* FluentBuiltInDateTimeFormatterFormat( + const ffi::RawDateTimeFormatter* aFormatter, double aUnixEpoch, + uint32_t* aOutCount) { + const auto* dtFormat = reinterpret_cast(aFormatter); + + SizeableUTF8Buffer buffer; + dtFormat->TryFormat(aUnixEpoch, buffer).unwrap(); + + *aOutCount = buffer.mWritten; + + return reinterpret_cast(buffer.mBuffer.release()); +} + +void FluentBuiltInDateTimeFormatterDestroy( + ffi::RawDateTimeFormatter* aFormatter) { + delete reinterpret_cast(aFormatter); +} +} + +} // namespace intl +} // namespace mozilla diff --git a/intl/l10n/FluentBundle.h b/intl/l10n/FluentBundle.h new file mode 100644 index 0000000000..5c28d30290 --- /dev/null +++ b/intl/l10n/FluentBundle.h @@ -0,0 +1,99 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FluentBundle_h +#define mozilla_intl_l10n_FluentBundle_h + +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" +#include "mozilla/dom/FluentBinding.h" +#include "mozilla/dom/LocalizationBinding.h" +#include "mozilla/intl/FluentResource.h" +#include "mozilla/intl/FluentBindings.h" + +class nsIGlobalObject; + +namespace mozilla { +class ErrorResult; + +namespace dom { +struct FluentMessage; +struct L10nMessage; +class OwningUTF8StringOrDouble; +class UTF8StringOrUTF8StringSequence; +struct FluentBundleOptions; +struct FluentBundleAddResourceOptions; +} // namespace dom + +namespace intl { + +class FluentResource; + +using L10nArgs = + dom::Record>; + +class FluentPattern : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentPattern) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentPattern) + + FluentPattern(nsISupports* aParent, const nsACString& aId); + FluentPattern(nsISupports* aParent, const nsACString& aId, + const nsACString& aAttrName); + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + nsISupports* GetParentObject() const { return mParent; } + + nsCString mId; + nsCString mAttrName; + + protected: + virtual ~FluentPattern(); + + nsCOMPtr mParent; +}; + +class FluentBundle final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentBundle) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentBundle) + + FluentBundle(nsISupports* aParent, UniquePtr aRaw); + + static already_AddRefed Constructor( + const dom::GlobalObject& aGlobal, + const dom::UTF8StringOrUTF8StringSequence& aLocales, + const dom::FluentBundleOptions& aOptions, ErrorResult& aRv); + JSObject* WrapObject(JSContext* aCx, JS::Handle aGivenProto) final; + nsISupports* GetParentObject() const { return mParent; } + + void GetLocales(nsTArray& aLocales); + + void AddResource(FluentResource& aResource, + const dom::FluentBundleAddResourceOptions& aOptions); + bool HasMessage(const nsACString& aId); + void GetMessage(const nsACString& aId, + dom::Nullable& aRetVal); + void FormatPattern(JSContext* aCx, const FluentPattern& aPattern, + const dom::Nullable& aArgs, + const dom::Optional>& aErrors, + nsACString& aRetVal, ErrorResult& aRv); + + static void ConvertArgs(const L10nArgs& aArgs, + nsTArray& aRetVal); + + protected: + virtual ~FluentBundle(); + + nsCOMPtr mParent; + UniquePtr mRaw; +}; + +} // namespace intl +} // namespace mozilla + +#endif diff --git a/intl/l10n/FluentResource.cpp b/intl/l10n/FluentResource.cpp new file mode 100644 index 0000000000..bef7d7adf4 --- /dev/null +++ b/intl/l10n/FluentResource.cpp @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsContentUtils.h" +#include "FluentResource.h" + +using namespace mozilla::dom; + +namespace mozilla { +namespace intl { + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentResource, mParent) + +FluentResource::FluentResource(nsISupports* aParent, + const ffi::FluentResource* aRaw) + : mParent(aParent), mRaw(std::move(aRaw)), mHasErrors(false) {} + +FluentResource::FluentResource(nsISupports* aParent, const nsACString& aSource) + : mParent(aParent), mHasErrors(false) { + mRaw = dont_AddRef(ffi::fluent_resource_new(&aSource, &mHasErrors)); +} + +already_AddRefed FluentResource::Constructor( + const GlobalObject& aGlobal, const nsACString& aSource) { + RefPtr res = + new FluentResource(aGlobal.GetAsSupports(), aSource); + + if (res->mHasErrors) { + nsContentUtils::LogSimpleConsoleError( + u"Errors encountered while parsing Fluent Resource."_ns, "chrome"_ns, + false, true /* from chrome context*/); + } + return res.forget(); +} + +JSObject* FluentResource::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return FluentResource_Binding::Wrap(aCx, this, aGivenProto); +} + +} // namespace intl +} // namespace mozilla diff --git a/intl/l10n/FluentResource.h b/intl/l10n/FluentResource.h new file mode 100644 index 0000000000..56e7f8198b --- /dev/null +++ b/intl/l10n/FluentResource.h @@ -0,0 +1,47 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_FluentResource_h +#define mozilla_intl_l10n_FluentResource_h + +#include "mozilla/dom/BindingDeclarations.h" +#include "nsCycleCollectionParticipant.h" +#include "nsWrapperCache.h" +#include "mozilla/dom/FluentBinding.h" +#include "mozilla/intl/FluentBindings.h" + +namespace mozilla { +namespace intl { + +class FluentResource : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentResource) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentResource) + + FluentResource(nsISupports* aParent, const ffi::FluentResource* aRaw); + FluentResource(nsISupports* aParent, const nsACString& aSource); + + static already_AddRefed Constructor( + const dom::GlobalObject& aGlobal, const nsACString& aSource); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + nsISupports* GetParentObject() const { return mParent; } + + const ffi::FluentResource* Raw() const { return mRaw; } + + protected: + virtual ~FluentResource() = default; + + nsCOMPtr mParent; + RefPtr mRaw; + bool mHasErrors; +}; + +} // namespace intl +} // namespace mozilla + +#endif diff --git a/intl/l10n/FluentSyntax.jsm b/intl/l10n/FluentSyntax.jsm new file mode 100644 index 0000000000..ec60dd5283 --- /dev/null +++ b/intl/l10n/FluentSyntax.jsm @@ -0,0 +1,1957 @@ +/* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */ + +/* Copyright 2019 Mozilla Foundation and others + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +/* fluent-syntax@0.12.0 */ + +/* + * Base class for all Fluent AST nodes. + * + * All productions described in the ASDL subclass BaseNode, including Span and + * Annotation. + * + */ +class BaseNode { + constructor() {} + + equals(other, ignoredFields = ["span"]) { + const thisKeys = new Set(Object.keys(this)); + const otherKeys = new Set(Object.keys(other)); + if (ignoredFields) { + for (const fieldName of ignoredFields) { + thisKeys.delete(fieldName); + otherKeys.delete(fieldName); + } + } + if (thisKeys.size !== otherKeys.size) { + return false; + } + for (const fieldName of thisKeys) { + if (!otherKeys.has(fieldName)) { + return false; + } + const thisVal = this[fieldName]; + const otherVal = other[fieldName]; + if (typeof thisVal !== typeof otherVal) { + return false; + } + if (thisVal instanceof Array) { + if (thisVal.length !== otherVal.length) { + return false; + } + for (let i = 0; i < thisVal.length; ++i) { + if (!scalarsEqual(thisVal[i], otherVal[i], ignoredFields)) { + return false; + } + } + } else if (!scalarsEqual(thisVal, otherVal, ignoredFields)) { + return false; + } + } + return true; + } + + clone() { + function visit(value) { + if (value instanceof BaseNode) { + return value.clone(); + } + if (Array.isArray(value)) { + return value.map(visit); + } + return value; + } + const clone = Object.create(this.constructor.prototype); + for (const prop of Object.keys(this)) { + clone[prop] = visit(this[prop]); + } + return clone; + } +} + +function scalarsEqual(thisVal, otherVal, ignoredFields) { + if (thisVal instanceof BaseNode) { + return thisVal.equals(otherVal, ignoredFields); + } + return thisVal === otherVal; +} + +/* + * Base class for AST nodes which can have Spans. + */ +class SyntaxNode extends BaseNode { + addSpan(start, end) { + this.span = new Span(start, end); + } +} + +class Resource extends SyntaxNode { + constructor(body = []) { + super(); + this.type = "Resource"; + this.body = body; + } +} + +/* + * An abstract base class for useful elements of Resource.body. + */ +class Entry extends SyntaxNode {} + +class Message extends Entry { + constructor(id, value = null, attributes = [], comment = null) { + super(); + this.type = "Message"; + this.id = id; + this.value = value; + this.attributes = attributes; + this.comment = comment; + } +} + +class Term extends Entry { + constructor(id, value, attributes = [], comment = null) { + super(); + this.type = "Term"; + this.id = id; + this.value = value; + this.attributes = attributes; + this.comment = comment; + } +} + +class Pattern extends SyntaxNode { + constructor(elements) { + super(); + this.type = "Pattern"; + this.elements = elements; + } +} + +/* + * An abstract base class for elements of Patterns. + */ +class PatternElement extends SyntaxNode {} + +class TextElement extends PatternElement { + constructor(value) { + super(); + this.type = "TextElement"; + this.value = value; + } +} + +class Placeable extends PatternElement { + constructor(expression) { + super(); + this.type = "Placeable"; + this.expression = expression; + } +} + +/* + * An abstract base class for expressions. + */ +class Expression extends SyntaxNode {} + +// An abstract base class for Literals. +class Literal extends Expression { + constructor(value) { + super(); + // The "value" field contains the exact contents of the literal, + // character-for-character. + this.value = value; + } + + parse() { + return {value: this.value}; + } +} + +class StringLiteral extends Literal { + constructor(value) { + super(value); + this.type = "StringLiteral"; + } + + parse() { + // Backslash backslash, backslash double quote, uHHHH, UHHHHHH. + const KNOWN_ESCAPES = + /(?:\\\\|\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g; + + function from_escape_sequence(match, codepoint4, codepoint6) { + switch (match) { + case "\\\\": + return "\\"; + case "\\\"": + return "\""; + default: + let codepoint = parseInt(codepoint4 || codepoint6, 16); + if (codepoint <= 0xD7FF || 0xE000 <= codepoint) { + // It's a Unicode scalar value. + return String.fromCodePoint(codepoint); + } + // Escape sequences reresenting surrogate code points are + // well-formed but invalid in Fluent. Replace them with U+FFFD + // REPLACEMENT CHARACTER. + return "�"; + } + } + + let value = this.value.replace(KNOWN_ESCAPES, from_escape_sequence); + return {value}; + } +} + +class NumberLiteral extends Literal { + constructor(value) { + super(value); + this.type = "NumberLiteral"; + } + + parse() { + let value = parseFloat(this.value); + let decimal_position = this.value.indexOf("."); + let precision = decimal_position > 0 + ? this.value.length - decimal_position - 1 + : 0; + return {value, precision}; + } +} + +class MessageReference extends Expression { + constructor(id, attribute = null) { + super(); + this.type = "MessageReference"; + this.id = id; + this.attribute = attribute; + } +} + +class TermReference extends Expression { + constructor(id, attribute = null, args = null) { + super(); + this.type = "TermReference"; + this.id = id; + this.attribute = attribute; + this.arguments = args; + } +} + +class VariableReference extends Expression { + constructor(id) { + super(); + this.type = "VariableReference"; + this.id = id; + } +} + +class FunctionReference extends Expression { + constructor(id, args) { + super(); + this.type = "FunctionReference"; + this.id = id; + this.arguments = args; + } +} + +class SelectExpression extends Expression { + constructor(selector, variants) { + super(); + this.type = "SelectExpression"; + this.selector = selector; + this.variants = variants; + } +} + +class CallArguments extends SyntaxNode { + constructor(positional = [], named = []) { + super(); + this.type = "CallArguments"; + this.positional = positional; + this.named = named; + } +} + +class Attribute extends SyntaxNode { + constructor(id, value) { + super(); + this.type = "Attribute"; + this.id = id; + this.value = value; + } +} + +class Variant extends SyntaxNode { + constructor(key, value, def = false) { + super(); + this.type = "Variant"; + this.key = key; + this.value = value; + this.default = def; + } +} + +class NamedArgument extends SyntaxNode { + constructor(name, value) { + super(); + this.type = "NamedArgument"; + this.name = name; + this.value = value; + } +} + +class Identifier extends SyntaxNode { + constructor(name) { + super(); + this.type = "Identifier"; + this.name = name; + } +} + +class BaseComment extends Entry { + constructor(content) { + super(); + this.type = "BaseComment"; + this.content = content; + } +} + +class Comment extends BaseComment { + constructor(content) { + super(content); + this.type = "Comment"; + } +} + +class GroupComment extends BaseComment { + constructor(content) { + super(content); + this.type = "GroupComment"; + } +} +class ResourceComment extends BaseComment { + constructor(content) { + super(content); + this.type = "ResourceComment"; + } +} + +class Junk extends SyntaxNode { + constructor(content) { + super(); + this.type = "Junk"; + this.annotations = []; + this.content = content; + } + + addAnnotation(annot) { + this.annotations.push(annot); + } +} + +class Span extends BaseNode { + constructor(start, end) { + super(); + this.type = "Span"; + this.start = start; + this.end = end; + } +} + +class Annotation extends SyntaxNode { + constructor(code, args = [], message) { + super(); + this.type = "Annotation"; + this.code = code; + this.arguments = args; + this.message = message; + } +} + +const ast = ({ + BaseNode: BaseNode, + Resource: Resource, + Entry: Entry, + Message: Message, + Term: Term, + Pattern: Pattern, + PatternElement: PatternElement, + TextElement: TextElement, + Placeable: Placeable, + Expression: Expression, + Literal: Literal, + StringLiteral: StringLiteral, + NumberLiteral: NumberLiteral, + MessageReference: MessageReference, + TermReference: TermReference, + VariableReference: VariableReference, + FunctionReference: FunctionReference, + SelectExpression: SelectExpression, + CallArguments: CallArguments, + Attribute: Attribute, + Variant: Variant, + NamedArgument: NamedArgument, + Identifier: Identifier, + BaseComment: BaseComment, + Comment: Comment, + GroupComment: GroupComment, + ResourceComment: ResourceComment, + Junk: Junk, + Span: Span, + Annotation: Annotation +}); + +class ParseError extends Error { + constructor(code, ...args) { + super(); + this.code = code; + this.args = args; + this.message = getErrorMessage(code, args); + } +} + +/* eslint-disable complexity */ +function getErrorMessage(code, args) { + switch (code) { + case "E0001": + return "Generic error"; + case "E0002": + return "Expected an entry start"; + case "E0003": { + const [token] = args; + return `Expected token: "${token}"`; + } + case "E0004": { + const [range] = args; + return `Expected a character from range: "${range}"`; + } + case "E0005": { + const [id] = args; + return `Expected message "${id}" to have a value or attributes`; + } + case "E0006": { + const [id] = args; + return `Expected term "-${id}" to have a value`; + } + case "E0007": + return "Keyword cannot end with a whitespace"; + case "E0008": + return "The callee has to be an upper-case identifier or a term"; + case "E0009": + return "The argument name has to be a simple identifier"; + case "E0010": + return "Expected one of the variants to be marked as default (*)"; + case "E0011": + return 'Expected at least one variant after "->"'; + case "E0012": + return "Expected value"; + case "E0013": + return "Expected variant key"; + case "E0014": + return "Expected literal"; + case "E0015": + return "Only one variant can be marked as default (*)"; + case "E0016": + return "Message references cannot be used as selectors"; + case "E0017": + return "Terms cannot be used as selectors"; + case "E0018": + return "Attributes of messages cannot be used as selectors"; + case "E0019": + return "Attributes of terms cannot be used as placeables"; + case "E0020": + return "Unterminated string expression"; + case "E0021": + return "Positional arguments must not follow named arguments"; + case "E0022": + return "Named arguments must be unique"; + case "E0024": + return "Cannot access variants of a message."; + case "E0025": { + const [char] = args; + return `Unknown escape sequence: \\${char}.`; + } + case "E0026": { + const [sequence] = args; + return `Invalid Unicode escape sequence: ${sequence}.`; + } + case "E0027": + return "Unbalanced closing brace in TextElement."; + case "E0028": + return "Expected an inline expression"; + default: + return code; + } +} + +function includes(arr, elem) { + return arr.indexOf(elem) > -1; +} + +/* eslint no-magic-numbers: "off" */ + +class ParserStream { + constructor(string) { + this.string = string; + this.index = 0; + this.peekOffset = 0; + } + + charAt(offset) { + // When the cursor is at CRLF, return LF but don't move the cursor. + // The cursor still points to the EOL position, which in this case is the + // beginning of the compound CRLF sequence. This ensures slices of + // [inclusive, exclusive) continue to work properly. + if (this.string[offset] === "\r" + && this.string[offset + 1] === "\n") { + return "\n"; + } + + return this.string[offset]; + } + + get currentChar() { + return this.charAt(this.index); + } + + get currentPeek() { + return this.charAt(this.index + this.peekOffset); + } + + next() { + this.peekOffset = 0; + // Skip over the CRLF as if it was a single character. + if (this.string[this.index] === "\r" + && this.string[this.index + 1] === "\n") { + this.index++; + } + this.index++; + return this.string[this.index]; + } + + peek() { + // Skip over the CRLF as if it was a single character. + if (this.string[this.index + this.peekOffset] === "\r" + && this.string[this.index + this.peekOffset + 1] === "\n") { + this.peekOffset++; + } + this.peekOffset++; + return this.string[this.index + this.peekOffset]; + } + + resetPeek(offset = 0) { + this.peekOffset = offset; + } + + skipToPeek() { + this.index += this.peekOffset; + this.peekOffset = 0; + } +} + +const EOL = "\n"; +const EOF = undefined; +const SPECIAL_LINE_START_CHARS = ["}", ".", "[", "*"]; + +class FluentParserStream extends ParserStream { + peekBlankInline() { + const start = this.index + this.peekOffset; + while (this.currentPeek === " ") { + this.peek(); + } + return this.string.slice(start, this.index + this.peekOffset); + } + + skipBlankInline() { + const blank = this.peekBlankInline(); + this.skipToPeek(); + return blank; + } + + peekBlankBlock() { + let blank = ""; + while (true) { + const lineStart = this.peekOffset; + this.peekBlankInline(); + if (this.currentPeek === EOL) { + blank += EOL; + this.peek(); + continue; + } + if (this.currentPeek === EOF) { + // Treat the blank line at EOF as a blank block. + return blank; + } + // Any other char; reset to column 1 on this line. + this.resetPeek(lineStart); + return blank; + } + } + + skipBlankBlock() { + const blank = this.peekBlankBlock(); + this.skipToPeek(); + return blank; + } + + peekBlank() { + while (this.currentPeek === " " || this.currentPeek === EOL) { + this.peek(); + } + } + + skipBlank() { + this.peekBlank(); + this.skipToPeek(); + } + + expectChar(ch) { + if (this.currentChar === ch) { + this.next(); + return true; + } + + throw new ParseError("E0003", ch); + } + + expectLineEnd() { + if (this.currentChar === EOF) { + // EOF is a valid line end in Fluent. + return true; + } + + if (this.currentChar === EOL) { + this.next(); + return true; + } + + // Unicode Character 'SYMBOL FOR NEWLINE' (U+2424) + throw new ParseError("E0003", "\u2424"); + } + + takeChar(f) { + const ch = this.currentChar; + if (ch === EOF) { + return EOF; + } + if (f(ch)) { + this.next(); + return ch; + } + return null; + } + + isCharIdStart(ch) { + if (ch === EOF) { + return false; + } + + const cc = ch.charCodeAt(0); + return (cc >= 97 && cc <= 122) || // a-z + (cc >= 65 && cc <= 90); // A-Z + } + + isIdentifierStart() { + return this.isCharIdStart(this.currentPeek); + } + + isNumberStart() { + const ch = this.currentChar === "-" + ? this.peek() + : this.currentChar; + + if (ch === EOF) { + this.resetPeek(); + return false; + } + + const cc = ch.charCodeAt(0); + const isDigit = cc >= 48 && cc <= 57; // 0-9 + this.resetPeek(); + return isDigit; + } + + isCharPatternContinuation(ch) { + if (ch === EOF) { + return false; + } + + return !includes(SPECIAL_LINE_START_CHARS, ch); + } + + isValueStart() { + // Inline Patterns may start with any char. + const ch = this.currentPeek; + return ch !== EOL && ch !== EOF; + } + + isValueContinuation() { + const column1 = this.peekOffset; + this.peekBlankInline(); + + if (this.currentPeek === "{") { + this.resetPeek(column1); + return true; + } + + if (this.peekOffset - column1 === 0) { + return false; + } + + if (this.isCharPatternContinuation(this.currentPeek)) { + this.resetPeek(column1); + return true; + } + + return false; + } + + // -1 - any + // 0 - comment + // 1 - group comment + // 2 - resource comment + isNextLineComment(level = -1) { + if (this.currentChar !== EOL) { + return false; + } + + let i = 0; + + while (i <= level || (level === -1 && i < 3)) { + if (this.peek() !== "#") { + if (i <= level && level !== -1) { + this.resetPeek(); + return false; + } + break; + } + i++; + } + + // The first char after #, ## or ###. + const ch = this.peek(); + if (ch === " " || ch === EOL) { + this.resetPeek(); + return true; + } + + this.resetPeek(); + return false; + } + + isVariantStart() { + const currentPeekOffset = this.peekOffset; + if (this.currentPeek === "*") { + this.peek(); + } + if (this.currentPeek === "[") { + this.resetPeek(currentPeekOffset); + return true; + } + this.resetPeek(currentPeekOffset); + return false; + } + + isAttributeStart() { + return this.currentPeek === "."; + } + + skipToNextEntryStart(junkStart) { + let lastNewline = this.string.lastIndexOf(EOL, this.index); + if (junkStart < lastNewline) { + // Last seen newline is _after_ the junk start. It's safe to rewind + // without the risk of resuming at the same broken entry. + this.index = lastNewline; + } + while (this.currentChar) { + // We're only interested in beginnings of line. + if (this.currentChar !== EOL) { + this.next(); + continue; + } + + // Break if the first char in this line looks like an entry start. + const first = this.next(); + if (this.isCharIdStart(first) || first === "-" || first === "#") { + break; + } + } + } + + takeIDStart() { + if (this.isCharIdStart(this.currentChar)) { + const ret = this.currentChar; + this.next(); + return ret; + } + + throw new ParseError("E0004", "a-zA-Z"); + } + + takeIDChar() { + const closure = ch => { + const cc = ch.charCodeAt(0); + return ((cc >= 97 && cc <= 122) || // a-z + (cc >= 65 && cc <= 90) || // A-Z + (cc >= 48 && cc <= 57) || // 0-9 + cc === 95 || cc === 45); // _- + }; + + return this.takeChar(closure); + } + + takeDigit() { + const closure = ch => { + const cc = ch.charCodeAt(0); + return (cc >= 48 && cc <= 57); // 0-9 + }; + + return this.takeChar(closure); + } + + takeHexDigit() { + const closure = ch => { + const cc = ch.charCodeAt(0); + return (cc >= 48 && cc <= 57) // 0-9 + || (cc >= 65 && cc <= 70) // A-F + || (cc >= 97 && cc <= 102); // a-f + }; + + return this.takeChar(closure); + } +} + +/* eslint no-magic-numbers: [0] */ + + +const trailingWSRe = /[ \t\n\r]+$/; + + +function withSpan(fn) { + return function(ps, ...args) { + if (!this.withSpans) { + return fn.call(this, ps, ...args); + } + + const start = ps.index; + const node = fn.call(this, ps, ...args); + + // Don't re-add the span if the node already has it. This may happen when + // one decorated function calls another decorated function. + if (node.span) { + return node; + } + + const end = ps.index; + node.addSpan(start, end); + return node; + }; +} + + +class FluentParser { + constructor({ + withSpans = true, + } = {}) { + this.withSpans = withSpans; + + // Poor man's decorators. + const methodNames = [ + "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier", + "getVariant", "getNumber", "getPattern", "getTextElement", + "getPlaceable", "getExpression", "getInlineExpression", + "getCallArgument", "getCallArguments", "getString", "getLiteral", + ]; + for (const name of methodNames) { + this[name] = withSpan(this[name]); + } + } + + parse(source) { + const ps = new FluentParserStream(source); + ps.skipBlankBlock(); + + const entries = []; + let lastComment = null; + + while (ps.currentChar) { + const entry = this.getEntryOrJunk(ps); + const blankLines = ps.skipBlankBlock(); + + // Regular Comments require special logic. Comments may be attached to + // Messages or Terms if they are followed immediately by them. However + // they should parse as standalone when they're followed by Junk. + // Consequently, we only attach Comments once we know that the Message + // or the Term parsed successfully. + if (entry.type === "Comment" + && blankLines.length === 0 + && ps.currentChar) { + // Stash the comment and decide what to do with it in the next pass. + lastComment = entry; + continue; + } + + if (lastComment) { + if (entry.type === "Message" || entry.type === "Term") { + entry.comment = lastComment; + if (this.withSpans) { + entry.span.start = entry.comment.span.start; + } + } else { + entries.push(lastComment); + } + // In either case, the stashed comment has been dealt with; clear it. + lastComment = null; + } + + // No special logic for other types of entries. + entries.push(entry); + } + + const res = new Resource(entries); + + if (this.withSpans) { + res.addSpan(0, ps.index); + } + + return res; + } + + /* + * Parse the first Message or Term in `source`. + * + * Skip all encountered comments and start parsing at the first Message or + * Term start. Return Junk if the parsing is not successful. + * + * Preceding comments are ignored unless they contain syntax errors + * themselves, in which case Junk for the invalid comment is returned. + */ + parseEntry(source) { + const ps = new FluentParserStream(source); + ps.skipBlankBlock(); + + while (ps.currentChar === "#") { + const skipped = this.getEntryOrJunk(ps); + if (skipped.type === "Junk") { + // Don't skip Junk comments. + return skipped; + } + ps.skipBlankBlock(); + } + + return this.getEntryOrJunk(ps); + } + + getEntryOrJunk(ps) { + const entryStartPos = ps.index; + + try { + const entry = this.getEntry(ps); + ps.expectLineEnd(); + return entry; + } catch (err) { + if (!(err instanceof ParseError)) { + throw err; + } + + let errorIndex = ps.index; + ps.skipToNextEntryStart(entryStartPos); + const nextEntryStart = ps.index; + if (nextEntryStart < errorIndex) { + // The position of the error must be inside of the Junk's span. + errorIndex = nextEntryStart; + } + + // Create a Junk instance + const slice = ps.string.substring(entryStartPos, nextEntryStart); + const junk = new Junk(slice); + if (this.withSpans) { + junk.addSpan(entryStartPos, nextEntryStart); + } + const annot = new Annotation(err.code, err.args, err.message); + annot.addSpan(errorIndex, errorIndex); + junk.addAnnotation(annot); + return junk; + } + } + + getEntry(ps) { + if (ps.currentChar === "#") { + return this.getComment(ps); + } + + if (ps.currentChar === "-") { + return this.getTerm(ps); + } + + if (ps.isIdentifierStart()) { + return this.getMessage(ps); + } + + throw new ParseError("E0002"); + } + + getComment(ps) { + // 0 - comment + // 1 - group comment + // 2 - resource comment + let level = -1; + let content = ""; + + while (true) { + let i = -1; + while (ps.currentChar === "#" && (i < (level === -1 ? 2 : level))) { + ps.next(); + i++; + } + + if (level === -1) { + level = i; + } + + if (ps.currentChar !== EOL) { + ps.expectChar(" "); + let ch; + while ((ch = ps.takeChar(x => x !== EOL))) { + content += ch; + } + } + + if (ps.isNextLineComment(level)) { + content += ps.currentChar; + ps.next(); + } else { + break; + } + } + + let Comment$$1; + switch (level) { + case 0: + Comment$$1 = Comment; + break; + case 1: + Comment$$1 = GroupComment; + break; + case 2: + Comment$$1 = ResourceComment; + break; + } + return new Comment$$1(content); + } + + getMessage(ps) { + const id = this.getIdentifier(ps); + + ps.skipBlankInline(); + ps.expectChar("="); + + const value = this.maybeGetPattern(ps); + const attrs = this.getAttributes(ps); + + if (value === null && attrs.length === 0) { + throw new ParseError("E0005", id.name); + } + + return new Message(id, value, attrs); + } + + getTerm(ps) { + ps.expectChar("-"); + const id = this.getIdentifier(ps); + + ps.skipBlankInline(); + ps.expectChar("="); + + const value = this.maybeGetPattern(ps); + if (value === null) { + throw new ParseError("E0006", id.name); + } + + const attrs = this.getAttributes(ps); + return new Term(id, value, attrs); + } + + getAttribute(ps) { + ps.expectChar("."); + + const key = this.getIdentifier(ps); + + ps.skipBlankInline(); + ps.expectChar("="); + + const value = this.maybeGetPattern(ps); + if (value === null) { + throw new ParseError("E0012"); + } + + return new Attribute(key, value); + } + + getAttributes(ps) { + const attrs = []; + ps.peekBlank(); + while (ps.isAttributeStart()) { + ps.skipToPeek(); + const attr = this.getAttribute(ps); + attrs.push(attr); + ps.peekBlank(); + } + return attrs; + } + + getIdentifier(ps) { + let name = ps.takeIDStart(); + + let ch; + while ((ch = ps.takeIDChar())) { + name += ch; + } + + return new Identifier(name); + } + + getVariantKey(ps) { + const ch = ps.currentChar; + + if (ch === EOF) { + throw new ParseError("E0013"); + } + + const cc = ch.charCodeAt(0); + + if ((cc >= 48 && cc <= 57) || cc === 45) { // 0-9, - + return this.getNumber(ps); + } + + return this.getIdentifier(ps); + } + + getVariant(ps, {hasDefault}) { + let defaultIndex = false; + + if (ps.currentChar === "*") { + if (hasDefault) { + throw new ParseError("E0015"); + } + ps.next(); + defaultIndex = true; + } + + ps.expectChar("["); + + ps.skipBlank(); + + const key = this.getVariantKey(ps); + + ps.skipBlank(); + ps.expectChar("]"); + + const value = this.maybeGetPattern(ps); + if (value === null) { + throw new ParseError("E0012"); + } + + return new Variant(key, value, defaultIndex); + } + + getVariants(ps) { + const variants = []; + let hasDefault = false; + + ps.skipBlank(); + while (ps.isVariantStart()) { + const variant = this.getVariant(ps, {hasDefault}); + + if (variant.default) { + hasDefault = true; + } + + variants.push(variant); + ps.expectLineEnd(); + ps.skipBlank(); + } + + if (variants.length === 0) { + throw new ParseError("E0011"); + } + + if (!hasDefault) { + throw new ParseError("E0010"); + } + + return variants; + } + + getDigits(ps) { + let num = ""; + + let ch; + while ((ch = ps.takeDigit())) { + num += ch; + } + + if (num.length === 0) { + throw new ParseError("E0004", "0-9"); + } + + return num; + } + + getNumber(ps) { + let value = ""; + + if (ps.currentChar === "-") { + ps.next(); + value += `-${this.getDigits(ps)}`; + } else { + value += this.getDigits(ps); + } + + if (ps.currentChar === ".") { + ps.next(); + value += `.${this.getDigits(ps)}`; + } + + return new NumberLiteral(value); + } + + // maybeGetPattern distinguishes between patterns which start on the same line + // as the identifier (a.k.a. inline signleline patterns and inline multiline + // patterns) and patterns which start on a new line (a.k.a. block multiline + // patterns). The distinction is important for the dedentation logic: the + // indent of the first line of a block pattern must be taken into account when + // calculating the maximum common indent. + maybeGetPattern(ps) { + ps.peekBlankInline(); + if (ps.isValueStart()) { + ps.skipToPeek(); + return this.getPattern(ps, {isBlock: false}); + } + + ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + return this.getPattern(ps, {isBlock: true}); + } + + return null; + } + + getPattern(ps, {isBlock}) { + const elements = []; + if (isBlock) { + // A block pattern is a pattern which starts on a new line. Store and + // measure the indent of this first line for the dedentation logic. + const blankStart = ps.index; + const firstIndent = ps.skipBlankInline(); + elements.push(this.getIndent(ps, firstIndent, blankStart)); + var commonIndentLength = firstIndent.length; + } else { + commonIndentLength = Infinity; + } + + let ch; + elements: while ((ch = ps.currentChar)) { + switch (ch) { + case EOL: { + const blankStart = ps.index; + const blankLines = ps.peekBlankBlock(); + if (ps.isValueContinuation()) { + ps.skipToPeek(); + const indent = ps.skipBlankInline(); + commonIndentLength = Math.min(commonIndentLength, indent.length); + elements.push(this.getIndent(ps, blankLines + indent, blankStart)); + continue elements; + } + + // The end condition for getPattern's while loop is a newline + // which is not followed by a valid pattern continuation. + ps.resetPeek(); + break elements; + } + case "{": + elements.push(this.getPlaceable(ps)); + continue elements; + case "}": + throw new ParseError("E0027"); + default: + const element = this.getTextElement(ps); + elements.push(element); + } + } + + const dedented = this.dedent(elements, commonIndentLength); + return new Pattern(dedented); + } + + // Create a token representing an indent. It's not part of the AST and it will + // be trimmed and merged into adjacent TextElements, or turned into a new + // TextElement, if it's surrounded by two Placeables. + getIndent(ps, value, start) { + return { + type: "Indent", + span: {start, end: ps.index}, + value, + }; + } + + // Dedent a list of elements by removing the maximum common indent from the + // beginning of text lines. The common indent is calculated in getPattern. + dedent(elements, commonIndent) { + const trimmed = []; + + for (let element of elements) { + if (element.type === "Placeable") { + trimmed.push(element); + continue; + } + + if (element.type === "Indent") { + // Strip common indent. + element.value = element.value.slice( + 0, element.value.length - commonIndent); + if (element.value.length === 0) { + continue; + } + } + + let prev = trimmed[trimmed.length - 1]; + if (prev && prev.type === "TextElement") { + // Join adjacent TextElements by replacing them with their sum. + const sum = new TextElement(prev.value + element.value); + if (this.withSpans) { + sum.addSpan(prev.span.start, element.span.end); + } + trimmed[trimmed.length - 1] = sum; + continue; + } + + if (element.type === "Indent") { + // If the indent hasn't been merged into a preceding TextElement, + // convert it into a new TextElement. + const textElement = new TextElement(element.value); + if (this.withSpans) { + textElement.addSpan(element.span.start, element.span.end); + } + element = textElement; + } + + trimmed.push(element); + } + + // Trim trailing whitespace from the Pattern. + const lastElement = trimmed[trimmed.length - 1]; + if (lastElement.type === "TextElement") { + lastElement.value = lastElement.value.replace(trailingWSRe, ""); + if (lastElement.value.length === 0) { + trimmed.pop(); + } + } + + return trimmed; + } + + getTextElement(ps) { + let buffer = ""; + + let ch; + while ((ch = ps.currentChar)) { + if (ch === "{" || ch === "}") { + return new TextElement(buffer); + } + + if (ch === EOL) { + return new TextElement(buffer); + } + + buffer += ch; + ps.next(); + } + + return new TextElement(buffer); + } + + getEscapeSequence(ps) { + const next = ps.currentChar; + + switch (next) { + case "\\": + case "\"": + ps.next(); + return `\\${next}`; + case "u": + return this.getUnicodeEscapeSequence(ps, next, 4); + case "U": + return this.getUnicodeEscapeSequence(ps, next, 6); + default: + throw new ParseError("E0025", next); + } + } + + getUnicodeEscapeSequence(ps, u, digits) { + ps.expectChar(u); + + let sequence = ""; + for (let i = 0; i < digits; i++) { + const ch = ps.takeHexDigit(); + + if (!ch) { + throw new ParseError( + "E0026", `\\${u}${sequence}${ps.currentChar}`); + } + + sequence += ch; + } + + return `\\${u}${sequence}`; + } + + getPlaceable(ps) { + ps.expectChar("{"); + ps.skipBlank(); + const expression = this.getExpression(ps); + ps.expectChar("}"); + return new Placeable(expression); + } + + getExpression(ps) { + const selector = this.getInlineExpression(ps); + ps.skipBlank(); + + if (ps.currentChar === "-") { + if (ps.peek() !== ">") { + ps.resetPeek(); + return selector; + } + + if (selector.type === "MessageReference") { + if (selector.attribute === null) { + throw new ParseError("E0016"); + } else { + throw new ParseError("E0018"); + } + } + + if (selector.type === "TermReference" && selector.attribute === null) { + throw new ParseError("E0017"); + } + + ps.next(); + ps.next(); + + ps.skipBlankInline(); + ps.expectLineEnd(); + + const variants = this.getVariants(ps); + return new SelectExpression(selector, variants); + } + + if (selector.type === "TermReference" && selector.attribute !== null) { + throw new ParseError("E0019"); + } + + return selector; + } + + getInlineExpression(ps) { + if (ps.currentChar === "{") { + return this.getPlaceable(ps); + } + + if (ps.isNumberStart()) { + return this.getNumber(ps); + } + + if (ps.currentChar === '"') { + return this.getString(ps); + } + + if (ps.currentChar === "$") { + ps.next(); + const id = this.getIdentifier(ps); + return new VariableReference(id); + } + + if (ps.currentChar === "-") { + ps.next(); + const id = this.getIdentifier(ps); + + let attr; + if (ps.currentChar === ".") { + ps.next(); + attr = this.getIdentifier(ps); + } + + let args; + if (ps.currentChar === "(") { + args = this.getCallArguments(ps); + } + + return new TermReference(id, attr, args); + } + + if (ps.isIdentifierStart()) { + const id = this.getIdentifier(ps); + + if (ps.currentChar === "(") { + // It's a Function. Ensure it's all upper-case. + if (!/^[A-Z][A-Z0-9_-]*$/.test(id.name)) { + throw new ParseError("E0008"); + } + + let args = this.getCallArguments(ps); + return new FunctionReference(id, args); + } + + let attr; + if (ps.currentChar === ".") { + ps.next(); + attr = this.getIdentifier(ps); + } + + return new MessageReference(id, attr); + } + + + throw new ParseError("E0028"); + } + + getCallArgument(ps) { + const exp = this.getInlineExpression(ps); + + ps.skipBlank(); + + if (ps.currentChar !== ":") { + return exp; + } + + if (exp.type === "MessageReference" && exp.attribute === null) { + ps.next(); + ps.skipBlank(); + + const value = this.getLiteral(ps); + return new NamedArgument(exp.id, value); + } + + throw new ParseError("E0009"); + } + + getCallArguments(ps) { + const positional = []; + const named = []; + const argumentNames = new Set(); + + ps.expectChar("("); + ps.skipBlank(); + + while (true) { + if (ps.currentChar === ")") { + break; + } + + const arg = this.getCallArgument(ps); + if (arg.type === "NamedArgument") { + if (argumentNames.has(arg.name.name)) { + throw new ParseError("E0022"); + } + named.push(arg); + argumentNames.add(arg.name.name); + } else if (argumentNames.size > 0) { + throw new ParseError("E0021"); + } else { + positional.push(arg); + } + + ps.skipBlank(); + + if (ps.currentChar === ",") { + ps.next(); + ps.skipBlank(); + continue; + } + + break; + } + + ps.expectChar(")"); + return new CallArguments(positional, named); + } + + getString(ps) { + ps.expectChar("\""); + let value = ""; + + let ch; + while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) { + if (ch === "\\") { + value += this.getEscapeSequence(ps); + } else { + value += ch; + } + } + + if (ps.currentChar === EOL) { + throw new ParseError("E0020"); + } + + ps.expectChar("\""); + + return new StringLiteral(value); + } + + getLiteral(ps) { + if (ps.isNumberStart()) { + return this.getNumber(ps); + } + + if (ps.currentChar === '"') { + return this.getString(ps); + } + + throw new ParseError("E0014"); + } +} + +function indent(content) { + return content.split("\n").join("\n "); +} + +function includesNewLine(elem) { + return elem.type === "TextElement" && includes(elem.value, "\n"); +} + +function isSelectExpr(elem) { + return elem.type === "Placeable" + && elem.expression.type === "SelectExpression"; +} + +const HAS_ENTRIES = 1; + +class FluentSerializer { + constructor({ withJunk = false } = {}) { + this.withJunk = withJunk; + } + + serialize(resource) { + if (resource.type !== "Resource") { + throw new Error(`Unknown resource type: ${resource.type}`); + } + + let state = 0; + const parts = []; + + for (const entry of resource.body) { + if (entry.type !== "Junk" || this.withJunk) { + parts.push(this.serializeEntry(entry, state)); + if (!(state & HAS_ENTRIES)) { + state |= HAS_ENTRIES; + } + } + } + + return parts.join(""); + } + + serializeEntry(entry, state = 0) { + switch (entry.type) { + case "Message": + return serializeMessage(entry); + case "Term": + return serializeTerm(entry); + case "Comment": + if (state & HAS_ENTRIES) { + return `\n${serializeComment(entry, "#")}\n`; + } + return `${serializeComment(entry, "#")}\n`; + case "GroupComment": + if (state & HAS_ENTRIES) { + return `\n${serializeComment(entry, "##")}\n`; + } + return `${serializeComment(entry, "##")}\n`; + case "ResourceComment": + if (state & HAS_ENTRIES) { + return `\n${serializeComment(entry, "###")}\n`; + } + return `${serializeComment(entry, "###")}\n`; + case "Junk": + return serializeJunk(entry); + default : + throw new Error(`Unknown entry type: ${entry.type}`); + } + } +} + + +function serializeComment(comment, prefix = "#") { + const prefixed = comment.content.split("\n").map( + line => line.length ? `${prefix} ${line}` : prefix + ).join("\n"); + // Add the trailing newline. + return `${prefixed}\n`; +} + + +function serializeJunk(junk) { + return junk.content; +} + + +function serializeMessage(message) { + const parts = []; + + if (message.comment) { + parts.push(serializeComment(message.comment)); + } + + parts.push(`${message.id.name} =`); + + if (message.value) { + parts.push(serializePattern(message.value)); + } + + for (const attribute of message.attributes) { + parts.push(serializeAttribute(attribute)); + } + + parts.push("\n"); + return parts.join(""); +} + + +function serializeTerm(term) { + const parts = []; + + if (term.comment) { + parts.push(serializeComment(term.comment)); + } + + parts.push(`-${term.id.name} =`); + parts.push(serializePattern(term.value)); + + for (const attribute of term.attributes) { + parts.push(serializeAttribute(attribute)); + } + + parts.push("\n"); + return parts.join(""); +} + + +function serializeAttribute(attribute) { + const value = indent(serializePattern(attribute.value)); + return `\n .${attribute.id.name} =${value}`; +} + + +function serializePattern(pattern) { + const content = pattern.elements.map(serializeElement).join(""); + const startOnNewLine = + pattern.elements.some(isSelectExpr) || + pattern.elements.some(includesNewLine); + + if (startOnNewLine) { + return `\n ${indent(content)}`; + } + + return ` ${content}`; +} + + +function serializeElement(element) { + switch (element.type) { + case "TextElement": + return element.value; + case "Placeable": + return serializePlaceable(element); + default: + throw new Error(`Unknown element type: ${element.type}`); + } +} + + +function serializePlaceable(placeable) { + const expr = placeable.expression; + switch (expr.type) { + case "Placeable": + return `{${serializePlaceable(expr)}}`; + case "SelectExpression": + // Special-case select expression to control the whitespace around the + // opening and the closing brace. + return `{ ${serializeExpression(expr)}}`; + default: + return `{ ${serializeExpression(expr)} }`; + } +} + + +function serializeExpression(expr) { + switch (expr.type) { + case "StringLiteral": + return `"${expr.value}"`; + case "NumberLiteral": + return expr.value; + case "VariableReference": + return `$${expr.id.name}`; + case "TermReference": { + let out = `-${expr.id.name}`; + if (expr.attribute) { + out += `.${expr.attribute.name}`; + } + if (expr.arguments) { + out += serializeCallArguments(expr.arguments); + } + return out; + } + case "MessageReference": { + let out = expr.id.name; + if (expr.attribute) { + out += `.${expr.attribute.name}`; + } + return out; + } + case "FunctionReference": + return `${expr.id.name}${serializeCallArguments(expr.arguments)}`; + case "SelectExpression": { + let out = `${serializeExpression(expr.selector)} ->`; + for (let variant of expr.variants) { + out += serializeVariant(variant); + } + return `${out}\n`; + } + case "Placeable": + return serializePlaceable(expr); + default: + throw new Error(`Unknown expression type: ${expr.type}`); + } +} + + +function serializeVariant(variant) { + const key = serializeVariantKey(variant.key); + const value = indent(serializePattern(variant.value)); + + if (variant.default) { + return `\n *[${key}]${value}`; + } + + return `\n [${key}]${value}`; +} + + +function serializeCallArguments(expr) { + const positional = expr.positional.map(serializeExpression).join(", "); + const named = expr.named.map(serializeNamedArgument).join(", "); + if (expr.positional.length > 0 && expr.named.length > 0) { + return `(${positional}, ${named})`; + } + return `(${positional || named})`; +} + + +function serializeNamedArgument(arg) { + const value = serializeExpression(arg.value); + return `${arg.name.name}: ${value}`; +} + + +function serializeVariantKey(key) { + switch (key.type) { + case "Identifier": + return key.name; + case "NumberLiteral": + return key.value; + default: + throw new Error(`Unknown variant key type: ${key.type}`); + } +} + +/* + * Abstract Visitor pattern + */ +class Visitor { + visit(node) { + if (Array.isArray(node)) { + node.forEach(child => this.visit(child)); + return; + } + if (!(node instanceof BaseNode)) { + return; + } + const visit = this[`visit${node.type}`] || this.genericVisit; + visit.call(this, node); + } + + genericVisit(node) { + for (const propname of Object.keys(node)) { + this.visit(node[propname]); + } + } +} + +/* + * Abstract Transformer pattern + */ +class Transformer extends Visitor { + visit(node) { + if (!(node instanceof BaseNode)) { + return node; + } + const visit = this[`visit${node.type}`] || this.genericVisit; + return visit.call(this, node); + } + + genericVisit(node) { + for (const propname of Object.keys(node)) { + const propvalue = node[propname]; + if (Array.isArray(propvalue)) { + const newvals = propvalue + .map(child => this.visit(child)) + .filter(newchild => newchild !== undefined); + node[propname] = newvals; + } + if (propvalue instanceof BaseNode) { + const new_val = this.visit(propvalue); + if (new_val === undefined) { + delete node[propname]; + } else { + node[propname] = new_val; + } + } + } + return node; + } +} + +const visitor = ({ + Visitor: Visitor, + Transformer: Transformer +}); + +/* eslint object-shorthand: "off", + comma-dangle: "off", + no-labels: "off" */ + +this.EXPORTED_SYMBOLS = [ + ...Object.keys({ + FluentParser, + FluentSerializer, + }), + ...Object.keys(ast), + ...Object.keys(visitor), +]; diff --git a/intl/l10n/L10nRegistry.cpp b/intl/l10n/L10nRegistry.cpp new file mode 100644 index 0000000000..faf075dbbb --- /dev/null +++ b/intl/l10n/L10nRegistry.cpp @@ -0,0 +1,442 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "L10nRegistry.h" +#include "mozilla/RefPtr.h" +#include "mozilla/URLPreloader.h" +#include "nsIChannel.h" +#include "nsILoadInfo.h" +#include "nsNetUtil.h" +#include "nsString.h" +#include "nsContentUtils.h" +#include "FluentResource.h" +#include "FileSource.h" +#include "nsICategoryManager.h" +#include "mozilla/SimpleEnumerator.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/PContent.h" +#include "mozilla/dom/ContentParent.h" +#include "mozilla/Preferences.h" + +using namespace mozilla; +using namespace mozilla::dom; + +namespace mozilla::intl { + +/* FluentBundleIterator */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentBundleIterator, mGlobal) + +FluentBundleIterator::FluentBundleIterator( + nsIGlobalObject* aGlobal, UniquePtr aRaw) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +JSObject* FluentBundleIterator::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return FluentBundleIterator_Binding::Wrap(aCx, this, aGivenProto); +} + +void FluentBundleIterator::Next(FluentBundleIteratorResult& aResult) { + UniquePtr raw( + ffi::fluent_bundle_iterator_next(mRaw.get())); + if (!raw) { + aResult.mDone = true; + return; + } + aResult.mDone = false; + aResult.mValue = new FluentBundle(mGlobal, std::move(raw)); +} + +already_AddRefed FluentBundleIterator::Values() { + return do_AddRef(this); +} + +/* FluentBundleAsyncIterator */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(FluentBundleAsyncIterator, mGlobal) + +FluentBundleAsyncIterator::FluentBundleAsyncIterator( + nsIGlobalObject* aGlobal, + UniquePtr aRaw) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +JSObject* FluentBundleAsyncIterator::WrapObject( + JSContext* aCx, JS::Handle aGivenProto) { + return FluentBundleAsyncIterator_Binding::Wrap(aCx, this, aGivenProto); +} + +already_AddRefed FluentBundleAsyncIterator::Next(ErrorResult& aError) { + RefPtr promise = Promise::Create(mGlobal, aError); + if (aError.Failed()) { + return nullptr; + } + + ffi::fluent_bundle_async_iterator_next( + mRaw.get(), promise, + // callback function which will be invoked by the rust code, passing the + // promise back in. + [](auto* aPromise, ffi::FluentBundleRc* aBundle) { + Promise* promise = const_cast(aPromise); + + FluentBundleIteratorResult res; + + if (aBundle) { + // The Rust caller will transfer the ownership to us. + UniquePtr b(aBundle); + nsIGlobalObject* global = promise->GetGlobalObject(); + res.mValue = new FluentBundle(global, std::move(b)); + res.mDone = false; + } else { + res.mDone = true; + } + promise->MaybeResolve(res); + }); + + return promise.forget(); +} + +already_AddRefed +FluentBundleAsyncIterator::Values() { + return do_AddRef(this); +} + +/* L10nRegistry */ + +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(L10nRegistry, mGlobal) + +L10nRegistry::L10nRegistry(nsIGlobalObject* aGlobal, bool aUseIsolating) + : mGlobal(aGlobal), + mRaw(dont_AddRef(ffi::l10nregistry_new(aUseIsolating))) {} + +L10nRegistry::L10nRegistry(nsIGlobalObject* aGlobal, + RefPtr aRaw) + : mGlobal(aGlobal), mRaw(std::move(aRaw)) {} + +/* static */ +already_AddRefed L10nRegistry::Constructor( + const GlobalObject& aGlobal, const L10nRegistryOptions& aOptions) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + return MakeAndAddRef(global, + aOptions.mBundleOptions.mUseIsolating); +} + +/* static */ +already_AddRefed L10nRegistry::GetInstance( + const GlobalObject& aGlobal) { + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + return MakeAndAddRef( + global, dont_AddRef(ffi::l10nregistry_instance_get())); +} + +JSObject* L10nRegistry::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return L10nRegistry_Binding::Wrap(aCx, this, aGivenProto); +} + +void L10nRegistry::GetAvailableLocales(nsTArray& aRetVal) { + ffi::l10nregistry_get_available_locales(mRaw.get(), &aRetVal); +} + +void L10nRegistry::RegisterSources( + const Sequence>& aSources) { + nsTArray sources(aSources.Length()); + for (const auto& source : aSources) { + sources.AppendElement(source->Raw()); + } + + ffi::l10nregistry_register_sources(mRaw.get(), &sources); +} + +void L10nRegistry::UpdateSources( + const Sequence>& aSources) { + nsTArray sources(aSources.Length()); + for (const auto& source : aSources) { + sources.AppendElement(source->Raw()); + } + + ffi::l10nregistry_update_sources(mRaw.get(), &sources); +} + +void L10nRegistry::RemoveSources(const Sequence& aSources) { + ffi::l10nregistry_remove_sources(mRaw.get(), aSources.Elements(), + aSources.Length()); +} + +bool L10nRegistry::HasSource(const nsACString& aName, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + + bool result = ffi::l10nregistry_has_source(mRaw.get(), &aName, &status); + PopulateError(aRv, status); + return result; +} + +already_AddRefed L10nRegistry::GetSource( + const nsACString& aName, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + + RefPtr raw( + dont_AddRef(ffi::l10nregistry_get_source(mRaw.get(), &aName, &status))); + if (PopulateError(aRv, status)) { + return nullptr; + } + + return MakeAndAddRef(std::move(raw)); +} + +void L10nRegistry::GetSourceNames(nsTArray& aRetVal) { + ffi::l10nregistry_get_source_names(mRaw.get(), &aRetVal); +} + +void L10nRegistry::ClearSources() { + ffi::l10nregistry_clear_sources(mRaw.get()); +} + +/* static */ +ffi::GeckoResourceId L10nRegistry::ResourceIdToFFI( + const nsCString& aResourceId) { + return ffi::GeckoResourceId{ + aResourceId, + ffi::GeckoResourceType::Required, + }; +} + +/* static */ +ffi::GeckoResourceId L10nRegistry::ResourceIdToFFI( + const dom::OwningUTF8StringOrResourceId& aResourceId) { + if (aResourceId.IsUTF8String()) { + return ffi::GeckoResourceId{ + aResourceId.GetAsUTF8String(), + ffi::GeckoResourceType::Required, + }; + } + return ffi::GeckoResourceId{ + aResourceId.GetAsResourceId().mPath, + aResourceId.GetAsResourceId().mOptional + ? ffi::GeckoResourceType::Optional + : ffi::GeckoResourceType::Required, + }; +} + +/* static */ +nsTArray L10nRegistry::ResourceIdsToFFI( + const nsTArray& aResourceIds) { + nsTArray ffiResourceIds; + for (const auto& resourceId : aResourceIds) { + ffiResourceIds.EmplaceBack(ResourceIdToFFI(resourceId)); + } + return ffiResourceIds; +} + +/* static */ +nsTArray L10nRegistry::ResourceIdsToFFI( + const nsTArray& aResourceIds) { + nsTArray ffiResourceIds; + for (const auto& resourceId : aResourceIds) { + ffiResourceIds.EmplaceBack(ResourceIdToFFI(resourceId)); + } + return ffiResourceIds; +} + +already_AddRefed L10nRegistry::GenerateBundlesSync( + const nsTArray& aLocales, + const nsTArray& aResourceIds, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + UniquePtr iter( + ffi::l10nregistry_generate_bundles_sync( + mRaw, aLocales.Elements(), aLocales.Length(), aResourceIds.Elements(), + aResourceIds.Length(), &status)); + + if (PopulateError(aRv, status) || !iter) { + return nullptr; + } + + return do_AddRef(new FluentBundleIterator(mGlobal, std::move(iter))); +} + +already_AddRefed L10nRegistry::GenerateBundlesSync( + const dom::Sequence& aLocales, + const dom::Sequence& aResourceIds, + ErrorResult& aRv) { + auto ffiResourceIds{ResourceIdsToFFI(aResourceIds)}; + return GenerateBundlesSync(aLocales, ffiResourceIds, aRv); +} + +already_AddRefed L10nRegistry::GenerateBundles( + const nsTArray& aLocales, + const nsTArray& aResourceIds, ErrorResult& aRv) { + ffi::L10nRegistryStatus status; + UniquePtr iter( + ffi::l10nregistry_generate_bundles( + mRaw, aLocales.Elements(), aLocales.Length(), aResourceIds.Elements(), + aResourceIds.Length(), &status)); + if (PopulateError(aRv, status) || !iter) { + return nullptr; + } + + return do_AddRef(new FluentBundleAsyncIterator(mGlobal, std::move(iter))); +} + +already_AddRefed L10nRegistry::GenerateBundles( + const dom::Sequence& aLocales, + const dom::Sequence& aResourceIds, + ErrorResult& aRv) { + nsTArray resourceIds; + for (const auto& resourceId : aResourceIds) { + resourceIds.EmplaceBack(ResourceIdToFFI(resourceId)); + } + return GenerateBundles(aLocales, resourceIds, aRv); +} + +/* static */ +void L10nRegistry::GetParentProcessFileSourceDescriptors( + nsTArray& aRetVal) { + MOZ_ASSERT(XRE_IsParentProcess()); + nsTArray sources; + ffi::l10nregistry_get_parent_process_sources(&sources); + for (const auto& source : sources) { + auto descriptor = aRetVal.AppendElement(); + descriptor->name() = source.name; + descriptor->metasource() = source.metasource; + descriptor->locales().AppendElements(std::move(source.locales)); + descriptor->prePath() = source.pre_path; + descriptor->index().AppendElements(std::move(source.index)); + } +} + +/* static */ +void L10nRegistry::RegisterFileSourcesFromParentProcess( + const nsTArray& aDescriptors) { + // This means that in content processes the L10nRegistry + // service instance is created eagerly, not lazily. + // It is necessary so that the instance can store the sources + // provided in the IPC init, which, in turn, is necessary + // for the service to be avialable for sync bundle generation. + // + // L10nRegistry is lightweight and performs no operations, so + // we believe this behavior to be acceptable. + MOZ_ASSERT(XRE_IsContentProcess()); + nsTArray sources; + for (const auto& desc : aDescriptors) { + auto source = sources.AppendElement(); + source->name = desc.name(); + source->metasource = desc.metasource(); + source->locales.AppendElements(desc.locales()); + source->pre_path = desc.prePath(); + source->index.AppendElements(desc.index()); + } + ffi::l10nregistry_register_parent_process_sources(&sources); +} + +/* static */ +nsresult L10nRegistry::Load(const nsACString& aPath, + nsIStreamLoaderObserver* aObserver) { + nsCOMPtr uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), aPath); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(uri, NS_ERROR_INVALID_ARG); + + RefPtr loader; + rv = NS_NewStreamLoader( + getter_AddRefs(loader), uri, aObserver, + nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + nsIContentPolicy::TYPE_OTHER); + + return rv; +} + +/* static */ +nsresult L10nRegistry::LoadSync(const nsACString& aPath, void** aData, + uint64_t* aSize) { + nsCOMPtr uri; + + nsresult rv = NS_NewURI(getter_AddRefs(uri), aPath); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ENSURE_TRUE(uri, NS_ERROR_INVALID_ARG); + + auto result = URLPreloader::ReadURI(uri); + if (result.isOk()) { + auto uri = result.unwrap(); + *aData = ToNewCString(uri); + *aSize = uri.Length(); + return NS_OK; + } + + auto err = result.unwrapErr(); + if (err != NS_ERROR_INVALID_ARG && err != NS_ERROR_NOT_INITIALIZED) { + return err; + } + + nsCOMPtr channel; + rv = NS_NewChannel( + getter_AddRefs(channel), uri, nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL, + nsIContentPolicy::TYPE_OTHER, nullptr, /* nsICookieJarSettings */ + nullptr, /* aPerformanceStorage */ + nullptr, /* aLoadGroup */ + nullptr, /* aCallbacks */ + nsIRequest::LOAD_NORMAL); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr input; + rv = channel->Open(getter_AddRefs(input)); + NS_ENSURE_SUCCESS(rv, NS_ERROR_INVALID_ARG); + + return NS_ReadInputStreamToBuffer(input, aData, -1, aSize); +} + +/* static */ +bool L10nRegistry::PopulateError(ErrorResult& aError, + ffi::L10nRegistryStatus& aStatus) { + switch (aStatus) { + case ffi::L10nRegistryStatus::InvalidLocaleCode: + aError.ThrowTypeError("Invalid locale code"); + return true; + case ffi::L10nRegistryStatus::EmptyName: + aError.ThrowTypeError("Name cannot be empty."); + return true; + + case ffi::L10nRegistryStatus::None: + return false; + } + MOZ_ASSERT_UNREACHABLE("Unknown status"); + return false; +} + +extern "C" { +nsresult L10nRegistryLoad(const nsACString* aPath, + const nsIStreamLoaderObserver* aObserver) { + if (!aPath || !aObserver) { + return NS_ERROR_INVALID_ARG; + } + + return mozilla::intl::L10nRegistry::Load( + *aPath, const_cast(aObserver)); +} + +nsresult L10nRegistryLoadSync(const nsACString* aPath, void** aData, + uint64_t* aSize) { + if (!aPath || !aData || !aSize) { + return NS_ERROR_INVALID_ARG; + } + + return mozilla::intl::L10nRegistry::LoadSync(*aPath, aData, aSize); +} + +void L10nRegistrySendUpdateL10nFileSources() { + MOZ_ASSERT(XRE_IsParentProcess()); + nsTArray sources; + L10nRegistry::GetParentProcessFileSourceDescriptors(sources); + + nsTArray parents; + ContentParent::GetAll(parents); + for (ContentParent* parent : parents) { + Unused << parent->SendUpdateL10nFileSources(sources); + } +} + +} // extern "C" + +} // namespace mozilla::intl diff --git a/intl/l10n/L10nRegistry.h b/intl/l10n/L10nRegistry.h new file mode 100644 index 0000000000..91047ec99c --- /dev/null +++ b/intl/l10n/L10nRegistry.h @@ -0,0 +1,152 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_L10nRegistry_h +#define mozilla_intl_l10n_L10nRegistry_h + +#include "nsIStreamLoader.h" +#include "nsWrapperCache.h" +#include "nsCycleCollectionParticipant.h" +#include "mozilla/dom/L10nRegistryBinding.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/intl/FluentBindings.h" +#include "mozilla/intl/RegistryBindings.h" + +class nsIGlobalObject; + +namespace mozilla::dom { +class L10nFileSourceDescriptor; +} + +namespace mozilla::intl { + +class FluentBundleAsyncIterator final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentBundleAsyncIterator) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentBundleAsyncIterator) + + FluentBundleAsyncIterator( + nsIGlobalObject* aGlobal, + UniquePtr aRaw); + + virtual JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + // WebIDL + already_AddRefed Next(ErrorResult& aError); + already_AddRefed Values(); + + protected: + ~FluentBundleAsyncIterator() = default; + nsCOMPtr mGlobal; + UniquePtr mRaw; +}; + +class FluentBundleIterator final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(FluentBundleIterator) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(FluentBundleIterator) + + FluentBundleIterator(nsIGlobalObject* aGlobal, + UniquePtr aRaw); + + virtual JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + // WebIDL + void Next(dom::FluentBundleIteratorResult& aResult); + already_AddRefed Values(); + + protected: + ~FluentBundleIterator() = default; + nsCOMPtr mGlobal; + UniquePtr mRaw; +}; + +class L10nRegistry final : public nsWrapperCache { + public: + NS_INLINE_DECL_CYCLE_COLLECTING_NATIVE_REFCOUNTING(L10nRegistry) + NS_DECL_CYCLE_COLLECTION_NATIVE_WRAPPERCACHE_CLASS(L10nRegistry) + + L10nRegistry(nsIGlobalObject* aGlobal, bool aUseIsolating); + + L10nRegistry(nsIGlobalObject* aGlobal, + RefPtr aRaw); + + static already_AddRefed Constructor( + const dom::GlobalObject& aGlobal, + const dom::L10nRegistryOptions& aOptions); + + static already_AddRefed GetInstance( + const dom::GlobalObject& aGlobal); + + static void GetParentProcessFileSourceDescriptors( + nsTArray& aRetVal); + static void RegisterFileSourcesFromParentProcess( + const nsTArray& aDescriptors); + + static nsresult Load(const nsACString& aPath, + nsIStreamLoaderObserver* aObserver); + static nsresult LoadSync(const nsACString& aPath, void** aData, + uint64_t* aSize); + + static ffi::GeckoResourceId ResourceIdToFFI(const nsCString& aResourceId); + static ffi::GeckoResourceId ResourceIdToFFI( + const dom::OwningUTF8StringOrResourceId& aResourceId); + static nsTArray ResourceIdsToFFI( + const nsTArray& aResourceIds); + static nsTArray ResourceIdsToFFI( + const nsTArray& aResourceIds); + + void GetAvailableLocales(nsTArray& aRetVal); + + void RegisterSources( + const dom::Sequence>& aSources); + void UpdateSources( + const dom::Sequence>& aSources); + void RemoveSources(const dom::Sequence& aSources); + bool HasSource(const nsACString& aName, ErrorResult& aRv); + already_AddRefed GetSource(const nsACString& aName, + ErrorResult& aRv); + void GetSourceNames(nsTArray& aRetVal); + void ClearSources(); + + already_AddRefed GenerateBundlesSync( + const nsTArray& aLocales, + const nsTArray& aResourceIds, ErrorResult& aRv); + already_AddRefed GenerateBundlesSync( + const dom::Sequence& aLocales, + const dom::Sequence& aResourceIds, + ErrorResult& aRv); + + already_AddRefed GenerateBundles( + const nsTArray& aLocales, + const nsTArray& aResourceIds, ErrorResult& aRv); + already_AddRefed GenerateBundles( + const dom::Sequence& aLocales, + const dom::Sequence& aResourceIds, + ErrorResult& aRv); + + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + + const ffi::GeckoL10nRegistry* Raw() const { return mRaw; } + + protected: + virtual ~L10nRegistry() = default; + nsCOMPtr mGlobal; + const RefPtr mRaw; + static bool PopulateError(ErrorResult& aError, + ffi::L10nRegistryStatus& aStatus); +}; + +} // namespace mozilla::intl + +#endif diff --git a/intl/l10n/Localization.cpp b/intl/l10n/Localization.cpp new file mode 100644 index 0000000000..0d6f1f9882 --- /dev/null +++ b/intl/l10n/Localization.cpp @@ -0,0 +1,522 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "Localization.h" +#include "nsIObserverService.h" +#include "mozilla/BasePrincipal.h" +#include "mozilla/Preferences.h" +#include "mozilla/Services.h" +#include "mozilla/dom/PromiseNativeHandler.h" + +#define INTL_APP_LOCALES_CHANGED "intl:app-locales-changed" +#define L10N_PSEUDO_PREF "intl.l10n.pseudo" + +using namespace mozilla; +using namespace mozilla::dom; +using namespace mozilla::intl; + +static const char* kObservedPrefs[] = {L10N_PSEUDO_PREF, nullptr}; + +static nsTArray ConvertFromL10nKeys( + const Sequence& aKeys) { + nsTArray l10nKeys(aKeys.Length()); + + for (const auto& entry : aKeys) { + if (entry.IsUTF8String()) { + const auto& id = entry.GetAsUTF8String(); + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &id; + } else { + const auto& e = entry.GetAsL10nIdArgs(); + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &e.mId; + if (!e.mArgs.IsNull()) { + FluentBundle::ConvertArgs(e.mArgs.Value(), key->args); + } + } + } + + return l10nKeys; +} + +[[nodiscard]] static bool ConvertToAttributeNameValue( + const nsTArray& aAttributes, + FallibleTArray& aValues) { + if (!aValues.SetCapacity(aAttributes.Length(), fallible)) { + return false; + } + for (const auto& attr : aAttributes) { + auto* cvtAttr = aValues.AppendElement(fallible); + MOZ_ASSERT(cvtAttr, "SetCapacity didn't set enough capacity somehow?"); + cvtAttr->mName = attr.name; + cvtAttr->mValue = attr.value; + } + return true; +} + +[[nodiscard]] static bool ConvertToL10nMessages( + const nsTArray& aMessages, + nsTArray>& aOut) { + if (!aOut.SetCapacity(aMessages.Length(), fallible)) { + return false; + } + + for (const auto& entry : aMessages) { + Nullable* msg = aOut.AppendElement(fallible); + MOZ_ASSERT(msg, "SetCapacity didn't set enough capacity somehow?"); + + if (!entry.is_present) { + continue; + } + + L10nMessage& m = msg->SetValue(); + if (!entry.message.value.IsVoid()) { + m.mValue = entry.message.value; + } + if (!entry.message.attributes.IsEmpty()) { + auto& value = m.mAttributes.SetValue(); + if (!ConvertToAttributeNameValue(entry.message.attributes, value)) { + return false; + } + } + } + + return true; +} + +NS_IMPL_CYCLE_COLLECTING_ADDREF(Localization) +NS_IMPL_CYCLE_COLLECTING_RELEASE(Localization) +NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(Localization) + NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY + NS_INTERFACE_MAP_ENTRY(nsIObserver) + NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference) + NS_INTERFACE_MAP_ENTRY_AMBIGUOUS(nsISupports, nsIObserver) +NS_INTERFACE_MAP_END +NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE_WEAK(Localization, mGlobal) + +/* static */ +already_AddRefed Localization::Create( + const nsTArray& aResourceIds, bool aIsSync) { + return MakeAndAddRef(aResourceIds, aIsSync); +} + +/* static */ +already_AddRefed Localization::Create( + const nsTArray& aResourceIds, bool aIsSync) { + return MakeAndAddRef(aResourceIds, aIsSync); +} + +Localization::Localization(const nsTArray& aResIds, bool aIsSync) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResIds)}; + ffi::localization_new(&ffiResourceIds, aIsSync, nullptr, + getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(const nsTArray& aResIds, + bool aIsSync) { + ffi::localization_new(&aResIds, aIsSync, nullptr, getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(nsIGlobalObject* aGlobal, + const nsTArray& aResIds, bool aIsSync) + : mGlobal(aGlobal) { + nsTArray resourceIds{ + L10nRegistry::ResourceIdsToFFI(aResIds)}; + ffi::localization_new(&resourceIds, aIsSync, nullptr, getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(nsIGlobalObject* aGlobal, bool aIsSync) + : mGlobal(aGlobal) { + nsTArray resIds; + ffi::localization_new(&resIds, aIsSync, nullptr, getter_AddRefs(mRaw)); + + RegisterObservers(); +} + +Localization::Localization(nsIGlobalObject* aGlobal, bool aIsSync, + const ffi::LocalizationRc* aRaw) + : mGlobal(aGlobal), mRaw(aRaw) { + RegisterObservers(); +} + +already_AddRefed Localization::Constructor( + const GlobalObject& aGlobal, + const Sequence& aResourceIds, bool aIsSync, + const Optional>& aRegistry, + const Optional>& aLocales, ErrorResult& aRv) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResourceIds)}; + Maybe> locales; + + if (aLocales.WasPassed()) { + locales.emplace(); + locales->SetCapacity(aLocales.Value().Length()); + for (const auto& locale : aLocales.Value()) { + locales->AppendElement(locale); + } + } + + RefPtr raw; + + bool result = ffi::localization_new_with_locales( + &ffiResourceIds, aIsSync, + aRegistry.WasPassed() ? aRegistry.Value().Raw() : nullptr, + locales.ptrOr(nullptr), getter_AddRefs(raw)); + + if (!result) { + aRv.ThrowInvalidStateError( + "Failed to create the Localization. Check the locales arguments."); + return nullptr; + } + + nsCOMPtr global = do_QueryInterface(aGlobal.GetAsSupports()); + + return do_AddRef(new Localization(global, aIsSync, raw)); +} + +JSObject* Localization::WrapObject(JSContext* aCx, + JS::Handle aGivenProto) { + return Localization_Binding::Wrap(aCx, this, aGivenProto); +} + +Localization::~Localization() = default; + +NS_IMETHODIMP +Localization::Observe(nsISupports* aSubject, const char* aTopic, + const char16_t* aData) { + if (!strcmp(aTopic, INTL_APP_LOCALES_CHANGED)) { + OnChange(); + } else { + MOZ_ASSERT(!strcmp("nsPref:changed", aTopic)); + nsDependentString pref(aData); + if (pref.EqualsLiteral(L10N_PSEUDO_PREF)) { + OnChange(); + } + } + + return NS_OK; +} + +void Localization::RegisterObservers() { + DebugOnly rv = Preferences::AddWeakObservers(this, kObservedPrefs); + MOZ_ASSERT(NS_SUCCEEDED(rv), "Adding observers failed."); + + nsCOMPtr obs = mozilla::services::GetObserverService(); + if (obs) { + obs->AddObserver(this, INTL_APP_LOCALES_CHANGED, true); + } +} + +void Localization::OnChange() { ffi::localization_on_change(mRaw.get()); } + +void Localization::AddResourceId(const ffi::GeckoResourceId& aResourceId) { + ffi::localization_add_res_id(mRaw.get(), &aResourceId); +} +void Localization::AddResourceId(const nsCString& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + AddResourceId(ffiResourceId); +} +void Localization::AddResourceId( + const dom::OwningUTF8StringOrResourceId& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + AddResourceId(ffiResourceId); +} + +uint32_t Localization::RemoveResourceId( + const ffi::GeckoResourceId& aResourceId) { + return ffi::localization_remove_res_id(mRaw.get(), &aResourceId); +} +uint32_t Localization::RemoveResourceId(const nsCString& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + return RemoveResourceId(ffiResourceId); +} +uint32_t Localization::RemoveResourceId( + const dom::OwningUTF8StringOrResourceId& aResourceId) { + auto ffiResourceId{L10nRegistry::ResourceIdToFFI(aResourceId)}; + return RemoveResourceId(ffiResourceId); +} + +void Localization::AddResourceIds( + const nsTArray& aResourceIds) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResourceIds)}; + ffi::localization_add_res_ids(mRaw.get(), &ffiResourceIds); +} + +uint32_t Localization::RemoveResourceIds( + const nsTArray& aResourceIds) { + auto ffiResourceIds{L10nRegistry::ResourceIdsToFFI(aResourceIds)}; + return ffi::localization_remove_res_ids(mRaw.get(), &ffiResourceIds); +} + +already_AddRefed Localization::FormatValue( + const nsACString& aId, const Optional& aArgs, ErrorResult& aRv) { + nsTArray l10nArgs; + nsTArray errors; + + if (aArgs.WasPassed()) { + const L10nArgs& args = aArgs.Value(); + FluentBundle::ConvertArgs(args, l10nArgs); + } + RefPtr promise = Promise::Create(mGlobal, aRv); + + ffi::localization_format_value( + mRaw.get(), &aId, &l10nArgs, promise, + [](const Promise* aPromise, const nsACString* aValue, + const nsTArray* aErrors) { + Promise* promise = const_cast(aPromise); + + ErrorResult rv; + if (MaybeReportErrorsToGecko(*aErrors, rv, + promise->GetParentObject())) { + promise->MaybeReject(std::move(rv)); + } else { + promise->MaybeResolve(aValue); + } + }); + + return MaybeWrapPromise(promise); +} + +already_AddRefed Localization::FormatValues( + const Sequence& aKeys, ErrorResult& aRv) { + nsTArray l10nKeys = ConvertFromL10nKeys(aKeys); + + RefPtr promise = Promise::Create(mGlobal, aRv); + if (aRv.Failed()) { + return nullptr; + } + + ffi::localization_format_values( + mRaw.get(), &l10nKeys, promise, + // callback function which will be invoked by the rust code, passing the + // promise back in. + [](const Promise* aPromise, const nsTArray* aValues, + const nsTArray* aErrors) { + Promise* promise = const_cast(aPromise); + + ErrorResult rv; + if (MaybeReportErrorsToGecko(*aErrors, rv, + promise->GetParentObject())) { + promise->MaybeReject(std::move(rv)); + } else { + promise->MaybeResolve(*aValues); + } + }); + + return MaybeWrapPromise(promise); +} + +already_AddRefed Localization::FormatMessages( + const Sequence& aKeys, ErrorResult& aRv) { + auto l10nKeys = ConvertFromL10nKeys(aKeys); + + RefPtr promise = Promise::Create(mGlobal, aRv); + if (aRv.Failed()) { + return nullptr; + } + + ffi::localization_format_messages( + mRaw.get(), &l10nKeys, promise, + // callback function which will be invoked by the rust code, passing the + // promise back in. + [](const Promise* aPromise, + const nsTArray* aRaw, + const nsTArray* aErrors) { + Promise* promise = const_cast(aPromise); + + ErrorResult rv; + if (MaybeReportErrorsToGecko(*aErrors, rv, + promise->GetParentObject())) { + promise->MaybeReject(std::move(rv)); + } else { + nsTArray> messages; + if (!ConvertToL10nMessages(*aRaw, messages)) { + promise->MaybeReject(NS_ERROR_OUT_OF_MEMORY); + } else { + promise->MaybeResolve(std::move(messages)); + } + } + }); + + return MaybeWrapPromise(promise); +} + +void Localization::FormatValueSync(const nsACString& aId, + const Optional& aArgs, + nsACString& aRetVal, ErrorResult& aRv) { + nsTArray l10nArgs; + nsTArray errors; + + if (aArgs.WasPassed()) { + const L10nArgs& args = aArgs.Value(); + FluentBundle::ConvertArgs(args, l10nArgs); + } + + bool rv = ffi::localization_format_value_sync(mRaw.get(), &aId, &l10nArgs, + &aRetVal, &errors); + + if (rv) { + MaybeReportErrorsToGecko(errors, aRv, GetParentObject()); + } else { + aRv.ThrowInvalidStateError( + "Can't use formatValueSync when state is async."); + } +} + +void Localization::FormatValuesSync( + const Sequence& aKeys, + nsTArray& aRetVal, ErrorResult& aRv) { + nsTArray l10nKeys(aKeys.Length()); + nsTArray errors; + + for (const auto& entry : aKeys) { + if (entry.IsUTF8String()) { + const auto& id = entry.GetAsUTF8String(); + nsTArray l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &id; + } else { + const auto& e = entry.GetAsL10nIdArgs(); + nsTArray l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &e.mId; + if (!e.mArgs.IsNull()) { + FluentBundle::ConvertArgs(e.mArgs.Value(), key->args); + } + } + } + + bool rv = ffi::localization_format_values_sync(mRaw.get(), &l10nKeys, + &aRetVal, &errors); + + if (rv) { + MaybeReportErrorsToGecko(errors, aRv, GetParentObject()); + } else { + aRv.ThrowInvalidStateError( + "Can't use formatValuesSync when state is async."); + } +} + +void Localization::FormatMessagesSync( + const Sequence& aKeys, + nsTArray>& aRetVal, ErrorResult& aRv) { + nsTArray l10nKeys(aKeys.Length()); + nsTArray errors; + + for (const auto& entry : aKeys) { + if (entry.IsUTF8String()) { + const auto& id = entry.GetAsUTF8String(); + nsTArray l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &id; + } else { + const auto& e = entry.GetAsL10nIdArgs(); + nsTArray l10nArgs; + ffi::L10nKey* key = l10nKeys.AppendElement(); + key->id = &e.mId; + if (!e.mArgs.IsNull()) { + FluentBundle::ConvertArgs(e.mArgs.Value(), key->args); + } + } + } + + nsTArray result(l10nKeys.Length()); + + bool rv = ffi::localization_format_messages_sync(mRaw.get(), &l10nKeys, + &result, &errors); + + if (!rv) { + return aRv.ThrowInvalidStateError( + "Can't use formatMessagesSync when state is async."); + } + MaybeReportErrorsToGecko(errors, aRv, GetParentObject()); + if (aRv.Failed()) { + return; + } + if (!ConvertToL10nMessages(result, aRetVal)) { + return aRv.Throw(NS_ERROR_OUT_OF_MEMORY); + } +} + +void Localization::SetAsync() { ffi::localization_set_async(mRaw.get()); } +bool Localization::IsSync() { return ffi::localization_is_sync(mRaw.get()); } + +/** + * PromiseResolver is a PromiseNativeHandler used + * by MaybeWrapPromise method. + */ +class PromiseResolver final : public PromiseNativeHandler { + public: + NS_DECL_ISUPPORTS + + explicit PromiseResolver(Promise* aPromise) : mPromise(aPromise) {} + void ResolvedCallback(JSContext* aCx, JS::Handle aValue, + ErrorResult& aRv) override; + void RejectedCallback(JSContext* aCx, JS::Handle aValue, + ErrorResult& aRv) override; + + protected: + virtual ~PromiseResolver(); + + RefPtr mPromise; +}; + +NS_INTERFACE_MAP_BEGIN(PromiseResolver) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END + +NS_IMPL_ADDREF(PromiseResolver) +NS_IMPL_RELEASE(PromiseResolver) + +void PromiseResolver::ResolvedCallback(JSContext* aCx, + JS::Handle aValue, + ErrorResult& aRv) { + mPromise->MaybeResolveWithClone(aCx, aValue); +} + +void PromiseResolver::RejectedCallback(JSContext* aCx, + JS::Handle aValue, + ErrorResult& aRv) { + mPromise->MaybeRejectWithClone(aCx, aValue); +} + +PromiseResolver::~PromiseResolver() { mPromise = nullptr; } + +/** + * MaybeWrapPromise is a helper method used by Localization + * API methods to clone the value returned by a promise + * into a new context. + * + * This allows for a promise from a privileged context + * to be returned into an unprivileged document. + * + * This method is only used for promises that carry values. + */ +already_AddRefed Localization::MaybeWrapPromise( + Promise* aInnerPromise) { + MOZ_ASSERT(aInnerPromise->State() == Promise::PromiseState::Pending); + // For system principal we don't need to wrap the + // result promise at all. + nsIPrincipal* principal = mGlobal->PrincipalOrNull(); + if (principal && principal->IsSystemPrincipal()) { + return do_AddRef(aInnerPromise); + } + + IgnoredErrorResult result; + RefPtr docPromise = Promise::Create(mGlobal, result); + if (NS_WARN_IF(result.Failed())) { + return nullptr; + } + + auto resolver = MakeRefPtr(docPromise); + aInnerPromise->AppendNativeHandler(resolver); + return docPromise.forget(); +} diff --git a/intl/l10n/Localization.h b/intl/l10n/Localization.h new file mode 100644 index 0000000000..54c78f68db --- /dev/null +++ b/intl/l10n/Localization.h @@ -0,0 +1,165 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_Localization_h +#define mozilla_intl_l10n_Localization_h + +#include "nsCycleCollectionParticipant.h" +#include "nsIObserver.h" +#include "nsWeakReference.h" +#include "nsWrapperCache.h" +#include "nsWeakReference.h" +#include "nsIScriptError.h" +#include "nsContentUtils.h" +#include "nsPIDOMWindow.h" +#include "mozilla/ErrorResult.h" +#include "mozilla/dom/Promise.h" +#include "mozilla/dom/BindingDeclarations.h" +#include "mozilla/dom/LocalizationBinding.h" +#include "mozilla/intl/LocalizationBindings.h" +#include "mozilla/intl/L10nRegistry.h" + +namespace mozilla { +namespace intl { + +// The state where the application contains incomplete localization resources +// is much more common than for other types of core resources. +// +// In result, our localization is designed to handle missing resources +// gracefully, and we need a more fine-tuned way to communicate those problems +// to developers. +// +// In particular, we want developers and early adopters to be able to reason +// about missing translations, without bothering end user in production, where +// the user cannot react to that. +// +// We currently differentiate between nightly/dev-edition builds or automation +// where we report the errors, and beta/release, where we silence them. +// +// A side effect of the conditional model of strict vs loose error handling is +// that we don't have a way to write integration tests for behavior we expect +// out of production environment. See bug 1741430. +[[maybe_unused]] static bool MaybeReportErrorsToGecko( + const nsTArray& aErrors, ErrorResult& aRv, + nsIGlobalObject* aGlobal) { + if (!aErrors.IsEmpty()) { + if (xpc::IsInAutomation()) { + aRv.ThrowInvalidStateError(aErrors.ElementAt(0)); + return true; + } + +#if defined(NIGHTLY_BUILD) || defined(MOZ_DEV_EDITION) || defined(DEBUG) + dom::Document* doc = nullptr; + if (aGlobal) { + nsPIDOMWindowInner* innerWindow = aGlobal->AsInnerWindow(); + if (innerWindow) { + doc = innerWindow->GetExtantDoc(); + } + } + + for (const auto& error : aErrors) { + nsContentUtils::ReportToConsoleNonLocalized(NS_ConvertUTF8toUTF16(error), + nsIScriptError::warningFlag, + "l10n"_ns, doc); + printf_stderr("%s\n", error.get()); + } +#endif + } + + return false; +} + +class Localization : public nsIObserver, + public nsWrapperCache, + public nsSupportsWeakReference { + template + friend already_AddRefed mozilla::MakeAndAddRef(Args&&... aArgs); + + public: + NS_DECL_CYCLE_COLLECTING_ISUPPORTS + NS_DECL_CYCLE_COLLECTION_WRAPPERCACHE_CLASS_AMBIGUOUS(Localization, + nsIObserver) + NS_DECL_NSIOBSERVER + + static already_AddRefed Constructor( + const dom::GlobalObject& aGlobal, + const dom::Sequence& aResourceIds, + bool aIsSync, const dom::Optional>& aRegistry, + const dom::Optional>& aLocales, + ErrorResult& aRv); + static already_AddRefed Create( + const nsTArray& aResourceIds, bool aIsSync); + static already_AddRefed Create( + const nsTArray& aResourceIds, bool aIsSync); + + JSObject* WrapObject(JSContext* aCx, + JS::Handle aGivenProto) override; + nsIGlobalObject* GetParentObject() const { return mGlobal; } + + void SetIsSync(bool aIsSync); + + already_AddRefed FormatValue( + const nsACString& aId, const dom::Optional& aArgs, + ErrorResult& aRv); + + already_AddRefed FormatValues( + const dom::Sequence& aKeys, + ErrorResult& aRv); + + already_AddRefed FormatMessages( + const dom::Sequence& aKeys, + ErrorResult& aRv); + + void FormatValueSync(const nsACString& aId, + const dom::Optional& aArgs, + nsACString& aRetVal, ErrorResult& aRv); + void FormatValuesSync( + const dom::Sequence& aKeys, + nsTArray& aRetVal, ErrorResult& aRv); + void FormatMessagesSync( + const dom::Sequence& aKeys, + nsTArray>& aRetVal, ErrorResult& aRv); + + void AddResourceId(const ffi::GeckoResourceId& aResourceId); + void AddResourceId(const nsCString& aResourceId); + void AddResourceId(const dom::OwningUTF8StringOrResourceId& aResourceId); + uint32_t RemoveResourceId(const ffi::GeckoResourceId& aResourceId); + uint32_t RemoveResourceId(const nsCString& aResourceId); + uint32_t RemoveResourceId( + const dom::OwningUTF8StringOrResourceId& aResourceId); + void AddResourceIds( + const nsTArray& aResourceIds); + uint32_t RemoveResourceIds( + const nsTArray& aResourceIds); + + void SetAsync(); + bool IsSync(); + + protected: + Localization(const nsTArray& aResIds, bool aIsSync); + Localization(const nsTArray& aResIds, bool aIsSync); + Localization(nsIGlobalObject* aGlobal, bool aIsSync); + + Localization(nsIGlobalObject* aGlobal, const nsTArray& aResIds, + bool aIsSync); + + Localization(nsIGlobalObject* aGlobal, bool aIsSync, + const ffi::LocalizationRc* aRaw); + + virtual ~Localization(); + + void RegisterObservers(); + virtual void OnChange(); + already_AddRefed MaybeWrapPromise(dom::Promise* aInnerPromise); + + nsCOMPtr mGlobal; + RefPtr mRaw; +}; + +} // namespace intl +} // namespace mozilla + +#endif diff --git a/intl/l10n/LocalizationBindings.h b/intl/l10n/LocalizationBindings.h new file mode 100644 index 0000000000..fbf4db2db0 --- /dev/null +++ b/intl/l10n/LocalizationBindings.h @@ -0,0 +1,26 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_LocalizationBindings_h +#define mozilla_intl_l10n_LocalizationBindings_h + +#include "mozilla/intl/localization_ffi_generated.h" + +#include "mozilla/RefPtr.h" + +namespace mozilla { + +template <> +struct RefPtrTraits { + static void AddRef(const intl::ffi::LocalizationRc* aPtr) { + intl::ffi::localization_addref(aPtr); + } + static void Release(const intl::ffi::LocalizationRc* aPtr) { + intl::ffi::localization_release(aPtr); + } +}; + +} // namespace mozilla + +#endif diff --git a/intl/l10n/README b/intl/l10n/README new file mode 100644 index 0000000000..d507db0522 --- /dev/null +++ b/intl/l10n/README @@ -0,0 +1,8 @@ +The content of this directory is partially sourced from the fluent.js project. + +The following files are affected: + - FluentSyntax.jsm + +At the moment, the tool used to produce those files in fluent.js repository, doesn't +fully align with how the code is structured here, so we perform a manual adjustments +mostly around header and footer. diff --git a/intl/l10n/RegistryBindings.h b/intl/l10n/RegistryBindings.h new file mode 100644 index 0000000000..916082e5cf --- /dev/null +++ b/intl/l10n/RegistryBindings.h @@ -0,0 +1,53 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_intl_l10n_RegistryBindings_h +#define mozilla_intl_l10n_RegistryBindings_h + +#include "mozilla/intl/l10nregistry_ffi_generated.h" + +#include "mozilla/RefPtr.h" + +namespace mozilla { + +template <> +struct RefPtrTraits { + static void AddRef(const intl::ffi::FileSource* aPtr) { + intl::ffi::l10nfilesource_addref(aPtr); + } + static void Release(const intl::ffi::FileSource* aPtr) { + intl::ffi::l10nfilesource_release(aPtr); + } +}; + +template <> +class DefaultDelete { + public: + void operator()(intl::ffi::GeckoFluentBundleIterator* aPtr) const { + fluent_bundle_iterator_destroy(aPtr); + } +}; + +template <> +class DefaultDelete { + public: + void operator()( + intl::ffi::GeckoFluentBundleAsyncIteratorWrapper* aPtr) const { + fluent_bundle_async_iterator_destroy(aPtr); + } +}; + +template <> +struct RefPtrTraits { + static void AddRef(const intl::ffi::GeckoL10nRegistry* aPtr) { + intl::ffi::l10nregistry_addref(aPtr); + } + static void Release(const intl::ffi::GeckoL10nRegistry* aPtr) { + intl::ffi::l10nregistry_release(aPtr); + } +}; + +} // namespace mozilla + +#endif diff --git a/intl/l10n/docs/crosschannel/commits.rst b/intl/l10n/docs/crosschannel/commits.rst new file mode 100644 index 0000000000..955baf734f --- /dev/null +++ b/intl/l10n/docs/crosschannel/commits.rst @@ -0,0 +1,33 @@ +Commits and Metadata +==================== + +When creating the commit for a particular revision, we need to find the +revisions on the other branches of cross-channel to unify the created +content with. + +To do so, the cross-channel algorithm keeps track of metadata associated with +a revision in the target repository. This metadata is stored in the commit +message: + +.. code-block:: bash + + X-Channel-Repo: mozilla-central + X-Channel-Converted-Revision: af4a1de0a11cb3afbb7e50bcdd0919f56c23959a + X-Channel-Repo: releases/mozilla-beta + X-Channel-Revision: 65fb3f6bce94f8696e1571c2d48104dbdc0b31e2 + X-Channel-Repo: releases/mozilla-release + X-Channel-Revision: 1c5bf69f887359645f1c3df4de0d0e3caf957e59 + X-Channel-Repo: releases/mozilla-esr68 + X-Channel-Revision: 4cbbc30e1ebc3254ec74dc041aff128c81220507 + +This metadata is appended to the original commit message when committing. +For each branch in the cross-channel configuration we have the name and +a revision. The revision that's currently converted is explicitly highlighted +by the ``-Converted-`` marker. On hg.mozilla.org, those revisions are also +marked up as links, so one can navigate from the converted changeset to the +original patch. + +When starting the update for an incremental graph from the previous section, +the metadata is read from the target repository, and the data for the +currently converted branch is updated for each commit. Each revision in +this metadata then goes into the algorithm to create the unified content. diff --git a/intl/l10n/docs/crosschannel/content.rst b/intl/l10n/docs/crosschannel/content.rst new file mode 100644 index 0000000000..01f5e1ab66 --- /dev/null +++ b/intl/l10n/docs/crosschannel/content.rst @@ -0,0 +1,129 @@ +===================== +Cross-channel Content +===================== + +When creating the actual content, there's a number of questions to answer. + +#. Where to take content from? +#. Which content to take? +#. Where to put the content? +#. What to put into each file? + +Content Sources +--------------- + +The content of each revision in ``gecko-strings`` corresponds to a given +revision in each original repository. For example, we could have + ++------------------+--------------+ +| Repository | Revision | ++==================+==============+ +| mozilla-central | 4c92802939c1 | ++------------------+--------------+ +| mozilla-beta | ace4081e8200 | ++------------------+--------------+ +| mozilla-release | 2cf08fbb92b2 | ++------------------+--------------+ +| mozilla-esr68 | 2cf9e0c91d51 | ++------------------+--------------+ +| comm-central | 3f3fc2c0d804 | ++------------------+--------------+ +| comm-beta | f95a6f4408a3 | ++------------------+--------------+ +| comm-release | dc2694f035fa | ++------------------+--------------+ +| comm-esr68 | d05d4d87d25c | ++------------------+--------------+ + +The assumption is that there's no content that's shared between ``mozilla-*`` and +``comm-*``, so we can just convert one repository and its branches at a time. + +Covered Content +--------------- + +Which content is included in ``gecko-strings`` is +controlled by the project configurations of each product, on each branch. +Currently, those are :file:`browser/locales/l10n.toml` and +:file:`mobile/android/locales/l10n.toml` in ``mozilla-central``. + +Created Content Structure +------------------------- + +The created content is laid out in the directory in the same structure as +the files in ``l10n-central``. The localizable files end up like this: + +.. code-block:: + + browser/ + browser/ + browser.ftl + chrome/ + browser.properties + toolkit/ + toolkit/ + about/aboutAbout.ftl + +This matches the file locations in ``mozilla-central`` with the +:file:`locales/en-US` part dropped. + +The project configuration files are also converted and added to the +created file structure. As they're commonly in the :file:`locales` folder +which we strip, they're added to the dedicated :file:`_configs` folder. + +.. code-block:: bash + + $ ls _configs + browser.toml devtools-client.toml devtools-shared.toml + mobile-android.toml toolkit.toml + + +L10n File Contents +------------------ + +Let's assume we have a file to localize in several revisions with different +content. + +== ======= ==== ======= +ID central beta release +== ======= ==== ======= +a one one one +b two two +c three +d four old old +== ======= ==== ======= + +The algorithm then creates content, taking localizable values from the left-most +branch, where *central* overrides *beta*, and *beta* overrides *release*. This +creates content as follows: + +== ======= +ID content +== ======= +a one +b two +c three +d four +== ======= + +If a file doesn't exist in one of the revisions, that revision is dropped +from the content generation for this particular file. + +.. note:: + + The example of the forth string here highlights the impact that changing + an existing string has. We ship one translation of *four* to central, + beta, and release. That's only a good idea if it doesn't matter which of the + two versions of the English copy got translated. + +Project configurations +---------------------- + +The TOML files for project configuration are processed, but not unified +across branches at this point. + +.. note:: + + The content of the ``-central`` branch determines what's localized + from ``gecko-strings``. Thus that TOML file needs to include all + directories across all branches for now. Removing entries requires + that the content is obsolete on all branches in cross-channel. diff --git a/intl/l10n/docs/crosschannel/index.rst b/intl/l10n/docs/crosschannel/index.rst new file mode 100644 index 0000000000..faa28d6157 --- /dev/null +++ b/intl/l10n/docs/crosschannel/index.rst @@ -0,0 +1,88 @@ +============= +Cross-channel +============= + +Firefox is localized with a process nick-named *cross-channel*. This document +explains both the general idea as well as some technical details of that +process. The gist of it is this: + + We use one localization for all release channels. + +There's a number of upsides to that: + +* Localizers maintain a single source of truth. Localizers can work on Nightly, + while updating Beta, Developer Edition or even Release and ESR. +* Localizers can work on strings at their timing. +* Uplifting string changes has less of an impact on the localization toolchain, + and their impact can be evaluated case by case. + +So the problem at hand is to have one localization source +and use that to build 5 different versions of Firefox. The goal is for that +localization to be as complete as possible for each version. While we do +allow for partial localizations, we don't want to enforce partial translations +on any version. + +The process to tackle these follows these steps: + +* Create resource to localize, ``gecko-strings``. + + * Review updates to that resource in *quarantine*. + * Expose a known good state of that resource to localizers. + +* The actual localization work happens in Pontoon. +* Write localizations back to ``l10n-central``. +* Get localizations into the builds. + +.. digraph:: full_tree + + graph [ rankdir=LR ]; + "m-c" -> "quarantine"; + "m-b" -> "quarantine"; + "m-r" -> "quarantine"; + "c-c" -> "quarantine"; + "c-b" -> "quarantine"; + "c-r" -> "quarantine"; + "quarantine" -> "gecko-strings"; + "gecko-strings" -> "Pontoon"; + "Pontoon" -> "l10n-central"; + "l10n-central" -> "Nightly"; + "l10n-central" -> "Beta"; + "l10n-central" -> "Firefox"; + "l10n-central" -> "Daily"; + "l10n-central" -> "Thunderbird"; + { + rank=same; + "quarantine"; + "gecko-strings"; + } + +.. note:: + + The concept behind the quarantine in the process above is to + protect localizers from churn on strings that have technical + problems. Examples like that could be missing localization notes + or copy that should be improved. + +The resource to localize is a Mercurial repository, unifying +all strings to localize for all covered products and branches. Each revision +of this repository holds all the strings for a particular point in time. + +There's three aspects that we'll want to unify here. + +#. Create a version history that allows the localization team + to learn where strings in the generated repository are coming from. +#. Unify the content across different branches for a single app. +#. Unify different apps, coming from different repositories. + +The last item is the easiest, as ``mozilla-*`` and ``comm-*`` don't share +code or history. Thus, they're converted individually to disjunct directories +and files in the target repository, and the Mercurial history of each is interleaved +in the target history. When parents are needed for one repository, they're +rebased over the commits for the other. + +.. toctree:: + :maxdepth: 1 + + commits + content + repositories diff --git a/intl/l10n/docs/crosschannel/repositories.rst b/intl/l10n/docs/crosschannel/repositories.rst new file mode 100644 index 0000000000..8461b32fbd --- /dev/null +++ b/intl/l10n/docs/crosschannel/repositories.rst @@ -0,0 +1,14 @@ +gecko-strings and Quarantine +============================ + +The actual generation is currently done via `taskcluster cron `_. +The state that is good to use by localizers at large is published at +https://hg.mozilla.org/l10n/gecko-strings/. + +The L10n team is doing a :ref:`review step ` before publishing the strings, and while +that is ongoing, the intermediate state is published to +https://hg.mozilla.org/l10n/gecko-strings-quarantine/. + +The code is in https://hg.mozilla.org/mozilla-central/file/tip/python/l10n/mozxchannel/, +supported as a mach subcommand in https://hg.mozilla.org/mozilla-central/file/tip/tools/compare-locales/mach_commands.py, +as a taskcluster kind in https://hg.mozilla.org/mozilla-central/file/tip/taskcluster/ci/l10n-cross-channel, and scheduled in cron in https://hg.mozilla.org/mozilla-central/file/tip/.cron.yml. diff --git a/intl/l10n/docs/fluent/index.rst b/intl/l10n/docs/fluent/index.rst new file mode 100644 index 0000000000..84103db5e4 --- /dev/null +++ b/intl/l10n/docs/fluent/index.rst @@ -0,0 +1,25 @@ +====== +Fluent +====== + +`Fluent`_ is a localization system developed by Mozilla, which aims to replace +all existing localization models currently used at Mozilla. + +In case of Firefox it directly supersedes DTD and StringBundle systems, providing +a large number of features and improvements over both of them, for developers +and localizers. + +.. toctree:: + :maxdepth: 2 + + tutorial + review + +Other resources: + + * `Fluent Syntax Guide `_ + * `Fluent Wiki `_ + * `Fluent.js Wiki `_ + * `Fluent DOM L10n Tutorial `_ + +.. _Fluent: http://projectfluent.org/ diff --git a/intl/l10n/docs/fluent/review.rst b/intl/l10n/docs/fluent/review.rst new file mode 100644 index 0000000000..83d65ebed9 --- /dev/null +++ b/intl/l10n/docs/fluent/review.rst @@ -0,0 +1,303 @@ +.. role:: bash(code) + :language: bash + +.. role:: js(code) + :language: javascript + +=============================== +Guidelines for Fluent Reviewers +=============================== + +This document is intended as a guideline for developers and reviewers when +working with FTL (Fluent) files. As such, it’s not meant to replace the +`existing extensive documentation`__ about Fluent. + +__ ./tutorial.html + +`Herald`_ is used to set the group `fluent-reviewers`_ as blocking reviewer for +any patch modifying FTL files committed to Phabricator. The person from this +group performing the review will have to manually set other reviewers as +blocking, if the original developer didn’t originally do it. + + +.. hint:: + + In case of doubt, you should always reach out to the l10n team for + clarifications. + + +Message Identifiers +=================== + +While in Fluent it’s possible to use both lowercase and uppercase characters in +message identifiers, the naming convention in Gecko is to use lowercase and +hyphens (*kebab-case*), avoiding CamelCase and underscores. For example, +:js:`allow-button` should be preferred to :js:`allow_button` or +:js:`allowButton`, unless there are technically constraints – like identifiers +generated at run-time from external sources – that make this impractical. + +When importing multiple FTL files, all messages share the same scope in the +Fluent bundle. For that reason, it’s suggested to add scope to the message +identifier itself: using :js:`cancel` as an identifier increases the chances of +having a conflict, :js:`save-dialog-cancel-button` would make it less likely. + +Message identifiers are also used as the ultimate fall back in case of run-time +errors. Having a descriptive message ID would make such fall back more useful +for the user. + +Comments +======== + +When a message includes placeables (variables), there should always be a +comment explaining the format of the variable, and what kind of content it will +be replaced with. This is the format suggested for such comments: + + +.. code-block:: fluent + + # This string is used on a new line below the add-on name + # Variables: + # $name (String) - Add-on author name + cfr-doorhanger-extension-author = by { $name } + + +By default, a comment is bound to the message immediately following it. Fluent +supports both `file-level and group-level comments`__. Be aware that a group +comment will apply to all messages following that comment until the end of the +file. If that shouldn’t be the case, you’ll need to “reset” the group comment, +by adding an empty one (:js:`##`), or moving the section of messages at the end +of the file. + +__ https://projectfluent.org/fluent/guide/comments.html + +Comments are fundamental for localizers, since they don’t see the file as a +whole, or changes as a fragment of a larger patch. Their work happens on a +message at a time, and the context is only provided by comments. + +License headers are standalone comments, that is, a single :js:`#` as prefix, +and the comment is followed by at least one empty line. + +Changes to Existing Messages +============================ + +You must update the message identifier if: + +- The meaning of the sentence has changed. +- You’re changing the morphology of the message, by adding or removing attributes. + +Messages are identified in the entire localization toolchain by their ID. For +this reason, there’s no need to change attribute names. + +If your changes are relevant only for English — for example, to correct a +typographical error or to make letter case consistent — then there is generally +no need to update the message identifier. + +There is a grey area between needing a new ID or not. In some cases, it will be +necessary to look at all the existing translations to determine if a new ID +would be beneficial. You should always reach out to the l10n team in case of +doubt. + +Changing the message ID will invalidate the existing translation, the new +message will be reported as missing in all tools, and localizers will have to +retranslate it. This is the only reliable method to ensure that localizers +update existing localizations, and run-time stop using obsolete translations. + +You must also update all instances where that message identifier is used in the +source code, including localization comments. + +Non-text Elements in Messages +============================= + +When a message includes non text-elements – like anchors or images – make sure +that they have a :js:`data-l10n-name` associated to them. Additional +attributes, like the URL for an anchor or CSS classes, should not be exposed +for localization in the FTL file. More details can be found in `this page`__ +dedicated to DOM overlays. + +__ https://github.com/projectfluent/fluent.js/wiki/DOM-Overlays#text-level-elements + +This information is not relevant if your code is using `fluent-react`_, where +DOM overlays `work differently`__. + +__ https://github.com/projectfluent/fluent.js/wiki/React-Overlays + +Message References +================== + +Consider the following example: + + +.. code-block:: fluent + + newtab-search-box-search-the-web-text = Search the Web + newtab-search-box-search-the-web-input = + .placeholder = { newtab-search-box-search-the-web-text } + .title = { newtab-search-box-search-the-web-text } + + +This might seem to reduce the work for localizers, but it actually doesn’t +help: + +- A change to the referenced message (:js:`newtab-search-box-search-the-web-text`) + would require a new ID also for all messages referencing it. +- Translation memory can help with matching text, not with message references. + +On the other hand, this approach is helpful if, for example, you want to +reference another element of the UI in your message: + + +.. code-block:: fluent + + help-button = Help + help-explanation = Click the { help-button} to access support + + +This enforces consistency and, if :js:`help-button` changes, all other messages +will need to be updated anyway. + +Terms +===== + +Fluent supports a specific type of message, called `term`_. Terms are similar +to regular messages but they can only be used as references in other messages. +They are best used to define vocabulary and glossary items which can be used +consistently across the localization of the entire product. + +Terms are typically used for brand names, like :js:`Firefox` or :js:`Mozilla`: +it allows to have them in one easily identifiable place, and raise warnings +when a localization is not using them. It helps enforcing consistency and brand +protection. If you simply need to reference a message from another message, you +don’t need a term: cross references between messages are allowed, but they +should not be abused, as already described. + +Variants and plurals +==================== + +Consider the following example: + + +.. code-block:: fluent + + items-selected = + { $num -> + [0] Select items. + [one] One item selected. + *[other] { $num } items selected. + } + + +In this example, there’s no guarantee that all localizations will have this +variant covered, since variants are private by design. The correct approach for +the example would be to have a separate message for the :js:`0` case: + + +.. code-block:: fluent + + # Separate messages which serve different purposes. + items-select = Select items + # The default variant works for all values of the selector. + items-selected = + { $num -> + [one] One item selected. + *[other] { $num } items selected. + } + + +As a rule of thumb: + +- Use variants only if the default variant makes sense for all possible values + of the selector. +- The code shouldn’t depend on the availability of a specific variant. + +More examples about selector and variant abuses can be found in `this wiki`__. + +__ https://github.com/projectfluent/fluent/wiki/Good-Practices-for-Developers#prefer-separate-messages-over-variants-for-ui-logic + +In general, also avoid putting a selector in the middle of a sentence, like in +the example below: + + +.. code-block:: fluent + + items-selected = + { $num -> + [one] One item. + *[other] { $num } items + } selected. + + +:js:`1` should only be used in case you want to cover the literal number. If +it’s a standard plural, you should use the :js:`one` category for singular. +Also make sure to always pass the variable to these messages as a number, not +as a string. + +Access Keys +=========== + +The following is a simple potential example of an access key: + +.. code-block:: fluent + + example-menu-item = + .label = Menu Item + .accesskey = M + +Access keys are used in menus in order to help provide easy keyboard shortcut access. They +are useful for both power users, and for users who have accessibility needs. It is +helpful to first read the `Access keys`__ guide in the Windows Developer documentation, +as it outlines the best practices for Windows applications. + +__ https://docs.microsoft.com/en-us/windows/uwp/design/input/access-keys + +There are some differences between operating systems. Linux mostly follows the same +practices as Windows. However, macOS in general does not have good support for accesskeys, +especially in menus. + +When choosing an access key, it's important that it's unique relative to the current level +of UI. It's preferable to avoid letters with descending parts, such as :code:`g`, +:code:`j`, :code:`p`, and :code:`q` as these will not be underlined nicely in Windows or +Linux. Other problematic characters are ones which are narrow, such as :code:`l`, +:code:`i` and :code:`I`. The underline may not be as visible as other letters in +sans-serif fonts. + +Linter +====== + +:bash:`mach lint` includes a :ref:`l10n linter `, called :bash:`moz-l10n-lint`. It +can be run locally by developers but also runs on Treeherder: in the Build +Status section of the diff on Phabricator, open the Treeherder Jobs link and +look for the :js:`l1nt` job. + +Besides displaying errors and warnings due to syntax errors, it’s particularly +important because it also checks for message changes without new IDs, and +conflicts with the cross-channel repository used to ship localized versions of +Firefox. + + +.. warning:: + + Currently, there’s an `issue`__ preventing warnings to be displayed in + Phabricator. Checks can be run locally using :bash:`./mach lint -l l10n -W`. + + __ https://github.com/mozilla/code-review/issues/32 + + +Migrating Strings From Legacy or Fluent Files +============================================= + +If a patch is moving legacy strings (.properties, .DTD) to Fluent, it should +also include a recipe to migrate existing strings to FTL messages. The same is +applicable if a patch moves existing Fluent messages to a different file, or +changes the morphology of existing messages without actual changes to the +content. + +Documentation on how to write and test migration recipes is available in `this +page`__. + +__ ./fluent_migrations.html + + +.. _Herald: https://phabricator.services.mozilla.com/herald/ +.. _fluent-reviewers: https://phabricator.services.mozilla.com/tag/fluent-reviewers/ +.. _fluent-react: https://github.com/projectfluent/fluent.js/wiki/React-Bindings +.. _term: https://projectfluent.org/fluent/guide/terms.html diff --git a/intl/l10n/docs/fluent/tutorial.rst b/intl/l10n/docs/fluent/tutorial.rst new file mode 100644 index 0000000000..1312efd448 --- /dev/null +++ b/intl/l10n/docs/fluent/tutorial.rst @@ -0,0 +1,750 @@ +.. role:: html(code) + :language: html + +.. role:: js(code) + :language: javascript + +============================= +Fluent for Firefox Developers +============================= + + +This tutorial is intended for Firefox engineers already familiar with the previous +localization systems offered by Gecko - `DTD`_ and `StringBundle`_ - and assumes +prior experience with those systems. + +For a more hands-on tutorial of understanding Fluent from the ground up, try +following the `Fluent DOMLocalization Tutorial`__, which provides some background on +how Fluent works and walks you through creating a basic web project from scratch that +uses Fluent for localization. + +__ https://projectfluent.org/dom-l10n-documentation/ + +Using Fluent in Gecko +===================== + +`Fluent`_ is a modern localization system introduced into +the Gecko platform with a focus on quality, performance, maintenance and completeness. + +The legacy DTD system is deprecated, and Fluent should be used where possible. + +Getting a Review +---------------- + +If you work on any patch that touches FTL files, you'll need to get a review +from `fluent-reviewers`__. There's a Herald hook that automatically sets +that group as a blocking reviewer. + +__ https://phabricator.services.mozilla.com/tag/fluent-reviewers/ + +Guidelines for the review process are available `here`__. + +__ ./fluent_review.html + +To lighten the burden on reviewers, please take a moment to review some +best practices before submitting your patch for review. + +- `ProjectFluent Good Practices for Developers`_ +- `Mozilla Localization Best Practices For Developers`_ + +.. _ProjectFluent Good Practices for Developers: https://github.com/projectfluent/fluent/wiki/Good-Practices-for-Developers +.. _Mozilla Localization Best Practices For Developers: https://mozilla-l10n.github.io/documentation/localization/dev_best_practices.html + +Major Benefits +============== + +Fluent `ties tightly`__ into the domain of internationalization +through `Unicode`_, `CLDR`_ and `ICU`_. + +__ https://github.com/projectfluent/fluent/wiki/Fluent-and-Standards + +More specifically, the most observable benefits for each group of consumers are + + +Developers +---------- + + - Support for XUL, XHTML, HTML, Web Components, React, JS, Python and Rust + - Strings are available in a single, unified localization context available for both DOM and runtime code + - Full internationalization (i18n) support: date and time formatting, number formatting, plurals, genders etc. + - Strong focus on `declarative API via DOM attributes`__ + - Extensible with custom formatters, Mozilla-specific APIs etc. + - `Separation of concerns`__: localization details, and the added complexity of some languages, don't leak onto the source code and are no concern for developers + - Compound messages link a single translation unit to a single UI element + - `DOM Overlays`__ allow for localization of DOM fragments + - Simplified build system model + - No need for pre-processing instructions + - Support for pseudolocalization + +__ https://github.com/projectfluent/fluent/wiki/Get-Started +__ https://github.com/projectfluent/fluent/wiki/Design-Principles +__ https://github.com/projectfluent/fluent.js/wiki/DOM-Overlays + + +Product Quality +------------------ + + - A robust, multilevel, `error fallback system`__ prevents XML errors and runtime errors + - Simplified l10n API reduces the amount of l10n specific code and resulting bugs + - Runtime localization allows for dynamic language changes and updates over-the-air + - DOM Overlays increase localization security + +__ https://github.com/projectfluent/fluent/wiki/Error-Handling + + +Fluent Translation List - FTL +============================= + +Fluent introduces a file format designed specifically for easy readability +and the localization features offered by the system. + +At first glance the format is a simple key-value store. It may look like this: + +.. code-block:: fluent + + home-page-header = Home Page + + # The label of a button opening a new tab + new-tab-open = Open New Tab + +But the FTL file format is significantly more powerful and the additional features +quickly add up. In order to familiarize yourself with the basic features, +consider reading through the `Fluent Syntax Guide`_ to understand +a more complex example like: + +.. code-block:: fluent + + ### These messages correspond to security and privacy user interface. + ### + ### Please choose simple and non-threatening language when localizing + ### to help user feel in control when interacting with the UI. + + ## General Section + + -brand-short-name = Firefox + .gender = masculine + + pref-pane = + .title = + { PLATFORM() -> + [windows] Options + *[other] Preferences + } + .accesskey = C + + # Variables: + # $tabCount (Number) - number of container tabs to be closed + containers-disable-alert-ok-button = + { $tabCount -> + [one] Close { $tabCount } Container Tab + *[other] Close { $tabCount } Container Tabs + } + + update-application-info = + You are using { -brand-short-name } Version: { $version }. + Please read the privacy policy. + +The above, of course, is a particular selection of complex strings intended to exemplify +the new features and concepts introduced by Fluent. + +.. important:: + + While in Fluent it’s possible to use both lowercase and uppercase characters in message + identifiers, the naming convention in Gecko is to use lowercase and hyphens, avoiding + CamelCase and underscores. For example, `allow-button` should be preferred to + `allow_button` or `allowButton`, unless there are technically constraints – like + identifiers generated at run-time from external sources – that make this impractical. + +In order to ensure the quality of the output, a lot of checks and tooling +is part of the build system. +`Pontoon`_, the main localization tool used to translate Firefox, also supports +Fluent and its features to help localizers in their work. + + +.. _fluent-tutorial-social-contract: + +Social Contract +=============== + +Fluent uses the concept of a `social contract` between developer and localizers. +This contract is established by the selection of a unique identifier, called :js:`l10n-id`, +which carries a promise of being used in a particular place to carry a particular meaning. + +The use of unique identifiers is shared with legacy localization systems in +Firefox. + +.. important:: + + An important part of the contract is that the developer commits to treat the + localization output as `opaque`. That means that no concatenations, replacements + or splitting should happen after the translation is completed to generate the + desired output. + +In return, localizers enter the social contract by promising to provide an accurate +and clean translation of the messages that match the request. + +In Fluent, the developer is not to be bothered with inner logic and complexity that the +localization will use to construct the response. Whether `declensions`__ or other +variant selection techniques are used is up to a localizer and their particular translation. +From the developer perspective, Fluent returns a final string to be presented to +the user, with no l10n logic required in the running code. + +__ https://en.wikipedia.org/wiki/Declension + + +Markup Localization +=================== + +To localize an element in Fluent, the developer adds a new message to +an FTL file and then has to associate an :js:`l10n-id` with the element +by defining a :js:`data-l10n-id` attribute: + +.. code-block:: html + +

+ +