summaryrefslogtreecommitdiffstats
path: root/intl/lwbrk/Segmenter.cpp
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-07 17:32:43 +0000
commit6bf0a5cb5034a7e684dcc3500e841785237ce2dd (patch)
treea68f146d7fa01f0134297619fbe7e33db084e0aa /intl/lwbrk/Segmenter.cpp
parentInitial commit. (diff)
downloadthunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.tar.xz
thunderbird-6bf0a5cb5034a7e684dcc3500e841785237ce2dd.zip
Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'intl/lwbrk/Segmenter.cpp')
-rw-r--r--intl/lwbrk/Segmenter.cpp260
1 files changed, 260 insertions, 0 deletions
diff --git a/intl/lwbrk/Segmenter.cpp b/intl/lwbrk/Segmenter.cpp
new file mode 100644
index 0000000000..53d87336a3
--- /dev/null
+++ b/intl/lwbrk/Segmenter.cpp
@@ -0,0 +1,260 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* Classes to iterate over grapheme, word, sentence, or line. */
+
+#include "mozilla/intl/Segmenter.h"
+
+#include "mozilla/intl/LineBreaker.h"
+#include "mozilla/intl/WordBreaker.h"
+#include "mozilla/intl/UnicodeProperties.h"
+#include "nsUnicodeProperties.h"
+#include "nsCharTraits.h"
+
+using namespace mozilla::unicode;
+
+namespace mozilla::intl {
+
+SegmentIteratorUtf16::SegmentIteratorUtf16(Span<const char16_t> aText)
+ : mText(aText) {}
+
+Maybe<uint32_t> SegmentIteratorUtf16::Seek(uint32_t aPos) {
+ if (mPos < aPos) {
+ mPos = aPos;
+ }
+ return Next();
+}
+
+LineBreakIteratorUtf16::LineBreakIteratorUtf16(Span<const char16_t> aText,
+ const LineBreakOptions& aOptions)
+ : SegmentIteratorUtf16(aText), mOptions(aOptions) {}
+
+Maybe<uint32_t> LineBreakIteratorUtf16::Next() {
+ const int32_t nextPos =
+ LineBreaker::Next(mText.Elements(), mText.Length(), mPos);
+ if (nextPos == NS_LINEBREAKER_NEED_MORE_TEXT) {
+ return Nothing();
+ }
+ mPos = nextPos;
+ return Some(mPos);
+}
+
+WordBreakIteratorUtf16::WordBreakIteratorUtf16(Span<const char16_t> aText)
+ : SegmentIteratorUtf16(aText) {}
+
+Maybe<uint32_t> WordBreakIteratorUtf16::Next() {
+ const int32_t nextPos =
+ WordBreaker::Next(mText.Elements(), mText.Length(), mPos);
+ if (nextPos == NS_WORDBREAKER_NEED_MORE_TEXT) {
+ return Nothing();
+ }
+ mPos = nextPos;
+ return Some(mPos);
+}
+
+GraphemeClusterBreakIteratorUtf16::GraphemeClusterBreakIteratorUtf16(
+ Span<const char16_t> aText)
+ : SegmentIteratorUtf16(aText) {}
+
+enum HSType {
+ HST_NONE = U_HST_NOT_APPLICABLE,
+ HST_L = U_HST_LEADING_JAMO,
+ HST_V = U_HST_VOWEL_JAMO,
+ HST_T = U_HST_TRAILING_JAMO,
+ HST_LV = U_HST_LV_SYLLABLE,
+ HST_LVT = U_HST_LVT_SYLLABLE
+};
+
+static HSType GetHangulSyllableType(uint32_t aCh) {
+ return HSType(UnicodeProperties::GetIntPropertyValue(
+ aCh, UnicodeProperties::IntProperty::HangulSyllableType));
+}
+
+Maybe<uint32_t> GraphemeClusterBreakIteratorUtf16::Next() {
+ const auto len = mText.Length();
+ if (mPos >= len) {
+ // The iterator has already reached the end.
+ return Nothing();
+ }
+
+ uint32_t ch = mText[mPos++];
+
+ if (mPos < len && NS_IS_SURROGATE_PAIR(ch, mText[mPos])) {
+ ch = SURROGATE_TO_UCS4(ch, mText[mPos++]);
+ } else if ((ch & ~0xff) == 0x1100 || (ch >= 0xa960 && ch <= 0xa97f) ||
+ (ch >= 0xac00 && ch <= 0xd7ff)) {
+ // Handle conjoining Jamo that make Hangul syllables
+ HSType hangulState = GetHangulSyllableType(ch);
+ while (mPos < len) {
+ ch = mText[mPos];
+ HSType hangulType = GetHangulSyllableType(ch);
+ switch (hangulType) {
+ case HST_L:
+ case HST_LV:
+ case HST_LVT:
+ if (hangulState == HST_L) {
+ hangulState = hangulType;
+ mPos++;
+ continue;
+ }
+ break;
+ case HST_V:
+ if ((hangulState != HST_NONE) && (hangulState != HST_T) &&
+ (hangulState != HST_LVT)) {
+ hangulState = hangulType;
+ mPos++;
+ continue;
+ }
+ break;
+ case HST_T:
+ if (hangulState != HST_NONE && hangulState != HST_L) {
+ hangulState = hangulType;
+ mPos++;
+ continue;
+ }
+ break;
+ default:
+ break;
+ }
+ break;
+ }
+ }
+
+ const uint32_t kVS16 = 0xfe0f;
+ const uint32_t kZWJ = 0x200d;
+ // UTF-16 surrogate values for Fitzpatrick type modifiers
+ const uint32_t kFitzpatrickHigh = 0xD83C;
+ const uint32_t kFitzpatrickLowFirst = 0xDFFB;
+ const uint32_t kFitzpatrickLowLast = 0xDFFF;
+
+ // Checking the emoji-presentation property of the base character is a bit
+ // expensive, so we do it lazily.
+ enum class EmojiStatus : uint8_t {
+ No,
+ Yes,
+ Unknown,
+ } baseIsEmojiStatus = EmojiStatus::Unknown;
+
+ // Remember the base character and the position of the next, in case we need
+ // to evaluate its emoji status.
+ uint32_t baseCh = ch;
+ uint32_t afterBase = mPos;
+
+ auto isFitzpatrickModifierAt = [&](uint32_t aPos) -> bool {
+ return aPos + 1 < len && mText[aPos] == kFitzpatrickHigh &&
+ mText[aPos + 1] >= kFitzpatrickLowFirst &&
+ mText[aPos + 1] <= kFitzpatrickLowLast;
+ };
+
+ auto baseIsEmoji = [&]() -> bool {
+ if (baseIsEmojiStatus == EmojiStatus::Unknown) {
+ auto basePresentation = GetEmojiPresentation(baseCh);
+ baseIsEmojiStatus =
+ basePresentation == EmojiDefault ||
+ (basePresentation == TextDefault &&
+ ((afterBase < len && mText[afterBase] == kVS16) ||
+ isFitzpatrickModifierAt(afterBase)))
+ ? EmojiStatus::Yes
+ : EmojiStatus::No;
+ }
+ return baseIsEmojiStatus == EmojiStatus::Yes;
+ };
+
+ bool prevWasZwj = false;
+
+ while (mPos < len) {
+ ch = mText[mPos];
+ size_t chLen = 1;
+
+ // Check for surrogate pairs; note that isolated surrogates will just
+ // be treated as generic (non-cluster-extending) characters here,
+ // which is fine for cluster-iterating purposes
+ if (mPos < len - 1 && NS_IS_SURROGATE_PAIR(ch, mText[mPos + 1])) {
+ ch = SURROGATE_TO_UCS4(ch, mText[mPos + 1]);
+ chLen = 2;
+ }
+
+ bool extendCluster =
+ IsClusterExtender(ch) ||
+ (prevWasZwj && baseIsEmoji() &&
+ ((GetEmojiPresentation(ch) == EmojiDefault) ||
+ (GetEmojiPresentation(ch) == TextDefault && mPos + chLen < len &&
+ mText[mPos + chLen] == kVS16)));
+ if (!extendCluster) {
+ break;
+ }
+
+ prevWasZwj = (ch == kZWJ);
+ mPos += chLen;
+ }
+
+ MOZ_ASSERT(mPos <= len, "Next() has overshot the string!");
+ return Some(mPos);
+}
+
+GraphemeClusterBreakReverseIteratorUtf16::
+ GraphemeClusterBreakReverseIteratorUtf16(Span<const char16_t> aText)
+ : SegmentIteratorUtf16(aText) {
+ mPos = mText.Length();
+}
+
+Maybe<uint32_t> GraphemeClusterBreakReverseIteratorUtf16::Next() {
+ if (mPos == 0) {
+ return Nothing();
+ }
+
+ uint32_t ch;
+ do {
+ ch = mText[--mPos];
+
+ if (mPos > 0 && NS_IS_SURROGATE_PAIR(mText[mPos - 1], ch)) {
+ ch = SURROGATE_TO_UCS4(mText[--mPos], ch);
+ }
+
+ if (!IsClusterExtender(ch)) {
+ break;
+ }
+ } while (mPos > 0);
+
+ // XXX May need to handle conjoining Jamo
+
+ return Some(mPos);
+}
+
+Maybe<uint32_t> GraphemeClusterBreakReverseIteratorUtf16::Seek(uint32_t aPos) {
+ if (mPos > aPos) {
+ mPos = aPos;
+ }
+ return Next();
+}
+
+Result<UniquePtr<Segmenter>, ICUError> Segmenter::TryCreate(
+ Span<const char> aLocale, const SegmenterOptions& aOptions) {
+ if (aOptions.mGranularity == SegmenterGranularity::Sentence) {
+ // Grapheme and Sentence iterator are not yet implemented.
+ return Err(ICUError::InternalError);
+ }
+ return MakeUnique<Segmenter>(aLocale, aOptions);
+}
+
+UniquePtr<SegmentIteratorUtf16> Segmenter::Segment(
+ Span<const char16_t> aText) const {
+ switch (mOptions.mGranularity) {
+ case SegmenterGranularity::Grapheme:
+ return MakeUnique<GraphemeClusterBreakIteratorUtf16>(aText);
+ case SegmenterGranularity::Sentence:
+ MOZ_ASSERT_UNREACHABLE("Unimplemented yet!");
+ return nullptr;
+ case SegmenterGranularity::Word:
+ return MakeUnique<WordBreakIteratorUtf16>(aText);
+ case SegmenterGranularity::Line:
+ return MakeUnique<LineBreakIteratorUtf16>(aText);
+ }
+ MOZ_ASSERT_UNREACHABLE("All granularities must be handled!");
+ return nullptr;
+}
+
+} // namespace mozilla::intl