summaryrefslogtreecommitdiffstats
path: root/i18npool/source/breakiterator/data/sent.txt
diff options
context:
space:
mode:
Diffstat (limited to 'i18npool/source/breakiterator/data/sent.txt')
-rw-r--r--i18npool/source/breakiterator/data/sent.txt128
1 files changed, 128 insertions, 0 deletions
diff --git a/i18npool/source/breakiterator/data/sent.txt b/i18npool/source/breakiterator/data/sent.txt
new file mode 100644
index 000000000..7fada89e6
--- /dev/null
+++ b/i18npool/source/breakiterator/data/sent.txt
@@ -0,0 +1,128 @@
+#
+# Copyright (C) 2002-2006, International Business Machines Corporation and others.
+# All Rights Reserved.
+#
+# file: sent.txt
+#
+# ICU Sentence Break Rules
+# See Unicode Standard Annex #29.
+# These rules are based on SA 29 version 5.0.0
+# Includes post 5.0 changes to treat Japanese half width voicing marks
+# as Grapheme Extend.
+#
+
+
+$VoiceMarks = [\uff9e\uff9f];
+$Thai = [:Script = Thai:];
+
+#
+# Character categories as defined in TR 29
+#
+$Sep = [\p{Sentence_Break = Sep}];
+$Format = [\p{Sentence_Break = Format}];
+$Sp = [\p{Sentence_Break = Sp}];
+$Lower = [\p{Sentence_Break = Lower}];
+$Upper = [\p{Sentence_Break = Upper}];
+$OLetter = [\p{Sentence_Break = OLetter}-$VoiceMarks];
+$Numeric = [\p{Sentence_Break = Numeric}];
+$ATerm = [\p{Sentence_Break = ATerm}];
+$STerm = [\p{Sentence_Break = STerm}];
+$Close = [\p{Sentence_Break = Close}];
+
+#
+# Define extended forms of the character classes,
+# incorporate grapheme cluster + format chars.
+# Rules 4 and 5.
+
+
+$CR = \u000d;
+$LF = \u000a;
+$Extend = [[:Grapheme_Extend = TRUE:]$VoiceMarks];
+
+$SpEx = $Sp ($Extend | $Format)*;
+$LowerEx = $Lower ($Extend | $Format)*;
+$UpperEx = $Upper ($Extend | $Format)*;
+$OLetterEx = $OLetter ($Extend | $Format)*;
+$NumericEx = $Numeric ($Extend | $Format)*;
+$ATermEx = $ATerm ($Extend | $Format)*;
+$STermEx = $STerm ($Extend | $Format)*;
+$CloseEx = $Close ($Extend | $Format)*;
+
+
+## -------------------------------------------------
+
+!!chain;
+!!forward;
+
+# Rule 3 - break after separators. Keep CR/LF together.
+#
+$CR $LF;
+
+$LettersEx = [$OLetter $Upper $Lower $Numeric $Close $STerm] ($Extend | $Format)*;
+$LettersEx* $Thai $LettersEx* ($ATermEx | $SpEx)*;
+
+# Rule 4 - Break after $Sep.
+# Rule 5 - Ignore $Format and $Extend
+#
+[^$Sep]? ($Extend | $Format)*;
+
+
+# Rule 6
+$ATermEx $NumericEx;
+
+# Rule 7
+$UpperEx $ATermEx $UpperEx;
+
+#Rule 8
+# Note: follows errata for Unicode 5.0 boundary rules.
+$NotLettersEx = [^$OLetter $Upper $Lower $Sep $ATerm $STerm] ($Extend | $Format)*;
+$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower;
+
+# Rule 8a
+($STermEx | $ATermEx) $CloseEx* $SpEx* ($STermEx | $ATermEx);
+
+#Rule 9, 10, 11
+($STermEx | $ATermEx) $CloseEx* $SpEx* $Sep?;
+
+#Rule 12
+[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend $Thai]{bof}] ($Extend | $Format | $Close | $Sp)* [^$Thai];
+[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep{eof}] | $CR $LF){100};
+
+## -------------------------------------------------
+
+!!reverse;
+
+$SpEx_R = ($Extend | $Format)* $Sp;
+$ATermEx_R = ($Extend | $Format)* $ATerm;
+$STermEx_R = ($Extend | $Format)* $STerm;
+$CloseEx_R = ($Extend | $Format)* $Close;
+
+#
+# Reverse rules.
+# For now, use the old style inexact reverse rules, which are easier
+# to write, but less efficient.
+# TODO: exact reverse rules. It appears that exact reverse rules
+# may require improving support for look-ahead breaks in the
+# builder. Needs more investigation.
+#
+
+[{bof}] (.? | $LF $CR) [^$Sep]* [$Sep {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*;
+#.*;
+
+# Explanation for this rule:
+#
+# It needs to back over
+# The $Sep at which we probably begin
+# All of the non $Sep chars leading to the preceding $Sep
+# The preceding $Sep, which will be the second one that the rule matches.
+# Any immediately preceding STerm or ATerm sequences. We need to see these
+# to get the correct rule status when moving forwards again.
+#
+# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match
+# the entire string.
+#
+# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be
+# at the beginning of the string at this point, and we don't want to fail.
+# Can only use {eof} once, and it is used later.
+#
+