diff options
Diffstat (limited to 'i18npool/source/breakiterator/data/sent.txt')
| -rw-r--r-- | i18npool/source/breakiterator/data/sent.txt | 128 |
1 files changed, 128 insertions, 0 deletions
diff --git a/i18npool/source/breakiterator/data/sent.txt b/i18npool/source/breakiterator/data/sent.txt new file mode 100644 index 000000000..7fada89e6 --- /dev/null +++ b/i18npool/source/breakiterator/data/sent.txt @@ -0,0 +1,128 @@ +# +# Copyright (C) 2002-2006, International Business Machines Corporation and others. +# All Rights Reserved. +# +# file: sent.txt +# +# ICU Sentence Break Rules +# See Unicode Standard Annex #29. +# These rules are based on SA 29 version 5.0.0 +# Includes post 5.0 changes to treat Japanese half width voicing marks +# as Grapheme Extend. +# + + +$VoiceMarks = [\uff9e\uff9f]; +$Thai = [:Script = Thai:]; + +# +# Character categories as defined in TR 29 +# +$Sep = [\p{Sentence_Break = Sep}]; +$Format = [\p{Sentence_Break = Format}]; +$Sp = [\p{Sentence_Break = Sp}]; +$Lower = [\p{Sentence_Break = Lower}]; +$Upper = [\p{Sentence_Break = Upper}]; +$OLetter = [\p{Sentence_Break = OLetter}-$VoiceMarks]; +$Numeric = [\p{Sentence_Break = Numeric}]; +$ATerm = [\p{Sentence_Break = ATerm}]; +$STerm = [\p{Sentence_Break = STerm}]; +$Close = [\p{Sentence_Break = Close}]; + +# +# Define extended forms of the character classes, +# incorporate grapheme cluster + format chars. +# Rules 4 and 5. + + +$CR = \u000d; +$LF = \u000a; +$Extend = [[:Grapheme_Extend = TRUE:]$VoiceMarks]; + +$SpEx = $Sp ($Extend | $Format)*; +$LowerEx = $Lower ($Extend | $Format)*; +$UpperEx = $Upper ($Extend | $Format)*; +$OLetterEx = $OLetter ($Extend | $Format)*; +$NumericEx = $Numeric ($Extend | $Format)*; +$ATermEx = $ATerm ($Extend | $Format)*; +$STermEx = $STerm ($Extend | $Format)*; +$CloseEx = $Close ($Extend | $Format)*; + + +## ------------------------------------------------- + +!!chain; +!!forward; + +# Rule 3 - break after separators. Keep CR/LF together. +# +$CR $LF; + +$LettersEx = [$OLetter $Upper $Lower $Numeric $Close $STerm] ($Extend | $Format)*; +$LettersEx* $Thai $LettersEx* ($ATermEx | $SpEx)*; + +# Rule 4 - Break after $Sep. +# Rule 5 - Ignore $Format and $Extend +# +[^$Sep]? ($Extend | $Format)*; + + +# Rule 6 +$ATermEx $NumericEx; + +# Rule 7 +$UpperEx $ATermEx $UpperEx; + +#Rule 8 +# Note: follows errata for Unicode 5.0 boundary rules. +$NotLettersEx = [^$OLetter $Upper $Lower $Sep $ATerm $STerm] ($Extend | $Format)*; +$ATermEx $CloseEx* $SpEx* $NotLettersEx* $Lower; + +# Rule 8a +($STermEx | $ATermEx) $CloseEx* $SpEx* ($STermEx | $ATermEx); + +#Rule 9, 10, 11 +($STermEx | $ATermEx) $CloseEx* $SpEx* $Sep?; + +#Rule 12 +[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend $Thai]{bof}] ($Extend | $Format | $Close | $Sp)* [^$Thai]; +[[^$STerm $ATerm $Close $Sp $Sep $Format $Extend]{bof}] ($Extend | $Format | $Close | $Sp)* ([$Sep{eof}] | $CR $LF){100}; + +## ------------------------------------------------- + +!!reverse; + +$SpEx_R = ($Extend | $Format)* $Sp; +$ATermEx_R = ($Extend | $Format)* $ATerm; +$STermEx_R = ($Extend | $Format)* $STerm; +$CloseEx_R = ($Extend | $Format)* $Close; + +# +# Reverse rules. +# For now, use the old style inexact reverse rules, which are easier +# to write, but less efficient. +# TODO: exact reverse rules. It appears that exact reverse rules +# may require improving support for look-ahead breaks in the +# builder. Needs more investigation. +# + +[{bof}] (.? | $LF $CR) [^$Sep]* [$Sep {eof}] ($SpEx_R* $CloseEx_R* ($STermEx_R | $ATermEx_R))*; +#.*; + +# Explanation for this rule: +# +# It needs to back over +# The $Sep at which we probably begin +# All of the non $Sep chars leading to the preceding $Sep +# The preceding $Sep, which will be the second one that the rule matches. +# Any immediately preceding STerm or ATerm sequences. We need to see these +# to get the correct rule status when moving forwards again. +# +# [{bof}] inhibit rule chaining. Without this, rule would loop on itself and match +# the entire string. +# +# (.? | $LF $CR) Match one $Sep instance. Use .? rather than $Sep because position might be +# at the beginning of the string at this point, and we don't want to fail. +# Can only use {eof} once, and it is used later. +# + |