diff options
Diffstat (limited to 'src/unicode/letter.go')
-rw-r--r-- | src/unicode/letter.go | 371 |
1 files changed, 371 insertions, 0 deletions
diff --git a/src/unicode/letter.go b/src/unicode/letter.go new file mode 100644 index 0000000..f3f8e52 --- /dev/null +++ b/src/unicode/letter.go @@ -0,0 +1,371 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package unicode provides data and functions to test some properties of +// Unicode code points. +package unicode + +const ( + MaxRune = '\U0010FFFF' // Maximum valid Unicode code point. + ReplacementChar = '\uFFFD' // Represents invalid code points. + MaxASCII = '\u007F' // maximum ASCII value. + MaxLatin1 = '\u00FF' // maximum Latin-1 value. +) + +// RangeTable defines a set of Unicode code points by listing the ranges of +// code points within the set. The ranges are listed in two slices +// to save space: a slice of 16-bit ranges and a slice of 32-bit ranges. +// The two slices must be in sorted order and non-overlapping. +// Also, R32 should contain only values >= 0x10000 (1<<16). +type RangeTable struct { + R16 []Range16 + R32 []Range32 + LatinOffset int // number of entries in R16 with Hi <= MaxLatin1 +} + +// Range16 represents of a range of 16-bit Unicode code points. The range runs from Lo to Hi +// inclusive and has the specified stride. +type Range16 struct { + Lo uint16 + Hi uint16 + Stride uint16 +} + +// Range32 represents of a range of Unicode code points and is used when one or +// more of the values will not fit in 16 bits. The range runs from Lo to Hi +// inclusive and has the specified stride. Lo and Hi must always be >= 1<<16. +type Range32 struct { + Lo uint32 + Hi uint32 + Stride uint32 +} + +// CaseRange represents a range of Unicode code points for simple (one +// code point to one code point) case conversion. +// The range runs from Lo to Hi inclusive, with a fixed stride of 1. Deltas +// are the number to add to the code point to reach the code point for a +// different case for that character. They may be negative. If zero, it +// means the character is in the corresponding case. There is a special +// case representing sequences of alternating corresponding Upper and Lower +// pairs. It appears with a fixed Delta of +// +// {UpperLower, UpperLower, UpperLower} +// +// The constant UpperLower has an otherwise impossible delta value. +type CaseRange struct { + Lo uint32 + Hi uint32 + Delta d +} + +// SpecialCase represents language-specific case mappings such as Turkish. +// Methods of SpecialCase customize (by overriding) the standard mappings. +type SpecialCase []CaseRange + +// BUG(r): There is no mechanism for full case folding, that is, for +// characters that involve multiple runes in the input or output. + +// Indices into the Delta arrays inside CaseRanges for case mapping. +const ( + UpperCase = iota + LowerCase + TitleCase + MaxCase +) + +type d [MaxCase]rune // to make the CaseRanges text shorter + +// If the Delta field of a CaseRange is UpperLower, it means +// this CaseRange represents a sequence of the form (say) +// Upper Lower Upper Lower. +const ( + UpperLower = MaxRune + 1 // (Cannot be a valid delta.) +) + +// linearMax is the maximum size table for linear search for non-Latin1 rune. +// Derived by running 'go test -calibrate'. +const linearMax = 18 + +// is16 reports whether r is in the sorted slice of 16-bit ranges. +func is16(ranges []Range16, r uint16) bool { + if len(ranges) <= linearMax || r <= MaxLatin1 { + for i := range ranges { + range_ := &ranges[i] + if r < range_.Lo { + return false + } + if r <= range_.Hi { + return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0 + } + } + return false + } + + // binary search over ranges + lo := 0 + hi := len(ranges) + for lo < hi { + m := lo + (hi-lo)/2 + range_ := &ranges[m] + if range_.Lo <= r && r <= range_.Hi { + return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0 + } + if r < range_.Lo { + hi = m + } else { + lo = m + 1 + } + } + return false +} + +// is32 reports whether r is in the sorted slice of 32-bit ranges. +func is32(ranges []Range32, r uint32) bool { + if len(ranges) <= linearMax { + for i := range ranges { + range_ := &ranges[i] + if r < range_.Lo { + return false + } + if r <= range_.Hi { + return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0 + } + } + return false + } + + // binary search over ranges + lo := 0 + hi := len(ranges) + for lo < hi { + m := lo + (hi-lo)/2 + range_ := ranges[m] + if range_.Lo <= r && r <= range_.Hi { + return range_.Stride == 1 || (r-range_.Lo)%range_.Stride == 0 + } + if r < range_.Lo { + hi = m + } else { + lo = m + 1 + } + } + return false +} + +// Is reports whether the rune is in the specified table of ranges. +func Is(rangeTab *RangeTable, r rune) bool { + r16 := rangeTab.R16 + // Compare as uint32 to correctly handle negative runes. + if len(r16) > 0 && uint32(r) <= uint32(r16[len(r16)-1].Hi) { + return is16(r16, uint16(r)) + } + r32 := rangeTab.R32 + if len(r32) > 0 && r >= rune(r32[0].Lo) { + return is32(r32, uint32(r)) + } + return false +} + +func isExcludingLatin(rangeTab *RangeTable, r rune) bool { + r16 := rangeTab.R16 + // Compare as uint32 to correctly handle negative runes. + if off := rangeTab.LatinOffset; len(r16) > off && uint32(r) <= uint32(r16[len(r16)-1].Hi) { + return is16(r16[off:], uint16(r)) + } + r32 := rangeTab.R32 + if len(r32) > 0 && r >= rune(r32[0].Lo) { + return is32(r32, uint32(r)) + } + return false +} + +// IsUpper reports whether the rune is an upper case letter. +func IsUpper(r rune) bool { + // See comment in IsGraphic. + if uint32(r) <= MaxLatin1 { + return properties[uint8(r)]&pLmask == pLu + } + return isExcludingLatin(Upper, r) +} + +// IsLower reports whether the rune is a lower case letter. +func IsLower(r rune) bool { + // See comment in IsGraphic. + if uint32(r) <= MaxLatin1 { + return properties[uint8(r)]&pLmask == pLl + } + return isExcludingLatin(Lower, r) +} + +// IsTitle reports whether the rune is a title case letter. +func IsTitle(r rune) bool { + if r <= MaxLatin1 { + return false + } + return isExcludingLatin(Title, r) +} + +// to maps the rune using the specified case mapping. +// It additionally reports whether caseRange contained a mapping for r. +func to(_case int, r rune, caseRange []CaseRange) (mappedRune rune, foundMapping bool) { + if _case < 0 || MaxCase <= _case { + return ReplacementChar, false // as reasonable an error as any + } + // binary search over ranges + lo := 0 + hi := len(caseRange) + for lo < hi { + m := lo + (hi-lo)/2 + cr := caseRange[m] + if rune(cr.Lo) <= r && r <= rune(cr.Hi) { + delta := cr.Delta[_case] + if delta > MaxRune { + // In an Upper-Lower sequence, which always starts with + // an UpperCase letter, the real deltas always look like: + // {0, 1, 0} UpperCase (Lower is next) + // {-1, 0, -1} LowerCase (Upper, Title are previous) + // The characters at even offsets from the beginning of the + // sequence are upper case; the ones at odd offsets are lower. + // The correct mapping can be done by clearing or setting the low + // bit in the sequence offset. + // The constants UpperCase and TitleCase are even while LowerCase + // is odd so we take the low bit from _case. + return rune(cr.Lo) + ((r-rune(cr.Lo))&^1 | rune(_case&1)), true + } + return r + delta, true + } + if r < rune(cr.Lo) { + hi = m + } else { + lo = m + 1 + } + } + return r, false +} + +// To maps the rune to the specified case: UpperCase, LowerCase, or TitleCase. +func To(_case int, r rune) rune { + r, _ = to(_case, r, CaseRanges) + return r +} + +// ToUpper maps the rune to upper case. +func ToUpper(r rune) rune { + if r <= MaxASCII { + if 'a' <= r && r <= 'z' { + r -= 'a' - 'A' + } + return r + } + return To(UpperCase, r) +} + +// ToLower maps the rune to lower case. +func ToLower(r rune) rune { + if r <= MaxASCII { + if 'A' <= r && r <= 'Z' { + r += 'a' - 'A' + } + return r + } + return To(LowerCase, r) +} + +// ToTitle maps the rune to title case. +func ToTitle(r rune) rune { + if r <= MaxASCII { + if 'a' <= r && r <= 'z' { // title case is upper case for ASCII + r -= 'a' - 'A' + } + return r + } + return To(TitleCase, r) +} + +// ToUpper maps the rune to upper case giving priority to the special mapping. +func (special SpecialCase) ToUpper(r rune) rune { + r1, hadMapping := to(UpperCase, r, []CaseRange(special)) + if r1 == r && !hadMapping { + r1 = ToUpper(r) + } + return r1 +} + +// ToTitle maps the rune to title case giving priority to the special mapping. +func (special SpecialCase) ToTitle(r rune) rune { + r1, hadMapping := to(TitleCase, r, []CaseRange(special)) + if r1 == r && !hadMapping { + r1 = ToTitle(r) + } + return r1 +} + +// ToLower maps the rune to lower case giving priority to the special mapping. +func (special SpecialCase) ToLower(r rune) rune { + r1, hadMapping := to(LowerCase, r, []CaseRange(special)) + if r1 == r && !hadMapping { + r1 = ToLower(r) + } + return r1 +} + +// caseOrbit is defined in tables.go as []foldPair. Right now all the +// entries fit in uint16, so use uint16. If that changes, compilation +// will fail (the constants in the composite literal will not fit in uint16) +// and the types here can change to uint32. +type foldPair struct { + From uint16 + To uint16 +} + +// SimpleFold iterates over Unicode code points equivalent under +// the Unicode-defined simple case folding. Among the code points +// equivalent to rune (including rune itself), SimpleFold returns the +// smallest rune > r if one exists, or else the smallest rune >= 0. +// If r is not a valid Unicode code point, SimpleFold(r) returns r. +// +// For example: +// +// SimpleFold('A') = 'a' +// SimpleFold('a') = 'A' +// +// SimpleFold('K') = 'k' +// SimpleFold('k') = '\u212A' (Kelvin symbol, K) +// SimpleFold('\u212A') = 'K' +// +// SimpleFold('1') = '1' +// +// SimpleFold(-2) = -2 +func SimpleFold(r rune) rune { + if r < 0 || r > MaxRune { + return r + } + + if int(r) < len(asciiFold) { + return rune(asciiFold[r]) + } + + // Consult caseOrbit table for special cases. + lo := 0 + hi := len(caseOrbit) + for lo < hi { + m := lo + (hi-lo)/2 + if rune(caseOrbit[m].From) < r { + lo = m + 1 + } else { + hi = m + } + } + if lo < len(caseOrbit) && rune(caseOrbit[lo].From) == r { + return rune(caseOrbit[lo].To) + } + + // No folding specified. This is a one- or two-element + // equivalence class containing rune and ToLower(rune) + // and ToUpper(rune) if they are different from rune. + if l := ToLower(r); l != r { + return l + } + return ToUpper(r) +} |