1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
// This file is part of ICU4X. For terms of use, please see the file
// called LICENSE at the top level of the ICU4X source tree
// (online at: https://github.com/unicode-org/icu4x/blob/main/LICENSE ).
/// Similar to [`core::str::CharIndices`] for Latin-1 strings, represented as `[u8]`.
///
/// Contrary to [`core::str::CharIndices`], the second element of the
/// [`Iterator::Item`] is a [`u8`], representing a Unicode scalar value in the
/// range U+0000–U+00FF.
#[derive(Clone, Debug)]
pub struct Latin1Indices<'a> {
front_offset: usize,
iter: &'a [u8],
}
impl<'a> Latin1Indices<'a> {
pub fn new(input: &'a [u8]) -> Self {
Self {
front_offset: 0,
iter: input,
}
}
}
impl<'a> Iterator for Latin1Indices<'a> {
type Item = (usize, u8);
#[inline]
fn next(&mut self) -> Option<(usize, u8)> {
self.iter.get(self.front_offset).map(|ch| {
self.front_offset += 1;
(self.front_offset - 1, *ch)
})
}
}
/// Similar to [`core::str::CharIndices`] for UTF-16 strings, represented as `[u16]`.
///
/// Contrary to [`core::str::CharIndices`], the second element of the
/// [`Iterator::Item`] is a Unicode code point represented by a [`u32`],
/// rather than a Unicode scalar value represented by a [`char`], because this
/// iterator preserves unpaired surrogates.
#[derive(Clone, Debug)]
pub struct Utf16Indices<'a> {
front_offset: usize,
iter: &'a [u16],
}
impl<'a> Utf16Indices<'a> {
pub fn new(input: &'a [u16]) -> Self {
Self {
front_offset: 0,
iter: input,
}
}
}
impl<'a> Iterator for Utf16Indices<'a> {
type Item = (usize, u32);
#[inline]
fn next(&mut self) -> Option<(usize, u32)> {
let (index, ch) = self.iter.get(self.front_offset).map(|ch| {
self.front_offset += 1;
(self.front_offset - 1, *ch)
})?;
let mut ch = ch as u32;
if (ch & 0xfc00) != 0xd800 {
return Some((index, ch));
}
if let Some(next) = self.iter.get(self.front_offset) {
let next = *next as u32;
if (next & 0xfc00) == 0xdc00 {
// Combine low and high surrogates to UTF-32 code point.
ch = ((ch & 0x3ff) << 10) + (next & 0x3ff) + 0x10000;
self.front_offset += 1;
}
}
Some((index, ch))
}
}
#[cfg(test)]
mod tests {
use crate::indices::*;
#[test]
fn latin1_indices() {
let latin1 = [0x30, 0x31, 0x32];
let mut indices = Latin1Indices::new(&latin1);
let n = indices.next().unwrap();
assert_eq!(n.0, 0);
assert_eq!(n.1, 0x30);
let n = indices.next().unwrap();
assert_eq!(n.0, 1);
assert_eq!(n.1, 0x31);
let n = indices.next().unwrap();
assert_eq!(n.0, 2);
assert_eq!(n.1, 0x32);
let n = indices.next();
assert_eq!(n, None);
}
#[test]
fn utf16_indices() {
let utf16 = [0xd83d, 0xde03, 0x0020, 0xd83c, 0xdf00, 0xd800, 0x0020];
let mut indices = Utf16Indices::new(&utf16);
let n = indices.next().unwrap();
assert_eq!(n.0, 0);
assert_eq!(n.1, 0x1f603);
let n = indices.next().unwrap();
assert_eq!(n.0, 2);
assert_eq!(n.1, 0x20);
let n = indices.next().unwrap();
assert_eq!(n.0, 3);
assert_eq!(n.1, 0x1f300);
// This is invalid surrogate pair.
let n = indices.next().unwrap();
assert_eq!(n.0, 5);
assert_eq!(n.1, 0xd800);
let n = indices.next().unwrap();
assert_eq!(n.0, 6);
assert_eq!(n.1, 0x0020);
let n = indices.next();
assert_eq!(n, None);
}
}
|
