1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
|
use std::path::Path;
use std::str::FromStr;
use crate::common::{
parse_break_test, parse_codepoint_association, CodepointIter, Codepoints,
UcdFile, UcdFileByCodepoint,
};
use crate::error::Error;
/// A single row in the `auxiliary/WordBreakProperty.txt` file.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct WordBreak {
/// The codepoint or codepoint range for this entry.
pub codepoints: Codepoints,
/// The property value assigned to the codepoints in this entry.
pub value: String,
}
impl UcdFile for WordBreak {
fn relative_file_path() -> &'static Path {
Path::new("auxiliary/WordBreakProperty.txt")
}
}
impl UcdFileByCodepoint for WordBreak {
fn codepoints(&self) -> CodepointIter {
self.codepoints.into_iter()
}
}
impl FromStr for WordBreak {
type Err = Error;
fn from_str(line: &str) -> Result<WordBreak, Error> {
let (codepoints, value) = parse_codepoint_association(line)?;
Ok(WordBreak { codepoints, value: value.to_string() })
}
}
/// A single row in the `auxiliary/WordBreakTest.txt` file.
///
/// This file defines tests for the word break algorithm.
#[derive(Clone, Debug, Default, Eq, PartialEq)]
pub struct WordBreakTest {
/// Each string is a UTF-8 encoded group of codepoints that make up a
/// single word.
pub words: Vec<String>,
/// A human readable description of this test.
pub comment: String,
}
impl UcdFile for WordBreakTest {
fn relative_file_path() -> &'static Path {
Path::new("auxiliary/WordBreakTest.txt")
}
}
impl FromStr for WordBreakTest {
type Err = Error;
fn from_str(line: &str) -> Result<WordBreakTest, Error> {
let (groups, comment) = parse_break_test(line)?;
Ok(WordBreakTest { words: groups, comment })
}
}
#[cfg(test)]
mod tests {
use super::{WordBreak, WordBreakTest};
#[test]
fn parse_single() {
let line = "0A83 ; Extend # Mc GUJARATI SIGN VISARGA\n";
let row: WordBreak = line.parse().unwrap();
assert_eq!(row.codepoints, 0x0A83);
assert_eq!(row.value, "Extend");
}
#[test]
fn parse_range() {
let line = "104A0..104A9 ; Numeric # Nd [10] OSMANYA DIGIT ZERO..OSMANYA DIGIT NINE\n";
let row: WordBreak = line.parse().unwrap();
assert_eq!(row.codepoints, (0x104A0, 0x104A9));
assert_eq!(row.value, "Numeric");
}
#[test]
fn parse_test() {
let line = "÷ 0031 ÷ 0027 × 0308 ÷ 0061 ÷ 0027 × 2060 ÷ # ÷ [0.2] DIGIT ONE (Numeric) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] COMBINING DIAERESIS (Extend_FE) ÷ [999.0] LATIN SMALL LETTER A (ALetter) ÷ [999.0] APOSTROPHE (Single_Quote) × [4.0] WORD JOINER (Format_FE) ÷ [0.3]";
let row: WordBreakTest = line.parse().unwrap();
assert_eq!(
row.words,
vec![
"\u{0031}",
"\u{0027}\u{0308}",
"\u{0061}",
"\u{0027}\u{2060}",
]
);
assert!(row.comment.contains("[4.0] COMBINING DIAERESIS (Extend_FE)"));
}
}
|
