1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
|
#![allow(
clippy::let_underscore_untyped,
clippy::manual_range_contains,
clippy::needless_pass_by_value,
clippy::type_complexity
)]
#[path = "../src/tokens.rs"]
#[allow(dead_code)]
mod tokens;
use crate::tokens::{Error, Token, Tokenizer};
use std::borrow::Cow;
fn err(input: &str, err: Error) {
let mut t = Tokenizer::new(input);
let token = t.next().unwrap_err();
assert_eq!(token, err);
assert!(t.next().unwrap().is_none());
}
#[test]
fn literal_strings() {
fn t(input: &str, val: &str, multiline: bool) {
let mut t = Tokenizer::new(input);
let (_, token) = t.next().unwrap().unwrap();
assert_eq!(
token,
Token::String {
src: input,
val: Cow::Borrowed(val),
multiline,
}
);
assert!(t.next().unwrap().is_none());
}
t("''", "", false);
t("''''''", "", true);
t("'''\n'''", "", true);
t("'a'", "a", false);
t("'\"a'", "\"a", false);
t("''''a'''", "'a", true);
t("'''\n'a\n'''", "'a\n", true);
t("'''a\n'a\r\n'''", "a\n'a\n", true);
}
#[test]
fn basic_strings() {
fn t(input: &str, val: &str, multiline: bool) {
let mut t = Tokenizer::new(input);
let (_, token) = t.next().unwrap().unwrap();
assert_eq!(
token,
Token::String {
src: input,
val: Cow::Borrowed(val),
multiline,
}
);
assert!(t.next().unwrap().is_none());
}
t(r#""""#, "", false);
t(r#""""""""#, "", true);
t(r#""a""#, "a", false);
t(r#""""a""""#, "a", true);
t(r#""\t""#, "\t", false);
t(r#""\u0000""#, "\0", false);
t(r#""\U00000000""#, "\0", false);
t(r#""\U000A0000""#, "\u{A0000}", false);
t(r#""\\t""#, "\\t", false);
t("\"\t\"", "\t", false);
t("\"\"\"\n\t\"\"\"", "\t", true);
t("\"\"\"\\\n\"\"\"", "", true);
t(
"\"\"\"\\\n \t \t \\\r\n \t \n \t \r\n\"\"\"",
"",
true,
);
t(r#""\r""#, "\r", false);
t(r#""\n""#, "\n", false);
t(r#""\b""#, "\u{8}", false);
t(r#""a\fa""#, "a\u{c}a", false);
t(r#""\"a""#, "\"a", false);
t("\"\"\"\na\"\"\"", "a", true);
t("\"\"\"\n\"\"\"", "", true);
t(r#""""a\"""b""""#, "a\"\"\"b", true);
err(r#""\a"#, Error::InvalidEscape(2, 'a'));
err("\"\\\n", Error::InvalidEscape(2, '\n'));
err("\"\\\r\n", Error::InvalidEscape(2, '\n'));
err("\"\\", Error::UnterminatedString(0));
err("\"\u{0}", Error::InvalidCharInString(1, '\u{0}'));
err(r#""\U00""#, Error::InvalidHexEscape(5, '"'));
err(r#""\U00"#, Error::UnterminatedString(0));
err(r#""\uD800"#, Error::InvalidEscapeValue(2, 0xd800));
err(r#""\UFFFFFFFF"#, Error::InvalidEscapeValue(2, 0xffff_ffff));
}
#[test]
fn keylike() {
fn t(input: &str) {
let mut t = Tokenizer::new(input);
let (_, token) = t.next().unwrap().unwrap();
assert_eq!(token, Token::Keylike(input));
assert!(t.next().unwrap().is_none());
}
t("foo");
t("0bar");
t("bar0");
t("1234");
t("a-b");
t("a_B");
t("-_-");
t("___");
}
#[test]
fn all() {
fn t(input: &str, expected: &[((usize, usize), Token, &str)]) {
let mut tokens = Tokenizer::new(input);
let mut actual: Vec<((usize, usize), Token, &str)> = Vec::new();
while let Some((span, token)) = tokens.next().unwrap() {
actual.push((span.into(), token, &input[span.start..span.end]));
}
for (a, b) in actual.iter().zip(expected) {
assert_eq!(a, b);
}
assert_eq!(actual.len(), expected.len());
}
t(
" a ",
&[
((0, 1), Token::Whitespace(" "), " "),
((1, 2), Token::Keylike("a"), "a"),
((2, 3), Token::Whitespace(" "), " "),
],
);
t(
" a\t [[]] \t [] {} , . =\n# foo \r\n#foo \n ",
&[
((0, 1), Token::Whitespace(" "), " "),
((1, 2), Token::Keylike("a"), "a"),
((2, 4), Token::Whitespace("\t "), "\t "),
((4, 5), Token::LeftBracket, "["),
((5, 6), Token::LeftBracket, "["),
((6, 7), Token::RightBracket, "]"),
((7, 8), Token::RightBracket, "]"),
((8, 11), Token::Whitespace(" \t "), " \t "),
((11, 12), Token::LeftBracket, "["),
((12, 13), Token::RightBracket, "]"),
((13, 14), Token::Whitespace(" "), " "),
((14, 15), Token::LeftBrace, "{"),
((15, 16), Token::RightBrace, "}"),
((16, 17), Token::Whitespace(" "), " "),
((17, 18), Token::Comma, ","),
((18, 19), Token::Whitespace(" "), " "),
((19, 20), Token::Period, "."),
((20, 21), Token::Whitespace(" "), " "),
((21, 22), Token::Equals, "="),
((22, 23), Token::Newline, "\n"),
((23, 29), Token::Comment("# foo "), "# foo "),
((29, 31), Token::Newline, "\r\n"),
((31, 36), Token::Comment("#foo "), "#foo "),
((36, 37), Token::Newline, "\n"),
((37, 38), Token::Whitespace(" "), " "),
],
);
}
#[test]
fn bare_cr_bad() {
err("\r", Error::Unexpected(0, '\r'));
err("'\n", Error::NewlineInString(1));
err("'\u{0}", Error::InvalidCharInString(1, '\u{0}'));
err("'", Error::UnterminatedString(0));
err("\u{0}", Error::Unexpected(0, '\u{0}'));
}
#[test]
fn bad_comment() {
let mut t = Tokenizer::new("#\u{0}");
t.next().unwrap().unwrap();
assert_eq!(t.next(), Err(Error::Unexpected(1, '\u{0}')));
assert!(t.next().unwrap().is_none());
}
|
