1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
// Copyright (c) 2001-2011 Hartmut Kaiser
//
// Distributed under the Boost Software License, Version 1.0. (See accompanying
// file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
#include <boost/detail/lightweight_test.hpp>
#include <boost/spirit/include/lex_lexertl.hpp>
#include "test.hpp"
///////////////////////////////////////////////////////////////////////////////
// test pattern definition capabilities
int main()
{
using namespace boost::spirit;
using namespace boost::spirit::lex;
using namespace spirit_test;
// initialize tokens
typedef lex::token_def<std::string> token_def;
std::size_t const CCOMMENT = 1;
std::size_t const CPPCOMMENT = 2;
std::size_t const TOKEN_ID_ABC = 1000;
std::size_t const TOKEN_ID_STR = 1001;
std::size_t const TOKEN_ID_WS = 1002;
typedef std::string::iterator base_iterator_type;
typedef lex::lexertl::token<base_iterator_type> token_type;
typedef lex::lexertl::lexer<token_type> lexer_type;
typedef lex::lexer<lexer_type> lexer_def;
std::string str("def");
{
// initialize lexer
lexer_def lex;
lex.self.add_pattern
("CCOMMENT", "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/")
("CPPCOMMENT", "\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)")
("WS", "[\\v\\f\\n\\r]+")
;
token_def c_comment ("{CCOMMENT}", CCOMMENT);
token_def cpp_comment ("{CPPCOMMENT}", CPPCOMMENT);
token_def ws_tok ("{WS}");
lex.self.add
(c_comment)(cpp_comment)
('1')('2')('3')
("abc", TOKEN_ID_ABC)
(str, TOKEN_ID_STR)
;
lex.self += token_def(' ') | '\t' | ws_tok;
// test lexer for different input strings
BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
BOOST_TEST(test (lex, "\n\n\v\f\r", ws_tok.id()));
BOOST_TEST(test (lex, " ", ' '));
BOOST_TEST(test (lex, "2", '2'));
BOOST_TEST(test (lex, "abc", TOKEN_ID_ABC));
BOOST_TEST(test (lex, "def", TOKEN_ID_STR));
}
{
// initialize lexer
lexer_def lex;
lex.self.add_pattern
("CCOMMENT", "\\/\\*[^*]*\\*+([^/*][^*]*\\*+)*\\/")
("CPPCOMMENT", "\\/\\/[^\\n\\r]*(\\n|\\r|\\r\\n)")
("WS", "[\\v\\f\\n\\r]+")
;
token_def c_comment ("{CCOMMENT}", CCOMMENT);
token_def cpp_comment ("{CPPCOMMENT}", CPPCOMMENT);
token_def ws_tok ("{WS}");
// init lexer
lex.self.add
(c_comment)(cpp_comment)
('1')('2')('3')
("abc", TOKEN_ID_ABC)
(str, TOKEN_ID_STR)
;
lex.self("WHITESPACE").add
(' ')('\t')
(ws_tok, TOKEN_ID_WS)
;
// test lexer for different input strings
BOOST_TEST(test (lex, "/* this is a comment */", CCOMMENT));
BOOST_TEST(test (lex, "// this is a comment as well\n", CPPCOMMENT));
BOOST_TEST(test (lex, "2", '2'));
BOOST_TEST(test (lex, "abc", TOKEN_ID_ABC));
BOOST_TEST(test (lex, "def", TOKEN_ID_STR));
BOOST_TEST(!test (lex, "\n\n\v\f\r", TOKEN_ID_WS));
BOOST_TEST(test (lex, " ", ' ', "WHITESPACE"));
BOOST_TEST(test (lex, "\n\n\v\f\r", TOKEN_ID_WS, "WHITESPACE"));
}
return boost::report_errors();
}
|
