blob: b284dc10017353f6d4868bedd09ede898ecded9e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
|
/* Copyright (c) 2023, MariaDB Corporation.
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; version 2 of the License.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1335 USA */
#ifndef SIMPLE_TOKENIZER_INCLUDED
#define SIMPLE_TOKENIZER_INCLUDED
class Simple_tokenizer
{
const char *m_ptr;
const char *m_end;
public:
Simple_tokenizer(const char *str, size_t length)
:m_ptr(str), m_end(str + length)
{ }
const char *ptr() const
{
return m_ptr;
}
bool eof() const
{
return m_ptr >= m_end;
}
void get_spaces()
{
for ( ; !eof(); m_ptr++)
{
if (m_ptr[0] != ' ')
break;
}
}
bool is_ident_start(char ch) const
{
return (ch >= 'a' && ch <= 'z') ||
(ch >= 'A' && ch <= 'Z') ||
ch == '_';
}
bool is_ident_body(char ch) const
{
return is_ident_start(ch) ||
(ch >= '0' && ch <= '9');
}
bool is_ident_start() const
{
return !eof() && is_ident_start(*m_ptr);
}
bool is_ident_body() const
{
return !eof() && is_ident_body(*m_ptr);
}
LEX_CSTRING get_ident()
{
get_spaces();
if (!is_ident_start())
return {m_ptr,0};
const char *start= m_ptr++;
for ( ; is_ident_body(); m_ptr++)
{ }
LEX_CSTRING res= {start, (size_t) (m_ptr - start)};
return res;
}
bool get_char(char ch)
{
get_spaces();
if (eof() || *m_ptr != ch)
return true;
m_ptr++;
return false;
}
};
#endif // SIMPLE_TOKENIZER_INCLUDED
|
