1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
|
%{
/*-------------------------------------------------------------------------
*
* exprscan.l
* lexical scanner for pgbench backslash commands
*
* This lexer supports two operating modes:
*
* In INITIAL state, just parse off whitespace-separated words (this mode
* is basically equivalent to strtok(), which is what we used to use).
*
* In EXPR state, lex for the simple expression syntax of exprparse.y.
*
* In either mode, stop upon hitting newline or end of string.
*
* Note that this lexer operates within the framework created by psqlscan.l,
*
* Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
* Portions Copyright (c) 1994, Regents of the University of California
*
* src/bin/pgbench/exprscan.l
*
*-------------------------------------------------------------------------
*/
#include "fe_utils/psqlscan_int.h"
/* context information for reporting errors in expressions */
static const char *expr_source = NULL;
static int expr_lineno = 0;
static int expr_start_offset = 0;
static const char *expr_command = NULL;
/* indicates whether last yylex() call read a newline */
static bool last_was_newline = false;
/*
* Work around a bug in flex 2.5.35: it emits a couple of functions that
* it forgets to emit declarations for. Since we use -Wmissing-prototypes,
* this would cause warnings. Providing our own declarations should be
* harmless even when the bug gets fixed.
*/
extern int expr_yyget_column(yyscan_t yyscanner);
extern void expr_yyset_column(int column_no, yyscan_t yyscanner);
/* LCOV_EXCL_START */
%}
/* Except for the prefix, these options should match psqlscan.l */
%option reentrant
%option bison-bridge
%option 8bit
%option never-interactive
%option nodefault
%option noinput
%option nounput
%option noyywrap
%option warn
%option prefix="expr_yy"
/* Character classes */
alpha [a-zA-Z\200-\377_]
digit [0-9]
alnum [A-Za-z\200-\377_0-9]
/* {space} + {nonspace} + {newline} should cover all characters */
space [ \t\r\f\v]
nonspace [^ \t\r\f\v\n]
newline [\n]
/* Line continuation marker */
continuation \\\r?{newline}
/* case insensitive keywords */
and [Aa][Nn][Dd]
or [Oo][Rr]
not [Nn][Oo][Tt]
case [Cc][Aa][Ss][Ee]
when [Ww][Hh][Ee][Nn]
then [Tt][Hh][Ee][Nn]
else [Ee][Ll][Ss][Ee]
end [Ee][Nn][Dd]
true [Tt][Rr][Uu][Ee]
false [Ff][Aa][Ll][Ss][Ee]
null [Nn][Uu][Ll][Ll]
is [Ii][Ss]
isnull [Ii][Ss][Nn][Uu][Ll][Ll]
notnull [Nn][Oo][Tt][Nn][Uu][Ll][Ll]
/* Exclusive states */
%x EXPR
%%
%{
/* Declare some local variables inside yylex(), for convenience */
PsqlScanState cur_state = yyextra;
/*
* Force flex into the state indicated by start_state. This has a
* couple of purposes: it lets some of the functions below set a new
* starting state without ugly direct access to flex variables, and it
* allows us to transition from one flex lexer to another so that we
* can lex different parts of the source string using separate lexers.
*/
BEGIN(cur_state->start_state);
/* Reset was-newline flag */
last_was_newline = false;
%}
/* INITIAL state */
{nonspace}+ {
/* Found a word, emit and return it */
psqlscan_emit(cur_state, yytext, yyleng);
return 1;
}
/*
* We need this rule to avoid returning "word\" instead of recognizing
* a continuation marker just after a word:
*/
{nonspace}+{continuation} {
/* Found "word\\\r?\n", emit and return just "word" */
int wordlen = yyleng - 2;
if (yytext[wordlen] == '\r')
wordlen--;
Assert(yytext[wordlen] == '\\');
psqlscan_emit(cur_state, yytext, wordlen);
return 1;
}
{space}+ { /* ignore */ }
{continuation} { /* ignore */ }
{newline} {
/* report end of command */
last_was_newline = true;
return 0;
}
/* EXPR state */
<EXPR>{
"+" { return '+'; }
"-" { return '-'; }
"*" { return '*'; }
"/" { return '/'; }
"%" { return '%'; } /* C version, also in Pg SQL */
"=" { return '='; }
"<>" { return NE_OP; }
"!=" { return NE_OP; } /* C version, also in Pg SQL */
"<=" { return LE_OP; }
">=" { return GE_OP; }
"<<" { return LS_OP; }
">>" { return RS_OP; }
"<" { return '<'; }
">" { return '>'; }
"|" { return '|'; }
"&" { return '&'; }
"#" { return '#'; }
"~" { return '~'; }
"(" { return '('; }
")" { return ')'; }
"," { return ','; }
{and} { return AND_OP; }
{or} { return OR_OP; }
{not} { return NOT_OP; }
{is} { return IS_OP; }
{isnull} { return ISNULL_OP; }
{notnull} { return NOTNULL_OP; }
{case} { return CASE_KW; }
{when} { return WHEN_KW; }
{then} { return THEN_KW; }
{else} { return ELSE_KW; }
{end} { return END_KW; }
:{alnum}+ {
yylval->str = pg_strdup(yytext + 1);
return VARIABLE;
}
{null} { return NULL_CONST; }
{true} {
yylval->bval = true;
return BOOLEAN_CONST;
}
{false} {
yylval->bval = false;
return BOOLEAN_CONST;
}
"9223372036854775808" {
/*
* Special handling for PG_INT64_MIN, which can't
* accurately be represented here, as the minus sign is
* lexed separately and INT64_MIN can't be represented as
* a positive integer.
*/
return MAXINT_PLUS_ONE_CONST;
}
{digit}+ {
if (!strtoint64(yytext, true, &yylval->ival))
expr_yyerror_more(yyscanner, "bigint constant overflow",
strdup(yytext));
return INTEGER_CONST;
}
{digit}+(\.{digit}*)?([eE][-+]?{digit}+)? {
if (!strtodouble(yytext, true, &yylval->dval))
expr_yyerror_more(yyscanner, "double constant overflow",
strdup(yytext));
return DOUBLE_CONST;
}
\.{digit}+([eE][-+]?{digit}+)? {
if (!strtodouble(yytext, true, &yylval->dval))
expr_yyerror_more(yyscanner, "double constant overflow",
strdup(yytext));
return DOUBLE_CONST;
}
{alpha}{alnum}* {
yylval->str = pg_strdup(yytext);
return FUNCTION;
}
{space}+ { /* ignore */ }
{continuation} { /* ignore */ }
{newline} {
/* report end of command */
last_was_newline = true;
return 0;
}
. {
/*
* must strdup yytext so that expr_yyerror_more doesn't
* change it while finding end of line
*/
expr_yyerror_more(yyscanner, "unexpected character",
pg_strdup(yytext));
/* NOTREACHED, syntax_error calls exit() */
return 0;
}
}
<<EOF>> {
if (cur_state->buffer_stack == NULL)
return 0; /* end of input reached */
/*
* We were expanding a variable, so pop the inclusion
* stack and keep lexing
*/
psqlscan_pop_buffer_stack(cur_state);
psqlscan_select_top_buffer(cur_state);
}
%%
/* LCOV_EXCL_STOP */
void
expr_yyerror_more(yyscan_t yyscanner, const char *message, const char *more)
{
PsqlScanState state = yyget_extra(yyscanner);
int error_detection_offset = expr_scanner_offset(state) - 1;
YYSTYPE lval;
char *full_line;
/*
* While parsing an expression, we may not have collected the whole line
* yet from the input source. Lex till EOL so we can report whole line.
* (If we're at EOF, it's okay to call yylex() an extra time.)
*/
if (!last_was_newline)
{
while (yylex(&lval, yyscanner))
/* skip */ ;
}
/* Extract the line, trimming trailing newline if any */
full_line = expr_scanner_get_substring(state,
expr_start_offset,
expr_scanner_offset(state),
true);
syntax_error(expr_source, expr_lineno, full_line, expr_command,
message, more, error_detection_offset - expr_start_offset);
}
void
expr_yyerror(yyscan_t yyscanner, const char *message)
{
expr_yyerror_more(yyscanner, message, NULL);
}
/*
* Collect a space-separated word from a backslash command and return it
* in word_buf, along with its starting string offset in *offset.
* Returns true if successful, false if at end of command.
*/
bool
expr_lex_one_word(PsqlScanState state, PQExpBuffer word_buf, int *offset)
{
int lexresult;
YYSTYPE lval;
/* Must be scanning already */
Assert(state->scanbufhandle != NULL);
/* Set current output target */
state->output_buf = word_buf;
resetPQExpBuffer(word_buf);
/* Set input source */
if (state->buffer_stack != NULL)
yy_switch_to_buffer(state->buffer_stack->buf, state->scanner);
else
yy_switch_to_buffer(state->scanbufhandle, state->scanner);
/* Set start state */
state->start_state = INITIAL;
/* And lex. */
lexresult = yylex(&lval, state->scanner);
/*
* Save start offset of word, if any. We could do this more efficiently,
* but for now this seems fine.
*/
if (lexresult)
*offset = expr_scanner_offset(state) - word_buf->len;
else
*offset = -1;
/*
* In case the caller returns to using the regular SQL lexer, reselect the
* appropriate initial state.
*/
psql_scan_reselect_sql_lexer(state);
return (bool) lexresult;
}
/*
* Prepare to lex an expression via expr_yyparse().
*
* Returns the yyscan_t that is to be passed to expr_yyparse().
* (This is just state->scanner, but callers don't need to know that.)
*/
yyscan_t
expr_scanner_init(PsqlScanState state,
const char *source, int lineno, int start_offset,
const char *command)
{
/* Save error context info */
expr_source = source;
expr_lineno = lineno;
expr_start_offset = start_offset;
expr_command = command;
/* Must be scanning already */
Assert(state->scanbufhandle != NULL);
/* Set current output target */
state->output_buf = NULL;
/* Set input source */
if (state->buffer_stack != NULL)
yy_switch_to_buffer(state->buffer_stack->buf, state->scanner);
else
yy_switch_to_buffer(state->scanbufhandle, state->scanner);
/* Set start state */
state->start_state = EXPR;
return state->scanner;
}
/*
* Finish lexing an expression.
*/
void
expr_scanner_finish(yyscan_t yyscanner)
{
PsqlScanState state = yyget_extra(yyscanner);
/*
* Reselect appropriate initial state for SQL lexer.
*/
psql_scan_reselect_sql_lexer(state);
}
/*
* Get offset from start of string to end of current lexer token.
*
* We rely on the knowledge that flex modifies the scan buffer by storing
* a NUL at the end of the current token (yytext). Note that this might
* not work quite right if we were parsing a sub-buffer, but since pgbench
* never invokes that functionality, it doesn't matter.
*/
int
expr_scanner_offset(PsqlScanState state)
{
return strlen(state->scanbuf);
}
/*
* Get a malloc'd copy of the lexer input string from start_offset
* to just before end_offset. If chomp is true, drop any trailing
* newline(s).
*/
char *
expr_scanner_get_substring(PsqlScanState state,
int start_offset, int end_offset,
bool chomp)
{
char *result;
const char *scanptr = state->scanbuf + start_offset;
int slen = end_offset - start_offset;
Assert(slen >= 0);
Assert(end_offset <= strlen(state->scanbuf));
if (chomp)
{
while (slen > 0 &&
(scanptr[slen - 1] == '\n' || scanptr[slen - 1] == '\r'))
slen--;
}
result = (char *) pg_malloc(slen + 1);
memcpy(result, scanptr, slen);
result[slen] = '\0';
return result;
}
/*
* Get the line number associated with the given string offset
* (which must not be past the end of where we've lexed to).
*/
int
expr_scanner_get_lineno(PsqlScanState state, int offset)
{
int lineno = 1;
const char *p = state->scanbuf;
while (*p && offset > 0)
{
if (*p == '\n')
lineno++;
p++, offset--;
}
return lineno;
}
|