/*-------------------------------------------------------------------------- * * test_regex.c * Test harness for the regular expression package. * * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * * IDENTIFICATION * src/test/modules/test_regex/test_regex.c * * ------------------------------------------------------------------------- */ #include "postgres.h" #include "funcapi.h" #include "miscadmin.h" #include "regex/regex.h" #include "utils/array.h" #include "utils/builtins.h" PG_MODULE_MAGIC; /* all the options of interest for regex functions */ typedef struct test_re_flags { int cflags; /* compile flags for Spencer's regex code */ int eflags; /* execute flags for Spencer's regex code */ long info; /* expected re_info bits */ bool glob; /* do it globally (for each occurrence) */ bool indices; /* report indices not actual strings */ bool partial; /* expect partial match */ } test_re_flags; /* cross-call state for test_regex() */ typedef struct test_regex_ctx { test_re_flags re_flags; /* flags */ rm_detail_t details; /* "details" from execution */ text *orig_str; /* data string in original TEXT form */ int nmatches; /* number of places where pattern matched */ int npatterns; /* number of capturing subpatterns */ /* We store start char index and end+1 char index for each match */ /* so the number of entries in match_locs is nmatches * npatterns * 2 */ int *match_locs; /* 0-based character indexes */ int next_match; /* 0-based index of next match to process */ /* workspace for build_test_match_result() */ Datum *elems; /* has npatterns+1 elements */ bool *nulls; /* has npatterns+1 elements */ pg_wchar *wide_str; /* wide-char version of original string */ char *conv_buf; /* conversion buffer, if needed */ int conv_bufsiz; /* size thereof */ } test_regex_ctx; /* Local functions */ static void test_re_compile(text *text_re, int cflags, Oid collation, regex_t *result_re); static void parse_test_flags(test_re_flags *flags, text *opts); static test_regex_ctx *setup_test_matches(text *orig_str, regex_t *cpattern, test_re_flags *flags, Oid collation, bool use_subpatterns); static ArrayType *build_test_info_result(regex_t *cpattern, test_re_flags *flags); static ArrayType *build_test_match_result(test_regex_ctx *matchctx); /* * test_regex(pattern text, string text, flags text) returns setof text[] * * This is largely based on regexp.c's regexp_matches, with additions * for debugging purposes. */ PG_FUNCTION_INFO_V1(test_regex); Datum test_regex(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; test_regex_ctx *matchctx; ArrayType *result_ary; if (SRF_IS_FIRSTCALL()) { text *pattern = PG_GETARG_TEXT_PP(0); text *flags = PG_GETARG_TEXT_PP(2); Oid collation = PG_GET_COLLATION(); test_re_flags re_flags; regex_t cpattern; MemoryContext oldcontext; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); /* Determine options */ parse_test_flags(&re_flags, flags); /* set up the compiled pattern */ test_re_compile(pattern, re_flags.cflags, collation, &cpattern); /* be sure to copy the input string into the multi-call ctx */ matchctx = setup_test_matches(PG_GETARG_TEXT_P_COPY(1), &cpattern, &re_flags, collation, true); /* Pre-create workspace that build_test_match_result needs */ matchctx->elems = (Datum *) palloc(sizeof(Datum) * (matchctx->npatterns + 1)); matchctx->nulls = (bool *) palloc(sizeof(bool) * (matchctx->npatterns + 1)); MemoryContextSwitchTo(oldcontext); funcctx->user_fctx = (void *) matchctx; /* * Return the first result row, which is info equivalent to Tcl's * "regexp -about" output */ result_ary = build_test_info_result(&cpattern, &re_flags); pg_regfree(&cpattern); SRF_RETURN_NEXT(funcctx, PointerGetDatum(result_ary)); } else { /* Each subsequent row describes one match */ funcctx = SRF_PERCALL_SETUP(); matchctx = (test_regex_ctx *) funcctx->user_fctx; if (matchctx->next_match < matchctx->nmatches) { result_ary = build_test_match_result(matchctx); matchctx->next_match++; SRF_RETURN_NEXT(funcctx, PointerGetDatum(result_ary)); } } SRF_RETURN_DONE(funcctx); } /* * test_re_compile - compile a RE * * text_re --- the pattern, expressed as a TEXT object * cflags --- compile options for the pattern * collation --- collation to use for LC_CTYPE-dependent behavior * result_re --- output, compiled RE is stored here * * Pattern is given in the database encoding. We internally convert to * an array of pg_wchar, which is what Spencer's regex package wants. * * Caller must eventually pg_regfree the resulting RE to avoid memory leaks. */ static void test_re_compile(text *text_re, int cflags, Oid collation, regex_t *result_re) { int text_re_len = VARSIZE_ANY_EXHDR(text_re); char *text_re_val = VARDATA_ANY(text_re); pg_wchar *pattern; int pattern_len; int regcomp_result; char errMsg[100]; /* Convert pattern string to wide characters */ pattern = (pg_wchar *) palloc((text_re_len + 1) * sizeof(pg_wchar)); pattern_len = pg_mb2wchar_with_len(text_re_val, pattern, text_re_len); regcomp_result = pg_regcomp(result_re, pattern, pattern_len, cflags, collation); pfree(pattern); if (regcomp_result != REG_OKAY) { /* re didn't compile (no need for pg_regfree, if so) */ /* * Here and in other places in this file, do CHECK_FOR_INTERRUPTS * before reporting a regex error. This is so that if the regex * library aborts and returns REG_CANCEL, we don't print an error * message that implies the regex was invalid. */ CHECK_FOR_INTERRUPTS(); pg_regerror(regcomp_result, result_re, errMsg, sizeof(errMsg)); ereport(ERROR, (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION), errmsg("invalid regular expression: %s", errMsg))); } } /* * test_re_execute - execute a RE on pg_wchar data * * Returns true on match, false on no match * Arguments are as for pg_regexec */ static bool test_re_execute(regex_t *re, pg_wchar *data, int data_len, int start_search, rm_detail_t *details, int nmatch, regmatch_t *pmatch, int eflags) { int regexec_result; char errMsg[100]; /* Initialize match locations in case engine doesn't */ details->rm_extend.rm_so = -1; details->rm_extend.rm_eo = -1; for (int i = 0; i < nmatch; i++) { pmatch[i].rm_so = -1; pmatch[i].rm_eo = -1; } /* Perform RE match and return result */ regexec_result = pg_regexec(re, data, data_len, start_search, details, nmatch, pmatch, eflags); if (regexec_result != REG_OKAY && regexec_result != REG_NOMATCH) { /* re failed??? */ CHECK_FOR_INTERRUPTS(); pg_regerror(regexec_result, re, errMsg, sizeof(errMsg)); ereport(ERROR, (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION), errmsg("regular expression failed: %s", errMsg))); } return (regexec_result == REG_OKAY); } /* * parse_test_flags - parse the flags argument * * flags --- output argument, filled with desired options * opts --- TEXT object, or NULL for defaults */ static void parse_test_flags(test_re_flags *flags, text *opts) { /* these defaults must match Tcl's */ int cflags = REG_ADVANCED; int eflags = 0; long info = 0; flags->glob = false; flags->indices = false; flags->partial = false; if (opts) { char *opt_p = VARDATA_ANY(opts); int opt_len = VARSIZE_ANY_EXHDR(opts); int i; for (i = 0; i < opt_len; i++) { switch (opt_p[i]) { case '-': /* allowed, no-op */ break; case '!': flags->partial = true; break; case '*': /* test requires Unicode --- ignored here */ break; case '0': flags->indices = true; break; /* These flags correspond to user-exposed RE options: */ case 'g': /* global match */ flags->glob = true; break; case 'i': /* case insensitive */ cflags |= REG_ICASE; break; case 'n': /* \n affects ^ $ . [^ */ cflags |= REG_NEWLINE; break; case 'p': /* ~Perl, \n affects . [^ */ cflags |= REG_NLSTOP; cflags &= ~REG_NLANCH; break; case 'w': /* weird, \n affects ^ $ only */ cflags &= ~REG_NLSTOP; cflags |= REG_NLANCH; break; case 'x': /* expanded syntax */ cflags |= REG_EXPANDED; break; /* These flags correspond to Tcl's -xflags options: */ case 'a': cflags |= REG_ADVF; break; case 'b': cflags &= ~REG_ADVANCED; break; case 'c': /* * Tcl calls this TCL_REG_CANMATCH, but it's really * REG_EXPECT. In this implementation we must also set * the partial and indices flags, so that * setup_test_matches and build_test_match_result will * emit the desired data. (They'll emit more fields than * Tcl would, but that's fine.) */ cflags |= REG_EXPECT; flags->partial = true; flags->indices = true; break; case 'e': cflags &= ~REG_ADVANCED; cflags |= REG_EXTENDED; break; case 'q': cflags &= ~REG_ADVANCED; cflags |= REG_QUOTE; break; case 'o': /* o for opaque */ cflags |= REG_NOSUB; break; case 's': /* s for start */ cflags |= REG_BOSONLY; break; case '+': cflags |= REG_FAKE; break; case ',': cflags |= REG_PROGRESS; break; case '.': cflags |= REG_DUMP; break; case ':': eflags |= REG_MTRACE; break; case ';': eflags |= REG_FTRACE; break; case '^': eflags |= REG_NOTBOL; break; case '$': eflags |= REG_NOTEOL; break; case 't': cflags |= REG_EXPECT; break; case '%': eflags |= REG_SMALL; break; /* These flags define expected info bits: */ case 'A': info |= REG_UBSALNUM; break; case 'B': info |= REG_UBRACES; break; case 'E': info |= REG_UBBS; break; case 'H': info |= REG_ULOOKAROUND; break; case 'I': info |= REG_UIMPOSSIBLE; break; case 'L': info |= REG_ULOCALE; break; case 'M': info |= REG_UUNPORT; break; case 'N': info |= REG_UEMPTYMATCH; break; case 'P': info |= REG_UNONPOSIX; break; case 'Q': info |= REG_UBOUNDS; break; case 'R': info |= REG_UBACKREF; break; case 'S': info |= REG_UUNSPEC; break; case 'T': info |= REG_USHORTEST; break; case 'U': info |= REG_UPBOTCH; break; default: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid regular expression test option: \"%.*s\"", pg_mblen(opt_p + i), opt_p + i))); break; } } } flags->cflags = cflags; flags->eflags = eflags; flags->info = info; } /* * setup_test_matches --- do the initial matching * * To simplify memory management, we do all the matching in one swoop. * The returned test_regex_ctx contains the locations of all the substrings * matching the pattern. */ static test_regex_ctx * setup_test_matches(text *orig_str, regex_t *cpattern, test_re_flags *re_flags, Oid collation, bool use_subpatterns) { test_regex_ctx *matchctx = palloc0(sizeof(test_regex_ctx)); int eml = pg_database_encoding_max_length(); int orig_len; pg_wchar *wide_str; int wide_len; regmatch_t *pmatch; int pmatch_len; int array_len; int array_idx; int prev_match_end; int start_search; int maxlen = 0; /* largest fetch length in characters */ /* save flags */ matchctx->re_flags = *re_flags; /* save original string --- we'll extract result substrings from it */ matchctx->orig_str = orig_str; /* convert string to pg_wchar form for matching */ orig_len = VARSIZE_ANY_EXHDR(orig_str); wide_str = (pg_wchar *) palloc(sizeof(pg_wchar) * (orig_len + 1)); wide_len = pg_mb2wchar_with_len(VARDATA_ANY(orig_str), wide_str, orig_len); /* do we want to remember subpatterns? */ if (use_subpatterns && cpattern->re_nsub > 0) { matchctx->npatterns = cpattern->re_nsub + 1; pmatch_len = cpattern->re_nsub + 1; } else { use_subpatterns = false; matchctx->npatterns = 1; pmatch_len = 1; } /* temporary output space for RE package */ pmatch = palloc(sizeof(regmatch_t) * pmatch_len); /* * the real output space (grown dynamically if needed) * * use values 2^n-1, not 2^n, so that we hit the limit at 2^28-1 rather * than at 2^27 */ array_len = re_flags->glob ? 255 : 31; matchctx->match_locs = (int *) palloc(sizeof(int) * array_len); array_idx = 0; /* search for the pattern, perhaps repeatedly */ prev_match_end = 0; start_search = 0; while (test_re_execute(cpattern, wide_str, wide_len, start_search, &matchctx->details, pmatch_len, pmatch, re_flags->eflags)) { /* enlarge output space if needed */ while (array_idx + matchctx->npatterns * 2 + 1 > array_len) { array_len += array_len + 1; /* 2^n-1 => 2^(n+1)-1 */ if (array_len > MaxAllocSize / sizeof(int)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("too many regular expression matches"))); matchctx->match_locs = (int *) repalloc(matchctx->match_locs, sizeof(int) * array_len); } /* save this match's locations */ for (int i = 0; i < matchctx->npatterns; i++) { int so = pmatch[i].rm_so; int eo = pmatch[i].rm_eo; matchctx->match_locs[array_idx++] = so; matchctx->match_locs[array_idx++] = eo; if (so >= 0 && eo >= 0 && (eo - so) > maxlen) maxlen = (eo - so); } matchctx->nmatches++; prev_match_end = pmatch[0].rm_eo; /* if not glob, stop after one match */ if (!re_flags->glob) break; /* * Advance search position. Normally we start the next search at the * end of the previous match; but if the match was of zero length, we * have to advance by one character, or we'd just find the same match * again. */ start_search = prev_match_end; if (pmatch[0].rm_so == pmatch[0].rm_eo) start_search++; if (start_search > wide_len) break; } /* * If we had no match, but "partial" and "indices" are set, emit the * details. */ if (matchctx->nmatches == 0 && re_flags->partial && re_flags->indices) { /* enlarge output space if needed */ while (array_idx + matchctx->npatterns * 2 + 1 > array_len) { array_len += array_len + 1; /* 2^n-1 => 2^(n+1)-1 */ if (array_len > MaxAllocSize / sizeof(int)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("too many regular expression matches"))); matchctx->match_locs = (int *) repalloc(matchctx->match_locs, sizeof(int) * array_len); } matchctx->match_locs[array_idx++] = matchctx->details.rm_extend.rm_so; matchctx->match_locs[array_idx++] = matchctx->details.rm_extend.rm_eo; /* we don't have pmatch data, so emit -1 */ for (int i = 1; i < matchctx->npatterns; i++) { matchctx->match_locs[array_idx++] = -1; matchctx->match_locs[array_idx++] = -1; } matchctx->nmatches++; } Assert(array_idx <= array_len); if (eml > 1) { int64 maxsiz = eml * (int64) maxlen; int conv_bufsiz; /* * Make the conversion buffer large enough for any substring of * interest. * * Worst case: assume we need the maximum size (maxlen*eml), but take * advantage of the fact that the original string length in bytes is * an upper bound on the byte length of any fetched substring (and we * know that len+1 is safe to allocate because the varlena header is * longer than 1 byte). */ if (maxsiz > orig_len) conv_bufsiz = orig_len + 1; else conv_bufsiz = maxsiz + 1; /* safe since maxsiz < 2^30 */ matchctx->conv_buf = palloc(conv_bufsiz); matchctx->conv_bufsiz = conv_bufsiz; matchctx->wide_str = wide_str; } else { /* No need to keep the wide string if we're in a single-byte charset. */ pfree(wide_str); matchctx->wide_str = NULL; matchctx->conv_buf = NULL; matchctx->conv_bufsiz = 0; } /* Clean up temp storage */ pfree(pmatch); return matchctx; } /* * build_test_info_result - build output array describing compiled regexp * * This borrows some code from Tcl's TclRegAbout(). */ static ArrayType * build_test_info_result(regex_t *cpattern, test_re_flags *flags) { /* Translation data for flag bits in regex_t.re_info */ struct infoname { int bit; const char *text; }; static const struct infoname infonames[] = { {REG_UBACKREF, "REG_UBACKREF"}, {REG_ULOOKAROUND, "REG_ULOOKAROUND"}, {REG_UBOUNDS, "REG_UBOUNDS"}, {REG_UBRACES, "REG_UBRACES"}, {REG_UBSALNUM, "REG_UBSALNUM"}, {REG_UPBOTCH, "REG_UPBOTCH"}, {REG_UBBS, "REG_UBBS"}, {REG_UNONPOSIX, "REG_UNONPOSIX"}, {REG_UUNSPEC, "REG_UUNSPEC"}, {REG_UUNPORT, "REG_UUNPORT"}, {REG_ULOCALE, "REG_ULOCALE"}, {REG_UEMPTYMATCH, "REG_UEMPTYMATCH"}, {REG_UIMPOSSIBLE, "REG_UIMPOSSIBLE"}, {REG_USHORTEST, "REG_USHORTEST"}, {0, NULL} }; const struct infoname *inf; Datum elems[lengthof(infonames) + 1]; int nresults = 0; char buf[80]; int dims[1]; int lbs[1]; /* Set up results: first, the number of subexpressions */ snprintf(buf, sizeof(buf), "%d", (int) cpattern->re_nsub); elems[nresults++] = PointerGetDatum(cstring_to_text(buf)); /* Report individual info bit states */ for (inf = infonames; inf->bit != 0; inf++) { if (cpattern->re_info & inf->bit) { if (flags->info & inf->bit) elems[nresults++] = PointerGetDatum(cstring_to_text(inf->text)); else { snprintf(buf, sizeof(buf), "unexpected %s!", inf->text); elems[nresults++] = PointerGetDatum(cstring_to_text(buf)); } } else { if (flags->info & inf->bit) { snprintf(buf, sizeof(buf), "missing %s!", inf->text); elems[nresults++] = PointerGetDatum(cstring_to_text(buf)); } } } /* And form an array */ dims[0] = nresults; lbs[0] = 1; /* XXX: this hardcodes assumptions about the text type */ return construct_md_array(elems, NULL, 1, dims, lbs, TEXTOID, -1, false, TYPALIGN_INT); } /* * build_test_match_result - build output array for current match * * Note that if the indices flag is set, we don't need any strings, * just the location data. */ static ArrayType * build_test_match_result(test_regex_ctx *matchctx) { char *buf = matchctx->conv_buf; Datum *elems = matchctx->elems; bool *nulls = matchctx->nulls; bool indices = matchctx->re_flags.indices; char bufstr[80]; int dims[1]; int lbs[1]; int loc; int i; /* Extract matching substrings from the original string */ loc = matchctx->next_match * matchctx->npatterns * 2; for (i = 0; i < matchctx->npatterns; i++) { int so = matchctx->match_locs[loc++]; int eo = matchctx->match_locs[loc++]; if (indices) { /* Report eo this way for consistency with Tcl */ snprintf(bufstr, sizeof(bufstr), "%d %d", so, so < 0 ? eo : eo - 1); elems[i] = PointerGetDatum(cstring_to_text(bufstr)); nulls[i] = false; } else if (so < 0 || eo < 0) { elems[i] = (Datum) 0; nulls[i] = true; } else if (buf) { int len = pg_wchar2mb_with_len(matchctx->wide_str + so, buf, eo - so); Assert(len < matchctx->conv_bufsiz); elems[i] = PointerGetDatum(cstring_to_text_with_len(buf, len)); nulls[i] = false; } else { elems[i] = DirectFunctionCall3(text_substr, PointerGetDatum(matchctx->orig_str), Int32GetDatum(so + 1), Int32GetDatum(eo - so)); nulls[i] = false; } } /* In EXPECT indices mode, also report the "details" */ if (indices && (matchctx->re_flags.cflags & REG_EXPECT)) { int so = matchctx->details.rm_extend.rm_so; int eo = matchctx->details.rm_extend.rm_eo; snprintf(bufstr, sizeof(bufstr), "%d %d", so, so < 0 ? eo : eo - 1); elems[i] = PointerGetDatum(cstring_to_text(bufstr)); nulls[i] = false; i++; } /* And form an array */ dims[0] = i; lbs[0] = 1; /* XXX: this hardcodes assumptions about the text type */ return construct_md_array(elems, nulls, 1, dims, lbs, TEXTOID, -1, false, TYPALIGN_INT); }