diff options
Diffstat (limited to 'src/backend/tsearch')
25 files changed, 10607 insertions, 0 deletions
diff --git a/src/backend/tsearch/Makefile b/src/backend/tsearch/Makefile new file mode 100644 index 0000000..cdb259e --- /dev/null +++ b/src/backend/tsearch/Makefile @@ -0,0 +1,54 @@ +#------------------------------------------------------------------------- +# +# Makefile for backend/tsearch +# +# Copyright (c) 2006-2021, PostgreSQL Global Development Group +# +# src/backend/tsearch/Makefile +# +#------------------------------------------------------------------------- +subdir = src/backend/tsearch +top_builddir = ../../.. +include $(top_builddir)/src/Makefile.global + +DICTDIR=tsearch_data + +# List of dictionaries files +DICTFILES=synonym_sample.syn thesaurus_sample.ths \ + hunspell_sample.affix \ + ispell_sample.affix ispell_sample.dict \ + hunspell_sample_long.affix hunspell_sample_long.dict \ + hunspell_sample_num.affix hunspell_sample_num.dict + +# Local paths to dictionaries files +DICTFILES_PATH=$(addprefix dicts/,$(DICTFILES)) + +OBJS = \ + dict.o \ + dict_ispell.o \ + dict_simple.o \ + dict_synonym.o \ + dict_thesaurus.o \ + regis.o \ + spell.o \ + to_tsany.o \ + ts_locale.o \ + ts_parse.o \ + ts_selfuncs.o \ + ts_typanalyze.o \ + ts_utils.o \ + wparser.o \ + wparser_def.o + +include $(top_srcdir)/src/backend/common.mk + +.PHONY: install-data +install-data: $(DICTFILES_PATH) installdirs + $(INSTALL_DATA) $(addprefix $(srcdir)/,$(DICTFILES_PATH)) '$(DESTDIR)$(datadir)/$(DICTDIR)/' + +installdirs: + $(MKDIR_P) '$(DESTDIR)$(datadir)' '$(DESTDIR)$(datadir)/$(DICTDIR)' + +.PHONY: uninstall-data +uninstall-data: + rm -rf $(addprefix '$(DESTDIR)$(datadir)/$(DICTDIR)/',$(DICTFILES)) diff --git a/src/backend/tsearch/dict.c b/src/backend/tsearch/dict.c new file mode 100644 index 0000000..1e1ccda --- /dev/null +++ b/src/backend/tsearch/dict.c @@ -0,0 +1,89 @@ +/*------------------------------------------------------------------------- + * + * dict.c + * Standard interface to dictionary + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/dict.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/pg_type.h" +#include "tsearch/ts_cache.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" + + +/* + * Lexize one word by dictionary, mostly debug function + */ +Datum +ts_lexize(PG_FUNCTION_ARGS) +{ + Oid dictId = PG_GETARG_OID(0); + text *in = PG_GETARG_TEXT_PP(1); + ArrayType *a; + TSDictionaryCacheEntry *dict; + TSLexeme *res, + *ptr; + Datum *da; + DictSubState dstate = {false, false, NULL}; + + dict = lookup_ts_dictionary_cache(dictId); + + res = (TSLexeme *) DatumGetPointer(FunctionCall4(&dict->lexize, + PointerGetDatum(dict->dictData), + PointerGetDatum(VARDATA_ANY(in)), + Int32GetDatum(VARSIZE_ANY_EXHDR(in)), + PointerGetDatum(&dstate))); + + if (dstate.getnext) + { + dstate.isend = true; + ptr = (TSLexeme *) DatumGetPointer(FunctionCall4(&dict->lexize, + PointerGetDatum(dict->dictData), + PointerGetDatum(VARDATA_ANY(in)), + Int32GetDatum(VARSIZE_ANY_EXHDR(in)), + PointerGetDatum(&dstate))); + if (ptr != NULL) + res = ptr; + } + + if (!res) + PG_RETURN_NULL(); + + ptr = res; + while (ptr->lexeme) + ptr++; + da = (Datum *) palloc(sizeof(Datum) * (ptr - res)); + ptr = res; + while (ptr->lexeme) + { + da[ptr - res] = CStringGetTextDatum(ptr->lexeme); + ptr++; + } + + a = construct_array(da, + ptr - res, + TEXTOID, + -1, + false, + TYPALIGN_INT); + + ptr = res; + while (ptr->lexeme) + { + pfree(DatumGetPointer(da[ptr - res])); + pfree(ptr->lexeme); + ptr++; + } + pfree(res); + pfree(da); + + PG_RETURN_POINTER(a); +} diff --git a/src/backend/tsearch/dict_ispell.c b/src/backend/tsearch/dict_ispell.c new file mode 100644 index 0000000..d93f601 --- /dev/null +++ b/src/backend/tsearch/dict_ispell.c @@ -0,0 +1,148 @@ +/*------------------------------------------------------------------------- + * + * dict_ispell.c + * Ispell dictionary interface + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/dict_ispell.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/defrem.h" +#include "tsearch/dicts/spell.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" + + +typedef struct +{ + StopList stoplist; + IspellDict obj; +} DictISpell; + +Datum +dispell_init(PG_FUNCTION_ARGS) +{ + List *dictoptions = (List *) PG_GETARG_POINTER(0); + DictISpell *d; + bool affloaded = false, + dictloaded = false, + stoploaded = false; + ListCell *l; + + d = (DictISpell *) palloc0(sizeof(DictISpell)); + + NIStartBuild(&(d->obj)); + + foreach(l, dictoptions) + { + DefElem *defel = (DefElem *) lfirst(l); + + if (strcmp(defel->defname, "dictfile") == 0) + { + if (dictloaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple DictFile parameters"))); + NIImportDictionary(&(d->obj), + get_tsearch_config_filename(defGetString(defel), + "dict")); + dictloaded = true; + } + else if (strcmp(defel->defname, "afffile") == 0) + { + if (affloaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple AffFile parameters"))); + NIImportAffixes(&(d->obj), + get_tsearch_config_filename(defGetString(defel), + "affix")); + affloaded = true; + } + else if (strcmp(defel->defname, "stopwords") == 0) + { + if (stoploaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple StopWords parameters"))); + readstoplist(defGetString(defel), &(d->stoplist), lowerstr); + stoploaded = true; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized Ispell parameter: \"%s\"", + defel->defname))); + } + } + + if (affloaded && dictloaded) + { + NISortDictionary(&(d->obj)); + NISortAffixes(&(d->obj)); + } + else if (!affloaded) + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing AffFile parameter"))); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing DictFile parameter"))); + } + + NIFinishBuild(&(d->obj)); + + PG_RETURN_POINTER(d); +} + +Datum +dispell_lexize(PG_FUNCTION_ARGS) +{ + DictISpell *d = (DictISpell *) PG_GETARG_POINTER(0); + char *in = (char *) PG_GETARG_POINTER(1); + int32 len = PG_GETARG_INT32(2); + char *txt; + TSLexeme *res; + TSLexeme *ptr, + *cptr; + + if (len <= 0) + PG_RETURN_POINTER(NULL); + + txt = lowerstr_with_len(in, len); + res = NINormalizeWord(&(d->obj), txt); + + if (res == NULL) + PG_RETURN_POINTER(NULL); + + cptr = res; + for (ptr = cptr; ptr->lexeme; ptr++) + { + if (searchstoplist(&(d->stoplist), ptr->lexeme)) + { + pfree(ptr->lexeme); + ptr->lexeme = NULL; + } + else + { + if (cptr != ptr) + memcpy(cptr, ptr, sizeof(TSLexeme)); + cptr++; + } + } + cptr->lexeme = NULL; + + PG_RETURN_POINTER(res); +} diff --git a/src/backend/tsearch/dict_simple.c b/src/backend/tsearch/dict_simple.c new file mode 100644 index 0000000..9cd4b6b --- /dev/null +++ b/src/backend/tsearch/dict_simple.c @@ -0,0 +1,105 @@ +/*------------------------------------------------------------------------- + * + * dict_simple.c + * Simple dictionary: just lowercase and check for stopword + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/dict_simple.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/defrem.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" + + +typedef struct +{ + StopList stoplist; + bool accept; +} DictSimple; + + +Datum +dsimple_init(PG_FUNCTION_ARGS) +{ + List *dictoptions = (List *) PG_GETARG_POINTER(0); + DictSimple *d = (DictSimple *) palloc0(sizeof(DictSimple)); + bool stoploaded = false, + acceptloaded = false; + ListCell *l; + + d->accept = true; /* default */ + + foreach(l, dictoptions) + { + DefElem *defel = (DefElem *) lfirst(l); + + if (strcmp(defel->defname, "stopwords") == 0) + { + if (stoploaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple StopWords parameters"))); + readstoplist(defGetString(defel), &d->stoplist, lowerstr); + stoploaded = true; + } + else if (strcmp(defel->defname, "accept") == 0) + { + if (acceptloaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple Accept parameters"))); + d->accept = defGetBoolean(defel); + acceptloaded = true; + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized simple dictionary parameter: \"%s\"", + defel->defname))); + } + } + + PG_RETURN_POINTER(d); +} + +Datum +dsimple_lexize(PG_FUNCTION_ARGS) +{ + DictSimple *d = (DictSimple *) PG_GETARG_POINTER(0); + char *in = (char *) PG_GETARG_POINTER(1); + int32 len = PG_GETARG_INT32(2); + char *txt; + TSLexeme *res; + + txt = lowerstr_with_len(in, len); + + if (*txt == '\0' || searchstoplist(&(d->stoplist), txt)) + { + /* reject as stopword */ + pfree(txt); + res = palloc0(sizeof(TSLexeme) * 2); + PG_RETURN_POINTER(res); + } + else if (d->accept) + { + /* accept */ + res = palloc0(sizeof(TSLexeme) * 2); + res[0].lexeme = txt; + PG_RETURN_POINTER(res); + } + else + { + /* report as unrecognized */ + pfree(txt); + PG_RETURN_POINTER(NULL); + } +} diff --git a/src/backend/tsearch/dict_synonym.c b/src/backend/tsearch/dict_synonym.c new file mode 100644 index 0000000..ed885ca --- /dev/null +++ b/src/backend/tsearch/dict_synonym.c @@ -0,0 +1,241 @@ +/*------------------------------------------------------------------------- + * + * dict_synonym.c + * Synonym dictionary: replace word by its synonym + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/dict_synonym.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "commands/defrem.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" + +typedef struct +{ + char *in; + char *out; + int outlen; + uint16 flags; +} Syn; + +typedef struct +{ + int len; /* length of syn array */ + Syn *syn; + bool case_sensitive; +} DictSyn; + +/* + * Finds the next whitespace-delimited word within the 'in' string. + * Returns a pointer to the first character of the word, and a pointer + * to the next byte after the last character in the word (in *end). + * Character '*' at the end of word will not be treated as word + * character if flags is not null. + */ +static char * +findwrd(char *in, char **end, uint16 *flags) +{ + char *start; + char *lastchar; + + /* Skip leading spaces */ + while (*in && t_isspace(in)) + in += pg_mblen(in); + + /* Return NULL on empty lines */ + if (*in == '\0') + { + *end = NULL; + return NULL; + } + + lastchar = start = in; + + /* Find end of word */ + while (*in && !t_isspace(in)) + { + lastchar = in; + in += pg_mblen(in); + } + + if (in - lastchar == 1 && t_iseq(lastchar, '*') && flags) + { + *flags = TSL_PREFIX; + *end = lastchar; + } + else + { + if (flags) + *flags = 0; + *end = in; + } + + return start; +} + +static int +compareSyn(const void *a, const void *b) +{ + return strcmp(((const Syn *) a)->in, ((const Syn *) b)->in); +} + + +Datum +dsynonym_init(PG_FUNCTION_ARGS) +{ + List *dictoptions = (List *) PG_GETARG_POINTER(0); + DictSyn *d; + ListCell *l; + char *filename = NULL; + bool case_sensitive = false; + tsearch_readline_state trst; + char *starti, + *starto, + *end = NULL; + int cur = 0; + char *line = NULL; + uint16 flags = 0; + + foreach(l, dictoptions) + { + DefElem *defel = (DefElem *) lfirst(l); + + if (strcmp(defel->defname, "synonyms") == 0) + filename = defGetString(defel); + else if (strcmp(defel->defname, "casesensitive") == 0) + case_sensitive = defGetBoolean(defel); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized synonym parameter: \"%s\"", + defel->defname))); + } + + if (!filename) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing Synonyms parameter"))); + + filename = get_tsearch_config_filename(filename, "syn"); + + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open synonym file \"%s\": %m", + filename))); + + d = (DictSyn *) palloc0(sizeof(DictSyn)); + + while ((line = tsearch_readline(&trst)) != NULL) + { + starti = findwrd(line, &end, NULL); + if (!starti) + { + /* Empty line */ + goto skipline; + } + if (*end == '\0') + { + /* A line with only one word. Ignore silently. */ + goto skipline; + } + *end = '\0'; + + starto = findwrd(end + 1, &end, &flags); + if (!starto) + { + /* A line with only one word (+whitespace). Ignore silently. */ + goto skipline; + } + *end = '\0'; + + /* + * starti now points to the first word, and starto to the second word + * on the line, with a \0 terminator at the end of both words. + */ + + if (cur >= d->len) + { + if (d->len == 0) + { + d->len = 64; + d->syn = (Syn *) palloc(sizeof(Syn) * d->len); + } + else + { + d->len *= 2; + d->syn = (Syn *) repalloc(d->syn, sizeof(Syn) * d->len); + } + } + + if (case_sensitive) + { + d->syn[cur].in = pstrdup(starti); + d->syn[cur].out = pstrdup(starto); + } + else + { + d->syn[cur].in = lowerstr(starti); + d->syn[cur].out = lowerstr(starto); + } + + d->syn[cur].outlen = strlen(starto); + d->syn[cur].flags = flags; + + cur++; + +skipline: + pfree(line); + } + + tsearch_readline_end(&trst); + + d->len = cur; + qsort(d->syn, d->len, sizeof(Syn), compareSyn); + + d->case_sensitive = case_sensitive; + + PG_RETURN_POINTER(d); +} + +Datum +dsynonym_lexize(PG_FUNCTION_ARGS) +{ + DictSyn *d = (DictSyn *) PG_GETARG_POINTER(0); + char *in = (char *) PG_GETARG_POINTER(1); + int32 len = PG_GETARG_INT32(2); + Syn key, + *found; + TSLexeme *res; + + /* note: d->len test protects against Solaris bsearch-of-no-items bug */ + if (len <= 0 || d->len <= 0) + PG_RETURN_POINTER(NULL); + + if (d->case_sensitive) + key.in = pnstrdup(in, len); + else + key.in = lowerstr_with_len(in, len); + + key.out = NULL; + + found = (Syn *) bsearch(&key, d->syn, d->len, sizeof(Syn), compareSyn); + pfree(key.in); + + if (!found) + PG_RETURN_POINTER(NULL); + + res = palloc0(sizeof(TSLexeme) * 2); + res[0].lexeme = pnstrdup(found->out, found->outlen); + res[0].flags = found->flags; + + PG_RETURN_POINTER(res); +} diff --git a/src/backend/tsearch/dict_thesaurus.c b/src/backend/tsearch/dict_thesaurus.c new file mode 100644 index 0000000..a95ed08 --- /dev/null +++ b/src/backend/tsearch/dict_thesaurus.c @@ -0,0 +1,877 @@ +/*------------------------------------------------------------------------- + * + * dict_thesaurus.c + * Thesaurus dictionary: phrase to phrase substitution + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/dict_thesaurus.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/namespace.h" +#include "commands/defrem.h" +#include "tsearch/ts_cache.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" +#include "utils/regproc.h" + + +/* + * Temporary we use TSLexeme.flags for inner use... + */ +#define DT_USEASIS 0x1000 + +typedef struct LexemeInfo +{ + uint32 idsubst; /* entry's number in DictThesaurus->subst */ + uint16 posinsubst; /* pos info in entry */ + uint16 tnvariant; /* total num lexemes in one variant */ + struct LexemeInfo *nextentry; + struct LexemeInfo *nextvariant; +} LexemeInfo; + +typedef struct +{ + char *lexeme; + LexemeInfo *entries; +} TheLexeme; + +typedef struct +{ + uint16 lastlexeme; /* number lexemes to substitute */ + uint16 reslen; + TSLexeme *res; /* prepared substituted result */ +} TheSubstitute; + +typedef struct +{ + /* subdictionary to normalize lexemes */ + Oid subdictOid; + TSDictionaryCacheEntry *subdict; + + /* Array to search lexeme by exact match */ + TheLexeme *wrds; + int nwrds; /* current number of words */ + int ntwrds; /* allocated array length */ + + /* + * Storage of substituted result, n-th element is for n-th expression + */ + TheSubstitute *subst; + int nsubst; +} DictThesaurus; + + +static void +newLexeme(DictThesaurus *d, char *b, char *e, uint32 idsubst, uint16 posinsubst) +{ + TheLexeme *ptr; + + if (d->nwrds >= d->ntwrds) + { + if (d->ntwrds == 0) + { + d->ntwrds = 16; + d->wrds = (TheLexeme *) palloc(sizeof(TheLexeme) * d->ntwrds); + } + else + { + d->ntwrds *= 2; + d->wrds = (TheLexeme *) repalloc(d->wrds, sizeof(TheLexeme) * d->ntwrds); + } + } + + ptr = d->wrds + d->nwrds; + d->nwrds++; + + ptr->lexeme = palloc(e - b + 1); + + memcpy(ptr->lexeme, b, e - b); + ptr->lexeme[e - b] = '\0'; + + ptr->entries = (LexemeInfo *) palloc(sizeof(LexemeInfo)); + + ptr->entries->nextentry = NULL; + ptr->entries->idsubst = idsubst; + ptr->entries->posinsubst = posinsubst; +} + +static void +addWrd(DictThesaurus *d, char *b, char *e, uint32 idsubst, uint16 nwrd, uint16 posinsubst, bool useasis) +{ + static int nres = 0; + static int ntres = 0; + TheSubstitute *ptr; + + if (nwrd == 0) + { + nres = ntres = 0; + + if (idsubst >= d->nsubst) + { + if (d->nsubst == 0) + { + d->nsubst = 16; + d->subst = (TheSubstitute *) palloc(sizeof(TheSubstitute) * d->nsubst); + } + else + { + d->nsubst *= 2; + d->subst = (TheSubstitute *) repalloc(d->subst, sizeof(TheSubstitute) * d->nsubst); + } + } + } + + ptr = d->subst + idsubst; + + ptr->lastlexeme = posinsubst - 1; + + if (nres + 1 >= ntres) + { + if (ntres == 0) + { + ntres = 2; + ptr->res = (TSLexeme *) palloc(sizeof(TSLexeme) * ntres); + } + else + { + ntres *= 2; + ptr->res = (TSLexeme *) repalloc(ptr->res, sizeof(TSLexeme) * ntres); + } + } + + ptr->res[nres].lexeme = palloc(e - b + 1); + memcpy(ptr->res[nres].lexeme, b, e - b); + ptr->res[nres].lexeme[e - b] = '\0'; + + ptr->res[nres].nvariant = nwrd; + if (useasis) + ptr->res[nres].flags = DT_USEASIS; + else + ptr->res[nres].flags = 0; + + ptr->res[++nres].lexeme = NULL; +} + +#define TR_WAITLEX 1 +#define TR_INLEX 2 +#define TR_WAITSUBS 3 +#define TR_INSUBS 4 + +static void +thesaurusRead(const char *filename, DictThesaurus *d) +{ + tsearch_readline_state trst; + uint32 idsubst = 0; + bool useasis = false; + char *line; + + filename = get_tsearch_config_filename(filename, "ths"); + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open thesaurus file \"%s\": %m", + filename))); + + while ((line = tsearch_readline(&trst)) != NULL) + { + char *ptr; + int state = TR_WAITLEX; + char *beginwrd = NULL; + uint32 posinsubst = 0; + uint32 nwrd = 0; + + ptr = line; + + /* is it a comment? */ + while (*ptr && t_isspace(ptr)) + ptr += pg_mblen(ptr); + + if (t_iseq(ptr, '#') || *ptr == '\0' || + t_iseq(ptr, '\n') || t_iseq(ptr, '\r')) + { + pfree(line); + continue; + } + + while (*ptr) + { + if (state == TR_WAITLEX) + { + if (t_iseq(ptr, ':')) + { + if (posinsubst == 0) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("unexpected delimiter"))); + state = TR_WAITSUBS; + } + else if (!t_isspace(ptr)) + { + beginwrd = ptr; + state = TR_INLEX; + } + } + else if (state == TR_INLEX) + { + if (t_iseq(ptr, ':')) + { + newLexeme(d, beginwrd, ptr, idsubst, posinsubst++); + state = TR_WAITSUBS; + } + else if (t_isspace(ptr)) + { + newLexeme(d, beginwrd, ptr, idsubst, posinsubst++); + state = TR_WAITLEX; + } + } + else if (state == TR_WAITSUBS) + { + if (t_iseq(ptr, '*')) + { + useasis = true; + state = TR_INSUBS; + beginwrd = ptr + pg_mblen(ptr); + } + else if (t_iseq(ptr, '\\')) + { + useasis = false; + state = TR_INSUBS; + beginwrd = ptr + pg_mblen(ptr); + } + else if (!t_isspace(ptr)) + { + useasis = false; + beginwrd = ptr; + state = TR_INSUBS; + } + } + else if (state == TR_INSUBS) + { + if (t_isspace(ptr)) + { + if (ptr == beginwrd) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("unexpected end of line or lexeme"))); + addWrd(d, beginwrd, ptr, idsubst, nwrd++, posinsubst, useasis); + state = TR_WAITSUBS; + } + } + else + elog(ERROR, "unrecognized thesaurus state: %d", state); + + ptr += pg_mblen(ptr); + } + + if (state == TR_INSUBS) + { + if (ptr == beginwrd) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("unexpected end of line or lexeme"))); + addWrd(d, beginwrd, ptr, idsubst, nwrd++, posinsubst, useasis); + } + + idsubst++; + + if (!(nwrd && posinsubst)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("unexpected end of line"))); + + if (nwrd != (uint16) nwrd || posinsubst != (uint16) posinsubst) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("too many lexemes in thesaurus entry"))); + + pfree(line); + } + + d->nsubst = idsubst; + + tsearch_readline_end(&trst); +} + +static TheLexeme * +addCompiledLexeme(TheLexeme *newwrds, int *nnw, int *tnm, TSLexeme *lexeme, LexemeInfo *src, uint16 tnvariant) +{ + if (*nnw >= *tnm) + { + *tnm *= 2; + newwrds = (TheLexeme *) repalloc(newwrds, sizeof(TheLexeme) * *tnm); + } + + newwrds[*nnw].entries = (LexemeInfo *) palloc(sizeof(LexemeInfo)); + + if (lexeme && lexeme->lexeme) + { + newwrds[*nnw].lexeme = pstrdup(lexeme->lexeme); + newwrds[*nnw].entries->tnvariant = tnvariant; + } + else + { + newwrds[*nnw].lexeme = NULL; + newwrds[*nnw].entries->tnvariant = 1; + } + + newwrds[*nnw].entries->idsubst = src->idsubst; + newwrds[*nnw].entries->posinsubst = src->posinsubst; + + newwrds[*nnw].entries->nextentry = NULL; + + (*nnw)++; + return newwrds; +} + +static int +cmpLexemeInfo(LexemeInfo *a, LexemeInfo *b) +{ + if (a == NULL || b == NULL) + return 0; + + if (a->idsubst == b->idsubst) + { + if (a->posinsubst == b->posinsubst) + { + if (a->tnvariant == b->tnvariant) + return 0; + + return (a->tnvariant > b->tnvariant) ? 1 : -1; + } + + return (a->posinsubst > b->posinsubst) ? 1 : -1; + } + + return (a->idsubst > b->idsubst) ? 1 : -1; +} + +static int +cmpLexeme(const TheLexeme *a, const TheLexeme *b) +{ + if (a->lexeme == NULL) + { + if (b->lexeme == NULL) + return 0; + else + return 1; + } + else if (b->lexeme == NULL) + return -1; + + return strcmp(a->lexeme, b->lexeme); +} + +static int +cmpLexemeQ(const void *a, const void *b) +{ + return cmpLexeme((const TheLexeme *) a, (const TheLexeme *) b); +} + +static int +cmpTheLexeme(const void *a, const void *b) +{ + const TheLexeme *la = (const TheLexeme *) a; + const TheLexeme *lb = (const TheLexeme *) b; + int res; + + if ((res = cmpLexeme(la, lb)) != 0) + return res; + + return -cmpLexemeInfo(la->entries, lb->entries); +} + +static void +compileTheLexeme(DictThesaurus *d) +{ + int i, + nnw = 0, + tnm = 16; + TheLexeme *newwrds = (TheLexeme *) palloc(sizeof(TheLexeme) * tnm), + *ptrwrds; + + for (i = 0; i < d->nwrds; i++) + { + TSLexeme *ptr; + + if (strcmp(d->wrds[i].lexeme, "?") == 0) /* Is stop word marker? */ + newwrds = addCompiledLexeme(newwrds, &nnw, &tnm, NULL, d->wrds[i].entries, 0); + else + { + ptr = (TSLexeme *) DatumGetPointer(FunctionCall4(&(d->subdict->lexize), + PointerGetDatum(d->subdict->dictData), + PointerGetDatum(d->wrds[i].lexeme), + Int32GetDatum(strlen(d->wrds[i].lexeme)), + PointerGetDatum(NULL))); + + if (!ptr) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("thesaurus sample word \"%s\" isn't recognized by subdictionary (rule %d)", + d->wrds[i].lexeme, + d->wrds[i].entries->idsubst + 1))); + else if (!(ptr->lexeme)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("thesaurus sample word \"%s\" is a stop word (rule %d)", + d->wrds[i].lexeme, + d->wrds[i].entries->idsubst + 1), + errhint("Use \"?\" to represent a stop word within a sample phrase."))); + else + { + while (ptr->lexeme) + { + TSLexeme *remptr = ptr + 1; + int tnvar = 1; + int curvar = ptr->nvariant; + + /* compute n words in one variant */ + while (remptr->lexeme) + { + if (remptr->nvariant != (remptr - 1)->nvariant) + break; + tnvar++; + remptr++; + } + + remptr = ptr; + while (remptr->lexeme && remptr->nvariant == curvar) + { + newwrds = addCompiledLexeme(newwrds, &nnw, &tnm, remptr, d->wrds[i].entries, tnvar); + remptr++; + } + + ptr = remptr; + } + } + } + + pfree(d->wrds[i].lexeme); + pfree(d->wrds[i].entries); + } + + if (d->wrds) + pfree(d->wrds); + d->wrds = newwrds; + d->nwrds = nnw; + d->ntwrds = tnm; + + if (d->nwrds > 1) + { + qsort(d->wrds, d->nwrds, sizeof(TheLexeme), cmpTheLexeme); + + /* uniq */ + newwrds = d->wrds; + ptrwrds = d->wrds + 1; + while (ptrwrds - d->wrds < d->nwrds) + { + if (cmpLexeme(ptrwrds, newwrds) == 0) + { + if (cmpLexemeInfo(ptrwrds->entries, newwrds->entries)) + { + ptrwrds->entries->nextentry = newwrds->entries; + newwrds->entries = ptrwrds->entries; + } + else + pfree(ptrwrds->entries); + + if (ptrwrds->lexeme) + pfree(ptrwrds->lexeme); + } + else + { + newwrds++; + *newwrds = *ptrwrds; + } + + ptrwrds++; + } + + d->nwrds = newwrds - d->wrds + 1; + d->wrds = (TheLexeme *) repalloc(d->wrds, sizeof(TheLexeme) * d->nwrds); + } +} + +static void +compileTheSubstitute(DictThesaurus *d) +{ + int i; + + for (i = 0; i < d->nsubst; i++) + { + TSLexeme *rem = d->subst[i].res, + *outptr, + *inptr; + int n = 2; + + outptr = d->subst[i].res = (TSLexeme *) palloc(sizeof(TSLexeme) * n); + outptr->lexeme = NULL; + inptr = rem; + + while (inptr && inptr->lexeme) + { + TSLexeme *lexized, + tmplex[2]; + + if (inptr->flags & DT_USEASIS) + { /* do not lexize */ + tmplex[0] = *inptr; + tmplex[0].flags = 0; + tmplex[1].lexeme = NULL; + lexized = tmplex; + } + else + { + lexized = (TSLexeme *) DatumGetPointer(FunctionCall4(&(d->subdict->lexize), + PointerGetDatum(d->subdict->dictData), + PointerGetDatum(inptr->lexeme), + Int32GetDatum(strlen(inptr->lexeme)), + PointerGetDatum(NULL))); + } + + if (lexized && lexized->lexeme) + { + int toset = (lexized->lexeme && outptr != d->subst[i].res) ? (outptr - d->subst[i].res) : -1; + + while (lexized->lexeme) + { + if (outptr - d->subst[i].res + 1 >= n) + { + int diff = outptr - d->subst[i].res; + + n *= 2; + d->subst[i].res = (TSLexeme *) repalloc(d->subst[i].res, sizeof(TSLexeme) * n); + outptr = d->subst[i].res + diff; + } + + *outptr = *lexized; + outptr->lexeme = pstrdup(lexized->lexeme); + + outptr++; + lexized++; + } + + if (toset > 0) + d->subst[i].res[toset].flags |= TSL_ADDPOS; + } + else if (lexized) + { + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("thesaurus substitute word \"%s\" is a stop word (rule %d)", + inptr->lexeme, i + 1))); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("thesaurus substitute word \"%s\" isn't recognized by subdictionary (rule %d)", + inptr->lexeme, i + 1))); + } + + if (inptr->lexeme) + pfree(inptr->lexeme); + inptr++; + } + + if (outptr == d->subst[i].res) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("thesaurus substitute phrase is empty (rule %d)", + i + 1))); + + d->subst[i].reslen = outptr - d->subst[i].res; + + pfree(rem); + } +} + +Datum +thesaurus_init(PG_FUNCTION_ARGS) +{ + List *dictoptions = (List *) PG_GETARG_POINTER(0); + DictThesaurus *d; + char *subdictname = NULL; + bool fileloaded = false; + ListCell *l; + + d = (DictThesaurus *) palloc0(sizeof(DictThesaurus)); + + foreach(l, dictoptions) + { + DefElem *defel = (DefElem *) lfirst(l); + + if (strcmp(defel->defname, "dictfile") == 0) + { + if (fileloaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple DictFile parameters"))); + thesaurusRead(defGetString(defel), d); + fileloaded = true; + } + else if (strcmp(defel->defname, "dictionary") == 0) + { + if (subdictname) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("multiple Dictionary parameters"))); + subdictname = pstrdup(defGetString(defel)); + } + else + { + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized Thesaurus parameter: \"%s\"", + defel->defname))); + } + } + + if (!fileloaded) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing DictFile parameter"))); + if (!subdictname) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("missing Dictionary parameter"))); + + d->subdictOid = get_ts_dict_oid(stringToQualifiedNameList(subdictname), false); + d->subdict = lookup_ts_dictionary_cache(d->subdictOid); + + compileTheLexeme(d); + compileTheSubstitute(d); + + PG_RETURN_POINTER(d); +} + +static LexemeInfo * +findTheLexeme(DictThesaurus *d, char *lexeme) +{ + TheLexeme key, + *res; + + if (d->nwrds == 0) + return NULL; + + key.lexeme = lexeme; + key.entries = NULL; + + res = bsearch(&key, d->wrds, d->nwrds, sizeof(TheLexeme), cmpLexemeQ); + + if (res == NULL) + return NULL; + return res->entries; +} + +static bool +matchIdSubst(LexemeInfo *stored, uint32 idsubst) +{ + bool res = true; + + if (stored) + { + res = false; + + for (; stored; stored = stored->nextvariant) + if (stored->idsubst == idsubst) + { + res = true; + break; + } + } + + return res; +} + +static LexemeInfo * +findVariant(LexemeInfo *in, LexemeInfo *stored, uint16 curpos, LexemeInfo **newin, int newn) +{ + for (;;) + { + int i; + LexemeInfo *ptr = newin[0]; + + for (i = 0; i < newn; i++) + { + while (newin[i] && newin[i]->idsubst < ptr->idsubst) + newin[i] = newin[i]->nextentry; + + if (newin[i] == NULL) + return in; + + if (newin[i]->idsubst > ptr->idsubst) + { + ptr = newin[i]; + i = -1; + continue; + } + + while (newin[i]->idsubst == ptr->idsubst) + { + if (newin[i]->posinsubst == curpos && newin[i]->tnvariant == newn) + { + ptr = newin[i]; + break; + } + + newin[i] = newin[i]->nextentry; + if (newin[i] == NULL) + return in; + } + + if (newin[i]->idsubst != ptr->idsubst) + { + ptr = newin[i]; + i = -1; + continue; + } + } + + if (i == newn && matchIdSubst(stored, ptr->idsubst) && (in == NULL || !matchIdSubst(in, ptr->idsubst))) + { /* found */ + + ptr->nextvariant = in; + in = ptr; + } + + /* step forward */ + for (i = 0; i < newn; i++) + newin[i] = newin[i]->nextentry; + } +} + +static TSLexeme * +copyTSLexeme(TheSubstitute *ts) +{ + TSLexeme *res; + uint16 i; + + res = (TSLexeme *) palloc(sizeof(TSLexeme) * (ts->reslen + 1)); + for (i = 0; i < ts->reslen; i++) + { + res[i] = ts->res[i]; + res[i].lexeme = pstrdup(ts->res[i].lexeme); + } + + res[ts->reslen].lexeme = NULL; + + return res; +} + +static TSLexeme * +checkMatch(DictThesaurus *d, LexemeInfo *info, uint16 curpos, bool *moreres) +{ + *moreres = false; + while (info) + { + Assert(info->idsubst < d->nsubst); + if (info->nextvariant) + *moreres = true; + if (d->subst[info->idsubst].lastlexeme == curpos) + return copyTSLexeme(d->subst + info->idsubst); + info = info->nextvariant; + } + + return NULL; +} + +Datum +thesaurus_lexize(PG_FUNCTION_ARGS) +{ + DictThesaurus *d = (DictThesaurus *) PG_GETARG_POINTER(0); + DictSubState *dstate = (DictSubState *) PG_GETARG_POINTER(3); + TSLexeme *res = NULL; + LexemeInfo *stored, + *info = NULL; + uint16 curpos = 0; + bool moreres = false; + + if (PG_NARGS() != 4 || dstate == NULL) + elog(ERROR, "forbidden call of thesaurus or nested call"); + + if (dstate->isend) + PG_RETURN_POINTER(NULL); + stored = (LexemeInfo *) dstate->private_state; + + if (stored) + curpos = stored->posinsubst + 1; + + if (!d->subdict->isvalid) + d->subdict = lookup_ts_dictionary_cache(d->subdictOid); + + res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(d->subdict->lexize), + PointerGetDatum(d->subdict->dictData), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2), + PointerGetDatum(NULL))); + + if (res && res->lexeme) + { + TSLexeme *ptr = res, + *basevar; + + while (ptr->lexeme) + { + uint16 nv = ptr->nvariant; + uint16 i, + nlex = 0; + LexemeInfo **infos; + + basevar = ptr; + while (ptr->lexeme && nv == ptr->nvariant) + { + nlex++; + ptr++; + } + + infos = (LexemeInfo **) palloc(sizeof(LexemeInfo *) * nlex); + for (i = 0; i < nlex; i++) + if ((infos[i] = findTheLexeme(d, basevar[i].lexeme)) == NULL) + break; + + if (i < nlex) + { + /* no chance to find */ + pfree(infos); + continue; + } + + info = findVariant(info, stored, curpos, infos, nlex); + } + } + else if (res) + { /* stop-word */ + LexemeInfo *infos = findTheLexeme(d, NULL); + + info = findVariant(NULL, stored, curpos, &infos, 1); + } + else + { + info = NULL; /* word isn't recognized */ + } + + dstate->private_state = (void *) info; + + if (!info) + { + dstate->getnext = false; + PG_RETURN_POINTER(NULL); + } + + if ((res = checkMatch(d, info, curpos, &moreres)) != NULL) + { + dstate->getnext = moreres; + PG_RETURN_POINTER(res); + } + + dstate->getnext = true; + + PG_RETURN_POINTER(NULL); +} diff --git a/src/backend/tsearch/dicts/hunspell_sample.affix b/src/backend/tsearch/dicts/hunspell_sample.affix new file mode 100644 index 0000000..9a64513 --- /dev/null +++ b/src/backend/tsearch/dicts/hunspell_sample.affix @@ -0,0 +1,23 @@ +COMPOUNDFLAG Z +ONLYINCOMPOUND L + +PFX B Y 1 +PFX B 0 re . + +PFX U N 1 +PFX U 0 un . + +SFX J Y 1 +SFX J 0 INGS [^E] + +SFX G Y 1 +SFX G 0 ING [^E] + +SFX S Y 1 +SFX S 0 S [^SXZHY] + +SFX A Y 1 +SFX A Y IES [^AEIOU]Y + +SFX \ N 1 +SFX \ 0 Y/L [^Y] diff --git a/src/backend/tsearch/dicts/hunspell_sample_long.affix b/src/backend/tsearch/dicts/hunspell_sample_long.affix new file mode 100644 index 0000000..d5df7a3 --- /dev/null +++ b/src/backend/tsearch/dicts/hunspell_sample_long.affix @@ -0,0 +1,53 @@ +FLAG long + +AF 11 +AF cZ #1 +AF cL #2 +AF sGsJpUsS #3 +AF sSpB #4 +AF cZsS #5 +AF sScZs\sE #6 +AF sA #7 +AF CaCp #8 +AF CcCp #9 +AF sD #10 +AF sB #11 + +COMPOUNDFLAG cZ +COMPOUNDBEGIN Ca +COMPOUNDMIDDLE Cb +COMPOUNDEND Cc +COMPOUNDPERMITFLAG Cp +ONLYINCOMPOUND cL + +PFX pB Y 1 +PFX pB 0 re . + +PFX pU N 1 +PFX pU 0 un . + +SFX sJ Y 1 +SFX sJ 0 INGS [^E] + +SFX sG Y 1 +SFX sG 0 ING [^E] + +SFX sS Y 1 +SFX sS 0 S [^SXZHY] + +SFX sA Y 1 +SFX sA Y IES [^AEIOU]Y{1} + +SFX sB Y 1 +SFX sB 0 ED K{1} + +# Affixes with compound flags +SFX s\ N 1 +SFX s\ 0 Y/2 [^Y] + +SFX sE N 1 +SFX sE 0 S/2 [^S] + +# Check duplicate affixes +SFX sD N 1 +SFX sD 0 S/2 [^S] diff --git a/src/backend/tsearch/dicts/hunspell_sample_long.dict b/src/backend/tsearch/dicts/hunspell_sample_long.dict new file mode 100644 index 0000000..370c27a --- /dev/null +++ b/src/backend/tsearch/dicts/hunspell_sample_long.dict @@ -0,0 +1,11 @@ +book/3 +book/11 +booking/4 +footballklubber +foot/5 +football/1 +ball/6 +klubber/1 +sky/7 +ex-/8 +machina/9 diff --git a/src/backend/tsearch/dicts/hunspell_sample_num.affix b/src/backend/tsearch/dicts/hunspell_sample_num.affix new file mode 100644 index 0000000..0c4766a --- /dev/null +++ b/src/backend/tsearch/dicts/hunspell_sample_num.affix @@ -0,0 +1,33 @@ +FLAG num + +COMPOUNDFLAG 101 +ONLYINCOMPOUND 102 + +PFX 201 Y 1 +PFX 201 0 re . + +PFX 202 N 1 +PFX 202 0 un . + +SFX 301 Y 1 +SFX 301 0 INGS [^E] + +SFX 302 Y 1 +SFX 302 0 ING [^E] + +SFX 303 Y 1 +SFX 303 0 S [^SXZHY] + +# Remove ED suffix from lexeme for base words with K ending +SFX 306 Y 1 +SFX 306 0 ED K{1} + +# Just add Y to lexeme for base words with Y ending +SFX 307 Y 1 +SFX 307 Y 0 Y* + +SFX 304 Y 1 +SFX 304 Y IES [^AEIOU]Y + +SFX 305 N 1 +SFX 305 0 Y/102 [^Y] diff --git a/src/backend/tsearch/dicts/hunspell_sample_num.dict b/src/backend/tsearch/dicts/hunspell_sample_num.dict new file mode 100644 index 0000000..fbc321d --- /dev/null +++ b/src/backend/tsearch/dicts/hunspell_sample_num.dict @@ -0,0 +1,9 @@ +book/302,301,202,303 +book/306 +booking/303,201 +footballklubber +foot/101,303 +football/101 +ball/303,101,305 +klubber/101 +sky/304,307 diff --git a/src/backend/tsearch/dicts/ispell_sample.affix b/src/backend/tsearch/dicts/ispell_sample.affix new file mode 100644 index 0000000..f29004f --- /dev/null +++ b/src/backend/tsearch/dicts/ispell_sample.affix @@ -0,0 +1,26 @@ +compoundwords controlled Z + +prefixes + +flag *B: + . > RE # As in enter > reenter + +flag U: + . > UN # As in natural > unnatural + +suffixes + +flag *J: + [^E] > INGS # As in cross > crossings + +flag *G: + [^E] > ING # As in cross > crossing + +flag *S: + [^SXZHY] > S # As in bat > bats + +flag *A: + [^AEIOU]Y > -Y,IES # As in imply > implies + +flag ~\\: + [^Y] > Y #~ advarsel > advarsely- diff --git a/src/backend/tsearch/dicts/ispell_sample.dict b/src/backend/tsearch/dicts/ispell_sample.dict new file mode 100644 index 0000000..44df196 --- /dev/null +++ b/src/backend/tsearch/dicts/ispell_sample.dict @@ -0,0 +1,8 @@ +book/GJUS +booking/SB +footballklubber +foot/ZS +football/Z +ball/SZ\ +klubber/Z +sky/A diff --git a/src/backend/tsearch/dicts/synonym_sample.syn b/src/backend/tsearch/dicts/synonym_sample.syn new file mode 100644 index 0000000..3ecbcf9 --- /dev/null +++ b/src/backend/tsearch/dicts/synonym_sample.syn @@ -0,0 +1,5 @@ +postgres pgsql +postgresql pgsql +postgre pgsql +gogle googl +indices index* diff --git a/src/backend/tsearch/dicts/thesaurus_sample.ths b/src/backend/tsearch/dicts/thesaurus_sample.ths new file mode 100644 index 0000000..718f54a --- /dev/null +++ b/src/backend/tsearch/dicts/thesaurus_sample.ths @@ -0,0 +1,17 @@ +# +# Theasurus config file. Character ':' separates string from replacement, eg +# sample-words : substitute-words +# +# Any substitute-word can be marked by preceding '*' character, +# which means do not lexize this word +# Docs: http://www.sai.msu.su/~megera/oddmuse/index.cgi/Thesaurus_dictionary + +one two three : *123 +one two : *12 +one : *1 +two : *2 + +supernovae stars : *sn +supernovae : *sn +booking tickets : order invitation cards +booking ? tickets : order invitation Cards diff --git a/src/backend/tsearch/regis.c b/src/backend/tsearch/regis.c new file mode 100644 index 0000000..8001717 --- /dev/null +++ b/src/backend/tsearch/regis.c @@ -0,0 +1,257 @@ +/*------------------------------------------------------------------------- + * + * regis.c + * Fast regex subset + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/regis.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "tsearch/dicts/regis.h" +#include "tsearch/ts_locale.h" + +#define RS_IN_ONEOF 1 +#define RS_IN_ONEOF_IN 2 +#define RS_IN_NONEOF 3 +#define RS_IN_WAIT 4 + + +/* + * Test whether a regex is of the subset supported here. + * Keep this in sync with RS_compile! + */ +bool +RS_isRegis(const char *str) +{ + int state = RS_IN_WAIT; + const char *c = str; + + while (*c) + { + if (state == RS_IN_WAIT) + { + if (t_isalpha(c)) + /* okay */ ; + else if (t_iseq(c, '[')) + state = RS_IN_ONEOF; + else + return false; + } + else if (state == RS_IN_ONEOF) + { + if (t_iseq(c, '^')) + state = RS_IN_NONEOF; + else if (t_isalpha(c)) + state = RS_IN_ONEOF_IN; + else + return false; + } + else if (state == RS_IN_ONEOF_IN || state == RS_IN_NONEOF) + { + if (t_isalpha(c)) + /* okay */ ; + else if (t_iseq(c, ']')) + state = RS_IN_WAIT; + else + return false; + } + else + elog(ERROR, "internal error in RS_isRegis: state %d", state); + c += pg_mblen(c); + } + + return (state == RS_IN_WAIT); +} + +static RegisNode * +newRegisNode(RegisNode *prev, int len) +{ + RegisNode *ptr; + + ptr = (RegisNode *) palloc0(RNHDRSZ + len + 1); + if (prev) + prev->next = ptr; + return ptr; +} + +void +RS_compile(Regis *r, bool issuffix, const char *str) +{ + int len = strlen(str); + int state = RS_IN_WAIT; + const char *c = str; + RegisNode *ptr = NULL; + + memset(r, 0, sizeof(Regis)); + r->issuffix = (issuffix) ? 1 : 0; + + while (*c) + { + if (state == RS_IN_WAIT) + { + if (t_isalpha(c)) + { + if (ptr) + ptr = newRegisNode(ptr, len); + else + ptr = r->node = newRegisNode(NULL, len); + COPYCHAR(ptr->data, c); + ptr->type = RSF_ONEOF; + ptr->len = pg_mblen(c); + } + else if (t_iseq(c, '[')) + { + if (ptr) + ptr = newRegisNode(ptr, len); + else + ptr = r->node = newRegisNode(NULL, len); + ptr->type = RSF_ONEOF; + state = RS_IN_ONEOF; + } + else /* shouldn't get here */ + elog(ERROR, "invalid regis pattern: \"%s\"", str); + } + else if (state == RS_IN_ONEOF) + { + if (t_iseq(c, '^')) + { + ptr->type = RSF_NONEOF; + state = RS_IN_NONEOF; + } + else if (t_isalpha(c)) + { + COPYCHAR(ptr->data, c); + ptr->len = pg_mblen(c); + state = RS_IN_ONEOF_IN; + } + else /* shouldn't get here */ + elog(ERROR, "invalid regis pattern: \"%s\"", str); + } + else if (state == RS_IN_ONEOF_IN || state == RS_IN_NONEOF) + { + if (t_isalpha(c)) + { + COPYCHAR(ptr->data + ptr->len, c); + ptr->len += pg_mblen(c); + } + else if (t_iseq(c, ']')) + state = RS_IN_WAIT; + else /* shouldn't get here */ + elog(ERROR, "invalid regis pattern: \"%s\"", str); + } + else + elog(ERROR, "internal error in RS_compile: state %d", state); + c += pg_mblen(c); + } + + if (state != RS_IN_WAIT) /* shouldn't get here */ + elog(ERROR, "invalid regis pattern: \"%s\"", str); + + ptr = r->node; + while (ptr) + { + r->nchar++; + ptr = ptr->next; + } +} + +void +RS_free(Regis *r) +{ + RegisNode *ptr = r->node, + *tmp; + + while (ptr) + { + tmp = ptr->next; + pfree(ptr); + ptr = tmp; + } + + r->node = NULL; +} + +static bool +mb_strchr(char *str, char *c) +{ + int clen, + plen, + i; + char *ptr = str; + bool res = false; + + clen = pg_mblen(c); + while (*ptr && !res) + { + plen = pg_mblen(ptr); + if (plen == clen) + { + i = plen; + res = true; + while (i--) + if (*(ptr + i) != *(c + i)) + { + res = false; + break; + } + } + + ptr += plen; + } + + return res; +} + +bool +RS_execute(Regis *r, char *str) +{ + RegisNode *ptr = r->node; + char *c = str; + int len = 0; + + while (*c) + { + len++; + c += pg_mblen(c); + } + + if (len < r->nchar) + return 0; + + c = str; + if (r->issuffix) + { + len -= r->nchar; + while (len-- > 0) + c += pg_mblen(c); + } + + + while (ptr) + { + switch (ptr->type) + { + case RSF_ONEOF: + if (!mb_strchr((char *) ptr->data, c)) + return false; + break; + case RSF_NONEOF: + if (mb_strchr((char *) ptr->data, c)) + return false; + break; + default: + elog(ERROR, "unrecognized regis node type: %d", ptr->type); + } + ptr = ptr->next; + c += pg_mblen(c); + } + + return true; +} diff --git a/src/backend/tsearch/spell.c b/src/backend/tsearch/spell.c new file mode 100644 index 0000000..ebc8960 --- /dev/null +++ b/src/backend/tsearch/spell.c @@ -0,0 +1,2617 @@ +/*------------------------------------------------------------------------- + * + * spell.c + * Normalizing word with ISpell + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * Ispell dictionary + * ----------------- + * + * Rules of dictionaries are defined in two files with .affix and .dict + * extensions. They are used by spell checker programs Ispell and Hunspell. + * + * An .affix file declares morphological rules to get a basic form of words. + * The format of an .affix file has different structure for Ispell and Hunspell + * dictionaries. The Hunspell format is more complicated. But when an .affix + * file is imported and compiled, it is stored in the same structure AffixNode. + * + * A .dict file stores a list of basic forms of words with references to + * affix rules. The format of a .dict file has the same structure for Ispell + * and Hunspell dictionaries. + * + * Compilation of a dictionary + * --------------------------- + * + * A compiled dictionary is stored in the IspellDict structure. Compilation of + * a dictionary is divided into the several steps: + * - NIImportDictionary() - stores each word of a .dict file in the + * temporary Spell field. + * - NIImportAffixes() - stores affix rules of an .affix file in the + * Affix field (not temporary) if an .affix file has the Ispell format. + * -> NIImportOOAffixes() - stores affix rules if an .affix file has the + * Hunspell format. The AffixData field is initialized if AF parameter + * is defined. + * - NISortDictionary() - builds a prefix tree (Trie) from the words list + * and stores it in the Dictionary field. The words list is got from the + * Spell field. The AffixData field is initialized if AF parameter is not + * defined. + * - NISortAffixes(): + * - builds a list of compound affixes from the affix list and stores it + * in the CompoundAffix. + * - builds prefix trees (Trie) from the affix list for prefixes and suffixes + * and stores them in Suffix and Prefix fields. + * The affix list is got from the Affix field. + * + * Memory management + * ----------------- + * + * The IspellDict structure has the Spell field which is used only in compile + * time. The Spell field stores a words list. It can take a lot of memory. + * Therefore when a dictionary is compiled this field is cleared by + * NIFinishBuild(). + * + * All resources which should cleared by NIFinishBuild() is initialized using + * tmpalloc() and tmpalloc0(). + * + * IDENTIFICATION + * src/backend/tsearch/spell.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "catalog/pg_collation.h" +#include "tsearch/dicts/spell.h" +#include "tsearch/ts_locale.h" +#include "utils/memutils.h" + + +/* + * Initialization requires a lot of memory that's not needed + * after the initialization is done. During initialization, + * CurrentMemoryContext is the long-lived memory context associated + * with the dictionary cache entry. We keep the short-lived stuff + * in the Conf->buildCxt context. + */ +#define tmpalloc(sz) MemoryContextAlloc(Conf->buildCxt, (sz)) +#define tmpalloc0(sz) MemoryContextAllocZero(Conf->buildCxt, (sz)) + +/* + * Prepare for constructing an ISpell dictionary. + * + * The IspellDict struct is assumed to be zeroed when allocated. + */ +void +NIStartBuild(IspellDict *Conf) +{ + /* + * The temp context is a child of CurTransactionContext, so that it will + * go away automatically on error. + */ + Conf->buildCxt = AllocSetContextCreate(CurTransactionContext, + "Ispell dictionary init context", + ALLOCSET_DEFAULT_SIZES); +} + +/* + * Clean up when dictionary construction is complete. + */ +void +NIFinishBuild(IspellDict *Conf) +{ + /* Release no-longer-needed temp memory */ + MemoryContextDelete(Conf->buildCxt); + /* Just for cleanliness, zero the now-dangling pointers */ + Conf->buildCxt = NULL; + Conf->Spell = NULL; + Conf->firstfree = NULL; + Conf->CompoundAffixFlags = NULL; +} + + +/* + * "Compact" palloc: allocate without extra palloc overhead. + * + * Since we have no need to free the ispell data items individually, there's + * not much value in the per-chunk overhead normally consumed by palloc. + * Getting rid of it is helpful since ispell can allocate a lot of small nodes. + * + * We currently pre-zero all data allocated this way, even though some of it + * doesn't need that. The cpalloc and cpalloc0 macros are just documentation + * to indicate which allocations actually require zeroing. + */ +#define COMPACT_ALLOC_CHUNK 8192 /* amount to get from palloc at once */ +#define COMPACT_MAX_REQ 1024 /* must be < COMPACT_ALLOC_CHUNK */ + +static void * +compact_palloc0(IspellDict *Conf, size_t size) +{ + void *result; + + /* Should only be called during init */ + Assert(Conf->buildCxt != NULL); + + /* No point in this for large chunks */ + if (size > COMPACT_MAX_REQ) + return palloc0(size); + + /* Keep everything maxaligned */ + size = MAXALIGN(size); + + /* Need more space? */ + if (size > Conf->avail) + { + Conf->firstfree = palloc0(COMPACT_ALLOC_CHUNK); + Conf->avail = COMPACT_ALLOC_CHUNK; + } + + result = (void *) Conf->firstfree; + Conf->firstfree += size; + Conf->avail -= size; + + return result; +} + +#define cpalloc(size) compact_palloc0(Conf, size) +#define cpalloc0(size) compact_palloc0(Conf, size) + +static char * +cpstrdup(IspellDict *Conf, const char *str) +{ + char *res = cpalloc(strlen(str) + 1); + + strcpy(res, str); + return res; +} + + +/* + * Apply lowerstr(), producing a temporary result (in the buildCxt). + */ +static char * +lowerstr_ctx(IspellDict *Conf, const char *src) +{ + MemoryContext saveCtx; + char *dst; + + saveCtx = MemoryContextSwitchTo(Conf->buildCxt); + dst = lowerstr(src); + MemoryContextSwitchTo(saveCtx); + + return dst; +} + +#define MAX_NORM 1024 +#define MAXNORMLEN 256 + +#define STRNCMP(s,p) strncmp( (s), (p), strlen(p) ) +#define GETWCHAR(W,L,N,T) ( ((const uint8*)(W))[ ((T)==FF_PREFIX) ? (N) : ( (L) - 1 - (N) ) ] ) +#define GETCHAR(A,N,T) GETWCHAR( (A)->repl, (A)->replen, N, T ) + +static char *VoidString = ""; + +static int +cmpspell(const void *s1, const void *s2) +{ + return strcmp((*(SPELL *const *) s1)->word, (*(SPELL *const *) s2)->word); +} + +static int +cmpspellaffix(const void *s1, const void *s2) +{ + return strcmp((*(SPELL *const *) s1)->p.flag, + (*(SPELL *const *) s2)->p.flag); +} + +static int +cmpcmdflag(const void *f1, const void *f2) +{ + CompoundAffixFlag *fv1 = (CompoundAffixFlag *) f1, + *fv2 = (CompoundAffixFlag *) f2; + + Assert(fv1->flagMode == fv2->flagMode); + + if (fv1->flagMode == FM_NUM) + { + if (fv1->flag.i == fv2->flag.i) + return 0; + + return (fv1->flag.i > fv2->flag.i) ? 1 : -1; + } + + return strcmp(fv1->flag.s, fv2->flag.s); +} + +static char * +findchar(char *str, int c) +{ + while (*str) + { + if (t_iseq(str, c)) + return str; + str += pg_mblen(str); + } + + return NULL; +} + +static char * +findchar2(char *str, int c1, int c2) +{ + while (*str) + { + if (t_iseq(str, c1) || t_iseq(str, c2)) + return str; + str += pg_mblen(str); + } + + return NULL; +} + + +/* backward string compare for suffix tree operations */ +static int +strbcmp(const unsigned char *s1, const unsigned char *s2) +{ + int l1 = strlen((const char *) s1) - 1, + l2 = strlen((const char *) s2) - 1; + + while (l1 >= 0 && l2 >= 0) + { + if (s1[l1] < s2[l2]) + return -1; + if (s1[l1] > s2[l2]) + return 1; + l1--; + l2--; + } + if (l1 < l2) + return -1; + if (l1 > l2) + return 1; + + return 0; +} + +static int +strbncmp(const unsigned char *s1, const unsigned char *s2, size_t count) +{ + int l1 = strlen((const char *) s1) - 1, + l2 = strlen((const char *) s2) - 1, + l = count; + + while (l1 >= 0 && l2 >= 0 && l > 0) + { + if (s1[l1] < s2[l2]) + return -1; + if (s1[l1] > s2[l2]) + return 1; + l1--; + l2--; + l--; + } + if (l == 0) + return 0; + if (l1 < l2) + return -1; + if (l1 > l2) + return 1; + return 0; +} + +/* + * Compares affixes. + * First compares the type of an affix. Prefixes should go before affixes. + * If types are equal then compares replaceable string. + */ +static int +cmpaffix(const void *s1, const void *s2) +{ + const AFFIX *a1 = (const AFFIX *) s1; + const AFFIX *a2 = (const AFFIX *) s2; + + if (a1->type < a2->type) + return -1; + if (a1->type > a2->type) + return 1; + if (a1->type == FF_PREFIX) + return strcmp(a1->repl, a2->repl); + else + return strbcmp((const unsigned char *) a1->repl, + (const unsigned char *) a2->repl); +} + +/* + * Gets an affix flag from the set of affix flags (sflagset). + * + * Several flags can be stored in a single string. Flags can be represented by: + * - 1 character (FM_CHAR). A character may be Unicode. + * - 2 characters (FM_LONG). A character may be Unicode. + * - numbers from 1 to 65000 (FM_NUM). + * + * Depending on the flagMode an affix string can have the following format: + * - FM_CHAR: ABCD + * Here we have 4 flags: A, B, C and D + * - FM_LONG: ABCDE* + * Here we have 3 flags: AB, CD and E* + * - FM_NUM: 200,205,50 + * Here we have 3 flags: 200, 205 and 50 + * + * Conf: current dictionary. + * sflagset: the set of affix flags. Returns a reference to the start of a next + * affix flag. + * sflag: returns an affix flag from sflagset. + */ +static void +getNextFlagFromString(IspellDict *Conf, char **sflagset, char *sflag) +{ + int32 s; + char *next, + *sbuf = *sflagset; + int maxstep; + bool stop = false; + bool met_comma = false; + + maxstep = (Conf->flagMode == FM_LONG) ? 2 : 1; + + while (**sflagset) + { + switch (Conf->flagMode) + { + case FM_LONG: + case FM_CHAR: + COPYCHAR(sflag, *sflagset); + sflag += pg_mblen(*sflagset); + + /* Go to start of the next flag */ + *sflagset += pg_mblen(*sflagset); + + /* Check if we get all characters of flag */ + maxstep--; + stop = (maxstep == 0); + break; + case FM_NUM: + s = strtol(*sflagset, &next, 10); + if (*sflagset == next || errno == ERANGE) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix flag \"%s\"", *sflagset))); + if (s < 0 || s > FLAGNUM_MAXSIZE) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("affix flag \"%s\" is out of range", + *sflagset))); + sflag += sprintf(sflag, "%0d", s); + + /* Go to start of the next flag */ + *sflagset = next; + while (**sflagset) + { + if (t_isdigit(*sflagset)) + { + if (!met_comma) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix flag \"%s\"", + *sflagset))); + break; + } + else if (t_iseq(*sflagset, ',')) + { + if (met_comma) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix flag \"%s\"", + *sflagset))); + met_comma = true; + } + else if (!t_isspace(*sflagset)) + { + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid character in affix flag \"%s\"", + *sflagset))); + } + + *sflagset += pg_mblen(*sflagset); + } + stop = true; + break; + default: + elog(ERROR, "unrecognized type of Conf->flagMode: %d", + Conf->flagMode); + } + + if (stop) + break; + } + + if (Conf->flagMode == FM_LONG && maxstep > 0) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix flag \"%s\" with \"long\" flag value", + sbuf))); + + *sflag = '\0'; +} + +/* + * Checks if the affix set Conf->AffixData[affix] contains affixflag. + * Conf->AffixData[affix] does not contain affixflag if this flag is not used + * actually by the .dict file. + * + * Conf: current dictionary. + * affix: index of the Conf->AffixData array. + * affixflag: the affix flag. + * + * Returns true if the string Conf->AffixData[affix] contains affixflag, + * otherwise returns false. + */ +static bool +IsAffixFlagInUse(IspellDict *Conf, int affix, const char *affixflag) +{ + char *flagcur; + char flag[BUFSIZ]; + + if (*affixflag == 0) + return true; + + Assert(affix < Conf->nAffixData); + + flagcur = Conf->AffixData[affix]; + + while (*flagcur) + { + getNextFlagFromString(Conf, &flagcur, flag); + /* Compare first affix flag in flagcur with affixflag */ + if (strcmp(flag, affixflag) == 0) + return true; + } + + /* Could not find affixflag */ + return false; +} + +/* + * Adds the new word into the temporary array Spell. + * + * Conf: current dictionary. + * word: new word. + * flag: set of affix flags. Single flag can be get by getNextFlagFromString(). + */ +static void +NIAddSpell(IspellDict *Conf, const char *word, const char *flag) +{ + if (Conf->nspell >= Conf->mspell) + { + if (Conf->mspell) + { + Conf->mspell *= 2; + Conf->Spell = (SPELL **) repalloc(Conf->Spell, Conf->mspell * sizeof(SPELL *)); + } + else + { + Conf->mspell = 1024 * 20; + Conf->Spell = (SPELL **) tmpalloc(Conf->mspell * sizeof(SPELL *)); + } + } + Conf->Spell[Conf->nspell] = (SPELL *) tmpalloc(SPELLHDRSZ + strlen(word) + 1); + strcpy(Conf->Spell[Conf->nspell]->word, word); + Conf->Spell[Conf->nspell]->p.flag = (*flag != '\0') + ? cpstrdup(Conf, flag) : VoidString; + Conf->nspell++; +} + +/* + * Imports dictionary into the temporary array Spell. + * + * Note caller must already have applied get_tsearch_config_filename. + * + * Conf: current dictionary. + * filename: path to the .dict file. + */ +void +NIImportDictionary(IspellDict *Conf, const char *filename) +{ + tsearch_readline_state trst; + char *line; + + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open dictionary file \"%s\": %m", + filename))); + + while ((line = tsearch_readline(&trst)) != NULL) + { + char *s, + *pstr; + + /* Set of affix flags */ + const char *flag; + + /* Extract flag from the line */ + flag = NULL; + if ((s = findchar(line, '/'))) + { + *s++ = '\0'; + flag = s; + while (*s) + { + /* we allow only single encoded flags for faster works */ + if (pg_mblen(s) == 1 && t_isprint(s) && !t_isspace(s)) + s++; + else + { + *s = '\0'; + break; + } + } + } + else + flag = ""; + + /* Remove trailing spaces */ + s = line; + while (*s) + { + if (t_isspace(s)) + { + *s = '\0'; + break; + } + s += pg_mblen(s); + } + pstr = lowerstr_ctx(Conf, line); + + NIAddSpell(Conf, pstr, flag); + pfree(pstr); + + pfree(line); + } + tsearch_readline_end(&trst); +} + +/* + * Searches a basic form of word in the prefix tree. This word was generated + * using an affix rule. This rule may not be presented in an affix set of + * a basic form of word. + * + * For example, we have the entry in the .dict file: + * meter/GMD + * + * The affix rule with the flag S: + * SFX S y ies [^aeiou]y + * is not presented here. + * + * The affix rule with the flag M: + * SFX M 0 's . + * is presented here. + * + * Conf: current dictionary. + * word: basic form of word. + * affixflag: affix flag, by which a basic form of word was generated. + * flag: compound flag used to compare with StopMiddle->compoundflag. + * + * Returns 1 if the word was found in the prefix tree, else returns 0. + */ +static int +FindWord(IspellDict *Conf, const char *word, const char *affixflag, int flag) +{ + SPNode *node = Conf->Dictionary; + SPNodeData *StopLow, + *StopHigh, + *StopMiddle; + const uint8 *ptr = (const uint8 *) word; + + flag &= FF_COMPOUNDFLAGMASK; + + while (node && *ptr) + { + StopLow = node->data; + StopHigh = node->data + node->length; + while (StopLow < StopHigh) + { + StopMiddle = StopLow + ((StopHigh - StopLow) >> 1); + if (StopMiddle->val == *ptr) + { + if (*(ptr + 1) == '\0' && StopMiddle->isword) + { + if (flag == 0) + { + /* + * The word can be formed only with another word. And + * in the flag parameter there is not a sign that we + * search compound words. + */ + if (StopMiddle->compoundflag & FF_COMPOUNDONLY) + return 0; + } + else if ((flag & StopMiddle->compoundflag) == 0) + return 0; + + /* + * Check if this affix rule is presented in the affix set + * with index StopMiddle->affix. + */ + if (IsAffixFlagInUse(Conf, StopMiddle->affix, affixflag)) + return 1; + } + node = StopMiddle->node; + ptr++; + break; + } + else if (StopMiddle->val < *ptr) + StopLow = StopMiddle + 1; + else + StopHigh = StopMiddle; + } + if (StopLow >= StopHigh) + break; + } + return 0; +} + +/* + * Context reset/delete callback for a regular expression used in an affix + */ +static void +regex_affix_deletion_callback(void *arg) +{ + aff_regex_struct *pregex = (aff_regex_struct *) arg; + + pg_regfree(&(pregex->regex)); +} + +/* + * Adds a new affix rule to the Affix field. + * + * Conf: current dictionary. + * flag: affix flag ('\' in the below example). + * flagflags: set of flags from the flagval field for this affix rule. This set + * is listed after '/' character in the added string (repl). + * + * For example L flag in the hunspell_sample.affix: + * SFX \ 0 Y/L [^Y] + * + * mask: condition for search ('[^Y]' in the above example). + * find: stripping characters from beginning (at prefix) or end (at suffix) + * of the word ('0' in the above example, 0 means that there is not + * stripping character). + * repl: adding string after stripping ('Y' in the above example). + * type: FF_SUFFIX or FF_PREFIX. + */ +static void +NIAddAffix(IspellDict *Conf, const char *flag, char flagflags, const char *mask, + const char *find, const char *repl, int type) +{ + AFFIX *Affix; + + if (Conf->naffixes >= Conf->maffixes) + { + if (Conf->maffixes) + { + Conf->maffixes *= 2; + Conf->Affix = (AFFIX *) repalloc((void *) Conf->Affix, Conf->maffixes * sizeof(AFFIX)); + } + else + { + Conf->maffixes = 16; + Conf->Affix = (AFFIX *) palloc(Conf->maffixes * sizeof(AFFIX)); + } + } + + Affix = Conf->Affix + Conf->naffixes; + + /* This affix rule can be applied for words with any ending */ + if (strcmp(mask, ".") == 0 || *mask == '\0') + { + Affix->issimple = 1; + Affix->isregis = 0; + } + /* This affix rule will use regis to search word ending */ + else if (RS_isRegis(mask)) + { + Affix->issimple = 0; + Affix->isregis = 1; + RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX), + *mask ? mask : VoidString); + } + /* This affix rule will use regex_t to search word ending */ + else + { + int masklen; + int wmasklen; + int err; + pg_wchar *wmask; + char *tmask; + aff_regex_struct *pregex; + + Affix->issimple = 0; + Affix->isregis = 0; + tmask = (char *) tmpalloc(strlen(mask) + 3); + if (type == FF_SUFFIX) + sprintf(tmask, "%s$", mask); + else + sprintf(tmask, "^%s", mask); + + masklen = strlen(tmask); + wmask = (pg_wchar *) tmpalloc((masklen + 1) * sizeof(pg_wchar)); + wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen); + + /* + * The regex engine stores its stuff using malloc not palloc, so we + * must arrange to explicitly clean up the regex when the dictionary's + * context is cleared. That means the regex_t has to stay in a fixed + * location within the context; we can't keep it directly in the AFFIX + * struct, since we may sort and resize the array of AFFIXes. + */ + Affix->reg.pregex = pregex = palloc(sizeof(aff_regex_struct)); + + err = pg_regcomp(&(pregex->regex), wmask, wmasklen, + REG_ADVANCED | REG_NOSUB, + DEFAULT_COLLATION_OID); + if (err) + { + char errstr[100]; + + pg_regerror(err, &(pregex->regex), errstr, sizeof(errstr)); + ereport(ERROR, + (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION), + errmsg("invalid regular expression: %s", errstr))); + } + + pregex->mcallback.func = regex_affix_deletion_callback; + pregex->mcallback.arg = (void *) pregex; + MemoryContextRegisterResetCallback(CurrentMemoryContext, + &pregex->mcallback); + } + + Affix->flagflags = flagflags; + if ((Affix->flagflags & FF_COMPOUNDONLY) || (Affix->flagflags & FF_COMPOUNDPERMITFLAG)) + { + if ((Affix->flagflags & FF_COMPOUNDFLAG) == 0) + Affix->flagflags |= FF_COMPOUNDFLAG; + } + Affix->flag = cpstrdup(Conf, flag); + Affix->type = type; + + Affix->find = (find && *find) ? cpstrdup(Conf, find) : VoidString; + if ((Affix->replen = strlen(repl)) > 0) + Affix->repl = cpstrdup(Conf, repl); + else + Affix->repl = VoidString; + Conf->naffixes++; +} + +/* Parsing states for parse_affentry() and friends */ +#define PAE_WAIT_MASK 0 +#define PAE_INMASK 1 +#define PAE_WAIT_FIND 2 +#define PAE_INFIND 3 +#define PAE_WAIT_REPL 4 +#define PAE_INREPL 5 +#define PAE_WAIT_TYPE 6 +#define PAE_WAIT_FLAG 7 + +/* + * Parse next space-separated field of an .affix file line. + * + * *str is the input pointer (will be advanced past field) + * next is where to copy the field value to, with null termination + * + * The buffer at "next" must be of size BUFSIZ; we truncate the input to fit. + * + * Returns true if we found a field, false if not. + */ +static bool +get_nextfield(char **str, char *next) +{ + int state = PAE_WAIT_MASK; + int avail = BUFSIZ; + + while (**str) + { + if (state == PAE_WAIT_MASK) + { + if (t_iseq(*str, '#')) + return false; + else if (!t_isspace(*str)) + { + int clen = pg_mblen(*str); + + if (clen < avail) + { + COPYCHAR(next, *str); + next += clen; + avail -= clen; + } + state = PAE_INMASK; + } + } + else /* state == PAE_INMASK */ + { + if (t_isspace(*str)) + { + *next = '\0'; + return true; + } + else + { + int clen = pg_mblen(*str); + + if (clen < avail) + { + COPYCHAR(next, *str); + next += clen; + avail -= clen; + } + } + } + *str += pg_mblen(*str); + } + + *next = '\0'; + + return (state == PAE_INMASK); /* OK if we got a nonempty field */ +} + +/* + * Parses entry of an .affix file of MySpell or Hunspell format. + * + * An .affix file entry has the following format: + * - header + * <type> <flag> <cross_flag> <flag_count> + * - fields after header: + * <type> <flag> <find> <replace> <mask> + * + * str is the input line + * field values are returned to type etc, which must be buffers of size BUFSIZ. + * + * Returns number of fields found; any omitted fields are set to empty strings. + */ +static int +parse_ooaffentry(char *str, char *type, char *flag, char *find, + char *repl, char *mask) +{ + int state = PAE_WAIT_TYPE; + int fields_read = 0; + bool valid = false; + + *type = *flag = *find = *repl = *mask = '\0'; + + while (*str) + { + switch (state) + { + case PAE_WAIT_TYPE: + valid = get_nextfield(&str, type); + state = PAE_WAIT_FLAG; + break; + case PAE_WAIT_FLAG: + valid = get_nextfield(&str, flag); + state = PAE_WAIT_FIND; + break; + case PAE_WAIT_FIND: + valid = get_nextfield(&str, find); + state = PAE_WAIT_REPL; + break; + case PAE_WAIT_REPL: + valid = get_nextfield(&str, repl); + state = PAE_WAIT_MASK; + break; + case PAE_WAIT_MASK: + valid = get_nextfield(&str, mask); + state = -1; /* force loop exit */ + break; + default: + elog(ERROR, "unrecognized state in parse_ooaffentry: %d", + state); + break; + } + if (valid) + fields_read++; + else + break; /* early EOL */ + if (state < 0) + break; /* got all fields */ + } + + return fields_read; +} + +/* + * Parses entry of an .affix file of Ispell format + * + * An .affix file entry has the following format: + * <mask> > [-<find>,]<replace> + */ +static bool +parse_affentry(char *str, char *mask, char *find, char *repl) +{ + int state = PAE_WAIT_MASK; + char *pmask = mask, + *pfind = find, + *prepl = repl; + + *mask = *find = *repl = '\0'; + + while (*str) + { + if (state == PAE_WAIT_MASK) + { + if (t_iseq(str, '#')) + return false; + else if (!t_isspace(str)) + { + COPYCHAR(pmask, str); + pmask += pg_mblen(str); + state = PAE_INMASK; + } + } + else if (state == PAE_INMASK) + { + if (t_iseq(str, '>')) + { + *pmask = '\0'; + state = PAE_WAIT_FIND; + } + else if (!t_isspace(str)) + { + COPYCHAR(pmask, str); + pmask += pg_mblen(str); + } + } + else if (state == PAE_WAIT_FIND) + { + if (t_iseq(str, '-')) + { + state = PAE_INFIND; + } + else if (t_isalpha(str) || t_iseq(str, '\'') /* english 's */ ) + { + COPYCHAR(prepl, str); + prepl += pg_mblen(str); + state = PAE_INREPL; + } + else if (!t_isspace(str)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("syntax error"))); + } + else if (state == PAE_INFIND) + { + if (t_iseq(str, ',')) + { + *pfind = '\0'; + state = PAE_WAIT_REPL; + } + else if (t_isalpha(str)) + { + COPYCHAR(pfind, str); + pfind += pg_mblen(str); + } + else if (!t_isspace(str)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("syntax error"))); + } + else if (state == PAE_WAIT_REPL) + { + if (t_iseq(str, '-')) + { + break; /* void repl */ + } + else if (t_isalpha(str)) + { + COPYCHAR(prepl, str); + prepl += pg_mblen(str); + state = PAE_INREPL; + } + else if (!t_isspace(str)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("syntax error"))); + } + else if (state == PAE_INREPL) + { + if (t_iseq(str, '#')) + { + *prepl = '\0'; + break; + } + else if (t_isalpha(str)) + { + COPYCHAR(prepl, str); + prepl += pg_mblen(str); + } + else if (!t_isspace(str)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("syntax error"))); + } + else + elog(ERROR, "unrecognized state in parse_affentry: %d", state); + + str += pg_mblen(str); + } + + *pmask = *pfind = *prepl = '\0'; + + return (*mask && (*find || *repl)); +} + +/* + * Sets a Hunspell options depending on flag type. + */ +static void +setCompoundAffixFlagValue(IspellDict *Conf, CompoundAffixFlag *entry, + char *s, uint32 val) +{ + if (Conf->flagMode == FM_NUM) + { + char *next; + int i; + + i = strtol(s, &next, 10); + if (s == next || errno == ERANGE) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix flag \"%s\"", s))); + if (i < 0 || i > FLAGNUM_MAXSIZE) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("affix flag \"%s\" is out of range", s))); + + entry->flag.i = i; + } + else + entry->flag.s = cpstrdup(Conf, s); + + entry->flagMode = Conf->flagMode; + entry->value = val; +} + +/* + * Sets up a correspondence for the affix parameter with the affix flag. + * + * Conf: current dictionary. + * s: affix flag in string. + * val: affix parameter. + */ +static void +addCompoundAffixFlagValue(IspellDict *Conf, char *s, uint32 val) +{ + CompoundAffixFlag *newValue; + char sbuf[BUFSIZ]; + char *sflag; + int clen; + + while (*s && t_isspace(s)) + s += pg_mblen(s); + + if (!*s) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("syntax error"))); + + /* Get flag without \n */ + sflag = sbuf; + while (*s && !t_isspace(s) && *s != '\n') + { + clen = pg_mblen(s); + COPYCHAR(sflag, s); + sflag += clen; + s += clen; + } + *sflag = '\0'; + + /* Resize array or allocate memory for array CompoundAffixFlag */ + if (Conf->nCompoundAffixFlag >= Conf->mCompoundAffixFlag) + { + if (Conf->mCompoundAffixFlag) + { + Conf->mCompoundAffixFlag *= 2; + Conf->CompoundAffixFlags = (CompoundAffixFlag *) + repalloc((void *) Conf->CompoundAffixFlags, + Conf->mCompoundAffixFlag * sizeof(CompoundAffixFlag)); + } + else + { + Conf->mCompoundAffixFlag = 10; + Conf->CompoundAffixFlags = (CompoundAffixFlag *) + tmpalloc(Conf->mCompoundAffixFlag * sizeof(CompoundAffixFlag)); + } + } + + newValue = Conf->CompoundAffixFlags + Conf->nCompoundAffixFlag; + + setCompoundAffixFlagValue(Conf, newValue, sbuf, val); + + Conf->usecompound = true; + Conf->nCompoundAffixFlag++; +} + +/* + * Returns a set of affix parameters which correspondence to the set of affix + * flags s. + */ +static int +getCompoundAffixFlagValue(IspellDict *Conf, char *s) +{ + uint32 flag = 0; + CompoundAffixFlag *found, + key; + char sflag[BUFSIZ]; + char *flagcur; + + if (Conf->nCompoundAffixFlag == 0) + return 0; + + flagcur = s; + while (*flagcur) + { + getNextFlagFromString(Conf, &flagcur, sflag); + setCompoundAffixFlagValue(Conf, &key, sflag, 0); + + found = (CompoundAffixFlag *) + bsearch(&key, (void *) Conf->CompoundAffixFlags, + Conf->nCompoundAffixFlag, sizeof(CompoundAffixFlag), + cmpcmdflag); + if (found != NULL) + flag |= found->value; + } + + return flag; +} + +/* + * Returns a flag set using the s parameter. + * + * If Conf->useFlagAliases is true then the s parameter is index of the + * Conf->AffixData array and function returns its entry. + * Else function returns the s parameter. + */ +static char * +getAffixFlagSet(IspellDict *Conf, char *s) +{ + if (Conf->useFlagAliases && *s != '\0') + { + int curaffix; + char *end; + + curaffix = strtol(s, &end, 10); + if (s == end || errno == ERANGE) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix alias \"%s\"", s))); + + if (curaffix > 0 && curaffix < Conf->nAffixData) + + /* + * Do not subtract 1 from curaffix because empty string was added + * in NIImportOOAffixes + */ + return Conf->AffixData[curaffix]; + else if (curaffix > Conf->nAffixData) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix alias \"%s\"", s))); + return VoidString; + } + else + return s; +} + +/* + * Import an affix file that follows MySpell or Hunspell format. + * + * Conf: current dictionary. + * filename: path to the .affix file. + */ +static void +NIImportOOAffixes(IspellDict *Conf, const char *filename) +{ + char type[BUFSIZ], + *ptype = NULL; + char sflag[BUFSIZ]; + char mask[BUFSIZ], + *pmask; + char find[BUFSIZ], + *pfind; + char repl[BUFSIZ], + *prepl; + bool isSuffix = false; + int naffix = 0, + curaffix = 0; + int sflaglen = 0; + char flagflags = 0; + tsearch_readline_state trst; + char *recoded; + + /* read file to find any flag */ + Conf->usecompound = false; + Conf->useFlagAliases = false; + Conf->flagMode = FM_CHAR; + + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open affix file \"%s\": %m", + filename))); + + while ((recoded = tsearch_readline(&trst)) != NULL) + { + if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#')) + { + pfree(recoded); + continue; + } + + if (STRNCMP(recoded, "COMPOUNDFLAG") == 0) + addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDFLAG"), + FF_COMPOUNDFLAG); + else if (STRNCMP(recoded, "COMPOUNDBEGIN") == 0) + addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDBEGIN"), + FF_COMPOUNDBEGIN); + else if (STRNCMP(recoded, "COMPOUNDLAST") == 0) + addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDLAST"), + FF_COMPOUNDLAST); + /* COMPOUNDLAST and COMPOUNDEND are synonyms */ + else if (STRNCMP(recoded, "COMPOUNDEND") == 0) + addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDEND"), + FF_COMPOUNDLAST); + else if (STRNCMP(recoded, "COMPOUNDMIDDLE") == 0) + addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDMIDDLE"), + FF_COMPOUNDMIDDLE); + else if (STRNCMP(recoded, "ONLYINCOMPOUND") == 0) + addCompoundAffixFlagValue(Conf, recoded + strlen("ONLYINCOMPOUND"), + FF_COMPOUNDONLY); + else if (STRNCMP(recoded, "COMPOUNDPERMITFLAG") == 0) + addCompoundAffixFlagValue(Conf, + recoded + strlen("COMPOUNDPERMITFLAG"), + FF_COMPOUNDPERMITFLAG); + else if (STRNCMP(recoded, "COMPOUNDFORBIDFLAG") == 0) + addCompoundAffixFlagValue(Conf, + recoded + strlen("COMPOUNDFORBIDFLAG"), + FF_COMPOUNDFORBIDFLAG); + else if (STRNCMP(recoded, "FLAG") == 0) + { + char *s = recoded + strlen("FLAG"); + + while (*s && t_isspace(s)) + s += pg_mblen(s); + + if (*s) + { + if (STRNCMP(s, "long") == 0) + Conf->flagMode = FM_LONG; + else if (STRNCMP(s, "num") == 0) + Conf->flagMode = FM_NUM; + else if (STRNCMP(s, "default") != 0) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("Ispell dictionary supports only " + "\"default\", \"long\", " + "and \"num\" flag values"))); + } + } + + pfree(recoded); + } + tsearch_readline_end(&trst); + + if (Conf->nCompoundAffixFlag > 1) + qsort((void *) Conf->CompoundAffixFlags, Conf->nCompoundAffixFlag, + sizeof(CompoundAffixFlag), cmpcmdflag); + + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open affix file \"%s\": %m", + filename))); + + while ((recoded = tsearch_readline(&trst)) != NULL) + { + int fields_read; + + if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#')) + goto nextline; + + fields_read = parse_ooaffentry(recoded, type, sflag, find, repl, mask); + + if (ptype) + pfree(ptype); + ptype = lowerstr_ctx(Conf, type); + + /* First try to parse AF parameter (alias compression) */ + if (STRNCMP(ptype, "af") == 0) + { + /* First line is the number of aliases */ + if (!Conf->useFlagAliases) + { + Conf->useFlagAliases = true; + naffix = atoi(sflag); + if (naffix <= 0) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid number of flag vector aliases"))); + + /* Also reserve place for empty flag set */ + naffix++; + + Conf->AffixData = (char **) palloc0(naffix * sizeof(char *)); + Conf->lenAffixData = Conf->nAffixData = naffix; + + /* Add empty flag set into AffixData */ + Conf->AffixData[curaffix] = VoidString; + curaffix++; + } + /* Other lines are aliases */ + else + { + if (curaffix < naffix) + { + Conf->AffixData[curaffix] = cpstrdup(Conf, sflag); + curaffix++; + } + else + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("number of aliases exceeds specified number %d", + naffix - 1))); + } + goto nextline; + } + /* Else try to parse prefixes and suffixes */ + if (fields_read < 4 || + (STRNCMP(ptype, "sfx") != 0 && STRNCMP(ptype, "pfx") != 0)) + goto nextline; + + sflaglen = strlen(sflag); + if (sflaglen == 0 + || (sflaglen > 1 && Conf->flagMode == FM_CHAR) + || (sflaglen > 2 && Conf->flagMode == FM_LONG)) + goto nextline; + + /*-------- + * Affix header. For example: + * SFX \ N 1 + *-------- + */ + if (fields_read == 4) + { + isSuffix = (STRNCMP(ptype, "sfx") == 0); + if (t_iseq(find, 'y') || t_iseq(find, 'Y')) + flagflags = FF_CROSSPRODUCT; + else + flagflags = 0; + } + /*-------- + * Affix fields. For example: + * SFX \ 0 Y/L [^Y] + *-------- + */ + else + { + char *ptr; + int aflg = 0; + + /* Get flags after '/' (flags are case sensitive) */ + if ((ptr = strchr(repl, '/')) != NULL) + aflg |= getCompoundAffixFlagValue(Conf, + getAffixFlagSet(Conf, + ptr + 1)); + /* Get lowercased version of string before '/' */ + prepl = lowerstr_ctx(Conf, repl); + if ((ptr = strchr(prepl, '/')) != NULL) + *ptr = '\0'; + pfind = lowerstr_ctx(Conf, find); + pmask = lowerstr_ctx(Conf, mask); + if (t_iseq(find, '0')) + *pfind = '\0'; + if (t_iseq(repl, '0')) + *prepl = '\0'; + + NIAddAffix(Conf, sflag, flagflags | aflg, pmask, pfind, prepl, + isSuffix ? FF_SUFFIX : FF_PREFIX); + pfree(prepl); + pfree(pfind); + pfree(pmask); + } + +nextline: + pfree(recoded); + } + + tsearch_readline_end(&trst); + if (ptype) + pfree(ptype); +} + +/* + * import affixes + * + * Note caller must already have applied get_tsearch_config_filename + * + * This function is responsible for parsing ispell ("old format") affix files. + * If we realize that the file contains new-format commands, we pass off the + * work to NIImportOOAffixes(), which will re-read the whole file. + */ +void +NIImportAffixes(IspellDict *Conf, const char *filename) +{ + char *pstr = NULL; + char flag[BUFSIZ]; + char mask[BUFSIZ]; + char find[BUFSIZ]; + char repl[BUFSIZ]; + char *s; + bool suffixes = false; + bool prefixes = false; + char flagflags = 0; + tsearch_readline_state trst; + bool oldformat = false; + char *recoded = NULL; + + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open affix file \"%s\": %m", + filename))); + + Conf->usecompound = false; + Conf->useFlagAliases = false; + Conf->flagMode = FM_CHAR; + + while ((recoded = tsearch_readline(&trst)) != NULL) + { + pstr = lowerstr(recoded); + + /* Skip comments and empty lines */ + if (*pstr == '#' || *pstr == '\n') + goto nextline; + + if (STRNCMP(pstr, "compoundwords") == 0) + { + /* Find case-insensitive L flag in non-lowercased string */ + s = findchar2(recoded, 'l', 'L'); + if (s) + { + while (*s && !t_isspace(s)) + s += pg_mblen(s); + while (*s && t_isspace(s)) + s += pg_mblen(s); + + if (*s && pg_mblen(s) == 1) + { + addCompoundAffixFlagValue(Conf, s, FF_COMPOUNDFLAG); + Conf->usecompound = true; + } + oldformat = true; + goto nextline; + } + } + if (STRNCMP(pstr, "suffixes") == 0) + { + suffixes = true; + prefixes = false; + oldformat = true; + goto nextline; + } + if (STRNCMP(pstr, "prefixes") == 0) + { + suffixes = false; + prefixes = true; + oldformat = true; + goto nextline; + } + if (STRNCMP(pstr, "flag") == 0) + { + s = recoded + 4; /* we need non-lowercased string */ + flagflags = 0; + + while (*s && t_isspace(s)) + s += pg_mblen(s); + + if (*s == '*') + { + flagflags |= FF_CROSSPRODUCT; + s++; + } + else if (*s == '~') + { + flagflags |= FF_COMPOUNDONLY; + s++; + } + + if (*s == '\\') + s++; + + /* + * An old-format flag is a single ASCII character; we expect it to + * be followed by EOL, whitespace, or ':'. Otherwise this is a + * new-format flag command. + */ + if (*s && pg_mblen(s) == 1) + { + COPYCHAR(flag, s); + flag[1] = '\0'; + + s++; + if (*s == '\0' || *s == '#' || *s == '\n' || *s == ':' || + t_isspace(s)) + { + oldformat = true; + goto nextline; + } + } + goto isnewformat; + } + if (STRNCMP(recoded, "COMPOUNDFLAG") == 0 || + STRNCMP(recoded, "COMPOUNDMIN") == 0 || + STRNCMP(recoded, "PFX") == 0 || + STRNCMP(recoded, "SFX") == 0) + goto isnewformat; + + if ((!suffixes) && (!prefixes)) + goto nextline; + + if (!parse_affentry(pstr, mask, find, repl)) + goto nextline; + + NIAddAffix(Conf, flag, flagflags, mask, find, repl, suffixes ? FF_SUFFIX : FF_PREFIX); + +nextline: + pfree(recoded); + pfree(pstr); + } + tsearch_readline_end(&trst); + return; + +isnewformat: + if (oldformat) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("affix file contains both old-style and new-style commands"))); + tsearch_readline_end(&trst); + + NIImportOOAffixes(Conf, filename); +} + +/* + * Merges two affix flag sets and stores a new affix flag set into + * Conf->AffixData. + * + * Returns index of a new affix flag set. + */ +static int +MergeAffix(IspellDict *Conf, int a1, int a2) +{ + char **ptr; + + Assert(a1 < Conf->nAffixData && a2 < Conf->nAffixData); + + /* Do not merge affix flags if one of affix flags is empty */ + if (*Conf->AffixData[a1] == '\0') + return a2; + else if (*Conf->AffixData[a2] == '\0') + return a1; + + while (Conf->nAffixData + 1 >= Conf->lenAffixData) + { + Conf->lenAffixData *= 2; + Conf->AffixData = (char **) repalloc(Conf->AffixData, + sizeof(char *) * Conf->lenAffixData); + } + + ptr = Conf->AffixData + Conf->nAffixData; + if (Conf->flagMode == FM_NUM) + { + *ptr = cpalloc(strlen(Conf->AffixData[a1]) + + strlen(Conf->AffixData[a2]) + + 1 /* comma */ + 1 /* \0 */ ); + sprintf(*ptr, "%s,%s", Conf->AffixData[a1], Conf->AffixData[a2]); + } + else + { + *ptr = cpalloc(strlen(Conf->AffixData[a1]) + + strlen(Conf->AffixData[a2]) + + 1 /* \0 */ ); + sprintf(*ptr, "%s%s", Conf->AffixData[a1], Conf->AffixData[a2]); + } + ptr++; + *ptr = NULL; + Conf->nAffixData++; + + return Conf->nAffixData - 1; +} + +/* + * Returns a set of affix parameters which correspondence to the set of affix + * flags with the given index. + */ +static uint32 +makeCompoundFlags(IspellDict *Conf, int affix) +{ + Assert(affix < Conf->nAffixData); + + return (getCompoundAffixFlagValue(Conf, Conf->AffixData[affix]) & + FF_COMPOUNDFLAGMASK); +} + +/* + * Makes a prefix tree for the given level. + * + * Conf: current dictionary. + * low: lower index of the Conf->Spell array. + * high: upper index of the Conf->Spell array. + * level: current prefix tree level. + */ +static SPNode * +mkSPNode(IspellDict *Conf, int low, int high, int level) +{ + int i; + int nchar = 0; + char lastchar = '\0'; + SPNode *rs; + SPNodeData *data; + int lownew = low; + + for (i = low; i < high; i++) + if (Conf->Spell[i]->p.d.len > level && lastchar != Conf->Spell[i]->word[level]) + { + nchar++; + lastchar = Conf->Spell[i]->word[level]; + } + + if (!nchar) + return NULL; + + rs = (SPNode *) cpalloc0(SPNHDRSZ + nchar * sizeof(SPNodeData)); + rs->length = nchar; + data = rs->data; + + lastchar = '\0'; + for (i = low; i < high; i++) + if (Conf->Spell[i]->p.d.len > level) + { + if (lastchar != Conf->Spell[i]->word[level]) + { + if (lastchar) + { + /* Next level of the prefix tree */ + data->node = mkSPNode(Conf, lownew, i, level + 1); + lownew = i; + data++; + } + lastchar = Conf->Spell[i]->word[level]; + } + data->val = ((uint8 *) (Conf->Spell[i]->word))[level]; + if (Conf->Spell[i]->p.d.len == level + 1) + { + bool clearCompoundOnly = false; + + if (data->isword && data->affix != Conf->Spell[i]->p.d.affix) + { + /* + * MergeAffix called a few times. If one of word is + * allowed to be in compound word and another isn't, then + * clear FF_COMPOUNDONLY flag. + */ + + clearCompoundOnly = (FF_COMPOUNDONLY & data->compoundflag + & makeCompoundFlags(Conf, Conf->Spell[i]->p.d.affix)) + ? false : true; + data->affix = MergeAffix(Conf, data->affix, Conf->Spell[i]->p.d.affix); + } + else + data->affix = Conf->Spell[i]->p.d.affix; + data->isword = 1; + + data->compoundflag = makeCompoundFlags(Conf, data->affix); + + if ((data->compoundflag & FF_COMPOUNDONLY) && + (data->compoundflag & FF_COMPOUNDFLAG) == 0) + data->compoundflag |= FF_COMPOUNDFLAG; + + if (clearCompoundOnly) + data->compoundflag &= ~FF_COMPOUNDONLY; + } + } + + /* Next level of the prefix tree */ + data->node = mkSPNode(Conf, lownew, high, level + 1); + + return rs; +} + +/* + * Builds the Conf->Dictionary tree and AffixData from the imported dictionary + * and affixes. + */ +void +NISortDictionary(IspellDict *Conf) +{ + int i; + int naffix; + int curaffix; + + /* compress affixes */ + + /* + * If we use flag aliases then we need to use Conf->AffixData filled in + * the NIImportOOAffixes(). + */ + if (Conf->useFlagAliases) + { + for (i = 0; i < Conf->nspell; i++) + { + char *end; + + if (*Conf->Spell[i]->p.flag != '\0') + { + curaffix = strtol(Conf->Spell[i]->p.flag, &end, 10); + if (Conf->Spell[i]->p.flag == end || errno == ERANGE) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix alias \"%s\"", + Conf->Spell[i]->p.flag))); + if (curaffix < 0 || curaffix >= Conf->nAffixData) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix alias \"%s\"", + Conf->Spell[i]->p.flag))); + if (*end != '\0' && !t_isdigit(end) && !t_isspace(end)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("invalid affix alias \"%s\"", + Conf->Spell[i]->p.flag))); + } + else + { + /* + * If Conf->Spell[i]->p.flag is empty, then get empty value of + * Conf->AffixData (0 index). + */ + curaffix = 0; + } + + Conf->Spell[i]->p.d.affix = curaffix; + Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word); + } + } + /* Otherwise fill Conf->AffixData here */ + else + { + /* Count the number of different flags used in the dictionary */ + qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), + cmpspellaffix); + + naffix = 0; + for (i = 0; i < Conf->nspell; i++) + { + if (i == 0 || + strcmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag) != 0) + naffix++; + } + + /* + * Fill in Conf->AffixData with the affixes that were used in the + * dictionary. Replace textual flag-field of Conf->Spell entries with + * indexes into Conf->AffixData array. + */ + Conf->AffixData = (char **) palloc0(naffix * sizeof(char *)); + + curaffix = -1; + for (i = 0; i < Conf->nspell; i++) + { + if (i == 0 || + strcmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix]) != 0) + { + curaffix++; + Assert(curaffix < naffix); + Conf->AffixData[curaffix] = cpstrdup(Conf, + Conf->Spell[i]->p.flag); + } + + Conf->Spell[i]->p.d.affix = curaffix; + Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word); + } + + Conf->lenAffixData = Conf->nAffixData = naffix; + } + + /* Start build a prefix tree */ + qsort((void *) Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell); + Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0); +} + +/* + * Makes a prefix tree for the given level using the repl string of an affix + * rule. Affixes with empty replace string do not include in the prefix tree. + * This affixes are included by mkVoidAffix(). + * + * Conf: current dictionary. + * low: lower index of the Conf->Affix array. + * high: upper index of the Conf->Affix array. + * level: current prefix tree level. + * type: FF_SUFFIX or FF_PREFIX. + */ +static AffixNode * +mkANode(IspellDict *Conf, int low, int high, int level, int type) +{ + int i; + int nchar = 0; + uint8 lastchar = '\0'; + AffixNode *rs; + AffixNodeData *data; + int lownew = low; + int naff; + AFFIX **aff; + + for (i = low; i < high; i++) + if (Conf->Affix[i].replen > level && lastchar != GETCHAR(Conf->Affix + i, level, type)) + { + nchar++; + lastchar = GETCHAR(Conf->Affix + i, level, type); + } + + if (!nchar) + return NULL; + + aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1)); + naff = 0; + + rs = (AffixNode *) cpalloc0(ANHRDSZ + nchar * sizeof(AffixNodeData)); + rs->length = nchar; + data = rs->data; + + lastchar = '\0'; + for (i = low; i < high; i++) + if (Conf->Affix[i].replen > level) + { + if (lastchar != GETCHAR(Conf->Affix + i, level, type)) + { + if (lastchar) + { + /* Next level of the prefix tree */ + data->node = mkANode(Conf, lownew, i, level + 1, type); + if (naff) + { + data->naff = naff; + data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * naff); + memcpy(data->aff, aff, sizeof(AFFIX *) * naff); + naff = 0; + } + data++; + lownew = i; + } + lastchar = GETCHAR(Conf->Affix + i, level, type); + } + data->val = GETCHAR(Conf->Affix + i, level, type); + if (Conf->Affix[i].replen == level + 1) + { /* affix stopped */ + aff[naff++] = Conf->Affix + i; + } + } + + /* Next level of the prefix tree */ + data->node = mkANode(Conf, lownew, high, level + 1, type); + if (naff) + { + data->naff = naff; + data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * naff); + memcpy(data->aff, aff, sizeof(AFFIX *) * naff); + naff = 0; + } + + pfree(aff); + + return rs; +} + +/* + * Makes the root void node in the prefix tree. The root void node is created + * for affixes which have empty replace string ("repl" field). + */ +static void +mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix) +{ + int i, + cnt = 0; + int start = (issuffix) ? startsuffix : 0; + int end = (issuffix) ? Conf->naffixes : startsuffix; + AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData)); + + Affix->length = 1; + Affix->isvoid = 1; + + if (issuffix) + { + Affix->data->node = Conf->Suffix; + Conf->Suffix = Affix; + } + else + { + Affix->data->node = Conf->Prefix; + Conf->Prefix = Affix; + } + + /* Count affixes with empty replace string */ + for (i = start; i < end; i++) + if (Conf->Affix[i].replen == 0) + cnt++; + + /* There is not affixes with empty replace string */ + if (cnt == 0) + return; + + Affix->data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * cnt); + Affix->data->naff = (uint32) cnt; + + cnt = 0; + for (i = start; i < end; i++) + if (Conf->Affix[i].replen == 0) + { + Affix->data->aff[cnt] = Conf->Affix + i; + cnt++; + } +} + +/* + * Checks if the affixflag is used by dictionary. Conf->AffixData does not + * contain affixflag if this flag is not used actually by the .dict file. + * + * Conf: current dictionary. + * affixflag: affix flag. + * + * Returns true if the Conf->AffixData array contains affixflag, otherwise + * returns false. + */ +static bool +isAffixInUse(IspellDict *Conf, char *affixflag) +{ + int i; + + for (i = 0; i < Conf->nAffixData; i++) + if (IsAffixFlagInUse(Conf, i, affixflag)) + return true; + + return false; +} + +/* + * Builds Conf->Prefix and Conf->Suffix trees from the imported affixes. + */ +void +NISortAffixes(IspellDict *Conf) +{ + AFFIX *Affix; + size_t i; + CMPDAffix *ptr; + int firstsuffix = Conf->naffixes; + + if (Conf->naffixes == 0) + return; + + /* Store compound affixes in the Conf->CompoundAffix array */ + if (Conf->naffixes > 1) + qsort((void *) Conf->Affix, Conf->naffixes, sizeof(AFFIX), cmpaffix); + Conf->CompoundAffix = ptr = (CMPDAffix *) palloc(sizeof(CMPDAffix) * Conf->naffixes); + ptr->affix = NULL; + + for (i = 0; i < Conf->naffixes; i++) + { + Affix = &(((AFFIX *) Conf->Affix)[i]); + if (Affix->type == FF_SUFFIX && i < firstsuffix) + firstsuffix = i; + + if ((Affix->flagflags & FF_COMPOUNDFLAG) && Affix->replen > 0 && + isAffixInUse(Conf, Affix->flag)) + { + bool issuffix = (Affix->type == FF_SUFFIX); + + if (ptr == Conf->CompoundAffix || + issuffix != (ptr - 1)->issuffix || + strbncmp((const unsigned char *) (ptr - 1)->affix, + (const unsigned char *) Affix->repl, + (ptr - 1)->len)) + { + /* leave only unique and minimal suffixes */ + ptr->affix = Affix->repl; + ptr->len = Affix->replen; + ptr->issuffix = issuffix; + ptr++; + } + } + } + ptr->affix = NULL; + Conf->CompoundAffix = (CMPDAffix *) repalloc(Conf->CompoundAffix, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1)); + + /* Start build a prefix tree */ + Conf->Prefix = mkANode(Conf, 0, firstsuffix, 0, FF_PREFIX); + Conf->Suffix = mkANode(Conf, firstsuffix, Conf->naffixes, 0, FF_SUFFIX); + mkVoidAffix(Conf, true, firstsuffix); + mkVoidAffix(Conf, false, firstsuffix); +} + +static AffixNodeData * +FindAffixes(AffixNode *node, const char *word, int wrdlen, int *level, int type) +{ + AffixNodeData *StopLow, + *StopHigh, + *StopMiddle; + uint8 symbol; + + if (node->isvoid) + { /* search void affixes */ + if (node->data->naff) + return node->data; + node = node->data->node; + } + + while (node && *level < wrdlen) + { + StopLow = node->data; + StopHigh = node->data + node->length; + while (StopLow < StopHigh) + { + StopMiddle = StopLow + ((StopHigh - StopLow) >> 1); + symbol = GETWCHAR(word, wrdlen, *level, type); + + if (StopMiddle->val == symbol) + { + (*level)++; + if (StopMiddle->naff) + return StopMiddle; + node = StopMiddle->node; + break; + } + else if (StopMiddle->val < symbol) + StopLow = StopMiddle + 1; + else + StopHigh = StopMiddle; + } + if (StopLow >= StopHigh) + break; + } + return NULL; +} + +static char * +CheckAffix(const char *word, size_t len, AFFIX *Affix, int flagflags, char *newword, int *baselen) +{ + /* + * Check compound allow flags + */ + + if (flagflags == 0) + { + if (Affix->flagflags & FF_COMPOUNDONLY) + return NULL; + } + else if (flagflags & FF_COMPOUNDBEGIN) + { + if (Affix->flagflags & FF_COMPOUNDFORBIDFLAG) + return NULL; + if ((Affix->flagflags & FF_COMPOUNDBEGIN) == 0) + if (Affix->type == FF_SUFFIX) + return NULL; + } + else if (flagflags & FF_COMPOUNDMIDDLE) + { + if ((Affix->flagflags & FF_COMPOUNDMIDDLE) == 0 || + (Affix->flagflags & FF_COMPOUNDFORBIDFLAG)) + return NULL; + } + else if (flagflags & FF_COMPOUNDLAST) + { + if (Affix->flagflags & FF_COMPOUNDFORBIDFLAG) + return NULL; + if ((Affix->flagflags & FF_COMPOUNDLAST) == 0) + if (Affix->type == FF_PREFIX) + return NULL; + } + + /* + * make replace pattern of affix + */ + if (Affix->type == FF_SUFFIX) + { + strcpy(newword, word); + strcpy(newword + len - Affix->replen, Affix->find); + if (baselen) /* store length of non-changed part of word */ + *baselen = len - Affix->replen; + } + else + { + /* + * if prefix is an all non-changed part's length then all word + * contains only prefix and suffix, so out + */ + if (baselen && *baselen + strlen(Affix->find) <= Affix->replen) + return NULL; + strcpy(newword, Affix->find); + strcat(newword, word + Affix->replen); + } + + /* + * check resulting word + */ + if (Affix->issimple) + return newword; + else if (Affix->isregis) + { + if (RS_execute(&(Affix->reg.regis), newword)) + return newword; + } + else + { + pg_wchar *data; + size_t data_len; + int newword_len; + + /* Convert data string to wide characters */ + newword_len = strlen(newword); + data = (pg_wchar *) palloc((newword_len + 1) * sizeof(pg_wchar)); + data_len = pg_mb2wchar_with_len(newword, data, newword_len); + + if (pg_regexec(&(Affix->reg.pregex->regex), data, data_len, + 0, NULL, 0, NULL, 0) == REG_OKAY) + { + pfree(data); + return newword; + } + pfree(data); + } + + return NULL; +} + +static int +addToResult(char **forms, char **cur, char *word) +{ + if (cur - forms >= MAX_NORM - 1) + return 0; + if (forms == cur || strcmp(word, *(cur - 1)) != 0) + { + *cur = pstrdup(word); + *(cur + 1) = NULL; + return 1; + } + + return 0; +} + +static char ** +NormalizeSubWord(IspellDict *Conf, char *word, int flag) +{ + AffixNodeData *suffix = NULL, + *prefix = NULL; + int slevel = 0, + plevel = 0; + int wrdlen = strlen(word), + swrdlen; + char **forms; + char **cur; + char newword[2 * MAXNORMLEN] = ""; + char pnewword[2 * MAXNORMLEN] = ""; + AffixNode *snode = Conf->Suffix, + *pnode; + int i, + j; + + if (wrdlen > MAXNORMLEN) + return NULL; + cur = forms = (char **) palloc(MAX_NORM * sizeof(char *)); + *cur = NULL; + + + /* Check that the word itself is normal form */ + if (FindWord(Conf, word, VoidString, flag)) + { + *cur = pstrdup(word); + cur++; + *cur = NULL; + } + + /* Find all other NORMAL forms of the 'word' (check only prefix) */ + pnode = Conf->Prefix; + plevel = 0; + while (pnode) + { + prefix = FindAffixes(pnode, word, wrdlen, &plevel, FF_PREFIX); + if (!prefix) + break; + for (j = 0; j < prefix->naff; j++) + { + if (CheckAffix(word, wrdlen, prefix->aff[j], flag, newword, NULL)) + { + /* prefix success */ + if (FindWord(Conf, newword, prefix->aff[j]->flag, flag)) + cur += addToResult(forms, cur, newword); + } + } + pnode = prefix->node; + } + + /* + * Find all other NORMAL forms of the 'word' (check suffix and then + * prefix) + */ + while (snode) + { + int baselen = 0; + + /* find possible suffix */ + suffix = FindAffixes(snode, word, wrdlen, &slevel, FF_SUFFIX); + if (!suffix) + break; + /* foreach suffix check affix */ + for (i = 0; i < suffix->naff; i++) + { + if (CheckAffix(word, wrdlen, suffix->aff[i], flag, newword, &baselen)) + { + /* suffix success */ + if (FindWord(Conf, newword, suffix->aff[i]->flag, flag)) + cur += addToResult(forms, cur, newword); + + /* now we will look changed word with prefixes */ + pnode = Conf->Prefix; + plevel = 0; + swrdlen = strlen(newword); + while (pnode) + { + prefix = FindAffixes(pnode, newword, swrdlen, &plevel, FF_PREFIX); + if (!prefix) + break; + for (j = 0; j < prefix->naff; j++) + { + if (CheckAffix(newword, swrdlen, prefix->aff[j], flag, pnewword, &baselen)) + { + /* prefix success */ + char *ff = (prefix->aff[j]->flagflags & suffix->aff[i]->flagflags & FF_CROSSPRODUCT) ? + VoidString : prefix->aff[j]->flag; + + if (FindWord(Conf, pnewword, ff, flag)) + cur += addToResult(forms, cur, pnewword); + } + } + pnode = prefix->node; + } + } + } + + snode = suffix->node; + } + + if (cur == forms) + { + pfree(forms); + return NULL; + } + return forms; +} + +typedef struct SplitVar +{ + int nstem; + int lenstem; + char **stem; + struct SplitVar *next; +} SplitVar; + +static int +CheckCompoundAffixes(CMPDAffix **ptr, char *word, int len, bool CheckInPlace) +{ + bool issuffix; + + /* in case CompoundAffix is null: */ + if (*ptr == NULL) + return -1; + + if (CheckInPlace) + { + while ((*ptr)->affix) + { + if (len > (*ptr)->len && strncmp((*ptr)->affix, word, (*ptr)->len) == 0) + { + len = (*ptr)->len; + issuffix = (*ptr)->issuffix; + (*ptr)++; + return (issuffix) ? len : 0; + } + (*ptr)++; + } + } + else + { + char *affbegin; + + while ((*ptr)->affix) + { + if (len > (*ptr)->len && (affbegin = strstr(word, (*ptr)->affix)) != NULL) + { + len = (*ptr)->len + (affbegin - word); + issuffix = (*ptr)->issuffix; + (*ptr)++; + return (issuffix) ? len : 0; + } + (*ptr)++; + } + } + return -1; +} + +static SplitVar * +CopyVar(SplitVar *s, int makedup) +{ + SplitVar *v = (SplitVar *) palloc(sizeof(SplitVar)); + + v->next = NULL; + if (s) + { + int i; + + v->lenstem = s->lenstem; + v->stem = (char **) palloc(sizeof(char *) * v->lenstem); + v->nstem = s->nstem; + for (i = 0; i < s->nstem; i++) + v->stem[i] = (makedup) ? pstrdup(s->stem[i]) : s->stem[i]; + } + else + { + v->lenstem = 16; + v->stem = (char **) palloc(sizeof(char *) * v->lenstem); + v->nstem = 0; + } + return v; +} + +static void +AddStem(SplitVar *v, char *word) +{ + if (v->nstem >= v->lenstem) + { + v->lenstem *= 2; + v->stem = (char **) repalloc(v->stem, sizeof(char *) * v->lenstem); + } + + v->stem[v->nstem] = word; + v->nstem++; +} + +static SplitVar * +SplitToVariants(IspellDict *Conf, SPNode *snode, SplitVar *orig, char *word, int wordlen, int startpos, int minpos) +{ + SplitVar *var = NULL; + SPNodeData *StopLow, + *StopHigh, + *StopMiddle = NULL; + SPNode *node = (snode) ? snode : Conf->Dictionary; + int level = (snode) ? minpos : startpos; /* recursive + * minpos==level */ + int lenaff; + CMPDAffix *caff; + char *notprobed; + int compoundflag = 0; + + notprobed = (char *) palloc(wordlen); + memset(notprobed, 1, wordlen); + var = CopyVar(orig, 1); + + while (level < wordlen) + { + /* find word with epenthetic or/and compound affix */ + caff = Conf->CompoundAffix; + while (level > startpos && (lenaff = CheckCompoundAffixes(&caff, word + level, wordlen - level, (node) ? true : false)) >= 0) + { + /* + * there is one of compound affixes, so check word for existings + */ + char buf[MAXNORMLEN]; + char **subres; + + lenaff = level - startpos + lenaff; + + if (!notprobed[startpos + lenaff - 1]) + continue; + + if (level + lenaff - 1 <= minpos) + continue; + + if (lenaff >= MAXNORMLEN) + continue; /* skip too big value */ + if (lenaff > 0) + memcpy(buf, word + startpos, lenaff); + buf[lenaff] = '\0'; + + if (level == 0) + compoundflag = FF_COMPOUNDBEGIN; + else if (level == wordlen - 1) + compoundflag = FF_COMPOUNDLAST; + else + compoundflag = FF_COMPOUNDMIDDLE; + subres = NormalizeSubWord(Conf, buf, compoundflag); + if (subres) + { + /* Yes, it was a word from dictionary */ + SplitVar *new = CopyVar(var, 0); + SplitVar *ptr = var; + char **sptr = subres; + + notprobed[startpos + lenaff - 1] = 0; + + while (*sptr) + { + AddStem(new, *sptr); + sptr++; + } + pfree(subres); + + while (ptr->next) + ptr = ptr->next; + ptr->next = SplitToVariants(Conf, NULL, new, word, wordlen, startpos + lenaff, startpos + lenaff); + + pfree(new->stem); + pfree(new); + } + } + + if (!node) + break; + + StopLow = node->data; + StopHigh = node->data + node->length; + while (StopLow < StopHigh) + { + StopMiddle = StopLow + ((StopHigh - StopLow) >> 1); + if (StopMiddle->val == ((uint8 *) (word))[level]) + break; + else if (StopMiddle->val < ((uint8 *) (word))[level]) + StopLow = StopMiddle + 1; + else + StopHigh = StopMiddle; + } + + if (StopLow < StopHigh) + { + if (startpos == 0) + compoundflag = FF_COMPOUNDBEGIN; + else if (level == wordlen - 1) + compoundflag = FF_COMPOUNDLAST; + else + compoundflag = FF_COMPOUNDMIDDLE; + + /* find infinitive */ + if (StopMiddle->isword && + (StopMiddle->compoundflag & compoundflag) && + notprobed[level]) + { + /* ok, we found full compoundallowed word */ + if (level > minpos) + { + /* and its length more than minimal */ + if (wordlen == level + 1) + { + /* well, it was last word */ + AddStem(var, pnstrdup(word + startpos, wordlen - startpos)); + pfree(notprobed); + return var; + } + else + { + /* then we will search more big word at the same point */ + SplitVar *ptr = var; + + while (ptr->next) + ptr = ptr->next; + ptr->next = SplitToVariants(Conf, node, var, word, wordlen, startpos, level); + /* we can find next word */ + level++; + AddStem(var, pnstrdup(word + startpos, level - startpos)); + node = Conf->Dictionary; + startpos = level; + continue; + } + } + } + node = StopMiddle->node; + } + else + node = NULL; + level++; + } + + AddStem(var, pnstrdup(word + startpos, wordlen - startpos)); + pfree(notprobed); + return var; +} + +static void +addNorm(TSLexeme **lres, TSLexeme **lcur, char *word, int flags, uint16 NVariant) +{ + if (*lres == NULL) + *lcur = *lres = (TSLexeme *) palloc(MAX_NORM * sizeof(TSLexeme)); + + if (*lcur - *lres < MAX_NORM - 1) + { + (*lcur)->lexeme = word; + (*lcur)->flags = flags; + (*lcur)->nvariant = NVariant; + (*lcur)++; + (*lcur)->lexeme = NULL; + } +} + +TSLexeme * +NINormalizeWord(IspellDict *Conf, char *word) +{ + char **res; + TSLexeme *lcur = NULL, + *lres = NULL; + uint16 NVariant = 1; + + res = NormalizeSubWord(Conf, word, 0); + + if (res) + { + char **ptr = res; + + while (*ptr && (lcur - lres) < MAX_NORM) + { + addNorm(&lres, &lcur, *ptr, 0, NVariant++); + ptr++; + } + pfree(res); + } + + if (Conf->usecompound) + { + int wordlen = strlen(word); + SplitVar *ptr, + *var = SplitToVariants(Conf, NULL, NULL, word, wordlen, 0, -1); + int i; + + while (var) + { + if (var->nstem > 1) + { + char **subres = NormalizeSubWord(Conf, var->stem[var->nstem - 1], FF_COMPOUNDLAST); + + if (subres) + { + char **subptr = subres; + + while (*subptr) + { + for (i = 0; i < var->nstem - 1; i++) + { + addNorm(&lres, &lcur, (subptr == subres) ? var->stem[i] : pstrdup(var->stem[i]), 0, NVariant); + } + + addNorm(&lres, &lcur, *subptr, 0, NVariant); + subptr++; + NVariant++; + } + + pfree(subres); + var->stem[0] = NULL; + pfree(var->stem[var->nstem - 1]); + } + } + + for (i = 0; i < var->nstem && var->stem[i]; i++) + pfree(var->stem[i]); + ptr = var->next; + pfree(var->stem); + pfree(var); + var = ptr; + } + } + + return lres; +} diff --git a/src/backend/tsearch/to_tsany.c b/src/backend/tsearch/to_tsany.c new file mode 100644 index 0000000..f4ddfc0 --- /dev/null +++ b/src/backend/tsearch/to_tsany.c @@ -0,0 +1,724 @@ +/*------------------------------------------------------------------------- + * + * to_tsany.c + * to_ts* function definitions + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/to_tsany.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "common/jsonapi.h" +#include "tsearch/ts_cache.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" +#include "utils/jsonfuncs.h" + + +/* + * Opaque data structure, which is passed by parse_tsquery() to pushval_morph(). + */ +typedef struct MorphOpaque +{ + Oid cfg_id; + + /* + * Single tsquery morph could be parsed into multiple words. When these + * words reside in adjacent positions, they are connected using this + * operator. Usually, that is OP_PHRASE, which requires word positions of + * a complex morph to exactly match the tsvector. + */ + int qoperator; +} MorphOpaque; + +typedef struct TSVectorBuildState +{ + ParsedText *prs; + Oid cfgId; +} TSVectorBuildState; + +static void add_to_tsvector(void *_state, char *elem_value, int elem_len); + + +Datum +get_current_ts_config(PG_FUNCTION_ARGS) +{ + PG_RETURN_OID(getTSCurrentConfig(true)); +} + +/* + * to_tsvector + */ +static int +compareWORD(const void *a, const void *b) +{ + int res; + + res = tsCompareString(((const ParsedWord *) a)->word, ((const ParsedWord *) a)->len, + ((const ParsedWord *) b)->word, ((const ParsedWord *) b)->len, + false); + + if (res == 0) + { + if (((const ParsedWord *) a)->pos.pos == ((const ParsedWord *) b)->pos.pos) + return 0; + + res = (((const ParsedWord *) a)->pos.pos > ((const ParsedWord *) b)->pos.pos) ? 1 : -1; + } + + return res; +} + +static int +uniqueWORD(ParsedWord *a, int32 l) +{ + ParsedWord *ptr, + *res; + int tmppos; + + if (l == 1) + { + tmppos = LIMITPOS(a->pos.pos); + a->alen = 2; + a->pos.apos = (uint16 *) palloc(sizeof(uint16) * a->alen); + a->pos.apos[0] = 1; + a->pos.apos[1] = tmppos; + return l; + } + + res = a; + ptr = a + 1; + + /* + * Sort words with its positions + */ + qsort((void *) a, l, sizeof(ParsedWord), compareWORD); + + /* + * Initialize first word and its first position + */ + tmppos = LIMITPOS(a->pos.pos); + a->alen = 2; + a->pos.apos = (uint16 *) palloc(sizeof(uint16) * a->alen); + a->pos.apos[0] = 1; + a->pos.apos[1] = tmppos; + + /* + * Summarize position information for each word + */ + while (ptr - a < l) + { + if (!(ptr->len == res->len && + strncmp(ptr->word, res->word, res->len) == 0)) + { + /* + * Got a new word, so put it in result + */ + res++; + res->len = ptr->len; + res->word = ptr->word; + tmppos = LIMITPOS(ptr->pos.pos); + res->alen = 2; + res->pos.apos = (uint16 *) palloc(sizeof(uint16) * res->alen); + res->pos.apos[0] = 1; + res->pos.apos[1] = tmppos; + } + else + { + /* + * The word already exists, so adjust position information. But + * before we should check size of position's array, max allowed + * value for position and uniqueness of position + */ + pfree(ptr->word); + if (res->pos.apos[0] < MAXNUMPOS - 1 && res->pos.apos[res->pos.apos[0]] != MAXENTRYPOS - 1 && + res->pos.apos[res->pos.apos[0]] != LIMITPOS(ptr->pos.pos)) + { + if (res->pos.apos[0] + 1 >= res->alen) + { + res->alen *= 2; + res->pos.apos = (uint16 *) repalloc(res->pos.apos, sizeof(uint16) * res->alen); + } + if (res->pos.apos[0] == 0 || res->pos.apos[res->pos.apos[0]] != LIMITPOS(ptr->pos.pos)) + { + res->pos.apos[res->pos.apos[0] + 1] = LIMITPOS(ptr->pos.pos); + res->pos.apos[0]++; + } + } + } + ptr++; + } + + return res + 1 - a; +} + +/* + * make value of tsvector, given parsed text + * + * Note: frees prs->words and subsidiary data. + */ +TSVector +make_tsvector(ParsedText *prs) +{ + int i, + j, + lenstr = 0, + totallen; + TSVector in; + WordEntry *ptr; + char *str; + int stroff; + + /* Merge duplicate words */ + if (prs->curwords > 0) + prs->curwords = uniqueWORD(prs->words, prs->curwords); + + /* Determine space needed */ + for (i = 0; i < prs->curwords; i++) + { + lenstr += prs->words[i].len; + if (prs->words[i].alen) + { + lenstr = SHORTALIGN(lenstr); + lenstr += sizeof(uint16) + prs->words[i].pos.apos[0] * sizeof(WordEntryPos); + } + } + + if (lenstr > MAXSTRPOS) + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("string is too long for tsvector (%d bytes, max %d bytes)", lenstr, MAXSTRPOS))); + + totallen = CALCDATASIZE(prs->curwords, lenstr); + in = (TSVector) palloc0(totallen); + SET_VARSIZE(in, totallen); + in->size = prs->curwords; + + ptr = ARRPTR(in); + str = STRPTR(in); + stroff = 0; + for (i = 0; i < prs->curwords; i++) + { + ptr->len = prs->words[i].len; + ptr->pos = stroff; + memcpy(str + stroff, prs->words[i].word, prs->words[i].len); + stroff += prs->words[i].len; + pfree(prs->words[i].word); + if (prs->words[i].alen) + { + int k = prs->words[i].pos.apos[0]; + WordEntryPos *wptr; + + if (k > 0xFFFF) + elog(ERROR, "positions array too long"); + + ptr->haspos = 1; + stroff = SHORTALIGN(stroff); + *(uint16 *) (str + stroff) = (uint16) k; + wptr = POSDATAPTR(in, ptr); + for (j = 0; j < k; j++) + { + WEP_SETWEIGHT(wptr[j], 0); + WEP_SETPOS(wptr[j], prs->words[i].pos.apos[j + 1]); + } + stroff += sizeof(uint16) + k * sizeof(WordEntryPos); + pfree(prs->words[i].pos.apos); + } + else + ptr->haspos = 0; + ptr++; + } + + if (prs->words) + pfree(prs->words); + + return in; +} + +Datum +to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + text *in = PG_GETARG_TEXT_PP(1); + ParsedText prs; + TSVector out; + + prs.lenwords = VARSIZE_ANY_EXHDR(in) / 6; /* just estimation of word's + * number */ + if (prs.lenwords < 2) + prs.lenwords = 2; + prs.curwords = 0; + prs.pos = 0; + prs.words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs.lenwords); + + parsetext(cfgId, &prs, VARDATA_ANY(in), VARSIZE_ANY_EXHDR(in)); + + PG_FREE_IF_COPY(in, 1); + + out = make_tsvector(&prs); + + PG_RETURN_TSVECTOR(out); +} + +Datum +to_tsvector(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(to_tsvector_byid, + ObjectIdGetDatum(cfgId), + PointerGetDatum(in))); +} + +/* + * Worker function for jsonb(_string)_to_tsvector(_byid) + */ +static TSVector +jsonb_to_tsvector_worker(Oid cfgId, Jsonb *jb, uint32 flags) +{ + TSVectorBuildState state; + ParsedText prs; + + prs.words = NULL; + prs.curwords = 0; + state.prs = &prs; + state.cfgId = cfgId; + + iterate_jsonb_values(jb, flags, &state, add_to_tsvector); + + return make_tsvector(&prs); +} + +Datum +jsonb_string_to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + Jsonb *jb = PG_GETARG_JSONB_P(1); + TSVector result; + + result = jsonb_to_tsvector_worker(cfgId, jb, jtiString); + PG_FREE_IF_COPY(jb, 1); + + PG_RETURN_TSVECTOR(result); +} + +Datum +jsonb_string_to_tsvector(PG_FUNCTION_ARGS) +{ + Jsonb *jb = PG_GETARG_JSONB_P(0); + Oid cfgId; + TSVector result; + + cfgId = getTSCurrentConfig(true); + result = jsonb_to_tsvector_worker(cfgId, jb, jtiString); + PG_FREE_IF_COPY(jb, 0); + + PG_RETURN_TSVECTOR(result); +} + +Datum +jsonb_to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + Jsonb *jb = PG_GETARG_JSONB_P(1); + Jsonb *jbFlags = PG_GETARG_JSONB_P(2); + TSVector result; + uint32 flags = parse_jsonb_index_flags(jbFlags); + + result = jsonb_to_tsvector_worker(cfgId, jb, flags); + PG_FREE_IF_COPY(jb, 1); + PG_FREE_IF_COPY(jbFlags, 2); + + PG_RETURN_TSVECTOR(result); +} + +Datum +jsonb_to_tsvector(PG_FUNCTION_ARGS) +{ + Jsonb *jb = PG_GETARG_JSONB_P(0); + Jsonb *jbFlags = PG_GETARG_JSONB_P(1); + Oid cfgId; + TSVector result; + uint32 flags = parse_jsonb_index_flags(jbFlags); + + cfgId = getTSCurrentConfig(true); + result = jsonb_to_tsvector_worker(cfgId, jb, flags); + PG_FREE_IF_COPY(jb, 0); + PG_FREE_IF_COPY(jbFlags, 1); + + PG_RETURN_TSVECTOR(result); +} + +/* + * Worker function for json(_string)_to_tsvector(_byid) + */ +static TSVector +json_to_tsvector_worker(Oid cfgId, text *json, uint32 flags) +{ + TSVectorBuildState state; + ParsedText prs; + + prs.words = NULL; + prs.curwords = 0; + state.prs = &prs; + state.cfgId = cfgId; + + iterate_json_values(json, flags, &state, add_to_tsvector); + + return make_tsvector(&prs); +} + +Datum +json_string_to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + text *json = PG_GETARG_TEXT_P(1); + TSVector result; + + result = json_to_tsvector_worker(cfgId, json, jtiString); + PG_FREE_IF_COPY(json, 1); + + PG_RETURN_TSVECTOR(result); +} + +Datum +json_string_to_tsvector(PG_FUNCTION_ARGS) +{ + text *json = PG_GETARG_TEXT_P(0); + Oid cfgId; + TSVector result; + + cfgId = getTSCurrentConfig(true); + result = json_to_tsvector_worker(cfgId, json, jtiString); + PG_FREE_IF_COPY(json, 0); + + PG_RETURN_TSVECTOR(result); +} + +Datum +json_to_tsvector_byid(PG_FUNCTION_ARGS) +{ + Oid cfgId = PG_GETARG_OID(0); + text *json = PG_GETARG_TEXT_P(1); + Jsonb *jbFlags = PG_GETARG_JSONB_P(2); + TSVector result; + uint32 flags = parse_jsonb_index_flags(jbFlags); + + result = json_to_tsvector_worker(cfgId, json, flags); + PG_FREE_IF_COPY(json, 1); + PG_FREE_IF_COPY(jbFlags, 2); + + PG_RETURN_TSVECTOR(result); +} + +Datum +json_to_tsvector(PG_FUNCTION_ARGS) +{ + text *json = PG_GETARG_TEXT_P(0); + Jsonb *jbFlags = PG_GETARG_JSONB_P(1); + Oid cfgId; + TSVector result; + uint32 flags = parse_jsonb_index_flags(jbFlags); + + cfgId = getTSCurrentConfig(true); + result = json_to_tsvector_worker(cfgId, json, flags); + PG_FREE_IF_COPY(json, 0); + PG_FREE_IF_COPY(jbFlags, 1); + + PG_RETURN_TSVECTOR(result); +} + +/* + * Parse lexemes in an element of a json(b) value, add to TSVectorBuildState. + */ +static void +add_to_tsvector(void *_state, char *elem_value, int elem_len) +{ + TSVectorBuildState *state = (TSVectorBuildState *) _state; + ParsedText *prs = state->prs; + int32 prevwords; + + if (prs->words == NULL) + { + /* + * First time through: initialize words array to a reasonable size. + * (parsetext() will realloc it bigger as needed.) + */ + prs->lenwords = 16; + prs->words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs->lenwords); + prs->curwords = 0; + prs->pos = 0; + } + + prevwords = prs->curwords; + + parsetext(state->cfgId, prs, elem_value, elem_len); + + /* + * If we extracted any words from this JSON element, advance pos to create + * an artificial break between elements. This is because we don't want + * phrase searches to think that the last word in this element is adjacent + * to the first word in the next one. + */ + if (prs->curwords > prevwords) + prs->pos += 1; +} + + +/* + * to_tsquery + */ + + +/* + * This function is used for morph parsing. + * + * The value is passed to parsetext which will call the right dictionary to + * lexize the word. If it turns out to be a stopword, we push a QI_VALSTOP + * to the stack. + * + * All words belonging to the same variant are pushed as an ANDed list, + * and different variants are ORed together. + */ +static void +pushval_morph(Datum opaque, TSQueryParserState state, char *strval, int lenval, int16 weight, bool prefix) +{ + int32 count = 0; + ParsedText prs; + uint32 variant, + pos = 0, + cntvar = 0, + cntpos = 0, + cnt = 0; + MorphOpaque *data = (MorphOpaque *) DatumGetPointer(opaque); + + prs.lenwords = 4; + prs.curwords = 0; + prs.pos = 0; + prs.words = (ParsedWord *) palloc(sizeof(ParsedWord) * prs.lenwords); + + parsetext(data->cfg_id, &prs, strval, lenval); + + if (prs.curwords > 0) + { + while (count < prs.curwords) + { + /* + * Were any stop words removed? If so, fill empty positions with + * placeholders linked by an appropriate operator. + */ + if (pos > 0 && pos + 1 < prs.words[count].pos.pos) + { + while (pos + 1 < prs.words[count].pos.pos) + { + /* put placeholders for each missing stop word */ + pushStop(state); + if (cntpos) + pushOperator(state, data->qoperator, 1); + cntpos++; + pos++; + } + } + + /* save current word's position */ + pos = prs.words[count].pos.pos; + + /* Go through all variants obtained from this token */ + cntvar = 0; + while (count < prs.curwords && pos == prs.words[count].pos.pos) + { + variant = prs.words[count].nvariant; + + /* Push all words belonging to the same variant */ + cnt = 0; + while (count < prs.curwords && + pos == prs.words[count].pos.pos && + variant == prs.words[count].nvariant) + { + pushValue(state, + prs.words[count].word, + prs.words[count].len, + weight, + ((prs.words[count].flags & TSL_PREFIX) || prefix)); + pfree(prs.words[count].word); + if (cnt) + pushOperator(state, OP_AND, 0); + cnt++; + count++; + } + + if (cntvar) + pushOperator(state, OP_OR, 0); + cntvar++; + } + + if (cntpos) + { + /* distance may be useful */ + pushOperator(state, data->qoperator, 1); + } + + cntpos++; + } + + pfree(prs.words); + + } + else + pushStop(state); +} + +Datum +to_tsquery_byid(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(1); + TSQuery query; + MorphOpaque data; + + data.cfg_id = PG_GETARG_OID(0); + + /* + * Passing OP_PHRASE as a qoperator makes tsquery require matching of word + * positions of a complex morph exactly match the tsvector. Also, when + * the complex morphs are connected with OP_PHRASE operator, we connect + * all their words into the OP_PHRASE sequence. + */ + data.qoperator = OP_PHRASE; + + query = parse_tsquery(text_to_cstring(in), + pushval_morph, + PointerGetDatum(&data), + 0); + + PG_RETURN_TSQUERY(query); +} + +Datum +to_tsquery(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(to_tsquery_byid, + ObjectIdGetDatum(cfgId), + PointerGetDatum(in))); +} + +Datum +plainto_tsquery_byid(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(1); + TSQuery query; + MorphOpaque data; + + data.cfg_id = PG_GETARG_OID(0); + + /* + * parse_tsquery() with P_TSQ_PLAIN flag takes the whole input text as a + * single morph. Passing OP_PHRASE as a qoperator makes tsquery require + * matching of all words independently on their positions. + */ + data.qoperator = OP_AND; + + query = parse_tsquery(text_to_cstring(in), + pushval_morph, + PointerGetDatum(&data), + P_TSQ_PLAIN); + + PG_RETURN_POINTER(query); +} + +Datum +plainto_tsquery(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(plainto_tsquery_byid, + ObjectIdGetDatum(cfgId), + PointerGetDatum(in))); +} + + +Datum +phraseto_tsquery_byid(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(1); + TSQuery query; + MorphOpaque data; + + data.cfg_id = PG_GETARG_OID(0); + + /* + * parse_tsquery() with P_TSQ_PLAIN flag takes the whole input text as a + * single morph. Passing OP_PHRASE as a qoperator makes tsquery require + * matching of word positions. + */ + data.qoperator = OP_PHRASE; + + query = parse_tsquery(text_to_cstring(in), + pushval_morph, + PointerGetDatum(&data), + P_TSQ_PLAIN); + + PG_RETURN_TSQUERY(query); +} + +Datum +phraseto_tsquery(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(phraseto_tsquery_byid, + ObjectIdGetDatum(cfgId), + PointerGetDatum(in))); +} + +Datum +websearch_to_tsquery_byid(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(1); + MorphOpaque data; + TSQuery query = NULL; + + data.cfg_id = PG_GETARG_OID(0); + + /* + * Passing OP_PHRASE as a qoperator makes tsquery require matching of word + * positions of a complex morph exactly match the tsvector. Also, when + * the complex morphs are given in quotes, we connect all their words into + * the OP_PHRASE sequence. + */ + data.qoperator = OP_PHRASE; + + query = parse_tsquery(text_to_cstring(in), + pushval_morph, + PointerGetDatum(&data), + P_TSQ_WEB); + + PG_RETURN_TSQUERY(query); +} + +Datum +websearch_to_tsquery(PG_FUNCTION_ARGS) +{ + text *in = PG_GETARG_TEXT_PP(0); + Oid cfgId; + + cfgId = getTSCurrentConfig(true); + PG_RETURN_DATUM(DirectFunctionCall2(websearch_to_tsquery_byid, + ObjectIdGetDatum(cfgId), + PointerGetDatum(in))); + +} diff --git a/src/backend/tsearch/ts_locale.c b/src/backend/tsearch/ts_locale.c new file mode 100644 index 0000000..f918cc8 --- /dev/null +++ b/src/backend/tsearch/ts_locale.c @@ -0,0 +1,325 @@ +/*------------------------------------------------------------------------- + * + * ts_locale.c + * locale compatibility layer for tsearch + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/ts_locale.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/pg_collation.h" +#include "common/string.h" +#include "storage/fd.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_public.h" + +static void tsearch_readline_callback(void *arg); + + +/* + * The reason these functions use a 3-wchar_t output buffer, not 2 as you + * might expect, is that on Windows "wchar_t" is 16 bits and what we'll be + * getting from char2wchar() is UTF16 not UTF32. A single input character + * may therefore produce a surrogate pair rather than just one wchar_t; + * we also need room for a trailing null. When we do get a surrogate pair, + * we pass just the first code to iswdigit() etc, so that these functions will + * always return false for characters outside the Basic Multilingual Plane. + */ +#define WC_BUF_LEN 3 + +int +t_isdigit(const char *ptr) +{ + int clen = pg_mblen(ptr); + wchar_t character[WC_BUF_LEN]; + Oid collation = DEFAULT_COLLATION_OID; /* TODO */ + pg_locale_t mylocale = 0; /* TODO */ + + if (clen == 1 || lc_ctype_is_c(collation)) + return isdigit(TOUCHAR(ptr)); + + char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale); + + return iswdigit((wint_t) character[0]); +} + +int +t_isspace(const char *ptr) +{ + int clen = pg_mblen(ptr); + wchar_t character[WC_BUF_LEN]; + Oid collation = DEFAULT_COLLATION_OID; /* TODO */ + pg_locale_t mylocale = 0; /* TODO */ + + if (clen == 1 || lc_ctype_is_c(collation)) + return isspace(TOUCHAR(ptr)); + + char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale); + + return iswspace((wint_t) character[0]); +} + +int +t_isalpha(const char *ptr) +{ + int clen = pg_mblen(ptr); + wchar_t character[WC_BUF_LEN]; + Oid collation = DEFAULT_COLLATION_OID; /* TODO */ + pg_locale_t mylocale = 0; /* TODO */ + + if (clen == 1 || lc_ctype_is_c(collation)) + return isalpha(TOUCHAR(ptr)); + + char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale); + + return iswalpha((wint_t) character[0]); +} + +int +t_isprint(const char *ptr) +{ + int clen = pg_mblen(ptr); + wchar_t character[WC_BUF_LEN]; + Oid collation = DEFAULT_COLLATION_OID; /* TODO */ + pg_locale_t mylocale = 0; /* TODO */ + + if (clen == 1 || lc_ctype_is_c(collation)) + return isprint(TOUCHAR(ptr)); + + char2wchar(character, WC_BUF_LEN, ptr, clen, mylocale); + + return iswprint((wint_t) character[0]); +} + + +/* + * Set up to read a file using tsearch_readline(). This facility is + * better than just reading the file directly because it provides error + * context pointing to the specific line where a problem is detected. + * + * Expected usage is: + * + * tsearch_readline_state trst; + * + * if (!tsearch_readline_begin(&trst, filename)) + * ereport(ERROR, + * (errcode(ERRCODE_CONFIG_FILE_ERROR), + * errmsg("could not open stop-word file \"%s\": %m", + * filename))); + * while ((line = tsearch_readline(&trst)) != NULL) + * process line; + * tsearch_readline_end(&trst); + * + * Note that the caller supplies the ereport() for file open failure; + * this is so that a custom message can be provided. The filename string + * passed to tsearch_readline_begin() must remain valid through + * tsearch_readline_end(). + */ +bool +tsearch_readline_begin(tsearch_readline_state *stp, + const char *filename) +{ + if ((stp->fp = AllocateFile(filename, "r")) == NULL) + return false; + stp->filename = filename; + stp->lineno = 0; + initStringInfo(&stp->buf); + stp->curline = NULL; + /* Setup error traceback support for ereport() */ + stp->cb.callback = tsearch_readline_callback; + stp->cb.arg = (void *) stp; + stp->cb.previous = error_context_stack; + error_context_stack = &stp->cb; + return true; +} + +/* + * Read the next line from a tsearch data file (expected to be in UTF-8), and + * convert it to database encoding if needed. The returned string is palloc'd. + * NULL return means EOF. + */ +char * +tsearch_readline(tsearch_readline_state *stp) +{ + char *recoded; + + /* Advance line number to use in error reports */ + stp->lineno++; + + /* Clear curline, it's no longer relevant */ + if (stp->curline) + { + if (stp->curline != stp->buf.data) + pfree(stp->curline); + stp->curline = NULL; + } + + /* Collect next line, if there is one */ + if (!pg_get_line_buf(stp->fp, &stp->buf)) + return NULL; + + /* Validate the input as UTF-8, then convert to DB encoding if needed */ + recoded = pg_any_to_server(stp->buf.data, stp->buf.len, PG_UTF8); + + /* Save the correctly-encoded string for possible error reports */ + stp->curline = recoded; /* might be equal to buf.data */ + + /* + * We always return a freshly pstrdup'd string. This is clearly necessary + * if pg_any_to_server() returned buf.data, and we need a second copy even + * if encoding conversion did occur. The caller is entitled to pfree the + * returned string at any time, which would leave curline pointing to + * recycled storage, causing problems if an error occurs after that point. + * (It's preferable to return the result of pstrdup instead of the output + * of pg_any_to_server, because the conversion result tends to be + * over-allocated. Since callers might save the result string directly + * into a long-lived dictionary structure, we don't want it to be a larger + * palloc chunk than necessary. We'll reclaim the conversion result on + * the next call.) + */ + return pstrdup(recoded); +} + +/* + * Close down after reading a file with tsearch_readline() + */ +void +tsearch_readline_end(tsearch_readline_state *stp) +{ + /* Suppress use of curline in any error reported below */ + if (stp->curline) + { + if (stp->curline != stp->buf.data) + pfree(stp->curline); + stp->curline = NULL; + } + + /* Release other resources */ + pfree(stp->buf.data); + FreeFile(stp->fp); + + /* Pop the error context stack */ + error_context_stack = stp->cb.previous; +} + +/* + * Error context callback for errors occurring while reading a tsearch + * configuration file. + */ +static void +tsearch_readline_callback(void *arg) +{ + tsearch_readline_state *stp = (tsearch_readline_state *) arg; + + /* + * We can't include the text of the config line for errors that occur + * during tsearch_readline() itself. The major cause of such errors is + * encoding violations, and we daren't try to print error messages + * containing badly-encoded data. + */ + if (stp->curline) + errcontext("line %d of configuration file \"%s\": \"%s\"", + stp->lineno, + stp->filename, + stp->curline); + else + errcontext("line %d of configuration file \"%s\"", + stp->lineno, + stp->filename); +} + + +/* + * lowerstr --- fold null-terminated string to lower case + * + * Returned string is palloc'd + */ +char * +lowerstr(const char *str) +{ + return lowerstr_with_len(str, strlen(str)); +} + +/* + * lowerstr_with_len --- fold string to lower case + * + * Input string need not be null-terminated. + * + * Returned string is palloc'd + */ +char * +lowerstr_with_len(const char *str, int len) +{ + char *out; + Oid collation = DEFAULT_COLLATION_OID; /* TODO */ + pg_locale_t mylocale = 0; /* TODO */ + + if (len == 0) + return pstrdup(""); + + /* + * Use wide char code only when max encoding length > 1 and ctype != C. + * Some operating systems fail with multi-byte encodings and a C locale. + * Also, for a C locale there is no need to process as multibyte. From + * backend/utils/adt/oracle_compat.c Teodor + */ + if (pg_database_encoding_max_length() > 1 && !lc_ctype_is_c(collation)) + { + wchar_t *wstr, + *wptr; + int wlen; + + /* + * alloc number of wchar_t for worst case, len contains number of + * bytes >= number of characters and alloc 1 wchar_t for 0, because + * wchar2char wants zero-terminated string + */ + wptr = wstr = (wchar_t *) palloc(sizeof(wchar_t) * (len + 1)); + + wlen = char2wchar(wstr, len + 1, str, len, mylocale); + Assert(wlen <= len); + + while (*wptr) + { + *wptr = towlower((wint_t) *wptr); + wptr++; + } + + /* + * Alloc result string for worst case + '\0' + */ + len = pg_database_encoding_max_length() * wlen + 1; + out = (char *) palloc(len); + + wlen = wchar2char(out, wstr, len, mylocale); + + pfree(wstr); + + if (wlen < 0) + ereport(ERROR, + (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE), + errmsg("conversion from wchar_t to server encoding failed: %m"))); + Assert(wlen < len); + } + else + { + const char *ptr = str; + char *outptr; + + outptr = out = (char *) palloc(sizeof(char) * (len + 1)); + while ((ptr - str) < len && *ptr) + { + *outptr++ = tolower(TOUCHAR(ptr)); + ptr++; + } + *outptr = '\0'; + } + + return out; +} diff --git a/src/backend/tsearch/ts_parse.c b/src/backend/tsearch/ts_parse.c new file mode 100644 index 0000000..92d95b4 --- /dev/null +++ b/src/backend/tsearch/ts_parse.c @@ -0,0 +1,667 @@ +/*------------------------------------------------------------------------- + * + * ts_parse.c + * main parse functions for tsearch + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/ts_parse.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include "tsearch/ts_cache.h" +#include "tsearch/ts_utils.h" + +#define IGNORE_LONGLEXEME 1 + +/* + * Lexize subsystem + */ + +typedef struct ParsedLex +{ + int type; + char *lemm; + int lenlemm; + struct ParsedLex *next; +} ParsedLex; + +typedef struct ListParsedLex +{ + ParsedLex *head; + ParsedLex *tail; +} ListParsedLex; + +typedef struct +{ + TSConfigCacheEntry *cfg; + Oid curDictId; + int posDict; + DictSubState dictState; + ParsedLex *curSub; + ListParsedLex towork; /* current list to work */ + ListParsedLex waste; /* list of lexemes that already lexized */ + + /* + * fields to store last variant to lexize (basically, thesaurus or similar + * to, which wants several lexemes + */ + + ParsedLex *lastRes; + TSLexeme *tmpRes; +} LexizeData; + +static void +LexizeInit(LexizeData *ld, TSConfigCacheEntry *cfg) +{ + ld->cfg = cfg; + ld->curDictId = InvalidOid; + ld->posDict = 0; + ld->towork.head = ld->towork.tail = ld->curSub = NULL; + ld->waste.head = ld->waste.tail = NULL; + ld->lastRes = NULL; + ld->tmpRes = NULL; +} + +static void +LPLAddTail(ListParsedLex *list, ParsedLex *newpl) +{ + if (list->tail) + { + list->tail->next = newpl; + list->tail = newpl; + } + else + list->head = list->tail = newpl; + newpl->next = NULL; +} + +static ParsedLex * +LPLRemoveHead(ListParsedLex *list) +{ + ParsedLex *res = list->head; + + if (list->head) + list->head = list->head->next; + + if (list->head == NULL) + list->tail = NULL; + + return res; +} + +static void +LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm) +{ + ParsedLex *newpl = (ParsedLex *) palloc(sizeof(ParsedLex)); + + newpl->type = type; + newpl->lemm = lemm; + newpl->lenlemm = lenlemm; + LPLAddTail(&ld->towork, newpl); + ld->curSub = ld->towork.tail; +} + +static void +RemoveHead(LexizeData *ld) +{ + LPLAddTail(&ld->waste, LPLRemoveHead(&ld->towork)); + + ld->posDict = 0; +} + +static void +setCorrLex(LexizeData *ld, ParsedLex **correspondLexem) +{ + if (correspondLexem) + { + *correspondLexem = ld->waste.head; + } + else + { + ParsedLex *tmp, + *ptr = ld->waste.head; + + while (ptr) + { + tmp = ptr->next; + pfree(ptr); + ptr = tmp; + } + } + ld->waste.head = ld->waste.tail = NULL; +} + +static void +moveToWaste(LexizeData *ld, ParsedLex *stop) +{ + bool go = true; + + while (ld->towork.head && go) + { + if (ld->towork.head == stop) + { + ld->curSub = stop->next; + go = false; + } + RemoveHead(ld); + } +} + +static void +setNewTmpRes(LexizeData *ld, ParsedLex *lex, TSLexeme *res) +{ + if (ld->tmpRes) + { + TSLexeme *ptr; + + for (ptr = ld->tmpRes; ptr->lexeme; ptr++) + pfree(ptr->lexeme); + pfree(ld->tmpRes); + } + ld->tmpRes = res; + ld->lastRes = lex; +} + +static TSLexeme * +LexizeExec(LexizeData *ld, ParsedLex **correspondLexem) +{ + int i; + ListDictionary *map; + TSDictionaryCacheEntry *dict; + TSLexeme *res; + + if (ld->curDictId == InvalidOid) + { + /* + * usual mode: dictionary wants only one word, but we should keep in + * mind that we should go through all stack + */ + + while (ld->towork.head) + { + ParsedLex *curVal = ld->towork.head; + char *curValLemm = curVal->lemm; + int curValLenLemm = curVal->lenlemm; + + map = ld->cfg->map + curVal->type; + + if (curVal->type == 0 || curVal->type >= ld->cfg->lenmap || map->len == 0) + { + /* skip this type of lexeme */ + RemoveHead(ld); + continue; + } + + for (i = ld->posDict; i < map->len; i++) + { + dict = lookup_ts_dictionary_cache(map->dictIds[i]); + + ld->dictState.isend = ld->dictState.getnext = false; + ld->dictState.private_state = NULL; + res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize), + PointerGetDatum(dict->dictData), + PointerGetDatum(curValLemm), + Int32GetDatum(curValLenLemm), + PointerGetDatum(&ld->dictState))); + + if (ld->dictState.getnext) + { + /* + * dictionary wants next word, so setup and store current + * position and go to multiword mode + */ + + ld->curDictId = DatumGetObjectId(map->dictIds[i]); + ld->posDict = i + 1; + ld->curSub = curVal->next; + if (res) + setNewTmpRes(ld, curVal, res); + return LexizeExec(ld, correspondLexem); + } + + if (!res) /* dictionary doesn't know this lexeme */ + continue; + + if (res->flags & TSL_FILTER) + { + curValLemm = res->lexeme; + curValLenLemm = strlen(res->lexeme); + continue; + } + + RemoveHead(ld); + setCorrLex(ld, correspondLexem); + return res; + } + + RemoveHead(ld); + } + } + else + { /* curDictId is valid */ + dict = lookup_ts_dictionary_cache(ld->curDictId); + + /* + * Dictionary ld->curDictId asks us about following words + */ + + while (ld->curSub) + { + ParsedLex *curVal = ld->curSub; + + map = ld->cfg->map + curVal->type; + + if (curVal->type != 0) + { + bool dictExists = false; + + if (curVal->type >= ld->cfg->lenmap || map->len == 0) + { + /* skip this type of lexeme */ + ld->curSub = curVal->next; + continue; + } + + /* + * We should be sure that current type of lexeme is recognized + * by our dictionary: we just check is it exist in list of + * dictionaries ? + */ + for (i = 0; i < map->len && !dictExists; i++) + if (ld->curDictId == DatumGetObjectId(map->dictIds[i])) + dictExists = true; + + if (!dictExists) + { + /* + * Dictionary can't work with current type of lexeme, + * return to basic mode and redo all stored lexemes + */ + ld->curDictId = InvalidOid; + return LexizeExec(ld, correspondLexem); + } + } + + ld->dictState.isend = (curVal->type == 0) ? true : false; + ld->dictState.getnext = false; + + res = (TSLexeme *) DatumGetPointer(FunctionCall4(&(dict->lexize), + PointerGetDatum(dict->dictData), + PointerGetDatum(curVal->lemm), + Int32GetDatum(curVal->lenlemm), + PointerGetDatum(&ld->dictState))); + + if (ld->dictState.getnext) + { + /* Dictionary wants one more */ + ld->curSub = curVal->next; + if (res) + setNewTmpRes(ld, curVal, res); + continue; + } + + if (res || ld->tmpRes) + { + /* + * Dictionary normalizes lexemes, so we remove from stack all + * used lexemes, return to basic mode and redo end of stack + * (if it exists) + */ + if (res) + { + moveToWaste(ld, ld->curSub); + } + else + { + res = ld->tmpRes; + moveToWaste(ld, ld->lastRes); + } + + /* reset to initial state */ + ld->curDictId = InvalidOid; + ld->posDict = 0; + ld->lastRes = NULL; + ld->tmpRes = NULL; + setCorrLex(ld, correspondLexem); + return res; + } + + /* + * Dict don't want next lexem and didn't recognize anything, redo + * from ld->towork.head + */ + ld->curDictId = InvalidOid; + return LexizeExec(ld, correspondLexem); + } + } + + setCorrLex(ld, correspondLexem); + return NULL; +} + +/* + * Parse string and lexize words. + * + * prs will be filled in. + */ +void +parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen) +{ + int type, + lenlemm; + char *lemm = NULL; + LexizeData ldata; + TSLexeme *norms; + TSConfigCacheEntry *cfg; + TSParserCacheEntry *prsobj; + void *prsdata; + + cfg = lookup_ts_config_cache(cfgId); + prsobj = lookup_ts_parser_cache(cfg->prsId); + + prsdata = (void *) DatumGetPointer(FunctionCall2(&prsobj->prsstart, + PointerGetDatum(buf), + Int32GetDatum(buflen))); + + LexizeInit(&ldata, cfg); + + do + { + type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken), + PointerGetDatum(prsdata), + PointerGetDatum(&lemm), + PointerGetDatum(&lenlemm))); + + if (type > 0 && lenlemm >= MAXSTRLEN) + { +#ifdef IGNORE_LONGLEXEME + ereport(NOTICE, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); + continue; +#else + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); +#endif + } + + LexizeAddLemm(&ldata, type, lemm, lenlemm); + + while ((norms = LexizeExec(&ldata, NULL)) != NULL) + { + TSLexeme *ptr = norms; + + prs->pos++; /* set pos */ + + while (ptr->lexeme) + { + if (prs->curwords == prs->lenwords) + { + prs->lenwords *= 2; + prs->words = (ParsedWord *) repalloc((void *) prs->words, prs->lenwords * sizeof(ParsedWord)); + } + + if (ptr->flags & TSL_ADDPOS) + prs->pos++; + prs->words[prs->curwords].len = strlen(ptr->lexeme); + prs->words[prs->curwords].word = ptr->lexeme; + prs->words[prs->curwords].nvariant = ptr->nvariant; + prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX; + prs->words[prs->curwords].alen = 0; + prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos); + ptr++; + prs->curwords++; + } + pfree(norms); + } + } while (type > 0); + + FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata)); +} + +/* + * Headline framework + */ +static void +hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type) +{ + while (prs->curwords >= prs->lenwords) + { + prs->lenwords *= 2; + prs->words = (HeadlineWordEntry *) repalloc((void *) prs->words, prs->lenwords * sizeof(HeadlineWordEntry)); + } + memset(&(prs->words[prs->curwords]), 0, sizeof(HeadlineWordEntry)); + prs->words[prs->curwords].type = (uint8) type; + prs->words[prs->curwords].len = buflen; + prs->words[prs->curwords].word = palloc(buflen); + memcpy(prs->words[prs->curwords].word, buf, buflen); + prs->curwords++; +} + +static void +hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen) +{ + int i; + QueryItem *item = GETQUERY(query); + HeadlineWordEntry *word; + + while (prs->curwords + query->size >= prs->lenwords) + { + prs->lenwords *= 2; + prs->words = (HeadlineWordEntry *) repalloc((void *) prs->words, prs->lenwords * sizeof(HeadlineWordEntry)); + } + + word = &(prs->words[prs->curwords - 1]); + word->pos = LIMITPOS(pos); + for (i = 0; i < query->size; i++) + { + if (item->type == QI_VAL && + tsCompareString(GETOPERAND(query) + item->qoperand.distance, item->qoperand.length, + buf, buflen, item->qoperand.prefix) == 0) + { + if (word->item) + { + memcpy(&(prs->words[prs->curwords]), word, sizeof(HeadlineWordEntry)); + prs->words[prs->curwords].item = &item->qoperand; + prs->words[prs->curwords].repeated = 1; + prs->curwords++; + } + else + word->item = &item->qoperand; + } + item++; + } +} + +static void +addHLParsedLex(HeadlineParsedText *prs, TSQuery query, ParsedLex *lexs, TSLexeme *norms) +{ + ParsedLex *tmplexs; + TSLexeme *ptr; + int32 savedpos; + + while (lexs) + { + if (lexs->type > 0) + hladdword(prs, lexs->lemm, lexs->lenlemm, lexs->type); + + ptr = norms; + savedpos = prs->vectorpos; + while (ptr && ptr->lexeme) + { + if (ptr->flags & TSL_ADDPOS) + savedpos++; + hlfinditem(prs, query, savedpos, ptr->lexeme, strlen(ptr->lexeme)); + ptr++; + } + + tmplexs = lexs->next; + pfree(lexs); + lexs = tmplexs; + } + + if (norms) + { + ptr = norms; + while (ptr->lexeme) + { + if (ptr->flags & TSL_ADDPOS) + prs->vectorpos++; + pfree(ptr->lexeme); + ptr++; + } + pfree(norms); + } +} + +void +hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen) +{ + int type, + lenlemm; + char *lemm = NULL; + LexizeData ldata; + TSLexeme *norms; + ParsedLex *lexs; + TSConfigCacheEntry *cfg; + TSParserCacheEntry *prsobj; + void *prsdata; + + cfg = lookup_ts_config_cache(cfgId); + prsobj = lookup_ts_parser_cache(cfg->prsId); + + prsdata = (void *) DatumGetPointer(FunctionCall2(&(prsobj->prsstart), + PointerGetDatum(buf), + Int32GetDatum(buflen))); + + LexizeInit(&ldata, cfg); + + do + { + type = DatumGetInt32(FunctionCall3(&(prsobj->prstoken), + PointerGetDatum(prsdata), + PointerGetDatum(&lemm), + PointerGetDatum(&lenlemm))); + + if (type > 0 && lenlemm >= MAXSTRLEN) + { +#ifdef IGNORE_LONGLEXEME + ereport(NOTICE, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); + continue; +#else + ereport(ERROR, + (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), + errmsg("word is too long to be indexed"), + errdetail("Words longer than %d characters are ignored.", + MAXSTRLEN))); +#endif + } + + LexizeAddLemm(&ldata, type, lemm, lenlemm); + + do + { + if ((norms = LexizeExec(&ldata, &lexs)) != NULL) + { + prs->vectorpos++; + addHLParsedLex(prs, query, lexs, norms); + } + else + addHLParsedLex(prs, query, lexs, NULL); + } while (norms); + + } while (type > 0); + + FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata)); +} + +text * +generateHeadline(HeadlineParsedText *prs) +{ + text *out; + char *ptr; + int len = 128; + int numfragments = 0; + int16 infrag = 0; + + HeadlineWordEntry *wrd = prs->words; + + out = (text *) palloc(len); + ptr = ((char *) out) + VARHDRSZ; + + while (wrd - prs->words < prs->curwords) + { + while (wrd->len + prs->stopsellen + prs->startsellen + prs->fragdelimlen + (ptr - ((char *) out)) >= len) + { + int dist = ptr - ((char *) out); + + len *= 2; + out = (text *) repalloc(out, len); + ptr = ((char *) out) + dist; + } + + if (wrd->in && !wrd->repeated) + { + if (!infrag) + { + + /* start of a new fragment */ + infrag = 1; + numfragments++; + /* add a fragment delimiter if this is after the first one */ + if (numfragments > 1) + { + memcpy(ptr, prs->fragdelim, prs->fragdelimlen); + ptr += prs->fragdelimlen; + } + + } + if (wrd->replace) + { + *ptr = ' '; + ptr++; + } + else if (!wrd->skip) + { + if (wrd->selected) + { + memcpy(ptr, prs->startsel, prs->startsellen); + ptr += prs->startsellen; + } + memcpy(ptr, wrd->word, wrd->len); + ptr += wrd->len; + if (wrd->selected) + { + memcpy(ptr, prs->stopsel, prs->stopsellen); + ptr += prs->stopsellen; + } + } + } + else if (!wrd->repeated) + { + if (infrag) + infrag = 0; + pfree(wrd->word); + } + + wrd++; + } + + SET_VARSIZE(out, ptr - ((char *) out)); + return out; +} diff --git a/src/backend/tsearch/ts_selfuncs.c b/src/backend/tsearch/ts_selfuncs.c new file mode 100644 index 0000000..be2546a --- /dev/null +++ b/src/backend/tsearch/ts_selfuncs.c @@ -0,0 +1,453 @@ +/*------------------------------------------------------------------------- + * + * ts_selfuncs.c + * Selectivity estimation functions for text search operators. + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/ts_selfuncs.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "access/htup_details.h" +#include "catalog/pg_statistic.h" +#include "catalog/pg_type.h" +#include "miscadmin.h" +#include "nodes/nodes.h" +#include "tsearch/ts_type.h" +#include "utils/builtins.h" +#include "utils/lsyscache.h" +#include "utils/selfuncs.h" +#include "utils/syscache.h" + + +/* + * The default text search selectivity is chosen to be small enough to + * encourage indexscans for typical table densities. See selfuncs.h and + * DEFAULT_EQ_SEL for details. + */ +#define DEFAULT_TS_MATCH_SEL 0.005 + +/* lookup table type for binary searching through MCELEMs */ +typedef struct +{ + text *element; + float4 frequency; +} TextFreq; + +/* type of keys for bsearch'ing through an array of TextFreqs */ +typedef struct +{ + char *lexeme; + int length; +} LexemeKey; + +static Selectivity tsquerysel(VariableStatData *vardata, Datum constval); +static Selectivity mcelem_tsquery_selec(TSQuery query, + Datum *mcelem, int nmcelem, + float4 *numbers, int nnumbers); +static Selectivity tsquery_opr_selec(QueryItem *item, char *operand, + TextFreq *lookup, int length, float4 minfreq); +static int compare_lexeme_textfreq(const void *e1, const void *e2); + +#define tsquery_opr_selec_no_stats(query) \ + tsquery_opr_selec(GETQUERY(query), GETOPERAND(query), NULL, 0, 0) + + +/* + * tsmatchsel -- Selectivity of "@@" + * + * restriction selectivity function for tsvector @@ tsquery and + * tsquery @@ tsvector + */ +Datum +tsmatchsel(PG_FUNCTION_ARGS) +{ + PlannerInfo *root = (PlannerInfo *) PG_GETARG_POINTER(0); + +#ifdef NOT_USED + Oid operator = PG_GETARG_OID(1); +#endif + List *args = (List *) PG_GETARG_POINTER(2); + int varRelid = PG_GETARG_INT32(3); + VariableStatData vardata; + Node *other; + bool varonleft; + Selectivity selec; + + /* + * If expression is not variable = something or something = variable, then + * punt and return a default estimate. + */ + if (!get_restriction_variable(root, args, varRelid, + &vardata, &other, &varonleft)) + PG_RETURN_FLOAT8(DEFAULT_TS_MATCH_SEL); + + /* + * Can't do anything useful if the something is not a constant, either. + */ + if (!IsA(other, Const)) + { + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(DEFAULT_TS_MATCH_SEL); + } + + /* + * The "@@" operator is strict, so we can cope with NULL right away + */ + if (((Const *) other)->constisnull) + { + ReleaseVariableStats(vardata); + PG_RETURN_FLOAT8(0.0); + } + + /* + * OK, there's a Var and a Const we're dealing with here. We need the + * Const to be a TSQuery, else we can't do anything useful. We have to + * check this because the Var might be the TSQuery not the TSVector. + */ + if (((Const *) other)->consttype == TSQUERYOID) + { + /* tsvector @@ tsquery or the other way around */ + Assert(vardata.vartype == TSVECTOROID); + + selec = tsquerysel(&vardata, ((Const *) other)->constvalue); + } + else + { + /* If we can't see the query structure, must punt */ + selec = DEFAULT_TS_MATCH_SEL; + } + + ReleaseVariableStats(vardata); + + CLAMP_PROBABILITY(selec); + + PG_RETURN_FLOAT8((float8) selec); +} + + +/* + * tsmatchjoinsel -- join selectivity of "@@" + * + * join selectivity function for tsvector @@ tsquery and tsquery @@ tsvector + */ +Datum +tsmatchjoinsel(PG_FUNCTION_ARGS) +{ + /* for the moment we just punt */ + PG_RETURN_FLOAT8(DEFAULT_TS_MATCH_SEL); +} + + +/* + * @@ selectivity for tsvector var vs tsquery constant + */ +static Selectivity +tsquerysel(VariableStatData *vardata, Datum constval) +{ + Selectivity selec; + TSQuery query; + + /* The caller made sure the const is a TSQuery, so get it now */ + query = DatumGetTSQuery(constval); + + /* Empty query matches nothing */ + if (query->size == 0) + return (Selectivity) 0.0; + + if (HeapTupleIsValid(vardata->statsTuple)) + { + Form_pg_statistic stats; + AttStatsSlot sslot; + + stats = (Form_pg_statistic) GETSTRUCT(vardata->statsTuple); + + /* MCELEM will be an array of TEXT elements for a tsvector column */ + if (get_attstatsslot(&sslot, vardata->statsTuple, + STATISTIC_KIND_MCELEM, InvalidOid, + ATTSTATSSLOT_VALUES | ATTSTATSSLOT_NUMBERS)) + { + /* + * There is a most-common-elements slot for the tsvector Var, so + * use that. + */ + selec = mcelem_tsquery_selec(query, sslot.values, sslot.nvalues, + sslot.numbers, sslot.nnumbers); + free_attstatsslot(&sslot); + } + else + { + /* No most-common-elements info, so do without */ + selec = tsquery_opr_selec_no_stats(query); + } + + /* + * MCE stats count only non-null rows, so adjust for null rows. + */ + selec *= (1.0 - stats->stanullfrac); + } + else + { + /* No stats at all, so do without */ + selec = tsquery_opr_selec_no_stats(query); + /* we assume no nulls here, so no stanullfrac correction */ + } + + return selec; +} + +/* + * Extract data from the pg_statistic arrays into useful format. + */ +static Selectivity +mcelem_tsquery_selec(TSQuery query, Datum *mcelem, int nmcelem, + float4 *numbers, int nnumbers) +{ + float4 minfreq; + TextFreq *lookup; + Selectivity selec; + int i; + + /* + * There should be two more Numbers than Values, because the last two + * cells are taken for minimal and maximal frequency. Punt if not. + * + * (Note: the MCELEM statistics slot definition allows for a third extra + * number containing the frequency of nulls, but we're not expecting that + * to appear for a tsvector column.) + */ + if (nnumbers != nmcelem + 2) + return tsquery_opr_selec_no_stats(query); + + /* + * Transpose the data into a single array so we can use bsearch(). + */ + lookup = (TextFreq *) palloc(sizeof(TextFreq) * nmcelem); + for (i = 0; i < nmcelem; i++) + { + /* + * The text Datums came from an array, so it cannot be compressed or + * stored out-of-line -- it's safe to use VARSIZE_ANY*. + */ + Assert(!VARATT_IS_COMPRESSED(mcelem[i]) && !VARATT_IS_EXTERNAL(mcelem[i])); + lookup[i].element = (text *) DatumGetPointer(mcelem[i]); + lookup[i].frequency = numbers[i]; + } + + /* + * Grab the lowest frequency. compute_tsvector_stats() stored it for us in + * the one before the last cell of the Numbers array. See ts_typanalyze.c + */ + minfreq = numbers[nnumbers - 2]; + + selec = tsquery_opr_selec(GETQUERY(query), GETOPERAND(query), lookup, + nmcelem, minfreq); + + pfree(lookup); + + return selec; +} + +/* + * Traverse the tsquery in preorder, calculating selectivity as: + * + * selec(left_oper) * selec(right_oper) in AND & PHRASE nodes, + * + * selec(left_oper) + selec(right_oper) - + * selec(left_oper) * selec(right_oper) in OR nodes, + * + * 1 - select(oper) in NOT nodes + * + * histogram-based estimation in prefix VAL nodes + * + * freq[val] in exact VAL nodes, if the value is in MCELEM + * min(freq[MCELEM]) / 2 in VAL nodes, if it is not + * + * The MCELEM array is already sorted (see ts_typanalyze.c), so we can use + * binary search for determining freq[MCELEM]. + * + * If we don't have stats for the tsvector, we still use this logic, + * except we use default estimates for VAL nodes. This case is signaled + * by lookup == NULL. + */ +static Selectivity +tsquery_opr_selec(QueryItem *item, char *operand, + TextFreq *lookup, int length, float4 minfreq) +{ + Selectivity selec; + + /* since this function recurses, it could be driven to stack overflow */ + check_stack_depth(); + + if (item->type == QI_VAL) + { + QueryOperand *oper = (QueryOperand *) item; + LexemeKey key; + + /* + * Prepare the key for bsearch(). + */ + key.lexeme = operand + oper->distance; + key.length = oper->length; + + if (oper->prefix) + { + /* Prefix match, ie the query item is lexeme:* */ + Selectivity matched, + allmces; + int i, + n_matched; + + /* + * Our strategy is to scan through the MCELEM list and combine the + * frequencies of the ones that match the prefix. We then + * extrapolate the fraction of matching MCELEMs to the remaining + * rows, assuming that the MCELEMs are representative of the whole + * lexeme population in this respect. (Compare + * histogram_selectivity().) Note that these are most common + * elements not most common values, so they're not mutually + * exclusive. We treat occurrences as independent events. + * + * This is only a good plan if we have a pretty fair number of + * MCELEMs available; we set the threshold at 100. If no stats or + * insufficient stats, arbitrarily use DEFAULT_TS_MATCH_SEL*4. + */ + if (lookup == NULL || length < 100) + return (Selectivity) (DEFAULT_TS_MATCH_SEL * 4); + + matched = allmces = 0; + n_matched = 0; + for (i = 0; i < length; i++) + { + TextFreq *t = lookup + i; + int tlen = VARSIZE_ANY_EXHDR(t->element); + + if (tlen >= key.length && + strncmp(key.lexeme, VARDATA_ANY(t->element), + key.length) == 0) + { + matched += t->frequency - matched * t->frequency; + n_matched++; + } + allmces += t->frequency - allmces * t->frequency; + } + + /* Clamp to ensure sanity in the face of roundoff error */ + CLAMP_PROBABILITY(matched); + CLAMP_PROBABILITY(allmces); + + selec = matched + (1.0 - allmces) * ((double) n_matched / length); + + /* + * In any case, never believe that a prefix match has selectivity + * less than we would assign for a non-MCELEM lexeme. This + * preserves the property that "word:*" should be estimated to + * match at least as many rows as "word" would be. + */ + selec = Max(Min(DEFAULT_TS_MATCH_SEL, minfreq / 2), selec); + } + else + { + /* Regular exact lexeme match */ + TextFreq *searchres; + + /* If no stats for the variable, use DEFAULT_TS_MATCH_SEL */ + if (lookup == NULL) + return (Selectivity) DEFAULT_TS_MATCH_SEL; + + searchres = (TextFreq *) bsearch(&key, lookup, length, + sizeof(TextFreq), + compare_lexeme_textfreq); + + if (searchres) + { + /* + * The element is in MCELEM. Return precise selectivity (or + * at least as precise as ANALYZE could find out). + */ + selec = searchres->frequency; + } + else + { + /* + * The element is not in MCELEM. Punt, but assume that the + * selectivity cannot be more than minfreq / 2. + */ + selec = Min(DEFAULT_TS_MATCH_SEL, minfreq / 2); + } + } + } + else + { + /* Current TSQuery node is an operator */ + Selectivity s1, + s2; + + switch (item->qoperator.oper) + { + case OP_NOT: + selec = 1.0 - tsquery_opr_selec(item + 1, operand, + lookup, length, minfreq); + break; + + case OP_PHRASE: + case OP_AND: + s1 = tsquery_opr_selec(item + 1, operand, + lookup, length, minfreq); + s2 = tsquery_opr_selec(item + item->qoperator.left, operand, + lookup, length, minfreq); + selec = s1 * s2; + break; + + case OP_OR: + s1 = tsquery_opr_selec(item + 1, operand, + lookup, length, minfreq); + s2 = tsquery_opr_selec(item + item->qoperator.left, operand, + lookup, length, minfreq); + selec = s1 + s2 - s1 * s2; + break; + + default: + elog(ERROR, "unrecognized operator: %d", item->qoperator.oper); + selec = 0; /* keep compiler quiet */ + break; + } + } + + /* Clamp intermediate results to stay sane despite roundoff error */ + CLAMP_PROBABILITY(selec); + + return selec; +} + +/* + * bsearch() comparator for a lexeme (non-NULL terminated string with length) + * and a TextFreq. Use length, then byte-for-byte comparison, because that's + * how ANALYZE code sorted data before storing it in a statistic tuple. + * See ts_typanalyze.c for details. + */ +static int +compare_lexeme_textfreq(const void *e1, const void *e2) +{ + const LexemeKey *key = (const LexemeKey *) e1; + const TextFreq *t = (const TextFreq *) e2; + int len1, + len2; + + len1 = key->length; + len2 = VARSIZE_ANY_EXHDR(t->element); + + /* Compare lengths first, possibly avoiding a strncmp call */ + if (len1 > len2) + return 1; + else if (len1 < len2) + return -1; + + /* Fall back on byte-for-byte comparison */ + return strncmp(key->lexeme, VARDATA_ANY(t->element), len1); +} diff --git a/src/backend/tsearch/ts_typanalyze.c b/src/backend/tsearch/ts_typanalyze.c new file mode 100644 index 0000000..56eeb6f --- /dev/null +++ b/src/backend/tsearch/ts_typanalyze.c @@ -0,0 +1,536 @@ +/*------------------------------------------------------------------------- + * + * ts_typanalyze.c + * functions for gathering statistics from tsvector columns + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/ts_typanalyze.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/pg_collation.h" +#include "catalog/pg_operator.h" +#include "commands/vacuum.h" +#include "common/hashfn.h" +#include "tsearch/ts_type.h" +#include "utils/builtins.h" + + +/* A hash key for lexemes */ +typedef struct +{ + char *lexeme; /* lexeme (not NULL terminated!) */ + int length; /* its length in bytes */ +} LexemeHashKey; + +/* A hash table entry for the Lossy Counting algorithm */ +typedef struct +{ + LexemeHashKey key; /* This is 'e' from the LC algorithm. */ + int frequency; /* This is 'f'. */ + int delta; /* And this is 'delta'. */ +} TrackItem; + +static void compute_tsvector_stats(VacAttrStats *stats, + AnalyzeAttrFetchFunc fetchfunc, + int samplerows, + double totalrows); +static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current); +static uint32 lexeme_hash(const void *key, Size keysize); +static int lexeme_match(const void *key1, const void *key2, Size keysize); +static int lexeme_compare(const void *key1, const void *key2); +static int trackitem_compare_frequencies_desc(const void *e1, const void *e2, + void *arg); +static int trackitem_compare_lexemes(const void *e1, const void *e2, + void *arg); + + +/* + * ts_typanalyze -- a custom typanalyze function for tsvector columns + */ +Datum +ts_typanalyze(PG_FUNCTION_ARGS) +{ + VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0); + Form_pg_attribute attr = stats->attr; + + /* If the attstattarget column is negative, use the default value */ + /* NB: it is okay to scribble on stats->attr since it's a copy */ + if (attr->attstattarget < 0) + attr->attstattarget = default_statistics_target; + + stats->compute_stats = compute_tsvector_stats; + /* see comment about the choice of minrows in commands/analyze.c */ + stats->minrows = 300 * attr->attstattarget; + + PG_RETURN_BOOL(true); +} + +/* + * compute_tsvector_stats() -- compute statistics for a tsvector column + * + * This functions computes statistics that are useful for determining @@ + * operations' selectivity, along with the fraction of non-null rows and + * average width. + * + * Instead of finding the most common values, as we do for most datatypes, + * we're looking for the most common lexemes. This is more useful, because + * there most probably won't be any two rows with the same tsvector and thus + * the notion of a MCV is a bit bogus with this datatype. With a list of the + * most common lexemes we can do a better job at figuring out @@ selectivity. + * + * For the same reasons we assume that tsvector columns are unique when + * determining the number of distinct values. + * + * The algorithm used is Lossy Counting, as proposed in the paper "Approximate + * frequency counts over data streams" by G. S. Manku and R. Motwani, in + * Proceedings of the 28th International Conference on Very Large Data Bases, + * Hong Kong, China, August 2002, section 4.2. The paper is available at + * http://www.vldb.org/conf/2002/S10P03.pdf + * + * The Lossy Counting (aka LC) algorithm goes like this: + * Let s be the threshold frequency for an item (the minimum frequency we + * are interested in) and epsilon the error margin for the frequency. Let D + * be a set of triples (e, f, delta), where e is an element value, f is that + * element's frequency (actually, its current occurrence count) and delta is + * the maximum error in f. We start with D empty and process the elements in + * batches of size w. (The batch size is also known as "bucket size" and is + * equal to 1/epsilon.) Let the current batch number be b_current, starting + * with 1. For each element e we either increment its f count, if it's + * already in D, or insert a new triple into D with values (e, 1, b_current + * - 1). After processing each batch we prune D, by removing from it all + * elements with f + delta <= b_current. After the algorithm finishes we + * suppress all elements from D that do not satisfy f >= (s - epsilon) * N, + * where N is the total number of elements in the input. We emit the + * remaining elements with estimated frequency f/N. The LC paper proves + * that this algorithm finds all elements with true frequency at least s, + * and that no frequency is overestimated or is underestimated by more than + * epsilon. Furthermore, given reasonable assumptions about the input + * distribution, the required table size is no more than about 7 times w. + * + * We set s to be the estimated frequency of the K'th word in a natural + * language's frequency table, where K is the target number of entries in + * the MCELEM array plus an arbitrary constant, meant to reflect the fact + * that the most common words in any language would usually be stopwords + * so we will not actually see them in the input. We assume that the + * distribution of word frequencies (including the stopwords) follows Zipf's + * law with an exponent of 1. + * + * Assuming Zipfian distribution, the frequency of the K'th word is equal + * to 1/(K * H(W)) where H(n) is 1/2 + 1/3 + ... + 1/n and W is the number of + * words in the language. Putting W as one million, we get roughly 0.07/K. + * Assuming top 10 words are stopwords gives s = 0.07/(K + 10). We set + * epsilon = s/10, which gives bucket width w = (K + 10)/0.007 and + * maximum expected hashtable size of about 1000 * (K + 10). + * + * Note: in the above discussion, s, epsilon, and f/N are in terms of a + * lexeme's frequency as a fraction of all lexemes seen in the input. + * However, what we actually want to store in the finished pg_statistic + * entry is each lexeme's frequency as a fraction of all rows that it occurs + * in. Assuming that the input tsvectors are correctly constructed, no + * lexeme occurs more than once per tsvector, so the final count f is a + * correct estimate of the number of input tsvectors it occurs in, and we + * need only change the divisor from N to nonnull_cnt to get the number we + * want. + */ +static void +compute_tsvector_stats(VacAttrStats *stats, + AnalyzeAttrFetchFunc fetchfunc, + int samplerows, + double totalrows) +{ + int num_mcelem; + int null_cnt = 0; + double total_width = 0; + + /* This is D from the LC algorithm. */ + HTAB *lexemes_tab; + HASHCTL hash_ctl; + HASH_SEQ_STATUS scan_status; + + /* This is the current bucket number from the LC algorithm */ + int b_current; + + /* This is 'w' from the LC algorithm */ + int bucket_width; + int vector_no, + lexeme_no; + LexemeHashKey hash_key; + TrackItem *item; + + /* + * We want statistics_target * 10 lexemes in the MCELEM array. This + * multiplier is pretty arbitrary, but is meant to reflect the fact that + * the number of individual lexeme values tracked in pg_statistic ought to + * be more than the number of values for a simple scalar column. + */ + num_mcelem = stats->attr->attstattarget * 10; + + /* + * We set bucket width equal to (num_mcelem + 10) / 0.007 as per the + * comment above. + */ + bucket_width = (num_mcelem + 10) * 1000 / 7; + + /* + * Create the hashtable. It will be in local memory, so we don't need to + * worry about overflowing the initial size. Also we don't need to pay any + * attention to locking and memory management. + */ + hash_ctl.keysize = sizeof(LexemeHashKey); + hash_ctl.entrysize = sizeof(TrackItem); + hash_ctl.hash = lexeme_hash; + hash_ctl.match = lexeme_match; + hash_ctl.hcxt = CurrentMemoryContext; + lexemes_tab = hash_create("Analyzed lexemes table", + num_mcelem, + &hash_ctl, + HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT); + + /* Initialize counters. */ + b_current = 1; + lexeme_no = 0; + + /* Loop over the tsvectors. */ + for (vector_no = 0; vector_no < samplerows; vector_no++) + { + Datum value; + bool isnull; + TSVector vector; + WordEntry *curentryptr; + char *lexemesptr; + int j; + + vacuum_delay_point(); + + value = fetchfunc(stats, vector_no, &isnull); + + /* + * Check for null/nonnull. + */ + if (isnull) + { + null_cnt++; + continue; + } + + /* + * Add up widths for average-width calculation. Since it's a + * tsvector, we know it's varlena. As in the regular + * compute_minimal_stats function, we use the toasted width for this + * calculation. + */ + total_width += VARSIZE_ANY(DatumGetPointer(value)); + + /* + * Now detoast the tsvector if needed. + */ + vector = DatumGetTSVector(value); + + /* + * We loop through the lexemes in the tsvector and add them to our + * tracking hashtable. + */ + lexemesptr = STRPTR(vector); + curentryptr = ARRPTR(vector); + for (j = 0; j < vector->size; j++) + { + bool found; + + /* + * Construct a hash key. The key points into the (detoasted) + * tsvector value at this point, but if a new entry is created, we + * make a copy of it. This way we can free the tsvector value + * once we've processed all its lexemes. + */ + hash_key.lexeme = lexemesptr + curentryptr->pos; + hash_key.length = curentryptr->len; + + /* Lookup current lexeme in hashtable, adding it if new */ + item = (TrackItem *) hash_search(lexemes_tab, + (const void *) &hash_key, + HASH_ENTER, &found); + + if (found) + { + /* The lexeme is already on the tracking list */ + item->frequency++; + } + else + { + /* Initialize new tracking list element */ + item->frequency = 1; + item->delta = b_current - 1; + + item->key.lexeme = palloc(hash_key.length); + memcpy(item->key.lexeme, hash_key.lexeme, hash_key.length); + } + + /* lexeme_no is the number of elements processed (ie N) */ + lexeme_no++; + + /* We prune the D structure after processing each bucket */ + if (lexeme_no % bucket_width == 0) + { + prune_lexemes_hashtable(lexemes_tab, b_current); + b_current++; + } + + /* Advance to the next WordEntry in the tsvector */ + curentryptr++; + } + + /* If the vector was toasted, free the detoasted copy. */ + if (TSVectorGetDatum(vector) != value) + pfree(vector); + } + + /* We can only compute real stats if we found some non-null values. */ + if (null_cnt < samplerows) + { + int nonnull_cnt = samplerows - null_cnt; + int i; + TrackItem **sort_table; + int track_len; + int cutoff_freq; + int minfreq, + maxfreq; + + stats->stats_valid = true; + /* Do the simple null-frac and average width stats */ + stats->stanullfrac = (double) null_cnt / (double) samplerows; + stats->stawidth = total_width / (double) nonnull_cnt; + + /* Assume it's a unique column (see notes above) */ + stats->stadistinct = -1.0 * (1.0 - stats->stanullfrac); + + /* + * Construct an array of the interesting hashtable items, that is, + * those meeting the cutoff frequency (s - epsilon)*N. Also identify + * the minimum and maximum frequencies among these items. + * + * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff + * frequency is 9*N / bucket_width. + */ + cutoff_freq = 9 * lexeme_no / bucket_width; + + i = hash_get_num_entries(lexemes_tab); /* surely enough space */ + sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * i); + + hash_seq_init(&scan_status, lexemes_tab); + track_len = 0; + minfreq = lexeme_no; + maxfreq = 0; + while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL) + { + if (item->frequency > cutoff_freq) + { + sort_table[track_len++] = item; + minfreq = Min(minfreq, item->frequency); + maxfreq = Max(maxfreq, item->frequency); + } + } + Assert(track_len <= i); + + /* emit some statistics for debug purposes */ + elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, " + "# lexemes = %d, hashtable size = %d, usable entries = %d", + num_mcelem, bucket_width, lexeme_no, i, track_len); + + /* + * If we obtained more lexemes than we really want, get rid of those + * with least frequencies. The easiest way is to qsort the array into + * descending frequency order and truncate the array. + */ + if (num_mcelem < track_len) + { + qsort_interruptible(sort_table, track_len, sizeof(TrackItem *), + trackitem_compare_frequencies_desc, NULL); + /* reset minfreq to the smallest frequency we're keeping */ + minfreq = sort_table[num_mcelem - 1]->frequency; + } + else + num_mcelem = track_len; + + /* Generate MCELEM slot entry */ + if (num_mcelem > 0) + { + MemoryContext old_context; + Datum *mcelem_values; + float4 *mcelem_freqs; + + /* + * We want to store statistics sorted on the lexeme value using + * first length, then byte-for-byte comparison. The reason for + * doing length comparison first is that we don't care about the + * ordering so long as it's consistent, and comparing lengths + * first gives us a chance to avoid a strncmp() call. + * + * This is different from what we do with scalar statistics -- + * they get sorted on frequencies. The rationale is that we + * usually search through most common elements looking for a + * specific value, so we can grab its frequency. When values are + * presorted we can employ binary search for that. See + * ts_selfuncs.c for a real usage scenario. + */ + qsort_interruptible(sort_table, num_mcelem, sizeof(TrackItem *), + trackitem_compare_lexemes, NULL); + + /* Must copy the target values into anl_context */ + old_context = MemoryContextSwitchTo(stats->anl_context); + + /* + * We sorted statistics on the lexeme value, but we want to be + * able to find out the minimal and maximal frequency without + * going through all the values. We keep those two extra + * frequencies in two extra cells in mcelem_freqs. + * + * (Note: the MCELEM statistics slot definition allows for a third + * extra number containing the frequency of nulls, but we don't + * create that for a tsvector column, since null elements aren't + * possible.) + */ + mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum)); + mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4)); + + /* + * See comments above about use of nonnull_cnt as the divisor for + * the final frequency estimates. + */ + for (i = 0; i < num_mcelem; i++) + { + TrackItem *item = sort_table[i]; + + mcelem_values[i] = + PointerGetDatum(cstring_to_text_with_len(item->key.lexeme, + item->key.length)); + mcelem_freqs[i] = (double) item->frequency / (double) nonnull_cnt; + } + mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt; + mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt; + MemoryContextSwitchTo(old_context); + + stats->stakind[0] = STATISTIC_KIND_MCELEM; + stats->staop[0] = TextEqualOperator; + stats->stacoll[0] = DEFAULT_COLLATION_OID; + stats->stanumbers[0] = mcelem_freqs; + /* See above comment about two extra frequency fields */ + stats->numnumbers[0] = num_mcelem + 2; + stats->stavalues[0] = mcelem_values; + stats->numvalues[0] = num_mcelem; + /* We are storing text values */ + stats->statypid[0] = TEXTOID; + stats->statyplen[0] = -1; /* typlen, -1 for varlena */ + stats->statypbyval[0] = false; + stats->statypalign[0] = 'i'; + } + } + else + { + /* We found only nulls; assume the column is entirely null */ + stats->stats_valid = true; + stats->stanullfrac = 1.0; + stats->stawidth = 0; /* "unknown" */ + stats->stadistinct = 0.0; /* "unknown" */ + } + + /* + * We don't need to bother cleaning up any of our temporary palloc's. The + * hashtable should also go away, as it used a child memory context. + */ +} + +/* + * A function to prune the D structure from the Lossy Counting algorithm. + * Consult compute_tsvector_stats() for wider explanation. + */ +static void +prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current) +{ + HASH_SEQ_STATUS scan_status; + TrackItem *item; + + hash_seq_init(&scan_status, lexemes_tab); + while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL) + { + if (item->frequency + item->delta <= b_current) + { + char *lexeme = item->key.lexeme; + + if (hash_search(lexemes_tab, (const void *) &item->key, + HASH_REMOVE, NULL) == NULL) + elog(ERROR, "hash table corrupted"); + pfree(lexeme); + } + } +} + +/* + * Hash functions for lexemes. They are strings, but not NULL terminated, + * so we need a special hash function. + */ +static uint32 +lexeme_hash(const void *key, Size keysize) +{ + const LexemeHashKey *l = (const LexemeHashKey *) key; + + return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme, + l->length)); +} + +/* + * Matching function for lexemes, to be used in hashtable lookups. + */ +static int +lexeme_match(const void *key1, const void *key2, Size keysize) +{ + /* The keysize parameter is superfluous, the keys store their lengths */ + return lexeme_compare(key1, key2); +} + +/* + * Comparison function for lexemes. + */ +static int +lexeme_compare(const void *key1, const void *key2) +{ + const LexemeHashKey *d1 = (const LexemeHashKey *) key1; + const LexemeHashKey *d2 = (const LexemeHashKey *) key2; + + /* First, compare by length */ + if (d1->length > d2->length) + return 1; + else if (d1->length < d2->length) + return -1; + /* Lengths are equal, do a byte-by-byte comparison */ + return strncmp(d1->lexeme, d2->lexeme, d1->length); +} + +/* + * Comparator for sorting TrackItems on frequencies (descending sort) + */ +static int +trackitem_compare_frequencies_desc(const void *e1, const void *e2, void *arg) +{ + const TrackItem *const *t1 = (const TrackItem *const *) e1; + const TrackItem *const *t2 = (const TrackItem *const *) e2; + + return (*t2)->frequency - (*t1)->frequency; +} + +/* + * Comparator for sorting TrackItems on lexemes + */ +static int +trackitem_compare_lexemes(const void *e1, const void *e2, void *arg) +{ + const TrackItem *const *t1 = (const TrackItem *const *) e1; + const TrackItem *const *t2 = (const TrackItem *const *) e2; + + return lexeme_compare(&(*t1)->key, &(*t2)->key); +} diff --git a/src/backend/tsearch/ts_utils.c b/src/backend/tsearch/ts_utils.c new file mode 100644 index 0000000..ed16a2e --- /dev/null +++ b/src/backend/tsearch/ts_utils.c @@ -0,0 +1,146 @@ +/*------------------------------------------------------------------------- + * + * ts_utils.c + * various support functions + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/ts_utils.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include <ctype.h> + +#include "miscadmin.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_utils.h" + + +/* + * Given the base name and extension of a tsearch config file, return + * its full path name. The base name is assumed to be user-supplied, + * and is checked to prevent pathname attacks. The extension is assumed + * to be safe. + * + * The result is a palloc'd string. + */ +char * +get_tsearch_config_filename(const char *basename, + const char *extension) +{ + char sharepath[MAXPGPATH]; + char *result; + + /* + * We limit the basename to contain a-z, 0-9, and underscores. This may + * be overly restrictive, but we don't want to allow access to anything + * outside the tsearch_data directory, so for instance '/' *must* be + * rejected, and on some platforms '\' and ':' are risky as well. Allowing + * uppercase might result in incompatible behavior between case-sensitive + * and case-insensitive filesystems, and non-ASCII characters create other + * interesting risks, so on the whole a tight policy seems best. + */ + if (strspn(basename, "abcdefghijklmnopqrstuvwxyz0123456789_") != strlen(basename)) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("invalid text search configuration file name \"%s\"", + basename))); + + get_share_path(my_exec_path, sharepath); + result = palloc(MAXPGPATH); + snprintf(result, MAXPGPATH, "%s/tsearch_data/%s.%s", + sharepath, basename, extension); + + return result; +} + +/* + * Reads a stop-word file. Each word is run through 'wordop' + * function, if given. wordop may either modify the input in-place, + * or palloc a new version. + */ +void +readstoplist(const char *fname, StopList *s, char *(*wordop) (const char *)) +{ + char **stop = NULL; + + s->len = 0; + if (fname && *fname) + { + char *filename = get_tsearch_config_filename(fname, "stop"); + tsearch_readline_state trst; + char *line; + int reallen = 0; + + if (!tsearch_readline_begin(&trst, filename)) + ereport(ERROR, + (errcode(ERRCODE_CONFIG_FILE_ERROR), + errmsg("could not open stop-word file \"%s\": %m", + filename))); + + while ((line = tsearch_readline(&trst)) != NULL) + { + char *pbuf = line; + + /* Trim trailing space */ + while (*pbuf && !t_isspace(pbuf)) + pbuf += pg_mblen(pbuf); + *pbuf = '\0'; + + /* Skip empty lines */ + if (*line == '\0') + { + pfree(line); + continue; + } + + if (s->len >= reallen) + { + if (reallen == 0) + { + reallen = 64; + stop = (char **) palloc(sizeof(char *) * reallen); + } + else + { + reallen *= 2; + stop = (char **) repalloc((void *) stop, + sizeof(char *) * reallen); + } + } + + if (wordop) + { + stop[s->len] = wordop(line); + if (stop[s->len] != line) + pfree(line); + } + else + stop[s->len] = line; + + (s->len)++; + } + + tsearch_readline_end(&trst); + pfree(filename); + } + + s->stop = stop; + + /* Sort to allow binary searching */ + if (s->stop && s->len > 0) + qsort(s->stop, s->len, sizeof(char *), pg_qsort_strcmp); +} + +bool +searchstoplist(StopList *s, char *key) +{ + return (s->stop && s->len > 0 && + bsearch(&key, s->stop, s->len, + sizeof(char *), pg_qsort_strcmp)) ? true : false; +} diff --git a/src/backend/tsearch/wparser.c b/src/backend/tsearch/wparser.c new file mode 100644 index 0000000..71882dc --- /dev/null +++ b/src/backend/tsearch/wparser.c @@ -0,0 +1,549 @@ +/*------------------------------------------------------------------------- + * + * wparser.c + * Standard interface to word parser + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/wparser.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "catalog/namespace.h" +#include "catalog/pg_type.h" +#include "commands/defrem.h" +#include "common/jsonapi.h" +#include "funcapi.h" +#include "tsearch/ts_cache.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" +#include "utils/jsonfuncs.h" +#include "utils/varlena.h" + +/******sql-level interface******/ + +typedef struct +{ + int cur; + LexDescr *list; +} TSTokenTypeStorage; + +/* state for ts_headline_json_* */ +typedef struct HeadlineJsonState +{ + HeadlineParsedText *prs; + TSConfigCacheEntry *cfg; + TSParserCacheEntry *prsobj; + TSQuery query; + List *prsoptions; + bool transformed; +} HeadlineJsonState; + +static text *headline_json_value(void *_state, char *elem_value, int elem_len); + +static void +tt_setup_firstcall(FuncCallContext *funcctx, Oid prsid) +{ + TupleDesc tupdesc; + MemoryContext oldcontext; + TSTokenTypeStorage *st; + TSParserCacheEntry *prs = lookup_ts_parser_cache(prsid); + + if (!OidIsValid(prs->lextypeOid)) + elog(ERROR, "method lextype isn't defined for text search parser %u", + prsid); + + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + st = (TSTokenTypeStorage *) palloc(sizeof(TSTokenTypeStorage)); + st->cur = 0; + /* lextype takes one dummy argument */ + st->list = (LexDescr *) DatumGetPointer(OidFunctionCall1(prs->lextypeOid, + (Datum) 0)); + funcctx->user_fctx = (void *) st; + + tupdesc = CreateTemplateTupleDesc(3); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "tokid", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "alias", + TEXTOID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 3, "description", + TEXTOID, -1, 0); + + funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc); + MemoryContextSwitchTo(oldcontext); +} + +static Datum +tt_process_call(FuncCallContext *funcctx) +{ + TSTokenTypeStorage *st; + + st = (TSTokenTypeStorage *) funcctx->user_fctx; + if (st->list && st->list[st->cur].lexid) + { + Datum result; + char *values[3]; + char txtid[16]; + HeapTuple tuple; + + sprintf(txtid, "%d", st->list[st->cur].lexid); + values[0] = txtid; + values[1] = st->list[st->cur].alias; + values[2] = st->list[st->cur].descr; + + tuple = BuildTupleFromCStrings(funcctx->attinmeta, values); + result = HeapTupleGetDatum(tuple); + + pfree(values[1]); + pfree(values[2]); + st->cur++; + return result; + } + return (Datum) 0; +} + +Datum +ts_token_type_byid(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + Datum result; + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + tt_setup_firstcall(funcctx, PG_GETARG_OID(0)); + } + + funcctx = SRF_PERCALL_SETUP(); + + if ((result = tt_process_call(funcctx)) != (Datum) 0) + SRF_RETURN_NEXT(funcctx, result); + SRF_RETURN_DONE(funcctx); +} + +Datum +ts_token_type_byname(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + Datum result; + + if (SRF_IS_FIRSTCALL()) + { + text *prsname = PG_GETARG_TEXT_PP(0); + Oid prsId; + + funcctx = SRF_FIRSTCALL_INIT(); + prsId = get_ts_parser_oid(textToQualifiedNameList(prsname), false); + tt_setup_firstcall(funcctx, prsId); + } + + funcctx = SRF_PERCALL_SETUP(); + + if ((result = tt_process_call(funcctx)) != (Datum) 0) + SRF_RETURN_NEXT(funcctx, result); + SRF_RETURN_DONE(funcctx); +} + +typedef struct +{ + int type; + char *lexeme; +} LexemeEntry; + +typedef struct +{ + int cur; + int len; + LexemeEntry *list; +} PrsStorage; + + +static void +prs_setup_firstcall(FuncCallContext *funcctx, Oid prsid, text *txt) +{ + TupleDesc tupdesc; + MemoryContext oldcontext; + PrsStorage *st; + TSParserCacheEntry *prs = lookup_ts_parser_cache(prsid); + char *lex = NULL; + int llen = 0, + type = 0; + void *prsdata; + + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + st = (PrsStorage *) palloc(sizeof(PrsStorage)); + st->cur = 0; + st->len = 16; + st->list = (LexemeEntry *) palloc(sizeof(LexemeEntry) * st->len); + + prsdata = (void *) DatumGetPointer(FunctionCall2(&prs->prsstart, + PointerGetDatum(VARDATA_ANY(txt)), + Int32GetDatum(VARSIZE_ANY_EXHDR(txt)))); + + while ((type = DatumGetInt32(FunctionCall3(&prs->prstoken, + PointerGetDatum(prsdata), + PointerGetDatum(&lex), + PointerGetDatum(&llen)))) != 0) + { + if (st->cur >= st->len) + { + st->len = 2 * st->len; + st->list = (LexemeEntry *) repalloc(st->list, sizeof(LexemeEntry) * st->len); + } + st->list[st->cur].lexeme = palloc(llen + 1); + memcpy(st->list[st->cur].lexeme, lex, llen); + st->list[st->cur].lexeme[llen] = '\0'; + st->list[st->cur].type = type; + st->cur++; + } + + FunctionCall1(&prs->prsend, PointerGetDatum(prsdata)); + + st->len = st->cur; + st->cur = 0; + + funcctx->user_fctx = (void *) st; + tupdesc = CreateTemplateTupleDesc(2); + TupleDescInitEntry(tupdesc, (AttrNumber) 1, "tokid", + INT4OID, -1, 0); + TupleDescInitEntry(tupdesc, (AttrNumber) 2, "token", + TEXTOID, -1, 0); + + funcctx->attinmeta = TupleDescGetAttInMetadata(tupdesc); + MemoryContextSwitchTo(oldcontext); +} + +static Datum +prs_process_call(FuncCallContext *funcctx) +{ + PrsStorage *st; + + st = (PrsStorage *) funcctx->user_fctx; + if (st->cur < st->len) + { + Datum result; + char *values[2]; + char tid[16]; + HeapTuple tuple; + + values[0] = tid; + sprintf(tid, "%d", st->list[st->cur].type); + values[1] = st->list[st->cur].lexeme; + tuple = BuildTupleFromCStrings(funcctx->attinmeta, values); + result = HeapTupleGetDatum(tuple); + + pfree(values[1]); + st->cur++; + return result; + } + return (Datum) 0; +} + +Datum +ts_parse_byid(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + Datum result; + + if (SRF_IS_FIRSTCALL()) + { + text *txt = PG_GETARG_TEXT_PP(1); + + funcctx = SRF_FIRSTCALL_INIT(); + prs_setup_firstcall(funcctx, PG_GETARG_OID(0), txt); + PG_FREE_IF_COPY(txt, 1); + } + + funcctx = SRF_PERCALL_SETUP(); + + if ((result = prs_process_call(funcctx)) != (Datum) 0) + SRF_RETURN_NEXT(funcctx, result); + SRF_RETURN_DONE(funcctx); +} + +Datum +ts_parse_byname(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + Datum result; + + if (SRF_IS_FIRSTCALL()) + { + text *prsname = PG_GETARG_TEXT_PP(0); + text *txt = PG_GETARG_TEXT_PP(1); + Oid prsId; + + funcctx = SRF_FIRSTCALL_INIT(); + prsId = get_ts_parser_oid(textToQualifiedNameList(prsname), false); + prs_setup_firstcall(funcctx, prsId, txt); + } + + funcctx = SRF_PERCALL_SETUP(); + + if ((result = prs_process_call(funcctx)) != (Datum) 0) + SRF_RETURN_NEXT(funcctx, result); + SRF_RETURN_DONE(funcctx); +} + +Datum +ts_headline_byid_opt(PG_FUNCTION_ARGS) +{ + Oid tsconfig = PG_GETARG_OID(0); + text *in = PG_GETARG_TEXT_PP(1); + TSQuery query = PG_GETARG_TSQUERY(2); + text *opt = (PG_NARGS() > 3 && PG_GETARG_POINTER(3)) ? PG_GETARG_TEXT_PP(3) : NULL; + HeadlineParsedText prs; + List *prsoptions; + text *out; + TSConfigCacheEntry *cfg; + TSParserCacheEntry *prsobj; + + cfg = lookup_ts_config_cache(tsconfig); + prsobj = lookup_ts_parser_cache(cfg->prsId); + + if (!OidIsValid(prsobj->headlineOid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("text search parser does not support headline creation"))); + + memset(&prs, 0, sizeof(HeadlineParsedText)); + prs.lenwords = 32; + prs.words = (HeadlineWordEntry *) palloc(sizeof(HeadlineWordEntry) * prs.lenwords); + + hlparsetext(cfg->cfgId, &prs, query, + VARDATA_ANY(in), VARSIZE_ANY_EXHDR(in)); + + if (opt) + prsoptions = deserialize_deflist(PointerGetDatum(opt)); + else + prsoptions = NIL; + + FunctionCall3(&(prsobj->prsheadline), + PointerGetDatum(&prs), + PointerGetDatum(prsoptions), + PointerGetDatum(query)); + + out = generateHeadline(&prs); + + PG_FREE_IF_COPY(in, 1); + PG_FREE_IF_COPY(query, 2); + if (opt) + PG_FREE_IF_COPY(opt, 3); + pfree(prs.words); + pfree(prs.startsel); + pfree(prs.stopsel); + + PG_RETURN_POINTER(out); +} + +Datum +ts_headline_byid(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_byid_opt, + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1))); +} + +Datum +ts_headline_opt(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall4(ts_headline_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_jsonb_byid_opt(PG_FUNCTION_ARGS) +{ + Oid tsconfig = PG_GETARG_OID(0); + Jsonb *jb = PG_GETARG_JSONB_P(1); + TSQuery query = PG_GETARG_TSQUERY(2); + text *opt = (PG_NARGS() > 3 && PG_GETARG_POINTER(3)) ? PG_GETARG_TEXT_P(3) : NULL; + Jsonb *out; + JsonTransformStringValuesAction action = (JsonTransformStringValuesAction) headline_json_value; + HeadlineParsedText prs; + HeadlineJsonState *state = palloc0(sizeof(HeadlineJsonState)); + + memset(&prs, 0, sizeof(HeadlineParsedText)); + prs.lenwords = 32; + prs.words = (HeadlineWordEntry *) palloc(sizeof(HeadlineWordEntry) * prs.lenwords); + + state->prs = &prs; + state->cfg = lookup_ts_config_cache(tsconfig); + state->prsobj = lookup_ts_parser_cache(state->cfg->prsId); + state->query = query; + if (opt) + state->prsoptions = deserialize_deflist(PointerGetDatum(opt)); + else + state->prsoptions = NIL; + + if (!OidIsValid(state->prsobj->headlineOid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("text search parser does not support headline creation"))); + + out = transform_jsonb_string_values(jb, state, action); + + PG_FREE_IF_COPY(jb, 1); + PG_FREE_IF_COPY(query, 2); + if (opt) + PG_FREE_IF_COPY(opt, 3); + + pfree(prs.words); + + if (state->transformed) + { + pfree(prs.startsel); + pfree(prs.stopsel); + } + + PG_RETURN_JSONB_P(out); +} + +Datum +ts_headline_jsonb(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_jsonb_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1))); +} + +Datum +ts_headline_jsonb_byid(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_jsonb_byid_opt, + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_jsonb_opt(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall4(ts_headline_jsonb_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_json_byid_opt(PG_FUNCTION_ARGS) +{ + Oid tsconfig = PG_GETARG_OID(0); + text *json = PG_GETARG_TEXT_P(1); + TSQuery query = PG_GETARG_TSQUERY(2); + text *opt = (PG_NARGS() > 3 && PG_GETARG_POINTER(3)) ? PG_GETARG_TEXT_P(3) : NULL; + text *out; + JsonTransformStringValuesAction action = (JsonTransformStringValuesAction) headline_json_value; + + HeadlineParsedText prs; + HeadlineJsonState *state = palloc0(sizeof(HeadlineJsonState)); + + memset(&prs, 0, sizeof(HeadlineParsedText)); + prs.lenwords = 32; + prs.words = (HeadlineWordEntry *) palloc(sizeof(HeadlineWordEntry) * prs.lenwords); + + state->prs = &prs; + state->cfg = lookup_ts_config_cache(tsconfig); + state->prsobj = lookup_ts_parser_cache(state->cfg->prsId); + state->query = query; + if (opt) + state->prsoptions = deserialize_deflist(PointerGetDatum(opt)); + else + state->prsoptions = NIL; + + if (!OidIsValid(state->prsobj->headlineOid)) + ereport(ERROR, + (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), + errmsg("text search parser does not support headline creation"))); + + out = transform_json_string_values(json, state, action); + + PG_FREE_IF_COPY(json, 1); + PG_FREE_IF_COPY(query, 2); + if (opt) + PG_FREE_IF_COPY(opt, 3); + pfree(prs.words); + + if (state->transformed) + { + pfree(prs.startsel); + pfree(prs.stopsel); + } + + PG_RETURN_TEXT_P(out); +} + +Datum +ts_headline_json(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_json_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1))); +} + +Datum +ts_headline_json_byid(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall3(ts_headline_json_byid_opt, + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + +Datum +ts_headline_json_opt(PG_FUNCTION_ARGS) +{ + PG_RETURN_DATUM(DirectFunctionCall4(ts_headline_json_byid_opt, + ObjectIdGetDatum(getTSCurrentConfig(true)), + PG_GETARG_DATUM(0), + PG_GETARG_DATUM(1), + PG_GETARG_DATUM(2))); +} + + +/* + * Return headline in text from, generated from a json(b) element + */ +static text * +headline_json_value(void *_state, char *elem_value, int elem_len) +{ + HeadlineJsonState *state = (HeadlineJsonState *) _state; + + HeadlineParsedText *prs = state->prs; + TSConfigCacheEntry *cfg = state->cfg; + TSParserCacheEntry *prsobj = state->prsobj; + TSQuery query = state->query; + List *prsoptions = state->prsoptions; + + prs->curwords = 0; + hlparsetext(cfg->cfgId, prs, query, elem_value, elem_len); + FunctionCall3(&(prsobj->prsheadline), + PointerGetDatum(prs), + PointerGetDatum(prsoptions), + PointerGetDatum(query)); + + state->transformed = true; + return generateHeadline(prs); +} diff --git a/src/backend/tsearch/wparser_def.c b/src/backend/tsearch/wparser_def.c new file mode 100644 index 0000000..559dff6 --- /dev/null +++ b/src/backend/tsearch/wparser_def.c @@ -0,0 +1,2634 @@ +/*------------------------------------------------------------------------- + * + * wparser_def.c + * Default text search parser + * + * Portions Copyright (c) 1996-2021, PostgreSQL Global Development Group + * + * + * IDENTIFICATION + * src/backend/tsearch/wparser_def.c + * + *------------------------------------------------------------------------- + */ + +#include "postgres.h" + +#include <limits.h> + +#include "catalog/pg_collation.h" +#include "commands/defrem.h" +#include "tsearch/ts_locale.h" +#include "tsearch/ts_public.h" +#include "tsearch/ts_type.h" +#include "tsearch/ts_utils.h" +#include "utils/builtins.h" + + +/* Define me to enable tracing of parser behavior */ +/* #define WPARSER_TRACE */ + + +/* Output token categories */ + +#define ASCIIWORD 1 +#define WORD_T 2 +#define NUMWORD 3 +#define EMAIL 4 +#define URL_T 5 +#define HOST 6 +#define SCIENTIFIC 7 +#define VERSIONNUMBER 8 +#define NUMPARTHWORD 9 +#define PARTHWORD 10 +#define ASCIIPARTHWORD 11 +#define SPACE 12 +#define TAG_T 13 +#define PROTOCOL 14 +#define NUMHWORD 15 +#define ASCIIHWORD 16 +#define HWORD 17 +#define URLPATH 18 +#define FILEPATH 19 +#define DECIMAL_T 20 +#define SIGNEDINT 21 +#define UNSIGNEDINT 22 +#define XMLENTITY 23 + +#define LASTNUM 23 + +static const char *const tok_alias[] = { + "", + "asciiword", + "word", + "numword", + "email", + "url", + "host", + "sfloat", + "version", + "hword_numpart", + "hword_part", + "hword_asciipart", + "blank", + "tag", + "protocol", + "numhword", + "asciihword", + "hword", + "url_path", + "file", + "float", + "int", + "uint", + "entity" +}; + +static const char *const lex_descr[] = { + "", + "Word, all ASCII", + "Word, all letters", + "Word, letters and digits", + "Email address", + "URL", + "Host", + "Scientific notation", + "Version number", + "Hyphenated word part, letters and digits", + "Hyphenated word part, all letters", + "Hyphenated word part, all ASCII", + "Space symbols", + "XML tag", + "Protocol head", + "Hyphenated word, letters and digits", + "Hyphenated word, all ASCII", + "Hyphenated word, all letters", + "URL path", + "File or path name", + "Decimal notation", + "Signed integer", + "Unsigned integer", + "XML entity" +}; + + +/* Parser states */ + +typedef enum +{ + TPS_Base = 0, + TPS_InNumWord, + TPS_InAsciiWord, + TPS_InWord, + TPS_InUnsignedInt, + TPS_InSignedIntFirst, + TPS_InSignedInt, + TPS_InSpace, + TPS_InUDecimalFirst, + TPS_InUDecimal, + TPS_InDecimalFirst, + TPS_InDecimal, + TPS_InVerVersion, + TPS_InSVerVersion, + TPS_InVersionFirst, + TPS_InVersion, + TPS_InMantissaFirst, + TPS_InMantissaSign, + TPS_InMantissa, + TPS_InXMLEntityFirst, + TPS_InXMLEntity, + TPS_InXMLEntityNumFirst, + TPS_InXMLEntityNum, + TPS_InXMLEntityHexNumFirst, + TPS_InXMLEntityHexNum, + TPS_InXMLEntityEnd, + TPS_InTagFirst, + TPS_InXMLBegin, + TPS_InTagCloseFirst, + TPS_InTagName, + TPS_InTagBeginEnd, + TPS_InTag, + TPS_InTagEscapeK, + TPS_InTagEscapeKK, + TPS_InTagBackSleshed, + TPS_InTagEnd, + TPS_InCommentFirst, + TPS_InCommentLast, + TPS_InComment, + TPS_InCloseCommentFirst, + TPS_InCloseCommentLast, + TPS_InCommentEnd, + TPS_InHostFirstDomain, + TPS_InHostDomainSecond, + TPS_InHostDomain, + TPS_InPortFirst, + TPS_InPort, + TPS_InHostFirstAN, + TPS_InHost, + TPS_InEmail, + TPS_InFileFirst, + TPS_InFileTwiddle, + TPS_InPathFirst, + TPS_InPathFirstFirst, + TPS_InPathSecond, + TPS_InFile, + TPS_InFileNext, + TPS_InURLPathFirst, + TPS_InURLPathStart, + TPS_InURLPath, + TPS_InFURL, + TPS_InProtocolFirst, + TPS_InProtocolSecond, + TPS_InProtocolEnd, + TPS_InHyphenAsciiWordFirst, + TPS_InHyphenAsciiWord, + TPS_InHyphenWordFirst, + TPS_InHyphenWord, + TPS_InHyphenNumWordFirst, + TPS_InHyphenNumWord, + TPS_InHyphenDigitLookahead, + TPS_InParseHyphen, + TPS_InParseHyphenHyphen, + TPS_InHyphenWordPart, + TPS_InHyphenAsciiWordPart, + TPS_InHyphenNumWordPart, + TPS_InHyphenUnsignedInt, + TPS_Null /* last state (fake value) */ +} TParserState; + +/* forward declaration */ +struct TParser; + +typedef int (*TParserCharTest) (struct TParser *); /* any p_is* functions + * except p_iseq */ +typedef void (*TParserSpecial) (struct TParser *); /* special handler for + * special cases... */ + +typedef struct +{ + TParserCharTest isclass; + char c; + uint16 flags; + TParserState tostate; + int type; + TParserSpecial special; +} TParserStateActionItem; + +/* Flag bits in TParserStateActionItem.flags */ +#define A_NEXT 0x0000 +#define A_BINGO 0x0001 +#define A_POP 0x0002 +#define A_PUSH 0x0004 +#define A_RERUN 0x0008 +#define A_CLEAR 0x0010 +#define A_MERGE 0x0020 +#define A_CLRALL 0x0040 + +typedef struct TParserPosition +{ + int posbyte; /* position of parser in bytes */ + int poschar; /* position of parser in characters */ + int charlen; /* length of current char */ + int lenbytetoken; /* length of token-so-far in bytes */ + int lenchartoken; /* and in chars */ + TParserState state; + struct TParserPosition *prev; + const TParserStateActionItem *pushedAtAction; +} TParserPosition; + +typedef struct TParser +{ + /* string and position information */ + char *str; /* multibyte string */ + int lenstr; /* length of mbstring */ + wchar_t *wstr; /* wide character string */ + pg_wchar *pgwstr; /* wide character string for C-locale */ + bool usewide; + + /* State of parse */ + int charmaxlen; + TParserPosition *state; + bool ignore; + bool wanthost; + + /* silly char */ + char c; + + /* out */ + char *token; + int lenbytetoken; + int lenchartoken; + int type; +} TParser; + + +/* forward decls here */ +static bool TParserGet(TParser *prs); + + +static TParserPosition * +newTParserPosition(TParserPosition *prev) +{ + TParserPosition *res = (TParserPosition *) palloc(sizeof(TParserPosition)); + + if (prev) + memcpy(res, prev, sizeof(TParserPosition)); + else + memset(res, 0, sizeof(TParserPosition)); + + res->prev = prev; + + res->pushedAtAction = NULL; + + return res; +} + +static TParser * +TParserInit(char *str, int len) +{ + TParser *prs = (TParser *) palloc0(sizeof(TParser)); + + prs->charmaxlen = pg_database_encoding_max_length(); + prs->str = str; + prs->lenstr = len; + + /* + * Use wide char code only when max encoding length > 1. + */ + if (prs->charmaxlen > 1) + { + Oid collation = DEFAULT_COLLATION_OID; /* TODO */ + pg_locale_t mylocale = 0; /* TODO */ + + prs->usewide = true; + if (lc_ctype_is_c(collation)) + { + /* + * char2wchar doesn't work for C-locale and sizeof(pg_wchar) could + * be different from sizeof(wchar_t) + */ + prs->pgwstr = (pg_wchar *) palloc(sizeof(pg_wchar) * (prs->lenstr + 1)); + pg_mb2wchar_with_len(prs->str, prs->pgwstr, prs->lenstr); + } + else + { + prs->wstr = (wchar_t *) palloc(sizeof(wchar_t) * (prs->lenstr + 1)); + char2wchar(prs->wstr, prs->lenstr + 1, prs->str, prs->lenstr, + mylocale); + } + } + else + prs->usewide = false; + + prs->state = newTParserPosition(NULL); + prs->state->state = TPS_Base; + +#ifdef WPARSER_TRACE + fprintf(stderr, "parsing \"%.*s\"\n", len, str); +#endif + + return prs; +} + +/* + * As an alternative to a full TParserInit one can create a + * TParserCopy which basically is a regular TParser without a private + * copy of the string - instead it uses the one from another TParser. + * This is useful because at some places TParsers are created + * recursively and the repeated copying around of the strings can + * cause major inefficiency if the source string is long. + * The new parser starts parsing at the original's current position. + * + * Obviously one must not close the original TParser before the copy. + */ +static TParser * +TParserCopyInit(const TParser *orig) +{ + TParser *prs = (TParser *) palloc0(sizeof(TParser)); + + prs->charmaxlen = orig->charmaxlen; + prs->str = orig->str + orig->state->posbyte; + prs->lenstr = orig->lenstr - orig->state->posbyte; + prs->usewide = orig->usewide; + + if (orig->pgwstr) + prs->pgwstr = orig->pgwstr + orig->state->poschar; + if (orig->wstr) + prs->wstr = orig->wstr + orig->state->poschar; + + prs->state = newTParserPosition(NULL); + prs->state->state = TPS_Base; + +#ifdef WPARSER_TRACE + fprintf(stderr, "parsing copy of \"%.*s\"\n", prs->lenstr, prs->str); +#endif + + return prs; +} + + +static void +TParserClose(TParser *prs) +{ + while (prs->state) + { + TParserPosition *ptr = prs->state->prev; + + pfree(prs->state); + prs->state = ptr; + } + + if (prs->wstr) + pfree(prs->wstr); + if (prs->pgwstr) + pfree(prs->pgwstr); + +#ifdef WPARSER_TRACE + fprintf(stderr, "closing parser\n"); +#endif + pfree(prs); +} + +/* + * Close a parser created with TParserCopyInit + */ +static void +TParserCopyClose(TParser *prs) +{ + while (prs->state) + { + TParserPosition *ptr = prs->state->prev; + + pfree(prs->state); + prs->state = ptr; + } + +#ifdef WPARSER_TRACE + fprintf(stderr, "closing parser copy\n"); +#endif + pfree(prs); +} + + +/* + * Character-type support functions, equivalent to is* macros, but + * working with any possible encodings and locales. Notes: + * - with multibyte encoding and C-locale isw* function may fail + * or give wrong result. + * - multibyte encoding and C-locale often are used for + * Asian languages. + * - if locale is C then we use pgwstr instead of wstr. + */ + +#define p_iswhat(type, nonascii) \ + \ +static int \ +p_is##type(TParser *prs) \ +{ \ + Assert(prs->state); \ + if (prs->usewide) \ + { \ + if (prs->pgwstr) \ + { \ + unsigned int c = *(prs->pgwstr + prs->state->poschar); \ + if (c > 0x7f) \ + return nonascii; \ + return is##type(c); \ + } \ + return isw##type(*(prs->wstr + prs->state->poschar)); \ + } \ + return is##type(*(unsigned char *) (prs->str + prs->state->posbyte)); \ +} \ + \ +static int \ +p_isnot##type(TParser *prs) \ +{ \ + return !p_is##type(prs); \ +} + +/* + * In C locale with a multibyte encoding, any non-ASCII symbol is considered + * an alpha character, but not a member of other char classes. + */ +p_iswhat(alnum, 1) +p_iswhat(alpha, 1) +p_iswhat(digit, 0) +p_iswhat(lower, 0) +p_iswhat(print, 0) +p_iswhat(punct, 0) +p_iswhat(space, 0) +p_iswhat(upper, 0) +p_iswhat(xdigit, 0) + +/* p_iseq should be used only for ascii symbols */ + +static int +p_iseq(TParser *prs, char c) +{ + Assert(prs->state); + return ((prs->state->charlen == 1 && *(prs->str + prs->state->posbyte) == c)) ? 1 : 0; +} + +static int +p_isEOF(TParser *prs) +{ + Assert(prs->state); + return (prs->state->posbyte == prs->lenstr || prs->state->charlen == 0) ? 1 : 0; +} + +static int +p_iseqC(TParser *prs) +{ + return p_iseq(prs, prs->c); +} + +static int +p_isneC(TParser *prs) +{ + return !p_iseq(prs, prs->c); +} + +static int +p_isascii(TParser *prs) +{ + return (prs->state->charlen == 1 && isascii((unsigned char) *(prs->str + prs->state->posbyte))) ? 1 : 0; +} + +static int +p_isasclet(TParser *prs) +{ + return (p_isascii(prs) && p_isalpha(prs)) ? 1 : 0; +} + +static int +p_isurlchar(TParser *prs) +{ + char ch; + + /* no non-ASCII need apply */ + if (prs->state->charlen != 1) + return 0; + ch = *(prs->str + prs->state->posbyte); + /* no spaces or control characters */ + if (ch <= 0x20 || ch >= 0x7F) + return 0; + /* reject characters disallowed by RFC 3986 */ + switch (ch) + { + case '"': + case '<': + case '>': + case '\\': + case '^': + case '`': + case '{': + case '|': + case '}': + return 0; + } + return 1; +} + + +/* deliberately suppress unused-function complaints for the above */ +void _make_compiler_happy(void); +void +_make_compiler_happy(void) +{ + p_isalnum(NULL); + p_isnotalnum(NULL); + p_isalpha(NULL); + p_isnotalpha(NULL); + p_isdigit(NULL); + p_isnotdigit(NULL); + p_islower(NULL); + p_isnotlower(NULL); + p_isprint(NULL); + p_isnotprint(NULL); + p_ispunct(NULL); + p_isnotpunct(NULL); + p_isspace(NULL); + p_isnotspace(NULL); + p_isupper(NULL); + p_isnotupper(NULL); + p_isxdigit(NULL); + p_isnotxdigit(NULL); + p_isEOF(NULL); + p_iseqC(NULL); + p_isneC(NULL); +} + + +static void +SpecialTags(TParser *prs) +{ + switch (prs->state->lenchartoken) + { + case 8: /* </script */ + if (pg_strncasecmp(prs->token, "</script", 8) == 0) + prs->ignore = false; + break; + case 7: /* <script || </style */ + if (pg_strncasecmp(prs->token, "</style", 7) == 0) + prs->ignore = false; + else if (pg_strncasecmp(prs->token, "<script", 7) == 0) + prs->ignore = true; + break; + case 6: /* <style */ + if (pg_strncasecmp(prs->token, "<style", 6) == 0) + prs->ignore = true; + break; + default: + break; + } +} + +static void +SpecialFURL(TParser *prs) +{ + prs->wanthost = true; + prs->state->posbyte -= prs->state->lenbytetoken; + prs->state->poschar -= prs->state->lenchartoken; +} + +static void +SpecialHyphen(TParser *prs) +{ + prs->state->posbyte -= prs->state->lenbytetoken; + prs->state->poschar -= prs->state->lenchartoken; +} + +static void +SpecialVerVersion(TParser *prs) +{ + prs->state->posbyte -= prs->state->lenbytetoken; + prs->state->poschar -= prs->state->lenchartoken; + prs->state->lenbytetoken = 0; + prs->state->lenchartoken = 0; +} + +static int +p_isstophost(TParser *prs) +{ + if (prs->wanthost) + { + prs->wanthost = false; + return 1; + } + return 0; +} + +static int +p_isignore(TParser *prs) +{ + return (prs->ignore) ? 1 : 0; +} + +static int +p_ishost(TParser *prs) +{ + TParser *tmpprs = TParserCopyInit(prs); + int res = 0; + + tmpprs->wanthost = true; + + if (TParserGet(tmpprs) && tmpprs->type == HOST) + { + prs->state->posbyte += tmpprs->lenbytetoken; + prs->state->poschar += tmpprs->lenchartoken; + prs->state->lenbytetoken += tmpprs->lenbytetoken; + prs->state->lenchartoken += tmpprs->lenchartoken; + prs->state->charlen = tmpprs->state->charlen; + res = 1; + } + TParserCopyClose(tmpprs); + + return res; +} + +static int +p_isURLPath(TParser *prs) +{ + TParser *tmpprs = TParserCopyInit(prs); + int res = 0; + + tmpprs->state = newTParserPosition(tmpprs->state); + tmpprs->state->state = TPS_InURLPathFirst; + + if (TParserGet(tmpprs) && tmpprs->type == URLPATH) + { + prs->state->posbyte += tmpprs->lenbytetoken; + prs->state->poschar += tmpprs->lenchartoken; + prs->state->lenbytetoken += tmpprs->lenbytetoken; + prs->state->lenchartoken += tmpprs->lenchartoken; + prs->state->charlen = tmpprs->state->charlen; + res = 1; + } + TParserCopyClose(tmpprs); + + return res; +} + +/* + * returns true if current character has zero display length or + * it's a special sign in several languages. Such characters + * aren't a word-breaker although they aren't an isalpha. + * In beginning of word they aren't a part of it. + */ +static int +p_isspecial(TParser *prs) +{ + /* + * pg_dsplen could return -1 which means error or control character + */ + if (pg_dsplen(prs->str + prs->state->posbyte) == 0) + return 1; + + /* + * Unicode Characters in the 'Mark, Spacing Combining' Category That + * characters are not alpha although they are not breakers of word too. + * Check that only in utf encoding, because other encodings aren't + * supported by postgres or even exists. + */ + if (GetDatabaseEncoding() == PG_UTF8 && prs->usewide) + { + static const pg_wchar strange_letter[] = { + /* + * use binary search, so elements should be ordered + */ + 0x0903, /* DEVANAGARI SIGN VISARGA */ + 0x093E, /* DEVANAGARI VOWEL SIGN AA */ + 0x093F, /* DEVANAGARI VOWEL SIGN I */ + 0x0940, /* DEVANAGARI VOWEL SIGN II */ + 0x0949, /* DEVANAGARI VOWEL SIGN CANDRA O */ + 0x094A, /* DEVANAGARI VOWEL SIGN SHORT O */ + 0x094B, /* DEVANAGARI VOWEL SIGN O */ + 0x094C, /* DEVANAGARI VOWEL SIGN AU */ + 0x0982, /* BENGALI SIGN ANUSVARA */ + 0x0983, /* BENGALI SIGN VISARGA */ + 0x09BE, /* BENGALI VOWEL SIGN AA */ + 0x09BF, /* BENGALI VOWEL SIGN I */ + 0x09C0, /* BENGALI VOWEL SIGN II */ + 0x09C7, /* BENGALI VOWEL SIGN E */ + 0x09C8, /* BENGALI VOWEL SIGN AI */ + 0x09CB, /* BENGALI VOWEL SIGN O */ + 0x09CC, /* BENGALI VOWEL SIGN AU */ + 0x09D7, /* BENGALI AU LENGTH MARK */ + 0x0A03, /* GURMUKHI SIGN VISARGA */ + 0x0A3E, /* GURMUKHI VOWEL SIGN AA */ + 0x0A3F, /* GURMUKHI VOWEL SIGN I */ + 0x0A40, /* GURMUKHI VOWEL SIGN II */ + 0x0A83, /* GUJARATI SIGN VISARGA */ + 0x0ABE, /* GUJARATI VOWEL SIGN AA */ + 0x0ABF, /* GUJARATI VOWEL SIGN I */ + 0x0AC0, /* GUJARATI VOWEL SIGN II */ + 0x0AC9, /* GUJARATI VOWEL SIGN CANDRA O */ + 0x0ACB, /* GUJARATI VOWEL SIGN O */ + 0x0ACC, /* GUJARATI VOWEL SIGN AU */ + 0x0B02, /* ORIYA SIGN ANUSVARA */ + 0x0B03, /* ORIYA SIGN VISARGA */ + 0x0B3E, /* ORIYA VOWEL SIGN AA */ + 0x0B40, /* ORIYA VOWEL SIGN II */ + 0x0B47, /* ORIYA VOWEL SIGN E */ + 0x0B48, /* ORIYA VOWEL SIGN AI */ + 0x0B4B, /* ORIYA VOWEL SIGN O */ + 0x0B4C, /* ORIYA VOWEL SIGN AU */ + 0x0B57, /* ORIYA AU LENGTH MARK */ + 0x0BBE, /* TAMIL VOWEL SIGN AA */ + 0x0BBF, /* TAMIL VOWEL SIGN I */ + 0x0BC1, /* TAMIL VOWEL SIGN U */ + 0x0BC2, /* TAMIL VOWEL SIGN UU */ + 0x0BC6, /* TAMIL VOWEL SIGN E */ + 0x0BC7, /* TAMIL VOWEL SIGN EE */ + 0x0BC8, /* TAMIL VOWEL SIGN AI */ + 0x0BCA, /* TAMIL VOWEL SIGN O */ + 0x0BCB, /* TAMIL VOWEL SIGN OO */ + 0x0BCC, /* TAMIL VOWEL SIGN AU */ + 0x0BD7, /* TAMIL AU LENGTH MARK */ + 0x0C01, /* TELUGU SIGN CANDRABINDU */ + 0x0C02, /* TELUGU SIGN ANUSVARA */ + 0x0C03, /* TELUGU SIGN VISARGA */ + 0x0C41, /* TELUGU VOWEL SIGN U */ + 0x0C42, /* TELUGU VOWEL SIGN UU */ + 0x0C43, /* TELUGU VOWEL SIGN VOCALIC R */ + 0x0C44, /* TELUGU VOWEL SIGN VOCALIC RR */ + 0x0C82, /* KANNADA SIGN ANUSVARA */ + 0x0C83, /* KANNADA SIGN VISARGA */ + 0x0CBE, /* KANNADA VOWEL SIGN AA */ + 0x0CC0, /* KANNADA VOWEL SIGN II */ + 0x0CC1, /* KANNADA VOWEL SIGN U */ + 0x0CC2, /* KANNADA VOWEL SIGN UU */ + 0x0CC3, /* KANNADA VOWEL SIGN VOCALIC R */ + 0x0CC4, /* KANNADA VOWEL SIGN VOCALIC RR */ + 0x0CC7, /* KANNADA VOWEL SIGN EE */ + 0x0CC8, /* KANNADA VOWEL SIGN AI */ + 0x0CCA, /* KANNADA VOWEL SIGN O */ + 0x0CCB, /* KANNADA VOWEL SIGN OO */ + 0x0CD5, /* KANNADA LENGTH MARK */ + 0x0CD6, /* KANNADA AI LENGTH MARK */ + 0x0D02, /* MALAYALAM SIGN ANUSVARA */ + 0x0D03, /* MALAYALAM SIGN VISARGA */ + 0x0D3E, /* MALAYALAM VOWEL SIGN AA */ + 0x0D3F, /* MALAYALAM VOWEL SIGN I */ + 0x0D40, /* MALAYALAM VOWEL SIGN II */ + 0x0D46, /* MALAYALAM VOWEL SIGN E */ + 0x0D47, /* MALAYALAM VOWEL SIGN EE */ + 0x0D48, /* MALAYALAM VOWEL SIGN AI */ + 0x0D4A, /* MALAYALAM VOWEL SIGN O */ + 0x0D4B, /* MALAYALAM VOWEL SIGN OO */ + 0x0D4C, /* MALAYALAM VOWEL SIGN AU */ + 0x0D57, /* MALAYALAM AU LENGTH MARK */ + 0x0D82, /* SINHALA SIGN ANUSVARAYA */ + 0x0D83, /* SINHALA SIGN VISARGAYA */ + 0x0DCF, /* SINHALA VOWEL SIGN AELA-PILLA */ + 0x0DD0, /* SINHALA VOWEL SIGN KETTI AEDA-PILLA */ + 0x0DD1, /* SINHALA VOWEL SIGN DIGA AEDA-PILLA */ + 0x0DD8, /* SINHALA VOWEL SIGN GAETTA-PILLA */ + 0x0DD9, /* SINHALA VOWEL SIGN KOMBUVA */ + 0x0DDA, /* SINHALA VOWEL SIGN DIGA KOMBUVA */ + 0x0DDB, /* SINHALA VOWEL SIGN KOMBU DEKA */ + 0x0DDC, /* SINHALA VOWEL SIGN KOMBUVA HAA AELA-PILLA */ + 0x0DDD, /* SINHALA VOWEL SIGN KOMBUVA HAA DIGA + * AELA-PILLA */ + 0x0DDE, /* SINHALA VOWEL SIGN KOMBUVA HAA GAYANUKITTA */ + 0x0DDF, /* SINHALA VOWEL SIGN GAYANUKITTA */ + 0x0DF2, /* SINHALA VOWEL SIGN DIGA GAETTA-PILLA */ + 0x0DF3, /* SINHALA VOWEL SIGN DIGA GAYANUKITTA */ + 0x0F3E, /* TIBETAN SIGN YAR TSHES */ + 0x0F3F, /* TIBETAN SIGN MAR TSHES */ + 0x0F7F, /* TIBETAN SIGN RNAM BCAD */ + 0x102B, /* MYANMAR VOWEL SIGN TALL AA */ + 0x102C, /* MYANMAR VOWEL SIGN AA */ + 0x1031, /* MYANMAR VOWEL SIGN E */ + 0x1038, /* MYANMAR SIGN VISARGA */ + 0x103B, /* MYANMAR CONSONANT SIGN MEDIAL YA */ + 0x103C, /* MYANMAR CONSONANT SIGN MEDIAL RA */ + 0x1056, /* MYANMAR VOWEL SIGN VOCALIC R */ + 0x1057, /* MYANMAR VOWEL SIGN VOCALIC RR */ + 0x1062, /* MYANMAR VOWEL SIGN SGAW KAREN EU */ + 0x1063, /* MYANMAR TONE MARK SGAW KAREN HATHI */ + 0x1064, /* MYANMAR TONE MARK SGAW KAREN KE PHO */ + 0x1067, /* MYANMAR VOWEL SIGN WESTERN PWO KAREN EU */ + 0x1068, /* MYANMAR VOWEL SIGN WESTERN PWO KAREN UE */ + 0x1069, /* MYANMAR SIGN WESTERN PWO KAREN TONE-1 */ + 0x106A, /* MYANMAR SIGN WESTERN PWO KAREN TONE-2 */ + 0x106B, /* MYANMAR SIGN WESTERN PWO KAREN TONE-3 */ + 0x106C, /* MYANMAR SIGN WESTERN PWO KAREN TONE-4 */ + 0x106D, /* MYANMAR SIGN WESTERN PWO KAREN TONE-5 */ + 0x1083, /* MYANMAR VOWEL SIGN SHAN AA */ + 0x1084, /* MYANMAR VOWEL SIGN SHAN E */ + 0x1087, /* MYANMAR SIGN SHAN TONE-2 */ + 0x1088, /* MYANMAR SIGN SHAN TONE-3 */ + 0x1089, /* MYANMAR SIGN SHAN TONE-5 */ + 0x108A, /* MYANMAR SIGN SHAN TONE-6 */ + 0x108B, /* MYANMAR SIGN SHAN COUNCIL TONE-2 */ + 0x108C, /* MYANMAR SIGN SHAN COUNCIL TONE-3 */ + 0x108F, /* MYANMAR SIGN RUMAI PALAUNG TONE-5 */ + 0x17B6, /* KHMER VOWEL SIGN AA */ + 0x17BE, /* KHMER VOWEL SIGN OE */ + 0x17BF, /* KHMER VOWEL SIGN YA */ + 0x17C0, /* KHMER VOWEL SIGN IE */ + 0x17C1, /* KHMER VOWEL SIGN E */ + 0x17C2, /* KHMER VOWEL SIGN AE */ + 0x17C3, /* KHMER VOWEL SIGN AI */ + 0x17C4, /* KHMER VOWEL SIGN OO */ + 0x17C5, /* KHMER VOWEL SIGN AU */ + 0x17C7, /* KHMER SIGN REAHMUK */ + 0x17C8, /* KHMER SIGN YUUKALEAPINTU */ + 0x1923, /* LIMBU VOWEL SIGN EE */ + 0x1924, /* LIMBU VOWEL SIGN AI */ + 0x1925, /* LIMBU VOWEL SIGN OO */ + 0x1926, /* LIMBU VOWEL SIGN AU */ + 0x1929, /* LIMBU SUBJOINED LETTER YA */ + 0x192A, /* LIMBU SUBJOINED LETTER RA */ + 0x192B, /* LIMBU SUBJOINED LETTER WA */ + 0x1930, /* LIMBU SMALL LETTER KA */ + 0x1931, /* LIMBU SMALL LETTER NGA */ + 0x1933, /* LIMBU SMALL LETTER TA */ + 0x1934, /* LIMBU SMALL LETTER NA */ + 0x1935, /* LIMBU SMALL LETTER PA */ + 0x1936, /* LIMBU SMALL LETTER MA */ + 0x1937, /* LIMBU SMALL LETTER RA */ + 0x1938, /* LIMBU SMALL LETTER LA */ + 0x19B0, /* NEW TAI LUE VOWEL SIGN VOWEL SHORTENER */ + 0x19B1, /* NEW TAI LUE VOWEL SIGN AA */ + 0x19B2, /* NEW TAI LUE VOWEL SIGN II */ + 0x19B3, /* NEW TAI LUE VOWEL SIGN U */ + 0x19B4, /* NEW TAI LUE VOWEL SIGN UU */ + 0x19B5, /* NEW TAI LUE VOWEL SIGN E */ + 0x19B6, /* NEW TAI LUE VOWEL SIGN AE */ + 0x19B7, /* NEW TAI LUE VOWEL SIGN O */ + 0x19B8, /* NEW TAI LUE VOWEL SIGN OA */ + 0x19B9, /* NEW TAI LUE VOWEL SIGN UE */ + 0x19BA, /* NEW TAI LUE VOWEL SIGN AY */ + 0x19BB, /* NEW TAI LUE VOWEL SIGN AAY */ + 0x19BC, /* NEW TAI LUE VOWEL SIGN UY */ + 0x19BD, /* NEW TAI LUE VOWEL SIGN OY */ + 0x19BE, /* NEW TAI LUE VOWEL SIGN OAY */ + 0x19BF, /* NEW TAI LUE VOWEL SIGN UEY */ + 0x19C0, /* NEW TAI LUE VOWEL SIGN IY */ + 0x19C8, /* NEW TAI LUE TONE MARK-1 */ + 0x19C9, /* NEW TAI LUE TONE MARK-2 */ + 0x1A19, /* BUGINESE VOWEL SIGN E */ + 0x1A1A, /* BUGINESE VOWEL SIGN O */ + 0x1A1B, /* BUGINESE VOWEL SIGN AE */ + 0x1B04, /* BALINESE SIGN BISAH */ + 0x1B35, /* BALINESE VOWEL SIGN TEDUNG */ + 0x1B3B, /* BALINESE VOWEL SIGN RA REPA TEDUNG */ + 0x1B3D, /* BALINESE VOWEL SIGN LA LENGA TEDUNG */ + 0x1B3E, /* BALINESE VOWEL SIGN TALING */ + 0x1B3F, /* BALINESE VOWEL SIGN TALING REPA */ + 0x1B40, /* BALINESE VOWEL SIGN TALING TEDUNG */ + 0x1B41, /* BALINESE VOWEL SIGN TALING REPA TEDUNG */ + 0x1B43, /* BALINESE VOWEL SIGN PEPET TEDUNG */ + 0x1B44, /* BALINESE ADEG ADEG */ + 0x1B82, /* SUNDANESE SIGN PANGWISAD */ + 0x1BA1, /* SUNDANESE CONSONANT SIGN PAMINGKAL */ + 0x1BA6, /* SUNDANESE VOWEL SIGN PANAELAENG */ + 0x1BA7, /* SUNDANESE VOWEL SIGN PANOLONG */ + 0x1BAA, /* SUNDANESE SIGN PAMAAEH */ + 0x1C24, /* LEPCHA SUBJOINED LETTER YA */ + 0x1C25, /* LEPCHA SUBJOINED LETTER RA */ + 0x1C26, /* LEPCHA VOWEL SIGN AA */ + 0x1C27, /* LEPCHA VOWEL SIGN I */ + 0x1C28, /* LEPCHA VOWEL SIGN O */ + 0x1C29, /* LEPCHA VOWEL SIGN OO */ + 0x1C2A, /* LEPCHA VOWEL SIGN U */ + 0x1C2B, /* LEPCHA VOWEL SIGN UU */ + 0x1C34, /* LEPCHA CONSONANT SIGN NYIN-DO */ + 0x1C35, /* LEPCHA CONSONANT SIGN KANG */ + 0xA823, /* SYLOTI NAGRI VOWEL SIGN A */ + 0xA824, /* SYLOTI NAGRI VOWEL SIGN I */ + 0xA827, /* SYLOTI NAGRI VOWEL SIGN OO */ + 0xA880, /* SAURASHTRA SIGN ANUSVARA */ + 0xA881, /* SAURASHTRA SIGN VISARGA */ + 0xA8B4, /* SAURASHTRA CONSONANT SIGN HAARU */ + 0xA8B5, /* SAURASHTRA VOWEL SIGN AA */ + 0xA8B6, /* SAURASHTRA VOWEL SIGN I */ + 0xA8B7, /* SAURASHTRA VOWEL SIGN II */ + 0xA8B8, /* SAURASHTRA VOWEL SIGN U */ + 0xA8B9, /* SAURASHTRA VOWEL SIGN UU */ + 0xA8BA, /* SAURASHTRA VOWEL SIGN VOCALIC R */ + 0xA8BB, /* SAURASHTRA VOWEL SIGN VOCALIC RR */ + 0xA8BC, /* SAURASHTRA VOWEL SIGN VOCALIC L */ + 0xA8BD, /* SAURASHTRA VOWEL SIGN VOCALIC LL */ + 0xA8BE, /* SAURASHTRA VOWEL SIGN E */ + 0xA8BF, /* SAURASHTRA VOWEL SIGN EE */ + 0xA8C0, /* SAURASHTRA VOWEL SIGN AI */ + 0xA8C1, /* SAURASHTRA VOWEL SIGN O */ + 0xA8C2, /* SAURASHTRA VOWEL SIGN OO */ + 0xA8C3, /* SAURASHTRA VOWEL SIGN AU */ + 0xA952, /* REJANG CONSONANT SIGN H */ + 0xA953, /* REJANG VIRAMA */ + 0xAA2F, /* CHAM VOWEL SIGN O */ + 0xAA30, /* CHAM VOWEL SIGN AI */ + 0xAA33, /* CHAM CONSONANT SIGN YA */ + 0xAA34, /* CHAM CONSONANT SIGN RA */ + 0xAA4D /* CHAM CONSONANT SIGN FINAL H */ + }; + const pg_wchar *StopLow = strange_letter, + *StopHigh = strange_letter + lengthof(strange_letter), + *StopMiddle; + pg_wchar c; + + if (prs->pgwstr) + c = *(prs->pgwstr + prs->state->poschar); + else + c = (pg_wchar) *(prs->wstr + prs->state->poschar); + + while (StopLow < StopHigh) + { + StopMiddle = StopLow + ((StopHigh - StopLow) >> 1); + if (*StopMiddle == c) + return 1; + else if (*StopMiddle < c) + StopLow = StopMiddle + 1; + else + StopHigh = StopMiddle; + } + } + + return 0; +} + +/* + * Table of state/action of parser + */ + +static const TParserStateActionItem actionTPS_Base[] = { + {p_isEOF, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '<', A_PUSH, TPS_InTagFirst, 0, NULL}, + {p_isignore, 0, A_NEXT, TPS_InSpace, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InAsciiWord, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InUnsignedInt, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InSignedIntFirst, 0, NULL}, + {p_iseqC, '+', A_PUSH, TPS_InSignedIntFirst, 0, NULL}, + {p_iseqC, '&', A_PUSH, TPS_InXMLEntityFirst, 0, NULL}, + {p_iseqC, '~', A_PUSH, TPS_InFileTwiddle, 0, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InPathFirstFirst, 0, NULL}, + {NULL, 0, A_NEXT, TPS_InSpace, 0, NULL} +}; + + +static const TParserStateActionItem actionTPS_InNumWord[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, NUMWORD, NULL}, + {p_isalnum, 0, A_NEXT, TPS_InNumWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InNumWord, 0, NULL}, + {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHyphenNumWordFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, NUMWORD, NULL} +}; + +static const TParserStateActionItem actionTPS_InAsciiWord[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL}, + {p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHyphenAsciiWordFirst, 0, NULL}, + {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL}, + {p_iseqC, ':', A_PUSH, TPS_InProtocolFirst, 0, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL}, + {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InNumWord, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InWord, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, ASCIIWORD, NULL} +}; + +static const TParserStateActionItem actionTPS_InWord[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, WORD_T, NULL}, + {p_isalpha, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InNumWord, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHyphenWordFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, WORD_T, NULL} +}; + +static const TParserStateActionItem actionTPS_InUnsignedInt[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, UNSIGNEDINT, NULL}, + {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InUDecimalFirst, 0, NULL}, + {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL}, + {p_isasclet, 0, A_PUSH, TPS_InHost, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InNumWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InNumWord, 0, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, UNSIGNEDINT, NULL} +}; + +static const TParserStateActionItem actionTPS_InSignedIntFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT | A_CLEAR, TPS_InSignedInt, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InSignedInt[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, SIGNEDINT, NULL}, + {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InDecimalFirst, 0, NULL}, + {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, SIGNEDINT, NULL} +}; + +static const TParserStateActionItem actionTPS_InSpace[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, SPACE, NULL}, + {p_iseqC, '<', A_BINGO, TPS_Base, SPACE, NULL}, + {p_isignore, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_BINGO, TPS_Base, SPACE, NULL}, + {p_iseqC, '+', A_BINGO, TPS_Base, SPACE, NULL}, + {p_iseqC, '&', A_BINGO, TPS_Base, SPACE, NULL}, + {p_iseqC, '/', A_BINGO, TPS_Base, SPACE, NULL}, + {p_isnotalnum, 0, A_NEXT, TPS_InSpace, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, SPACE, NULL} +}; + +static const TParserStateActionItem actionTPS_InUDecimalFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_CLEAR, TPS_InUDecimal, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InUDecimal[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InUDecimal, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InVersionFirst, 0, NULL}, + {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL} +}; + +static const TParserStateActionItem actionTPS_InDecimalFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_CLEAR, TPS_InDecimal, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InDecimal[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InDecimal, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InVerVersion, 0, NULL}, + {p_iseqC, 'e', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {p_iseqC, 'E', A_PUSH, TPS_InMantissaFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, DECIMAL_T, NULL} +}; + +static const TParserStateActionItem actionTPS_InVerVersion[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_RERUN, TPS_InSVerVersion, 0, SpecialVerVersion}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InSVerVersion[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_BINGO | A_CLRALL, TPS_InUnsignedInt, SPACE, NULL}, + {NULL, 0, A_NEXT, TPS_Null, 0, NULL} +}; + + +static const TParserStateActionItem actionTPS_InVersionFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_CLEAR, TPS_InVersion, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InVersion[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, VERSIONNUMBER, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InVersion, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InVersionFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, VERSIONNUMBER, NULL} +}; + +static const TParserStateActionItem actionTPS_InMantissaFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_CLEAR, TPS_InMantissa, 0, NULL}, + {p_iseqC, '+', A_NEXT, TPS_InMantissaSign, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InMantissaSign, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InMantissaSign[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_CLEAR, TPS_InMantissa, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InMantissa[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, SCIENTIFIC, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InMantissa, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, SCIENTIFIC, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntityFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '#', A_NEXT, TPS_InXMLEntityNumFirst, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, ':', A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_InXMLEntity, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntity[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isalnum, 0, A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, ':', A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, '.', A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InXMLEntity, 0, NULL}, + {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntityNumFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, 'x', A_NEXT, TPS_InXMLEntityHexNumFirst, 0, NULL}, + {p_iseqC, 'X', A_NEXT, TPS_InXMLEntityHexNumFirst, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InXMLEntityNum, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntityHexNumFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isxdigit, 0, A_NEXT, TPS_InXMLEntityHexNum, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntityNum[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InXMLEntityNum, 0, NULL}, + {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntityHexNum[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isxdigit, 0, A_NEXT, TPS_InXMLEntityHexNum, 0, NULL}, + {p_iseqC, ';', A_NEXT, TPS_InXMLEntityEnd, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLEntityEnd[] = { + {NULL, 0, A_BINGO | A_CLEAR, TPS_Base, XMLENTITY, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InTagCloseFirst, 0, NULL}, + {p_iseqC, '!', A_PUSH, TPS_InCommentFirst, 0, NULL}, + {p_iseqC, '?', A_PUSH, TPS_InXMLBegin, 0, NULL}, + {p_isasclet, 0, A_PUSH, TPS_InTagName, 0, NULL}, + {p_iseqC, ':', A_PUSH, TPS_InTagName, 0, NULL}, + {p_iseqC, '_', A_PUSH, TPS_InTagName, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InXMLBegin[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + /* <?xml ... */ + /* XXX do we wants states for the m and l ? Right now this accepts <?xZ */ + {p_iseqC, 'x', A_NEXT, TPS_InTag, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagCloseFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InTagName, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagName[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + /* <br/> case */ + {p_iseqC, '/', A_NEXT, TPS_InTagBeginEnd, 0, NULL}, + {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, SpecialTags}, + {p_isspace, 0, A_NEXT, TPS_InTag, 0, SpecialTags}, + {p_isalnum, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, ':', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '.', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagBeginEnd[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTag[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '>', A_NEXT, TPS_InTagEnd, 0, SpecialTags}, + {p_iseqC, '\'', A_NEXT, TPS_InTagEscapeK, 0, NULL}, + {p_iseqC, '"', A_NEXT, TPS_InTagEscapeKK, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '=', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '#', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '/', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, ':', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '.', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '&', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '?', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '%', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '~', A_NEXT, TPS_Null, 0, NULL}, + {p_isspace, 0, A_NEXT, TPS_Null, 0, SpecialTags}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagEscapeK[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '\\', A_PUSH, TPS_InTagBackSleshed, 0, NULL}, + {p_iseqC, '\'', A_NEXT, TPS_InTag, 0, NULL}, + {NULL, 0, A_NEXT, TPS_InTagEscapeK, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagEscapeKK[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '\\', A_PUSH, TPS_InTagBackSleshed, 0, NULL}, + {p_iseqC, '"', A_NEXT, TPS_InTag, 0, NULL}, + {NULL, 0, A_NEXT, TPS_InTagEscapeKK, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagBackSleshed[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {NULL, 0, A_MERGE, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InTagEnd[] = { + {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, TAG_T, NULL} +}; + +static const TParserStateActionItem actionTPS_InCommentFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InCommentLast, 0, NULL}, + /* <!DOCTYPE ...> */ + {p_iseqC, 'D', A_NEXT, TPS_InTag, 0, NULL}, + {p_iseqC, 'd', A_NEXT, TPS_InTag, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InCommentLast[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InComment, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InComment[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InCloseCommentFirst, 0, NULL}, + {NULL, 0, A_NEXT, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InCloseCommentFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InCloseCommentLast, 0, NULL}, + {NULL, 0, A_NEXT, TPS_InComment, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InCloseCommentLast[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_Null, 0, NULL}, + {p_iseqC, '>', A_NEXT, TPS_InCommentEnd, 0, NULL}, + {NULL, 0, A_NEXT, TPS_InComment, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InCommentEnd[] = { + {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, TAG_T, NULL} +}; + +static const TParserStateActionItem actionTPS_InHostFirstDomain[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHostDomainSecond, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHostDomainSecond[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL}, + {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL}, + {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHostDomain[] = { + {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHostDomain, 0, NULL}, + {p_isdigit, 0, A_PUSH, TPS_InHost, 0, NULL}, + {p_iseqC, ':', A_PUSH, TPS_InPortFirst, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL}, + {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL}, + {p_isdigit, 0, A_POP, TPS_Null, 0, NULL}, + {p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL}, + {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL} +}; + +static const TParserStateActionItem actionTPS_InPortFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InPort, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InPort[] = { + {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InPort, 0, NULL}, + {p_isstophost, 0, A_BINGO | A_CLRALL, TPS_InURLPathStart, HOST, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFURL, 0, NULL}, + {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, HOST, NULL} +}; + +static const TParserStateActionItem actionTPS_InHostFirstAN[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHost, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHost[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHost, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHost, 0, NULL}, + {p_iseqC, '@', A_PUSH, TPS_InEmail, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InHostFirstDomain, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {p_iseqC, '_', A_PUSH, TPS_InHostFirstAN, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InEmail[] = { + {p_isstophost, 0, A_POP, TPS_Null, 0, NULL}, + {p_ishost, 0, A_BINGO | A_CLRALL, TPS_Base, EMAIL, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InFileFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '.', A_NEXT, TPS_InPathFirst, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '~', A_PUSH, TPS_InFileTwiddle, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InFileTwiddle[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InPathFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '.', A_NEXT, TPS_InPathSecond, 0, NULL}, + {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InPathFirstFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '.', A_NEXT, TPS_InPathSecond, 0, NULL}, + {p_iseqC, '/', A_NEXT, TPS_InFileFirst, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InPathSecond[] = { + {p_isEOF, 0, A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL}, + {p_iseqC, '/', A_NEXT | A_PUSH, TPS_InFileFirst, 0, NULL}, + {p_iseqC, '/', A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL}, + {p_isspace, 0, A_BINGO | A_CLEAR, TPS_Base, FILEPATH, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InFile[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, FILEPATH, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '.', A_PUSH, TPS_InFileNext, 0, NULL}, + {p_iseqC, '_', A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '-', A_NEXT, TPS_InFile, 0, NULL}, + {p_iseqC, '/', A_PUSH, TPS_InFileFirst, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, FILEPATH, NULL} +}; + +static const TParserStateActionItem actionTPS_InFileNext[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_CLEAR, TPS_InFile, 0, NULL}, + {p_isdigit, 0, A_CLEAR, TPS_InFile, 0, NULL}, + {p_iseqC, '_', A_CLEAR, TPS_InFile, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InURLPathFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isurlchar, 0, A_NEXT, TPS_InURLPath, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL}, +}; + +static const TParserStateActionItem actionTPS_InURLPathStart[] = { + {NULL, 0, A_NEXT, TPS_InURLPath, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InURLPath[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, URLPATH, NULL}, + {p_isurlchar, 0, A_NEXT, TPS_InURLPath, 0, NULL}, + {NULL, 0, A_BINGO, TPS_Base, URLPATH, NULL} +}; + +static const TParserStateActionItem actionTPS_InFURL[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isURLPath, 0, A_BINGO | A_CLRALL, TPS_Base, URL_T, SpecialFURL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InProtocolFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '/', A_NEXT, TPS_InProtocolSecond, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InProtocolSecond[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_iseqC, '/', A_NEXT, TPS_InProtocolEnd, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InProtocolEnd[] = { + {NULL, 0, A_BINGO | A_CLRALL, TPS_Base, PROTOCOL, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenAsciiWordFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWord, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenAsciiWord[] = { + {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, ASCIIHWORD, SpecialHyphen}, + {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWord, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenWord, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHyphenAsciiWordFirst, 0, NULL}, + {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, ASCIIHWORD, SpecialHyphen} +}; + +static const TParserStateActionItem actionTPS_InHyphenWordFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenWord[] = { + {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, HWORD, SpecialHyphen}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenWord, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHyphenWordFirst, 0, NULL}, + {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, HWORD, SpecialHyphen} +}; + +static const TParserStateActionItem actionTPS_InHyphenNumWordFirst[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenNumWord[] = { + {p_isEOF, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, NUMHWORD, SpecialHyphen}, + {p_isalnum, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InHyphenNumWordFirst, 0, NULL}, + {NULL, 0, A_BINGO | A_CLRALL, TPS_InParseHyphen, NUMHWORD, SpecialHyphen} +}; + +static const TParserStateActionItem actionTPS_InHyphenDigitLookahead[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenDigitLookahead, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWord, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InParseHyphen[] = { + {p_isEOF, 0, A_RERUN, TPS_Base, 0, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWordPart, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL}, + {p_isdigit, 0, A_PUSH, TPS_InHyphenUnsignedInt, 0, NULL}, + {p_iseqC, '-', A_PUSH, TPS_InParseHyphenHyphen, 0, NULL}, + {NULL, 0, A_RERUN, TPS_Base, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InParseHyphenHyphen[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isalnum, 0, A_BINGO | A_CLEAR, TPS_InParseHyphen, SPACE, NULL}, + {p_isspecial, 0, A_BINGO | A_CLEAR, TPS_InParseHyphen, SPACE, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenWordPart[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, PARTHWORD, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL}, + {NULL, 0, A_BINGO, TPS_InParseHyphen, PARTHWORD, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenAsciiWordPart[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, ASCIIPARTHWORD, NULL}, + {p_isasclet, 0, A_NEXT, TPS_InHyphenAsciiWordPart, 0, NULL}, + {p_isalpha, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenWordPart, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL}, + {NULL, 0, A_BINGO, TPS_InParseHyphen, ASCIIPARTHWORD, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenNumWordPart[] = { + {p_isEOF, 0, A_BINGO, TPS_Base, NUMPARTHWORD, NULL}, + {p_isalnum, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL}, + {p_isspecial, 0, A_NEXT, TPS_InHyphenNumWordPart, 0, NULL}, + {NULL, 0, A_BINGO, TPS_InParseHyphen, NUMPARTHWORD, NULL} +}; + +static const TParserStateActionItem actionTPS_InHyphenUnsignedInt[] = { + {p_isEOF, 0, A_POP, TPS_Null, 0, NULL}, + {p_isdigit, 0, A_NEXT, TPS_Null, 0, NULL}, + {p_isalpha, 0, A_CLEAR, TPS_InHyphenNumWordPart, 0, NULL}, + {p_isspecial, 0, A_CLEAR, TPS_InHyphenNumWordPart, 0, NULL}, + {NULL, 0, A_POP, TPS_Null, 0, NULL} +}; + + +/* + * main table of per-state parser actions + */ +typedef struct +{ + const TParserStateActionItem *action; /* the actual state info */ + TParserState state; /* only for Assert crosscheck */ +#ifdef WPARSER_TRACE + const char *state_name; /* only for debug printout */ +#endif +} TParserStateAction; + +#ifdef WPARSER_TRACE +#define TPARSERSTATEACTION(state) \ + { CppConcat(action,state), state, CppAsString(state) } +#else +#define TPARSERSTATEACTION(state) \ + { CppConcat(action,state), state } +#endif + +/* + * order must be the same as in typedef enum {} TParserState!! + */ + +static const TParserStateAction Actions[] = { + TPARSERSTATEACTION(TPS_Base), + TPARSERSTATEACTION(TPS_InNumWord), + TPARSERSTATEACTION(TPS_InAsciiWord), + TPARSERSTATEACTION(TPS_InWord), + TPARSERSTATEACTION(TPS_InUnsignedInt), + TPARSERSTATEACTION(TPS_InSignedIntFirst), + TPARSERSTATEACTION(TPS_InSignedInt), + TPARSERSTATEACTION(TPS_InSpace), + TPARSERSTATEACTION(TPS_InUDecimalFirst), + TPARSERSTATEACTION(TPS_InUDecimal), + TPARSERSTATEACTION(TPS_InDecimalFirst), + TPARSERSTATEACTION(TPS_InDecimal), + TPARSERSTATEACTION(TPS_InVerVersion), + TPARSERSTATEACTION(TPS_InSVerVersion), + TPARSERSTATEACTION(TPS_InVersionFirst), + TPARSERSTATEACTION(TPS_InVersion), + TPARSERSTATEACTION(TPS_InMantissaFirst), + TPARSERSTATEACTION(TPS_InMantissaSign), + TPARSERSTATEACTION(TPS_InMantissa), + TPARSERSTATEACTION(TPS_InXMLEntityFirst), + TPARSERSTATEACTION(TPS_InXMLEntity), + TPARSERSTATEACTION(TPS_InXMLEntityNumFirst), + TPARSERSTATEACTION(TPS_InXMLEntityNum), + TPARSERSTATEACTION(TPS_InXMLEntityHexNumFirst), + TPARSERSTATEACTION(TPS_InXMLEntityHexNum), + TPARSERSTATEACTION(TPS_InXMLEntityEnd), + TPARSERSTATEACTION(TPS_InTagFirst), + TPARSERSTATEACTION(TPS_InXMLBegin), + TPARSERSTATEACTION(TPS_InTagCloseFirst), + TPARSERSTATEACTION(TPS_InTagName), + TPARSERSTATEACTION(TPS_InTagBeginEnd), + TPARSERSTATEACTION(TPS_InTag), + TPARSERSTATEACTION(TPS_InTagEscapeK), + TPARSERSTATEACTION(TPS_InTagEscapeKK), + TPARSERSTATEACTION(TPS_InTagBackSleshed), + TPARSERSTATEACTION(TPS_InTagEnd), + TPARSERSTATEACTION(TPS_InCommentFirst), + TPARSERSTATEACTION(TPS_InCommentLast), + TPARSERSTATEACTION(TPS_InComment), + TPARSERSTATEACTION(TPS_InCloseCommentFirst), + TPARSERSTATEACTION(TPS_InCloseCommentLast), + TPARSERSTATEACTION(TPS_InCommentEnd), + TPARSERSTATEACTION(TPS_InHostFirstDomain), + TPARSERSTATEACTION(TPS_InHostDomainSecond), + TPARSERSTATEACTION(TPS_InHostDomain), + TPARSERSTATEACTION(TPS_InPortFirst), + TPARSERSTATEACTION(TPS_InPort), + TPARSERSTATEACTION(TPS_InHostFirstAN), + TPARSERSTATEACTION(TPS_InHost), + TPARSERSTATEACTION(TPS_InEmail), + TPARSERSTATEACTION(TPS_InFileFirst), + TPARSERSTATEACTION(TPS_InFileTwiddle), + TPARSERSTATEACTION(TPS_InPathFirst), + TPARSERSTATEACTION(TPS_InPathFirstFirst), + TPARSERSTATEACTION(TPS_InPathSecond), + TPARSERSTATEACTION(TPS_InFile), + TPARSERSTATEACTION(TPS_InFileNext), + TPARSERSTATEACTION(TPS_InURLPathFirst), + TPARSERSTATEACTION(TPS_InURLPathStart), + TPARSERSTATEACTION(TPS_InURLPath), + TPARSERSTATEACTION(TPS_InFURL), + TPARSERSTATEACTION(TPS_InProtocolFirst), + TPARSERSTATEACTION(TPS_InProtocolSecond), + TPARSERSTATEACTION(TPS_InProtocolEnd), + TPARSERSTATEACTION(TPS_InHyphenAsciiWordFirst), + TPARSERSTATEACTION(TPS_InHyphenAsciiWord), + TPARSERSTATEACTION(TPS_InHyphenWordFirst), + TPARSERSTATEACTION(TPS_InHyphenWord), + TPARSERSTATEACTION(TPS_InHyphenNumWordFirst), + TPARSERSTATEACTION(TPS_InHyphenNumWord), + TPARSERSTATEACTION(TPS_InHyphenDigitLookahead), + TPARSERSTATEACTION(TPS_InParseHyphen), + TPARSERSTATEACTION(TPS_InParseHyphenHyphen), + TPARSERSTATEACTION(TPS_InHyphenWordPart), + TPARSERSTATEACTION(TPS_InHyphenAsciiWordPart), + TPARSERSTATEACTION(TPS_InHyphenNumWordPart), + TPARSERSTATEACTION(TPS_InHyphenUnsignedInt) +}; + + +static bool +TParserGet(TParser *prs) +{ + const TParserStateActionItem *item = NULL; + + Assert(prs->state); + + if (prs->state->posbyte >= prs->lenstr) + return false; + + prs->token = prs->str + prs->state->posbyte; + prs->state->pushedAtAction = NULL; + + /* look at string */ + while (prs->state->posbyte <= prs->lenstr) + { + if (prs->state->posbyte == prs->lenstr) + prs->state->charlen = 0; + else + prs->state->charlen = (prs->charmaxlen == 1) ? prs->charmaxlen : + pg_mblen(prs->str + prs->state->posbyte); + + Assert(prs->state->posbyte + prs->state->charlen <= prs->lenstr); + Assert(prs->state->state >= TPS_Base && prs->state->state < TPS_Null); + Assert(Actions[prs->state->state].state == prs->state->state); + + if (prs->state->pushedAtAction) + { + /* After a POP, pick up at the next test */ + item = prs->state->pushedAtAction + 1; + prs->state->pushedAtAction = NULL; + } + else + { + item = Actions[prs->state->state].action; + Assert(item != NULL); + } + + /* find action by character class */ + while (item->isclass) + { + prs->c = item->c; + if (item->isclass(prs) != 0) + break; + item++; + } + +#ifdef WPARSER_TRACE + { + TParserPosition *ptr; + + fprintf(stderr, "state "); + /* indent according to stack depth */ + for (ptr = prs->state->prev; ptr; ptr = ptr->prev) + fprintf(stderr, " "); + fprintf(stderr, "%s ", Actions[prs->state->state].state_name); + if (prs->state->posbyte < prs->lenstr) + fprintf(stderr, "at %c", *(prs->str + prs->state->posbyte)); + else + fprintf(stderr, "at EOF"); + fprintf(stderr, " matched rule %d flags%s%s%s%s%s%s%s%s%s%s%s\n", + (int) (item - Actions[prs->state->state].action), + (item->flags & A_BINGO) ? " BINGO" : "", + (item->flags & A_POP) ? " POP" : "", + (item->flags & A_PUSH) ? " PUSH" : "", + (item->flags & A_RERUN) ? " RERUN" : "", + (item->flags & A_CLEAR) ? " CLEAR" : "", + (item->flags & A_MERGE) ? " MERGE" : "", + (item->flags & A_CLRALL) ? " CLRALL" : "", + (item->tostate != TPS_Null) ? " tostate " : "", + (item->tostate != TPS_Null) ? Actions[item->tostate].state_name : "", + (item->type > 0) ? " type " : "", + tok_alias[item->type]); + } +#endif + + /* call special handler if exists */ + if (item->special) + item->special(prs); + + /* BINGO, token is found */ + if (item->flags & A_BINGO) + { + Assert(item->type > 0); + prs->lenbytetoken = prs->state->lenbytetoken; + prs->lenchartoken = prs->state->lenchartoken; + prs->state->lenbytetoken = prs->state->lenchartoken = 0; + prs->type = item->type; + } + + /* do various actions by flags */ + if (item->flags & A_POP) + { /* pop stored state in stack */ + TParserPosition *ptr = prs->state->prev; + + pfree(prs->state); + prs->state = ptr; + Assert(prs->state); + } + else if (item->flags & A_PUSH) + { /* push (store) state in stack */ + prs->state->pushedAtAction = item; /* remember where we push */ + prs->state = newTParserPosition(prs->state); + } + else if (item->flags & A_CLEAR) + { /* clear previous pushed state */ + TParserPosition *ptr; + + Assert(prs->state->prev); + ptr = prs->state->prev->prev; + pfree(prs->state->prev); + prs->state->prev = ptr; + } + else if (item->flags & A_CLRALL) + { /* clear all previous pushed state */ + TParserPosition *ptr; + + while (prs->state->prev) + { + ptr = prs->state->prev->prev; + pfree(prs->state->prev); + prs->state->prev = ptr; + } + } + else if (item->flags & A_MERGE) + { /* merge posinfo with current and pushed state */ + TParserPosition *ptr = prs->state; + + Assert(prs->state->prev); + prs->state = prs->state->prev; + + prs->state->posbyte = ptr->posbyte; + prs->state->poschar = ptr->poschar; + prs->state->charlen = ptr->charlen; + prs->state->lenbytetoken = ptr->lenbytetoken; + prs->state->lenchartoken = ptr->lenchartoken; + pfree(ptr); + } + + /* set new state if pointed */ + if (item->tostate != TPS_Null) + prs->state->state = item->tostate; + + /* check for go away */ + if ((item->flags & A_BINGO) || + (prs->state->posbyte >= prs->lenstr && + (item->flags & A_RERUN) == 0)) + break; + + /* go to beginning of loop if we should rerun or we just restore state */ + if (item->flags & (A_RERUN | A_POP)) + continue; + + /* move forward */ + if (prs->state->charlen) + { + prs->state->posbyte += prs->state->charlen; + prs->state->lenbytetoken += prs->state->charlen; + prs->state->poschar++; + prs->state->lenchartoken++; + } + } + + return (item && (item->flags & A_BINGO)) ? true : false; +} + +Datum +prsd_lextype(PG_FUNCTION_ARGS) +{ + LexDescr *descr = (LexDescr *) palloc(sizeof(LexDescr) * (LASTNUM + 1)); + int i; + + for (i = 1; i <= LASTNUM; i++) + { + descr[i - 1].lexid = i; + descr[i - 1].alias = pstrdup(tok_alias[i]); + descr[i - 1].descr = pstrdup(lex_descr[i]); + } + + descr[LASTNUM].lexid = 0; + + PG_RETURN_POINTER(descr); +} + +Datum +prsd_start(PG_FUNCTION_ARGS) +{ + PG_RETURN_POINTER(TParserInit((char *) PG_GETARG_POINTER(0), PG_GETARG_INT32(1))); +} + +Datum +prsd_nexttoken(PG_FUNCTION_ARGS) +{ + TParser *p = (TParser *) PG_GETARG_POINTER(0); + char **t = (char **) PG_GETARG_POINTER(1); + int *tlen = (int *) PG_GETARG_POINTER(2); + + if (!TParserGet(p)) + PG_RETURN_INT32(0); + + *t = p->token; + *tlen = p->lenbytetoken; + + PG_RETURN_INT32(p->type); +} + +Datum +prsd_end(PG_FUNCTION_ARGS) +{ + TParser *p = (TParser *) PG_GETARG_POINTER(0); + + TParserClose(p); + PG_RETURN_VOID(); +} + + +/* + * ts_headline support begins here + */ + +/* token type classification macros */ +#define LEAVETOKEN(x) ( (x)==SPACE ) +#define COMPLEXTOKEN(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD ) +#define ENDPUNCTOKEN(x) ( (x)==SPACE ) + +#define TS_IDIGNORE(x) ( (x)==TAG_T || (x)==PROTOCOL || (x)==SPACE || (x)==XMLENTITY ) +#define HLIDREPLACE(x) ( (x)==TAG_T ) +#define HLIDSKIP(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD ) +#define XMLHLIDSKIP(x) ( (x)==URL_T || (x)==NUMHWORD || (x)==ASCIIHWORD || (x)==HWORD ) +#define NONWORDTOKEN(x) ( (x)==SPACE || HLIDREPLACE(x) || HLIDSKIP(x) ) +#define NOENDTOKEN(x) ( NONWORDTOKEN(x) || (x)==SCIENTIFIC || (x)==VERSIONNUMBER || (x)==DECIMAL_T || (x)==SIGNEDINT || (x)==UNSIGNEDINT || TS_IDIGNORE(x) ) + +/* + * Macros useful in headline selection. These rely on availability of + * "HeadlineParsedText *prs" describing some text, and "int shortword" + * describing the "short word" length parameter. + */ + +/* Interesting words are non-repeated search terms */ +#define INTERESTINGWORD(j) \ + (prs->words[j].item && !prs->words[j].repeated) + +/* Don't want to end at a non-word or a short word, unless interesting */ +#define BADENDPOINT(j) \ + ((NOENDTOKEN(prs->words[j].type) || prs->words[j].len <= shortword) && \ + !INTERESTINGWORD(j)) + +typedef struct +{ + /* one cover (well, really one fragment) for mark_hl_fragments */ + int32 startpos; /* fragment's starting word index */ + int32 endpos; /* ending word index (inclusive) */ + int32 poslen; /* number of interesting words */ + int32 curlen; /* total number of words */ + bool chosen; /* chosen? */ + bool excluded; /* excluded? */ +} CoverPos; + +typedef struct +{ + /* callback data for checkcondition_HL */ + HeadlineWordEntry *words; + int len; +} hlCheck; + + +/* + * TS_execute callback for matching a tsquery operand to headline words + */ +static TSTernaryValue +checkcondition_HL(void *opaque, QueryOperand *val, ExecPhraseData *data) +{ + hlCheck *checkval = (hlCheck *) opaque; + int i; + + /* scan words array for marching items */ + for (i = 0; i < checkval->len; i++) + { + if (checkval->words[i].item == val) + { + /* if data == NULL, don't need to report positions */ + if (!data) + return TS_YES; + + if (!data->pos) + { + data->pos = palloc(sizeof(WordEntryPos) * checkval->len); + data->allocated = true; + data->npos = 1; + data->pos[0] = checkval->words[i].pos; + } + else if (data->pos[data->npos - 1] < checkval->words[i].pos) + { + data->pos[data->npos++] = checkval->words[i].pos; + } + } + } + + if (data && data->npos > 0) + return TS_YES; + + return TS_NO; +} + +/* + * hlFirstIndex: find first index >= pos containing any word used in query + * + * Returns -1 if no such index + */ +static int +hlFirstIndex(HeadlineParsedText *prs, int pos) +{ + int i; + + for (i = pos; i < prs->curwords; i++) + { + if (prs->words[i].item != NULL) + return i; + } + return -1; +} + +/* + * hlCover: try to find a substring of prs' word list that satisfies query + * + * At entry, *p must be the first word index to consider (initialize this + * to zero, or to the next index after a previous successful search). + * We will consider all substrings starting at or after that word, and + * containing no more than max_cover words. (We need a length limit to + * keep this from taking O(N^2) time for a long document with many query + * words but few complete matches. Actually, since checkcondition_HL is + * roughly O(N) in the length of the substring being checked, it's even + * worse than that.) + * + * On success, sets *p to first word index and *q to last word index of the + * cover substring, and returns true. + * + * The result is a minimal cover, in the sense that both *p and *q will be + * words used in the query. + */ +static bool +hlCover(HeadlineParsedText *prs, TSQuery query, int max_cover, + int *p, int *q) +{ + int pmin, + pmax, + nextpmin, + nextpmax; + hlCheck ch; + + /* + * We look for the earliest, shortest substring of prs->words that + * satisfies the query. Both the pmin and pmax indices must be words + * appearing in the query; there's no point in trying endpoints in between + * such points. + */ + pmin = hlFirstIndex(prs, *p); + while (pmin >= 0) + { + /* This useless assignment just keeps stupider compilers quiet */ + nextpmin = -1; + /* Consider substrings starting at pmin */ + ch.words = &(prs->words[pmin]); + /* Consider the length-one substring first, then longer substrings */ + pmax = pmin; + do + { + /* Try to match query against pmin .. pmax substring */ + ch.len = pmax - pmin + 1; + if (TS_execute(GETQUERY(query), &ch, + TS_EXEC_EMPTY, checkcondition_HL)) + { + *p = pmin; + *q = pmax; + return true; + } + /* Nope, so advance pmax to next feasible endpoint */ + nextpmax = hlFirstIndex(prs, pmax + 1); + + /* + * If this is our first advance past pmin, then the result is also + * the next feasible value of pmin; remember it to save a + * redundant search. + */ + if (pmax == pmin) + nextpmin = nextpmax; + pmax = nextpmax; + } + while (pmax >= 0 && pmax - pmin < max_cover); + /* No luck here, so try next feasible startpoint */ + pmin = nextpmin; + } + return false; +} + +/* + * Apply suitable highlight marking to words selected by headline selector + * + * The words from startpos to endpos inclusive are marked per highlightall + */ +static void +mark_fragment(HeadlineParsedText *prs, bool highlightall, + int startpos, int endpos) +{ + int i; + + for (i = startpos; i <= endpos; i++) + { + if (prs->words[i].item) + prs->words[i].selected = 1; + if (!highlightall) + { + if (HLIDREPLACE(prs->words[i].type)) + prs->words[i].replace = 1; + else if (HLIDSKIP(prs->words[i].type)) + prs->words[i].skip = 1; + } + else + { + if (XMLHLIDSKIP(prs->words[i].type)) + prs->words[i].skip = 1; + } + + prs->words[i].in = (prs->words[i].repeated) ? 0 : 1; + } +} + +/* + * split a cover substring into fragments not longer than max_words + * + * At entry, *startpos and *endpos are the (remaining) bounds of the cover + * substring. They are updated to hold the bounds of the next fragment. + * + * *curlen and *poslen are set to the fragment's length, in words and + * interesting words respectively. + */ +static void +get_next_fragment(HeadlineParsedText *prs, int *startpos, int *endpos, + int *curlen, int *poslen, int max_words) +{ + int i; + + /* + * Objective: select a fragment of words between startpos and endpos such + * that it has at most max_words and both ends have query words. If the + * startpos and endpos are the endpoints of the cover and the cover has + * fewer words than max_words, then this function should just return the + * cover + */ + /* first move startpos to an item */ + for (i = *startpos; i <= *endpos; i++) + { + *startpos = i; + if (INTERESTINGWORD(i)) + break; + } + /* cut endpos to have only max_words */ + *curlen = 0; + *poslen = 0; + for (i = *startpos; i <= *endpos && *curlen < max_words; i++) + { + if (!NONWORDTOKEN(prs->words[i].type)) + *curlen += 1; + if (INTERESTINGWORD(i)) + *poslen += 1; + } + /* if the cover was cut then move back endpos to a query item */ + if (*endpos > i) + { + *endpos = i; + for (i = *endpos; i >= *startpos; i--) + { + *endpos = i; + if (INTERESTINGWORD(i)) + break; + if (!NONWORDTOKEN(prs->words[i].type)) + *curlen -= 1; + } + } +} + +/* + * Headline selector used when MaxFragments > 0 + * + * Note: in this mode, highlightall is disregarded for phrase selection; + * it only controls presentation details. + */ +static void +mark_hl_fragments(HeadlineParsedText *prs, TSQuery query, bool highlightall, + int shortword, int min_words, + int max_words, int max_fragments, int max_cover) +{ + int32 poslen, + curlen, + i, + f, + num_f = 0; + int32 stretch, + maxstretch, + posmarker; + + int32 startpos = 0, + endpos = 0, + p = 0, + q = 0; + + int32 numcovers = 0, + maxcovers = 32; + + int32 minI, + minwords, + maxitems; + CoverPos *covers; + + covers = palloc(maxcovers * sizeof(CoverPos)); + + /* get all covers */ + while (hlCover(prs, query, max_cover, &p, &q)) + { + startpos = p; + endpos = q; + + /* + * Break the cover into smaller fragments such that each fragment has + * at most max_words. Also ensure that each end of each fragment is a + * query word. This will allow us to stretch the fragment in either + * direction + */ + + while (startpos <= endpos) + { + get_next_fragment(prs, &startpos, &endpos, &curlen, &poslen, max_words); + if (numcovers >= maxcovers) + { + maxcovers *= 2; + covers = repalloc(covers, sizeof(CoverPos) * maxcovers); + } + covers[numcovers].startpos = startpos; + covers[numcovers].endpos = endpos; + covers[numcovers].curlen = curlen; + covers[numcovers].poslen = poslen; + covers[numcovers].chosen = false; + covers[numcovers].excluded = false; + numcovers++; + startpos = endpos + 1; + endpos = q; + } + + /* move p to generate the next cover */ + p++; + } + + /* choose best covers */ + for (f = 0; f < max_fragments; f++) + { + maxitems = 0; + minwords = PG_INT32_MAX; + minI = -1; + + /* + * Choose the cover that contains max items. In case of tie choose the + * one with smaller number of words. + */ + for (i = 0; i < numcovers; i++) + { + if (!covers[i].chosen && !covers[i].excluded && + (maxitems < covers[i].poslen || + (maxitems == covers[i].poslen && + minwords > covers[i].curlen))) + { + maxitems = covers[i].poslen; + minwords = covers[i].curlen; + minI = i; + } + } + /* if a cover was found mark it */ + if (minI >= 0) + { + covers[minI].chosen = true; + /* adjust the size of cover */ + startpos = covers[minI].startpos; + endpos = covers[minI].endpos; + curlen = covers[minI].curlen; + /* stretch the cover if cover size is lower than max_words */ + if (curlen < max_words) + { + /* divide the stretch on both sides of cover */ + maxstretch = (max_words - curlen) / 2; + + /* + * first stretch the startpos stop stretching if 1. we hit the + * beginning of document 2. exceed maxstretch 3. we hit an + * already marked fragment + */ + stretch = 0; + posmarker = startpos; + for (i = startpos - 1; i >= 0 && stretch < maxstretch && !prs->words[i].in; i--) + { + if (!NONWORDTOKEN(prs->words[i].type)) + { + curlen++; + stretch++; + } + posmarker = i; + } + /* cut back startpos till we find a good endpoint */ + for (i = posmarker; i < startpos && BADENDPOINT(i); i++) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen--; + } + startpos = i; + /* now stretch the endpos as much as possible */ + posmarker = endpos; + for (i = endpos + 1; i < prs->curwords && curlen < max_words && !prs->words[i].in; i++) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen++; + posmarker = i; + } + /* cut back endpos till we find a good endpoint */ + for (i = posmarker; i > endpos && BADENDPOINT(i); i--) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen--; + } + endpos = i; + } + covers[minI].startpos = startpos; + covers[minI].endpos = endpos; + covers[minI].curlen = curlen; + /* Mark the chosen fragments (covers) */ + mark_fragment(prs, highlightall, startpos, endpos); + num_f++; + /* Exclude covers overlapping this one from future consideration */ + for (i = 0; i < numcovers; i++) + { + if (i != minI && + ((covers[i].startpos >= startpos && + covers[i].startpos <= endpos) || + (covers[i].endpos >= startpos && + covers[i].endpos <= endpos) || + (covers[i].startpos < startpos && + covers[i].endpos > endpos))) + covers[i].excluded = true; + } + } + else + break; /* no selectable covers remain */ + } + + /* show the first min_words words if we have not marked anything */ + if (num_f <= 0) + { + startpos = endpos = curlen = 0; + for (i = 0; i < prs->curwords && curlen < min_words; i++) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen++; + endpos = i; + } + mark_fragment(prs, highlightall, startpos, endpos); + } + + pfree(covers); +} + +/* + * Headline selector used when MaxFragments == 0 + */ +static void +mark_hl_words(HeadlineParsedText *prs, TSQuery query, bool highlightall, + int shortword, int min_words, int max_words, int max_cover) +{ + int p = 0, + q = 0; + int bestb = -1, + beste = -1; + int bestlen = -1; + bool bestcover = false; + int pose, + posb, + poslen, + curlen; + bool poscover; + int i; + + if (!highlightall) + { + /* examine all covers, select a headline using the best one */ + while (hlCover(prs, query, max_cover, &p, &q)) + { + /* + * Count words (curlen) and interesting words (poslen) within + * cover, but stop once we reach max_words. This step doesn't + * consider whether that's a good stopping point. posb and pose + * are set to the start and end indexes of the possible headline. + */ + curlen = 0; + poslen = 0; + posb = pose = p; + for (i = p; i <= q && curlen < max_words; i++) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen++; + if (INTERESTINGWORD(i)) + poslen++; + pose = i; + } + + if (curlen < max_words) + { + /* + * We have room to lengthen the headline, so search forward + * until it's full or we find a good stopping point. We'll + * reconsider the word at "q", then move forward. + */ + for (i = i - 1; i < prs->curwords && curlen < max_words; i++) + { + if (i > q) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen++; + if (INTERESTINGWORD(i)) + poslen++; + } + pose = i; + if (BADENDPOINT(i)) + continue; + if (curlen >= min_words) + break; + } + if (curlen < min_words) + { + /* + * Reached end of text and our headline is still shorter + * than min_words, so try to extend it to the left. + */ + for (i = p - 1; i >= 0; i--) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen++; + if (INTERESTINGWORD(i)) + poslen++; + if (curlen >= max_words) + break; + if (BADENDPOINT(i)) + continue; + if (curlen >= min_words) + break; + } + posb = (i >= 0) ? i : 0; + } + } + else + { + /* + * Can't make headline longer, so consider making it shorter + * if needed to avoid a bad endpoint. + */ + if (i > q) + i = q; + for (; curlen > min_words; i--) + { + if (!BADENDPOINT(i)) + break; + if (!NONWORDTOKEN(prs->words[i].type)) + curlen--; + if (INTERESTINGWORD(i)) + poslen--; + pose = i - 1; + } + } + + /* + * Check whether the proposed headline includes the original + * cover; it might not if we trimmed it due to max_words. + */ + poscover = (posb <= p && pose >= q); + + /* + * Adopt this headline if it's better than the last one, giving + * highest priority to headlines including the cover, then to + * headlines with more interesting words, then to headlines with + * good stopping points. (Since bestlen is initially -1, we will + * certainly adopt the first headline.) + */ + if (poscover > bestcover || + (poscover == bestcover && poslen > bestlen) || + (poscover == bestcover && poslen == bestlen && + !BADENDPOINT(pose) && BADENDPOINT(beste))) + { + bestb = posb; + beste = pose; + bestlen = poslen; + bestcover = poscover; + } + + /* move p to generate the next cover */ + p++; + } + + /* + * If we found nothing acceptable, select min_words words starting at + * the beginning. + */ + if (bestlen < 0) + { + curlen = 0; + pose = 0; + for (i = 0; i < prs->curwords && curlen < min_words; i++) + { + if (!NONWORDTOKEN(prs->words[i].type)) + curlen++; + pose = i; + } + bestb = 0; + beste = pose; + } + } + else + { + /* highlightall mode: headline is whole document */ + bestb = 0; + beste = prs->curwords - 1; + } + + mark_fragment(prs, highlightall, bestb, beste); +} + +/* + * Default parser's prsheadline function + */ +Datum +prsd_headline(PG_FUNCTION_ARGS) +{ + HeadlineParsedText *prs = (HeadlineParsedText *) PG_GETARG_POINTER(0); + List *prsoptions = (List *) PG_GETARG_POINTER(1); + TSQuery query = PG_GETARG_TSQUERY(2); + + /* default option values: */ + int min_words = 15; + int max_words = 35; + int shortword = 3; + int max_fragments = 0; + bool highlightall = false; + int max_cover; + ListCell *l; + + /* Extract configuration option values */ + prs->startsel = NULL; + prs->stopsel = NULL; + prs->fragdelim = NULL; + foreach(l, prsoptions) + { + DefElem *defel = (DefElem *) lfirst(l); + char *val = defGetString(defel); + + if (pg_strcasecmp(defel->defname, "MaxWords") == 0) + max_words = pg_strtoint32(val); + else if (pg_strcasecmp(defel->defname, "MinWords") == 0) + min_words = pg_strtoint32(val); + else if (pg_strcasecmp(defel->defname, "ShortWord") == 0) + shortword = pg_strtoint32(val); + else if (pg_strcasecmp(defel->defname, "MaxFragments") == 0) + max_fragments = pg_strtoint32(val); + else if (pg_strcasecmp(defel->defname, "StartSel") == 0) + prs->startsel = pstrdup(val); + else if (pg_strcasecmp(defel->defname, "StopSel") == 0) + prs->stopsel = pstrdup(val); + else if (pg_strcasecmp(defel->defname, "FragmentDelimiter") == 0) + prs->fragdelim = pstrdup(val); + else if (pg_strcasecmp(defel->defname, "HighlightAll") == 0) + highlightall = (pg_strcasecmp(val, "1") == 0 || + pg_strcasecmp(val, "on") == 0 || + pg_strcasecmp(val, "true") == 0 || + pg_strcasecmp(val, "t") == 0 || + pg_strcasecmp(val, "y") == 0 || + pg_strcasecmp(val, "yes") == 0); + else + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("unrecognized headline parameter: \"%s\"", + defel->defname))); + } + + /* + * We might eventually make max_cover a user-settable parameter, but for + * now, just compute a reasonable value based on max_words and + * max_fragments. + */ + max_cover = Max(max_words * 10, 100); + if (max_fragments > 0) + max_cover *= max_fragments; + + /* in HighlightAll mode these parameters are ignored */ + if (!highlightall) + { + if (min_words >= max_words) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("MinWords should be less than MaxWords"))); + if (min_words <= 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("MinWords should be positive"))); + if (shortword < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("ShortWord should be >= 0"))); + if (max_fragments < 0) + ereport(ERROR, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("MaxFragments should be >= 0"))); + } + + /* Apply appropriate headline selector */ + if (max_fragments == 0) + mark_hl_words(prs, query, highlightall, shortword, + min_words, max_words, max_cover); + else + mark_hl_fragments(prs, query, highlightall, shortword, + min_words, max_words, max_fragments, max_cover); + + /* Fill in default values for string options */ + if (!prs->startsel) + prs->startsel = pstrdup("<b>"); + if (!prs->stopsel) + prs->stopsel = pstrdup("</b>"); + if (!prs->fragdelim) + prs->fragdelim = pstrdup(" ... "); + + /* Caller will need these lengths, too */ + prs->startsellen = strlen(prs->startsel); + prs->stopsellen = strlen(prs->stopsel); + prs->fragdelimlen = strlen(prs->fragdelim); + + PG_RETURN_POINTER(prs); +} |