diff options
Diffstat (limited to 'ext/fts1/fts1.c')
| -rw-r--r-- | ext/fts1/fts1.c | 3348 |
1 files changed, 3348 insertions, 0 deletions
diff --git a/ext/fts1/fts1.c b/ext/fts1/fts1.c new file mode 100644 index 0000000..77fa9e2 --- /dev/null +++ b/ext/fts1/fts1.c @@ -0,0 +1,3348 @@ +/* fts1 has a design flaw which can lead to database corruption (see +** below). It is recommended not to use it any longer, instead use +** fts3 (or higher). If you believe that your use of fts1 is safe, +** add -DSQLITE_ENABLE_BROKEN_FTS1=1 to your CFLAGS. +*/ +#if (!defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1)) \ + && !defined(SQLITE_ENABLE_BROKEN_FTS1) +#error fts1 has a design flaw and has been deprecated. +#endif +/* The flaw is that fts1 uses the content table's unaliased rowid as +** the unique docid. fts1 embeds the rowid in the index it builds, +** and expects the rowid to not change. The SQLite VACUUM operation +** will renumber such rowids, thereby breaking fts1. If you are using +** fts1 in a system which has disabled VACUUM, then you can continue +** to use it safely. Note that PRAGMA auto_vacuum does NOT disable +** VACUUM, though systems using auto_vacuum are unlikely to invoke +** VACUUM. +** +** fts1 should be safe even across VACUUM if you only insert documents +** and never delete. +*/ + +/* The author disclaims copyright to this source code. + * + * This is an SQLite module implementing full-text search. + */ + +/* +** The code in this file is only compiled if: +** +** * The FTS1 module is being built as an extension +** (in which case SQLITE_CORE is not defined), or +** +** * The FTS1 module is being built into the core of +** SQLite (in which case SQLITE_ENABLE_FTS1 is defined). +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) + +#if defined(SQLITE_ENABLE_FTS1) && !defined(SQLITE_CORE) +# define SQLITE_CORE 1 +#endif + +#include <assert.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <ctype.h> + +#include "fts1.h" +#include "fts1_hash.h" +#include "fts1_tokenizer.h" +#include "sqlite3.h" +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + + +#if 0 +# define TRACE(A) printf A; fflush(stdout) +#else +# define TRACE(A) +#endif + +/* utility functions */ + +typedef struct StringBuffer { + int len; /* length, not including null terminator */ + int alloced; /* Space allocated for s[] */ + char *s; /* Content of the string */ +} StringBuffer; + +static void initStringBuffer(StringBuffer *sb){ + sb->len = 0; + sb->alloced = 100; + sb->s = malloc(100); + sb->s[0] = '\0'; +} + +static void nappend(StringBuffer *sb, const char *zFrom, int nFrom){ + if( sb->len + nFrom >= sb->alloced ){ + sb->alloced = sb->len + nFrom + 100; + sb->s = realloc(sb->s, sb->alloced+1); + if( sb->s==0 ){ + initStringBuffer(sb); + return; + } + } + memcpy(sb->s + sb->len, zFrom, nFrom); + sb->len += nFrom; + sb->s[sb->len] = 0; +} +static void append(StringBuffer *sb, const char *zFrom){ + nappend(sb, zFrom, strlen(zFrom)); +} + +/* We encode variable-length integers in little-endian order using seven bits + * per byte as follows: +** +** KEY: +** A = 0xxxxxxx 7 bits of data and one flag bit +** B = 1xxxxxxx 7 bits of data and one flag bit +** +** 7 bits - A +** 14 bits - BA +** 21 bits - BBA +** and so on. +*/ + +/* We may need up to VARINT_MAX bytes to store an encoded 64-bit integer. */ +#define VARINT_MAX 10 + +/* Write a 64-bit variable-length integer to memory starting at p[0]. + * The length of data written will be between 1 and VARINT_MAX bytes. + * The number of bytes written is returned. */ +static int putVarint(char *p, sqlite_int64 v){ + unsigned char *q = (unsigned char *) p; + sqlite_uint64 vu = v; + do{ + *q++ = (unsigned char) ((vu & 0x7f) | 0x80); + vu >>= 7; + }while( vu!=0 ); + q[-1] &= 0x7f; /* turn off high bit in final byte */ + assert( q - (unsigned char *)p <= VARINT_MAX ); + return (int) (q - (unsigned char *)p); +} + +/* Read a 64-bit variable-length integer from memory starting at p[0]. + * Return the number of bytes read, or 0 on error. + * The value is stored in *v. */ +static int getVarint(const char *p, sqlite_int64 *v){ + const unsigned char *q = (const unsigned char *) p; + sqlite_uint64 x = 0, y = 1; + while( (*q & 0x80) == 0x80 ){ + x += y * (*q++ & 0x7f); + y <<= 7; + if( q - (unsigned char *)p >= VARINT_MAX ){ /* bad data */ + assert( 0 ); + return 0; + } + } + x += y * (*q++); + *v = (sqlite_int64) x; + return (int) (q - (unsigned char *)p); +} + +static int getVarint32(const char *p, int *pi){ + sqlite_int64 i; + int ret = getVarint(p, &i); + *pi = (int) i; + assert( *pi==i ); + return ret; +} + +/*** Document lists *** + * + * A document list holds a sorted list of varint-encoded document IDs. + * + * A doclist with type DL_POSITIONS_OFFSETS is stored like this: + * + * array { + * varint docid; + * array { + * varint position; (delta from previous position plus POS_BASE) + * varint startOffset; (delta from previous startOffset) + * varint endOffset; (delta from startOffset) + * } + * } + * + * Here, array { X } means zero or more occurrences of X, adjacent in memory. + * + * A position list may hold positions for text in multiple columns. A position + * POS_COLUMN is followed by a varint containing the index of the column for + * following positions in the list. Any positions appearing before any + * occurrences of POS_COLUMN are for column 0. + * + * A doclist with type DL_POSITIONS is like the above, but holds only docids + * and positions without offset information. + * + * A doclist with type DL_DOCIDS is like the above, but holds only docids + * without positions or offset information. + * + * On disk, every document list has positions and offsets, so we don't bother + * to serialize a doclist's type. + * + * We don't yet delta-encode document IDs; doing so will probably be a + * modest win. + * + * NOTE(shess) I've thought of a slightly (1%) better offset encoding. + * After the first offset, estimate the next offset by using the + * current token position and the previous token position and offset, + * offset to handle some variance. So the estimate would be + * (iPosition*w->iStartOffset/w->iPosition-64), which is delta-encoded + * as normal. Offsets more than 64 chars from the estimate are + * encoded as the delta to the previous start offset + 128. An + * additional tiny increment can be gained by using the end offset of + * the previous token to make the estimate a tiny bit more precise. +*/ + +/* It is not safe to call isspace(), tolower(), or isalnum() on +** hi-bit-set characters. This is the same solution used in the +** tokenizer. +*/ +/* TODO(shess) The snippet-generation code should be using the +** tokenizer-generated tokens rather than doing its own local +** tokenization. +*/ +/* TODO(shess) Is __isascii() a portable version of (c&0x80)==0? */ +static int safe_isspace(char c){ + return (c&0x80)==0 ? isspace((unsigned char)c) : 0; +} +static int safe_tolower(char c){ + return (c&0x80)==0 ? tolower((unsigned char)c) : c; +} +static int safe_isalnum(char c){ + return (c&0x80)==0 ? isalnum((unsigned char)c) : 0; +} + +typedef enum DocListType { + DL_DOCIDS, /* docids only */ + DL_POSITIONS, /* docids + positions */ + DL_POSITIONS_OFFSETS /* docids + positions + offsets */ +} DocListType; + +/* +** By default, only positions and not offsets are stored in the doclists. +** To change this so that offsets are stored too, compile with +** +** -DDL_DEFAULT=DL_POSITIONS_OFFSETS +** +*/ +#ifndef DL_DEFAULT +# define DL_DEFAULT DL_POSITIONS +#endif + +typedef struct DocList { + char *pData; + int nData; + DocListType iType; + int iLastColumn; /* the last column written */ + int iLastPos; /* the last position written */ + int iLastOffset; /* the last start offset written */ +} DocList; + +enum { + POS_END = 0, /* end of this position list */ + POS_COLUMN, /* followed by new column number */ + POS_BASE +}; + +/* Initialize a new DocList to hold the given data. */ +static void docListInit(DocList *d, DocListType iType, + const char *pData, int nData){ + d->nData = nData; + if( nData>0 ){ + d->pData = malloc(nData); + memcpy(d->pData, pData, nData); + } else { + d->pData = NULL; + } + d->iType = iType; + d->iLastColumn = 0; + d->iLastPos = d->iLastOffset = 0; +} + +/* Create a new dynamically-allocated DocList. */ +static DocList *docListNew(DocListType iType){ + DocList *d = (DocList *) malloc(sizeof(DocList)); + docListInit(d, iType, 0, 0); + return d; +} + +static void docListDestroy(DocList *d){ + free(d->pData); +#ifndef NDEBUG + memset(d, 0x55, sizeof(*d)); +#endif +} + +static void docListDelete(DocList *d){ + docListDestroy(d); + free(d); +} + +static char *docListEnd(DocList *d){ + return d->pData + d->nData; +} + +/* Append a varint to a DocList's data. */ +static void appendVarint(DocList *d, sqlite_int64 i){ + char c[VARINT_MAX]; + int n = putVarint(c, i); + d->pData = realloc(d->pData, d->nData + n); + memcpy(d->pData + d->nData, c, n); + d->nData += n; +} + +static void docListAddDocid(DocList *d, sqlite_int64 iDocid){ + appendVarint(d, iDocid); + if( d->iType>=DL_POSITIONS ){ + appendVarint(d, POS_END); /* initially empty position list */ + d->iLastColumn = 0; + d->iLastPos = d->iLastOffset = 0; + } +} + +/* helper function for docListAddPos and docListAddPosOffset */ +static void addPos(DocList *d, int iColumn, int iPos){ + assert( d->nData>0 ); + --d->nData; /* remove previous terminator */ + if( iColumn!=d->iLastColumn ){ + assert( iColumn>d->iLastColumn ); + appendVarint(d, POS_COLUMN); + appendVarint(d, iColumn); + d->iLastColumn = iColumn; + d->iLastPos = d->iLastOffset = 0; + } + assert( iPos>=d->iLastPos ); + appendVarint(d, iPos-d->iLastPos+POS_BASE); + d->iLastPos = iPos; +} + +/* Add a position to the last position list in a doclist. */ +static void docListAddPos(DocList *d, int iColumn, int iPos){ + assert( d->iType==DL_POSITIONS ); + addPos(d, iColumn, iPos); + appendVarint(d, POS_END); /* add new terminator */ +} + +/* +** Add a position and starting and ending offsets to a doclist. +** +** If the doclist is setup to handle only positions, then insert +** the position only and ignore the offsets. +*/ +static void docListAddPosOffset( + DocList *d, /* Doclist under construction */ + int iColumn, /* Column the inserted term is part of */ + int iPos, /* Position of the inserted term */ + int iStartOffset, /* Starting offset of inserted term */ + int iEndOffset /* Ending offset of inserted term */ +){ + assert( d->iType>=DL_POSITIONS ); + addPos(d, iColumn, iPos); + if( d->iType==DL_POSITIONS_OFFSETS ){ + assert( iStartOffset>=d->iLastOffset ); + appendVarint(d, iStartOffset-d->iLastOffset); + d->iLastOffset = iStartOffset; + assert( iEndOffset>=iStartOffset ); + appendVarint(d, iEndOffset-iStartOffset); + } + appendVarint(d, POS_END); /* add new terminator */ +} + +/* +** A DocListReader object is a cursor into a doclist. Initialize +** the cursor to the beginning of the doclist by calling readerInit(). +** Then use routines +** +** peekDocid() +** readDocid() +** readPosition() +** skipPositionList() +** and so forth... +** +** to read information out of the doclist. When we reach the end +** of the doclist, atEnd() returns TRUE. +*/ +typedef struct DocListReader { + DocList *pDoclist; /* The document list we are stepping through */ + char *p; /* Pointer to next unread byte in the doclist */ + int iLastColumn; + int iLastPos; /* the last position read, or -1 when not in a position list */ +} DocListReader; + +/* +** Initialize the DocListReader r to point to the beginning of pDoclist. +*/ +static void readerInit(DocListReader *r, DocList *pDoclist){ + r->pDoclist = pDoclist; + if( pDoclist!=NULL ){ + r->p = pDoclist->pData; + } + r->iLastColumn = -1; + r->iLastPos = -1; +} + +/* +** Return TRUE if we have reached then end of pReader and there is +** nothing else left to read. +*/ +static int atEnd(DocListReader *pReader){ + return pReader->pDoclist==0 || (pReader->p >= docListEnd(pReader->pDoclist)); +} + +/* Peek at the next docid without advancing the read pointer. +*/ +static sqlite_int64 peekDocid(DocListReader *pReader){ + sqlite_int64 ret; + assert( !atEnd(pReader) ); + assert( pReader->iLastPos==-1 ); + getVarint(pReader->p, &ret); + return ret; +} + +/* Read the next docid. See also nextDocid(). +*/ +static sqlite_int64 readDocid(DocListReader *pReader){ + sqlite_int64 ret; + assert( !atEnd(pReader) ); + assert( pReader->iLastPos==-1 ); + pReader->p += getVarint(pReader->p, &ret); + if( pReader->pDoclist->iType>=DL_POSITIONS ){ + pReader->iLastColumn = 0; + pReader->iLastPos = 0; + } + return ret; +} + +/* Read the next position and column index from a position list. + * Returns the position, or -1 at the end of the list. */ +static int readPosition(DocListReader *pReader, int *iColumn){ + int i; + int iType = pReader->pDoclist->iType; + + if( pReader->iLastPos==-1 ){ + return -1; + } + assert( !atEnd(pReader) ); + + if( iType<DL_POSITIONS ){ + return -1; + } + pReader->p += getVarint32(pReader->p, &i); + if( i==POS_END ){ + pReader->iLastColumn = pReader->iLastPos = -1; + *iColumn = -1; + return -1; + } + if( i==POS_COLUMN ){ + pReader->p += getVarint32(pReader->p, &pReader->iLastColumn); + pReader->iLastPos = 0; + pReader->p += getVarint32(pReader->p, &i); + assert( i>=POS_BASE ); + } + pReader->iLastPos += ((int) i)-POS_BASE; + if( iType>=DL_POSITIONS_OFFSETS ){ + /* Skip over offsets, ignoring them for now. */ + int iStart, iEnd; + pReader->p += getVarint32(pReader->p, &iStart); + pReader->p += getVarint32(pReader->p, &iEnd); + } + *iColumn = pReader->iLastColumn; + return pReader->iLastPos; +} + +/* Skip past the end of a position list. */ +static void skipPositionList(DocListReader *pReader){ + DocList *p = pReader->pDoclist; + if( p && p->iType>=DL_POSITIONS ){ + int iColumn; + while( readPosition(pReader, &iColumn)!=-1 ){} + } +} + +/* Skip over a docid, including its position list if the doclist has + * positions. */ +static void skipDocument(DocListReader *pReader){ + readDocid(pReader); + skipPositionList(pReader); +} + +/* Skip past all docids which are less than [iDocid]. Returns 1 if a docid + * matching [iDocid] was found. */ +static int skipToDocid(DocListReader *pReader, sqlite_int64 iDocid){ + sqlite_int64 d = 0; + while( !atEnd(pReader) && (d=peekDocid(pReader))<iDocid ){ + skipDocument(pReader); + } + return !atEnd(pReader) && d==iDocid; +} + +/* Return the first document in a document list. +*/ +static sqlite_int64 firstDocid(DocList *d){ + DocListReader r; + readerInit(&r, d); + return readDocid(&r); +} + +#ifdef SQLITE_DEBUG +/* +** This routine is used for debugging purpose only. +** +** Write the content of a doclist to standard output. +*/ +static void printDoclist(DocList *p){ + DocListReader r; + const char *zSep = ""; + + readerInit(&r, p); + while( !atEnd(&r) ){ + sqlite_int64 docid = readDocid(&r); + if( docid==0 ){ + skipPositionList(&r); + continue; + } + printf("%s%lld", zSep, docid); + zSep = ","; + if( p->iType>=DL_POSITIONS ){ + int iPos, iCol; + const char *zDiv = ""; + printf("("); + while( (iPos = readPosition(&r, &iCol))>=0 ){ + printf("%s%d:%d", zDiv, iCol, iPos); + zDiv = ":"; + } + printf(")"); + } + } + printf("\n"); + fflush(stdout); +} +#endif /* SQLITE_DEBUG */ + +/* Trim the given doclist to contain only positions in column + * [iRestrictColumn]. */ +static void docListRestrictColumn(DocList *in, int iRestrictColumn){ + DocListReader r; + DocList out; + + assert( in->iType>=DL_POSITIONS ); + readerInit(&r, in); + docListInit(&out, DL_POSITIONS, NULL, 0); + + while( !atEnd(&r) ){ + sqlite_int64 iDocid = readDocid(&r); + int iPos, iColumn; + + docListAddDocid(&out, iDocid); + while( (iPos = readPosition(&r, &iColumn)) != -1 ){ + if( iColumn==iRestrictColumn ){ + docListAddPos(&out, iColumn, iPos); + } + } + } + + docListDestroy(in); + *in = out; +} + +/* Trim the given doclist by discarding any docids without any remaining + * positions. */ +static void docListDiscardEmpty(DocList *in) { + DocListReader r; + DocList out; + + /* TODO: It would be nice to implement this operation in place; that + * could save a significant amount of memory in queries with long doclists. */ + assert( in->iType>=DL_POSITIONS ); + readerInit(&r, in); + docListInit(&out, DL_POSITIONS, NULL, 0); + + while( !atEnd(&r) ){ + sqlite_int64 iDocid = readDocid(&r); + int match = 0; + int iPos, iColumn; + while( (iPos = readPosition(&r, &iColumn)) != -1 ){ + if( !match ){ + docListAddDocid(&out, iDocid); + match = 1; + } + docListAddPos(&out, iColumn, iPos); + } + } + + docListDestroy(in); + *in = out; +} + +/* Helper function for docListUpdate() and docListAccumulate(). +** Splices a doclist element into the doclist represented by r, +** leaving r pointing after the newly spliced element. +*/ +static void docListSpliceElement(DocListReader *r, sqlite_int64 iDocid, + const char *pSource, int nSource){ + DocList *d = r->pDoclist; + char *pTarget; + int nTarget, found; + + found = skipToDocid(r, iDocid); + + /* Describe slice in d to place pSource/nSource. */ + pTarget = r->p; + if( found ){ + skipDocument(r); + nTarget = r->p-pTarget; + }else{ + nTarget = 0; + } + + /* The sense of the following is that there are three possibilities. + ** If nTarget==nSource, we should not move any memory nor realloc. + ** If nTarget>nSource, trim target and realloc. + ** If nTarget<nSource, realloc then expand target. + */ + if( nTarget>nSource ){ + memmove(pTarget+nSource, pTarget+nTarget, docListEnd(d)-(pTarget+nTarget)); + } + if( nTarget!=nSource ){ + int iDoclist = pTarget-d->pData; + d->pData = realloc(d->pData, d->nData+nSource-nTarget); + pTarget = d->pData+iDoclist; + } + if( nTarget<nSource ){ + memmove(pTarget+nSource, pTarget+nTarget, docListEnd(d)-(pTarget+nTarget)); + } + + memcpy(pTarget, pSource, nSource); + d->nData += nSource-nTarget; + r->p = pTarget+nSource; +} + +/* Insert/update pUpdate into the doclist. */ +static void docListUpdate(DocList *d, DocList *pUpdate){ + DocListReader reader; + + assert( d!=NULL && pUpdate!=NULL ); + assert( d->iType==pUpdate->iType); + + readerInit(&reader, d); + docListSpliceElement(&reader, firstDocid(pUpdate), + pUpdate->pData, pUpdate->nData); +} + +/* Propagate elements from pUpdate to pAcc, overwriting elements with +** matching docids. +*/ +static void docListAccumulate(DocList *pAcc, DocList *pUpdate){ + DocListReader accReader, updateReader; + + /* Handle edge cases where one doclist is empty. */ + assert( pAcc!=NULL ); + if( pUpdate==NULL || pUpdate->nData==0 ) return; + if( pAcc->nData==0 ){ + pAcc->pData = malloc(pUpdate->nData); + memcpy(pAcc->pData, pUpdate->pData, pUpdate->nData); + pAcc->nData = pUpdate->nData; + return; + } + + readerInit(&accReader, pAcc); + readerInit(&updateReader, pUpdate); + + while( !atEnd(&updateReader) ){ + char *pSource = updateReader.p; + sqlite_int64 iDocid = readDocid(&updateReader); + skipPositionList(&updateReader); + docListSpliceElement(&accReader, iDocid, pSource, updateReader.p-pSource); + } +} + +/* +** Read the next docid off of pIn. Return 0 if we reach the end. +* +* TODO: This assumes that docids are never 0, but they may actually be 0 since +* users can choose docids when inserting into a full-text table. Fix this. +*/ +static sqlite_int64 nextDocid(DocListReader *pIn){ + skipPositionList(pIn); + return atEnd(pIn) ? 0 : readDocid(pIn); +} + +/* +** pLeft and pRight are two DocListReaders that are pointing to +** positions lists of the same document: iDocid. +** +** If there are no instances in pLeft or pRight where the position +** of pLeft is one less than the position of pRight, then this +** routine adds nothing to pOut. +** +** If there are one or more instances where positions from pLeft +** are exactly one less than positions from pRight, then add a new +** document record to pOut. If pOut wants to hold positions, then +** include the positions from pRight that are one more than a +** position in pLeft. In other words: pRight.iPos==pLeft.iPos+1. +** +** pLeft and pRight are left pointing at the next document record. +*/ +static void mergePosList( + DocListReader *pLeft, /* Left position list */ + DocListReader *pRight, /* Right position list */ + sqlite_int64 iDocid, /* The docid from pLeft and pRight */ + DocList *pOut /* Write the merged document record here */ +){ + int iLeftCol, iLeftPos = readPosition(pLeft, &iLeftCol); + int iRightCol, iRightPos = readPosition(pRight, &iRightCol); + int match = 0; + + /* Loop until we've reached the end of both position lists. */ + while( iLeftPos!=-1 && iRightPos!=-1 ){ + if( iLeftCol==iRightCol && iLeftPos+1==iRightPos ){ + if( !match ){ + docListAddDocid(pOut, iDocid); + match = 1; + } + if( pOut->iType>=DL_POSITIONS ){ + docListAddPos(pOut, iRightCol, iRightPos); + } + iLeftPos = readPosition(pLeft, &iLeftCol); + iRightPos = readPosition(pRight, &iRightCol); + }else if( iRightCol<iLeftCol || + (iRightCol==iLeftCol && iRightPos<iLeftPos+1) ){ + iRightPos = readPosition(pRight, &iRightCol); + }else{ + iLeftPos = readPosition(pLeft, &iLeftCol); + } + } + if( iLeftPos>=0 ) skipPositionList(pLeft); + if( iRightPos>=0 ) skipPositionList(pRight); +} + +/* We have two doclists: pLeft and pRight. +** Write the phrase intersection of these two doclists into pOut. +** +** A phrase intersection means that two documents only match +** if pLeft.iPos+1==pRight.iPos. +** +** The output pOut may or may not contain positions. If pOut +** does contain positions, they are the positions of pRight. +*/ +static void docListPhraseMerge( + DocList *pLeft, /* Doclist resulting from the words on the left */ + DocList *pRight, /* Doclist for the next word to the right */ + DocList *pOut /* Write the combined doclist here */ +){ + DocListReader left, right; + sqlite_int64 docidLeft, docidRight; + + readerInit(&left, pLeft); + readerInit(&right, pRight); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); + + while( docidLeft>0 && docidRight>0 ){ + if( docidLeft<docidRight ){ + docidLeft = nextDocid(&left); + }else if( docidRight<docidLeft ){ + docidRight = nextDocid(&right); + }else{ + mergePosList(&left, &right, docidLeft, pOut); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); + } + } +} + +/* We have two doclists: pLeft and pRight. +** Write the intersection of these two doclists into pOut. +** Only docids are matched. Position information is ignored. +** +** The output pOut never holds positions. +*/ +static void docListAndMerge( + DocList *pLeft, /* Doclist resulting from the words on the left */ + DocList *pRight, /* Doclist for the next word to the right */ + DocList *pOut /* Write the combined doclist here */ +){ + DocListReader left, right; + sqlite_int64 docidLeft, docidRight; + + assert( pOut->iType<DL_POSITIONS ); + + readerInit(&left, pLeft); + readerInit(&right, pRight); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); + + while( docidLeft>0 && docidRight>0 ){ + if( docidLeft<docidRight ){ + docidLeft = nextDocid(&left); + }else if( docidRight<docidLeft ){ + docidRight = nextDocid(&right); + }else{ + docListAddDocid(pOut, docidLeft); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); + } + } +} + +/* We have two doclists: pLeft and pRight. +** Write the union of these two doclists into pOut. +** Only docids are matched. Position information is ignored. +** +** The output pOut never holds positions. +*/ +static void docListOrMerge( + DocList *pLeft, /* Doclist resulting from the words on the left */ + DocList *pRight, /* Doclist for the next word to the right */ + DocList *pOut /* Write the combined doclist here */ +){ + DocListReader left, right; + sqlite_int64 docidLeft, docidRight, priorLeft; + + readerInit(&left, pLeft); + readerInit(&right, pRight); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); + + while( docidLeft>0 && docidRight>0 ){ + if( docidLeft<=docidRight ){ + docListAddDocid(pOut, docidLeft); + }else{ + docListAddDocid(pOut, docidRight); + } + priorLeft = docidLeft; + if( docidLeft<=docidRight ){ + docidLeft = nextDocid(&left); + } + if( docidRight>0 && docidRight<=priorLeft ){ + docidRight = nextDocid(&right); + } + } + while( docidLeft>0 ){ + docListAddDocid(pOut, docidLeft); + docidLeft = nextDocid(&left); + } + while( docidRight>0 ){ + docListAddDocid(pOut, docidRight); + docidRight = nextDocid(&right); + } +} + +/* We have two doclists: pLeft and pRight. +** Write into pOut all documents that occur in pLeft but not +** in pRight. +** +** Only docids are matched. Position information is ignored. +** +** The output pOut never holds positions. +*/ +static void docListExceptMerge( + DocList *pLeft, /* Doclist resulting from the words on the left */ + DocList *pRight, /* Doclist for the next word to the right */ + DocList *pOut /* Write the combined doclist here */ +){ + DocListReader left, right; + sqlite_int64 docidLeft, docidRight, priorLeft; + + readerInit(&left, pLeft); + readerInit(&right, pRight); + docidLeft = nextDocid(&left); + docidRight = nextDocid(&right); + + while( docidLeft>0 && docidRight>0 ){ + priorLeft = docidLeft; + if( docidLeft<docidRight ){ + docListAddDocid(pOut, docidLeft); + } + if( docidLeft<=docidRight ){ + docidLeft = nextDocid(&left); + } + if( docidRight>0 && docidRight<=priorLeft ){ + docidRight = nextDocid(&right); + } + } + while( docidLeft>0 ){ + docListAddDocid(pOut, docidLeft); + docidLeft = nextDocid(&left); + } +} + +static char *string_dup_n(const char *s, int n){ + char *str = malloc(n + 1); + memcpy(str, s, n); + str[n] = '\0'; + return str; +} + +/* Duplicate a string; the caller must free() the returned string. + * (We don't use strdup() since it is not part of the standard C library and + * may not be available everywhere.) */ +static char *string_dup(const char *s){ + return string_dup_n(s, strlen(s)); +} + +/* Format a string, replacing each occurrence of the % character with + * zDb.zName. This may be more convenient than sqlite_mprintf() + * when one string is used repeatedly in a format string. + * The caller must free() the returned string. */ +static char *string_format(const char *zFormat, + const char *zDb, const char *zName){ + const char *p; + size_t len = 0; + size_t nDb = strlen(zDb); + size_t nName = strlen(zName); + size_t nFullTableName = nDb+1+nName; + char *result; + char *r; + + /* first compute length needed */ + for(p = zFormat ; *p ; ++p){ + len += (*p=='%' ? nFullTableName : 1); + } + len += 1; /* for null terminator */ + + r = result = malloc(len); + for(p = zFormat; *p; ++p){ + if( *p=='%' ){ + memcpy(r, zDb, nDb); + r += nDb; + *r++ = '.'; + memcpy(r, zName, nName); + r += nName; + } else { + *r++ = *p; + } + } + *r++ = '\0'; + assert( r == result + len ); + return result; +} + +static int sql_exec(sqlite3 *db, const char *zDb, const char *zName, + const char *zFormat){ + char *zCommand = string_format(zFormat, zDb, zName); + int rc; + TRACE(("FTS1 sql: %s\n", zCommand)); + rc = sqlite3_exec(db, zCommand, NULL, 0, NULL); + free(zCommand); + return rc; +} + +static int sql_prepare(sqlite3 *db, const char *zDb, const char *zName, + sqlite3_stmt **ppStmt, const char *zFormat){ + char *zCommand = string_format(zFormat, zDb, zName); + int rc; + TRACE(("FTS1 prepare: %s\n", zCommand)); + rc = sqlite3_prepare(db, zCommand, -1, ppStmt, NULL); + free(zCommand); + return rc; +} + +/* end utility functions */ + +/* Forward reference */ +typedef struct fulltext_vtab fulltext_vtab; + +/* A single term in a query is represented by an instances of +** the following structure. +*/ +typedef struct QueryTerm { + short int nPhrase; /* How many following terms are part of the same phrase */ + short int iPhrase; /* This is the i-th term of a phrase. */ + short int iColumn; /* Column of the index that must match this term */ + signed char isOr; /* this term is preceded by "OR" */ + signed char isNot; /* this term is preceded by "-" */ + char *pTerm; /* text of the term. '\000' terminated. malloced */ + int nTerm; /* Number of bytes in pTerm[] */ +} QueryTerm; + + +/* A query string is parsed into a Query structure. + * + * We could, in theory, allow query strings to be complicated + * nested expressions with precedence determined by parentheses. + * But none of the major search engines do this. (Perhaps the + * feeling is that an parenthesized expression is two complex of + * an idea for the average user to grasp.) Taking our lead from + * the major search engines, we will allow queries to be a list + * of terms (with an implied AND operator) or phrases in double-quotes, + * with a single optional "-" before each non-phrase term to designate + * negation and an optional OR connector. + * + * OR binds more tightly than the implied AND, which is what the + * major search engines seem to do. So, for example: + * + * [one two OR three] ==> one AND (two OR three) + * [one OR two three] ==> (one OR two) AND three + * + * A "-" before a term matches all entries that lack that term. + * The "-" must occur immediately before the term with in intervening + * space. This is how the search engines do it. + * + * A NOT term cannot be the right-hand operand of an OR. If this + * occurs in the query string, the NOT is ignored: + * + * [one OR -two] ==> one OR two + * + */ +typedef struct Query { + fulltext_vtab *pFts; /* The full text index */ + int nTerms; /* Number of terms in the query */ + QueryTerm *pTerms; /* Array of terms. Space obtained from malloc() */ + int nextIsOr; /* Set the isOr flag on the next inserted term */ + int nextColumn; /* Next word parsed must be in this column */ + int dfltColumn; /* The default column */ +} Query; + + +/* +** An instance of the following structure keeps track of generated +** matching-word offset information and snippets. +*/ +typedef struct Snippet { + int nMatch; /* Total number of matches */ + int nAlloc; /* Space allocated for aMatch[] */ + struct snippetMatch { /* One entry for each matching term */ + char snStatus; /* Status flag for use while constructing snippets */ + short int iCol; /* The column that contains the match */ + short int iTerm; /* The index in Query.pTerms[] of the matching term */ + short int nByte; /* Number of bytes in the term */ + int iStart; /* The offset to the first character of the term */ + } *aMatch; /* Points to space obtained from malloc */ + char *zOffset; /* Text rendering of aMatch[] */ + int nOffset; /* strlen(zOffset) */ + char *zSnippet; /* Snippet text */ + int nSnippet; /* strlen(zSnippet) */ +} Snippet; + + +typedef enum QueryType { + QUERY_GENERIC, /* table scan */ + QUERY_ROWID, /* lookup by rowid */ + QUERY_FULLTEXT /* QUERY_FULLTEXT + [i] is a full-text search for column i*/ +} QueryType; + +/* TODO(shess) CHUNK_MAX controls how much data we allow in segment 0 +** before we start aggregating into larger segments. Lower CHUNK_MAX +** means that for a given input we have more individual segments per +** term, which means more rows in the table and a bigger index (due to +** both more rows and bigger rowids). But it also reduces the average +** cost of adding new elements to the segment 0 doclist, and it seems +** to reduce the number of pages read and written during inserts. 256 +** was chosen by measuring insertion times for a certain input (first +** 10k documents of Enron corpus), though including query performance +** in the decision may argue for a larger value. +*/ +#define CHUNK_MAX 256 + +typedef enum fulltext_statement { + CONTENT_INSERT_STMT, + CONTENT_SELECT_STMT, + CONTENT_UPDATE_STMT, + CONTENT_DELETE_STMT, + + TERM_SELECT_STMT, + TERM_SELECT_ALL_STMT, + TERM_INSERT_STMT, + TERM_UPDATE_STMT, + TERM_DELETE_STMT, + + MAX_STMT /* Always at end! */ +} fulltext_statement; + +/* These must exactly match the enum above. */ +/* TODO(adam): Is there some risk that a statement (in particular, +** pTermSelectStmt) will be used in two cursors at once, e.g. if a +** query joins a virtual table to itself? If so perhaps we should +** move some of these to the cursor object. +*/ +static const char *const fulltext_zStatement[MAX_STMT] = { + /* CONTENT_INSERT */ NULL, /* generated in contentInsertStatement() */ + /* CONTENT_SELECT */ "select * from %_content where rowid = ?", + /* CONTENT_UPDATE */ NULL, /* generated in contentUpdateStatement() */ + /* CONTENT_DELETE */ "delete from %_content where rowid = ?", + + /* TERM_SELECT */ + "select rowid, doclist from %_term where term = ? and segment = ?", + /* TERM_SELECT_ALL */ + "select doclist from %_term where term = ? order by segment", + /* TERM_INSERT */ + "insert into %_term (rowid, term, segment, doclist) values (?, ?, ?, ?)", + /* TERM_UPDATE */ "update %_term set doclist = ? where rowid = ?", + /* TERM_DELETE */ "delete from %_term where rowid = ?", +}; + +/* +** A connection to a fulltext index is an instance of the following +** structure. The xCreate and xConnect methods create an instance +** of this structure and xDestroy and xDisconnect free that instance. +** All other methods receive a pointer to the structure as one of their +** arguments. +*/ +struct fulltext_vtab { + sqlite3_vtab base; /* Base class used by SQLite core */ + sqlite3 *db; /* The database connection */ + const char *zDb; /* logical database name */ + const char *zName; /* virtual table name */ + int nColumn; /* number of columns in virtual table */ + char **azColumn; /* column names. malloced */ + char **azContentColumn; /* column names in content table; malloced */ + sqlite3_tokenizer *pTokenizer; /* tokenizer for inserts and queries */ + + /* Precompiled statements which we keep as long as the table is + ** open. + */ + sqlite3_stmt *pFulltextStatements[MAX_STMT]; +}; + +/* +** When the core wants to do a query, it create a cursor using a +** call to xOpen. This structure is an instance of a cursor. It +** is destroyed by xClose. +*/ +typedef struct fulltext_cursor { + sqlite3_vtab_cursor base; /* Base class used by SQLite core */ + QueryType iCursorType; /* Copy of sqlite3_index_info.idxNum */ + sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ + int eof; /* True if at End Of Results */ + Query q; /* Parsed query string */ + Snippet snippet; /* Cached snippet for the current row */ + int iColumn; /* Column being searched */ + DocListReader result; /* used when iCursorType == QUERY_FULLTEXT */ +} fulltext_cursor; + +static struct fulltext_vtab *cursor_vtab(fulltext_cursor *c){ + return (fulltext_vtab *) c->base.pVtab; +} + +static const sqlite3_module fulltextModule; /* forward declaration */ + +/* Append a list of strings separated by commas to a StringBuffer. */ +static void appendList(StringBuffer *sb, int nString, char **azString){ + int i; + for(i=0; i<nString; ++i){ + if( i>0 ) append(sb, ", "); + append(sb, azString[i]); + } +} + +/* Return a dynamically generated statement of the form + * insert into %_content (rowid, ...) values (?, ...) + */ +static const char *contentInsertStatement(fulltext_vtab *v){ + StringBuffer sb; + int i; + + initStringBuffer(&sb); + append(&sb, "insert into %_content (rowid, "); + appendList(&sb, v->nColumn, v->azContentColumn); + append(&sb, ") values (?"); + for(i=0; i<v->nColumn; ++i) + append(&sb, ", ?"); + append(&sb, ")"); + return sb.s; +} + +/* Return a dynamically generated statement of the form + * update %_content set [col_0] = ?, [col_1] = ?, ... + * where rowid = ? + */ +static const char *contentUpdateStatement(fulltext_vtab *v){ + StringBuffer sb; + int i; + + initStringBuffer(&sb); + append(&sb, "update %_content set "); + for(i=0; i<v->nColumn; ++i) { + if( i>0 ){ + append(&sb, ", "); + } + append(&sb, v->azContentColumn[i]); + append(&sb, " = ?"); + } + append(&sb, " where rowid = ?"); + return sb.s; +} + +/* Puts a freshly-prepared statement determined by iStmt in *ppStmt. +** If the indicated statement has never been prepared, it is prepared +** and cached, otherwise the cached version is reset. +*/ +static int sql_get_statement(fulltext_vtab *v, fulltext_statement iStmt, + sqlite3_stmt **ppStmt){ + assert( iStmt<MAX_STMT ); + if( v->pFulltextStatements[iStmt]==NULL ){ + const char *zStmt; + int rc; + switch( iStmt ){ + case CONTENT_INSERT_STMT: + zStmt = contentInsertStatement(v); break; + case CONTENT_UPDATE_STMT: + zStmt = contentUpdateStatement(v); break; + default: + zStmt = fulltext_zStatement[iStmt]; + } + rc = sql_prepare(v->db, v->zDb, v->zName, &v->pFulltextStatements[iStmt], + zStmt); + if( zStmt != fulltext_zStatement[iStmt]) free((void *) zStmt); + if( rc!=SQLITE_OK ) return rc; + } else { + int rc = sqlite3_reset(v->pFulltextStatements[iStmt]); + if( rc!=SQLITE_OK ) return rc; + } + + *ppStmt = v->pFulltextStatements[iStmt]; + return SQLITE_OK; +} + +/* Step the indicated statement, handling errors SQLITE_BUSY (by +** retrying) and SQLITE_SCHEMA (by re-preparing and transferring +** bindings to the new statement). +** TODO(adam): We should extend this function so that it can work with +** statements declared locally, not only globally cached statements. +*/ +static int sql_step_statement(fulltext_vtab *v, fulltext_statement iStmt, + sqlite3_stmt **ppStmt){ + int rc; + sqlite3_stmt *s = *ppStmt; + assert( iStmt<MAX_STMT ); + assert( s==v->pFulltextStatements[iStmt] ); + + while( (rc=sqlite3_step(s))!=SQLITE_DONE && rc!=SQLITE_ROW ){ + if( rc==SQLITE_BUSY ) continue; + if( rc!=SQLITE_ERROR ) return rc; + + /* If an SQLITE_SCHEMA error has occurred, then finalizing this + * statement is going to delete the fulltext_vtab structure. If + * the statement just executed is in the pFulltextStatements[] + * array, it will be finalized twice. So remove it before + * calling sqlite3_finalize(). + */ + v->pFulltextStatements[iStmt] = NULL; + rc = sqlite3_finalize(s); + break; + } + return rc; + + err: + sqlite3_finalize(s); + return rc; +} + +/* Like sql_step_statement(), but convert SQLITE_DONE to SQLITE_OK. +** Useful for statements like UPDATE, where we expect no results. +*/ +static int sql_single_step_statement(fulltext_vtab *v, + fulltext_statement iStmt, + sqlite3_stmt **ppStmt){ + int rc = sql_step_statement(v, iStmt, ppStmt); + return (rc==SQLITE_DONE) ? SQLITE_OK : rc; +} + +/* insert into %_content (rowid, ...) values ([rowid], [pValues]) */ +static int content_insert(fulltext_vtab *v, sqlite3_value *rowid, + sqlite3_value **pValues){ + sqlite3_stmt *s; + int i; + int rc = sql_get_statement(v, CONTENT_INSERT_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_value(s, 1, rowid); + if( rc!=SQLITE_OK ) return rc; + + for(i=0; i<v->nColumn; ++i){ + rc = sqlite3_bind_value(s, 2+i, pValues[i]); + if( rc!=SQLITE_OK ) return rc; + } + + return sql_single_step_statement(v, CONTENT_INSERT_STMT, &s); +} + +/* update %_content set col0 = pValues[0], col1 = pValues[1], ... + * where rowid = [iRowid] */ +static int content_update(fulltext_vtab *v, sqlite3_value **pValues, + sqlite_int64 iRowid){ + sqlite3_stmt *s; + int i; + int rc = sql_get_statement(v, CONTENT_UPDATE_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + for(i=0; i<v->nColumn; ++i){ + rc = sqlite3_bind_value(s, 1+i, pValues[i]); + if( rc!=SQLITE_OK ) return rc; + } + + rc = sqlite3_bind_int64(s, 1+v->nColumn, iRowid); + if( rc!=SQLITE_OK ) return rc; + + return sql_single_step_statement(v, CONTENT_UPDATE_STMT, &s); +} + +static void freeStringArray(int nString, const char **pString){ + int i; + + for (i=0 ; i < nString ; ++i) { + if( pString[i]!=NULL ) free((void *) pString[i]); + } + free((void *) pString); +} + +/* select * from %_content where rowid = [iRow] + * The caller must delete the returned array and all strings in it. + * null fields will be NULL in the returned array. + * + * TODO: Perhaps we should return pointer/length strings here for consistency + * with other code which uses pointer/length. */ +static int content_select(fulltext_vtab *v, sqlite_int64 iRow, + const char ***pValues){ + sqlite3_stmt *s; + const char **values; + int i; + int rc; + + *pValues = NULL; + + rc = sql_get_statement(v, CONTENT_SELECT_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int64(s, 1, iRow); + if( rc!=SQLITE_OK ) return rc; + + rc = sql_step_statement(v, CONTENT_SELECT_STMT, &s); + if( rc!=SQLITE_ROW ) return rc; + + values = (const char **) malloc(v->nColumn * sizeof(const char *)); + for(i=0; i<v->nColumn; ++i){ + if( sqlite3_column_type(s, i)==SQLITE_NULL ){ + values[i] = NULL; + }else{ + values[i] = string_dup((char*)sqlite3_column_text(s, i)); + } + } + + /* We expect only one row. We must execute another sqlite3_step() + * to complete the iteration; otherwise the table will remain locked. */ + rc = sqlite3_step(s); + if( rc==SQLITE_DONE ){ + *pValues = values; + return SQLITE_OK; + } + + freeStringArray(v->nColumn, values); + return rc; +} + +/* delete from %_content where rowid = [iRow ] */ +static int content_delete(fulltext_vtab *v, sqlite_int64 iRow){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, CONTENT_DELETE_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int64(s, 1, iRow); + if( rc!=SQLITE_OK ) return rc; + + return sql_single_step_statement(v, CONTENT_DELETE_STMT, &s); +} + +/* select rowid, doclist from %_term + * where term = [pTerm] and segment = [iSegment] + * If found, returns SQLITE_ROW; the caller must free the + * returned doclist. If no rows found, returns SQLITE_DONE. */ +static int term_select(fulltext_vtab *v, const char *pTerm, int nTerm, + int iSegment, + sqlite_int64 *rowid, DocList *out){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, TERM_SELECT_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_text(s, 1, pTerm, nTerm, SQLITE_STATIC); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int(s, 2, iSegment); + if( rc!=SQLITE_OK ) return rc; + + rc = sql_step_statement(v, TERM_SELECT_STMT, &s); + if( rc!=SQLITE_ROW ) return rc; + + *rowid = sqlite3_column_int64(s, 0); + docListInit(out, DL_DEFAULT, + sqlite3_column_blob(s, 1), sqlite3_column_bytes(s, 1)); + + /* We expect only one row. We must execute another sqlite3_step() + * to complete the iteration; otherwise the table will remain locked. */ + rc = sqlite3_step(s); + return rc==SQLITE_DONE ? SQLITE_ROW : rc; +} + +/* Load the segment doclists for term pTerm and merge them in +** appropriate order into out. Returns SQLITE_OK if successful. If +** there are no segments for pTerm, successfully returns an empty +** doclist in out. +** +** Each document consists of 1 or more "columns". The number of +** columns is v->nColumn. If iColumn==v->nColumn, then return +** position information about all columns. If iColumn<v->nColumn, +** then only return position information about the iColumn-th column +** (where the first column is 0). +*/ +static int term_select_all( + fulltext_vtab *v, /* The fulltext index we are querying against */ + int iColumn, /* If <nColumn, only look at the iColumn-th column */ + const char *pTerm, /* The term whose posting lists we want */ + int nTerm, /* Number of bytes in pTerm */ + DocList *out /* Write the resulting doclist here */ +){ + DocList doclist; + sqlite3_stmt *s; + int rc = sql_get_statement(v, TERM_SELECT_ALL_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_text(s, 1, pTerm, nTerm, SQLITE_STATIC); + if( rc!=SQLITE_OK ) return rc; + + docListInit(&doclist, DL_DEFAULT, 0, 0); + + /* TODO(shess) Handle schema and busy errors. */ + while( (rc=sql_step_statement(v, TERM_SELECT_ALL_STMT, &s))==SQLITE_ROW ){ + DocList old; + + /* TODO(shess) If we processed doclists from oldest to newest, we + ** could skip the malloc() involved with the following call. For + ** now, I'd rather keep this logic similar to index_insert_term(). + ** We could additionally drop elements when we see deletes, but + ** that would require a distinct version of docListAccumulate(). + */ + docListInit(&old, DL_DEFAULT, + sqlite3_column_blob(s, 0), sqlite3_column_bytes(s, 0)); + + if( iColumn<v->nColumn ){ /* querying a single column */ + docListRestrictColumn(&old, iColumn); + } + + /* doclist contains the newer data, so write it over old. Then + ** steal accumulated result for doclist. + */ + docListAccumulate(&old, &doclist); + docListDestroy(&doclist); + doclist = old; + } + if( rc!=SQLITE_DONE ){ + docListDestroy(&doclist); + return rc; + } + + docListDiscardEmpty(&doclist); + *out = doclist; + return SQLITE_OK; +} + +/* insert into %_term (rowid, term, segment, doclist) + values ([piRowid], [pTerm], [iSegment], [doclist]) +** Lets sqlite select rowid if piRowid is NULL, else uses *piRowid. +** +** NOTE(shess) piRowid is IN, with values of "space of int64" plus +** null, it is not used to pass data back to the caller. +*/ +static int term_insert(fulltext_vtab *v, sqlite_int64 *piRowid, + const char *pTerm, int nTerm, + int iSegment, DocList *doclist){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, TERM_INSERT_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + if( piRowid==NULL ){ + rc = sqlite3_bind_null(s, 1); + }else{ + rc = sqlite3_bind_int64(s, 1, *piRowid); + } + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_text(s, 2, pTerm, nTerm, SQLITE_STATIC); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int(s, 3, iSegment); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_blob(s, 4, doclist->pData, doclist->nData, SQLITE_STATIC); + if( rc!=SQLITE_OK ) return rc; + + return sql_single_step_statement(v, TERM_INSERT_STMT, &s); +} + +/* update %_term set doclist = [doclist] where rowid = [rowid] */ +static int term_update(fulltext_vtab *v, sqlite_int64 rowid, + DocList *doclist){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, TERM_UPDATE_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_blob(s, 1, doclist->pData, doclist->nData, SQLITE_STATIC); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int64(s, 2, rowid); + if( rc!=SQLITE_OK ) return rc; + + return sql_single_step_statement(v, TERM_UPDATE_STMT, &s); +} + +static int term_delete(fulltext_vtab *v, sqlite_int64 rowid){ + sqlite3_stmt *s; + int rc = sql_get_statement(v, TERM_DELETE_STMT, &s); + if( rc!=SQLITE_OK ) return rc; + + rc = sqlite3_bind_int64(s, 1, rowid); + if( rc!=SQLITE_OK ) return rc; + + return sql_single_step_statement(v, TERM_DELETE_STMT, &s); +} + +/* +** Free the memory used to contain a fulltext_vtab structure. +*/ +static void fulltext_vtab_destroy(fulltext_vtab *v){ + int iStmt, i; + + TRACE(("FTS1 Destroy %p\n", v)); + for( iStmt=0; iStmt<MAX_STMT; iStmt++ ){ + if( v->pFulltextStatements[iStmt]!=NULL ){ + sqlite3_finalize(v->pFulltextStatements[iStmt]); + v->pFulltextStatements[iStmt] = NULL; + } + } + + if( v->pTokenizer!=NULL ){ + v->pTokenizer->pModule->xDestroy(v->pTokenizer); + v->pTokenizer = NULL; + } + + free(v->azColumn); + for(i = 0; i < v->nColumn; ++i) { + sqlite3_free(v->azContentColumn[i]); + } + free(v->azContentColumn); + free(v); +} + +/* +** Token types for parsing the arguments to xConnect or xCreate. +*/ +#define TOKEN_EOF 0 /* End of file */ +#define TOKEN_SPACE 1 /* Any kind of whitespace */ +#define TOKEN_ID 2 /* An identifier */ +#define TOKEN_STRING 3 /* A string literal */ +#define TOKEN_PUNCT 4 /* A single punctuation character */ + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identfiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +static const char isIdChar[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ + 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 2x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 3x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, /* 5x */ + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 7x */ +}; +#define IdChar(C) (((c=C)&0x80)!=0 || (c>0x1f && isIdChar[c-0x20])) + + +/* +** Return the length of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +static int getToken(const char *z, int *tokenType){ + int i, c; + switch( *z ){ + case 0: { + *tokenType = TOKEN_EOF; + return 0; + } + case ' ': case '\t': case '\n': case '\f': case '\r': { + for(i=1; safe_isspace(z[i]); i++){} + *tokenType = TOKEN_SPACE; + return i; + } + case '`': + case '\'': + case '"': { + int delim = z[0]; + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + *tokenType = TOKEN_STRING; + return i + (c!=0); + } + case '[': { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = TOKEN_ID; + return i; + } + default: { + if( !IdChar(*z) ){ + break; + } + for(i=1; IdChar(z[i]); i++){} + *tokenType = TOKEN_ID; + return i; + } + } + *tokenType = TOKEN_PUNCT; + return 1; +} + +/* +** A token extracted from a string is an instance of the following +** structure. +*/ +typedef struct Token { + const char *z; /* Pointer to token text. Not '\000' terminated */ + short int n; /* Length of the token text in bytes. */ +} Token; + +/* +** Given a input string (which is really one of the argv[] parameters +** passed into xConnect or xCreate) split the string up into tokens. +** Return an array of pointers to '\000' terminated strings, one string +** for each non-whitespace token. +** +** The returned array is terminated by a single NULL pointer. +** +** Space to hold the returned array is obtained from a single +** malloc and should be freed by passing the return value to free(). +** The individual strings within the token list are all a part of +** the single memory allocation and will all be freed at once. +*/ +static char **tokenizeString(const char *z, int *pnToken){ + int nToken = 0; + Token *aToken = malloc( strlen(z) * sizeof(aToken[0]) ); + int n = 1; + int e, i; + int totalSize = 0; + char **azToken; + char *zCopy; + while( n>0 ){ + n = getToken(z, &e); + if( e!=TOKEN_SPACE ){ + aToken[nToken].z = z; + aToken[nToken].n = n; + nToken++; + totalSize += n+1; + } + z += n; + } + azToken = (char**)malloc( nToken*sizeof(char*) + totalSize ); + zCopy = (char*)&azToken[nToken]; + nToken--; + for(i=0; i<nToken; i++){ + azToken[i] = zCopy; + n = aToken[i].n; + memcpy(zCopy, aToken[i].z, n); + zCopy[n] = 0; + zCopy += n+1; + } + azToken[nToken] = 0; + free(aToken); + *pnToken = nToken; + return azToken; +} + +/* +** Convert an SQL-style quoted string into a normal string by removing +** the quote characters. The conversion is done in-place. If the +** input does not begin with a quote character, then this routine +** is a no-op. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static void dequoteString(char *z){ + int quote; + int i, j; + if( z==0 ) return; + quote = z[0]; + switch( quote ){ + case '\'': break; + case '"': break; + case '`': break; /* For MySQL compatibility */ + case '[': quote = ']'; break; /* For MS SqlServer compatibility */ + default: return; + } + for(i=1, j=0; z[i]; i++){ + if( z[i]==quote ){ + if( z[i+1]==quote ){ + z[j++] = quote; + i++; + }else{ + z[j++] = 0; + break; + } + }else{ + z[j++] = z[i]; + } + } +} + +/* +** The input azIn is a NULL-terminated list of tokens. Remove the first +** token and all punctuation tokens. Remove the quotes from +** around string literal tokens. +** +** Example: +** +** input: tokenize chinese ( 'simplifed' , 'mixed' ) +** output: chinese simplifed mixed +** +** Another example: +** +** input: delimiters ( '[' , ']' , '...' ) +** output: [ ] ... +*/ +static void tokenListToIdList(char **azIn){ + int i, j; + if( azIn ){ + for(i=0, j=-1; azIn[i]; i++){ + if( safe_isalnum(azIn[i][0]) || azIn[i][1] ){ + dequoteString(azIn[i]); + if( j>=0 ){ + azIn[j] = azIn[i]; + } + j++; + } + } + azIn[j] = 0; + } +} + + +/* +** Find the first alphanumeric token in the string zIn. Null-terminate +** this token. Remove any quotation marks. And return a pointer to +** the result. +*/ +static char *firstToken(char *zIn, char **pzTail){ + int n, ttype; + while(1){ + n = getToken(zIn, &ttype); + if( ttype==TOKEN_SPACE ){ + zIn += n; + }else if( ttype==TOKEN_EOF ){ + *pzTail = zIn; + return 0; + }else{ + zIn[n] = 0; + *pzTail = &zIn[1]; + dequoteString(zIn); + return zIn; + } + } + /*NOTREACHED*/ +} + +/* Return true if... +** +** * s begins with the string t, ignoring case +** * s is longer than t +** * The first character of s beyond t is not a alphanumeric +** +** Ignore leading space in *s. +** +** To put it another way, return true if the first token of +** s[] is t[]. +*/ +static int startsWith(const char *s, const char *t){ + while( safe_isspace(*s) ){ s++; } + while( *t ){ + if( safe_tolower(*s++)!=safe_tolower(*t++) ) return 0; + } + return *s!='_' && !safe_isalnum(*s); +} + +/* +** An instance of this structure defines the "spec" of a +** full text index. This structure is populated by parseSpec +** and use by fulltextConnect and fulltextCreate. +*/ +typedef struct TableSpec { + const char *zDb; /* Logical database name */ + const char *zName; /* Name of the full-text index */ + int nColumn; /* Number of columns to be indexed */ + char **azColumn; /* Original names of columns to be indexed */ + char **azContentColumn; /* Column names for %_content */ + char **azTokenizer; /* Name of tokenizer and its arguments */ +} TableSpec; + +/* +** Reclaim all of the memory used by a TableSpec +*/ +static void clearTableSpec(TableSpec *p) { + free(p->azColumn); + free(p->azContentColumn); + free(p->azTokenizer); +} + +/* Parse a CREATE VIRTUAL TABLE statement, which looks like this: + * + * CREATE VIRTUAL TABLE email + * USING fts1(subject, body, tokenize mytokenizer(myarg)) + * + * We return parsed information in a TableSpec structure. + * + */ +static int parseSpec(TableSpec *pSpec, int argc, const char *const*argv, + char**pzErr){ + int i, n; + char *z, *zDummy; + char **azArg; + const char *zTokenizer = 0; /* argv[] entry describing the tokenizer */ + + assert( argc>=3 ); + /* Current interface: + ** argv[0] - module name + ** argv[1] - database name + ** argv[2] - table name + ** argv[3..] - columns, optionally followed by tokenizer specification + ** and snippet delimiters specification. + */ + + /* Make a copy of the complete argv[][] array in a single allocation. + ** The argv[][] array is read-only and transient. We can write to the + ** copy in order to modify things and the copy is persistent. + */ + memset(pSpec, 0, sizeof(*pSpec)); + for(i=n=0; i<argc; i++){ + n += strlen(argv[i]) + 1; + } + azArg = malloc( sizeof(char*)*argc + n ); + if( azArg==0 ){ + return SQLITE_NOMEM; + } + z = (char*)&azArg[argc]; + for(i=0; i<argc; i++){ + azArg[i] = z; + strcpy(z, argv[i]); + z += strlen(z)+1; + } + + /* Identify the column names and the tokenizer and delimiter arguments + ** in the argv[][] array. + */ + pSpec->zDb = azArg[1]; + pSpec->zName = azArg[2]; + pSpec->nColumn = 0; + pSpec->azColumn = azArg; + zTokenizer = "tokenize simple"; + for(i=3; i<argc; ++i){ + if( startsWith(azArg[i],"tokenize") ){ + zTokenizer = azArg[i]; + }else{ + z = azArg[pSpec->nColumn] = firstToken(azArg[i], &zDummy); + pSpec->nColumn++; + } + } + if( pSpec->nColumn==0 ){ + azArg[0] = "content"; + pSpec->nColumn = 1; + } + + /* + ** Construct the list of content column names. + ** + ** Each content column name will be of the form cNNAAAA + ** where NN is the column number and AAAA is the sanitized + ** column name. "sanitized" means that special characters are + ** converted to "_". The cNN prefix guarantees that all column + ** names are unique. + ** + ** The AAAA suffix is not strictly necessary. It is included + ** for the convenience of people who might examine the generated + ** %_content table and wonder what the columns are used for. + */ + pSpec->azContentColumn = malloc( pSpec->nColumn * sizeof(char *) ); + if( pSpec->azContentColumn==0 ){ + clearTableSpec(pSpec); + return SQLITE_NOMEM; + } + for(i=0; i<pSpec->nColumn; i++){ + char *p; + pSpec->azContentColumn[i] = sqlite3_mprintf("c%d%s", i, azArg[i]); + for (p = pSpec->azContentColumn[i]; *p ; ++p) { + if( !safe_isalnum(*p) ) *p = '_'; + } + } + + /* + ** Parse the tokenizer specification string. + */ + pSpec->azTokenizer = tokenizeString(zTokenizer, &n); + tokenListToIdList(pSpec->azTokenizer); + + return SQLITE_OK; +} + +/* +** Generate a CREATE TABLE statement that describes the schema of +** the virtual table. Return a pointer to this schema string. +** +** Space is obtained from sqlite3_mprintf() and should be freed +** using sqlite3_free(). +*/ +static char *fulltextSchema( + int nColumn, /* Number of columns */ + const char *const* azColumn, /* List of columns */ + const char *zTableName /* Name of the table */ +){ + int i; + char *zSchema, *zNext; + const char *zSep = "("; + zSchema = sqlite3_mprintf("CREATE TABLE x"); + for(i=0; i<nColumn; i++){ + zNext = sqlite3_mprintf("%s%s%Q", zSchema, zSep, azColumn[i]); + sqlite3_free(zSchema); + zSchema = zNext; + zSep = ","; + } + zNext = sqlite3_mprintf("%s,%Q)", zSchema, zTableName); + sqlite3_free(zSchema); + return zNext; +} + +/* +** Build a new sqlite3_vtab structure that will describe the +** fulltext index defined by spec. +*/ +static int constructVtab( + sqlite3 *db, /* The SQLite database connection */ + TableSpec *spec, /* Parsed spec information from parseSpec() */ + sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ + char **pzErr /* Write any error message here */ +){ + int rc; + int n; + fulltext_vtab *v = 0; + const sqlite3_tokenizer_module *m = NULL; + char *schema; + + v = (fulltext_vtab *) malloc(sizeof(fulltext_vtab)); + if( v==0 ) return SQLITE_NOMEM; + memset(v, 0, sizeof(*v)); + /* sqlite will initialize v->base */ + v->db = db; + v->zDb = spec->zDb; /* Freed when azColumn is freed */ + v->zName = spec->zName; /* Freed when azColumn is freed */ + v->nColumn = spec->nColumn; + v->azContentColumn = spec->azContentColumn; + spec->azContentColumn = 0; + v->azColumn = spec->azColumn; + spec->azColumn = 0; + + if( spec->azTokenizer==0 ){ + return SQLITE_NOMEM; + } + /* TODO(shess) For now, add new tokenizers as else if clauses. */ + if( spec->azTokenizer[0]==0 || startsWith(spec->azTokenizer[0], "simple") ){ + sqlite3Fts1SimpleTokenizerModule(&m); + }else if( startsWith(spec->azTokenizer[0], "porter") ){ + sqlite3Fts1PorterTokenizerModule(&m); + }else{ + *pzErr = sqlite3_mprintf("unknown tokenizer: %s", spec->azTokenizer[0]); + rc = SQLITE_ERROR; + goto err; + } + for(n=0; spec->azTokenizer[n]; n++){} + if( n ){ + rc = m->xCreate(n-1, (const char*const*)&spec->azTokenizer[1], + &v->pTokenizer); + }else{ + rc = m->xCreate(0, 0, &v->pTokenizer); + } + if( rc!=SQLITE_OK ) goto err; + v->pTokenizer->pModule = m; + + /* TODO: verify the existence of backing tables foo_content, foo_term */ + + schema = fulltextSchema(v->nColumn, (const char*const*)v->azColumn, + spec->zName); + rc = sqlite3_declare_vtab(db, schema); + sqlite3_free(schema); + if( rc!=SQLITE_OK ) goto err; + + memset(v->pFulltextStatements, 0, sizeof(v->pFulltextStatements)); + + *ppVTab = &v->base; + TRACE(("FTS1 Connect %p\n", v)); + + return rc; + +err: + fulltext_vtab_destroy(v); + return rc; +} + +static int fulltextConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVTab, + char **pzErr +){ + TableSpec spec; + int rc = parseSpec(&spec, argc, argv, pzErr); + if( rc!=SQLITE_OK ) return rc; + + rc = constructVtab(db, &spec, ppVTab, pzErr); + clearTableSpec(&spec); + return rc; +} + + /* The %_content table holds the text of each document, with + ** the rowid used as the docid. + ** + ** The %_term table maps each term to a document list blob + ** containing elements sorted by ascending docid, each element + ** encoded as: + ** + ** docid varint-encoded + ** token elements: + ** position+1 varint-encoded as delta from previous position + ** start offset varint-encoded as delta from previous start offset + ** end offset varint-encoded as delta from start offset + ** + ** The sentinel position of 0 indicates the end of the token list. + ** + ** Additionally, doclist blobs are chunked into multiple segments, + ** using segment to order the segments. New elements are added to + ** the segment at segment 0, until it exceeds CHUNK_MAX. Then + ** segment 0 is deleted, and the doclist is inserted at segment 1. + ** If there is already a doclist at segment 1, the segment 0 doclist + ** is merged with it, the segment 1 doclist is deleted, and the + ** merged doclist is inserted at segment 2, repeating those + ** operations until an insert succeeds. + ** + ** Since this structure doesn't allow us to update elements in place + ** in case of deletion or update, these are simply written to + ** segment 0 (with an empty token list in case of deletion), with + ** docListAccumulate() taking care to retain lower-segment + ** information in preference to higher-segment information. + */ + /* TODO(shess) Provide a VACUUM type operation which both removes + ** deleted elements which are no longer necessary, and duplicated + ** elements. I suspect this will probably not be necessary in + ** practice, though. + */ +static int fulltextCreate(sqlite3 *db, void *pAux, + int argc, const char * const *argv, + sqlite3_vtab **ppVTab, char **pzErr){ + int rc; + TableSpec spec; + StringBuffer schema; + TRACE(("FTS1 Create\n")); + + rc = parseSpec(&spec, argc, argv, pzErr); + if( rc!=SQLITE_OK ) return rc; + + initStringBuffer(&schema); + append(&schema, "CREATE TABLE %_content("); + appendList(&schema, spec.nColumn, spec.azContentColumn); + append(&schema, ")"); + rc = sql_exec(db, spec.zDb, spec.zName, schema.s); + free(schema.s); + if( rc!=SQLITE_OK ) goto out; + + rc = sql_exec(db, spec.zDb, spec.zName, + "create table %_term(term text, segment integer, doclist blob, " + "primary key(term, segment));"); + if( rc!=SQLITE_OK ) goto out; + + rc = constructVtab(db, &spec, ppVTab, pzErr); + +out: + clearTableSpec(&spec); + return rc; +} + +/* Decide how to handle an SQL query. */ +static int fulltextBestIndex(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ + int i; + TRACE(("FTS1 BestIndex\n")); + + for(i=0; i<pInfo->nConstraint; ++i){ + const struct sqlite3_index_constraint *pConstraint; + pConstraint = &pInfo->aConstraint[i]; + if( pConstraint->usable ) { + if( pConstraint->iColumn==-1 && + pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pInfo->idxNum = QUERY_ROWID; /* lookup by rowid */ + TRACE(("FTS1 QUERY_ROWID\n")); + } else if( pConstraint->iColumn>=0 && + pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + /* full-text search */ + pInfo->idxNum = QUERY_FULLTEXT + pConstraint->iColumn; + TRACE(("FTS1 QUERY_FULLTEXT %d\n", pConstraint->iColumn)); + } else continue; + + pInfo->aConstraintUsage[i].argvIndex = 1; + pInfo->aConstraintUsage[i].omit = 1; + + /* An arbitrary value for now. + * TODO: Perhaps rowid matches should be considered cheaper than + * full-text searches. */ + pInfo->estimatedCost = 1.0; + + return SQLITE_OK; + } + } + pInfo->idxNum = QUERY_GENERIC; + return SQLITE_OK; +} + +static int fulltextDisconnect(sqlite3_vtab *pVTab){ + TRACE(("FTS1 Disconnect %p\n", pVTab)); + fulltext_vtab_destroy((fulltext_vtab *)pVTab); + return SQLITE_OK; +} + +static int fulltextDestroy(sqlite3_vtab *pVTab){ + fulltext_vtab *v = (fulltext_vtab *)pVTab; + int rc; + + TRACE(("FTS1 Destroy %p\n", pVTab)); + rc = sql_exec(v->db, v->zDb, v->zName, + "drop table if exists %_content;" + "drop table if exists %_term;" + ); + if( rc!=SQLITE_OK ) return rc; + + fulltext_vtab_destroy((fulltext_vtab *)pVTab); + return SQLITE_OK; +} + +static int fulltextOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + fulltext_cursor *c; + + c = (fulltext_cursor *) calloc(sizeof(fulltext_cursor), 1); + /* sqlite will initialize c->base */ + *ppCursor = &c->base; + TRACE(("FTS1 Open %p: %p\n", pVTab, c)); + + return SQLITE_OK; +} + + +/* Free all of the dynamically allocated memory held by *q +*/ +static void queryClear(Query *q){ + int i; + for(i = 0; i < q->nTerms; ++i){ + free(q->pTerms[i].pTerm); + } + free(q->pTerms); + memset(q, 0, sizeof(*q)); +} + +/* Free all of the dynamically allocated memory held by the +** Snippet +*/ +static void snippetClear(Snippet *p){ + free(p->aMatch); + free(p->zOffset); + free(p->zSnippet); + memset(p, 0, sizeof(*p)); +} +/* +** Append a single entry to the p->aMatch[] log. +*/ +static void snippetAppendMatch( + Snippet *p, /* Append the entry to this snippet */ + int iCol, int iTerm, /* The column and query term */ + int iStart, int nByte /* Offset and size of the match */ +){ + int i; + struct snippetMatch *pMatch; + if( p->nMatch+1>=p->nAlloc ){ + p->nAlloc = p->nAlloc*2 + 10; + p->aMatch = realloc(p->aMatch, p->nAlloc*sizeof(p->aMatch[0]) ); + if( p->aMatch==0 ){ + p->nMatch = 0; + p->nAlloc = 0; + return; + } + } + i = p->nMatch++; + pMatch = &p->aMatch[i]; + pMatch->iCol = iCol; + pMatch->iTerm = iTerm; + pMatch->iStart = iStart; + pMatch->nByte = nByte; +} + +/* +** Sizing information for the circular buffer used in snippetOffsetsOfColumn() +*/ +#define FTS1_ROTOR_SZ (32) +#define FTS1_ROTOR_MASK (FTS1_ROTOR_SZ-1) + +/* +** Add entries to pSnippet->aMatch[] for every match that occurs against +** document zDoc[0..nDoc-1] which is stored in column iColumn. +*/ +static void snippetOffsetsOfColumn( + Query *pQuery, + Snippet *pSnippet, + int iColumn, + const char *zDoc, + int nDoc +){ + const sqlite3_tokenizer_module *pTModule; /* The tokenizer module */ + sqlite3_tokenizer *pTokenizer; /* The specific tokenizer */ + sqlite3_tokenizer_cursor *pTCursor; /* Tokenizer cursor */ + fulltext_vtab *pVtab; /* The full text index */ + int nColumn; /* Number of columns in the index */ + const QueryTerm *aTerm; /* Query string terms */ + int nTerm; /* Number of query string terms */ + int i, j; /* Loop counters */ + int rc; /* Return code */ + unsigned int match, prevMatch; /* Phrase search bitmasks */ + const char *zToken; /* Next token from the tokenizer */ + int nToken; /* Size of zToken */ + int iBegin, iEnd, iPos; /* Offsets of beginning and end */ + + /* The following variables keep a circular buffer of the last + ** few tokens */ + unsigned int iRotor = 0; /* Index of current token */ + int iRotorBegin[FTS1_ROTOR_SZ]; /* Beginning offset of token */ + int iRotorLen[FTS1_ROTOR_SZ]; /* Length of token */ + + pVtab = pQuery->pFts; + nColumn = pVtab->nColumn; + pTokenizer = pVtab->pTokenizer; + pTModule = pTokenizer->pModule; + rc = pTModule->xOpen(pTokenizer, zDoc, nDoc, &pTCursor); + if( rc ) return; + pTCursor->pTokenizer = pTokenizer; + aTerm = pQuery->pTerms; + nTerm = pQuery->nTerms; + if( nTerm>=FTS1_ROTOR_SZ ){ + nTerm = FTS1_ROTOR_SZ - 1; + } + prevMatch = 0; + while(1){ + rc = pTModule->xNext(pTCursor, &zToken, &nToken, &iBegin, &iEnd, &iPos); + if( rc ) break; + iRotorBegin[iRotor&FTS1_ROTOR_MASK] = iBegin; + iRotorLen[iRotor&FTS1_ROTOR_MASK] = iEnd-iBegin; + match = 0; + for(i=0; i<nTerm; i++){ + int iCol; + iCol = aTerm[i].iColumn; + if( iCol>=0 && iCol<nColumn && iCol!=iColumn ) continue; + if( aTerm[i].nTerm!=nToken ) continue; + if( memcmp(aTerm[i].pTerm, zToken, nToken) ) continue; + if( aTerm[i].iPhrase>1 && (prevMatch & (1<<i))==0 ) continue; + match |= 1<<i; + if( i==nTerm-1 || aTerm[i+1].iPhrase==1 ){ + for(j=aTerm[i].iPhrase-1; j>=0; j--){ + int k = (iRotor-j) & FTS1_ROTOR_MASK; + snippetAppendMatch(pSnippet, iColumn, i-j, + iRotorBegin[k], iRotorLen[k]); + } + } + } + prevMatch = match<<1; + iRotor++; + } + pTModule->xClose(pTCursor); +} + + +/* +** Compute all offsets for the current row of the query. +** If the offsets have already been computed, this routine is a no-op. +*/ +static void snippetAllOffsets(fulltext_cursor *p){ + int nColumn; + int iColumn, i; + int iFirst, iLast; + fulltext_vtab *pFts; + + if( p->snippet.nMatch ) return; + if( p->q.nTerms==0 ) return; + pFts = p->q.pFts; + nColumn = pFts->nColumn; + iColumn = p->iCursorType - QUERY_FULLTEXT; + if( iColumn<0 || iColumn>=nColumn ){ + iFirst = 0; + iLast = nColumn-1; + }else{ + iFirst = iColumn; + iLast = iColumn; + } + for(i=iFirst; i<=iLast; i++){ + const char *zDoc; + int nDoc; + zDoc = (const char*)sqlite3_column_text(p->pStmt, i+1); + nDoc = sqlite3_column_bytes(p->pStmt, i+1); + snippetOffsetsOfColumn(&p->q, &p->snippet, i, zDoc, nDoc); + } +} + +/* +** Convert the information in the aMatch[] array of the snippet +** into the string zOffset[0..nOffset-1]. +*/ +static void snippetOffsetText(Snippet *p){ + int i; + int cnt = 0; + StringBuffer sb; + char zBuf[200]; + if( p->zOffset ) return; + initStringBuffer(&sb); + for(i=0; i<p->nMatch; i++){ + struct snippetMatch *pMatch = &p->aMatch[i]; + zBuf[0] = ' '; + sqlite3_snprintf(sizeof(zBuf)-1, &zBuf[cnt>0], "%d %d %d %d", + pMatch->iCol, pMatch->iTerm, pMatch->iStart, pMatch->nByte); + append(&sb, zBuf); + cnt++; + } + p->zOffset = sb.s; + p->nOffset = sb.len; +} + +/* +** zDoc[0..nDoc-1] is phrase of text. aMatch[0..nMatch-1] are a set +** of matching words some of which might be in zDoc. zDoc is column +** number iCol. +** +** iBreak is suggested spot in zDoc where we could begin or end an +** excerpt. Return a value similar to iBreak but possibly adjusted +** to be a little left or right so that the break point is better. +*/ +static int wordBoundary( + int iBreak, /* The suggested break point */ + const char *zDoc, /* Document text */ + int nDoc, /* Number of bytes in zDoc[] */ + struct snippetMatch *aMatch, /* Matching words */ + int nMatch, /* Number of entries in aMatch[] */ + int iCol /* The column number for zDoc[] */ +){ + int i; + if( iBreak<=10 ){ + return 0; + } + if( iBreak>=nDoc-10 ){ + return nDoc; + } + for(i=0; i<nMatch && aMatch[i].iCol<iCol; i++){} + while( i<nMatch && aMatch[i].iStart+aMatch[i].nByte<iBreak ){ i++; } + if( i<nMatch ){ + if( aMatch[i].iStart<iBreak+10 ){ + return aMatch[i].iStart; + } + if( i>0 && aMatch[i-1].iStart+aMatch[i-1].nByte>=iBreak ){ + return aMatch[i-1].iStart; + } + } + for(i=1; i<=10; i++){ + if( safe_isspace(zDoc[iBreak-i]) ){ + return iBreak - i + 1; + } + if( safe_isspace(zDoc[iBreak+i]) ){ + return iBreak + i + 1; + } + } + return iBreak; +} + +/* +** If the StringBuffer does not end in white space, add a single +** space character to the end. +*/ +static void appendWhiteSpace(StringBuffer *p){ + if( p->len==0 ) return; + if( safe_isspace(p->s[p->len-1]) ) return; + append(p, " "); +} + +/* +** Remove white space from teh end of the StringBuffer +*/ +static void trimWhiteSpace(StringBuffer *p){ + while( p->len>0 && safe_isspace(p->s[p->len-1]) ){ + p->len--; + } +} + + + +/* +** Allowed values for Snippet.aMatch[].snStatus +*/ +#define SNIPPET_IGNORE 0 /* It is ok to omit this match from the snippet */ +#define SNIPPET_DESIRED 1 /* We want to include this match in the snippet */ + +/* +** Generate the text of a snippet. +*/ +static void snippetText( + fulltext_cursor *pCursor, /* The cursor we need the snippet for */ + const char *zStartMark, /* Markup to appear before each match */ + const char *zEndMark, /* Markup to appear after each match */ + const char *zEllipsis /* Ellipsis mark */ +){ + int i, j; + struct snippetMatch *aMatch; + int nMatch; + int nDesired; + StringBuffer sb; + int tailCol; + int tailOffset; + int iCol; + int nDoc; + const char *zDoc; + int iStart, iEnd; + int tailEllipsis = 0; + int iMatch; + + + free(pCursor->snippet.zSnippet); + pCursor->snippet.zSnippet = 0; + aMatch = pCursor->snippet.aMatch; + nMatch = pCursor->snippet.nMatch; + initStringBuffer(&sb); + + for(i=0; i<nMatch; i++){ + aMatch[i].snStatus = SNIPPET_IGNORE; + } + nDesired = 0; + for(i=0; i<pCursor->q.nTerms; i++){ + for(j=0; j<nMatch; j++){ + if( aMatch[j].iTerm==i ){ + aMatch[j].snStatus = SNIPPET_DESIRED; + nDesired++; + break; + } + } + } + + iMatch = 0; + tailCol = -1; + tailOffset = 0; + for(i=0; i<nMatch && nDesired>0; i++){ + if( aMatch[i].snStatus!=SNIPPET_DESIRED ) continue; + nDesired--; + iCol = aMatch[i].iCol; + zDoc = (const char*)sqlite3_column_text(pCursor->pStmt, iCol+1); + nDoc = sqlite3_column_bytes(pCursor->pStmt, iCol+1); + iStart = aMatch[i].iStart - 40; + iStart = wordBoundary(iStart, zDoc, nDoc, aMatch, nMatch, iCol); + if( iStart<=10 ){ + iStart = 0; + } + if( iCol==tailCol && iStart<=tailOffset+20 ){ + iStart = tailOffset; + } + if( (iCol!=tailCol && tailCol>=0) || iStart!=tailOffset ){ + trimWhiteSpace(&sb); + appendWhiteSpace(&sb); + append(&sb, zEllipsis); + appendWhiteSpace(&sb); + } + iEnd = aMatch[i].iStart + aMatch[i].nByte + 40; + iEnd = wordBoundary(iEnd, zDoc, nDoc, aMatch, nMatch, iCol); + if( iEnd>=nDoc-10 ){ + iEnd = nDoc; + tailEllipsis = 0; + }else{ + tailEllipsis = 1; + } + while( iMatch<nMatch && aMatch[iMatch].iCol<iCol ){ iMatch++; } + while( iStart<iEnd ){ + while( iMatch<nMatch && aMatch[iMatch].iStart<iStart + && aMatch[iMatch].iCol<=iCol ){ + iMatch++; + } + if( iMatch<nMatch && aMatch[iMatch].iStart<iEnd + && aMatch[iMatch].iCol==iCol ){ + nappend(&sb, &zDoc[iStart], aMatch[iMatch].iStart - iStart); + iStart = aMatch[iMatch].iStart; + append(&sb, zStartMark); + nappend(&sb, &zDoc[iStart], aMatch[iMatch].nByte); + append(&sb, zEndMark); + iStart += aMatch[iMatch].nByte; + for(j=iMatch+1; j<nMatch; j++){ + if( aMatch[j].iTerm==aMatch[iMatch].iTerm + && aMatch[j].snStatus==SNIPPET_DESIRED ){ + nDesired--; + aMatch[j].snStatus = SNIPPET_IGNORE; + } + } + }else{ + nappend(&sb, &zDoc[iStart], iEnd - iStart); + iStart = iEnd; + } + } + tailCol = iCol; + tailOffset = iEnd; + } + trimWhiteSpace(&sb); + if( tailEllipsis ){ + appendWhiteSpace(&sb); + append(&sb, zEllipsis); + } + pCursor->snippet.zSnippet = sb.s; + pCursor->snippet.nSnippet = sb.len; +} + + +/* +** Close the cursor. For additional information see the documentation +** on the xClose method of the virtual table interface. +*/ +static int fulltextClose(sqlite3_vtab_cursor *pCursor){ + fulltext_cursor *c = (fulltext_cursor *) pCursor; + TRACE(("FTS1 Close %p\n", c)); + sqlite3_finalize(c->pStmt); + queryClear(&c->q); + snippetClear(&c->snippet); + if( c->result.pDoclist!=NULL ){ + docListDelete(c->result.pDoclist); + } + free(c); + return SQLITE_OK; +} + +static int fulltextNext(sqlite3_vtab_cursor *pCursor){ + fulltext_cursor *c = (fulltext_cursor *) pCursor; + sqlite_int64 iDocid; + int rc; + + TRACE(("FTS1 Next %p\n", pCursor)); + snippetClear(&c->snippet); + if( c->iCursorType < QUERY_FULLTEXT ){ + /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ + rc = sqlite3_step(c->pStmt); + switch( rc ){ + case SQLITE_ROW: + c->eof = 0; + return SQLITE_OK; + case SQLITE_DONE: + c->eof = 1; + return SQLITE_OK; + default: + c->eof = 1; + return rc; + } + } else { /* full-text query */ + rc = sqlite3_reset(c->pStmt); + if( rc!=SQLITE_OK ) return rc; + + iDocid = nextDocid(&c->result); + if( iDocid==0 ){ + c->eof = 1; + return SQLITE_OK; + } + rc = sqlite3_bind_int64(c->pStmt, 1, iDocid); + if( rc!=SQLITE_OK ) return rc; + /* TODO(shess) Handle SQLITE_SCHEMA AND SQLITE_BUSY. */ + rc = sqlite3_step(c->pStmt); + if( rc==SQLITE_ROW ){ /* the case we expect */ + c->eof = 0; + return SQLITE_OK; + } + /* an error occurred; abort */ + return rc==SQLITE_DONE ? SQLITE_ERROR : rc; + } +} + + +/* Return a DocList corresponding to the query term *pTerm. If *pTerm +** is the first term of a phrase query, go ahead and evaluate the phrase +** query and return the doclist for the entire phrase query. +** +** The result is stored in pTerm->doclist. +*/ +static int docListOfTerm( + fulltext_vtab *v, /* The full text index */ + int iColumn, /* column to restrict to. No restrition if >=nColumn */ + QueryTerm *pQTerm, /* Term we are looking for, or 1st term of a phrase */ + DocList **ppResult /* Write the result here */ +){ + DocList *pLeft, *pRight, *pNew; + int i, rc; + + pLeft = docListNew(DL_POSITIONS); + rc = term_select_all(v, iColumn, pQTerm->pTerm, pQTerm->nTerm, pLeft); + if( rc ){ + docListDelete(pLeft); + return rc; + } + for(i=1; i<=pQTerm->nPhrase; i++){ + pRight = docListNew(DL_POSITIONS); + rc = term_select_all(v, iColumn, pQTerm[i].pTerm, pQTerm[i].nTerm, pRight); + if( rc ){ + docListDelete(pLeft); + return rc; + } + pNew = docListNew(i<pQTerm->nPhrase ? DL_POSITIONS : DL_DOCIDS); + docListPhraseMerge(pLeft, pRight, pNew); + docListDelete(pLeft); + docListDelete(pRight); + pLeft = pNew; + } + *ppResult = pLeft; + return SQLITE_OK; +} + +/* Add a new term pTerm[0..nTerm-1] to the query *q. +*/ +static void queryAdd(Query *q, const char *pTerm, int nTerm){ + QueryTerm *t; + ++q->nTerms; + q->pTerms = realloc(q->pTerms, q->nTerms * sizeof(q->pTerms[0])); + if( q->pTerms==0 ){ + q->nTerms = 0; + return; + } + t = &q->pTerms[q->nTerms - 1]; + memset(t, 0, sizeof(*t)); + t->pTerm = malloc(nTerm+1); + memcpy(t->pTerm, pTerm, nTerm); + t->pTerm[nTerm] = 0; + t->nTerm = nTerm; + t->isOr = q->nextIsOr; + q->nextIsOr = 0; + t->iColumn = q->nextColumn; + q->nextColumn = q->dfltColumn; +} + +/* +** Check to see if the string zToken[0...nToken-1] matches any +** column name in the virtual table. If it does, +** return the zero-indexed column number. If not, return -1. +*/ +static int checkColumnSpecifier( + fulltext_vtab *pVtab, /* The virtual table */ + const char *zToken, /* Text of the token */ + int nToken /* Number of characters in the token */ +){ + int i; + for(i=0; i<pVtab->nColumn; i++){ + if( memcmp(pVtab->azColumn[i], zToken, nToken)==0 + && pVtab->azColumn[i][nToken]==0 ){ + return i; + } + } + return -1; +} + +/* +** Parse the text at pSegment[0..nSegment-1]. Add additional terms +** to the query being assemblied in pQuery. +** +** inPhrase is true if pSegment[0..nSegement-1] is contained within +** double-quotes. If inPhrase is true, then the first term +** is marked with the number of terms in the phrase less one and +** OR and "-" syntax is ignored. If inPhrase is false, then every +** term found is marked with nPhrase=0 and OR and "-" syntax is significant. +*/ +static int tokenizeSegment( + sqlite3_tokenizer *pTokenizer, /* The tokenizer to use */ + const char *pSegment, int nSegment, /* Query expression being parsed */ + int inPhrase, /* True if within "..." */ + Query *pQuery /* Append results here */ +){ + const sqlite3_tokenizer_module *pModule = pTokenizer->pModule; + sqlite3_tokenizer_cursor *pCursor; + int firstIndex = pQuery->nTerms; + int iCol; + int nTerm = 1; + + int rc = pModule->xOpen(pTokenizer, pSegment, nSegment, &pCursor); + if( rc!=SQLITE_OK ) return rc; + pCursor->pTokenizer = pTokenizer; + + while( 1 ){ + const char *pToken; + int nToken, iBegin, iEnd, iPos; + + rc = pModule->xNext(pCursor, + &pToken, &nToken, + &iBegin, &iEnd, &iPos); + if( rc!=SQLITE_OK ) break; + if( !inPhrase && + pSegment[iEnd]==':' && + (iCol = checkColumnSpecifier(pQuery->pFts, pToken, nToken))>=0 ){ + pQuery->nextColumn = iCol; + continue; + } + if( !inPhrase && pQuery->nTerms>0 && nToken==2 + && pSegment[iBegin]=='O' && pSegment[iBegin+1]=='R' ){ + pQuery->nextIsOr = 1; + continue; + } + queryAdd(pQuery, pToken, nToken); + if( !inPhrase && iBegin>0 && pSegment[iBegin-1]=='-' ){ + pQuery->pTerms[pQuery->nTerms-1].isNot = 1; + } + pQuery->pTerms[pQuery->nTerms-1].iPhrase = nTerm; + if( inPhrase ){ + nTerm++; + } + } + + if( inPhrase && pQuery->nTerms>firstIndex ){ + pQuery->pTerms[firstIndex].nPhrase = pQuery->nTerms - firstIndex - 1; + } + + return pModule->xClose(pCursor); +} + +/* Parse a query string, yielding a Query object pQuery. +** +** The calling function will need to queryClear() to clean up +** the dynamically allocated memory held by pQuery. +*/ +static int parseQuery( + fulltext_vtab *v, /* The fulltext index */ + const char *zInput, /* Input text of the query string */ + int nInput, /* Size of the input text */ + int dfltColumn, /* Default column of the index to match against */ + Query *pQuery /* Write the parse results here. */ +){ + int iInput, inPhrase = 0; + + if( zInput==0 ) nInput = 0; + if( nInput<0 ) nInput = strlen(zInput); + pQuery->nTerms = 0; + pQuery->pTerms = NULL; + pQuery->nextIsOr = 0; + pQuery->nextColumn = dfltColumn; + pQuery->dfltColumn = dfltColumn; + pQuery->pFts = v; + + for(iInput=0; iInput<nInput; ++iInput){ + int i; + for(i=iInput; i<nInput && zInput[i]!='"'; ++i){} + if( i>iInput ){ + tokenizeSegment(v->pTokenizer, zInput+iInput, i-iInput, inPhrase, + pQuery); + } + iInput = i; + if( i<nInput ){ + assert( zInput[i]=='"' ); + inPhrase = !inPhrase; + } + } + + if( inPhrase ){ + /* unmatched quote */ + queryClear(pQuery); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + +/* Perform a full-text query using the search expression in +** zInput[0..nInput-1]. Return a list of matching documents +** in pResult. +** +** Queries must match column iColumn. Or if iColumn>=nColumn +** they are allowed to match against any column. +*/ +static int fulltextQuery( + fulltext_vtab *v, /* The full text index */ + int iColumn, /* Match against this column by default */ + const char *zInput, /* The query string */ + int nInput, /* Number of bytes in zInput[] */ + DocList **pResult, /* Write the result doclist here */ + Query *pQuery /* Put parsed query string here */ +){ + int i, iNext, rc; + DocList *pLeft = NULL; + DocList *pRight, *pNew, *pOr; + int nNot = 0; + QueryTerm *aTerm; + + rc = parseQuery(v, zInput, nInput, iColumn, pQuery); + if( rc!=SQLITE_OK ) return rc; + + /* Merge AND terms. */ + aTerm = pQuery->pTerms; + for(i = 0; i<pQuery->nTerms; i=iNext){ + if( aTerm[i].isNot ){ + /* Handle all NOT terms in a separate pass */ + nNot++; + iNext = i + aTerm[i].nPhrase+1; + continue; + } + iNext = i + aTerm[i].nPhrase + 1; + rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &pRight); + if( rc ){ + queryClear(pQuery); + return rc; + } + while( iNext<pQuery->nTerms && aTerm[iNext].isOr ){ + rc = docListOfTerm(v, aTerm[iNext].iColumn, &aTerm[iNext], &pOr); + iNext += aTerm[iNext].nPhrase + 1; + if( rc ){ + queryClear(pQuery); + return rc; + } + pNew = docListNew(DL_DOCIDS); + docListOrMerge(pRight, pOr, pNew); + docListDelete(pRight); + docListDelete(pOr); + pRight = pNew; + } + if( pLeft==0 ){ + pLeft = pRight; + }else{ + pNew = docListNew(DL_DOCIDS); + docListAndMerge(pLeft, pRight, pNew); + docListDelete(pRight); + docListDelete(pLeft); + pLeft = pNew; + } + } + + if( nNot && pLeft==0 ){ + /* We do not yet know how to handle a query of only NOT terms */ + return SQLITE_ERROR; + } + + /* Do the EXCEPT terms */ + for(i=0; i<pQuery->nTerms; i += aTerm[i].nPhrase + 1){ + if( !aTerm[i].isNot ) continue; + rc = docListOfTerm(v, aTerm[i].iColumn, &aTerm[i], &pRight); + if( rc ){ + queryClear(pQuery); + docListDelete(pLeft); + return rc; + } + pNew = docListNew(DL_DOCIDS); + docListExceptMerge(pLeft, pRight, pNew); + docListDelete(pRight); + docListDelete(pLeft); + pLeft = pNew; + } + + *pResult = pLeft; + return rc; +} + +/* +** This is the xFilter interface for the virtual table. See +** the virtual table xFilter method documentation for additional +** information. +** +** If idxNum==QUERY_GENERIC then do a full table scan against +** the %_content table. +** +** If idxNum==QUERY_ROWID then do a rowid lookup for a single entry +** in the %_content table. +** +** If idxNum>=QUERY_FULLTEXT then use the full text index. The +** column on the left-hand side of the MATCH operator is column +** number idxNum-QUERY_FULLTEXT, 0 indexed. argv[0] is the right-hand +** side of the MATCH operator. +*/ +/* TODO(shess) Upgrade the cursor initialization and destruction to +** account for fulltextFilter() being called multiple times on the +** same cursor. The current solution is very fragile. Apply fix to +** fts2 as appropriate. +*/ +static int fulltextFilter( + sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ + int idxNum, const char *idxStr, /* Which indexing scheme to use */ + int argc, sqlite3_value **argv /* Arguments for the indexing scheme */ +){ + fulltext_cursor *c = (fulltext_cursor *) pCursor; + fulltext_vtab *v = cursor_vtab(c); + int rc; + char *zSql; + + TRACE(("FTS1 Filter %p\n",pCursor)); + + zSql = sqlite3_mprintf("select rowid, * from %%_content %s", + idxNum==QUERY_GENERIC ? "" : "where rowid=?"); + sqlite3_finalize(c->pStmt); + rc = sql_prepare(v->db, v->zDb, v->zName, &c->pStmt, zSql); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ) return rc; + + c->iCursorType = idxNum; + switch( idxNum ){ + case QUERY_GENERIC: + break; + + case QUERY_ROWID: + rc = sqlite3_bind_int64(c->pStmt, 1, sqlite3_value_int64(argv[0])); + if( rc!=SQLITE_OK ) return rc; + break; + + default: /* full-text search */ + { + const char *zQuery = (const char *)sqlite3_value_text(argv[0]); + DocList *pResult; + assert( idxNum<=QUERY_FULLTEXT+v->nColumn); + assert( argc==1 ); + queryClear(&c->q); + rc = fulltextQuery(v, idxNum-QUERY_FULLTEXT, zQuery, -1, &pResult, &c->q); + if( rc!=SQLITE_OK ) return rc; + if( c->result.pDoclist!=NULL ) docListDelete(c->result.pDoclist); + readerInit(&c->result, pResult); + break; + } + } + + return fulltextNext(pCursor); +} + +/* This is the xEof method of the virtual table. The SQLite core +** calls this routine to find out if it has reached the end of +** a query's results set. +*/ +static int fulltextEof(sqlite3_vtab_cursor *pCursor){ + fulltext_cursor *c = (fulltext_cursor *) pCursor; + return c->eof; +} + +/* This is the xColumn method of the virtual table. The SQLite +** core calls this method during a query when it needs the value +** of a column from the virtual table. This method needs to use +** one of the sqlite3_result_*() routines to store the requested +** value back in the pContext. +*/ +static int fulltextColumn(sqlite3_vtab_cursor *pCursor, + sqlite3_context *pContext, int idxCol){ + fulltext_cursor *c = (fulltext_cursor *) pCursor; + fulltext_vtab *v = cursor_vtab(c); + + if( idxCol<v->nColumn ){ + sqlite3_value *pVal = sqlite3_column_value(c->pStmt, idxCol+1); + sqlite3_result_value(pContext, pVal); + }else if( idxCol==v->nColumn ){ + /* The extra column whose name is the same as the table. + ** Return a blob which is a pointer to the cursor + */ + sqlite3_result_blob(pContext, &c, sizeof(c), SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* This is the xRowid method. The SQLite core calls this routine to +** retrive the rowid for the current row of the result set. The +** rowid should be written to *pRowid. +*/ +static int fulltextRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + fulltext_cursor *c = (fulltext_cursor *) pCursor; + + *pRowid = sqlite3_column_int64(c->pStmt, 0); + return SQLITE_OK; +} + +/* Add all terms in [zText] to the given hash table. If [iColumn] > 0, + * we also store positions and offsets in the hash table using the given + * column number. */ +static int buildTerms(fulltext_vtab *v, fts1Hash *terms, sqlite_int64 iDocid, + const char *zText, int iColumn){ + sqlite3_tokenizer *pTokenizer = v->pTokenizer; + sqlite3_tokenizer_cursor *pCursor; + const char *pToken; + int nTokenBytes; + int iStartOffset, iEndOffset, iPosition; + int rc; + + rc = pTokenizer->pModule->xOpen(pTokenizer, zText, -1, &pCursor); + if( rc!=SQLITE_OK ) return rc; + + pCursor->pTokenizer = pTokenizer; + while( SQLITE_OK==pTokenizer->pModule->xNext(pCursor, + &pToken, &nTokenBytes, + &iStartOffset, &iEndOffset, + &iPosition) ){ + DocList *p; + + /* Positions can't be negative; we use -1 as a terminator internally. */ + if( iPosition<0 ){ + pTokenizer->pModule->xClose(pCursor); + return SQLITE_ERROR; + } + + p = fts1HashFind(terms, pToken, nTokenBytes); + if( p==NULL ){ + p = docListNew(DL_DEFAULT); + docListAddDocid(p, iDocid); + fts1HashInsert(terms, pToken, nTokenBytes, p); + } + if( iColumn>=0 ){ + docListAddPosOffset(p, iColumn, iPosition, iStartOffset, iEndOffset); + } + } + + /* TODO(shess) Check return? Should this be able to cause errors at + ** this point? Actually, same question about sqlite3_finalize(), + ** though one could argue that failure there means that the data is + ** not durable. *ponder* + */ + pTokenizer->pModule->xClose(pCursor); + return rc; +} + +/* Update the %_terms table to map the term [pTerm] to the given rowid. */ +static int index_insert_term(fulltext_vtab *v, const char *pTerm, int nTerm, + DocList *d){ + sqlite_int64 iIndexRow; + DocList doclist; + int iSegment = 0, rc; + + rc = term_select(v, pTerm, nTerm, iSegment, &iIndexRow, &doclist); + if( rc==SQLITE_DONE ){ + docListInit(&doclist, DL_DEFAULT, 0, 0); + docListUpdate(&doclist, d); + /* TODO(shess) Consider length(doclist)>CHUNK_MAX? */ + rc = term_insert(v, NULL, pTerm, nTerm, iSegment, &doclist); + goto err; + } + if( rc!=SQLITE_ROW ) return SQLITE_ERROR; + + docListUpdate(&doclist, d); + if( doclist.nData<=CHUNK_MAX ){ + rc = term_update(v, iIndexRow, &doclist); + goto err; + } + + /* Doclist doesn't fit, delete what's there, and accumulate + ** forward. + */ + rc = term_delete(v, iIndexRow); + if( rc!=SQLITE_OK ) goto err; + + /* Try to insert the doclist into a higher segment bucket. On + ** failure, accumulate existing doclist with the doclist from that + ** bucket, and put results in the next bucket. + */ + iSegment++; + while( (rc=term_insert(v, &iIndexRow, pTerm, nTerm, iSegment, + &doclist))!=SQLITE_OK ){ + sqlite_int64 iSegmentRow; + DocList old; + int rc2; + + /* Retain old error in case the term_insert() error was really an + ** error rather than a bounced insert. + */ + rc2 = term_select(v, pTerm, nTerm, iSegment, &iSegmentRow, &old); + if( rc2!=SQLITE_ROW ) goto err; + + rc = term_delete(v, iSegmentRow); + if( rc!=SQLITE_OK ) goto err; + + /* Reusing lowest-number deleted row keeps the index smaller. */ + if( iSegmentRow<iIndexRow ) iIndexRow = iSegmentRow; + + /* doclist contains the newer data, so accumulate it over old. + ** Then steal accumulated data for doclist. + */ + docListAccumulate(&old, &doclist); + docListDestroy(&doclist); + doclist = old; + + iSegment++; + } + + err: + docListDestroy(&doclist); + return rc; +} + +/* Add doclists for all terms in [pValues] to the hash table [terms]. */ +static int insertTerms(fulltext_vtab *v, fts1Hash *terms, sqlite_int64 iRowid, + sqlite3_value **pValues){ + int i; + for(i = 0; i < v->nColumn ; ++i){ + char *zText = (char*)sqlite3_value_text(pValues[i]); + int rc = buildTerms(v, terms, iRowid, zText, i); + if( rc!=SQLITE_OK ) return rc; + } + return SQLITE_OK; +} + +/* Add empty doclists for all terms in the given row's content to the hash + * table [pTerms]. */ +static int deleteTerms(fulltext_vtab *v, fts1Hash *pTerms, sqlite_int64 iRowid){ + const char **pValues; + int i; + + int rc = content_select(v, iRowid, &pValues); + if( rc!=SQLITE_OK ) return rc; + + for(i = 0 ; i < v->nColumn; ++i) { + rc = buildTerms(v, pTerms, iRowid, pValues[i], -1); + if( rc!=SQLITE_OK ) break; + } + + freeStringArray(v->nColumn, pValues); + return SQLITE_OK; +} + +/* Insert a row into the %_content table; set *piRowid to be the ID of the + * new row. Fill [pTerms] with new doclists for the %_term table. */ +static int index_insert(fulltext_vtab *v, sqlite3_value *pRequestRowid, + sqlite3_value **pValues, + sqlite_int64 *piRowid, fts1Hash *pTerms){ + int rc; + + rc = content_insert(v, pRequestRowid, pValues); /* execute an SQL INSERT */ + if( rc!=SQLITE_OK ) return rc; + *piRowid = sqlite3_last_insert_rowid(v->db); + return insertTerms(v, pTerms, *piRowid, pValues); +} + +/* Delete a row from the %_content table; fill [pTerms] with empty doclists + * to be written to the %_term table. */ +static int index_delete(fulltext_vtab *v, sqlite_int64 iRow, fts1Hash *pTerms){ + int rc = deleteTerms(v, pTerms, iRow); + if( rc!=SQLITE_OK ) return rc; + return content_delete(v, iRow); /* execute an SQL DELETE */ +} + +/* Update a row in the %_content table; fill [pTerms] with new doclists for the + * %_term table. */ +static int index_update(fulltext_vtab *v, sqlite_int64 iRow, + sqlite3_value **pValues, fts1Hash *pTerms){ + /* Generate an empty doclist for each term that previously appeared in this + * row. */ + int rc = deleteTerms(v, pTerms, iRow); + if( rc!=SQLITE_OK ) return rc; + + rc = content_update(v, pValues, iRow); /* execute an SQL UPDATE */ + if( rc!=SQLITE_OK ) return rc; + + /* Now add positions for terms which appear in the updated row. */ + return insertTerms(v, pTerms, iRow, pValues); +} + +/* This function implements the xUpdate callback; it is the top-level entry + * point for inserting, deleting or updating a row in a full-text table. */ +static int fulltextUpdate(sqlite3_vtab *pVtab, int nArg, sqlite3_value **ppArg, + sqlite_int64 *pRowid){ + fulltext_vtab *v = (fulltext_vtab *) pVtab; + fts1Hash terms; /* maps term string -> PosList */ + int rc; + fts1HashElem *e; + + TRACE(("FTS1 Update %p\n", pVtab)); + + fts1HashInit(&terms, FTS1_HASH_STRING, 1); + + if( nArg<2 ){ + rc = index_delete(v, sqlite3_value_int64(ppArg[0]), &terms); + } else if( sqlite3_value_type(ppArg[0]) != SQLITE_NULL ){ + /* An update: + * ppArg[0] = old rowid + * ppArg[1] = new rowid + * ppArg[2..2+v->nColumn-1] = values + * ppArg[2+v->nColumn] = value for magic column (we ignore this) + */ + sqlite_int64 rowid = sqlite3_value_int64(ppArg[0]); + if( sqlite3_value_type(ppArg[1]) != SQLITE_INTEGER || + sqlite3_value_int64(ppArg[1]) != rowid ){ + rc = SQLITE_ERROR; /* we don't allow changing the rowid */ + } else { + assert( nArg==2+v->nColumn+1); + rc = index_update(v, rowid, &ppArg[2], &terms); + } + } else { + /* An insert: + * ppArg[1] = requested rowid + * ppArg[2..2+v->nColumn-1] = values + * ppArg[2+v->nColumn] = value for magic column (we ignore this) + */ + assert( nArg==2+v->nColumn+1); + rc = index_insert(v, ppArg[1], &ppArg[2], pRowid, &terms); + } + + if( rc==SQLITE_OK ){ + /* Write updated doclists to disk. */ + for(e=fts1HashFirst(&terms); e; e=fts1HashNext(e)){ + DocList *p = fts1HashData(e); + rc = index_insert_term(v, fts1HashKey(e), fts1HashKeysize(e), p); + if( rc!=SQLITE_OK ) break; + } + } + + /* clean up */ + for(e=fts1HashFirst(&terms); e; e=fts1HashNext(e)){ + DocList *p = fts1HashData(e); + docListDelete(p); + } + fts1HashClear(&terms); + + return rc; +} + +/* +** Implementation of the snippet() function for FTS1 +*/ +static void snippetFunc( + sqlite3_context *pContext, + int argc, + sqlite3_value **argv +){ + fulltext_cursor *pCursor; + if( argc<1 ) return; + if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ + sqlite3_result_error(pContext, "illegal first argument to html_snippet",-1); + }else{ + const char *zStart = "<b>"; + const char *zEnd = "</b>"; + const char *zEllipsis = "<b>...</b>"; + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); + if( argc>=2 ){ + zStart = (const char*)sqlite3_value_text(argv[1]); + if( argc>=3 ){ + zEnd = (const char*)sqlite3_value_text(argv[2]); + if( argc>=4 ){ + zEllipsis = (const char*)sqlite3_value_text(argv[3]); + } + } + } + snippetAllOffsets(pCursor); + snippetText(pCursor, zStart, zEnd, zEllipsis); + sqlite3_result_text(pContext, pCursor->snippet.zSnippet, + pCursor->snippet.nSnippet, SQLITE_STATIC); + } +} + +/* +** Implementation of the offsets() function for FTS1 +*/ +static void snippetOffsetsFunc( + sqlite3_context *pContext, + int argc, + sqlite3_value **argv +){ + fulltext_cursor *pCursor; + if( argc<1 ) return; + if( sqlite3_value_type(argv[0])!=SQLITE_BLOB || + sqlite3_value_bytes(argv[0])!=sizeof(pCursor) ){ + sqlite3_result_error(pContext, "illegal first argument to offsets",-1); + }else{ + memcpy(&pCursor, sqlite3_value_blob(argv[0]), sizeof(pCursor)); + snippetAllOffsets(pCursor); + snippetOffsetText(&pCursor->snippet); + sqlite3_result_text(pContext, + pCursor->snippet.zOffset, pCursor->snippet.nOffset, + SQLITE_STATIC); + } +} + +/* +** This routine implements the xFindFunction method for the FTS1 +** virtual table. +*/ +static int fulltextFindFunction( + sqlite3_vtab *pVtab, + int nArg, + const char *zName, + void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), + void **ppArg +){ + if( strcmp(zName,"snippet")==0 ){ + *pxFunc = snippetFunc; + return 1; + }else if( strcmp(zName,"offsets")==0 ){ + *pxFunc = snippetOffsetsFunc; + return 1; + } + return 0; +} + +/* +** Rename an fts1 table. +*/ +static int fulltextRename( + sqlite3_vtab *pVtab, + const char *zName +){ + fulltext_vtab *p = (fulltext_vtab *)pVtab; + int rc = SQLITE_NOMEM; + char *zSql = sqlite3_mprintf( + "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';" + "ALTER TABLE %Q.'%q_term' RENAME TO '%q_term';" + , p->zDb, p->zName, zName + , p->zDb, p->zName, zName + ); + if( zSql ){ + rc = sqlite3_exec(p->db, zSql, 0, 0, 0); + sqlite3_free(zSql); + } + return rc; +} + +static const sqlite3_module fulltextModule = { + /* iVersion */ 0, + /* xCreate */ fulltextCreate, + /* xConnect */ fulltextConnect, + /* xBestIndex */ fulltextBestIndex, + /* xDisconnect */ fulltextDisconnect, + /* xDestroy */ fulltextDestroy, + /* xOpen */ fulltextOpen, + /* xClose */ fulltextClose, + /* xFilter */ fulltextFilter, + /* xNext */ fulltextNext, + /* xEof */ fulltextEof, + /* xColumn */ fulltextColumn, + /* xRowid */ fulltextRowid, + /* xUpdate */ fulltextUpdate, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindFunction */ fulltextFindFunction, + /* xRename */ fulltextRename, +}; + +int sqlite3Fts1Init(sqlite3 *db){ + sqlite3_overload_function(db, "snippet", -1); + sqlite3_overload_function(db, "offsets", -1); + return sqlite3_create_module(db, "fts1", &fulltextModule, 0); +} + +#if !SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fts1_init(sqlite3 *db, char **pzErrMsg, + const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi) + return sqlite3Fts1Init(db); +} +#endif + +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS1) */ |