summaryrefslogtreecommitdiffstats
path: root/ext/fts5
diff options
context:
space:
mode:
Diffstat (limited to 'ext/fts5')
-rw-r--r--ext/fts5/fts5_expr.c7
-rw-r--r--ext/fts5/fts5_main.c29
-rw-r--r--ext/fts5/fts5_tokenize.c12
-rw-r--r--ext/fts5/test/fts5aux.test24
-rw-r--r--ext/fts5/test/fts5integrity.test27
-rw-r--r--ext/fts5/test/fts5secure8.test16
-rw-r--r--ext/fts5/test/fts5tokenizer2.test20
-rw-r--r--ext/fts5/test/fts5trigram.test5
-rw-r--r--ext/fts5/test/fts5trigram2.test3
9 files changed, 126 insertions, 17 deletions
diff --git a/ext/fts5/fts5_expr.c b/ext/fts5/fts5_expr.c
index 05c1b59..960a4d4 100644
--- a/ext/fts5/fts5_expr.c
+++ b/ext/fts5/fts5_expr.c
@@ -324,7 +324,11 @@ int sqlite3Fts5ExprNew(
}
sqlite3_free(sParse.apPhrase);
- *pzErr = sParse.zErr;
+ if( 0==*pzErr ){
+ *pzErr = sParse.zErr;
+ }else{
+ sqlite3_free(sParse.zErr);
+ }
return sParse.rc;
}
@@ -2452,6 +2456,7 @@ Fts5ExprNode *sqlite3Fts5ParseImplicitAnd(
assert( pRight->eType==FTS5_STRING
|| pRight->eType==FTS5_TERM
|| pRight->eType==FTS5_EOF
+ || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd)
);
if( pLeft->eType==FTS5_AND ){
diff --git a/ext/fts5/fts5_main.c b/ext/fts5/fts5_main.c
index f609f7f..837ea40 100644
--- a/ext/fts5/fts5_main.c
+++ b/ext/fts5/fts5_main.c
@@ -1700,6 +1700,7 @@ static int fts5UpdateMethod(
rc = SQLITE_ERROR;
}else{
rc = fts5SpecialDelete(pTab, apVal);
+ bUpdateOrDelete = 1;
}
}else{
rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]);
@@ -2874,14 +2875,16 @@ int sqlite3Fts5GetTokenizer(
if( pMod==0 ){
assert( nArg>0 );
rc = SQLITE_ERROR;
- *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
+ if( pzErr ) *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]);
}else{
rc = pMod->x.xCreate(
pMod->pUserData, (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->pTok
);
pConfig->pTokApi = &pMod->x;
if( rc!=SQLITE_OK ){
- if( pzErr ) *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+ if( pzErr && rc!=SQLITE_NOMEM ){
+ *pzErr = sqlite3_mprintf("error in tokenizer constructor");
+ }
}else{
pConfig->ePattern = sqlite3Fts5TokenizerPattern(
pMod->x.xCreate, pConfig->pTok
@@ -2975,17 +2978,23 @@ static int fts5IntegrityMethod(
assert( pzErr!=0 && *pzErr==0 );
UNUSED_PARAM(isQuick);
+ assert( pTab->p.pConfig->pzErrmsg==0 );
+ pTab->p.pConfig->pzErrmsg = pzErr;
rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0);
- if( (rc&0xff)==SQLITE_CORRUPT ){
- *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
- zSchema, zTabname);
- rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
- }else if( rc!=SQLITE_OK ){
- *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
- " FTS5 table %s.%s: %s",
- zSchema, zTabname, sqlite3_errstr(rc));
+ if( *pzErr==0 && rc!=SQLITE_OK ){
+ if( (rc&0xff)==SQLITE_CORRUPT ){
+ *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s",
+ zSchema, zTabname);
+ rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM;
+ }else{
+ *pzErr = sqlite3_mprintf("unable to validate the inverted index for"
+ " FTS5 table %s.%s: %s",
+ zSchema, zTabname, sqlite3_errstr(rc));
+ }
}
+
sqlite3Fts5IndexCloseReader(pTab->p.pIndex);
+ pTab->p.pConfig->pzErrmsg = 0;
return rc;
}
diff --git a/ext/fts5/fts5_tokenize.c b/ext/fts5/fts5_tokenize.c
index f120561..2200e78 100644
--- a/ext/fts5/fts5_tokenize.c
+++ b/ext/fts5/fts5_tokenize.c
@@ -79,7 +79,7 @@ static int fts5AsciiCreate(
int i;
memset(p, 0, sizeof(AsciiTokenizer));
memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar));
- for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+ for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){
fts5AsciiAddExceptions(p, zArg, 1);
@@ -90,6 +90,7 @@ static int fts5AsciiCreate(
rc = SQLITE_ERROR;
}
}
+ if( rc==SQLITE_OK && i<nArg ) rc = SQLITE_ERROR;
if( rc!=SQLITE_OK ){
fts5AsciiDelete((Fts5Tokenizer*)p);
p = 0;
@@ -381,17 +382,16 @@ static int fts5UnicodeCreate(
}
/* Search for a "categories" argument */
- for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+ for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
if( 0==sqlite3_stricmp(azArg[i], "categories") ){
zCat = azArg[i+1];
}
}
-
if( rc==SQLITE_OK ){
rc = unicodeSetCategories(p, zCat);
}
- for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+ for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){
if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){
@@ -416,6 +416,7 @@ static int fts5UnicodeCreate(
rc = SQLITE_ERROR;
}
}
+ if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
}else{
rc = SQLITE_NOMEM;
@@ -1298,7 +1299,7 @@ static int fts5TriCreate(
int i;
pNew->bFold = 1;
pNew->iFoldParam = 0;
- for(i=0; rc==SQLITE_OK && i<nArg; i+=2){
+ for(i=0; rc==SQLITE_OK && i<nArg-1; i+=2){
const char *zArg = azArg[i+1];
if( 0==sqlite3_stricmp(azArg[i], "case_sensitive") ){
if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1] ){
@@ -1316,6 +1317,7 @@ static int fts5TriCreate(
rc = SQLITE_ERROR;
}
}
+ if( i<nArg && rc==SQLITE_OK ) rc = SQLITE_ERROR;
if( pNew->iFoldParam!=0 && pNew->bFold==0 ){
rc = SQLITE_ERROR;
diff --git a/ext/fts5/test/fts5aux.test b/ext/fts5/test/fts5aux.test
index 5569f48..7963006 100644
--- a/ext/fts5/test/fts5aux.test
+++ b/ext/fts5/test/fts5aux.test
@@ -377,4 +377,28 @@ do_catchsql_test 12.3.3 {
SELECT fts5_collist(t1, 1) FROM t1('one AND two');
} {0 1}
+#-------------------------------------------------------------------------
+reset_db
+do_execsql_test 13.1 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
+ INSERT INTO t1 VALUES('a b c'), ('d e f');
+ PRAGMA integrity_check;
+} {ok}
+
+do_catchsql_test 13.2 {
+ SELECT highlight(t1, 0, '[', ']') FROM t1
+} {0 {{a b c} {d e f}}}
+
+do_execsql_test 13.3 {
+ PRAGMA writable_schema = 1;
+ UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
+ WHERE name = 't1';
+}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 13.4 {
+ SELECT highlight(t1, 0, '[', ']') FROM t1
+} {1 {no such tokenizer: blah}}
+
finish_test
diff --git a/ext/fts5/test/fts5integrity.test b/ext/fts5/test/fts5integrity.test
index 1bb3675..5ed659b 100644
--- a/ext/fts5/test/fts5integrity.test
+++ b/ext/fts5/test/fts5integrity.test
@@ -380,5 +380,32 @@ do_execsql_test 12.3 {
} {ok}
+#-------------------------------------------------------------------
+reset_db
+do_execsql_test 13.1 {
+ CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=ascii);
+ INSERT INTO t1 VALUES('a b c'), ('d e f');
+ PRAGMA integrity_check;
+} {ok}
+
+db close
+sqlite3 db test.db
+do_catchsql_test 13.2 {
+ PRAGMA integrity_check;
+} {0 ok}
+
+do_execsql_test 13.3 {
+ PRAGMA writable_schema = 1;
+ UPDATE sqlite_schema SET sql = 'CREATE VIRTUAL TABLE t1 USING fts5(a, tokenize=blah)'
+ WHERE name = 't1';
+}
+
+db close
+sqlite3 db test.db
+breakpoint
+do_catchsql_test 13.4 {
+ PRAGMA integrity_check;
+} {1 {no such tokenizer: blah}}
+
finish_test
diff --git a/ext/fts5/test/fts5secure8.test b/ext/fts5/test/fts5secure8.test
index 8ceb963..0216bb6 100644
--- a/ext/fts5/test/fts5secure8.test
+++ b/ext/fts5/test/fts5secure8.test
@@ -42,6 +42,22 @@ do_execsql_test 1.2 {
PRAGMA integrity_check;
} {ok}
+do_execsql_test 2.0 {
+ CREATE VIRTUAL TABLE xyz USING fts5 (
+ name,
+ content=''
+ );
+
+ INSERT INTO xyz(xyz, rank) VALUES('secure-delete', 1);
+ INSERT INTO xyz (rowid, name) VALUES(1, 'A');
+ INSERT INTO xyz (rowid, name) VALUES(2, 'A');
+ INSERT INTO xyz(xyz, rowid, name) VALUES('delete', 2, 'A');
+}
+
+do_execsql_test 2.1 {
+ pragma quick_check;
+} {ok}
+
diff --git a/ext/fts5/test/fts5tokenizer2.test b/ext/fts5/test/fts5tokenizer2.test
index bdabd53..52b3032 100644
--- a/ext/fts5/test/fts5tokenizer2.test
+++ b/ext/fts5/test/fts5tokenizer2.test
@@ -85,5 +85,25 @@ do_execsql_test 1.7 {
SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
} {AAdont>BBmess<}
+# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd
+# Error handling of tokenize= arguments.
+#
+foreach {n tkz} {
+ 1 {ascii none}
+ 2 {unicode61 none}
+ 3 {porter none}
+ 4 {trigram none}
+ 5 {ascii none 0}
+ 6 {unicode61 none 0}
+ 7 {porter none 0}
+ 8 {trigram none 0}
+} {
+ db eval {DROP TABLE IF EXISTS t2;}
+ do_catchsql_test 2.$n "
+ DROP TABLE IF EXISTS t2;
+ CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz');
+ " {1 {error in tokenizer constructor}}
+}
+
finish_test
diff --git a/ext/fts5/test/fts5trigram.test b/ext/fts5/test/fts5trigram.test
index 351c059..7526866 100644
--- a/ext/fts5/test/fts5trigram.test
+++ b/ext/fts5/test/fts5trigram.test
@@ -69,6 +69,9 @@ do_execsql_test 2.0 {
INSERT INTO t1 VALUES('abcdefghijklm');
INSERT INTO t1 VALUES('กรุงเทพมหานคร');
}
+do_catchsql_test 2.0.1 {
+ CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram case_sensitive');
+} {1 {error in tokenizer constructor}}
foreach {tn s res} {
1 abc "(abc)defghijklm"
@@ -206,7 +209,7 @@ do_execsql_test 7.0 {
(20, "жираф.png"),
(30, "cat.png"),
(40, "кот.png"),
- (50, "misic-🎵-.mp3");
+ (50, "misic-🎵-.mp3");
}
do_execsql_test 7.1 {
SELECT rowid FROM f WHERE +filename GLOB '*ир*';
diff --git a/ext/fts5/test/fts5trigram2.test b/ext/fts5/test/fts5trigram2.test
index f5beae5..395d899 100644
--- a/ext/fts5/test/fts5trigram2.test
+++ b/ext/fts5/test/fts5trigram2.test
@@ -21,6 +21,9 @@ do_execsql_test 1.0 "
INSERT INTO t1 VALUES('abc\u0303defghijklm');
INSERT INTO t1 VALUES('a\u0303b\u0303c\u0303defghijklm');
"
+do_catchsql_test 1.0.1 {
+ CREATE VIRTUAL TABLE t2 USING fts5(z, tokenize='trigram remove_diacritics');
+} {1 {error in tokenizer constructor}}
do_execsql_test 1.1 {
SELECT highlight(t1, 0, '(', ')') FROM t1('abc');
generated by cgit v1.2.3 (git 2.39.1) at 2025-02-17 13:03:23 +0000