1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
|
# 2023 Nov 03
#
# The author disclaims copyright to this source code. In place of
# a legal notice, here is a blessing:
#
# May you do good and not evil.
# May you find forgiveness for yourself and forgive others.
# May you share freely, never taking more than you give.
#
#***********************************************************************
#
# Tests focusing on the built-in fts5 tokenizers.
#
source [file join [file dirname [info script]] fts5_common.tcl]
set testprefix fts5tokenizer2
# If SQLITE_ENABLE_FTS5 is defined, omit this file.
ifcapable !fts5 {
finish_test
return
}
sqlite3_fts5_create_tokenizer db tst get_tst_tokenizer
proc get_tst_tokenizer {args} {
return "tst_tokenizer"
}
proc tst_tokenizer {flags txt} {
set token ""
set lTok [list]
foreach c [split $txt {}] {
if {$token==""} {
append token $c
} else {
set t1 [string is upper $token]
set t2 [string is upper $c]
if {$t1!=$t2} {
lappend lTok $token
set token ""
}
append token $c
}
}
if {$token!=""} { lappend lTok $token }
set iOff 0
foreach t $lTok {
set n [string length $t]
sqlite3_fts5_token $t $iOff [expr $iOff+$n]
incr iOff $n
}
}
do_execsql_test 1.0 {
CREATE VIRTUAL TABLE t1 USING fts5(t, tokenize=tst);
}
do_execsql_test 1.1 {
INSERT INTO t1 VALUES('AAdontBBmess');
}
do_execsql_test 1.2 {
SELECT snippet(t1, 0, '>', '<', '...', 4) FROM t1('BB');
} {AAdont>BB<mess}
do_execsql_test 1.3 {
SELECT highlight(t1, 0, '>', '<') FROM t1('BB');
} {AAdont>BB<mess}
do_execsql_test 1.4 {
SELECT highlight(t1, 0, '>', '<') FROM t1('AA');
} {>AA<dontBBmess}
do_execsql_test 1.5 {
SELECT highlight(t1, 0, '>', '<') FROM t1('dont');
} {AA>dont<BBmess}
do_execsql_test 1.6 {
SELECT highlight(t1, 0, '>', '<') FROM t1('mess');
} {AAdontBB>mess<}
do_execsql_test 1.7 {
SELECT highlight(t1, 0, '>', '<') FROM t1('BB mess');
} {AAdont>BBmess<}
# 2024-08-06 https://sqlite.org/forum/forumpost/171bcc2bcd
# Error handling of tokenize= arguments.
#
foreach {n tkz} {
1 {ascii none}
2 {unicode61 none}
3 {porter none}
4 {trigram none}
5 {ascii none 0}
6 {unicode61 none 0}
7 {porter none 0}
8 {trigram none 0}
} {
db eval {DROP TABLE IF EXISTS t2;}
do_catchsql_test 2.$n "
DROP TABLE IF EXISTS t2;
CREATE VIRTUAL TABLE t2 USING fts5(a,b,c,tokenize='$tkz');
" {1 {error in tokenizer constructor}}
}
finish_test
|
