summaryrefslogtreecommitdiffstats
path: root/test/fts4unicode.test
diff options
context:
space:
mode:
Diffstat (limited to 'test/fts4unicode.test')
-rw-r--r--test/fts4unicode.test588
1 files changed, 588 insertions, 0 deletions
diff --git a/test/fts4unicode.test b/test/fts4unicode.test
new file mode 100644
index 0000000..facf2bf
--- /dev/null
+++ b/test/fts4unicode.test
@@ -0,0 +1,588 @@
+# 2012 May 25
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#*************************************************************************
+#
+# The tests in this file focus on testing the "unicode" FTS tokenizer.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+ifcapable !fts3_unicode { finish_test ; return }
+set ::testprefix fts4unicode
+
+proc do_unicode_token_test {tn input res} {
+ set input [string map {' ''} $input]
+ uplevel [list do_execsql_test $tn "
+ SELECT fts3_tokenizer_test('unicode61', 'remove_diacritics=0', '$input');
+ " [list [list {*}$res]]]
+}
+
+proc do_unicode_token_test2 {tn input res} {
+ set input [string map {' ''} $input]
+ uplevel [list do_execsql_test $tn "
+ SELECT fts3_tokenizer_test('unicode61', '$input');
+ " [list [list {*}$res]]]
+}
+
+proc do_unicode_token_test3 {tn args} {
+ set res [lindex $args end]
+ set sql "SELECT fts3_tokenizer_test('unicode61'"
+ foreach a [lrange $args 0 end-1] {
+ append sql ", '"
+ append sql [string map {' ''} $a]
+ append sql "'"
+ }
+ append sql ")"
+ uplevel [list do_execsql_test $tn $sql [list [list {*}$res]]]
+}
+
+do_unicode_token_test 1.0 {a B c D} {0 a a 1 b B 2 c c 3 d D}
+
+do_unicode_token_test 1.1 "\uC4 \uD6 \uDC" \
+ "0 \uE4 \uC4 1 \uF6 \uD6 2 \uFC \uDC"
+
+do_unicode_token_test 1.2 "x\uC4x x\uD6x x\uDCx" \
+ "0 x\uE4x x\uC4x 1 x\uF6x x\uD6x 2 x\uFCx x\uDCx"
+
+# 0x00DF is a small "sharp s". 0x1E9E is a capital sharp s.
+do_unicode_token_test 1.3 "\uDF" "0 \uDF \uDF"
+do_unicode_token_test 1.4 "\u1E9E" "0 \uDF \u1E9E"
+
+do_unicode_token_test 1.5 "The quick brown fox" {
+ 0 the The 1 quick quick 2 brown brown 3 fox fox
+}
+do_unicode_token_test 1.6 "The\u00bfquick\u224ebrown\u2263fox" {
+ 0 the The 1 quick quick 2 brown brown 3 fox fox
+}
+
+do_unicode_token_test2 1.7 {a B c D} {0 a a 1 b B 2 c c 3 d D}
+do_unicode_token_test2 1.8 "\uC4 \uD6 \uDC" "0 a \uC4 1 o \uD6 2 u \uDC"
+
+do_unicode_token_test2 1.9 "x\uC4x x\uD6x x\uDCx" \
+ "0 xax x\uC4x 1 xox x\uD6x 2 xux x\uDCx"
+
+# Check that diacritics are removed if remove_diacritics=1 is specified.
+# And that they do not break tokens.
+do_unicode_token_test2 1.10 "xx\u0301xx" "0 xxxx xx\u301xx"
+
+# Title-case mappings work
+do_unicode_token_test 1.11 "\u01c5" "0 \u01c6 \u01c5"
+
+#-------------------------------------------------------------------------
+#
+set docs [list {
+ Enhance the INSERT syntax to allow multiple rows to be inserted via the
+ VALUES clause.
+} {
+ Enhance the CREATE VIRTUAL TABLE command to support the IF NOT EXISTS clause.
+} {
+ Added the sqlite3_stricmp() interface as a counterpart to sqlite3_strnicmp().
+} {
+ Added the sqlite3_db_readonly() interface.
+} {
+ Added the SQLITE_FCNTL_PRAGMA file control, giving VFS implementations the
+ ability to add new PRAGMA statements or to override built-in PRAGMAs.
+} {
+ Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
+ the same row that contains the maximum x value.
+} {
+ Added support for the FTS4 languageid option.
+} {
+ Documented support for the FTS4 content option. This feature has actually
+ been in the code since version 3.7.9 but is only now considered to be
+ officially supported.
+} {
+ Pending statements no longer block ROLLBACK. Instead, the pending statement
+ will return SQLITE_ABORT upon next access after the ROLLBACK.
+} {
+ Improvements to the handling of CSV inputs in the command-line shell
+} {
+ Fix a bug introduced in version 3.7.10 that might cause a LEFT JOIN to be
+ incorrectly converted into an INNER JOIN if the WHERE clause indexable terms
+ connected by OR.
+}]
+
+set map(a) [list "\u00C4" "\u00E4"] ; # LATIN LETTER A WITH DIAERESIS
+set map(e) [list "\u00CB" "\u00EB"] ; # LATIN LETTER E WITH DIAERESIS
+set map(i) [list "\u00CF" "\u00EF"] ; # LATIN LETTER I WITH DIAERESIS
+set map(o) [list "\u00D6" "\u00F6"] ; # LATIN LETTER O WITH DIAERESIS
+set map(u) [list "\u00DC" "\u00FC"] ; # LATIN LETTER U WITH DIAERESIS
+set map(y) [list "\u0178" "\u00FF"] ; # LATIN LETTER Y WITH DIAERESIS
+set map(h) [list "\u1E26" "\u1E27"] ; # LATIN LETTER H WITH DIAERESIS
+set map(w) [list "\u1E84" "\u1E85"] ; # LATIN LETTER W WITH DIAERESIS
+set map(x) [list "\u1E8C" "\u1E8D"] ; # LATIN LETTER X WITH DIAERESIS
+foreach k [array names map] {
+ lappend mappings [string toupper $k] [lindex $map($k) 0]
+ lappend mappings $k [lindex $map($k) 1]
+}
+proc mapdoc {doc} {
+ set doc [regsub -all {[[:space:]]+} $doc " "]
+ string map $::mappings [string trim $doc]
+}
+
+do_test 2.0 {
+ execsql { CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61, x); }
+ foreach doc $docs {
+ set d [mapdoc $doc]
+ execsql { INSERT INTO t2 VALUES($d) }
+ }
+} {}
+
+do_test 2.1 {
+ set q [mapdoc "row"]
+ execsql { SELECT * FROM t2 WHERE t2 MATCH $q }
+} [list [mapdoc {
+ Queries of the form: "SELECT max(x), y FROM table" returns the value of y on
+ the same row that contains the maximum x value.
+}]]
+
+foreach {tn query snippet} {
+ 2 "row" {
+ ...returns the value of y on the same [row] that contains
+ the maximum x value.
+ }
+ 3 "ROW" {
+ ...returns the value of y on the same [row] that contains
+ the maximum x value.
+ }
+ 4 "rollback" {
+ ...[ROLLBACK]. Instead, the pending statement
+ will return SQLITE_ABORT upon next access after the [ROLLBACK].
+ }
+ 5 "rOllback" {
+ ...[ROLLBACK]. Instead, the pending statement
+ will return SQLITE_ABORT upon next access after the [ROLLBACK].
+ }
+ 6 "lang*" {
+ Added support for the FTS4 [languageid] option.
+ }
+} {
+ do_test 2.$tn {
+ set q [mapdoc $query]
+ execsql { SELECT snippet(t2, '[', ']', '...') FROM t2 WHERE t2 MATCH $q }
+ } [list [mapdoc $snippet]]
+}
+
+#-------------------------------------------------------------------------
+# Make sure the unicode61 tokenizer does not crash if it is passed a
+# NULL pointer.
+reset_db
+do_execsql_test 3.1 {
+ CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x, y);
+ INSERT INTO t1 VALUES(NULL, 'a b c');
+}
+
+do_execsql_test 3.2 {
+ SELECT snippet(t1, '[', ']') FROM t1 WHERE t1 MATCH 'b'
+} {{a [b] c}}
+
+do_execsql_test 3.3 {
+ BEGIN;
+ DELETE FROM t1;
+ INSERT INTO t1 VALUES('b b b b b b b b b b b', 'b b b b b b b b b b b b b');
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 SELECT * FROM t1;
+ INSERT INTO t1 VALUES('a b c', NULL);
+ INSERT INTO t1 VALUES('a x c', NULL);
+ COMMIT;
+}
+
+do_execsql_test 3.4 {
+ SELECT * FROM t1 WHERE t1 MATCH 'a b';
+} {{a b c} {}}
+
+#-------------------------------------------------------------------------
+#
+reset_db
+
+do_test 4.1 {
+ set a "abc\uFFFEdef"
+ set b "abc\uD800def"
+ set c "\uFFFEdef"
+ set d "\uD800def"
+ execsql {
+ CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61, x);
+ INSERT INTO t1 VALUES($a);
+ INSERT INTO t1 VALUES($b);
+ INSERT INTO t1 VALUES($c);
+ INSERT INTO t1 VALUES($d);
+ }
+} {}
+
+do_test 4.2 {
+ set a [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0x62}]
+ set b [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0x62}]
+ set c [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
+ set d [binary format c* {0x61 0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF 0x62}]
+ execsql {
+ INSERT INTO t1 VALUES($a);
+ INSERT INTO t1 VALUES($b);
+ INSERT INTO t1 VALUES($c);
+ INSERT INTO t1 VALUES($d);
+ }
+} {}
+
+do_test 4.3 {
+ set a [binary format c* {0xF7 0xBF 0xBF 0xBF}]
+ set b [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF}]
+ set c [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF}]
+ set d [binary format c* {0xF7 0xBF 0xBF 0xBF 0xBF 0xBF 0xBF}]
+ execsql {
+ INSERT INTO t1 VALUES($a);
+ INSERT INTO t1 VALUES($b);
+ INSERT INTO t1 VALUES($c);
+ INSERT INTO t1 VALUES($d);
+ }
+} {}
+
+#-------------------------------------------------------------------------
+
+do_unicode_token_test3 5.1 {tokenchars=} {
+ sqlite3_reset sqlite3_column_int
+} {
+ 0 sqlite3 sqlite3
+ 1 reset reset
+ 2 sqlite3 sqlite3
+ 3 column column
+ 4 int int
+}
+
+do_unicode_token_test3 5.2 {tokenchars=_} {
+ sqlite3_reset sqlite3_column_int
+} {
+ 0 sqlite3_reset sqlite3_reset
+ 1 sqlite3_column_int sqlite3_column_int
+}
+
+do_unicode_token_test3 5.3 {separators=xyz} {
+ Laotianxhorseyrunszfast
+} {
+ 0 laotian Laotian
+ 1 horse horse
+ 2 runs runs
+ 3 fast fast
+}
+
+do_unicode_token_test3 5.4 {tokenchars=xyz} {
+ Laotianxhorseyrunszfast
+} {
+ 0 laotianxhorseyrunszfast Laotianxhorseyrunszfast
+}
+
+do_unicode_token_test3 5.5 {tokenchars=_} {separators=zyx} {
+ sqlite3_resetxsqlite3_column_intyhonda_phantom
+} {
+ 0 sqlite3_reset sqlite3_reset
+ 1 sqlite3_column_int sqlite3_column_int
+ 2 honda_phantom honda_phantom
+}
+
+do_unicode_token_test3 5.6 "separators=\u05D1" "abc\u05D1def" {
+ 0 abc abc 1 def def
+}
+
+do_unicode_token_test3 5.7 \
+ "tokenchars=\u2444\u2445" \
+ "separators=\u05D0\u05D1\u05D2" \
+ "\u2444fre\u2445sh\u05D0water\u05D2fish.\u2445timer" \
+ [list \
+ 0 \u2444fre\u2445sh \u2444fre\u2445sh \
+ 1 water water \
+ 2 fish fish \
+ 3 \u2445timer \u2445timer \
+ ]
+
+# Check that it is not possible to add a standalone diacritic codepoint
+# to either separators or tokenchars.
+do_unicode_token_test3 5.8 "separators=\u0301" \
+ "hello\u0301world \u0301helloworld" \
+ "0 helloworld hello\u0301world 1 helloworld helloworld"
+
+do_unicode_token_test3 5.9 "tokenchars=\u0301" \
+ "hello\u0301world \u0301helloworld" \
+ "0 helloworld hello\u0301world 1 helloworld helloworld"
+
+do_unicode_token_test3 5.10 "separators=\u0301" \
+ "remove_diacritics=0" \
+ "hello\u0301world \u0301helloworld" \
+ "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
+
+do_unicode_token_test3 5.11 "tokenchars=\u0301" \
+ "remove_diacritics=0" \
+ "hello\u0301world \u0301helloworld" \
+ "0 hello\u0301world hello\u0301world 1 helloworld helloworld"
+
+
+#-------------------------------------------------------------------------
+
+proc do_tokenize {tokenizer txt} {
+ set res [list]
+ foreach {a b c} [db one {SELECT fts3_tokenizer_test($tokenizer, $txt)}] {
+ lappend res $b
+ }
+ set res
+}
+
+# Argument $lCodepoint must be a list of codepoints (integers) that
+# correspond to whitespace characters. This command creates a string
+# $W from the codepoints, then tokenizes "${W}hello{$W}world${W}"
+# using tokenizer $tokenizer. The test passes if the tokenizer successfully
+# extracts the two 5 character tokens.
+#
+proc do_isspace_test {tn tokenizer lCp} {
+ set whitespace [format [string repeat %c [llength $lCp]] {*}$lCp]
+ set txt "${whitespace}hello${whitespace}world${whitespace}"
+ uplevel [list do_test $tn [list do_tokenize $tokenizer $txt] {hello world}]
+}
+
+set tokenizers [list unicode61]
+ifcapable icu { lappend tokenizers icu }
+
+# Some tests to check that the tokenizers can both identify white-space
+# codepoints. All codepoints tested below are of type "Zs" in the
+# UnicodeData.txt file.
+#
+# Note that codepoint 6158 has changed from Zs to Cf in recent versions
+# of UnicodeData.txt. So take that into account for the "icu" tests.
+#
+foreach T $tokenizers {
+ do_isspace_test 6.$T.1 $T 32
+ do_isspace_test 6.$T.2 $T 160
+ do_isspace_test 6.$T.3 $T 5760
+ if {$T!="icu"} {
+ do_isspace_test 6.$T.4 $T 6158
+ }
+ do_isspace_test 6.$T.5 $T 8192
+ do_isspace_test 6.$T.6 $T 8193
+ do_isspace_test 6.$T.7 $T 8194
+ do_isspace_test 6.$T.8 $T 8195
+ do_isspace_test 6.$T.9 $T 8196
+ do_isspace_test 6.$T.10 $T 8197
+ do_isspace_test 6.$T.11 $T 8198
+ do_isspace_test 6.$T.12 $T 8199
+ do_isspace_test 6.$T.13 $T 8200
+ do_isspace_test 6.$T.14 $T 8201
+ do_isspace_test 6.$T.15 $T 8202
+ if {$T!="icu"} {
+ do_isspace_test 6.$T.16 $T 8239
+ }
+ do_isspace_test 6.$T.17 $T 8287
+ do_isspace_test 6.$T.18 $T 12288
+
+ if {$T!="icu"} {
+ do_isspace_test 6.$T.19 $T {32 160 5760 6158}
+ } else {
+ do_isspace_test 6.$T.19 $T {32 160 5760 8192}
+ }
+ do_isspace_test 6.$T.20 $T {8192 8193 8194 8195}
+ do_isspace_test 6.$T.21 $T {8196 8197 8198 8199}
+ do_isspace_test 6.$T.22 $T {8200 8201 8202 8239}
+ do_isspace_test 6.$T.23 $T {8287 12288}
+}
+
+#-------------------------------------------------------------------------
+# Test that the private use ranges are treated as alphanumeric.
+#
+foreach {tn1 c} {
+ 1 \ue000 2 \ue001 3 \uf000 4 \uf8fe 5 \uf8ff
+} {
+ foreach {tn2 config res} {
+ 1 "" "0 hello*world hello*world"
+ 2 "separators=*" "0 hello hello 1 world world"
+ } {
+ set config [string map [list * $c] $config]
+ set input [string map [list * $c] "hello*world"]
+ set output [string map [list * $c] $res]
+ do_unicode_token_test3 7.$tn1.$tn2 {*}$config $input $output
+ }
+}
+
+#-------------------------------------------------------------------------
+# Cursory test of remove_diacritics=0.
+#
+# 00C4;LATIN CAPITAL LETTER A WITH DIAERESIS
+# 00D6;LATIN CAPITAL LETTER O WITH DIAERESIS
+# 00E4;LATIN SMALL LETTER A WITH DIAERESIS
+# 00F6;LATIN SMALL LETTER O WITH DIAERESIS
+#
+do_execsql_test 8.1.1 "
+ CREATE VIRTUAL TABLE t3 USING fts4(tokenize=unicode61 'remove_diacritics=1');
+ INSERT INTO t3 VALUES('o');
+ INSERT INTO t3 VALUES('a');
+ INSERT INTO t3 VALUES('O');
+ INSERT INTO t3 VALUES('A');
+ INSERT INTO t3 VALUES('\xD6');
+ INSERT INTO t3 VALUES('\xC4');
+ INSERT INTO t3 VALUES('\xF6');
+ INSERT INTO t3 VALUES('\xE4');
+"
+do_execsql_test 8.1.2 {
+ SELECT rowid FROM t3 WHERE t3 MATCH 'o';
+} {1 3 5 7}
+do_execsql_test 8.1.3 {
+ SELECT rowid FROM t3 WHERE t3 MATCH 'a';
+} {2 4 6 8}
+do_execsql_test 8.2.1 {
+ CREATE VIRTUAL TABLE t4 USING fts4(tokenize=unicode61 "remove_diacritics=0");
+ INSERT INTO t4 SELECT * FROM t3;
+}
+do_execsql_test 8.2.2 {
+ SELECT rowid FROM t4 WHERE t4 MATCH 'o';
+} {1 3}
+do_execsql_test 8.2.3 {
+ SELECT rowid FROM t4 WHERE t4 MATCH 'a';
+} {2 4}
+
+#-------------------------------------------------------------------------
+#
+foreach {tn sql} {
+ 1 {
+ CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 [tokenchars= .]);
+ CREATE VIRTUAL TABLE t6 USING fts4(
+ tokenize=unicode61 [tokenchars=="] "tokenchars=[]");
+ CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 [separators=x\xC4]);
+ }
+ 2 {
+ CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 "tokenchars= .");
+ CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 "tokenchars=[=""]");
+ CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 "separators=x\xC4");
+ }
+ 3 {
+ CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 'tokenchars= .');
+ CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 'tokenchars=="[]');
+ CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 'separators=x\xC4');
+ }
+ 4 {
+ CREATE VIRTUAL TABLE t5 USING fts4(tokenize=unicode61 `tokenchars= .`);
+ CREATE VIRTUAL TABLE t6 USING fts4(tokenize=unicode61 `tokenchars=[="]`);
+ CREATE VIRTUAL TABLE t7 USING fts4(tokenize=unicode61 `separators=x\xC4`);
+ }
+} {
+ do_execsql_test 9.$tn.0 {
+ DROP TABLE IF EXISTS t5;
+ DROP TABLE IF EXISTS t5aux;
+ DROP TABLE IF EXISTS t6;
+ DROP TABLE IF EXISTS t6aux;
+ DROP TABLE IF EXISTS t7;
+ DROP TABLE IF EXISTS t7aux;
+ }
+ do_execsql_test 9.$tn.1 $sql
+
+ do_execsql_test 9.$tn.2 {
+ CREATE VIRTUAL TABLE t5aux USING fts4aux(t5);
+ INSERT INTO t5 VALUES('one two three/four.five.six');
+ SELECT * FROM t5aux;
+ } {
+ four.five.six * 1 1 four.five.six 0 1 1
+ {one two three} * 1 1 {one two three} 0 1 1
+ }
+
+ do_execsql_test 9.$tn.3 {
+ CREATE VIRTUAL TABLE t6aux USING fts4aux(t6);
+ INSERT INTO t6 VALUES('alpha=beta"gamma/delta[epsilon]zeta');
+ SELECT * FROM t6aux;
+ } {
+ {alpha=beta"gamma} * 1 1 {alpha=beta"gamma} 0 1 1
+ {delta[epsilon]zeta} * 1 1 {delta[epsilon]zeta} 0 1 1
+ }
+
+ do_execsql_test 9.$tn.4 {
+ CREATE VIRTUAL TABLE t7aux USING fts4aux(t7);
+ INSERT INTO t7 VALUES('alephxbeth\xC4gimel');
+ SELECT * FROM t7aux;
+ } {
+ aleph * 1 1 aleph 0 1 1
+ beth * 1 1 beth 0 1 1
+ gimel * 1 1 gimel 0 1 1
+ }
+}
+
+# Check that multiple options are handled correctly.
+#
+do_execsql_test 10.1 {
+ DROP TABLE IF EXISTS t1;
+ CREATE VIRTUAL TABLE t1 USING fts4(tokenize=unicode61
+ "tokenchars=xyz" "tokenchars=.=" "separators=.=" "separators=xy"
+ "separators=a" "separators=a" "tokenchars=a" "tokenchars=a"
+ );
+
+ INSERT INTO t1 VALUES('oneatwoxthreeyfour');
+ INSERT INTO t1 VALUES('a.single=word');
+ CREATE VIRTUAL TABLE t1aux USING fts4aux(t1);
+ SELECT * FROM t1aux;
+} {
+ .single=word * 1 1 .single=word 0 1 1
+ four * 1 1 four 0 1 1
+ one * 1 1 one 0 1 1
+ three * 1 1 three 0 1 1
+ two * 1 1 two 0 1 1
+}
+
+# Test that case folding happens after tokenization, not before.
+#
+do_execsql_test 10.2 {
+ DROP TABLE IF EXISTS t2;
+ CREATE VIRTUAL TABLE t2 USING fts4(tokenize=unicode61 "separators=aB");
+ INSERT INTO t2 VALUES('oneatwoBthree');
+ INSERT INTO t2 VALUES('onebtwoAthree');
+ CREATE VIRTUAL TABLE t2aux USING fts4aux(t2);
+ SELECT * FROM t2aux;
+} {
+ one * 1 1 one 0 1 1
+ onebtwoathree * 1 1 onebtwoathree 0 1 1
+ three * 1 1 three 0 1 1
+ two * 1 1 two 0 1 1
+}
+
+# Test that the tokenchars and separators options work with the
+# fts3tokenize table.
+#
+do_execsql_test 11.1 {
+ CREATE VIRTUAL TABLE ft1 USING fts3tokenize(
+ "unicode61", "tokenchars=@.", "separators=1234567890"
+ );
+ SELECT token FROM ft1 WHERE input = 'berlin@street123sydney.road';
+} {
+ berlin@street sydney.road
+}
+
+# Test for embedded nul characters in fts4 unicode index.
+#
+do_execsql_test 12.0 {
+ CREATE VIRTUAL TABLE t12 USING fts4(tokenize=unicode61);
+ INSERT INTO t12 VALUES('abc' || char(0) || 'def');
+ SELECT hex(CAST(content AS blob)) FROM t12;
+} {61626300646566}
+do_execsql_test 12.1 {
+ INSERT INTO t12(t12) VALUES('integrity-check');
+} {}
+do_execsql_test 12.2 {
+ CREATE VIRTUAL TABLE t12aux USING fts4aux(t12);
+ SELECT * FROM t12aux;
+} {abc * 1 1 abc 0 1 1}
+do_execsql_test 12.3 {
+ SELECT hex(CAST(content AS blob)) FROM t12 WHERE t12 MATCH 'abc'
+} {61626300646566}
+
+finish_test
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-20 02:12:30 +0000