From 63847496f14c813a5d80efd5b7de0f1294ffe1e3 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 16:07:11 +0200 Subject: Adding upstream version 3.45.1. Signed-off-by: Daniel Baumann --- ext/rbu/rbu.c | 195 ++ ext/rbu/rbu1.test | 676 ++++++ ext/rbu/rbu10.test | 184 ++ ext/rbu/rbu11.test | 195 ++ ext/rbu/rbu12.test | 232 ++ ext/rbu/rbu13.test | 62 + ext/rbu/rbu14.test | 92 + ext/rbu/rbu3.test | 200 ++ ext/rbu/rbu5.test | 303 +++ ext/rbu/rbu6.test | 100 + ext/rbu/rbu7.test | 106 + ext/rbu/rbu8.test | 72 + ext/rbu/rbu9.test | 125 ++ ext/rbu/rbuA.test | 80 + ext/rbu/rbuB.test | 62 + ext/rbu/rbuC.test | 142 ++ ext/rbu/rbu_common.tcl | 120 + ext/rbu/rbubusy.test | 89 + ext/rbu/rbucollate.test | 58 + ext/rbu/rbucrash.test | 139 ++ ext/rbu/rbucrash2.test | 97 + ext/rbu/rbudiff.test | 309 +++ ext/rbu/rbudor.test | 56 + ext/rbu/rbuexlock.test | 292 +++ ext/rbu/rbuexpr.test | 88 + ext/rbu/rbufault.test | 234 ++ ext/rbu/rbufault2.test | 64 + ext/rbu/rbufault3.test | 96 + ext/rbu/rbufault4.test | 66 + ext/rbu/rbufts.test | 131 ++ ext/rbu/rbumisc.test | 176 ++ ext/rbu/rbumulti.test | 175 ++ ext/rbu/rbupartial.test | 98 + ext/rbu/rbupass.test | 81 + ext/rbu/rbuprogress.test | 451 ++++ ext/rbu/rburename.test | 55 + ext/rbu/rburesume.test | 256 +++ ext/rbu/rbusave.test | 102 + ext/rbu/rbusplit.test | 93 + ext/rbu/rbutemplimit.test | 129 ++ ext/rbu/rbuvacuum.test | 400 ++++ ext/rbu/rbuvacuum2.test | 249 +++ ext/rbu/rbuvacuum3.test | 64 + ext/rbu/rbuvacuum4.test | 118 + ext/rbu/sqlite3rbu.c | 5416 +++++++++++++++++++++++++++++++++++++++++++++ ext/rbu/sqlite3rbu.h | 633 ++++++ ext/rbu/test_rbu.c | 442 ++++ 47 files changed, 13603 insertions(+) create mode 100644 ext/rbu/rbu.c create mode 100644 ext/rbu/rbu1.test create mode 100644 ext/rbu/rbu10.test create mode 100644 ext/rbu/rbu11.test create mode 100644 ext/rbu/rbu12.test create mode 100644 ext/rbu/rbu13.test create mode 100644 ext/rbu/rbu14.test create mode 100644 ext/rbu/rbu3.test create mode 100644 ext/rbu/rbu5.test create mode 100644 ext/rbu/rbu6.test create mode 100644 ext/rbu/rbu7.test create mode 100644 ext/rbu/rbu8.test create mode 100644 ext/rbu/rbu9.test create mode 100644 ext/rbu/rbuA.test create mode 100644 ext/rbu/rbuB.test create mode 100644 ext/rbu/rbuC.test create mode 100644 ext/rbu/rbu_common.tcl create mode 100644 ext/rbu/rbubusy.test create mode 100644 ext/rbu/rbucollate.test create mode 100644 ext/rbu/rbucrash.test create mode 100644 ext/rbu/rbucrash2.test create mode 100644 ext/rbu/rbudiff.test create mode 100644 ext/rbu/rbudor.test create mode 100644 ext/rbu/rbuexlock.test create mode 100644 ext/rbu/rbuexpr.test create mode 100644 ext/rbu/rbufault.test create mode 100644 ext/rbu/rbufault2.test create mode 100644 ext/rbu/rbufault3.test create mode 100644 ext/rbu/rbufault4.test create mode 100644 ext/rbu/rbufts.test create mode 100644 ext/rbu/rbumisc.test create mode 100644 ext/rbu/rbumulti.test create mode 100644 ext/rbu/rbupartial.test create mode 100644 ext/rbu/rbupass.test create mode 100644 ext/rbu/rbuprogress.test create mode 100644 ext/rbu/rburename.test create mode 100644 ext/rbu/rburesume.test create mode 100644 ext/rbu/rbusave.test create mode 100644 ext/rbu/rbusplit.test create mode 100644 ext/rbu/rbutemplimit.test create mode 100644 ext/rbu/rbuvacuum.test create mode 100644 ext/rbu/rbuvacuum2.test create mode 100644 ext/rbu/rbuvacuum3.test create mode 100644 ext/rbu/rbuvacuum4.test create mode 100644 ext/rbu/sqlite3rbu.c create mode 100644 ext/rbu/sqlite3rbu.h create mode 100644 ext/rbu/test_rbu.c (limited to 'ext/rbu') diff --git a/ext/rbu/rbu.c b/ext/rbu/rbu.c new file mode 100644 index 0000000..7a24db2 --- /dev/null +++ b/ext/rbu/rbu.c @@ -0,0 +1,195 @@ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains a command-line application that uses the RBU +** extension. See the usage() function below for an explanation. +*/ + +#include "sqlite3rbu.h" +#include +#include +#include + +/* +** Print a usage message and exit. +*/ +void usage(const char *zArgv0){ + fprintf(stderr, +"Usage: %s ?OPTIONS? TARGET-DB RBU-DB\n" +"\n" +"Where options are:\n" +"\n" +" -step NSTEP\n" +" -statstep NSTATSTEP\n" +" -vacuum\n" +" -presql SQL\n" +"\n" +" If the -vacuum switch is not present, argument RBU-DB must be an RBU\n" +" database containing an update suitable for target database TARGET-DB.\n" +" Or, if -vacuum is specified, then TARGET-DB is a database to vacuum using\n" +" RBU, and RBU-DB is used as the state database for the vacuum (refer to\n" +" API documentation for details).\n" +"\n" +" If NSTEP is set to less than or equal to zero (the default value), this \n" +" program attempts to perform the entire update or vacuum operation before\n" +" exiting\n" +"\n" +" If NSTEP is greater than zero, then a maximum of NSTEP calls are made\n" +" to sqlite3rbu_step(). If the RBU update has not been completely applied\n" +" after the NSTEP'th call is made, the state is saved in the database RBU-DB\n" +" and the program exits. Subsequent invocations of this (or any other RBU)\n" +" application will use this state to resume applying the RBU update to the\n" +" target db.\n" +"\n" +, zArgv0); + exit(1); +} + +void report_default_vfs(){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + fprintf(stdout, "default vfs is \"%s\"\n", pVfs ? pVfs->zName : "NULL"); +} + +void report_rbu_vfs(sqlite3rbu *pRbu){ + sqlite3 *db = sqlite3rbu_db(pRbu, 0); + if( db ){ + char *zName = 0; + sqlite3_file_control(db, "main", SQLITE_FCNTL_VFSNAME, &zName); + if( zName ){ + fprintf(stdout, "using vfs \"%s\"\n", zName); + }else{ + fprintf(stdout, "vfs name not available\n"); + } + sqlite3_free(zName); + } +} + +int main(int argc, char **argv){ + int i; + const char *zTarget; /* Target database to apply RBU to */ + const char *zRbu; /* Database containing RBU */ + char zBuf[200]; /* Buffer for printf() */ + char *zErrmsg = 0; /* Error message, if any */ + sqlite3rbu *pRbu; /* RBU handle */ + int nStep = 0; /* Maximum number of step() calls */ + int nStatStep = 0; /* Report stats after this many step calls */ + int bVacuum = 0; + const char *zPreSql = 0; + int rc = SQLITE_OK; + sqlite3_int64 nProgress = 0; + int nArgc = argc-2; + + if( argc<3 ) usage(argv[0]); + for(i=1; i1 && nArg<=8 && 0==memcmp(zArg, "-vacuum", nArg) ){ + bVacuum = 1; + }else if( nArg>1 && nArg<=7 + && 0==memcmp(zArg, "-presql", nArg) && i1 && nArg<=5 && 0==memcmp(zArg, "-step", nArg) && i1 && nArg<=9 + && 0==memcmp(zArg, "-statstep", nArg) && i0 && (i % nStatStep)==0 ){ + sqlite3_int64 nUsed; + sqlite3_int64 nHighwater; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0); + fprintf(stdout, "memory used=%lld highwater=%lld", nUsed, nHighwater); + if( bVacuum==0 ){ + int one; + int two; + sqlite3rbu_bp_progress(pRbu, &one, &two); + fprintf(stdout, " progress=%d/%d\n", one, two); + }else{ + fprintf(stdout, "\n"); + } + fflush(stdout); + } + } + nProgress = sqlite3rbu_progress(pRbu); + rc = sqlite3rbu_close(pRbu, &zErrmsg); + } + + /* Let the user know what happened. */ + switch( rc ){ + case SQLITE_OK: + sqlite3_snprintf(sizeof(zBuf), zBuf, + "SQLITE_OK: rbu update incomplete (%lld operations so far)\n", + nProgress + ); + fprintf(stdout, "%s", zBuf); + break; + + case SQLITE_DONE: + sqlite3_snprintf(sizeof(zBuf), zBuf, + "SQLITE_DONE: rbu update completed (%lld operations)\n", + nProgress + ); + fprintf(stdout, "%s", zBuf); + break; + + default: + fprintf(stderr, "error=%d: %s\n", rc, zErrmsg); + break; + } + + if( nStatStep>0 ){ + sqlite3_int64 nUsed; + sqlite3_int64 nHighwater; + sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &nUsed, &nHighwater, 0); + fprintf(stdout, "memory used=%lld highwater=%lld\n", nUsed, nHighwater); + } + + sqlite3_free(zErrmsg); + return (rc==SQLITE_OK || rc==SQLITE_DONE) ? 0 : 1; +} diff --git a/ext/rbu/rbu1.test b/ext/rbu/rbu1.test new file mode 100644 index 0000000..418857d --- /dev/null +++ b/ext/rbu/rbu1.test @@ -0,0 +1,676 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu1 + +db close + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + } + rbu1 close + return $filename +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +# This RBU includes both insert and delete operations. +# +proc create_rbu4 {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, NULL, 5, 1); + INSERT INTO data_t1 VALUES(3, 8, 9, 0); + INSERT INTO data_t1 VALUES(4, NULL, 11, 1); + } + rbu1 close + return $filename +} +# +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(c, b, '(a)' INTEGER PRIMARY KEY); +# +# This RBU includes both insert and delete operations. +# +proc create_rbu4b {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(c, b, '(a)', rbu_control); + INSERT INTO data_t1 VALUES(3, 2, 1, 0); + INSERT INTO data_t1 VALUES(5, NULL, 2, 1); + INSERT INTO data_t1 VALUES(9, 8, 3, 0); + INSERT INTO data_t1 VALUES(11, NULL, 4, 1); + } + rbu1 close + return $filename +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); +# +# This RBU includes update statements. +# +proc create_rbu5 {filename} { + forcedelete $filename + sqlite3 rbu5 $filename + rbu5 eval { + CREATE TABLE data_t1(a, b, c, d, rbu_control); + INSERT INTO data_t1 VALUES(1, NULL, NULL, 5, '...x'); -- SET d = 5 + INSERT INTO data_t1 VALUES(2, NULL, 10, 5, '..xx'); -- SET c=10, d = 5 + INSERT INTO data_t1 VALUES(3, 11, NULL, NULL, '.x..'); -- SET b=11 + } + rbu5 close + return $filename +} + + +# Same as [step_rbu], except using a URI to open the target db. +# +proc step_rbu_uri {target rbu} { + while 1 { + sqlite3rbu rbu file:$target?xyz=&abc=123 $rbu + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +# Same as [step_rbu], except using an external state database - "state.db" +# +proc step_rbu_state {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu state.db + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +proc dbfilecksum {file} { + sqlite3 ck $file + set cksum [dbcksum ck main] + ck close + set cksum +} + +foreach {tn3 create_vfs destroy_vfs} { + 1 {} {} + 2 { + sqlite3rbu_create_vfs -default myrbu "" + } { + sqlite3rbu_destroy_vfs myrbu + } + 3 { + sqlite3_register_cksumvfs + } { + sqlite3_unregister_cksumvfs + } +} { + + eval $create_vfs + + foreach {tn2 cmd} { + 1 run_rbu + 2 step_rbu 3 step_rbu_uri 4 step_rbu_state + 5 step_rbu_legacy + } { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 5 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + 6 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b, a); + } + 7 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 8 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, b, c, a, b, c); + } + + 9 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c)); + CREATE INDEX i1 ON t1(b); + } + + 10 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC); + } + + 11 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b DESC, a ASC, c DESC); + } + + 12 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + } + + 13 { + CREATE TABLE t1(a INT, b, c, PRIMARY KEY(a DESC)) WITHOUT ROWID; + } + + 14 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a DESC, c)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 15 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, c DESC)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + } + + 16 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(c DESC, a)) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b DESC, c, a); + } + } { + reset_db + execsql $schema + create_rbu1 rbu.db + set check [dbfilecksum rbu.db] + forcedelete state.db + + do_test $tn3.1.$tn2.$tn.1 { + $cmd test.db rbu.db + } {SQLITE_DONE} + + do_execsql_test $tn3.1.$tn2.$tn.2 { SELECT * FROM t1 ORDER BY a ASC } { + 1 2 3 + 2 two three + 3 {} 8.2 + } + do_execsql_test $tn3.1.$tn2.$tn.3 { SELECT * FROM t1 ORDER BY b ASC } { + 3 {} 8.2 + 1 2 3 + 2 two three + } + do_execsql_test $tn3.1.$tn2.$tn.4 { SELECT * FROM t1 ORDER BY c ASC } { + 1 2 3 + 3 {} 8.2 + 2 two three + } + + do_execsql_test $tn3.1.$tn2.$tn.5 { PRAGMA integrity_check } ok + + if {$cmd=="step_rbu_state"} { + do_test $tn3.1.$tn2.$tn.6 { file exists state.db } 1 + do_test $tn3.1.$tn2.$tn.7 { expr {$check == [dbfilecksum rbu.db]} } 1 + } else { + do_test $tn3.1.$tn2.$tn.8 { file exists state.db } 0 + do_test $tn3.1.$tn2.$tn.9 { expr {$check == [dbfilecksum rbu.db]} } 0 + } + } + } + + #------------------------------------------------------------------------- + # Check that an RBU cannot be applied to a table that has no PK. + # + # UPDATE: At one point RBU required that all tables featured either + # explicit IPK columns or were declared WITHOUT ROWID. This has been + # relaxed so that external PRIMARY KEYs on tables with automatic rowids + # are now allowed. + # + # UPDATE 2: Tables without any PRIMARY KEY declaration are now allowed. + # However the input table must feature an "rbu_rowid" column. + # + reset_db + create_rbu1 rbu.db + do_execsql_test $tn3.2.1 { CREATE TABLE t1(a, b, c) } + do_test $tn3.2.2 { + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_ERROR} + do_test $tn3.2.3 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_ERROR - table data_t1 requires rbu_rowid column}} + reset_db + do_execsql_test $tn3.2.4 { CREATE TABLE t1(a PRIMARY KEY, b, c) } + do_test $tn3.2.5 { + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_OK} + do_test $tn3.2.6 { + list [catch { rbu close } msg] $msg + } {0 SQLITE_OK} + + #------------------------------------------------------------------------- + # Check that if a UNIQUE constraint is violated the current and all + # subsequent [rbu step] calls return SQLITE_CONSTRAINT. And that the RBU + # transaction is rolled back by the [rbu close] that deletes the rbu + # handle. + # + foreach {tn errcode errmsg schema} { + 1 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } + + 2 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } + + 3 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.a" { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(3, 2, 1); + } + + 4 SQLITE_CONSTRAINT "UNIQUE constraint failed: t1.c" { + CREATE TABLE t1(a PRIMARY KEY, b, c UNIQUE); + INSERT INTO t1 VALUES(4, 2, 'three'); + } + + } { + reset_db + execsql $schema + set cksum [dbcksum db main] + + do_test $tn3.3.$tn.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + while {[set res [rbu step]]=="SQLITE_OK"} {} + set res + } $errcode + + do_test $tn3.3.$tn.2 { rbu step } $errcode + + do_test $tn3.3.$tn.3 { + list [catch { rbu close } msg] $msg + } [list 1 "$errcode - $errmsg"] + + do_test $tn3.3.$tn.4 { dbcksum db main } $cksum + } + + #------------------------------------------------------------------------- + # + foreach {tn2 cmd} {1 run_rbu 2 step_rbu 3 step_rbu_state } { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + } + 3 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + 4 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + 5 { + CREATE TABLE t1(a INT PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + + 6 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c); + CREATE INDEX i1 ON t1(b DESC); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c DESC, b, c); + } + 7 { + CREATE TABLE t1(a INT PRIMARY KEY DESC, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(c, b, c); + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(2, 'hello', 'world'); + INSERT INTO t1 VALUES(4, 'hello', 'planet'); + INSERT INTO t1 VALUES(6, 'hello', 'xyz'); + } + + create_rbu4 rbu.db + set check [dbfilecksum rbu.db] + forcedelete state.db + + do_test $tn3.4.$tn2.$tn.1 { + $cmd test.db rbu.db + } {SQLITE_DONE} + + do_execsql_test $tn3.4.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 + 3 8 9 + 6 hello xyz + } + + do_execsql_test $tn3.4.$tn2.$tn.3 { PRAGMA integrity_check } ok + + if {$cmd=="step_rbu_state"} { + do_test $tn3.4.$tn2.$tn.4 { file exists state.db } 1 + do_test $tn3.4.$tn2.$tn.5 { expr {$check == [dbfilecksum rbu.db]} } 1 + } else { + do_test $tn3.4.$tn2.$tn.6 { file exists state.db } 0 + do_test $tn3.4.$tn2.$tn.7 { expr {$check == [dbfilecksum rbu.db]} } 0 + } + } + } + + foreach {tn2 cmd} {1 run_rbu 2 step_rbu 3 step_rbu_state} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(c, b, '(a)' INTEGER PRIMARY KEY); + CREATE INDEX i1 ON t1(c, b); + } + 2 { + CREATE TABLE t1(c, b, '(a)' PRIMARY KEY); + } + 3 { + CREATE TABLE t1(c, b, '(a)' PRIMARY KEY) WITHOUT ROWID; + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1('(a)', b, c) VALUES(2, 'hello', 'world'); + INSERT INTO t1('(a)', b, c) VALUES(4, 'hello', 'planet'); + INSERT INTO t1('(a)', b, c) VALUES(6, 'hello', 'xyz'); + } + + create_rbu4b rbu.db + set check [dbfilecksum rbu.db] + forcedelete state.db + + do_test $tn3.5.$tn2.$tn.1 { + $cmd test.db rbu.db + } {SQLITE_DONE} + + do_execsql_test $tn3.5.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY "(a)" ASC; + } { + 3 2 1 + 9 8 3 + xyz hello 6 + } + + do_execsql_test $tn3.4.$tn2.$tn.3 { PRAGMA integrity_check } ok + + if {$cmd=="step_rbu_state"} { + do_test $tn3.5.$tn2.$tn.4 { file exists state.db } 1 + do_test $tn3.5.$tn2.$tn.5 { expr {$check == [dbfilecksum rbu.db]} } 1 + } else { + do_test $tn3.5.$tn2.$tn.6 { file exists state.db } 0 + do_test $tn3.5.$tn2.$tn.7 { expr {$check == [dbfilecksum rbu.db]} } 0 + } + } + } + + #------------------------------------------------------------------------- + # + foreach {tn2 cmd} {1 run_rbu 2 step_rbu 3 step_rbu_state} { + foreach {tn schema} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + } + 2 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d) WITHOUT ROWID; + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d); + CREATE INDEX i1 ON t1(d); + CREATE INDEX i2 ON t1(d, c); + CREATE INDEX i3 ON t1(d, c, b); + CREATE INDEX i4 ON t1(b); + CREATE INDEX i5 ON t1(c); + CREATE INDEX i6 ON t1(c, b); + } + } { + reset_db + execsql $schema + execsql { + INSERT INTO t1 VALUES(1, 2, 3, 4); + INSERT INTO t1 VALUES(2, 5, 6, 7); + INSERT INTO t1 VALUES(3, 8, 9, 10); + } + + create_rbu5 rbu.db + set check [dbfilecksum rbu.db] + forcedelete state.db + + do_test $tn3.5.$tn2.$tn.1 { + $cmd test.db rbu.db + } {SQLITE_DONE} + + do_execsql_test $tn3.5.$tn2.$tn.2 { + SELECT * FROM t1 ORDER BY a ASC; + } { + 1 2 3 5 + 2 5 10 5 + 3 11 9 10 + } + + do_execsql_test $tn3.6.$tn2.$tn.3 { PRAGMA integrity_check } ok + + if {$cmd=="step_rbu_state"} { + do_test $tn3.6.$tn2.$tn.4 { file exists state.db } 1 + do_test $tn3.6.$tn2.$tn.5 { expr {$check == [dbfilecksum rbu.db]} } 1 + } else { + do_test $tn3.6.$tn2.$tn.6 { file exists state.db } 0 + do_test $tn3.6.$tn2.$tn.7 { expr {$check == [dbfilecksum rbu.db]} } 0 + } + } + } + + #------------------------------------------------------------------------- + # Test some error cases: + # + # * A virtual table with no rbu_rowid column. + # * A no-PK table with no rbu_rowid column. + # * A PK table with an rbu_rowid column. + # + # 6: An update string of the wrong length + # + ifcapable fts3 { + foreach {tn schema error} { + 1 { + CREATE TABLE t1(a, b); + CREATE TABLE rbu.data_t1(a, b, rbu_control); + } {SQLITE_ERROR - table data_t1 requires rbu_rowid column} + + 2 { + CREATE VIRTUAL TABLE t1 USING fts4(a, b); + CREATE TABLE rbu.data_t1(a, b, rbu_control); + } {SQLITE_ERROR - table data_t1 requires rbu_rowid column} + + 3 { + CREATE TABLE t1(a PRIMARY KEY, b); + CREATE TABLE rbu.data_t1(a, b, rbu_rowid, rbu_control); + } {SQLITE_ERROR - table data_t1 may not have rbu_rowid column} + + 4 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE rbu.data_t1(a, b, rbu_rowid, rbu_control); + } {SQLITE_ERROR - table data_t1 may not have rbu_rowid column} + + 5 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE rbu.data_t1(a, b, rbu_rowid, rbu_control); + } {SQLITE_ERROR - table data_t1 may not have rbu_rowid column} + + 6 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE rbu.data_t1(a, b, rbu_control); + INSERT INTO rbu.data_t1 VALUES(1, 2, 'x.x'); + } {SQLITE_ERROR - invalid rbu_control value} + + 7 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE rbu.data_t1(a, b, rbu_control); + INSERT INTO rbu.data_t1 VALUES(1, 2, NULL); + } {SQLITE_ERROR - invalid rbu_control value} + + 8 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE rbu.data_t1(a, b, rbu_control); + INSERT INTO rbu.data_t1 VALUES(1, 2, 4); + } {SQLITE_ERROR - invalid rbu_control value} + + 9 { + CREATE TABLE t1(a, b PRIMARY KEY) WITHOUT ROWID; + CREATE TABLE rbu.data_t1(a, b, rbu_control); + INSERT INTO rbu.data_t1 VALUES(1, 2, 3); + } {SQLITE_ERROR - invalid rbu_control value} + + 10 { + CREATE TABLE t2(a, b); + CREATE TABLE rbu.data_t1(a, b, rbu_control); + INSERT INTO rbu.data_t1 VALUES(1, 2, 2); + } {SQLITE_ERROR - no such table: t1} + + 11 { + CREATE TABLE rbu.data_t2(a, b, rbu_control); + INSERT INTO rbu.data_t2 VALUES(1, 2, 2); + } {SQLITE_ERROR - no such table: t2} + + } { + reset_db + forcedelete rbu.db + execsql { ATTACH 'rbu.db' AS rbu } + execsql $schema + + do_test $tn3.7.$tn { + list [catch { run_rbu test.db rbu.db } msg] $msg + } [list 1 $error] + } + } + + # Test that an RBU database containing no input tables is handled + # correctly. + reset_db + forcedelete rbu.db + do_test $tn3.8.1 { + list [catch { run_rbu test.db rbu.db } msg] $msg + } {0 SQLITE_DONE} + + # Test that an RBU database containing only empty data_xxx tables is + # also handled correctly. + reset_db + forcedelete rbu.db + do_execsql_test $tn3.8.2.1 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + ATTACH 'rbu.db' AS rbu; + CREATE TABLE data_t1(a, b, rbu_control); + DETACH rbu; + } + do_test $tn3.8.2.1 { + list [catch { run_rbu test.db rbu.db } msg] $msg + } {0 SQLITE_DONE} + + # Test that RBU can update indexes containing NULL values. + # + reset_db + forcedelete rbu.db + do_execsql_test $tn3.9.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + INSERT INTO t1 VALUES(1, 1, NULL); + INSERT INTO t1 VALUES(2, NULL, 2); + INSERT INTO t1 VALUES(3, NULL, NULL); + + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(3, NULL, NULL, 1); + } {} + + do_test $tn3.9.2 { + list [catch { run_rbu test.db rbu.db } msg] $msg + } {0 SQLITE_DONE} + + do_execsql_test $tn3.9.3 { + SELECT * FROM t1 + } {2 {} 2} + do_execsql_test $tn3.9.4 { PRAGMA integrity_check } {ok} + + catch { db close } + eval $destroy_vfs +} + + +finish_test diff --git a/ext/rbu/rbu10.test b/ext/rbu/rbu10.test new file mode 100644 index 0000000..8705c7b --- /dev/null +++ b/ext/rbu/rbu10.test @@ -0,0 +1,184 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu10 + +#-------------------------------------------------------------------- +# Test that UPDATE commands work even if the input columns are in a +# different order to the output columns. +# +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 'b', 'c'); +} + +proc apply_rbu {sql} { + forcedelete rbu.db + sqlite3 db2 rbu.db + db2 eval $sql + db2 close + sqlite3rbu rbu test.db rbu.db + while { [rbu step]=="SQLITE_OK" } {} + rbu close +} + +do_test 1.1 { + apply_rbu { + CREATE TABLE data_t1(a, c, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 'xxx', NULL, '.x.'); + } + db eval { SELECT * FROM t1 } +} {1 b xxx} + +#-------------------------------------------------------------------- +# Test that the hidden languageid column of an fts4 table can be +# written. +# +ifcapable fts3 { + do_execsql_test 2.0 { + create virtual TABLE ft USING fts4(a, b, languageid='langid'); + } + do_test 2.1 { + apply_rbu { + CREATE TABLE data_ft(a, b, rbu_rowid, langid, rbu_control); + INSERT INTO data_ft VALUES('a', 'b', 22, 1, 0); -- insert + INSERT INTO data_ft VALUES('a', 'b', 23, 10, 0); -- insert + INSERT INTO data_ft VALUES('a', 'b', 24, 100, 0); -- insert + } + db eval { SELECT a, b, rowid, langid FROM ft } + } [list {*}{ + a b 22 1 + a b 23 10 + a b 24 100 + }] + + # Or not - this data_xxx table has no langid column, so langid + # defaults to 0. + # + do_test 2.2 { + apply_rbu { + CREATE TABLE data_ft(a, b, rbu_rowid, rbu_control); + INSERT INTO data_ft VALUES('a', 'b', 25, 0); -- insert + } + db eval { SELECT a, b, rowid, langid FROM ft } + } [list {*}{ + a b 22 1 + a b 23 10 + a b 24 100 + a b 25 0 + }] + + # Update langid. + # + do_test 2.3 { + apply_rbu { + CREATE TABLE data_ft(a, b, rbu_rowid, langid, rbu_control); + INSERT INTO data_ft VALUES(NULL, NULL, 23, 50, '..x'); + INSERT INTO data_ft VALUES(NULL, NULL, 25, 500, '..x'); + } + db eval { SELECT a, b, rowid, langid FROM ft } + } [list {*}{ + a b 22 1 + a b 23 50 + a b 24 100 + a b 25 500 + }] +} + +#-------------------------------------------------------------------- +# Test that if writing a hidden virtual table column is an error, +# attempting to do so via rbu is also an error. +# +ifcapable fts3 { + do_execsql_test 3.0 { + CREATE VIRTUAL TABLE xt USING fts4(a); + } + do_test 3.1 { + list [catch { + apply_rbu { + CREATE TABLE data_xt(a, xt, rbu_rowid, rbu_control); + INSERT INTO data_xt VALUES('a', 'b', 1, 0); + } + } msg] $msg + } {1 {SQLITE_ERROR - SQL logic error}} +} + +#-------------------------------------------------------------------- +# Test that it is not possible to violate a NOT NULL constraint by +# applying an RBU update. +# +do_execsql_test 4.1 { + CREATE TABLE t2(a INTEGER NOT NULL, b TEXT NOT NULL, c PRIMARY KEY); + CREATE TABLE t3(a INTEGER NOT NULL, b TEXT NOT NULL, c INTEGER PRIMARY KEY); + CREATE TABLE t4(a, b, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(10, 10, 10); + INSERT INTO t3 VALUES(10, 10, 10); + INSERT INTO t4 VALUES(10, 10); +} + +foreach {tn error rbu} { + 2 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.a} { + INSERT INTO data_t2 VALUES(NULL, 'abc', 1, 0); + } + 3 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.b} { + INSERT INTO data_t2 VALUES(2, NULL, 1, 0); + } + 4 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.c} { + INSERT INTO data_t2 VALUES(1, 'abc', NULL, 0); + } + + 5 {SQLITE_MISMATCH - datatype mismatch} { + INSERT INTO data_t3 VALUES(1, 'abc', NULL, 0); + } + + 6 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t4.b} { + INSERT INTO data_t4 VALUES('a', NULL, 0); + } + 7 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t4.a} { + INSERT INTO data_t4 VALUES(NULL, 'a', 0); + } + 8 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t2.a} { + INSERT INTO data_t2 VALUES(NULL, 0, 10, 'x..'); + } + 9 {SQLITE_CONSTRAINT - NOT NULL constraint failed: t3.b} { + INSERT INTO data_t3 VALUES(10, NULL, 10, '.x.'); + } + + 10 {SQLITE_MISMATCH - datatype mismatch} { + INSERT INTO data_t3 VALUES(1, 'abc', 'text', 0); + } +} { + set rbu " + CREATE TABLE data_t2(a, b, c, rbu_control); + CREATE TABLE data_t3(a, b, c, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + $rbu + " + do_test 4.2.$tn { + list [catch { apply_rbu $rbu } msg] $msg + } [list 1 $error] +} + +do_test 4.3 { + set rbu { + CREATE TABLE data_t3(a, b, c, rbu_control); + INSERT INTO data_t3 VALUES(1, 'abc', '5', 0); + INSERT INTO data_t3 VALUES(1, 'abc', '-6.0', 0); + } + list [catch { apply_rbu $rbu } msg] $msg +} {0 SQLITE_DONE} + + +finish_test diff --git a/ext/rbu/rbu11.test b/ext/rbu/rbu11.test new file mode 100644 index 0000000..a42163c --- /dev/null +++ b/ext/rbu/rbu11.test @@ -0,0 +1,195 @@ +# 2015 February 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu11 + + +#-------------------------------------------------------------------- +# Test that the xAccess() method of an rbu vfs handles queries other +# than SQLITE_ACCESS_EXISTS correctly. The test code below causes +# SQLite to call xAccess(SQLITE_ACCESS_READWRITE) on the directory +# path argument passed to "PRAGMA temp_store_directory". +# +do_test 1.1 { + sqlite3rbu_create_vfs -default rbu "" + reset_db + catchsql { PRAGMA temp_store_directory = '/no/such/directory' } +} {1 {not a writable directory}} + +do_test 1.2 { + catchsql " PRAGMA temp_store_directory = '[pwd]' " +} {0 {}} + +do_test 1.3 { + catchsql " PRAGMA temp_store_directory = '' " +} {0 {}} + +do_test 1.4 { + db close + sqlite3rbu_destroy_vfs rbu +} {} + +#-------------------------------------------------------------------- +# Try to trick rbu into operating on a database opened in wal mode. +# +reset_db +do_execsql_test 2.1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + PRAGMA journal_mode = 'wal'; + CREATE TABLE t2(d PRIMARY KEY, e, f); +} {wal} + +do_test 2.2 { + db_save + db close + + forcedelete rbu.db + sqlite3 dbo rbu.db + dbo eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); + } + dbo close + + db_restore + hexio_write test.db 18 0101 + file exists test.db-wal +} {1} + +do_test 2.3 { + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} + +do_test 2.4 { + list [catch {rbu close} msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + +#-------------------------------------------------------------------- +# Test a constraint violation message with an unusual table name. +# Specifically, one for which the first character is a codepoint +# smaller than 30 (character '0'). +# +reset_db +do_execsql_test 3.1 { + CREATE TABLE "(t1)"(a PRIMARY KEY, b, c); + INSERT INTO "(t1)" VALUES(1, 2, 3); + INSERT INTO "(t1)" VALUES(4, 5, 6); +} +db close + +do_test 3.2 { + forcedelete rbu.db + sqlite3 dbo rbu.db + dbo eval { + CREATE TABLE "data_(t1)"(a, b, c, rbu_control); + INSERT INTO "data_(t1)" VALUES(4, 8, 9, 0); + } + dbo close + + sqlite3rbu rbu test.db rbu.db + rbu step + rbu step +} {SQLITE_CONSTRAINT} + +do_test 3.3 { + list [catch {rbu close} msg] $msg +} {1 {SQLITE_CONSTRAINT - UNIQUE constraint failed: (t1).a}} + +#-------------------------------------------------------------------- +# Check that once an RBU update has been applied, attempting to apply +# it a second time is a no-op (as the state stored in the RBU database is +# "all steps completed"). +# +reset_db +do_execsql_test 4.1 { + CREATE TABLE "(t1)"(a, b, c, PRIMARY KEY(c, b, a)); + INSERT INTO "(t1)" VALUES(1, 2, 3); + INSERT INTO "(t1)" VALUES(4, 5, 6); +} +db close + +do_test 4.2 { + forcedelete rbu.db + sqlite3 dbo rbu.db + dbo eval { + CREATE TABLE "data_(t1)"(a, b, c, rbu_control); + INSERT INTO "data_(t1)" VALUES(7, 8, 9, 0); + INSERT INTO "data_(t1)" VALUES(1, 2, 3, 1); + } + dbo close + + sqlite3rbu rbu test.db rbu.db + while {[rbu step]=="SQLITE_OK"} { } + rbu close +} {SQLITE_DONE} + +do_test 4.3 { + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_DONE} + +do_test 4.4 { + rbu close +} {SQLITE_DONE} + +do_test 4.5.1 { + sqlite3 dbo rbu.db + dbo eval { INSERT INTO rbu_state VALUES(100, 100) } + dbo close + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_CORRUPT} +do_test 4.5.2 { + list [catch {rbu close} msg] $msg +} {1 SQLITE_CORRUPT} +do_test 4.5.3 { + sqlite3 dbo rbu.db + dbo eval { DELETE FROM rbu_state WHERE k = 100 } + dbo close +} {} + +# Also, check that an invalid state value in the rbu_state table is +# detected and reported as corruption. +do_test 4.6.1 { + sqlite3 dbo rbu.db + dbo eval { UPDATE rbu_state SET v = v*-1 WHERE k = 1 } + dbo close + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_CORRUPT} +do_test 4.6.2 { + list [catch {rbu close} msg] $msg +} {1 SQLITE_CORRUPT} +do_test 4.6.3 { + sqlite3 dbo rbu.db + dbo eval { UPDATE rbu_state SET v = v*-1 WHERE k = 1 } + dbo close +} {} + +do_test 4.7.1 { + sqlite3 dbo rbu.db + dbo eval { UPDATE rbu_state SET v = 1 WHERE k = 1 } + dbo eval { UPDATE rbu_state SET v = 'nosuchtable' WHERE k = 2 } + dbo close + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} +do_test 4.7.2 { + list [catch {rbu close} msg] $msg +} {1 {SQLITE_ERROR - rbu_state mismatch error}} + +finish_test diff --git a/ext/rbu/rbu12.test b/ext/rbu/rbu12.test new file mode 100644 index 0000000..a753ef2 --- /dev/null +++ b/ext/rbu/rbu12.test @@ -0,0 +1,232 @@ +# 2015 February 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/lock_common.tcl +set ::testprefix rbu12 + +set setup_sql { + DROP TABLE IF EXISTS xx; + DROP TABLE IF EXISTS xy; + CREATE TABLE xx(a, b, c PRIMARY KEY); + INSERT INTO xx VALUES(1, 2, 3); + CREATE TABLE xy(a, b, c PRIMARY KEY); + + ATTACH 'rbu.db' AS rbu; + DROP TABLE IF EXISTS data_xx; + CREATE TABLE rbu.data_xx(a, b, c, rbu_control); + INSERT INTO data_xx VALUES(4, 5, 6, 0); + INSERT INTO data_xx VALUES(7, 8, 9, 0); + CREATE TABLE rbu.data_xy(a, b, c, rbu_control); + INSERT INTO data_xy VALUES(10, 11, 12, 0); + DETACH rbu; +} + +do_multiclient_test tn { + + # Initialize a target (test.db) and rbu (rbu.db) database. + # + forcedelete rbu.db + sql1 $setup_sql + + # Using connection 2, open a read transaction on the target database. + # RBU will still be able to generate "test.db-oal", but it will not be + # able to rename it to "test.db-wal". + # + do_test 1.$tn.1 { + sql2 { BEGIN; SELECT * FROM xx; } + } {1 2 3} + do_test 1.$tn.2 { + sqlite3rbu rbu test.db rbu.db + while 1 { + set res [rbu step] + if {$res!="SQLITE_OK"} break + } + set res + } {SQLITE_BUSY} + + do_test 1.$tn.3 { sql2 { SELECT * FROM xx; } } {1 2 3} + do_test 1.$tn.4 { sql2 { SELECT * FROM xy; } } {} + do_test 1.$tn.5 { + list [file exists test.db-wal] [file exists test.db-oal] + } {0 1} + do_test 1.$tn.6 { sql2 COMMIT } {} + + # The rbu object that hit the SQLITE_BUSY error above cannot be reused. + # It is stuck in a permanent SQLITE_BUSY state at this point. + # + do_test 1.$tn.7 { rbu step } {SQLITE_BUSY} + do_test 1.$tn.8 { + list [catch { rbu close } msg] $msg + } {1 SQLITE_BUSY} + + do_test 1.$tn.9.1 { sql2 { BEGIN EXCLUSIVE } } {} + do_test 1.$tn.9.2 { + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_BUSY} + do_test 1.$tn.9.3 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database is locked}} + do_test 1.$tn.9.4 { sql2 COMMIT } {} + + sqlite3rbu rbu test.db rbu.db + do_test 1.$tn.10.1 { sql2 { BEGIN EXCLUSIVE } } {} + do_test 1.$tn.10.2 { + rbu step + } {SQLITE_BUSY} + do_test 1.$tn.10.3 { + list [catch { rbu close } msg] $msg + } {1 SQLITE_BUSY} + do_test 1.$tn.10.4 { sql2 COMMIT } {} + + # A new rbu object can finish the work though. + # + do_test 1.$tn.11 { + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_OK} + do_test 1.$tn.12 { + list [file exists test.db-wal] [file exists test.db-oal] + } {1 0} + do_test 1.$tn.13 { + while 1 { + set res [rbu step] + if {$res!="SQLITE_OK"} break + } + set res + } {SQLITE_DONE} + + do_test 1.$tn.14 { + rbu close + } {SQLITE_DONE} +} + +do_multiclient_test tn { + + # Initialize a target (test.db) and rbu (rbu.db) database. + # + forcedelete rbu.db + sql1 $setup_sql + + do_test 2.$tn.1 { + sqlite3rbu rbu test.db rbu.db + while {[file exists test.db-wal]==0} { + if {[rbu step]!="SQLITE_OK"} {error "problem here...."} + } + rbu close + } {SQLITE_OK} + + + do_test 2.$tn.2 { sql2 { BEGIN IMMEDIATE } } {} + + do_test 2.$tn.3 { + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_BUSY} + + do_test 2.$tn.4 { list [catch { rbu close } msg] $msg } {1 SQLITE_BUSY} + + do_test 2.$tn.5 { + sql2 { SELECT * FROM xx ; COMMIT } + } {1 2 3 4 5 6 7 8 9} + + do_test 2.$tn.6 { + sqlite3rbu rbu test.db rbu.db + rbu step + rbu close + } {SQLITE_OK} + + do_test 2.$tn.7 { sql2 { BEGIN EXCLUSIVE } } {} + + do_test 2.$tn.8 { + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_BUSY} + do_test 2.$tn.9 { list [catch { rbu close } msg] $msg } {1 SQLITE_BUSY} + do_test 2.$tn.10 { + sql2 { SELECT * FROM xx ; COMMIT } + } {1 2 3 4 5 6 7 8 9} + + do_test 2.$tn.11 { + sqlite3rbu rbu test.db rbu.db + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + +} + +#------------------------------------------------------------------------- +# Test that "PRAGMA data_version" works when an RBU client writes the +# database. +# +do_multiclient_test tn { + + # Initialize a target (test.db) and rbu (rbu.db) database. + # + forcedelete rbu.db + sql1 $setup_sql + + # Check the initial database contains table "xx" with a single row. + # Also save the current values of "PRAGMA data-version" for [db1] + # and [db2]. + # + do_test 2.$tn.1 { + list [sql1 { SELECT count(*) FROM xx }] [sql2 { SELECT count(*) FROM xx }] + } {1 1} + set V1 [sql1 {PRAGMA data_version}] + set V2 [sql2 {PRAGMA data_version}] + + # Check the values of data-version have not magically changed. + # + do_test 2.$tn.2 { + list [sql1 {PRAGMA data_version}] [sql2 {PRAGMA data_version}] + } [list $V1 $V2] + + # Start stepping the RBU. From the point of view of [db1] and [db2], the + # data-version values remain unchanged until the database contents are + # modified. At which point the values are incremented. + # + sqlite3rbu rbu test.db rbu.db + set x 0 + while {[db one {SELECT count(*) FROM xx}]==1} { + do_test 2.$tn.3.[incr x] { + list [sql1 {PRAGMA data_version}] [sql2 {PRAGMA data_version}] + } [list $V1 $V2] + rbu step + } + do_test 2.$tn.5.1 { expr {$V1 < [sql1 {PRAGMA data_version}]} } 1 + do_test 2.$tn.5.2 { expr {$V2 < [sql2 {PRAGMA data_version}]} } 1 + + # Check the db contents is as expected. + # + do_test 2.$tn.4 { + list [sql1 {SELECT count(*) FROM xx}] [sql2 {SELECT count(*) FROM xx}] + } {3 3} + + set V1 [sql1 {PRAGMA data_version}] + set V2 [sql2 {PRAGMA data_version}] + + # Finish applying the RBU (i.e. do the incremental checkpoint). Check that + # this does not cause the data-version values to change. + # + while {[rbu step]=="SQLITE_OK"} { } + rbu close + + do_test 2.$tn.6 { + list [sql1 {PRAGMA data_version}] [sql2 {PRAGMA data_version}] + } [list $V1 $V2] + +} + +finish_test diff --git a/ext/rbu/rbu13.test b/ext/rbu/rbu13.test new file mode 100644 index 0000000..9bf2adc --- /dev/null +++ b/ext/rbu/rbu13.test @@ -0,0 +1,62 @@ +# 2015 February 16 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test an RBU update that features lots of different rbu_control strings +# for UPDATE statements. This tests RBU's internal UPDATE statement cache. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/lock_common.tcl +set ::testprefix rbu13 + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c, d, e, f, g, h); + WITH ii(i) AS (SELECT 0 UNION ALL SELECT i+1 FROM ii WHERE i<127) + INSERT INTO t1 SELECT i, 0, 0, 0, 0, 0, 0, 0 FROM ii; +} + +forcedelete rbu.db +do_execsql_test 1.1 { + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, d, e, f, g, h, rbu_control); +} + +do_test 1.2 { + for {set i 0} {$i<128} {incr i} { + set control "." + for {set bit 6} {$bit>=0} {incr bit -1} { + if { $i & (1<<$bit) } { + append control "x" + } else { + append control "." + } + } + execsql { INSERT INTO data_t1 VALUES($i, 1, 1, 1, 1, 1, 1, 1, $control) } + } +} {} + +do_test 1.3 { + sqlite3rbu rbu test.db rbu.db + while 1 { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + } + rbu close +} {SQLITE_DONE} + +do_execsql_test 1.4 { + SELECT count(*) FROM t1 WHERE + a == ( (b<<6) + (c<<5) + (d<<4) + (e<<3) + (f<<2) + (g<<1) + (h<<0) ) +} {128} + + +finish_test diff --git a/ext/rbu/rbu14.test b/ext/rbu/rbu14.test new file mode 100644 index 0000000..ac5b92c --- /dev/null +++ b/ext/rbu/rbu14.test @@ -0,0 +1,92 @@ +# 2015 July 25 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test that an RBU data_xxx table may be a view instead of a regular +# table. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/lock_common.tcl +set ::testprefix rbu14 + + +foreach {tn schema} { + 1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c); + } + 2 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t2(b, c); + } + 3 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + } + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t2(b, c); + } +} { + reset_db + + execsql $schema + execsql { + INSERT INTO t1 VALUES(50, 50, 50); + INSERT INTO t1 VALUES(51, 51, 51); + INSERT INTO t2 VALUES(50, 50, 50); + INSERT INTO t2 VALUES(51, 51, 51); + } + + forcedelete rbu.db + do_execsql_test $tn.1 { + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.stuff(tbl, a, b, c, rbu_control); + INSERT INTO stuff VALUES + ('t1', 1, 2, 3, 0), -- insert into t1 + ('t2', 4, 5, 6, 0), -- insert into t2 + ('t1', 50, NULL, NULL, 1), -- delete from t1 + ('t2', 51, NULL, NULL, 1); -- delete from t2 + + CREATE VIEW rbu.data_t1 AS + SELECT a, b, c, rbu_control FROM stuff WHERE tbl='t1'; + CREATE VIEW rbu.data_t2 AS + SELECT a, b, c, rbu_control FROM stuff WHERE tbl='t2'; + } + + do_test $tn.2 { + while 1 { + sqlite3rbu rbu test.db rbu.db + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc + } {SQLITE_DONE} + + do_execsql_test $tn.3.1 { + SELECT * FROM t1 ORDER BY a; + } {1 2 3 51 51 51} + + do_execsql_test $tn.3.2 { + SELECT * FROM t2 ORDER BY a; + } {4 5 6 50 50 50} + + integrity_check $tn.4 +} + + +finish_test diff --git a/ext/rbu/rbu3.test b/ext/rbu/rbu3.test new file mode 100644 index 0000000..4ea6adb --- /dev/null +++ b/ext/rbu/rbu3.test @@ -0,0 +1,200 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu3 + + +# Run the RBU in file $rbu on target database $target until completion. +# +proc run_rbu {target rbu} { + sqlite3rbu rbu $target $rbu + while { [rbu step]=="SQLITE_OK" } {} + rbu close +} + +forcedelete test.db-oal rbu.db +reset_db + +#-------------------------------------------------------------------- +# Test that for an RBU to be applied, no corruption results if the +# affinities on the source and target table do not match. +# +do_execsql_test 1.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b TEXT, c REAL); + CREATE INDEX i1 ON x1(b, c); +} {} + +do_test 1.1 { + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_x1(a, b, c, rbu_control); + INSERT INTO data_x1 VALUES(1, '123', '123', 0); + INSERT INTO data_x1 VALUES(2, 123, 123, 0); + } + db2 close + run_rbu test.db rbu.db +} {SQLITE_DONE} + +do_execsql_test 1.2 { + PRAGMA integrity_check; +} {ok} + +#-------------------------------------------------------------------- +# Test that NULL values may not be inserted into INTEGER PRIMARY KEY +# columns. +# +forcedelete rbu.db +reset_db + +do_execsql_test 2.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b TEXT, c REAL); + CREATE INDEX i1 ON x1(b, c); +} {} + +foreach {tn rbudb} { + 1 { + CREATE TABLE data_x1(a, b, c, rbu_control); + INSERT INTO data_x1 VALUES(NULL, 'a', 'b', 0); + } + + 2 { + CREATE TABLE data_x1(c, b, a, rbu_control); + INSERT INTO data_x1 VALUES('b', 'a', NULL, 0); + } +} { + do_test 2.$tn.1 { + forcedelete rbu.db + sqlite3 db2 rbu.db + db2 eval $rbudb + db2 close + list [catch { run_rbu test.db rbu.db } msg] $msg + } {1 {SQLITE_MISMATCH - datatype mismatch}} + + do_execsql_test 2.1.2 { + PRAGMA integrity_check; + } {ok} +} + +#-------------------------------------------------------------------- +# Test that missing columns are detected. +# +forcedelete rbu.db +reset_db + +do_execsql_test 2.0 { + CREATE TABLE x1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON x1(b, c); +} {} + +do_test 2.1 { + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_x1(a, b, rbu_control); + INSERT INTO data_x1 VALUES(1, 'a', 0); + } + db2 close + list [catch { run_rbu test.db rbu.db } msg] $msg +} {1 {SQLITE_ERROR - column missing from data_x1: c}} + +do_execsql_test 2.2 { + PRAGMA integrity_check; +} {ok} + +# Also extra columns. +# +do_execsql_test 2.3 { + CREATE TABLE x2(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i2 ON x2(b, c); +} {} + +do_test 2.4 { + forcedelete rbu.db + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_x2(a, b, c, d, rbu_control); + INSERT INTO data_x2 VALUES(1, 'a', 2, 3, 0); + } + db2 close + list [catch { run_rbu test.db rbu.db } msg] $msg +} {1 SQLITE_ERROR} + +do_execsql_test 2.5 { + PRAGMA integrity_check; +} {ok} + + +#------------------------------------------------------------------------- +# Test that sqlite3rbu_create_vfs() returns an error if the requested +# parent VFS is unknown. +# +# And that nothing disasterous happens if a VFS name passed to +# sqlite3rbu_destroy_vfs() is unknown or not an RBU vfs. +# +do_test 3.1 { + list [catch {sqlite3rbu_create_vfs xyz nosuchparent} msg] $msg +} {1 SQLITE_NOTFOUND} + +do_test 3.2 { + sqlite3rbu_destroy_vfs nosuchvfs + sqlite3rbu_destroy_vfs unix + sqlite3rbu_destroy_vfs win32 +} {} + +#------------------------------------------------------------------------- +# Test that it is an error to specify an explicit VFS that does not +# include rbu VFS functionality. +# +do_test 4.1 { + testvfs tvfs + sqlite3rbu rbu file:test.db?vfs=tvfs rbu.db + list [catch { rbu step } msg] $msg +} {0 SQLITE_ERROR} +do_test 4.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - rbu vfs not found}} +tvfs delete + +#------------------------------------------------------------------------- +# Test a large rbu update to ensure that wal_autocheckpoint does not get +# in the way. +# +forcedelete rbu.db +reset_db +do_execsql_test 5.1 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(a)) WITHOUT ROWID; + CREATE INDEX i1 ON x1(a); + + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_x1(a, b, c, rbu_control); + WITH s(a, b, c) AS ( + SELECT randomblob(300), randomblob(300), 1 + UNION ALL + SELECT randomblob(300), randomblob(300), c+1 FROM s WHERE c<2000 + ) + INSERT INTO data_x1 SELECT a, b, c, 0 FROM s; +} + +do_test 5.2 { + sqlite3rbu rbu test.db rbu.db + while {[rbu step]=="SQLITE_OK" && [file exists test.db-wal]==0} {} + rbu close +} {SQLITE_OK} + +do_test 5.3 { + expr {[file size test.db-wal] > (1024 * 1200)} +} 1 + +do_test 6.1 { sqlite3rbu_internal_test } {} + +finish_test diff --git a/ext/rbu/rbu5.test b/ext/rbu/rbu5.test new file mode 100644 index 0000000..2ae69e9 --- /dev/null +++ b/ext/rbu/rbu5.test @@ -0,0 +1,303 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test some properties of the pager_rbu_mode and rbu_mode pragmas. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu5 + + +# Return a list of the primary key columns for table $tbl in the database +# opened by database handle $db. +# +proc pkcols {db tbl} { + set ret [list] + $db eval "PRAGMA table_info = '$tbl'" { + if {$pk} { lappend ret $name } + } + return $ret +} + +# Return a list of all columns for table $tbl in the database opened by +# database handle $db. +# +proc allcols {db tbl} { + set ret [list] + $db eval "PRAGMA table_info = '$tbl'" { + lappend ret $name + } + return $ret +} + +# Return a checksum on all tables and data in the main database attached +# to database handle $db. It is possible to add indexes without changing +# the checksum. +# +proc datacksum {db} { + + $db eval { SELECT name FROM sqlite_master WHERE type='table' } { + append txt $name + set cols [list] + set order [list] + set cnt 0 + $db eval "PRAGMA table_info = $name" x { + lappend cols "quote($x(name))" + lappend order [incr cnt] + } + set cols [join $cols ,] + set order [join $order ,] + append txt [$db eval "SELECT $cols FROM $name ORDER BY $order"] + } + return "[string length $txt]-[md5 $txt]" +} + +proc ucontrol {args} { + set ret "" + foreach a $args { + if {$a} { + append ret . + } else { + append ret x + } + } + return $ret +} + +# Argument $target is the name of an SQLite database file. $sql is an SQL +# script containing INSERT, UPDATE and DELETE statements to execute against +# it. This command creates an RBU update database in file $rbu that has +# the same effect as the script. The target database is not modified by +# this command. +# +proc generate_rbu_db {target rbu sql} { + + forcedelete $rbu + forcecopy $target copy.db + + # Evaluate the SQL script to modify the contents of copy.db. + # + sqlite3 dbRbu copy.db + dbRbu eval $sql + + dbRbu function ucontrol ucontrol + + # Evaluate the SQL script to modify the contents of copy.db. + set ret [datacksum dbRbu] + + dbRbu eval { ATTACH $rbu AS rbu } + dbRbu eval { ATTACH $target AS orig } + + dbRbu eval { SELECT name AS tbl FROM sqlite_master WHERE type = 'table' } { + set pk [pkcols dbRbu $tbl] + set cols [allcols dbRbu $tbl] + + # A WHERE clause to test that the PK columns match. + # + set where [list] + foreach c $pk { lappend where "main.$tbl.$c IS orig.$tbl.$c" } + set where [join $where " AND "] + + # A WHERE clause to test that all columns match. + # + set where2 [list] + foreach c $cols { lappend where2 "main.$tbl.$c IS orig.$tbl.$c" } + set ucontrol "ucontrol([join $where2 ,])" + set where2 [join $where2 " AND "] + + # Create a data_xxx table in the RBU update database. + dbRbu eval " + CREATE TABLE rbu.data_$tbl AS SELECT *, '' AS rbu_control + FROM main.$tbl LIMIT 0 + " + + # Find all new rows INSERTed by the script. + dbRbu eval " + INSERT INTO rbu.data_$tbl + SELECT *, 0 AS rbu_control FROM main.$tbl + WHERE NOT EXISTS ( + SELECT 1 FROM orig.$tbl WHERE $where + ) + " + + # Find all old rows DELETEd by the script. + dbRbu eval " + INSERT INTO rbu.data_$tbl + SELECT *, 1 AS rbu_control FROM orig.$tbl + WHERE NOT EXISTS ( + SELECT 1 FROM main.$tbl WHERE $where + ) + " + + # Find all rows UPDATEd by the script. + set origcols [list] + foreach c $cols { lappend origcols "main.$tbl.$c" } + set origcols [join $origcols ,] + dbRbu eval " + INSERT INTO rbu.data_$tbl + SELECT $origcols, $ucontrol AS rbu_control + FROM orig.$tbl, main.$tbl + WHERE $where AND NOT ($where2) + " + + } + + dbRbu close + forcedelete copy.db + + return $ret +} + +#------------------------------------------------------------------------- +# +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t2(x, y, z, PRIMARY KEY(y, z)) WITHOUT ROWID; + + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(2, 4, 6); + INSERT INTO t1 VALUES(3, 6, 9); + + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(2, 4, 6); + INSERT INTO t2 VALUES(3, 6, 9); +} +db close + +set cksum [generate_rbu_db test.db rbu.db { + INSERT INTO t1 VALUES(4, 8, 12); + DELETE FROM t1 WHERE a = 2; + UPDATE t1 SET c = 15 WHERE a=3; + + INSERT INTO t2 VALUES(4, 8, 12); + DELETE FROM t2 WHERE x = 2; + UPDATE t2 SET x = 15 WHERE z=9; +}] + +foreach {tn idx} { + 1 { + } + 2 { + CREATE INDEX i1 ON t1(a, b, c); + CREATE INDEX i2 ON t2(x, y, z); + } +} { + foreach cmd {run step} { + forcecopy test.db test.db2 + forcecopy rbu.db rbu.db2 + + sqlite3 db test.db2 + db eval $idx + + do_test 1.$tn.$cmd.1 { + ${cmd}_rbu test.db2 rbu.db2 + datacksum db + } $cksum + + do_test 1.$tn.$cmd.2 { + db eval { PRAGMA integrity_check } + } {ok} + + db close + } +} + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c, d, e); + INSERT INTO t1 VALUES(-750250,'fyetckfaagjkzqjx',-185831,X'FEAD',444258.29); + INSERT INTO t1 VALUES(649081,NULL,X'7DF25BF78778',-342324.63,'akvspktocwozo'); + INSERT INTO t1 VALUES(-133045,-44822.31,X'',287935,NULL); + INSERT INTO t1 VALUES(202132,NULL,X'5399','cujsjtspryqeyovcdpz','m'); + INSERT INTO t1 VALUES(302910,NULL,'dvdhivtfkaedzhdcnn',-717113.41,688487); + INSERT INTO t1 VALUES(-582327,X'7A267A',X'7E6B3CFE5CB9','zacuzilrok',-196478); + INSERT INTO t1 VALUES(-190462,X'D1A087E7D68D9578','lsmleti',NULL,-928094); + INSERT INTO t1 VALUES(-467665,176344.57,-536684.23,828876.22,X'903E'); + INSERT INTO t1 VALUES(-629138,632630.29,X'28D6',-774501,X'819BBBFC65'); + INSERT INTO t1 VALUES(-828110,-54379.24,-881121.44,X'',X'8D5A894F0D'); + + CREATE TABLE t2(a PRIMARY KEY, b, c, d, e) WITHOUT ROWID; + INSERT INTO t2 VALUES(-65174,X'AC1DBFFE27310F',-194471.08,347988,X'84041BA6F9BDDE86A8'); + INSERT INTO t2 VALUES('bzbpi',-952693.69,811628.25,NULL,-817434); + INSERT INTO t2 VALUES(-643830,NULL,'n',NULL,'dio'); + INSERT INTO t2 VALUES('rovoenxxj',NULL,'owupbtdcoxxnvg',-119676,X'55431DFA'); + INSERT INTO t2 VALUES(899770,'jlygdl',X'DBCA4D1A',NULL,-631773); + INSERT INTO t2 VALUES(334698.80,NULL,-697585.58,-89277,-817352); + INSERT INTO t2 VALUES(X'1A9EB7547A4AAF38','aiprdhkpzdz','anw','szvjbwdvzucybpwwqjt',X'53'); + INSERT INTO t2 VALUES(713220,NULL,'hfcqhqzjuqplvkum',X'20B076075649DE','fthgpvqdyy'); + INSERT INTO t2 VALUES(763908,NULL,'xgslzcpvwfknbr',X'75',X'668146'); + INSERT INTO t2 VALUES(X'E1BA2B6BA27278','wjbpd',NULL,139341,-290086.15); +} +db close + +set cksum [generate_rbu_db test.db rbu.db { +INSERT INTO t2 VALUES(222916.23,'idh',X'472C517405',X'E3',X'7C4F31824669'); +INSERT INTO t2 VALUES('xcndjwafcoxwxizoktd',-319567.21,NULL,-720906.43,-577170); +INSERT INTO t2 VALUES(376369.99,-536058,'yoaiurfqupdscwc',X'29EC8A2542EC3953E9',-740485.22); +INSERT INTO t2 VALUES(X'0EFB4DC50693',-175590.83,X'1779E253CAB5B1789E',X'BC6903',NULL); +INSERT INTO t2 VALUES(-288299,'hfrp',NULL,528477,730676.77); +DELETE FROM t2 WHERE a < -60000; + +UPDATE t2 SET b = 'pgnnaaoflnw' WHERE a = 'bzbpi'; +UPDATE t2 SET c = -675583 WHERE a = 'rovoenxxj'; +UPDATE t2 SET d = X'09CDF2B2C241' WHERE a = 713220; + +INSERT INTO t1 VALUES(224938,'bmruycvfznhhnfmgqys','fr',854381,789143); +INSERT INTO t1 VALUES(-863931,-1386.26,X'2A058540C2FB5C',NULL,X'F9D5990A'); +INSERT INTO t1 VALUES(673696,X'97301F0AC5735F44B5',X'440C',227999.92,-709599.79); +INSERT INTO t1 VALUES(-243640,NULL,-71718.11,X'1EEFEB38',X'8CC7C55D95E142FBA5'); +INSERT INTO t1 VALUES(275893,X'',375606.30,X'0AF9EC334711FB',-468194); +DELETE FROM t1 WHERE a > 200000; + +UPDATE t1 SET b = 'pgnnaaoflnw' WHERE a = -190462; +UPDATE t1 SET c = -675583 WHERE a = -467665; +UPDATE t1 SET d = X'09CDF2B2C241' WHERE a = -133045; + +}] + +foreach {tn idx} { + 1 { + } + 2 { + CREATE UNIQUE INDEX i1 ON t1(b, c, d); + CREATE UNIQUE INDEX i2 ON t1(d, e, a); + CREATE UNIQUE INDEX i3 ON t1(e, d, c, b); + + CREATE UNIQUE INDEX i4 ON t2(b, c, d); + CREATE UNIQUE INDEX i5 ON t2(d, e, a); + CREATE UNIQUE INDEX i6 ON t2(e, d, c, b); + } +} { + foreach cmd {run step} { + forcecopy test.db test.db2 + forcecopy rbu.db rbu.db2 + + sqlite3 db test.db2 + db eval $idx + + do_test 2.$tn.$cmd.1 { + ${cmd}_rbu test.db2 rbu.db2 + datacksum db + } $cksum + + do_test 2.$tn.$cmd.2 { + db eval { PRAGMA integrity_check } + } {ok} + + db close + } +} + + +finish_test diff --git a/ext/rbu/rbu6.test b/ext/rbu/rbu6.test new file mode 100644 index 0000000..29d4c5e --- /dev/null +++ b/ext/rbu/rbu6.test @@ -0,0 +1,100 @@ +# 2014 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. Specifically, it tests the +# outcome of some other client writing to the database while an RBU update +# is being applied. + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu6 + +proc setup_test {} { + reset_db + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b UNIQUE); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b UNIQUE); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b UNIQUE); + } + db close + + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 't1', 0); + INSERT INTO data_t2 VALUES(2, 't2', 0); + INSERT INTO data_t3 VALUES(3, 't3', 0); + } + rbu close +} + +# Test the outcome of some other client writing the db while the *-oal +# file is being generated. Once this has happened, the update cannot be +# progressed. +# +for {set nStep 1} {$nStep < 8} {incr nStep} { + do_test 1.$nStep.1 { + setup_test + sqlite3rbu rbu test.db rbu.db + for {set i 0} {$i<$nStep} {incr i} {rbu step} + + rbu close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'hello') } + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_BUSY} + do_test 1.$nStep.2 { + rbu step + } {SQLITE_BUSY} + do_test 1.$nStep.3 { + list [file exists test.db-oal] [file exists test.db-wal] + } {1 0} + do_test 1.$nStep.4 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database modified during rbu update}} +} + +# Test the outcome of some other client writing the db after the *-oal +# file has been copied to the *-wal path. Once this has happened, any +# other client writing to the db causes RBU to consider its job finished. +# +for {set nStep 8} {$nStep < 20} {incr nStep} { + do_test 1.$nStep.1 { + setup_test + sqlite3rbu rbu test.db rbu.db + for {set i 0} {$i<$nStep} {incr i} {rbu step} + rbu close + sqlite3 db test.db + execsql { INSERT INTO t1 VALUES(5, 'hello') } + sqlite3rbu rbu test.db rbu.db + rbu step + } {SQLITE_DONE} + do_test 1.$nStep.2 { + rbu step + } {SQLITE_DONE} + do_test 1.$nStep.3 { + file exists test.db-oal + } {0} + do_test 1.$nStep.4 { + list [catch { rbu close } msg] $msg + } {0 SQLITE_DONE} + + do_execsql_test 1.$nStep.5 { + SELECT * FROM t1; + } {1 t1 5 hello} +} + + +finish_test diff --git a/ext/rbu/rbu7.test b/ext/rbu/rbu7.test new file mode 100644 index 0000000..ffe6ebe --- /dev/null +++ b/ext/rbu/rbu7.test @@ -0,0 +1,106 @@ +# 2014 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. +# + + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu7 + +# Test index: +# +# 1.*: That affinities are correctly applied to values within the +# RBU database. +# +# 2.*: Tests for multi-column primary keys. +# + +do_test 1.0 { + execsql { + CREATE TABLE t1(a INT PRIMARY KEY, b) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 'abc'); + INSERT INTO t1 VALUES(2, 'def'); + } + + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES('1', NULL, 1); + } + rbu close +} {} + +do_test 1.1 { + sqlite3rbu rbu test.db rbu.db + while { [rbu step]=="SQLITE_OK" } {} + rbu close +} {SQLITE_DONE} + +sqlite3 db test.db +do_execsql_test 1.2 { + SELECT * FROM t1 +} {2 def} + +#------------------------------------------------------------------------- +# +foreach {tn tbl} { + 1 { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID } + 2 { CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b)) } +} { + reset_db + + execsql $tbl + do_execsql_test 2.$tn.1 { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 'a'); + INSERT INTO t1 VALUES(1, 2, 'b'); + INSERT INTO t1 VALUES(2, 1, 'c'); + INSERT INTO t1 VALUES(2, 2, 'd'); + } + + do_test 2.$tn.2 { + forcedelete rbu.db + sqlite3 rbu rbu.db + execsql { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(3, 1, 'e', 0); + INSERT INTO data_t1 VALUES(3, 2, 'f', 0); + INSERT INTO data_t1 VALUES(1, 2, NULL, 1); + INSERT INTO data_t1 VALUES(2, 1, 'X', '..x'); + } rbu + rbu close + } {} + + do_test 2.$tn.3 { + set rc "SQLITE_OK" + while {$rc == "SQLITE_OK"} { + sqlite3rbu rbu test.db rbu.db + rbu step + set rc [rbu close] + } + set rc + } {SQLITE_DONE} + + do_execsql_test 2.$tn.1 { + SELECT * FROM t1 ORDER BY a, b + } { + 1 1 a + 2 1 X + 2 2 d + 3 1 e + 3 2 f + } +} + +finish_test diff --git a/ext/rbu/rbu8.test b/ext/rbu/rbu8.test new file mode 100644 index 0000000..b73bde2 --- /dev/null +++ b/ext/rbu/rbu8.test @@ -0,0 +1,72 @@ +# 2014 November 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test the rbu_delta() feature. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu8 + +do_execsql_test 1.0 { + CREATE TABLE t1(x, y PRIMARY KEY, z); + INSERT INTO t1 VALUES(NULL, 1, 'one'); + INSERT INTO t1 VALUES(NULL, 2, 'two'); + INSERT INTO t1 VALUES(NULL, 3, 'three'); + CREATE INDEX i1z ON t1(z, x); +} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_t1(x, y, z, rbu_control); + INSERT INTO data_t1 VALUES('a', 1, '_i' , 'x.d'); + INSERT INTO data_t1 VALUES('b', 2, 2 , '..x'); + INSERT INTO data_t1 VALUES('_iii', 3, '-III' , 'd.d'); + } + db2 close +} {} + +do_test 1.2.1 { + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} +do_test 1.2.2 { + list [catch {rbu close} msg] $msg +} {1 {SQLITE_ERROR - no such function: rbu_delta}} + +proc rbu_delta {orig new} { + return "${orig}${new}" +} + +do_test 1.3.1 { + while 1 { + sqlite3rbu rbu test.db rbu.db + rbu create_rbu_delta + set rc [rbu step] + if {$rc != "SQLITE_OK"} break + rbu close + } + rbu close +} {SQLITE_DONE} + +do_execsql_test 1.3.2 { + SELECT * FROM t1 +} { + a 1 one_i + {} 2 2 + _iii 3 three-III +} +integrity_check 1.3.3 + + +finish_test diff --git a/ext/rbu/rbu9.test b/ext/rbu/rbu9.test new file mode 100644 index 0000000..499ff09 --- /dev/null +++ b/ext/rbu/rbu9.test @@ -0,0 +1,125 @@ +# 2014 November 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Test RBU with virtual tables. And tables with no PRIMARY KEY declarations. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbu9 + +ifcapable !fts3 { + finish_test + return +} + +do_execsql_test 1.1 { + CREATE VIRTUAL TABLE f1 USING fts4(a, b, c); + INSERT INTO f1(rowid, a, b, c) VALUES(11, 'a', 'b', 'c'); + INSERT INTO f1(rowid, a, b, c) VALUES(12, 'd', 'e', 'f'); + INSERT INTO f1(rowid, a, b, c) VALUES(13, 'g', 'h', 'i'); +} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_f1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data_f1 VALUES(14, 'x', 'y', 'z', 0); -- INSERT + INSERT INTO data_f1 VALUES(11, NULL, NULL, NULL, 1); -- DELETE + INSERT INTO data_f1 VALUES(13, NULL, NULL, 'X', '..x'); -- UPDATE + } + db2 close +} {} + +do_test 1.2.1 { + while 1 { + sqlite3rbu rbu test.db rbu.db + set rc [rbu step] + if {$rc != "SQLITE_OK"} break + rbu close + } + rbu close +} {SQLITE_DONE} + +do_execsql_test 1.2.2 { SELECT rowid, * FROM f1 } { + 12 d e f + 13 g h X + 14 x y z +} +do_execsql_test 1.2.3 { INSERT INTO f1(f1) VALUES('integrity-check') } +integrity_check 1.2.4 + +#------------------------------------------------------------------------- +# Tables with no PK declaration. +# + +# Run the RBU in file $rbu on target database $target until completion. +# +proc run_rbu {target rbu} { + sqlite3rbu rbu $target $rbu + while { [rbu step]=="SQLITE_OK" } {} + rbu close +} + +foreach {tn idx} { + 1 { } + 2 { + CREATE INDEX i1 ON t1(a); + } + 3 { + CREATE INDEX i1 ON t1(b, c); + CREATE INDEX i2 ON t1(c, b); + CREATE INDEX i3 ON t1(a, a, a, b, b, b, c, c, c); + } +} { + + reset_db + do_execsql_test 2.$tn.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1(rowid, a, b, c) VALUES(-1, 'a', 'b', 'c'); + INSERT INTO t1(rowid, a, b, c) VALUES(-2, 'd', 'e', 'f'); + } + + db eval $idx + + do_test 2.$tn.2 { + forcedelete rbu.db + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(3, 'x', 'y', 'z', 0); + INSERT INTO data_t1 VALUES(NULL, 'X', 'Y', 'Z', 0); + INSERT INTO data_t1 VALUES('1', NULL, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(-2, NULL, NULL, 'fff', '..x'); + } + db2 close + } {} + + run_rbu test.db rbu.db + + do_execsql_test 2.$tn.3 { + SELECT rowid, a, b, c FROM t1 ORDER BY rowid; + } { + -2 d e fff + -1 a b c + 2 4 5 6 + 3 x y z + 4 X Y Z + } + + integrity_check 2.$tn.4 +} + + +finish_test diff --git a/ext/rbu/rbuA.test b/ext/rbu/rbuA.test new file mode 100644 index 0000000..b43d30d --- /dev/null +++ b/ext/rbu/rbuA.test @@ -0,0 +1,80 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that it is an error to attempt to update +# a wal mode database via RBU. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuA + +set db_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c); +} +set rbu_sql { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); +} + +do_test 1.0 { + db close + forcedelete test.db rbu.db + + sqlite3 db test.db + db eval $db_sql + db eval { PRAGMA journal_mode = wal } + db close + + sqlite3 db rbu.db + db eval $rbu_sql + db close + + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} +do_test 1.1 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + +do_test 2.0 { + forcedelete test.db rbu.db + + sqlite3 db test.db + db eval $db_sql + db close + + sqlite3 db rbu.db + db eval $rbu_sql + db close + + sqlite3rbu rbu test.db rbu.db + rbu step + rbu close +} {SQLITE_OK} + +do_test 2.1 { + sqlite3 db test.db + db eval {PRAGMA journal_mode = wal} + db close + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_ERROR} + +do_test 2.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + + +finish_test diff --git a/ext/rbu/rbuB.test b/ext/rbu/rbuB.test new file mode 100644 index 0000000..3e77763 --- /dev/null +++ b/ext/rbu/rbuB.test @@ -0,0 +1,62 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuB + +db close +sqlite3_shutdown +test_sqlite3_log xLog +reset_db + +proc xLog {args} { } + +set db_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c); +} +set rbu_sql { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); +} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval $rbu_sql + rbu close + + db eval $db_sql +} {} + +set ::errlog [list] +proc xLog {err msg} { lappend ::errlog $err } +do_test 1.2 { + run_rbu test.db rbu.db +} {SQLITE_DONE} + +do_test 1.3 { + set ::errlog +} {SQLITE_NOTICE_RECOVER_WAL SQLITE_NOTICE_RBU} + +do_execsql_test 1.4 { + SELECT * FROM t1 +} {1 2 3 4 5 6 7 8 9} + +db close +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test diff --git a/ext/rbu/rbuC.test b/ext/rbu/rbuC.test new file mode 100644 index 0000000..a38a691 --- /dev/null +++ b/ext/rbu/rbuC.test @@ -0,0 +1,142 @@ +# 2016 March 7 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# Tests for RBU focused on the REPLACE operation (rbu_control column +# contains integer value 2). +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuC + +#------------------------------------------------------------------------- +# This test is actually of an UPDATE directive. Just to establish that +# these work with UNIQUE indexes before preceding to REPLACE. +# +do_execsql_test 1.0 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b, c UNIQUE); + INSERT INTO t1 VALUES(1, 'a', 'b', 'c'); +} + +forcedelete rbu.db +do_execsql_test 1.1 { + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_t1(i, a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 'a', 'b', 'c', '.xxx'); +} + +do_test 1.2 { + step_rbu test.db rbu.db +} {SQLITE_DONE} + +do_execsql_test 1.3 { + SELECT * FROM t1 +} { + 1 a b c +} + +#------------------------------------------------------------------------- +# +foreach {tn schema} { + 1 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b, c UNIQUE); + CREATE INDEX t1a ON t1(a); + } + 2 { + CREATE TABLE t1(i PRIMARY KEY, a, b, c UNIQUE); + CREATE INDEX t1a ON t1(a); + } + 3 { + CREATE TABLE t1(i PRIMARY KEY, a, b, c UNIQUE) WITHOUT ROWID; + CREATE INDEX t1a ON t1(a); + } +} { + reset_db + forcedelete rbu.db + execsql $schema + + do_execsql_test 2.$tn.0 { + INSERT INTO t1 VALUES(1, 'a', 'b', 'c'); + INSERT INTO t1 VALUES(2, 'b', 'c', 'd'); + INSERT INTO t1 VALUES(3, 'c', 'd', 'e'); + } + + do_execsql_test 2.$tn.1 { + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_t1(i, a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 1, 2, 3, 2); + INSERT INTO data_t1 VALUES(3, 'c', 'd', 'e', 2); + INSERT INTO data_t1 VALUES(4, 'd', 'e', 'f', 2); + } + + do_test 2.$tn.2 { + step_rbu test.db rbu.db + } {SQLITE_DONE} + + do_execsql_test 2.$tn.3 { + SELECT * FROM t1 ORDER BY i + } { + 1 1 2 3 + 2 b c d + 3 c d e + 4 d e f + } + + integrity_check 2.$tn.4 +} + +foreach {tn schema} { + 1 { + CREATE TABLE t1(a, b, c UNIQUE); + CREATE INDEX t1a ON t1(a); + } + + 2 { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + } +} { + if {$tn==2} { ifcapable !fts5 break } + reset_db + forcedelete rbu.db + execsql $schema + + do_execsql_test 3.$tn.0 { + INSERT INTO t1 VALUES('a', 'b', 'c'); + INSERT INTO t1 VALUES('b', 'c', 'd'); + INSERT INTO t1 VALUES('c', 'd', 'e'); + } + + do_execsql_test 3.$tn.1 { + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 1, 2, 3, 2); + INSERT INTO data_t1 VALUES(3, 'c', 'd', 'e', 2); + INSERT INTO data_t1 VALUES(4, 'd', 'e', 'f', 2); + } + + do_test 3.$tn.2 { + step_rbu test.db rbu.db + } {SQLITE_DONE} + + do_execsql_test 3.$tn.3 { + SELECT rowid, * FROM t1 ORDER BY 1 + } { + 1 1 2 3 + 2 b c d + 3 c d e + 4 d e f + } + + integrity_check 3.$tn.4 +} + + + +finish_test diff --git a/ext/rbu/rbu_common.tcl b/ext/rbu/rbu_common.tcl new file mode 100644 index 0000000..57db5a5 --- /dev/null +++ b/ext/rbu/rbu_common.tcl @@ -0,0 +1,120 @@ +# 2015 Aug 8 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +if {![info exists testdir]} { + set testdir [file join [file dirname [info script]] .. .. test] +} +source $testdir/tester.tcl + +proc if_no_rbu_support {tcl} { + set bOk 1 + ifcapable !rbu { set bOk 0 } + if {[permutation]=="journaltest"} { set bOk 0 } + if {$bOk==0} { + set c [catch {uplevel 1 $tcl} r] + return -code $c $r + } +} + +proc check_prestep_state {target state} { + set oal_exists [file exists $target-oal] + set wal_exists [file exists $target-wal] + set progress [rbu progress] + + if {($progress==0 && $state!="oal" && $state!="done") + || ($oal_exists && $wal_exists) + || ($progress>0 && $state=="oal" && (!$oal_exists || $wal_exists)) + || ($state=="move" && (!$oal_exists || $wal_exists)) + || ($state=="checkpoint" && ($oal_exists || !$wal_exists)) + || ($state=="done" && ($oal_exists && $progress!=0)) + } { + error "B: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } +} + +proc check_poststep_state {rc target state} { + if {$rc=="SQLITE_OK" || $rc=="SQLITE_DONE"} { + set oal_exists [file exists $target-oal] + set wal_exists [file exists $target-wal] + if {$state=="move" && ($oal_exists || !$wal_exists)} { + error "A: state=$state progress=$progress oal=$oal_exists wal=$wal_exists" + } + } +} + +# Run the RBU in file $rbu on target database $target until completion. +# +proc run_rbu {target rbu} { + sqlite3rbu rbu $target $rbu + while 1 { + set state [rbu state] + + check_prestep_state $target $state + set rc [rbu step] + check_poststep_state $rc $target $state + + if {$rc!="SQLITE_OK"} break + } + rbu close +} + +proc step_rbu {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set state [rbu state] + check_prestep_state $target $state + set rc [rbu step] + check_poststep_state $rc $target $state + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +proc step_rbu_legacy {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set state [rbu state] + check_prestep_state $target $state + set rc [rbu step] + check_poststep_state $rc $target $state + rbu close + if {$rc != "SQLITE_OK"} break + sqlite3 tmpdb $rbu + tmpdb eval { DELETE FROM rbu_state WHERE k==10 } + tmpdb close + } + set rc +} + +proc do_rbu_vacuum_test {tn step {statedb state.db}} { + forcedelete $statedb + if {$statedb=="" && $step==1} breakpoint + uplevel [list do_test $tn.1 [string map [list %state% $statedb %step% $step] { + if {%step%==0} { sqlite3rbu_vacuum rbu test.db {%state%}} + while 1 { + if {%step%==1} { sqlite3rbu_vacuum rbu test.db {%state%}} + set state [rbu state] + check_prestep_state test.db $state + set rc [rbu step] + check_poststep_state $rc test.db $state + if {$rc!="SQLITE_OK"} break + if {%step%==1} { rbu close } + } + rbu close + }] {SQLITE_DONE}] + + uplevel [list do_execsql_test $tn.2 { + PRAGMA integrity_check + } ok] +} + diff --git a/ext/rbu/rbubusy.test b/ext/rbu/rbubusy.test new file mode 100644 index 0000000..b4b4321 --- /dev/null +++ b/ext/rbu/rbubusy.test @@ -0,0 +1,89 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +ifcapable !rbu { finish_test ; return } +if_no_rbu_support { finish_test ; return } +set ::testprefix rbubusy + +db close +sqlite3_shutdown +test_sqlite3_log xLog +reset_db + +set db_sql { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1000, 2000, 3000); +} + +set rbu_sql { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(4, 5, 6, 0); + INSERT INTO data_t1 VALUES(7, 8, 9, 0); +} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval $rbu_sql + rbu close + + db eval $db_sql +} {} + +do_execsql_test 1.2 { + BEGIN; + SELECT * FROM t1 +} {1000 2000 3000} + +do_test 1.3 { + sqlite3rbu rbu test.db rbu.db + rbu step +} {SQLITE_OK} + +do_test 1.4 { + while 1 { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + } + set rc +} {SQLITE_BUSY} + +do_test 1.5 { + rbu step +} {SQLITE_BUSY} + +do_test 1.6 { + db eval COMMIT + rbu step +} {SQLITE_BUSY} +catch { rbu close } + +do_test 1.7 { + sqlite3rbu rbu test.db rbu.db + while 1 { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + } + set rc +} {SQLITE_DONE} + +rbu close + +db close +sqlite3_shutdown +test_sqlite3_log +sqlite3_initialize +finish_test + diff --git a/ext/rbu/rbucollate.test b/ext/rbu/rbucollate.test new file mode 100644 index 0000000..beb54aa --- /dev/null +++ b/ext/rbu/rbucollate.test @@ -0,0 +1,58 @@ +# 2018 March 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbucollate + +ifcapable !icu_collations { + finish_test + return +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES('a', 'one', 1, 0); + INSERT INTO data_t1 VALUES('b', 'two', 2, 0); + INSERT INTO data_t1 VALUES('c', 'three', 3, 0); + } + rbu1 close + return $filename +} + +do_execsql_test 1.0 { + SELECT icu_load_collation('en_US', 'my-collate'); + CREATE TABLE t1(a COLLATE "my-collate" PRIMARY KEY, b, c); +} {{}} + +do_test 1.2 { + create_rbu1 testrbu.db + sqlite3rbu rbu test.db testrbu.db + rbu dbMain_eval { SELECT icu_load_collation('en_US', 'my-collate') } + rbu dbRbu_eval { SELECT icu_load_collation('en_US', 'my-collate') } + while 1 { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + } + rbu close + db eval { SELECT * FROM t1 } +} {a one 1 b two 2 c three 3} + +#forcedelete testrbu.db +finish_test diff --git a/ext/rbu/rbucrash.test b/ext/rbu/rbucrash.test new file mode 100644 index 0000000..e187e8d --- /dev/null +++ b/ext/rbu/rbucrash.test @@ -0,0 +1,139 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbucrash + +# Set up a target database and an rbu update database. The target +# db is the usual "test.db", the rbu db is "test.db2". +# +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(10, 11, 12, 0); + INSERT INTO data_t1 VALUES(13, 14, 15, 0); + INSERT INTO data_t1 VALUES(4, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(1, NULL, 100, '..x'); +} +db_save_and_close + + +# Determine the number of steps in applying the rbu update to the test +# target database created above. Set $::rbu_num_steps accordingly +# +# Check that the same number of steps are required to apply the rbu +# update using many calls to sqlite3rbu_step() on a single rbu handle +# as required to apply it using a series of rbu handles, on each of +# which sqlite3rbu_step() is called once. +# +do_test 1.1 { + db_restore + sqlite3rbu rbu test.db test.db2 + set nStep 0 + while {[rbu step]=="SQLITE_OK"} { incr nStep } + rbu close +} {SQLITE_DONE} +set rbu_num_steps $nStep +do_test 1.2 { + db_restore + set nStep 0 + while {1} { + sqlite3rbu rbu test.db test.db2 + rbu step + if {[rbu close]=="SQLITE_DONE"} break + incr nStep + } + set nStep +} $rbu_num_steps + + +# Run one or more tests using the target (test.db) and rbu (test.db2) +# databases created above. As follows: +# +# 1. This process starts the rbu update and calls sqlite3rbu_step() +# $nPre times. Then closes the rbu update handle. +# +# 2. A second process resumes the rbu update and attempts to call +# sqlite3rbu_step() $nStep times before closing the handle. A +# crash is simulated during each xSync() of file test.db2. +# +# 3. This process attempts to resume the rbu update from whatever +# state it was left in by step (2). Test that it is successful +# in doing so and that the final target database is as expected. +# +# In total (nSync+1) tests are run, where nSync is the number of times +# xSync() is called on test.db2. +# +proc do_rbu_crash_test {tn nPre nStep} { + + set script [subst -nocommands { + sqlite3rbu rbu test.db file:test.db2?vfs=crash + set i 0 + while {[set i] < $nStep} { + if {[rbu step]!="SQLITE_OK"} break + incr i + } + rbu close + }] + + set bDone 0 + for {set iDelay 1} {$bDone==0} {incr iDelay} { + forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal + db_restore + + if {$nPre>0} { + sqlite3rbu rbu test.db file:test.db2 + set i 0 + for {set i 0} {$i < $nPre} {incr i} { + if {[rbu step]!="SQLITE_OK"} break + } + rbu close + } + + set res [ + crashsql -file test.db2 -delay $iDelay -tclbody $script -opendb {} {} + ] + + set bDone 1 + if {$res == "1 {child process exited abnormally}"} { + set bDone 0 + } elseif {$res != "0 {}"} { + error "unexected catchsql result: $res" + } + + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + + sqlite3 db test.db + do_execsql_test $tn.delay=$iDelay { + SELECT * FROM t1; + PRAGMA integrity_check; + } {1 2 100 7 8 9 10 11 12 13 14 15 ok} + db close + } +} + +for {set nPre 0} {$nPre < $rbu_num_steps} {incr nPre} { + for {set is 1} {$is <= ($rbu_num_steps - $nPre)} {incr is} { + do_rbu_crash_test 2.pre=$nPre.step=$is $nPre $is + } +} + +finish_test diff --git a/ext/rbu/rbucrash2.test b/ext/rbu/rbucrash2.test new file mode 100644 index 0000000..10721df --- /dev/null +++ b/ext/rbu/rbucrash2.test @@ -0,0 +1,97 @@ +# 2017 March 02 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbucrash2 + +# Set up a target database and an rbu update database. The target +# db is the usual "test.db", the rbu db is "test.db2". +# +forcedelete test.db2 +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a), UNIQUE(b)); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES('one', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('two', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('three', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('four', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('five', randomblob(3500), NULL, 0); + INSERT INTO data_t1 VALUES('six', randomblob(3500), NULL, 0); +} +db_save_and_close + +proc do_rbu_crash_test2 {tn script} { + + foreach {f blksz} { + test.db 512 + test.db2 512 + test.db 4096 + test.db2 4096 + } { + set bDone 0 + for {set iDelay 1} {$bDone==0} {incr iDelay} { + forcedelete test.db2 test.db2-journal test.db test.db-oal test.db-wal + db_restore + + set res [ + crashsql -file $f -delay $iDelay -tclbody $script -dflt 1 -opendb {} \ + -blocksize $blksz {} + ] + + set bDone 1 + if {$res == "1 {child process exited abnormally}"} { + set bDone 0 + } elseif {$res != "0 {}"} { + error "unexected catchsql result: $res" + } + + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + + sqlite3 db test.db + do_execsql_test $tn.delay=$iDelay.f=$f.blksz=$blksz { + PRAGMA integrity_check; + } {ok} + db close + } + } +} + +for {set x 1} {$x < 10} {incr x} { + do_rbu_crash_test2 1.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu savestate + } + rbu close + } +} + +for {set x 1} {$x < 2} {incr x} { + do_rbu_crash_test2 2.$x { + sqlite3rbu rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} { + rbu close + sqlite3rbu rbu test.db test.db2 + } + rbu close + } +} + +finish_test diff --git a/ext/rbu/rbudiff.test b/ext/rbu/rbudiff.test new file mode 100644 index 0000000..e5e3df1 --- /dev/null +++ b/ext/rbu/rbudiff.test @@ -0,0 +1,309 @@ +# 2015-07-31 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# Tests for the [sqldiff --rbu] command. +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set testprefix rbudiff + +set PROG [test_find_sqldiff] +db close + +proc get_rbudiff_sql {db1 db2} { + exec $::PROG --rbu $db1 $db2 +} + +proc get_vtab_rbudiff_sql {db1 db2} { + exec $::PROG --vtab --rbu $db1 $db2 +} + +proc step_rbu {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +proc apply_rbudiff {sql target} { + test_rbucount $sql + forcedelete rbu.db + sqlite3 rbudb rbu.db + rbudb eval $sql + rbudb close + step_rbu $target rbu.db +} + +proc sqlesc {id} { + set ret "'[string map {' ''} $id]'" + set ret +} + +# The only argument is the output of an [sqldiff -rbu] run. This command +# tests that the contents of the rbu_count table is correct. An exception +# is thrown if it is not. +# +proc test_rbucount {sql} { + sqlite3 tmpdb "" + tmpdb eval $sql + tmpdb eval { + SELECT name FROM sqlite_master WHERE name LIKE 'data%' AND type='table' + } { + set a [tmpdb eval "SELECT count(*) FROM [sqlesc $name]"] + set b [tmpdb eval {SELECT cnt FROM rbu_count WHERE tbl = $name}] + if {$a != $b} { + tmpdb close + error "rbu_count error - tbl = $name" + } + } + tmpdb close + return "" +} + +proc rbudiff_cksum {db1} { + set txt "" + + sqlite3 dbtmp $db1 + foreach tbl [dbtmp eval {SELECT name FROM sqlite_master WHERE type='table'}] { + set cols [list] + dbtmp eval "PRAGMA table_info = [sqlesc $tbl]" { + lappend cols "quote( $name )" + } + append txt [dbtmp eval \ + "SELECT [join $cols {||'.'||}] FROM [sqlesc $tbl] ORDER BY 1" + ] + } + dbtmp close + + md5 $txt +} + +foreach {tn init mod} { + 1 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + + CREATE TABLE t2(a, b, c, PRIMARY KEY(b, c)); + INSERT INTO t2 VALUES(1, 2, 3); + INSERT INTO t2 VALUES(4, 5, 6); + } { + INSERT INTO t1 VALUES(7, 8, 9); + DELETE FROM t1 WHERE a=4; + UPDATE t1 SET c = 11 WHERE a = 1; + + INSERT INTO t2 VALUES(7, 8, 9); + DELETE FROM t2 WHERE a=4; + UPDATE t2 SET c = 11 WHERE a = 1; + } + + 2 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO t1 VALUES('u', 'v', 'w'); + INSERT INTO t1 VALUES('x', 'y', 'z'); + } { + DELETE FROM t1 WHERE a='u'; + INSERT INTO t1 VALUES('a', 'b', 'c'); + } + + 3 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, x); + INSERT INTO t1 VALUES(1, + X'0000000000000000111111111111111122222222222222223333333333333333' + ); + CREATE TABLE t2(y INTEGER PRIMARY KEY, x); + INSERT INTO t2 VALUES(1, + X'0000000000000000111111111111111122222222222222223333333333333333' + ); + } { + DELETE FROM t1; + INSERT INTO t1 VALUES(1, + X'0000000000000000111111111111111122222555555552223333333333333333' + ); + DELETE FROM t2; + INSERT INTO t2 VALUES(1, + X'0000000000000000111111111111111122222222222222223333333FFF333333' + ); + } + + 4 { + CREATE TABLE x1(a, b, c, PRIMARY KEY(a, b, c)); + INSERT INTO x1 VALUES('u', 'v', NULL); + INSERT INTO x1 VALUES('x', 'y', 'z'); + INSERT INTO x1 VALUES('a', NULL, 'b'); + } { + INSERT INTO x1 VALUES('a', 'b', 'c'); + } + + 5 { + CREATE TABLE t1(a PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, NULL); + INSERT INTO t1 VALUES(2, X''); + } { + UPDATE t1 SET b = X'' WHERE a=1; + UPDATE t1 SET b = NULL WHERE a=2; + } + +} { + catch { db close } + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db close + + do_test 1.$tn.2 { + set sql [get_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 1.$tn.3 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init ; $mod" + sqlite3 db test.db2 + db eval "$init" + db close + + do_test 1.$tn.4 { + set sql [get_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 1.$tn.5 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] +} + +#------------------------------------------------------------------------- +# Test that if the --vtab switch is present, [sqldiff] handles virtual +# table types fts[345] and rtree correctly. +# +ifcapable fts3&&fts5&&rtree { + +foreach {tn init mod} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 2 { + CREATE VIRTUAL TABLE "x y" USING 'rtree'(id, x1, x2); + INSERT INTO "x y" VALUES(1, 2, 3); + INSERT INTO "x y" VALUES(2, 4, 6); + } { + DELETE FROM "x y" WHERE rowid = 1; + INSERT INTO "x y" VALUES(3, 6, 9); + } + + 3 { + CREATE VIRTUAL TABLE 'x''y' USING fts3; + INSERT INTO 'x''y' VALUES('one two three'); + INSERT INTO 'x''y' VALUES('four five six'); + } { + DELETE FROM 'x''y' WHERE rowid = 1; + INSERT INTO 'x''y' VALUES('one two three'); + } +} { + + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db close + + do_test 2.$tn.1 { + set sql [get_vtab_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + do_test 2.$tn.2 { rbudiff_cksum test.db } [rbudiff_cksum test.db2] +} + +} + +ifcapable fts5 { + foreach {tn init mod} { + 1 { + CREATE VIRTUAL TABLE t1 USING fts5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 2 { + CREATE VIRTUAL TABLE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + 3 { + creAte virTUal +tablE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + 4 { + creAte virTUal tablE t1 USING FTs5(c); + INSERT INTO t1 VALUES('a b c'); + INSERT INTO t1 VALUES('a b c'); + } { + DELETE FROM t1 WHERE rowid = 1; + INSERT INTO t1 VALUES('a b c'); + } + + } { + forcedelete test.db test.db2 + sqlite3 db test.db + db eval "$init" + sqlite3 db test.db2 + db eval "$init ; $mod" + db eval { INSERT INTO t1(t1) VALUES('optimize') } + db close + + do_test 3.$tn.1 { + set sql [get_vtab_rbudiff_sql test.db test.db2] + apply_rbudiff $sql test.db + } {SQLITE_DONE} + + sqlite3 db test.db + sqlite3 db2 test.db2 + do_test 3.$tn.2 { + db2 eval { SELECT * FROM t1 ORDER BY rowid } + } [db eval { SELECT * FROM t1 ORDER BY rowid }] + + do_test 3.$tn.3 { + db2 eval { INSERT INTO t1(t1) VALUES('integrity-check') } + } {} + + db close + db2 close + } +} + + +finish_test diff --git a/ext/rbu/rbudor.test b/ext/rbu/rbudor.test new file mode 100644 index 0000000..c456d6a --- /dev/null +++ b/ext/rbu/rbudor.test @@ -0,0 +1,56 @@ +# 2016 October 21 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This test file focuses on interactions between RBU and the feature +# enabled by SQLITE_DIRECT_OVERFLOW_READ - Direct Overflow Read. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbudor + +set bigA [string repeat a 5000] +set bigB [string repeat b 5000] +do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB); + INSERT INTO t1 VALUES(1, $bigA); +} {} + +do_test 1.1 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, $bigB, 0); + } + rbu close +} {} + +do_test 1.2 { + sqlite3rbu rbu test.db rbu.db + while {[rbu state]!="checkpoint"} { + rbu step + } + rbu step + db eval { SELECT * FROM t1 } +} [list 1 $bigA 2 $bigB] + +do_test 1.3 { + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} {SQLITE_DONE} + +do_execsql_test 1.4 { + SELECT * FROM t1 +} [list 1 $bigA 2 $bigB] + +finish_test diff --git a/ext/rbu/rbuexlock.test b/ext/rbu/rbuexlock.test new file mode 100644 index 0000000..28ab308 --- /dev/null +++ b/ext/rbu/rbuexlock.test @@ -0,0 +1,292 @@ +# 2021 November 06 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuexlock + +db close + +set journalmode delete +if {[permutation]=="inmemory_journal"} { + set journalmode memory +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu {filename} { + forcedelete $filename + sqlite3 rbu1 $filename + rbu1 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(10, random(), random(), 0); + INSERT INTO data_t1 VALUES(20, random(), random(), 0); + INSERT INTO data_t1 VALUES(30, random(), random(), 0); + INSERT INTO data_t1 VALUES(40, random(), random(), 0); + INSERT INTO data_t1 VALUES(50, random(), random(), 0); + INSERT INTO data_t1 VALUES(60, random(), random(), 0); + INSERT INTO data_t1 VALUES(70, random(), random(), 0); + INSERT INTO data_t1 VALUES(80, random(), random(), 0); + } + rbu1 close + return $filename +} + +reset_db + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b INT, c INT); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); +} +create_rbu rbu1.db + +do_test 1.1.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + rbu step +} SQLITE_OK +do_catchsql_test 1.1.1 { SELECT * FROM t1 } {0 {1 2 3}} + +do_test 1.2.0 { + for {set ii 0} {$ii < 10} {incr ii} { + rbu step + } + rbu step +} SQLITE_OK +do_catchsql_test 1.2.1 { SELECT * FROM t1 } {0 {1 2 3}} +do_test 1.2.2 { + db eval {PRAGMA journal_mode} +} $journalmode + +do_test 1.3.0 { + while {[file exists test.db-wal]==0} { + rbu step + } +} {} +do_catchsql_test 1.3.1 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.3.2 { + db eval {PRAGMA journal_mode} +} $journalmode + + +do_test 1.4.0 { + rbu step +} SQLITE_OK +do_catchsql_test 1.4.1 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.4.2 { + db eval {PRAGMA journal_mode} +} $journalmode + + +rbu close + +do_test 1.5.0 { + file exists test.db-wal +} {1} +do_test 1.5.1 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + file exists test.db-wal +} 1 +do_catchsql_test 1.5.2 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.5.2 { + db eval {PRAGMA journal_mode} +} $journalmode + + +do_test 1.6.0 { + rbu step +} SQLITE_OK +do_catchsql_test 1.6.1 { SELECT * FROM t1 } {1 {database is locked}} +do_test 1.6.2 { + db eval {PRAGMA journal_mode} +} $journalmode + +do_test 1.7.0 { + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} SQLITE_DONE +do_catchsql_test 1.7.2 { SELECT count(*) FROM t1 } {0 9} +do_test 1.7.2 { + db eval {PRAGMA journal_mode} +} $journalmode + +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(a PRIMARY KEY, b INT, c INT); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); +} +create_rbu rbu1.db + +do_test 2.1.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_OK +do_catchsql_test 2.1.1 { SELECT * FROM t1 } {0 {1 2 3}} + +do_test 2.2.0 { + for {set ii 0} {$ii < 10} {incr ii} { + rbu step + } + rbu step +} SQLITE_OK +do_catchsql_test 2.2.1 { SELECT * FROM t1 } {0 {1 2 3}} + +do_test 2.3.0 { + while {[file exists test.db-wal]==0} { + rbu step + } +} {} +do_test 2.3.1 { + llength [db eval {SELECT * FROM t1}] +} {27} +do_test 2.3.2 { + db eval {PRAGMA journal_mode} +} {wal} + +do_test 2.4.0 { + rbu step +} SQLITE_OK +do_test 2.4.1 { + llength [db eval {SELECT * FROM t1}] +} {27} +do_test 2.4.2 { + db eval {PRAGMA journal_mode} +} {wal} + +rbu close + +do_test 2.5.0 { + db eval {PRAGMA journal_mode} +} {wal} +do_execsql_test 2.5.1 { + DELETE FROM t1; +} {} + +create_rbu rbu1.db +do_test 3.1.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_ERROR + +do_test 3.1.1 { + set rc [catch {rbu close} msg] + lappend rc $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} +db eval {PRAGMA journal_mode=DELETE} + +create_rbu rbu1.db +do_test 3.2.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_OK + +do_test 3.3.1 { + set rc [catch {rbu close} msg] + lappend rc $msg +} {0 SQLITE_OK} + +db close +create_rbu rbu1.db +do_test 3.4.0 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=0 rbu1.db + rbu step +} SQLITE_OK +rbu close + +#------------------------------------------------------------------------- +reset_db +forcedelete rbu1.db +forcedelete rbu2.db + +sqlite3 rbu rbu1.db +do_execsql_test -db rbu 4.1 { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 'one', 0); +} +rbu close +sqlite3 rbu rbu2.db +do_execsql_test -db rbu 4.2 { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, 'two', 0); +} +rbu close + +do_execsql_test 4.3 { + CREATE TABLE t1(a PRIMARY KEY, b); +} +db close + +do_test 4.4 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu1.db + rbu step + rbu state +} {oal} + +sqlite3 cons test.db +do_execsql_test -db cons 4.5 { + SELECT * FROM t1 +} {} + +do_test 4.6 { rbu step ; rbu state } {oal} +do_test 4.7 { rbu step ; rbu state } {move} +do_execsql_test -db cons 4.8 { + SELECT * FROM t1 +} {} +do_test 4.9 { rbu step ; rbu state } {checkpoint} +do_test 4.10 { + catchsql { SELECT * FROM t1 } cons +} {1 {database is locked}} +do_test 4.11 { rbu step ; rbu state } {checkpoint} +do_test 4.11 { rbu step ; rbu state } {done} +rbu close + +do_test 4.12 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.13 { + sqlite3rbu rbu file:test.db?rbu_exclusive_checkpoint=1 rbu2.db + rbu step + rbu state +} {oal} + +do_test 4.14 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.15 { rbu step ; rbu state } {oal} +do_test 4.16 { rbu step ; rbu state } {move} + +do_test 4.17 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one}} + +do_test 4.18 { rbu step ; rbu state } {checkpoint} +do_test 4.19 { + catchsql { SELECT * FROM t1 } cons +} {1 {database is locked}} +do_test 4.20 { rbu step ; rbu state } {checkpoint} +do_test 4.21 { rbu step ; rbu state } {done} +rbu close + +do_test 4.22 { + catchsql { SELECT * FROM t1 } cons +} {0 {1 one 2 two}} + +cons close + +finish_test diff --git a/ext/rbu/rbuexpr.test b/ext/rbu/rbuexpr.test new file mode 100644 index 0000000..4635d9f --- /dev/null +++ b/ext/rbu/rbuexpr.test @@ -0,0 +1,88 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuexpr + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b, c PRIMARY KEY); + CREATE INDEX i1 ON t1(a, null, b+1); + CREATE INDEX i2 ON t1(a+1, b+1, c+1); + + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + INSERT INTO t1 VALUES(10, 11, 12); + + PRAGMA integrity_check; +} {ok} + +forcedelete rbu.db +sqlite3 db2 rbu.db +do_execsql_test -db db2 1.1 { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(13, 14, 15, 0); + INSERT INTO data_t1 VALUES(NULL, NULL, 6, 1); + INSERT INTO data_t1 VALUES(NULL, 'three', 3, '.x.'); +} +db2 close +db close + +do_test 1.2 { + run_rbu test.db rbu.db +} {SQLITE_DONE} + +sqlite3 db test.db + +do_execsql_test 1.3 { + SELECT * FROM t1 WHERE a=4; +} + +integrity_check 1.4 + +#------------------------------------------------------------------------- +# +reset_db +do_execsql_test 2.0 { + CREATE TABLE t1(c1, c2, c3, i INTEGER PRIMARY KEY); + INSERT INTO t1 VALUES('one', 'one', 'one', 1); + INSERT INTO t1 VALUES('two', 'two', 'two', 2); + INSERT INTO t1 VALUES('three', 'three', 'three', 3); + INSERT INTO t1 VALUES('four', 'four', 'four', 4); + + CREATE INDEX i1 ON t1( substr(c1, 1, 2) ); + CREATE INDEX i2 ON t1( c1 || c2 || c3 ); + CREATE INDEX i3 ON t1( length(c1) + length(c2) - 1, c3||i ); +} + +forcedelete rbu.db +sqlite3 db2 rbu.db +do_execsql_test -db db2 2.1 { + CREATE TABLE data_t1(c1, c2, c3, i, rbu_control); + INSERT INTO data_t1 VALUES(NULL, NULL, NULL, 2, 1); + INSERT INTO data_t1 VALUES('thirty', NULL, NULL, 3, 'xx..'); + INSERT INTO data_t1 VALUES('five', 'five', 'five', 5, 0); +} +db2 close + +db close + +do_test 2.2 { + run_rbu test.db rbu.db +} {SQLITE_DONE} + +sqlite3 db test.db +integrity_check 2.3 + +finish_test + diff --git a/ext/rbu/rbufault.test b/ext/rbu/rbufault.test new file mode 100644 index 0000000..d35c389 --- /dev/null +++ b/ext/rbu/rbufault.test @@ -0,0 +1,234 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/malloc_common.tcl +set ::testprefix rbufault + +proc copy_if_exists {src target} { + if {[file exists $src]} { + forcecopy $src $target + } else { + forcedelete $target + } +} + +foreach {tn2 setup sql expect} { + 1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX t1cb ON t1(c, b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(3, 'three', NULL, '.x.'); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + } { + SELECT * FROM t1 + } {1 1 1 3 three 3 4 4 4} + + 2 { + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX t2cb ON t2(c, b); + INSERT INTO t2 VALUES('a', 'a', 'a'); + INSERT INTO t2 VALUES('b', 'b', 'b'); + INSERT INTO t2 VALUES('c', 'c', 'c'); + + CREATE TABLE rbu.data_t2(a, b, c, rbu_control); + INSERT INTO data_t2 VALUES('b', NULL, NULL, 1); + INSERT INTO data_t2 VALUES('c', 'see', NULL, '.x.'); + INSERT INTO data_t2 VALUES('d', 'd', 'd', 0); + } { + SELECT * FROM t2 + } {a a a c see c d d d} + + 3 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE INDEX t1cb ON t1(c, b); + CREATE INDEX t2cb ON t2(c, b); + + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + CREATE TABLE rbu.data_t2(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t2 VALUES(4, 5, 6, 0); + } { + SELECT * FROM t1 UNION ALL SELECT * FROM t2 + } {1 2 3 4 5 6} + + 4 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES('A', 'B', 'C'); + INSERT INTO t1 VALUES('D', 'E', 'F'); + + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES('D', NULL, NULL, 1); + INSERT INTO data_t1 VALUES('A', 'Z', NULL, '.x.'); + INSERT INTO data_t1 VALUES('G', 'H', 'I', 0); + } { + SELECT * FROM t1 ORDER BY a; + } {A Z C G H I} + + 5 { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1c ON t1(c, b); + + CREATE TABLE rbu.data_t1(a, b, c, rbu_rowid, rbu_control); + INSERT INTO data_t1 VALUES('a', 'b', 'c', 1, 0); + INSERT INTO data_t1 VALUES('d', 'e', 'f', '2', 0); + } { + SELECT * FROM t1 ORDER BY a; + } {a b c d e f} + +} { + catch {db close} + forcedelete rbu.db test.db + sqlite3 db test.db + execsql { + PRAGMA encoding = utf16; + ATTACH 'rbu.db' AS rbu; + } + execsql $setup + db close + + forcecopy test.db test.db.bak + forcecopy rbu.db rbu.db.bak + + foreach {tn f reslist} { + 1 oom-tra* { + {0 SQLITE_DONE} + {1 {SQLITE_NOMEM - out of memory}} + {1 SQLITE_NOMEM} + {1 SQLITE_IOERR_NOMEM} + {1 {SQLITE_NOMEM - unable to open a temporary database file for storing temporary tables}} + } + + 2 ioerr-* { + {0 SQLITE_DONE} + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + {1 SQLITE_IOERR_WRITE} + {1 SQLITE_IOERR_READ} + {1 SQLITE_IOERR_FSYNC} + {1 {SQLITE_ERROR - SQL logic error}} + {1 {SQLITE_ERROR - unable to open database: rbu.db}} + {1 {SQLITE_IOERR - unable to open database: rbu.db}} + } + + 3 shmerr-* { + {0 SQLITE_DONE} + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + } + } { + + catch {db close} + sqlite3_shutdown + set lookaside_config [sqlite3_config_lookaside 0 0] + sqlite3_initialize + autoinstall_test_functions + + do_faultsim_test 2.$tn2 -faults $::f -prep { + catch { db close } + forcedelete test.db-journal test.db-wal rbu.db-journal rbu.db-wal + forcecopy test.db.bak test.db + forcecopy rbu.db.bak rbu.db + } -body { + sqlite3rbu rbu test.db rbu.db + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } -test { + faultsim_test_result {*}$::reslist + if {$testrc==0} { + sqlite3 db test.db + faultsim_integrity_check + set res [db eval $::sql] + if {$res != [list {*}$::expect]} { + puts "" + puts "res: $res" + puts "exp: $expect" + error "data not as expected!" + } + } + } + + catch {db close} + sqlite3_shutdown + sqlite3_config_lookaside {*}$lookaside_config + sqlite3_initialize + autoinstall_test_functions + + + for {set iStep 0} {$iStep<=21} {incr iStep} { + + forcedelete test.db-journal test.db-wal rbu.db-journal rbu.db-wal + + copy_if_exists test.db.bak test.db + copy_if_exists rbu.db.bak rbu.db + + sqlite3rbu rbu test.db rbu.db + for {set x 0} {$x < $::iStep} {incr x} { rbu step } + rbu close + +# sqlite3 x rbu.db ; puts "XYZ [x eval { SELECT * FROM rbu_state } ]" ; x close + + copy_if_exists test.db test.db.bak.2 + copy_if_exists test.db-wal test.db.bak.2-wal + copy_if_exists test.db-oal test.db.bak.2-oal + copy_if_exists rbu.db rbu.db.bak.2 + + do_faultsim_test 3.$tn.$iStep -faults $::f -prep { + catch { db close } + forcedelete test.db-journal test.db-wal rbu.db-journal rbu.db-wal + copy_if_exists test.db.bak.2 test.db + copy_if_exists test.db.bak.2-wal test.db-wal + copy_if_exists test.db.bak.2-oal test.db-oal + copy_if_exists rbu.db.bak.2 rbu.db + } -body { + sqlite3rbu rbu test.db rbu.db + rbu step + rbu close + } -test { + + if {$testresult=="SQLITE_OK"} {set testresult "SQLITE_DONE"} + faultsim_test_result {*}$::reslist + + if {$testrc==0} { + # No error occurred. If the RBU has not already been fully applied, + # apply the rest of it now. Then ensure that the final state of the + # target db is as expected. And that "PRAGMA integrity_check" + # passes. + sqlite3rbu rbu test.db rbu.db + while {[rbu step] == "SQLITE_OK"} {} + rbu close + + sqlite3 db test.db + faultsim_integrity_check + + set res [db eval $::sql] + if {$res != [list {*}$::expect]} { + puts "" + puts "res: $res" + puts "exp: $::expect" + error "data not as expected!" + } + } + } + } + } +} + +finish_test diff --git a/ext/rbu/rbufault2.test b/ext/rbu/rbufault2.test new file mode 100644 index 0000000..fafe0ed --- /dev/null +++ b/ext/rbu/rbufault2.test @@ -0,0 +1,64 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/malloc_common.tcl +set ::testprefix rbufault2 + +forcedelete rbu.db +do_execsql_test 1.0 { + CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y)); + INSERT INTO target VALUES(1, 2, 3); + INSERT INTO target VALUES(4, 5, 6); + + ATTACH 'rbu.db' AS rbu; + CREATE TABLE rbu.data_target(x, y, z, rbu_control); + INSERT INTO data_target VALUES(7, 8, 9, 0); + INSERT INTO data_target VALUES(1, 11, 12, 0); + DETACH rbu; +} +db close + +forcecopy test.db test.db-bak +forcecopy rbu.db rbu.db-bak + +do_faultsim_test 1 -faults oom* -prep { + forcecopy test.db-bak test.db + forcecopy rbu.db-bak rbu.db + forcedelete test.db-oal test.db-wal rbu.db-journal + sqlite3rbu rbu test.db rbu.db +} -body { + while {[rbu step]=="SQLITE_OK"} { } + rbu close +} -test { + faultsim_test_result \ + {1 {SQLITE_CONSTRAINT - UNIQUE constraint failed: target.x}} \ + {1 SQLITE_CONSTRAINT} \ + {1 SQLITE_NOMEM} \ + {1 {SQLITE_NOMEM - unable to open a temporary database file for storing temporary tables}} \ + {1 {SQLITE_NOMEM - out of memory}} +} + + +sqlite3rbu_create_vfs -default rbu "" +sqlite3 db test.db +set ::vfsname [file_control_vfsname db] +do_faultsim_test 2 -faults oom* -prep { +} -body { + file_control_vfsname db +} +db close +sqlite3rbu_destroy_vfs rbu + + +finish_test diff --git a/ext/rbu/rbufault3.test b/ext/rbu/rbufault3.test new file mode 100644 index 0000000..543e237 --- /dev/null +++ b/ext/rbu/rbufault3.test @@ -0,0 +1,96 @@ +# 2016 April 20 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains fault injection tests for RBU vacuum operations. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/malloc_common.tcl +set ::testprefix rbufault3 + +foreach {fault errlist} { + oom-* { + {1 SQLITE_NOMEM} + {1 SQLITE_IOERR_NOMEM} + {1 {SQLITE_NOMEM - out of memory}} + } + + ioerr-* { + {1 {SQLITE_IOERR - disk I/O error}} + {1 SQLITE_IOERR} + {1 SQLITE_IOERR_WRITE} + {1 SQLITE_IOERR_FSYNC} + {1 SQLITE_IOERR_READ} + {1 {SQLITE_IOERR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - unable to open database: test.db2}} + {1 {SQLITE_ERROR - SQL logic error}} + } + + cantopen* { + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database: test.db2}} + {1 {SQLITE_CANTOPEN - unable to open database file}} + {1 SQLITE_CANTOPEN} + } + +} { + + reset_db + do_execsql_test 0 { + CREATE TABLE target(x UNIQUE, y, z, PRIMARY KEY(y)); + INSERT INTO target VALUES(1, 2, 3); + INSERT INTO target VALUES(4, 5, 6); + INSERT INTO target VALUES(7, 8, 9); + CREATE INDEX i1 ON target(z); + } + faultsim_save_and_close + + do_faultsim_test 1 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_DONE} {*}$::errlist] + } + + do_faultsim_test 2 -faults $fault -prep { + faultsim_restore_and_reopen + forcedelete test.db2 + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + faultsim_save_and_close + + do_faultsim_test 3 -faults $fault -prep { + faultsim_restore_and_reopen + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + rbu step + rbu close + } -test { + eval [list faultsim_test_result {0 SQLITE_OK} {*}$::errlist] + } +} + +finish_test diff --git a/ext/rbu/rbufault4.test b/ext/rbu/rbufault4.test new file mode 100644 index 0000000..b956124 --- /dev/null +++ b/ext/rbu/rbufault4.test @@ -0,0 +1,66 @@ +# 2014 October 22 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +source $testdir/malloc_common.tcl +set ::testprefix rbufault4 + +for {set tn 1} {1} {incr tn} { + reset_db + do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + } + db close + + forcedelete test.db2 + sqlite3rbu_vacuum rbu test.db test.db2 + for {set i 0} {$i < $tn} {incr i} { rbu step } + set rc [rbu close] + if {$rc!="SQLITE_OK"} { + if {$rc!="SQLITE_DONE"} {error $rc} + break + } + faultsim_save + + do_faultsim_test $tn -faults oom-t* -prep { + faultsim_restore + } -body { + sqlite3rbu_vacuum rbu test.db test.db2 + while 1 { + set rc [rbu step] + if {$rc=="SQLITE_DONE"} break + if {$rc!="SQLITE_OK"} { error $rc } + } + } -test { + catch {rbu close} + faultsim_test_result {0 {}} {1 SQLITE_NOMEM} {1 SQLITE_IOERR_NOMEM} + + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]=="SQLITE_OK"} {} + set trc [rbu close] + if {$trc!="SQLITE_DONE"} { error "Got $trc instead of SQLITE_DONE!" } + + sqlite3 db test.db + set rc [db one {PRAGMA integrity_check}] + if {$rc!="ok"} { error "Got $rc instead of ok!" } + db close + } +} + + + +finish_test diff --git a/ext/rbu/rbufts.test b/ext/rbu/rbufts.test new file mode 100644 index 0000000..b46c32f --- /dev/null +++ b/ext/rbu/rbufts.test @@ -0,0 +1,131 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that RBU works with FTS tables. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbufts + +ifcapable !fts3 { + finish_test + return +} + +proc step_rbu {target rbu} { + while 1 { + sqlite3rbu rbu $target $rbu + set rc [rbu step] + rbu close + if {$rc != "SQLITE_OK"} break + } + set rc +} + +proc apply_rbu_update {target sql} { + forcedelete rbu.db + sqlite3 dbrbu rbu.db + execsql $sql dbrbu + dbrbu close + + step_rbu $target rbu.db +} + +do_execsql_test 1.1.0 { + CREATE TABLE t1(i INTEGER PRIMARY KEY, a, b); + CREATE VIRTUAL TABLE xx USING fts4(content=t1, a, b); + INSERT INTO t1(rowid, a, b) VALUES(10, 'a b c', 'c b a'); + INSERT INTO t1(rowid, a, b) VALUES(20, 'a b c', 'd e f'); + INSERT INTO t1(rowid, a, b) VALUES(30, 'd e f', 'a b c'); + INSERT INTO t1(rowid, a, b) VALUES(40, 'd e f', 'd e f'); +} + +do_execsql_test 1.1.1 { + INSERT INTO xx(xx) VALUES('rebuild'); + INSERT INTO xx(xx) VALUES('integrity-check'); +} + +do_test 1.1.2 { + apply_rbu_update test.db { + CREATE TABLE data_t1(i, a, b, rbu_control); + INSERT INTO data_t1 VALUES(20, NULL, NULL, 1); -- delete + INSERT INTO data_t1 VALUES(30, 'x y z', NULL, '.x.'); -- update + INSERT INTO data_t1 VALUES(50, '1 2 3', 'x y z', 0); -- insert + + CREATE VIEW data0_xx AS + SELECT i AS rbu_rowid, a, b, + CASE WHEN rbu_control IN (0, 1) + THEN rbu_control ELSE substr(rbu_control, 2) END AS rbu_control + FROM data_t1; + } +} {SQLITE_DONE} + +do_execsql_test 1.1.3 { + INSERT INTO xx(xx) VALUES('integrity-check'); +} + +reset_db +do_execsql_test 1.2.1 { + CREATE TABLE ccc(addr, text); + CREATE VIRTUAL TABLE ccc_fts USING fts4(addr, text, content=ccc); + INSERT INTO ccc VALUES('a b c', 'd e f'); + INSERT INTO ccc VALUES('a b c', 'd e f'); + INSERT INTO ccc_fts(ccc_fts) VALUES('rebuild'); + INSERT INTO ccc_fts(ccc_fts) VALUES('integrity-check'); +} + +do_test 1.2.2 { + apply_rbu_update test.db { + CREATE TABLE data_ccc(addr, text, rbu_rowid, rbu_control); + CREATE VIEW data0_ccc_fts AS SELECT * FROM data_ccc; + INSERT INTO data_ccc VALUES(NULL, NULL, 1, 1); + INSERT INTO data_ccc VALUES('x y z', NULL, 2, 'x.'); + INSERT INTO data_ccc VALUES('y y y', '1 1 1', 3, 0); + } +} {SQLITE_DONE} + +do_execsql_test 1.2.3 { + INSERT INTO ccc_fts(ccc_fts) VALUES('integrity-check'); +} +do_execsql_test 1.2.4 { + SELECT rowid, * FROM ccc_fts; +} {2 {x y z} {d e f} 3 {y y y} {1 1 1}} + +#------------------------------------------------------------------------- +# Test the outcome of attempting to delete or update a row within a +# contentless FTS table using RBU. An error. +# +reset_db +do_execsql_test 3.1 { + CREATE VIRTUAL TABLE ft USING fts4(x, content=); + INSERT INTO ft(rowid, x) VALUES(1, '1 2 3'); + INSERT INTO ft(rowid, x) VALUES(2, '4 5 6'); +} + +do_test 3.2 { + list [catch { apply_rbu_update test.db { + CREATE TABLE data_ft(x, rbu_rowid, rbu_control); + INSERT INTO data_ft VALUES(NULL, 2, 1); + } } msg] $msg] +} {1 {SQLITE_ERROR - SQL logic error]}} + +do_test 3.3 { + list [catch { apply_rbu_update test.db { + CREATE TABLE data_ft(x, rbu_rowid, rbu_control); + INSERT INTO data_ft VALUES('7 8 9', 1, 'x'); + } } msg] $msg] +} {1 {SQLITE_ERROR - SQL logic error]}} + + + +finish_test diff --git a/ext/rbu/rbumisc.test b/ext/rbu/rbumisc.test new file mode 100644 index 0000000..1629314 --- /dev/null +++ b/ext/rbu/rbumisc.test @@ -0,0 +1,176 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbumisc + +proc populate_rbu_db {} { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_x1(a, b, c, rbu_control); + INSERT INTO data_x1 VALUES(1, 1, 1, 0); + INSERT INTO data_x1 VALUES(2, 2, 2, 0); + + CREATE TABLE dat(a, b, c, rbu_control); + CREATE TABLE "data x1"(a, b, c, rbu_control); + CREATE TABLE datax1(a, b, c, rbu_control); + CREATE TABLE data_(a, b, c, rbu_control); + + INSERT INTO "data x1" VALUES(3, 3, 3, 0); + INSERT INTO datax1 VALUES(3, 3, 3, 0); + INSERT INTO data_ VALUES(3, 3, 3, 0); + INSERT INTO dat VALUES(3, 3, 3, 0); + } + rbu close +} + +#------------------------------------------------------------------------- +# Ensure that RBU is not confused by oddly named tables in an RBU +# database. +# +do_execsql_test 1.0 { + CREATE TABLE x1(a, b, c INTEGER PRIMARY KEY); +} +do_test 1.1 { + populate_rbu_db +} {} + +do_test 1.2 { + step_rbu test.db rbu.db + db eval { SELECT * FROM x1 } +} {1 1 1 2 2 2} + +do_test 1.3 { + db eval { DELETE FROM x1 } + sqlite3 rbu rbu.db + rbu eval { DELETE FROM rbu_state } + rbu close + step_rbu test.db rbu.db + db eval { SELECT * FROM x1 } +} {1 1 1 2 2 2} + +do_test 1.4 { + db eval { DELETE FROM x1 } + populate_rbu_db + + sqlite3rbu rbu test.db rbu.db + rbu step + rbu step + rbu close + + forcecopy test.db-oal test.db-wal + sqlite3rbu rbu test.db rbu.db + rbu step + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot update wal mode database}} + +#------------------------------------------------------------------------- +# Test the effect of a wal file appearing after the target database has +# been opened, but before it has been locked. +# +catch { db close } +testvfs tvfs -default 1 + +for {set N 1} {$N < 10} {incr N} { + reset_db + populate_rbu_db + do_execsql_test 2.$N.0 { + CREATE TABLE x1(a, b, c INTEGER PRIMARY KEY); + } + + set nAccessCnt 0 + do_test 2.$N.1 { + sqlite3rbu rbu test.db rbu.db + rbu step + rbu step + rbu close + } {SQLITE_OK} + + tvfs script xAccess + tvfs filter xAccess + set nAccessCnt 0 + proc xAccess {method file args} { + global nAccessCnt + if {[file tail $file]=="test.db-wal"} { + incr nAccessCnt -1 + if {$nAccessCnt==0} { + set fd [open test.db-wal w] + puts -nonewline $fd [string repeat 0 2000] + close $fd + } + } + return SQLITE_OK + } + + foreach r { + {1 {SQLITE_ERROR - cannot update wal mode database}} + {0 SQLITE_OK} + {1 {SQLITE_CANTOPEN - unable to open database file}} + } { + set RES($r) 1 + } + do_test 2.$N.2 { + set ::nAccessCnt $N + set res [list [catch { + sqlite3rbu rbu test.db rbu.db + rbu step + rbu close + } msg ] $msg] + set RES($res) + } {1} + catch {rbu close} +} +catch {db close} +catch {tvfs delete} + +#------------------------------------------------------------------------- +testvfs tvfs -default 1 +reset_db +populate_rbu_db +do_execsql_test 3.0 { + CREATE TABLE x1(a, b, c INTEGER PRIMARY KEY); +} + +tvfs script xFileControl +tvfs filter xFileControl + +proc xFileControl {method file verb args} { + if {$verb=="ZIPVFS" && [info exists ::zipvfs_filecontrol]} { + return $::zipvfs_filecontrol + } + return "SQLITE_NOTFOUND" +} + +breakpoint +foreach {tn ret err} { + 1 SQLITE_OK 0 + 2 SQLITE_ERROR 1 + 3 SQLITE_NOTFOUND 0 + 4 SQLITE_OMIT 1 +} { + set ::zipvfs_filecontrol $ret + do_test 3.$tn.1 { + catch { + sqlite3rbu rbu test.db rbu.db + rbu step + rbu close + } + } $err +} +catch {db close} +catch {tvfs delete} + +#------------------------------------------------------------------------- + +finish_test diff --git a/ext/rbu/rbumulti.test b/ext/rbu/rbumulti.test new file mode 100644 index 0000000..c24da35 --- /dev/null +++ b/ext/rbu/rbumulti.test @@ -0,0 +1,175 @@ +# 2018 January 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# +# This file contains tests of multiple RBU operations running +# concurrently within the same process. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbumulti + +db close + +autoinstall_test_functions + +proc build_db {db} { + $db eval { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t1(c); + + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<500 ) + INSERT INTO t1 + SELECT randomblob(10), randomblob(100), randomblob(100) FROM s; + } +} + +proc build_rbu {db} { + $db eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + WITH s(i) AS ( SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 ) + INSERT INTO data_t1 + SELECT randomblob(10), randomblob(100), randomblob(100), 0 FROM s; + } +} + +proc step_rbu2 {bOpenClose openr1 openr2} { + + forcedelete teststate.db1 + forcedelete teststate.db2 + + if {$bOpenClose!=0 && $bOpenClose!=1} { error $bOpenClose } + if {$bOpenClose==0} { + eval $openr1 + eval $openr2 + } + + set b1 0 + set b2 0 + + while {$b1==0 || $b2==0} { + if {$bOpenClose==1} { + if {$b1==0} { eval $openr1 teststate.db1 } + if {$b2==0} { eval $openr2 teststate.db2 } + } + if {$b1==0} { + set rc1 [r1 step] + if {$rc1 != "SQLITE_OK"} { set b1 1 } + } + if {$b2==0} { + set rc2 [r2 step] + if {$rc2 != "SQLITE_OK"} { set b2 1 } + } + if {$bOpenClose==1} { + if {$b1==0} { r1 close } + if {$b2==0} { r2 close } + } + } + + set rc1 [r1 close] + set rc2 [r2 close] + + list $rc1 $rc2 +} + + +for {set i 0} {$i<=3} {incr i} { + + if {$i & 0x01} { + sqlite3rbu_create_vfs -default myrbu "" + } + set bOpenClose [expr $i>>1] + + forcedelete test.db + forcedelete test.db2 + forcedelete rbu.db + forcedelete rbu.db2 + + do_test 1.$i.0 { + sqlite3 db test.db + sqlite3 db2 test.db2 + build_db db + build_db db2 + + sqlite3 rbu1 rbu.db + sqlite3 rbu2 rbu.db2 + + build_rbu rbu1 + build_rbu rbu2 + + rbu1 close + rbu2 close + } {} + + set m1 [db eval {SELECT md5sum(a, b, c) FROM t1}] + set m2 [db2 eval {SELECT md5sum(a, b, c) FROM t1}] + + do_test 1.$i.1 { + step_rbu2 $bOpenClose { + sqlite3rbu r1 test.db rbu.db + } { + sqlite3rbu r2 test.db2 rbu.db2 + } + } {SQLITE_DONE SQLITE_DONE} + + do_execsql_test -db db 1.$i.2.1 { PRAGMA integrity_check } ok + do_execsql_test -db db2 1.$i.2.2 { PRAGMA integrity_check } ok + + do_execsql_test -db db 1.$i.3.1 { SELECT md5sum(a, b, c)==$m1 FROM t1 } 0 + do_execsql_test -db db2 1.$i.3.2 { SELECT md5sum(a, b, c)==$m2 FROM t1 } 0 + + catch { db close } + catch { db2 close } + #----------------------------------------------------------------------- + forcedelete test.db2 + forcedelete test.db + forcedelete rbu.db2 + + do_test 1.$i.4 { + sqlite3 db test.db + sqlite3 db2 test.db2 + build_db db + build_db db2 + sqlite3 rbu2 rbu.db2 + build_rbu rbu2 + rbu2 close + } {} + + set m1 [db eval {SELECT md5sum(a, b, c) FROM t1}] + set m2 [db2 eval {SELECT md5sum(a, b, c) FROM t1}] + + do_test 1.$i.5 { + step_rbu2 $bOpenClose { + sqlite3rbu_vacuum r1 test.db + } { + sqlite3rbu r2 test.db2 rbu.db2 + } + } {SQLITE_DONE SQLITE_DONE} + + do_execsql_test -db db 1.$i.6.1 { SELECT md5sum(a, b, c)==$m1 FROM t1 } 1 + do_execsql_test -db db2 1.$i.6.2 { SELECT md5sum(a, b, c)==$m2 FROM t1 } 0 + + do_execsql_test -db db 1.$i.7.1 { PRAGMA integrity_check } ok + do_execsql_test -db db2 1.$i.7.2 { PRAGMA integrity_check } ok + + catch { db close } + catch { db2 close } + if {$i & 0x01} { + sqlite3rbu_destroy_vfs myrbu + } + +} + + +finish_test diff --git a/ext/rbu/rbupartial.test b/ext/rbu/rbupartial.test new file mode 100644 index 0000000..0c8191d --- /dev/null +++ b/ext/rbu/rbupartial.test @@ -0,0 +1,98 @@ +# 2019 April 11 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbupartial + +db close + +foreach {tn without_rowid a b c d} { + 1 "" a b c d + 2 "WITHOUT ROWID" aaa bbb ccc ddd + 3 "WITHOUT ROWID" "\"hello\"" {"one'two"} {[c]} ddd + 4 "WITHOUT ROWID" {`a b`} {"one'two"} {[c c c]} ddd + 5 "" a b c {"d""d"} + 6 "" {'one''two'} b {"c""c"} {"d""d"} +} { + eval [string map [list \ + %WITHOUT_ROWID% $without_rowid %A% $a %B% $b %C% $c %D% $d + ] { + reset_db + do_execsql_test $tn.1.0 { + CREATE TABLE t1(%A% PRIMARY KEY, %B%, %C%, %D%) %WITHOUT_ROWID% ; + CREATE INDEX i1b ON t1(%B%); + CREATE INDEX i1b2 ON t1(%B%) WHERE %C%<5; + CREATE INDEX i1b3 ON t1(%B%) WHERE %C%>=5; + + CREATE INDEX i1c ON t1(%C%); + CREATE INDEX i1c2 ON t1(%C%) WHERE %C% IS NULL; + CREATE INDEX i1c3 ON t1(%C%) WHERE %C% IS NOT NULL; + + CREATE INDEX i1c4 ON t1(%C%) WHERE %D% < 'd'; + CREATE INDEX i1c5 ON t1( + %C% -- for (c = ... expressions + ) WHERE %D% < 'd'; + CREATE INDEX i1c6 ON t1( + %C% /* Again, for (c=... expr */, %D% + ) WHERE %D% < 'd'; + + CREATE INDEX i1c7 ON t1( + %C% /* As before, for (c=... "expr */) WHERE %D% < 'd'; + } + + do_execsql_test $tn.1.1 { + INSERT INTO t1 VALUES(0, NULL, NULL, 'a'); + INSERT INTO t1 VALUES(1, 2, 3, 'b'); + INSERT INTO t1 VALUES(4, 5, 6, 'c'); + INSERT INTO t1 VALUES(7, 8, 9, 'd'); + } + + forcedelete rbu.db + do_test $tn.1.2 { + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(%A%, %B%, %C%, %D%, rbu_control); + + INSERT INTO data_t1 VALUES(10, 11, 12, 'e', 0); + INSERT INTO data_t1 VALUES(13, 14, NULL, 'f', 0); + + INSERT INTO data_t1 VALUES(0, NULL, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(4, NULL, NULL, NULL, 1); + + INSERT INTO data_t1 VALUES(7, NULL, 4, NULL, '..x.'); + INSERT INTO data_t1 VALUES(1, 10, NULL, NULL, '.xx.'); + } + rbu close + } {} + + do_test $tn.1.3 { + run_rbu test.db rbu.db + execsql { PRAGMA integrity_check } + } {ok} + + do_execsql_test $tn.1.4 { + SELECT * FROM t1 ORDER BY %A%; + } { + 1 10 {} b 7 8 4 d 10 11 12 e 13 14 {} f + } + + set step 0 + do_rbu_vacuum_test $tn.1.5 0 + + do_test $tn.1.6 { + execsql { PRAGMA integrity_check } + } {ok} + }] +} + +finish_test diff --git a/ext/rbu/rbupass.test b/ext/rbu/rbupass.test new file mode 100644 index 0000000..8b921cb --- /dev/null +++ b/ext/rbu/rbupass.test @@ -0,0 +1,81 @@ +# 2023 January 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbupass + +if {[info commands register_demovfs]==""} { + finish_test + return +} + +db close + +register_demovfs +sqlite3rbu_create_vfs myvfs demo + +sqlite3 db file:test.db?vfs=myvfs -uri 1 +do_execsql_test 1.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + SELECT * FROM t1; +} {1 2} + +if {[permutation]!="inmemory_journal"} { + do_execsql_test 1.1 { + PRAGMA journal_mode = wal; + } {delete} +} + +do_execsql_test 1.2 { + SELECT * FROM t1; +} {1 2} + +do_test 1.3 { + forcedelete rbu.db + sqlite3 rbu rbu.db + rbu eval { + CREATE TABLE data_t1(a, b, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 0); + } + rbu close +} {} + +do_test 1.4 { + sqlite3rbu rbu test.db rbu.db +} {rbu} +do_test 1.5 { + rbu step +} {SQLITE_CANTOPEN} +do_test 1.6 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_CANTOPEN - unable to open database file}} + +do_test 1.7 { + sqlite3rbu_vacuum rbu test.db +} {rbu} +do_test 1.8 { + rbu step + catch { rbu close } +} {1} + +do_execsql_test 1.9 { + SELECT * FROM t1; +} {1 2} + +db close +sqlite3rbu_destroy_vfs myvfs +unregister_demovfs +sqlite3_shutdown +finish_test + diff --git a/ext/rbu/rbuprogress.test b/ext/rbu/rbuprogress.test new file mode 100644 index 0000000..0726837 --- /dev/null +++ b/ext/rbu/rbuprogress.test @@ -0,0 +1,451 @@ +# 2016 March 18 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuprogress + + +proc create_db_file {filename sql} { + forcedelete $filename + sqlite3 tmpdb $filename + tmpdb eval $sql + tmpdb close +} + +# Create a simple RBU database. That expects to write to a table: +# +# CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +# +proc create_rbu1 {filename} { + create_db_file $filename { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 3, 0); + INSERT INTO data_t1 VALUES(2, 'two', 'three', 0); + INSERT INTO data_t1 VALUES(3, NULL, 8.2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + return $filename +} + + +do_execsql_test 1.0 { + PRAGMA page_size = 4096; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); +} + +do_test 1.1 { + create_rbu1 rbu.db + sqlite3rbu rbu test.db rbu.db + rbu bp_progress +} {0 0} +do_test 1.2 { rbu step ; rbu bp_progress } {3333 0} +do_test 1.3 { rbu step ; rbu bp_progress } {6666 0} +do_test 1.4 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.5 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.6 { rbu step ; rbu bp_progress } {10000 0} +do_test 1.7 { rbu step ; rbu bp_progress } {10000 5000} +do_test 1.8 { rbu step ; rbu bp_progress } {10000 10000} +do_test 1.9 { rbu step ; rbu bp_progress } {10000 10000} + +do_test 1.10 { + rbu close +} {SQLITE_DONE} + +#------------------------------------------------------------------------- +# +proc do_sp_test {tn bReopen target rbu reslist} { + uplevel [list do_test $tn [subst -nocommands { + if {$bReopen==0} { sqlite3rbu rbu $target $rbu } + set res [list] + while 1 { + if {$bReopen} { sqlite3rbu rbu $target $rbu } + set rc [rbu step] + if {[set rc] != "SQLITE_OK"} { rbu close ; error "error 1" } + lappend res [lindex [rbu bp_progress] 0] + if {[lindex [set res] end]==10000} break + if {$bReopen} { rbu close } + } + if {[set res] != [list $reslist]} { + rbu close + error "1. reslist incorrect (expect=$reslist got=[set res])" + } + + # One step to clean up the temporary tables used to update the only + # target table in the rbu database. And one more to move the *-oal + # file to *-wal. After each of these steps, the progress remains + # at "10000 0". + # + if {[lindex [list $reslist] 0]!=-1} { + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "2. reslist incorrect (expect=10000 0 got=[set res])" + } + } + + rbu step + set res [rbu bp_progress] + if {[set res] != [list 10000 0]} { + rbu close + error "3. reslist incorrect (expect=10000 0 got=[set res])" + } + + # Do the checkpoint. + while {[rbu step]=="SQLITE_OK"} { + foreach {a b} [rbu bp_progress] {} + if {[set a]!=10000 || [set b]<=0 || [set b]>10000} { + rbu close + error "4. reslist incorrect (expect=10000 1..10000 got=[set a] [set b])" + } + } + + set res [rbu bp_progress] + if {[set res] != [list 10000 10000]} { + rbu close + error "5. reslist is incorrect (expect=10000 10000 got=[set res])" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach {bReopen} { 0 1 } { + reset_db + do_test 2.$bReopen.1.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(5, 5, 5, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 2); + } + } {} + do_sp_test 2.$bReopen.1.1 $bReopen test.db rbu.db {5000 10000} + + reset_db + do_test 2.$bReopen.2.0 { + execsql { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + create_rbu1 rbu.db + } {rbu.db} + do_sp_test 2.$bReopen.2.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.3.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 3); + } + } {} + do_sp_test 2.$bReopen.3.1 $bReopen test.db rbu.db {1666 3333 6000 8000 10000} + + reset_db + do_test 2.$bReopen.4.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(2, 4, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.4.1 $bReopen test.db rbu.db {3333 6666 10000} + + reset_db + do_test 2.$bReopen.5.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, NULL, 4, '.xx'); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + } + } {} + do_sp_test 2.$bReopen.5.1 $bReopen test.db rbu.db {10000} + + reset_db + do_test 2.$bReopen.6.0 { + execsql { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(5, NULL, NULL, 1); + } + } {} + do_sp_test 2.$bReopen.6.1 $bReopen test.db rbu.db {-1 -1 -1 -1 -1 10000} +} + +#------------------------------------------------------------------------- +# The following tests verify that the API works when resuming an update +# during the incremental checkpoint stage. +# +proc do_phase2_test {tn bReopen target rbu nStep} { + uplevel [list do_test $tn [subst -nocommands { + + # Build the OAL/WAL file: + sqlite3rbu rbu $target $rbu + while {[lindex [rbu bp_progress] 0]<10000} { + set rc [rbu step] + if {"SQLITE_OK" != [set rc]} { rbu close } + } + + # Clean up the temp tables and move the *-oal file to *-wal. + rbu step + rbu step + + for {set i 0} {[set i] < $nStep} {incr i} { + if {$bReopen} { + rbu close + sqlite3rbu rbu $target $rbu + } + rbu step + set res [rbu bp_progress] + set expect [expr (1 + [set i]) * 10000 / $nStep] + if {[lindex [set res] 1] != [set expect]} { + error "Have [set res], expected 10000 [set expect]" + } + } + + set rc [rbu step] + if {[set rc] != "SQLITE_DONE"} { + error "Have [set rc], expected SQLITE_DONE" + } + + rbu close + }] {SQLITE_DONE}] +} + +foreach bReopen {0 1} { + do_test 3.$bReopen.1.0 { + reset_db + execsql { + PRAGMA page_size = 4096; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + } + create_db_file rbu.db { + CREATE TABLE data_t1(a, b, rbu_control); + CREATE TABLE data_t2(a, b, rbu_control); + CREATE TABLE data_t3(a, b, rbu_control); + CREATE TABLE data_t4(a, b, rbu_control); + INSERT INTO data_t1 VALUES(1, 2, 0); + INSERT INTO data_t2 VALUES(1, 2, 0); + INSERT INTO data_t3 VALUES(1, 2, 0); + INSERT INTO data_t4 VALUES(1, 2, 0); + + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data_t1', 1); + INSERT INTO rbu_count VALUES('data_t2', 1); + INSERT INTO rbu_count VALUES('data_t3', 1); + INSERT INTO rbu_count VALUES('data_t4', 1); + } + } {} + do_phase2_test 3.$bReopen.1.1 $bReopen test.db rbu.db 5 +} + + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + ipk { CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c) } + wr { CREATE TABLE t1(a INT PRIMARY KEY, b, c) WITHOUT ROWID } + pk { CREATE TABLE t1(a INT PRIMARY KEY, b, c) } + } { + + foreach {tn2 rbusql r1 r3} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {1666 3333 5000 6666 8333 10000} + + 2 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 1); + } + {3333 6666 10000} + {2000 4000 6000 8000 10000} + + 3 { + CREATE TABLE data0_t1(a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {1666 2500 3750 5000 6250 7500 8750 10000} + + } { + + reset_db ; execsql $tbl + do_test 4.$tn.$bReopen.$tn2.0 { + execsql { + CREATE INDEX t1c ON t1(c); + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(ipk) $r1 + set R(wr) $r1 + set R(pk) $r3 + do_sp_test 4.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + +foreach {bReopen} { 0 1 } { + foreach {tn tbl} { + nopk { + CREATE TABLE t1(a, b, c); + CREATE INDEX t1c ON t1(c); + } + vtab { + CREATE VIRTUAL TABLE t1 USING fts5(a, b, c); + } + } { + + if {$tn=="vtab"} { ifcapable !fts5 break } + + foreach {tn2 rbusql r1 r2} { + 1 { + CREATE TABLE data0_t1(a, b, c, rbu_rowid, rbu_control); + INSERT INTO data0_t1 VALUES(15, 15, 15, 4, 0); + INSERT INTO data0_t1 VALUES(20, 20, 20, 5, 0); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 5000 7500 10000} + {5000 10000} + + 2 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(0, 7, 7, 7, 2); + INSERT INTO data0_t1 VALUES(2, 10, 10, 10, 2); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + + 3 { + CREATE TABLE data0_t1(rbu_rowid, a, b, c, rbu_control); + INSERT INTO data0_t1 VALUES(1, NULL, NULL, NULL, 1); + INSERT INTO data0_t1 VALUES(2, NULL, NULL, 7, '..x'); + CREATE TABLE rbu_count(tbl, cnt); + INSERT INTO rbu_count VALUES('data0_t1', 2); + } + {2500 4000 6000 8000 10000} + {5000 10000} + } { + + reset_db ; execsql $tbl + do_test 5.$tn.$bReopen.$tn2.0 { + execsql { + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(5, 5, 5); + INSERT INTO t1 VALUES(10, 10, 10); + } + create_db_file rbu.db $rbusql + } {} + + set R(nopk) $r1 + set R(vtab) $r2 + do_sp_test 5.$tn.$bReopen.$tn2.1 $bReopen test.db rbu.db $R($tn) + } + } +} + +#------------------------------------------------------------------------- +# Test that sqlite3_bp_progress() works with an RBU vacuum if there +# is an rbu_count table in the db being vacuumed. +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a, b, c); + CREATE INDEX i1 ON t1(a); + CREATE INDEX i2 ON t1(b); + WITH s(i) AS ( + SELECT 1 UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO t1 SELECT i, i, i FROM s; + CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; + INSERT INTO rbu_count VALUES('t1', (SELECT count(*) FROM t1)); + INSERT INTO rbu_count VALUES('rbu_count', 2); +} + +forcedelete state.db +do_test 6.1 { + set maxA 0 + set maxB 0 + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} { + foreach {a b} [rbu bp_progress] { + if {$a > $maxA} { set maxA $a } + if {$b > $maxB} { set maxB $b } + } + } + list [rbu close] $maxA $maxB +} {SQLITE_DONE 10000 10000} + + +finish_test diff --git a/ext/rbu/rburename.test b/ext/rbu/rburename.test new file mode 100644 index 0000000..e2bacae --- /dev/null +++ b/ext/rbu/rburename.test @@ -0,0 +1,55 @@ +# 2022 November 07 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rburename + + +do_execsql_test 1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); +} + +forcedelete test.db-vacuum + +proc my_rename {old new} { + lappend ::my_rename_calls [list [file tail $old] [file tail $new]] + file rename $old $new +} + +do_test 1.1 { + sqlite3rbu_vacuum rbu test.db + rbu rename_handler my_rename + while {[rbu step]=="SQLITE_OK"} {} + rbu close +} SQLITE_DONE + +do_test 1.2 { + set ::my_rename_calls +} {{test.db-oal test.db-wal}} + +proc my_rename {old new} { + error "something went wrong" +} + +do_test 1.3 { + sqlite3rbu_vacuum rbu test.db + rbu rename_handler my_rename + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 SQLITE_IOERR} + +finish_test diff --git a/ext/rbu/rburesume.test b/ext/rbu/rburesume.test new file mode 100644 index 0000000..cdd8908 --- /dev/null +++ b/ext/rbu/rburesume.test @@ -0,0 +1,256 @@ +# 2017 January 13 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow +# +# This file contains tests for resumption of RBU operations in the +# case where the previous RBU process crashed. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rburesume + +forcedelete test.db-shm test.db-oal +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX t1a ON t1(a); + CREATE INDEX t1b ON t1(b); + CREATE INDEX t1c ON t1(c); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<50 + ) + INSERT INTO t1 SELECT randomblob(50), randomblob(75), randomblob(100) FROM s; +} +db_save_and_close + +do_test 1.1 { + list [file exists test.db] \ + [file exists test.db-wal] \ + [file exists test.db-shm] \ + [file exists test.db-oal] +} {1 0 0 0} + +# Each iteration of the following loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Closes the RBU vacuum handled opened in (2). +# 4. Opens a second RBU vacuum handle, resumes and completes the vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + rbu close + if {$rc == "SQLITE_DONE"} break + + do_test 1.2.$n.1 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.2.$n.2 { + sqlite3 db2 test.db + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (2) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.3.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.3.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU vacuum and steps it 10 times. Then closes it. +# 2. Opens an RBU vacuum and steps it $n times. +# 3. Takes a copy of all database files and the state db. +# 4. Opens a second RBU vacuum handle on the copy, resumes and completes the +# vacuum op. +# +# The loop runs until $n is large enough that step (3) vacuums the entire +# database. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<10} {incr i} { + rbu step + } + rbu close + + sqlite3rbu_vacuum rbu test.db state.db + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 1.4.$n.1 { + sqlite3rbu_vacuum rbu test.db2 state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 1.4.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {50 ok} + db2 close +} + +forcedelete rbu.db +do_test 2.0 { + sqlite3 db2 rbu.db + db2 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10 + ) + INSERT INTO data_t1 + SELECT randomblob(50), randomblob(75), randomblob(100), 0 FROM s; + } + db2 close +} {} + +# Each iteration of this loop: +# +# 1. Restores the db to the state it was in following test case 1.0 +# 2. Opens an RBU handle to apply the RBU update created in test case 2.0. +# 3. Steps the RBU handle $n times. +# 4. Takes a copy of all database files and the state db. +# 5. Opens a second RBU handle on the copy, resumes and completes the +# RBU op. Checks it worked as expected. +# +# The loop runs until $n is large enough that step (3) applies the entire +# update. +# +for {set n 1} {$n < 5000} {incr n} { + db_restore + forcedelete state.db state.db-shm state.db-oal state.db-wal + sqlite3rbu rbu test.db rbu.db state.db + + for {set i 0} {$i<$n} {incr i} { + set rc [rbu step] + if {$rc == "SQLITE_DONE"} break + } + if {$rc == "SQLITE_DONE"} { + rbu close + break + } + + foreach f {test.db test.db-oal test.db-wal test.db-vacuum} { + set f2 [string map [list test.db test.db2] $f] + if {[file exists $f]} { + forcecopy $f $f2 + } else { + forcedelete $f2 + } + } + forcecopy state.db state.db2 + rbu close + + do_test 2.$n.1 { + sqlite3rbu rbu test.db2 rbu.db state.db2 + while {[rbu step]=="SQLITE_OK"} {} + rbu close + } {SQLITE_DONE} + + do_test 2.$n.2 { + sqlite3 db2 test.db2 + db2 eval { + SELECT count(*) FROM t1; + PRAGMA integrity_check; + } + } {60 ok} + db2 close +} + +finish_test diff --git a/ext/rbu/rbusave.test b/ext/rbu/rbusave.test new file mode 100644 index 0000000..8e97089 --- /dev/null +++ b/ext/rbu/rbusave.test @@ -0,0 +1,102 @@ +# 2015 August 14 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbusave + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + CREATE TABLE t2(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 ON t1(b); + CREATE INDEX i2 ON t2(c, b); + + INSERT INTO t1 VALUES(1, 1, 1); + INSERT INTO t1 VALUES(2, 2, 2); + INSERT INTO t1 VALUES(3, 3, 3); + + INSERT INTO t2 VALUES(1, 1, 1); + INSERT INTO t2 VALUES(2, 2, 2); + INSERT INTO t2 VALUES(3, 3, 3); +} + +do_test 1.1 { + forcedelete test.db2 + sqlite3 db2 test.db2 + db2 eval { + CREATE TABLE data_t1(a, b, c, rbu_control); + INSERT INTO data_t1 VALUES(4, 4, 4, 0); + INSERT INTO data_t1 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t1 VALUES(1, 'one', NULL, '.x.'); + + CREATE TABLE data_t2(a, b, c, rbu_control); + INSERT INTO data_t2 VALUES(4, 4, 4, 0); + INSERT INTO data_t2 VALUES(2, NULL, NULL, 1); + INSERT INTO data_t2 VALUES(1, 'one', NULL, '.x.'); + } +} {} + +proc test_to_bak {} { + foreach f { + test.db test.db-wal test.db-oal test.db-journal + test.db2 test.db2-wal test.db2-oal test.db2-journal + } { + set t [string map {test bak} $f] + forcedelete $t + if {[file exists $f]} { forcecopy $f $t } + } +} + +do_test 1.2 { + test_to_bak + sqlite3rbu rrr bak.db bak.db2 + set nStep 0 + while {[rrr step]=="SQLITE_OK"} {incr nStep} + set res2 [rrr close] +} {SQLITE_DONE} + + +sqlite3rbu rbu test.db test.db2 +set res "SQLITE_OK" +for {set i 1} {$res=="SQLITE_OK"} {incr i} { + set res [rbu step] + + do_test 1.3.$i.1 { + rbu savestate + test_to_bak + sqlite3rbu rrr bak.db bak.db2 + set nRem 0 + while {[rrr step]=="SQLITE_OK"} {incr nRem} + set res2 [rrr close] + } {SQLITE_DONE} + + do_test 1.3.$i.3 { expr $nRem+$i } [expr {$nStep + ($res=="SQLITE_DONE")}] + + do_test 1.3.$i.3 { + sqlite3 bak bak.db + bak eval { + SELECT * FROM t1; + SELECT * FROM t2; + } + } {1 one 1 3 3 3 4 4 4 1 one 1 3 3 3 4 4 4} + + bak close +} + +do_test 1.4 { rbu close } {SQLITE_DONE} + +do_execsql_test 1.5 { + SELECT * FROM t1; + SELECT * FROM t2; +} {1 one 1 3 3 3 4 4 4 1 one 1 3 3 3 4 4 4} + +finish_test diff --git a/ext/rbu/rbusplit.test b/ext/rbu/rbusplit.test new file mode 100644 index 0000000..fb1d723 --- /dev/null +++ b/ext/rbu/rbusplit.test @@ -0,0 +1,93 @@ +# 2018 April 28 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbusplit + +db close + +autoinstall_test_functions + +proc build_db {db} { + $db eval { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE TABLE t2(a PRIMARY KEY, b, c); + + CREATE INDEX t1c ON t1(c); + } +} + +proc build_rbu {filename} { + forcedelete $filename + sqlite3 dbRbu $filename + dbRbu eval { + CREATE TABLE data0_t1(a, b, c, rbu_control); + CREATE TABLE data1_t1(a, b, c, rbu_control); + CREATE TABLE data2_t1(a, b, c, rbu_control); + CREATE TABLE data3_t1(a, b, c, rbu_control); + + CREATE TABLE data_t2(a, b, c, rbu_control); + + INSERT INTO data0_t1 VALUES(1, 1, 1, 0); + INSERT INTO data0_t1 VALUES(2, 2, 2, 0); + INSERT INTO data0_t1 VALUES(3, 3, 3, 0); + INSERT INTO data0_t1 VALUES(4, 4, 4, 0); + INSERT INTO data1_t1 VALUES(5, 5, 5, 0); + INSERT INTO data1_t1 VALUES(6, 6, 6, 0); + INSERT INTO data1_t1 VALUES(7, 7, 7, 0); + INSERT INTO data1_t1 VALUES(8, 8, 8, 0); + INSERT INTO data3_t1 VALUES(9, 9, 9, 0); + + INSERT INTO data_t2 VALUES(1, 1, 1, 0); + INSERT INTO data_t2 VALUES(2, 2, 2, 0); + INSERT INTO data_t2 VALUES(3, 3, 3, 0); + INSERT INTO data_t2 VALUES(4, 4, 4, 0); + INSERT INTO data_t2 VALUES(5, 5, 5, 0); + INSERT INTO data_t2 VALUES(6, 6, 6, 0); + INSERT INTO data_t2 VALUES(7, 7, 7, 0); + INSERT INTO data_t2 VALUES(8, 8, 8, 0); + INSERT INTO data_t2 VALUES(9, 9, 9, 0); + } + + dbRbu close +} + +foreach {tn cmd} { + 1 run_rbu + 2 step_rbu +} { + reset_db + build_db db + build_rbu testrbu.db + + do_test 1.$tn.1 { + $cmd test.db testrbu.db + } {SQLITE_DONE} + do_execsql_test 1.$tn.1 { + SELECT * FROM t1; + } { + 1 1 1 2 2 2 3 3 3 4 4 4 + 5 5 5 6 6 6 7 7 7 8 8 8 + 9 9 9 + } + do_execsql_test 1.$tn.2 { + SELECT * FROM t2; + } { + 1 1 1 2 2 2 3 3 3 4 4 4 + 5 5 5 6 6 6 7 7 7 8 8 8 + 9 9 9 + } +} + +finish_test diff --git a/ext/rbu/rbutemplimit.test b/ext/rbu/rbutemplimit.test new file mode 100644 index 0000000..88b5f41 --- /dev/null +++ b/ext/rbu/rbutemplimit.test @@ -0,0 +1,129 @@ +# 2014 August 30 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# TESTRUNNER: slow + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbutemplimit + +db close + +proc setup_databases {} { + forcedelete test.db2 + forcedelete test.db + sqlite3 db test.db + execsql { + -- Create target database schema. + -- + CREATE TABLE t1(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100)); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b BLOB(100), c BLOB(100)); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + CREATE INDEX i2b ON t2(b); + CREATE INDEX i2c ON t2(c); + + -- Create a large RBU database. + -- + ATTACH 'test.db2' AS rbu; + CREATE TABLE rbu.data_t1(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<10000 + ) + INSERT INTO data_t1 SELECT i, randomblob(100), randomblob(100), 0 FROM s; + CREATE TABLE rbu.data_t2(a, b, c, rbu_control); + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<15000 + ) + INSERT INTO data_t2 SELECT i, randomblob(100), randomblob(100), 0 FROM s; + } + db close +} + +proc run_rbu_cachesize {target rbu cachesize temp_limit} { + sqlite3rbu rbu $target $rbu + rbu temp_size_limit $temp_limit + sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize" + while 1 { + set rc [rbu step] + set ::A([rbu temp_size]) 1 + if {$rc!="SQLITE_OK"} break + } + list [catch {rbu close} msg] $msg +} + +proc step_rbu_cachesize {target rbu stepsize cachesize temp_limit} { + set res "" + while 1 { + sqlite3rbu rbu $target $rbu + rbu temp_size_limit $temp_limit + if { [rbu temp_size_limit -1]!=$temp_limit } { error "round trip problem!" } + sqlite3_exec_nr [rbu db 1] "PRAGMA cache_size = $cachesize" + for {set i 0} {$i < $stepsize} {incr i} { + set rc [rbu step] + set ::A([rbu temp_size]) 1 + if {$rc!="SQLITE_OK"} break + } + set res [list [catch {rbu close} msg] $msg] + if {$res != "0 SQLITE_OK"} break + } + set res +} + +do_test 1.1.0 { setup_databases } {} + +do_test 1.1.1 { + unset -nocomplain ::A + run_rbu_cachesize test.db test.db2 10 0 +} {0 SQLITE_DONE} + +do_test 1.1.2 { llength [array names ::A] } 3 + +do_test 1.1.3 { + foreach {a0 a1 a2} [lsort -integer [array names ::A]] {} + list [expr $a0==0] \ + [expr $a1>1048576] [expr $a1<1200000] \ + [expr $a2>1500000] [expr $a2<1700000] +} {1 1 1 1 1} + +do_test 1.2.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1000000 +} {1 SQLITE_FULL} +do_test 1.2.2 { info commands rbu } {} + +do_test 1.3.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1300000 +} {1 SQLITE_FULL} +do_test 1.3.2 { info commands rbu } {} + +do_test 1.4.1 { + setup_databases + run_rbu_cachesize test.db test.db2 10 1800000 +} {0 SQLITE_DONE} +do_test 1.4.2 { info commands rbu } {} + +do_test 1.5.1 { + setup_databases + unset -nocomplain ::A + step_rbu_cachesize test.db test.db2 1000 10 2400000 +} {0 SQLITE_DONE} +do_test 1.5.2 { info commands rbu } {} + +do_test 1.6.1 { + setup_databases + unset -nocomplain ::A + step_rbu_cachesize test.db test.db2 1000 10 1400000 +} {1 SQLITE_FULL} +do_test 1.6.2 { info commands rbu } {} + +finish_test diff --git a/ext/rbu/rbuvacuum.test b/ext/rbu/rbuvacuum.test new file mode 100644 index 0000000..ad377a4 --- /dev/null +++ b/ext/rbu/rbuvacuum.test @@ -0,0 +1,400 @@ +# 2016 April 15 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set ::testprefix rbuvacuum + +foreach step {0 1} { + + set ::testprefix rbuvacuum-step=$step + reset_db + + # Simplest possible vacuum. + do_execsql_test 1.0 { + PRAGMA page_size = 1024; + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + PRAGMA integrity_check; + } {ok} + do_rbu_vacuum_test 1.1 $step + + # A vacuum that actually reclaims space. + do_execsql_test 1.2.1 { + INSERT INTO t1 VALUES(8, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(9, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(10, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(11, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(12, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {12} + do_execsql_test 1.2.2 { + DELETE FROM t1 WHERE rowid BETWEEN 8 AND 11; + PRAGMA page_count; + } {12} + do_rbu_vacuum_test 1.2.3 $step + do_execsql_test 1.2.4 { + PRAGMA page_count; + } {3} + + # Add an index to the table. + do_execsql_test 1.3.1 { + CREATE INDEX t1b ON t1(b); + INSERT INTO t1 VALUES(13, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(14, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(15, randomblob(900), randomblob(900)); + INSERT INTO t1 VALUES(16, randomblob(900), randomblob(900)); + PRAGMA page_count; + } {18} + do_execsql_test 1.3.2 { + DELETE FROM t1 WHERE rowid BETWEEN 12 AND 15; + PRAGMA page_count; + } {18} + do_rbu_vacuum_test 1.3.3 $step + do_execsql_test 1.3.4 { + PRAGMA page_count; + } {5} + + # WITHOUT ROWID table. + do_execsql_test 1.4.1 { + CREATE TABLE t2(a, b, c, PRIMARY KEY(a, b)) WITHOUT ROWID; + + INSERT INTO t2 VALUES(randomblob(900), 1, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 2, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 3, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 4, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 6, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 7, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 8, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 2 AND 7; + PRAGMA page_count; + } {20} + do_rbu_vacuum_test 1.4.2 $step + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {10} + + # WITHOUT ROWID table with an index. + do_execsql_test 1.4.1 { + CREATE INDEX t2c ON t2(c); + + INSERT INTO t2 VALUES(randomblob(900), 9, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 10, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 11, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 12, randomblob(900)); + INSERT INTO t2 VALUES(randomblob(900), 13, randomblob(900)); + + DELETE FROM t2 WHERE b BETWEEN 8 AND 12; + PRAGMA page_count; + } {35} + do_rbu_vacuum_test 1.4.2 $step + do_execsql_test 1.4.3 { + PRAGMA page_count; + } {15} + do_execsql_test 1.4.4 { + VACUUM; + PRAGMA page_count; + } {15} + + do_execsql_test 1.5.1 { + CREATE TABLE t3(a, b, c); + INSERT INTO t3 VALUES('a', 'b', 'c'); + INSERT INTO t3 VALUES('d', 'e', 'f'); + INSERT INTO t3 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.5.2 $step + do_execsql_test 1.5.3 { + SELECT * FROM t3 + } {a b c d e f g h i} + do_execsql_test 1.5.4 { + CREATE INDEX t3a ON t3(a); + CREATE INDEX t3b ON t3(b); + CREATE INDEX t3c ON t3(c); + INSERT INTO t3 VALUES('j', 'k', 'l'); + DELETE FROM t3 WHERE a = 'g'; + } + do_rbu_vacuum_test 1.5.5 $step + do_execsql_test 1.5.6 { + SELECT rowid, * FROM t3 ORDER BY b + } {1 a b c 2 d e f 4 j k l} + + do_execsql_test 1.6.1 { + CREATE TABLE t4(a PRIMARY KEY, b, c); + INSERT INTO t4 VALUES('a', 'b', 'c'); + INSERT INTO t4 VALUES('d', 'e', 'f'); + INSERT INTO t4 VALUES('g', 'h', 'i'); + } + do_rbu_vacuum_test 1.6.2 $step + do_execsql_test 1.6.3 { + SELECT * FROM t4 + } {a b c d e f g h i} + do_execsql_test 1.6.4 { + CREATE INDEX t4a ON t4(a); + CREATE INDEX t4b ON t4(b); + CREATE INDEX t4c ON t4(c); + + INSERT INTO t4 VALUES('j', 'k', 'l'); + DELETE FROM t4 WHERE a='g'; + } + do_rbu_vacuum_test 1.6.5 $step + do_execsql_test 1.6.6 { + SELECT * FROM t4 ORDER BY b + } {a b c d e f j k l} + + reset_db + do_execsql_test 1.7.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + DELETE FROM t1 WHERE a=2; + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + DELETE FROM t1 WHERE a=4; + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + DELETE FROM t1 WHERE a=6; + SELECT * FROM t1; + } {1 one 3 three 5 five} + do_rbu_vacuum_test 1.7.1 $step + do_execsql_test 1.7.2 { + INSERT INTO t1 VALUES(NULL, 'seven'); + SELECT * FROM t1; + } {1 one 3 three 5 five 7 seven} + + reset_db + do_execsql_test 1.8.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + ANALYZE; + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + do_rbu_vacuum_test 1.8.1 $step + do_execsql_test 1.7.2 { + SELECT * FROM sqlite_stat1; + } {t1 i1 {6 1}} + + reset_db + do_execsql_test 1.9.0 { + PRAGMA page_size = 8192; + PRAGMA auto_vacuum = 2; + PRAGMA user_version = 412; + PRAGMA application_id = 413; + + CREATE TABLE t1(a INTEGER PRIMARY KEY AUTOINCREMENT, b); + CREATE INDEX i1 ON t1(b); + INSERT INTO t1 VALUES(NULL, 'one'); + INSERT INTO t1 VALUES(NULL, 'two'); + INSERT INTO t1 VALUES(NULL, 'three'); + INSERT INTO t1 VALUES(NULL, 'four'); + INSERT INTO t1 VALUES(NULL, 'five'); + INSERT INTO t1 VALUES(NULL, 'six'); + + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + do_rbu_vacuum_test 1.9.1 $step + do_execsql_test 1.9.2 { + PRAGMA main.page_size; + PRAGMA main.auto_vacuum; + PRAGMA main.user_version; + PRAGMA main.application_id; + } {8192 2 412 413} + + # Vacuum a database with a large sqlite_master table. + # + reset_db + do_test 1.10.1 { + for {set i 1} {$i < 50} {incr i} { + execsql "PRAGMA page_size = 1024" + execsql "CREATE TABLE t$i (a, b, c, PRIMARY KEY(a, b));" + execsql " + INSERT INTO t$i VALUES(1, 2, 3); + INSERT INTO t$i VALUES(4, 5, 6); + " + } + } {} + do_rbu_vacuum_test 1.10.2 $step + + # Database with empty tables. + # + reset_db + do_execsql_test 1.11.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t4 VALUES(1, 2); + } + do_rbu_vacuum_test 1.11.2 $step + do_execsql_test 1.11.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} + reset_db + do_execsql_test 1.12.1 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + CREATE TABLE t2(a INTEGER PRIMARY KEY, b); + CREATE TABLE t3(a INTEGER PRIMARY KEY, b); + CREATE TABLE t4(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 2); + } + do_rbu_vacuum_test 1.12.2 $step + do_execsql_test 1.12.3 { + SELECT * FROM t1; + SELECT * FROM t2; + SELECT * FROM t3; + SELECT * FROM t4; + } {1 2} +} +set ::testprefix rbuvacuum + +#------------------------------------------------------------------------- +# Test some error cases: +# +# 2.1.* the db being vacuumed being in wal mode already. +# 2.2.* database modified mid vacuum. +# +reset_db +do_execsql_test 2.1.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + PRAGMA journal_mode = wal; + INSERT INTO t1 VALUES(9, 10); +} wal +do_test 2.1.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step +} {SQLITE_ERROR} +do_test 2.1.2 { + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - cannot vacuum wal mode database}} + +do_test 2.1.3 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step +} {SQLITE_ERROR} +do_test 2.1.4 { + list [catch { rbu close_no_error } msg] $msg +} {1 SQLITE_ERROR} + +reset_db +do_execsql_test 2.2.0 { + CREATE TABLE tx(a PRIMARY KEY, b BLOB); + INSERT INTO tx VALUES(1, randomblob(900)); + INSERT INTO tx SELECT a+1, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+2, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+4, randomblob(900) FROM tx; + INSERT INTO tx SELECT a+8, randomblob(900) FROM tx; +} +db_save_and_close +for {set i 1} 1 {incr i} { + db_restore_and_reopen + + sqlite3rbu_vacuum rbu test.db state.db + for {set step 0} {$step<$i} {incr step} { rbu step } + rbu close + if {[file exists test.db-wal]} break + + execsql { INSERT INTO tx VALUES(20, 20) } + + do_test 2.2.$i.1 { + sqlite3rbu_vacuum rbu test.db state.db + rbu step + } {SQLITE_BUSY} + do_test 2.2.$i.2 { + list [catch { rbu close } msg] $msg + } {1 {SQLITE_BUSY - database modified during rbu vacuum}} +} + +#------------------------------------------------------------------------- +# Test that a database that uses custom collation sequences can be RBU +# vacuumed. +# +reset_db +forcedelete state.db +proc noop {args} {} +proc length_cmp {x y} { + set n1 [string length $x] + set n2 [string length $y] + return [expr $n1 - $n2] +} +sqlite3_create_collation_v2 db length length_cmp noop + +do_execsql_test 3.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b); + INSERT INTO t1 VALUES(1, 'i'); + INSERT INTO t1 VALUES(2, 'iiii'); + INSERT INTO t1 VALUES(3, 'ii'); + INSERT INTO t1 VALUES(4, 'iii'); + SELECT a FROM t1 ORDER BY b COLLATE length; +} {1 3 4 2} +do_execsql_test 3.1 { + CREATE INDEX i1 ON t1(b COLLATE length); +} + +do_test 3.2 { + sqlite3rbu_vacuum rbu test.db state.db + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.3 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.4 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 1] + sqlite3_create_collation_v2 $db1 length length_cmp noop + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {1 {SQLITE_ERROR - no such collation sequence: length}} + +do_test 3.5 { + sqlite3rbu_vacuum rbu test.db state.db + set db1 [rbu db 0] + set db2 [rbu db 1] + + sqlite3_create_collation_v2 $db1 length length_cmp noop + sqlite3_create_collation_v2 $db2 length length_cmp noop + + while {[rbu step]=="SQLITE_OK"} {} + list [catch { rbu close } msg] $msg +} {0 SQLITE_DONE} + +catch { db close } +finish_test diff --git a/ext/rbu/rbuvacuum2.test b/ext/rbu/rbuvacuum2.test new file mode 100644 index 0000000..34ec261 --- /dev/null +++ b/ext/rbu/rbuvacuum2.test @@ -0,0 +1,249 @@ +# 2016 June 1 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } + +foreach {step} {0 1} { +foreach {ttt state} { + s state.db t test.db-vacuum n {} +} { + set ::testprefix rbuvacuum2-$step$ttt + + #------------------------------------------------------------------------- + # Test that a database that contains fts3 tables can be vacuumed. + # + ifcapable fts3 { + reset_db + do_execsql_test 1.1 { + CREATE VIRTUAL TABLE t1 USING fts3(z, y); + INSERT INTO t1 VALUES('fix this issue', 'at some point'); + } + + do_rbu_vacuum_test 1.2 $step $state + + do_execsql_test 1.3 { + SELECT * FROM t1; + } {{fix this issue} {at some point}} + + do_execsql_test 1.4 { + SELECT rowid FROM t1 WHERE t1 MATCH 'fix'; + } {1} + + do_execsql_test 1.5 { + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('l h i', 'd e f'); + DELETE FROM t1 WHERE docid = 2; + INSERT INTO t1 VALUES('a b c', 'x y z'); + } + + do_rbu_vacuum_test 1.6 $step $state + do_execsql_test 1.7 { + INSERT INTO t1(t1) VALUES('integrity-check'); + SELECT * FROM t1; + } { + {fix this issue} {at some point} + {l h i} {d e f} + {a b c} {x y z} + } + } + + #------------------------------------------------------------------------- + # Test that a database that contains fts5 tables can be vacuumed. + # + ifcapable fts5 { + reset_db + do_execsql_test 2.1 { + CREATE VIRTUAL TABLE t1 USING fts5(z, y); + INSERT INTO t1 VALUES('fix this issue', 'at some point'); + } + + do_rbu_vacuum_test 2.2 $step $state + + do_execsql_test 2.3 { + SELECT * FROM t1; + } {{fix this issue} {at some point}} + + do_execsql_test 2.4 { + SELECT rowid FROM t1 ('fix'); + } {1} + + do_execsql_test 2.5 { + INSERT INTO t1 VALUES('a b c', 'd e f'); + INSERT INTO t1 VALUES('l h i', 'd e f'); + DELETE FROM t1 WHERE rowid = 2; + INSERT INTO t1 VALUES('a b c', 'x y z'); + } + + do_rbu_vacuum_test 2.6 $step $state + do_execsql_test 2.7 { + INSERT INTO t1(t1) VALUES('integrity-check'); + SELECT * FROM t1; + } { + {fix this issue} {at some point} + {l h i} {d e f} + {a b c} {x y z} + } + } + + #------------------------------------------------------------------------- + # Test that a database that contains an rtree table can be vacuumed. + # + ifcapable rtree { + reset_db + do_execsql_test 3.1 { + CREATE VIRTUAL TABLE rt USING rtree(id, x1, x2); + INSERT INTO rt VALUES(1, 45, 55); + INSERT INTO rt VALUES(2, 50, 60); + INSERT INTO rt VALUES(3, 55, 65); + } + + do_rbu_vacuum_test 3.2 $step $state + + do_execsql_test 3.3 { + SELECT * FROM rt; + } {1 45.0 55.0 2 50.0 60.0 3 55.0 65.0} + + do_execsql_test 3.4.1 { + SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 + } {1 2} + do_execsql_test 3.4.2 { + SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 + } {2 3} + + do_rbu_vacuum_test 3.5 $step $state + + do_execsql_test 3.6.1 { + SELECT rowid FROM rt WHERE x2>51 AND x1 < 51 + } {1 2} + do_execsql_test 3.6.2 { + SELECT rowid FROM rt WHERE x2>59 AND x1 < 59 + } {2 3} + } + + ifcapable trigger { + reset_db + do_execsql_test 4.1 { + CREATE TABLE t1(a, b, c); + INSERT INTO t1 VALUES(1, 2, 3); + CREATE VIEW v1 AS SELECT * FROM t1; + CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END; + } + + do_execsql_test 4.2 { + SELECT * FROM sqlite_master; + } { + table t1 t1 2 {CREATE TABLE t1(a, b, c)} + view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} + trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} + } + + do_rbu_vacuum_test 4.3 $step $state + do_execsql_test 4.4 { + SELECT * FROM sqlite_master; + } { + table t1 t1 2 {CREATE TABLE t1(a, b, c)} + view v1 v1 0 {CREATE VIEW v1 AS SELECT * FROM t1} + trigger tr1 t1 0 {CREATE TRIGGER tr1 AFTER INSERT ON t1 BEGIN SELECT 1; END} + } + } +} +} + +#------------------------------------------------------------------------- +# Test that passing a NULL value as the second argument to +# sqlite3rbu_vacuum() causes it to: +# +# * Use -vacuum as the state db, and +# * Set the state db permissions to the same as those on the db file. +# +db close +if {$::tcl_platform(platform)=="unix"} { + forcedelete test.db + + sqlite3 db test.db + do_execsql_test 5.0 { + CREATE TABLE t1(a, b); + INSERT INTO t1 VALUES(1, 2); + INSERT INTO t1 VALUES(3, 4); + INSERT INTO t1 VALUES(5, 6); + INSERT INTO t1 VALUES(7, 8); + } + db close + + foreach {tn perm} { + 1 00755 + 2 00666 + 3 00644 + 4 00444 + } { + forcedelete test.db-vacuum + + do_test 5.$tn.1 { + file attributes test.db -permissions $perm + sqlite3rbu_vacuum rbu test.db + rbu step + } {SQLITE_OK} + + do_test 5.$tn.2 { file exists test.db-vacuum } 1 + # The result pattern might be 00xxx or 0oxxx depending on which + # version of TCL is being used. So make perm2 into a regexp that + # will match either + regsub {^00} $perm {0.} perm2 + do_test 5.$tn.3 { file attributes test.db-vacuum -permissions} /$perm2/ + rbu close + } +} + +#------------------------------------------------------------------------- +# Test the outcome of some other connection running a checkpoint while +# the incremental checkpoint is suspended. +# +reset_db +do_execsql_test 6.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); +} +forcedelete test.db2 + +do_test 6.1 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu state]!="checkpoint"} { rbu step } + rbu close +} {SQLITE_OK} + +do_test 6.2 { + execsql { SELECT 1 FROM sqlite_master LIMIT 1 } + execsql { PRAGMA wal_checkpoint } + execsql { SELECT 1 FROM sqlite_master LIMIT 1 } +} {1} + +do_test 6.3 { + sqlite3rbu_vacuum rbu test.db test.db2 + while {[rbu step]!="SQLITE_DONE"} { rbu step } + rbu close + execsql { PRAGMA integrity_check } +} {ok} + +do_test 6.4 { + sqlite3rbu_vacuum rbu test.db test.db-vactmp + list [catch { rbu close } msg] $msg +} {1 SQLITE_MISUSE} + +finish_test diff --git a/ext/rbu/rbuvacuum3.test b/ext/rbu/rbuvacuum3.test new file mode 100644 index 0000000..d70c094 --- /dev/null +++ b/ext/rbu/rbuvacuum3.test @@ -0,0 +1,64 @@ +# 2019 Jan 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set testprefix rbuvacuum3 + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c); + CREATE INDEX i1b ON t1(b); + CREATE INDEX i1c ON t1(c); + + WITH s(i) AS ( + VALUES(1) UNION ALL SELECT i+1 FROM s WHERE i<100 + ) + INSERT INTO t1 SELECT i, randomblob(100), randomblob(100) FROM s; +} + +forcedelete state.db +do_test 1.1 { + sqlite3rbu_vacuum rbu test.db state.db + while {1} { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + rbu savestate + } + rbu close +} {SQLITE_DONE} + +do_test 1.2 { + sqlite3rbu_vacuum rbu test.db state.db + while {1} { + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + rbu savestate + } + rbu close +} {SQLITE_DONE} + +do_test 1.3 { + while {1} { + sqlite3rbu_vacuum rbu test.db state.db + set rc [rbu step] + if {$rc!="SQLITE_OK"} break + rbu savestate + rbu close + } + rbu close +} {SQLITE_DONE} + +finish_test diff --git a/ext/rbu/rbuvacuum4.test b/ext/rbu/rbuvacuum4.test new file mode 100644 index 0000000..7b0db97 --- /dev/null +++ b/ext/rbu/rbuvacuum4.test @@ -0,0 +1,118 @@ +# 2019 Jan 3 +# +# The author disclaims copyright to this source code. In place of +# a legal notice, here is a blessing: +# +# May you do good and not evil. +# May you find forgiveness for yourself and forgive others. +# May you share freely, never taking more than you give. +# +#*********************************************************************** +# +# This file contains tests for the RBU module. More specifically, it +# contains tests to ensure that the sqlite3rbu_vacuum() API works as +# expected. +# + +source [file join [file dirname [info script]] rbu_common.tcl] +if_no_rbu_support { finish_test ; return } +set testprefix rbuvacuum4 + +set step 1 + +do_execsql_test 1.0 { + CREATE TABLE t1(a PRIMARY KEY, b, c) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); +} +do_rbu_vacuum_test 1.1 1 + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 2.0 { + CREATE TABLE t1(a, b, c, PRIMARY KEY(a, b, c)) WITHOUT ROWID; + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); +} +do_rbu_vacuum_test 2.1 1 +do_execsql_test 2.2 { + SELECT * FROM t1; +} {1 2 3 4 5 6 7 8 9} + +#------------------------------------------------------------------------- +reset_db + +do_execsql_test 3.0 { + CREATE TABLE t1(a INTEGER PRIMARY KEY, b, c); + CREATE INDEX i1 oN t1(b, c); + INSERT INTO t1 VALUES(1, 2, 3); + INSERT INTO t1 VALUES(4, 5, 6); + INSERT INTO t1 VALUES(7, 8, 9); + + CREATE TABLE t2(a, b, c INTEGER, PRIMARY KEY(c)); + CREATE INDEX i2 oN t2(b, a); + INSERT INTO t2 VALUES('a', 'b', -1); + INSERT INTO t2 VALUES('c', 'd', -2); + INSERT INTO t2 VALUES('e', 'f', -3); +} + +do_rbu_vacuum_test 3.1 1 + +do_execsql_test 3.2 { + SELECT * FROM t1; + SELECT * FROM t2; +} {1 2 3 4 5 6 7 8 9 e f -3 c d -2 a b -1} + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 4.0 { + CREATE TABLE x1(a, b, c, d, PRIMARY KEY(c, b)) WITHOUT ROWID; + INSERT INTO x1 VALUES(1, 1, 1, 1); + INSERT INTO x1 VALUES(1, 1, 2, 1); + INSERT INTO x1 VALUES(1, 2, 2, 1); + + INSERT INTO x1 VALUES(NULL, 2, 3, NULL); + INSERT INTO x1 VALUES(NULL, 2, 4, NULL); + INSERT INTO x1 VALUES(NULL, 2, 5, NULL); + + CREATE INDEX x1ad ON x1(d, a); + CREATE INDEX x1null ON x1(d, a) WHERE d>15; +} + +do_rbu_vacuum_test 4.1.1 1 + +do_execsql_test 4.2 { + SELECT count(*) fROM x1 +} 6 + +db cache flush +do_rbu_vacuum_test 4.1.2 0 + +#------------------------------------------------------------------------- +reset_db +do_execsql_test 5.0 { + CREATE TABLE "a b c"(a, "b b" PRIMARY KEY, "c c"); + CREATE INDEX abc1 ON "a b c"(a, "c c"); + + INSERT INTO "a b c" VALUES(NULL, 'a', NULL); + INSERT INTO "a b c" VALUES(NULL, 'b', NULL); + INSERT INTO "a b c" VALUES(NULL, 'c', NULL); + + INSERT INTO "a b c" VALUES(1, 2, 3); + INSERT INTO "a b c" VALUES(3, 9, 1); + INSERT INTO "a b c" VALUES('aaa', 'bbb', 'ccc'); + + CREATE INDEX abc2 ON "a b c"("c c" DESC, a); + + CREATE TABLE x(a); + INSERT INTO x VALUES('a'), ('b'), ('d'); + CREATE UNIQUE INDEX y ON x(a); +} + +do_rbu_vacuum_test 5.1 1 + +finish_test + diff --git a/ext/rbu/sqlite3rbu.c b/ext/rbu/sqlite3rbu.c new file mode 100644 index 0000000..15b05ce --- /dev/null +++ b/ext/rbu/sqlite3rbu.c @@ -0,0 +1,5416 @@ +/* +** 2014 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** +** OVERVIEW +** +** The RBU extension requires that the RBU update be packaged as an +** SQLite database. The tables it expects to find are described in +** sqlite3rbu.h. Essentially, for each table xyz in the target database +** that the user wishes to write to, a corresponding data_xyz table is +** created in the RBU database and populated with one row for each row to +** update, insert or delete from the target table. +** +** The update proceeds in three stages: +** +** 1) The database is updated. The modified database pages are written +** to a *-oal file. A *-oal file is just like a *-wal file, except +** that it is named "-oal" instead of "-wal". +** Because regular SQLite clients do not look for file named +** "-oal", they go on using the original database in +** rollback mode while the *-oal file is being generated. +** +** During this stage RBU does not update the database by writing +** directly to the target tables. Instead it creates "imposter" +** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses +** to update each b-tree individually. All updates required by each +** b-tree are completed before moving on to the next, and all +** updates are done in sorted key order. +** +** 2) The "-oal" file is moved to the equivalent "-wal" +** location using a call to rename(2). Before doing this the RBU +** module takes an EXCLUSIVE lock on the database file, ensuring +** that there are no other active readers. +** +** Once the EXCLUSIVE lock is released, any other database readers +** detect the new *-wal file and read the database in wal mode. At +** this point they see the new version of the database - including +** the updates made as part of the RBU update. +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way +** as a regular database checkpoint, except that a single frame is +** checkpointed each time sqlite3rbu_step() is called. If the RBU +** handle is closed before the entire *-wal file is checkpointed, +** the checkpoint progress is saved in the RBU database and the +** checkpoint can be resumed by another RBU client at some point in +** the future. +** +** POTENTIAL PROBLEMS +** +** The rename() call might not be portable. And RBU is not currently +** syncing the directory after renaming the file. +** +** When state is saved, any commit to the *-oal file and the commit to +** the RBU update database are not atomic. So if the power fails at the +** wrong moment they might get out of sync. As the main database will be +** committed before the RBU update database this will likely either just +** pass unnoticed, or result in SQLITE_CONSTRAINT errors (due to UNIQUE +** constraint violations). +** +** If some client does modify the target database mid RBU update, or some +** other error occurs, the RBU extension will keep throwing errors. It's +** not really clear how to get out of this state. The system could just +** by delete the RBU update database and *-oal file and have the device +** download the update again and start over. +** +** At present, for an UPDATE, both the new.* and old.* records are +** collected in the rbu_xyz table. And for both UPDATEs and DELETEs all +** fields are collected. This means we're probably writing a lot more +** data to disk when saving the state of an ongoing update to the RBU +** update database than is strictly necessary. +** +*/ + +#include +#include +#include + +#include "sqlite3.h" + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_RBU) +#include "sqlite3rbu.h" + +#if defined(_WIN32_WCE) +#include "windows.h" +#endif + +/* Maximum number of prepared UPDATE statements held by this module */ +#define SQLITE_RBU_UPDATE_CACHESIZE 16 + +/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM +** to enable checksum verification. +*/ +#ifndef RBU_ENABLE_DELTA_CKSUM +# define RBU_ENABLE_DELTA_CKSUM 0 +#endif + +/* +** Swap two objects of type TYPE. +*/ +#if !defined(SQLITE_AMALGAMATION) +# define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} +#endif + +/* +** Name of the URI option that causes RBU to take an exclusive lock as +** part of the incremental checkpoint operation. +*/ +#define RBU_EXCLUSIVE_CHECKPOINT "rbu_exclusive_checkpoint" + + +/* +** The rbu_state table is used to save the state of a partially applied +** update so that it can be resumed later. The table consists of integer +** keys mapped to values as follows: +** +** RBU_STATE_STAGE: +** May be set to integer values 1, 2, 4 or 5. As follows: +** 1: the *-rbu file is currently under construction. +** 2: the *-rbu file has been constructed, but not yet moved +** to the *-wal path. +** 4: the checkpoint is underway. +** 5: the rbu update has been checkpointed. +** +** RBU_STATE_TBL: +** Only valid if STAGE==1. The target database name of the table +** currently being written. +** +** RBU_STATE_IDX: +** Only valid if STAGE==1. The target database name of the index +** currently being written, or NULL if the main table is currently being +** updated. +** +** RBU_STATE_ROW: +** Only valid if STAGE==1. Number of rows already processed for the current +** table/index. +** +** RBU_STATE_PROGRESS: +** Trbul number of sqlite3rbu_step() calls made so far as part of this +** rbu update. +** +** RBU_STATE_CKPT: +** Valid if STAGE==4. The 64-bit checksum associated with the wal-index +** header created by recovering the *-wal file. This is used to detect +** cases when another client appends frames to the *-wal file in the +** middle of an incremental checkpoint (an incremental checkpoint cannot +** be continued if this happens). +** +** RBU_STATE_COOKIE: +** Valid if STAGE==1. The current change-counter cookie value in the +** target db file. +** +** RBU_STATE_OALSZ: +** Valid if STAGE==1. The size in bytes of the *-oal file. +** +** RBU_STATE_DATATBL: +** Only valid if STAGE==1. The RBU database name of the table +** currently being read. +*/ +#define RBU_STATE_STAGE 1 +#define RBU_STATE_TBL 2 +#define RBU_STATE_IDX 3 +#define RBU_STATE_ROW 4 +#define RBU_STATE_PROGRESS 5 +#define RBU_STATE_CKPT 6 +#define RBU_STATE_COOKIE 7 +#define RBU_STATE_OALSZ 8 +#define RBU_STATE_PHASEONESTEP 9 +#define RBU_STATE_DATATBL 10 + +#define RBU_STAGE_OAL 1 +#define RBU_STAGE_MOVE 2 +#define RBU_STAGE_CAPTURE 3 +#define RBU_STAGE_CKPT 4 +#define RBU_STAGE_DONE 5 + + +#define RBU_CREATE_STATE \ + "CREATE TABLE IF NOT EXISTS %s.rbu_state(k INTEGER PRIMARY KEY, v)" + +typedef struct RbuFrame RbuFrame; +typedef struct RbuObjIter RbuObjIter; +typedef struct RbuState RbuState; +typedef struct RbuSpan RbuSpan; +typedef struct rbu_vfs rbu_vfs; +typedef struct rbu_file rbu_file; +typedef struct RbuUpdateStmt RbuUpdateStmt; + +#if !defined(SQLITE_AMALGAMATION) +typedef unsigned int u32; +typedef unsigned short u16; +typedef unsigned char u8; +typedef sqlite3_int64 i64; +#endif + +/* +** These values must match the values defined in wal.c for the equivalent +** locks. These are not magic numbers as they are part of the SQLite file +** format. +*/ +#define WAL_LOCK_WRITE 0 +#define WAL_LOCK_CKPT 1 +#define WAL_LOCK_READ0 3 + +#define SQLITE_FCNTL_RBUCNT 5149216 + +/* +** A structure to store values read from the rbu_state table in memory. +*/ +struct RbuState { + int eStage; + char *zTbl; + char *zDataTbl; + char *zIdx; + i64 iWalCksum; + int nRow; + i64 nProgress; + u32 iCookie; + i64 iOalSz; + i64 nPhaseOneStep; +}; + +struct RbuUpdateStmt { + char *zMask; /* Copy of update mask used with pUpdate */ + sqlite3_stmt *pUpdate; /* Last update statement (or NULL) */ + RbuUpdateStmt *pNext; +}; + +struct RbuSpan { + const char *zSpan; + int nSpan; +}; + +/* +** An iterator of this type is used to iterate through all objects in +** the target database that require updating. For each such table, the +** iterator visits, in order: +** +** * the table itself, +** * each index of the table (zero or more points to visit), and +** * a special "cleanup table" state. +** +** abIndexed: +** If the table has no indexes on it, abIndexed is set to NULL. Otherwise, +** it points to an array of flags nTblCol elements in size. The flag is +** set for each column that is either a part of the PK or a part of an +** index. Or clear otherwise. +** +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. +** +*/ +struct RbuObjIter { + sqlite3_stmt *pTblIter; /* Iterate through tables */ + sqlite3_stmt *pIdxIter; /* Index iterator */ + int nTblCol; /* Size of azTblCol[] array */ + char **azTblCol; /* Array of unquoted target column names */ + char **azTblType; /* Array of target column types */ + int *aiSrcOrder; /* src table col -> target table col */ + u8 *abTblPk; /* Array of flags, set on target PK columns */ + u8 *abNotNull; /* Array of flags, set on NOT NULL columns */ + u8 *abIndexed; /* Array of flags, set on indexed & PK cols */ + int eType; /* Table type - an RBU_PK_XXX value */ + + /* Output variables. zTbl==0 implies EOF. */ + int bCleanup; /* True in "cleanup" state */ + const char *zTbl; /* Name of target db table */ + const char *zDataTbl; /* Name of rbu db table (or null) */ + const char *zIdx; /* Name of target db index (or null) */ + int iTnum; /* Root page of current object */ + int iPkTnum; /* If eType==EXTERNAL, root of PK index */ + int bUnique; /* Current index is unique */ + int nIndex; /* Number of aux. indexes on table zTbl */ + + /* Statements created by rbuObjIterPrepareAll() */ + int nCol; /* Number of columns in current object */ + sqlite3_stmt *pSelect; /* Source data */ + sqlite3_stmt *pInsert; /* Statement for INSERT operations */ + sqlite3_stmt *pDelete; /* Statement for DELETE ops */ + sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + int nIdxCol; + RbuSpan *aIdxCol; + char *zIdxSql; + + /* Last UPDATE used (for PK b-tree updates only), or NULL. */ + RbuUpdateStmt *pRbuUpdate; +}; + +/* +** Values for RbuObjIter.eType +** +** 0: Table does not exist (error) +** 1: Table has an implicit rowid. +** 2: Table has an explicit IPK column. +** 3: Table has an external PK index. +** 4: Table is WITHOUT ROWID. +** 5: Table is a virtual table. +*/ +#define RBU_PK_NOTABLE 0 +#define RBU_PK_NONE 1 +#define RBU_PK_IPK 2 +#define RBU_PK_EXTERNAL 3 +#define RBU_PK_WITHOUT_ROWID 4 +#define RBU_PK_VTAB 5 + + +/* +** Within the RBU_STAGE_OAL stage, each call to sqlite3rbu_step() performs +** one of the following operations. +*/ +#define RBU_INSERT 1 /* Insert on a main table b-tree */ +#define RBU_DELETE 2 /* Delete a row from a main table b-tree */ +#define RBU_REPLACE 3 /* Delete and then insert a row */ +#define RBU_IDX_DELETE 4 /* Delete a row from an aux. index b-tree */ +#define RBU_IDX_INSERT 5 /* Insert on an aux. index b-tree */ + +#define RBU_UPDATE 6 /* Update a row in a main table b-tree */ + +/* +** A single step of an incremental checkpoint - frame iWalFrame of the wal +** file should be copied to page iDbPage of the database file. +*/ +struct RbuFrame { + u32 iDbPage; + u32 iWalFrame; +}; + +/* +** RBU handle. +** +** nPhaseOneStep: +** If the RBU database contains an rbu_count table, this value is set to +** a running estimate of the number of b-tree operations required to +** finish populating the *-oal file. This allows the sqlite3_bp_progress() +** API to calculate the permyriadage progress of populating the *-oal file +** using the formula: +** +** permyriadage = (10000 * nProgress) / nPhaseOneStep +** +** nPhaseOneStep is initialized to the sum of: +** +** nRow * (nIndex + 1) +** +** for all source tables in the RBU database, where nRow is the number +** of rows in the source table and nIndex the number of indexes on the +** corresponding target database table. +** +** This estimate is accurate if the RBU update consists entirely of +** INSERT operations. However, it is inaccurate if: +** +** * the RBU update contains any UPDATE operations. If the PK specified +** for an UPDATE operation does not exist in the target table, then +** no b-tree operations are required on index b-trees. Or if the +** specified PK does exist, then (nIndex*2) such operations are +** required (one delete and one insert on each index b-tree). +** +** * the RBU update contains any DELETE operations for which the specified +** PK does not exist. In this case no operations are required on index +** b-trees. +** +** * the RBU update contains REPLACE operations. These are similar to +** UPDATE operations. +** +** nPhaseOneStep is updated to account for the conditions above during the +** first pass of each source table. The updated nPhaseOneStep value is +** stored in the rbu_state table if the RBU update is suspended. +*/ +struct sqlite3rbu { + int eStage; /* Value of RBU_STATE_STAGE field */ + sqlite3 *dbMain; /* target database handle */ + sqlite3 *dbRbu; /* rbu database handle */ + char *zTarget; /* Path to target db */ + char *zRbu; /* Path to rbu db */ + char *zState; /* Path to state db (or NULL if zRbu) */ + char zStateDb[5]; /* Db name for state ("stat" or "main") */ + int rc; /* Value returned by last rbu_step() call */ + char *zErrmsg; /* Error message if rc!=SQLITE_OK */ + int nStep; /* Rows processed for current object */ + int nProgress; /* Rows processed for all objects */ + RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ + const char *zVfsName; /* Name of automatically created rbu vfs */ + rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ + i64 iOalSz; + i64 nPhaseOneStep; + void *pRenameArg; + int (*xRename)(void*, const char*, const char*); + + /* The following state variables are used as part of the incremental + ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding + ** function rbuSetupCheckpoint() for details. */ + u32 iMaxFrame; /* Largest iWalFrame value in aFrame[] */ + u32 mLock; + int nFrame; /* Entries in aFrame[] array */ + int nFrameAlloc; /* Allocated size of aFrame[] array */ + RbuFrame *aFrame; + int pgsz; + u8 *aBuf; + i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ + + /* Used in RBU vacuum mode only */ + int nRbu; /* Number of RBU VFS in the stack */ + rbu_file *pRbuFd; /* Fd for main db of dbRbu */ +}; + +/* +** An rbu VFS is implemented using an instance of this structure. +** +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. +*/ +struct rbu_vfs { + sqlite3_vfs base; /* rbu VFS shim methods */ + sqlite3_vfs *pRealVfs; /* Underlying VFS */ + sqlite3_mutex *mutex; /* Mutex to protect pMain */ + sqlite3rbu *pRbu; /* Owner RBU object */ + rbu_file *pMain; /* List of main db files */ + rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */ +}; + +/* +** Each file opened by an rbu VFS is represented by an instance of +** the following structure. +** +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. +*/ +struct rbu_file { + sqlite3_file base; /* sqlite3_file methods */ + sqlite3_file *pReal; /* Underlying file handle */ + rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ + sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ + + int openFlags; /* Flags this file was opened with */ + u32 iCookie; /* Cookie value for main db files */ + u8 iWriteVer; /* "write-version" value for main db files */ + u8 bNolock; /* True to fail EXCLUSIVE locks */ + + int nShm; /* Number of entries in apShm[] array */ + char **apShm; /* Array of mmap'd *-shm regions */ + char *zDel; /* Delete this when closing file */ + + const char *zWal; /* Wal filename for this main db file */ + rbu_file *pWalFd; /* Wal file descriptor for this main db */ + rbu_file *pMainNext; /* Next MAIN_DB file */ + rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */ +}; + +/* +** True for an RBU vacuum handle, or false otherwise. +*/ +#define rbuIsVacuum(p) ((p)->zTarget==0) + + +/************************************************************************* +** The following three functions, found below: +** +** rbuDeltaGetInt() +** rbuDeltaChecksum() +** rbuDeltaApply() +** +** are lifted from the fossil source code (http://fossil-scm.org). They +** are used to implement the scalar SQL function rbu_fossil_delta(). +*/ + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +#if RBU_ENABLE_DELTA_CKSUM +/* +** Compute a 32-bit checksum on the N-byte buffer. Return the result. +*/ +static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ + const unsigned char *z = (const unsigned char *)zIn; + unsigned sum0 = 0; + unsigned sum1 = 0; + unsigned sum2 = 0; + unsigned sum3 = 0; + while(N >= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum3 += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum3 += z[3]; + z += 4; + N -= 4; + } + sum3 += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); + switch(N){ + case 3: sum3 += (z[2] << 8); + case 2: sum3 += (z[1] << 16); + case 1: sum3 += (z[0] << 24); + default: ; + } + return sum3; +} +#endif + +/* +** Apply a delta. +** +** The output buffer should be big enough to hold the whole output +** file and a NUL terminator at the end. The delta_output_size() +** routine will determine this size for you. +** +** The delta string should be null-terminated. But the delta string +** may contain embedded NUL characters (if the input and output are +** binary files) so we also have to pass in the length of the delta in +** the lenDelta parameter. +** +** This function returns the size of the output file in bytes (excluding +** the final NUL terminator character). Except, if the delta string is +** malformed or intended for use with a source file other than zSrc, +** then this routine returns -1. +** +** Refer to the delta_create() documentation above for a description +** of the delta file format. +*/ +static int rbuDeltaApply( + const char *zSrc, /* The source or pattern file */ + int lenSrc, /* Length of the source file */ + const char *zDelta, /* Delta to apply to the pattern */ + int lenDelta, /* Length of the delta */ + char *zOut /* Write the output into this preallocated buffer */ +){ + unsigned int limit; + unsigned int total = 0; +#if RBU_ENABLE_DELTA_CKSUM + char *zOrigOut = zOut; +#endif + + limit = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + zDelta++; lenDelta--; + while( *zDelta && lenDelta>0 ){ + unsigned int cnt, ofst; + cnt = rbuDeltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = rbuDeltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( (int)(ofst+cnt) > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( (int)cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#if RBU_ENABLE_DELTA_CKSUM + if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +static int rbuDeltaOutputSize(const char *zDelta, int lenDelta){ + int size; + size = rbuDeltaGetInt(&zDelta, &lenDelta); + if( *zDelta!='\n' ){ + /* ERROR: size integer not terminated by "\n" */ + return -1; + } + return size; +} + +/* +** End of code taken from fossil. +*************************************************************************/ + +/* +** Implementation of SQL scalar function rbu_fossil_delta(). +** +** This function applies a fossil delta patch to a blob. Exactly two +** arguments must be passed to this function. The first is the blob to +** patch and the second the patch to apply. If no error occurs, this +** function returns the patched blob. +*/ +static void rbuFossilDeltaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; + int nDelta; + const char *aOrig; + int nOrig; + + int nOut; + int nOut2; + char *aOut; + + assert( argc==2 ); + + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = rbuDeltaOutputSize(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + + aOut = sqlite3_malloc(nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + +/* +** Prepare the SQL statement in buffer zSql against database handle db. +** If successful, set *ppStmt to point to the new statement and return +** SQLITE_OK. +** +** Otherwise, if an error does occur, set *ppStmt to NULL and return +** an SQLite error code. Additionally, set output variable *pzErrmsg to +** point to a buffer containing an error message. It is the responsibility +** of the caller to (eventually) free this buffer using sqlite3_free(). +*/ +static int prepareAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + const char *zSql +){ + int rc = sqlite3_prepare_v2(db, zSql, -1, ppStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + *ppStmt = 0; + } + return rc; +} + +/* +** Reset the SQL statement passed as the first argument. Return a copy +** of the value returned by sqlite3_reset(). +** +** If an error has occurred, then set *pzErrmsg to point to a buffer +** containing an error message. It is the responsibility of the caller +** to eventually free this buffer using sqlite3_free(). +*/ +static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ + int rc = sqlite3_reset(pStmt); + if( rc!=SQLITE_OK ){ + *pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(sqlite3_db_handle(pStmt))); + } + return rc; +} + +/* +** Unless it is NULL, argument zSql points to a buffer allocated using +** sqlite3_malloc containing an SQL statement. This function prepares the SQL +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. +** +** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code +** returned. In this case, *pzErrmsg may also be set to point to an error +** message. It is the responsibility of the caller to free this error message +** buffer using sqlite3_free(). +** +** If argument zSql is NULL, this function assumes that an OOM has occurred. +** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. +*/ +static int prepareFreeAndCollectError( + sqlite3 *db, + sqlite3_stmt **ppStmt, + char **pzErrmsg, + char *zSql +){ + int rc; + assert( *pzErrmsg==0 ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + *ppStmt = 0; + }else{ + rc = prepareAndCollectError(db, ppStmt, pzErrmsg, zSql); + sqlite3_free(zSql); + } + return rc; +} + +/* +** Free the RbuObjIter.azTblCol[] and RbuObjIter.abTblPk[] arrays allocated +** by an earlier call to rbuObjIterCacheTableInfo(). +*/ +static void rbuObjIterFreeCols(RbuObjIter *pIter){ + int i; + for(i=0; inTblCol; i++){ + sqlite3_free(pIter->azTblCol[i]); + sqlite3_free(pIter->azTblType[i]); + } + sqlite3_free(pIter->azTblCol); + pIter->azTblCol = 0; + pIter->azTblType = 0; + pIter->aiSrcOrder = 0; + pIter->abTblPk = 0; + pIter->abNotNull = 0; + pIter->nTblCol = 0; + pIter->eType = 0; /* Invalid value */ +} + +/* +** Finalize all statements and free all allocations that are specific to +** the current object (table/index pair). +*/ +static void rbuObjIterClearStatements(RbuObjIter *pIter){ + RbuUpdateStmt *pUp; + + sqlite3_finalize(pIter->pSelect); + sqlite3_finalize(pIter->pInsert); + sqlite3_finalize(pIter->pDelete); + sqlite3_finalize(pIter->pTmpInsert); + pUp = pIter->pRbuUpdate; + while( pUp ){ + RbuUpdateStmt *pTmp = pUp->pNext; + sqlite3_finalize(pUp->pUpdate); + sqlite3_free(pUp); + pUp = pTmp; + } + sqlite3_free(pIter->aIdxCol); + sqlite3_free(pIter->zIdxSql); + + pIter->pSelect = 0; + pIter->pInsert = 0; + pIter->pDelete = 0; + pIter->pRbuUpdate = 0; + pIter->pTmpInsert = 0; + pIter->nCol = 0; + pIter->nIdxCol = 0; + pIter->aIdxCol = 0; + pIter->zIdxSql = 0; +} + +/* +** Clean up any resources allocated as part of the iterator object passed +** as the only argument. +*/ +static void rbuObjIterFinalize(RbuObjIter *pIter){ + rbuObjIterClearStatements(pIter); + sqlite3_finalize(pIter->pTblIter); + sqlite3_finalize(pIter->pIdxIter); + rbuObjIterFreeCols(pIter); + memset(pIter, 0, sizeof(RbuObjIter)); +} + +/* +** Advance the iterator to the next position. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ + int rc = p->rc; + if( rc==SQLITE_OK ){ + + /* Free any SQLite statements used while processing the previous object */ + rbuObjIterClearStatements(pIter); + if( pIter->zIdx==0 ){ + rc = sqlite3_exec(p->dbMain, + "DROP TRIGGER IF EXISTS temp.rbu_insert_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update1_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_update2_tr;" + "DROP TRIGGER IF EXISTS temp.rbu_delete_tr;" + , 0, 0, &p->zErrmsg + ); + } + + if( rc==SQLITE_OK ){ + if( pIter->bCleanup ){ + rbuObjIterFreeCols(pIter); + pIter->bCleanup = 0; + rc = sqlite3_step(pIter->pTblIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); + pIter->zTbl = 0; + }else{ + pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); + pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); + rc = (pIter->zDataTbl && pIter->zTbl) ? SQLITE_OK : SQLITE_NOMEM; + } + }else{ + if( pIter->zIdx==0 ){ + sqlite3_stmt *pIdx = pIter->pIdxIter; + rc = sqlite3_bind_text(pIdx, 1, pIter->zTbl, -1, SQLITE_STATIC); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_step(pIter->pIdxIter); + if( rc!=SQLITE_ROW ){ + rc = resetAndCollectError(pIter->pIdxIter, &p->zErrmsg); + pIter->bCleanup = 1; + pIter->zIdx = 0; + }else{ + pIter->zIdx = (const char*)sqlite3_column_text(pIter->pIdxIter, 0); + pIter->iTnum = sqlite3_column_int(pIter->pIdxIter, 1); + pIter->bUnique = sqlite3_column_int(pIter->pIdxIter, 2); + rc = pIter->zIdx ? SQLITE_OK : SQLITE_NOMEM; + } + } + } + } + } + + if( rc!=SQLITE_OK ){ + rbuObjIterFinalize(pIter); + p->rc = rc; + } + return rc; +} + + +/* +** The implementation of the rbu_target_name() SQL function. This function +** accepts one or two arguments. The first argument is the name of a table - +** the name of a table in the RBU database. The second, if it is present, is 1 +** for a view or 0 for a table. +** +** For a non-vacuum RBU handle, if the table name matches the pattern: +** +** data[0-9]_ +** +** where is any sequence of 1 or more characters, is returned. +** Otherwise, if the only argument does not match the above pattern, an SQL +** NULL is returned. +** +** "data_t1" -> "t1" +** "data0123_t2" -> "t2" +** "dataAB_t3" -> NULL +** +** For an rbu vacuum handle, a copy of the first argument is returned if +** the second argument is either missing or 0 (not a view). +*/ +static void rbuTargetNameFunc( + sqlite3_context *pCtx, + int argc, + sqlite3_value **argv +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + const char *zIn; + assert( argc==1 || argc==2 ); + + zIn = (const char*)sqlite3_value_text(argv[0]); + if( zIn ){ + if( rbuIsVacuum(p) ){ + assert( argc==2 || argc==1 ); + if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ + sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); + } + }else{ + if( strlen(zIn)>4 && memcmp("data", zIn, 4)==0 ){ + int i; + for(i=4; zIn[i]>='0' && zIn[i]<='9'; i++); + if( zIn[i]=='_' && zIn[i+1] ){ + sqlite3_result_text(pCtx, &zIn[i+1], -1, SQLITE_STATIC); + } + } + } + } +} + +/* +** Initialize the iterator structure passed as the second argument. +** +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the +** error code is returned. +*/ +static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ + int rc; + memset(pIter, 0, sizeof(RbuObjIter)); + + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( + "SELECT rbu_target_name(name, type='view') AS target, name " + "FROM sqlite_schema " + "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " + "ORDER BY name" + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, + "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " + " FROM main.sqlite_schema " + " WHERE type='index' AND tbl_name = ?" + ); + } + + pIter->bCleanup = 1; + p->rc = rc; + return rbuObjIterNext(p, pIter); +} + +/* +** This is a wrapper around "sqlite3_mprintf(zFmt, ...)". If an OOM occurs, +** an error code is stored in the RBU handle passed as the first argument. +** +** If an error has already occurred (p->rc is already set to something other +** than SQLITE_OK), then this function returns NULL without modifying the +** stored error code. In this case it still calls sqlite3_free() on any +** printf() parameters associated with %z conversions. +*/ +static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ + char *zSql = 0; + va_list ap; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ) p->rc = SQLITE_NOMEM; + }else{ + sqlite3_free(zSql); + zSql = 0; + } + va_end(ap); + return zSql; +} + +/* +** Argument zFmt is a sqlite3_mprintf() style format string. The trailing +** arguments are the usual subsitution values. This function performs +** the printf() style substitutions and executes the result as an SQL +** statement on the RBU handles database. +** +** If an error occurs, an error code and error message is stored in the +** RBU handle. If an error has already occurred when this function is +** called, it is a no-op. +*/ +static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ + va_list ap; + char *zSql; + va_start(ap, zFmt); + zSql = sqlite3_vmprintf(zFmt, ap); + if( p->rc==SQLITE_OK ){ + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + p->rc = sqlite3_exec(db, zSql, 0, 0, &p->zErrmsg); + } + } + sqlite3_free(zSql); + va_end(ap); + return p->rc; +} + +/* +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. +** +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL +** immediately without attempting the allocation or modifying the stored +** error code. +*/ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ + void *pRet = 0; + if( p->rc==SQLITE_OK ){ + assert( nByte>0 ); + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + memset(pRet, 0, nByte); + } + } + return pRet; +} + + +/* +** Allocate and zero the pIter->azTblCol[] and abTblPk[] arrays so that +** there is room for at least nCol elements. If an OOM occurs, store an +** error code in the RBU handle passed as the first argument. +*/ +static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + char **azNew; + + azNew = (char**)rbuMalloc(p, nByte); + if( azNew ){ + pIter->azTblCol = azNew; + pIter->azTblType = &azNew[nCol]; + pIter->aiSrcOrder = (int*)&pIter->azTblType[nCol]; + pIter->abTblPk = (u8*)&pIter->aiSrcOrder[nCol]; + pIter->abNotNull = (u8*)&pIter->abTblPk[nCol]; + pIter->abIndexed = (u8*)&pIter->abNotNull[nCol]; + } +} + +/* +** The first argument must be a nul-terminated string. This function +** returns a copy of the string in memory obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free this memory +** using sqlite3_free(). +** +** If an OOM condition is encountered when attempting to allocate memory, +** output variable (*pRc) is set to SQLITE_NOMEM before returning. Otherwise, +** if the allocation succeeds, (*pRc) is left unchanged. +*/ +static char *rbuStrndup(const char *zStr, int *pRc){ + char *zRet = 0; + + if( *pRc==SQLITE_OK ){ + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } + } + } + + return zRet; +} + +/* +** Finalize the statement passed as the second argument. +** +** If the sqlite3_finalize() call indicates that an error occurs, and the +** rbu handle error code is not already set, set the error code and error +** message accordingly. +*/ +static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ + sqlite3 *db = sqlite3_db_handle(pStmt); + int rc = sqlite3_finalize(pStmt); + if( p->rc==SQLITE_OK && rc!=SQLITE_OK ){ + p->rc = rc; + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } +} + +/* Determine the type of a table. +** +** peType is of type (int*), a pointer to an output parameter of type +** (int). This call sets the output parameter as follows, depending +** on the type of the table specified by parameters dbName and zTbl. +** +** RBU_PK_NOTABLE: No such table. +** RBU_PK_NONE: Table has an implicit rowid. +** RBU_PK_IPK: Table has an explicit IPK column. +** RBU_PK_EXTERNAL: Table has an external PK index. +** RBU_PK_WITHOUT_ROWID: Table is WITHOUT ROWID. +** RBU_PK_VTAB: Table is a virtual table. +** +** Argument *piPk is also of type (int*), and also points to an output +** parameter. Unless the table has an external primary key index +** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, +** if the table does have an external primary key index, then *piPk +** is set to the root page number of the primary key index before +** returning. +** +** ALGORITHM: +** +** if( no entry exists in sqlite_schema ){ +** return RBU_PK_NOTABLE +** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ +** return RBU_PK_VTAB +** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ +** if( the index that is the pk exists in sqlite_schema ){ +** *piPK = rootpage of that index. +** return RBU_PK_EXTERNAL +** }else{ +** return RBU_PK_WITHOUT_ROWID +** } +** }else if( "PRAGMA table_info()" lists one or more "pk" columns ){ +** return RBU_PK_IPK +** }else{ +** return RBU_PK_NONE +** } +*/ +static void rbuTableType( + sqlite3rbu *p, + const char *zTab, + int *peType, + int *piTnum, + int *piPk +){ + /* + ** 0) SELECT count(*) FROM sqlite_schema where name=%Q AND IsVirtual(%Q) + ** 1) PRAGMA index_list = ? + ** 2) SELECT count(*) FROM sqlite_schema where name=%Q + ** 3) PRAGMA table_info = ? + */ + sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; + + *peType = RBU_PK_NOTABLE; + *piPk = 0; + + assert( p->rc==SQLITE_OK ); + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + sqlite3_mprintf( + "SELECT " + " (sql COLLATE nocase BETWEEN 'CREATE VIRTUAL' AND 'CREATE VIRTUAM')," + " rootpage" + " FROM sqlite_schema" + " WHERE name=%Q", zTab + )); + if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ + /* Either an error, or no such table. */ + goto rbuTableType_end; + } + if( sqlite3_column_int(aStmt[0], 0) ){ + *peType = RBU_PK_VTAB; /* virtual table */ + goto rbuTableType_end; + } + *piTnum = sqlite3_column_int(aStmt[0], 1); + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + sqlite3_mprintf("PRAGMA index_list=%Q",zTab) + ); + if( p->rc ) goto rbuTableType_end; + while( sqlite3_step(aStmt[1])==SQLITE_ROW ){ + const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); + const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); + if( zOrig && zIdx && zOrig[0]=='p' ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + sqlite3_mprintf( + "SELECT rootpage FROM sqlite_schema WHERE name = %Q", zIdx + )); + if( p->rc==SQLITE_OK ){ + if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ + *piPk = sqlite3_column_int(aStmt[2], 0); + *peType = RBU_PK_EXTERNAL; + }else{ + *peType = RBU_PK_WITHOUT_ROWID; + } + } + goto rbuTableType_end; + } + } + + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info=%Q",zTab) + ); + if( p->rc==SQLITE_OK ){ + while( sqlite3_step(aStmt[3])==SQLITE_ROW ){ + if( sqlite3_column_int(aStmt[3],5)>0 ){ + *peType = RBU_PK_IPK; /* explicit IPK column */ + goto rbuTableType_end; + } + } + *peType = RBU_PK_NONE; + } + +rbuTableType_end: { + unsigned int i; + for(i=0; iabIndexed[] array. +*/ +static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pList = 0; + int bIndex = 0; + + if( p->rc==SQLITE_OK ){ + memcpy(pIter->abIndexed, pIter->abTblPk, sizeof(u8)*pIter->nTblCol); + p->rc = prepareFreeAndCollectError(p->dbMain, &pList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + } + + pIter->nIndex = 0; + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ + const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); + sqlite3_stmt *pXInfo = 0; + if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + if( iCid==-2 ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } + } + rbuFinalize(p, pXInfo); + bIndex = 1; + pIter->nIndex++; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + /* "PRAGMA index_list" includes the main PK b-tree */ + pIter->nIndex--; + } + + rbuFinalize(p, pList); + if( bIndex==0 ) pIter->abIndexed = 0; +} + + +/* +** If they are not already populated, populate the pIter->azTblCol[], +** pIter->abTblPk[], pIter->nTblCol and pIter->bRowid variables according to +** the table (not index) that the iterator currently points to. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. If +** an error does occur, an error code and error message are also left in +** the RBU handle. +*/ +static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ + if( pIter->azTblCol==0 ){ + sqlite3_stmt *pStmt = 0; + int nCol = 0; + int i; /* for() loop iterator variable */ + int bRbuRowid = 0; /* If input table has column "rbu_rowid" */ + int iOrder = 0; + int iTnum = 0; + + /* Figure out the type of table this step will deal with. */ + assert( pIter->eType==0 ); + rbuTableType(p, pIter->zTbl, &pIter->eType, &iTnum, &pIter->iPkTnum); + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_NOTABLE ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("no such table: %s", pIter->zTbl); + } + if( p->rc ) return p->rc; + if( pIter->zIdx==0 ) pIter->iTnum = iTnum; + + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID + || pIter->eType==RBU_PK_VTAB + ); + + /* Populate the azTblCol[] and nTblCol variables based on the columns + ** of the input table. Ignore any input table columns that begin with + ** "rbu_". */ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) + ); + if( p->rc==SQLITE_OK ){ + nCol = sqlite3_column_count(pStmt); + rbuAllocateIterArrays(p, pIter, nCol); + } + for(i=0; p->rc==SQLITE_OK && irc); + pIter->aiSrcOrder[pIter->nTblCol] = pIter->nTblCol; + pIter->azTblCol[pIter->nTblCol++] = zCopy; + } + else if( 0==sqlite3_stricmp("rbu_rowid", zName) ){ + bRbuRowid = 1; + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + + if( p->rc==SQLITE_OK + && rbuIsVacuum(p)==0 + && bRbuRowid!=(pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf( + "table %q %s rbu_rowid column", pIter->zDataTbl, + (bRbuRowid ? "may not have" : "requires") + ); + } + + /* Check that all non-HIDDEN columns in the destination table are also + ** present in the input table. Populate the abTblPk[], azTblType[] and + ** aiTblOrder[] arrays at the same time. */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) + ); + } + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char*)sqlite3_column_text(pStmt, 1); + if( zName==0 ) break; /* An OOM - finalize() below returns S_NOMEM */ + for(i=iOrder; inTblCol; i++){ + if( 0==strcmp(zName, pIter->azTblCol[i]) ) break; + } + if( i==pIter->nTblCol ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("column missing from %q: %s", + pIter->zDataTbl, zName + ); + }else{ + int iPk = sqlite3_column_int(pStmt, 5); + int bNotNull = sqlite3_column_int(pStmt, 3); + const char *zType = (const char*)sqlite3_column_text(pStmt, 2); + + if( i!=iOrder ){ + SWAP(int, pIter->aiSrcOrder[i], pIter->aiSrcOrder[iOrder]); + SWAP(char*, pIter->azTblCol[i], pIter->azTblCol[iOrder]); + } + + pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); + assert( iPk>=0 ); + pIter->abTblPk[iOrder] = (u8)iPk; + pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); + iOrder++; + } + } + + rbuFinalize(p, pStmt); + rbuObjIterCacheIndexedCols(p, pIter); + assert( pIter->eType!=RBU_PK_VTAB || pIter->abIndexed==0 ); + assert( pIter->eType!=RBU_PK_VTAB || pIter->nIndex==0 ); + } + + return p->rc; +} + +/* +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the +** column names currently stored in the pIter->azTblCol[] array. +*/ +static char *rbuObjIterGetCollist( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter /* Object iterator for column names */ +){ + char *zList = 0; + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + const char *z = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"", zList, zSep, z); + zSep = ", "; + } + return zList; +} + +/* +** Return a comma separated list of the quoted PRIMARY KEY column names, +** in order, for the current table. Before each column name, add the text +** zPre. After each column name, add the zPost text. Use zSeparator as +** the separator text (usually ", "). +*/ +static char *rbuObjIterGetPkList( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + const char *zPre, /* Before each quoted column name */ + const char *zSeparator, /* Separator to use between columns */ + const char *zPost /* After each quoted column name */ +){ + int iPk = 1; + char *zRet = 0; + const char *zSep = ""; + while( 1 ){ + int i; + for(i=0; inTblCol; i++){ + if( (int)pIter->abTblPk[i]==iPk ){ + const char *zCol = pIter->azTblCol[i]; + zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost); + zSep = zSeparator; + break; + } + } + if( i==pIter->nTblCol ) break; + iPk++; + } + return zRet; +} + +/* +** This function is called as part of restarting an RBU vacuum within +** stage 1 of the process (while the *-oal file is being built) while +** updating a table (not an index). The table may be a rowid table or +** a WITHOUT ROWID table. It queries the target database to find the +** largest key that has already been written to the target table and +** constructs a WHERE clause that can be used to extract the remaining +** rows from the source table. For a rowid table, the WHERE clause +** is of the form: +** +** "WHERE _rowid_ > ?" +** +** and for WITHOUT ROWID tables: +** +** "WHERE (key1, key2) > (?, ?)" +** +** Instead of "?" placeholders, the actual WHERE clauses created by +** this function contain literal SQL values. +*/ +static char *rbuVacuumTableStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* RBU iterator object */ + int bRowid, /* True for a rowid table */ + const char *zWrite /* Target table name prefix */ +){ + sqlite3_stmt *pMax = 0; + char *zRet = 0; + if( bRowid ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0); + zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax); + } + rbuFinalize(p, pMax); + }else{ + char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC"); + char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")"); + char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1", + zSelect, zWrite, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + const char *zVal = (const char*)sqlite3_column_text(pMax, 0); + zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal); + } + rbuFinalize(p, pMax); + } + + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zList); + } + return zRet; +} + +/* +** This function is called as part of restating an RBU vacuum when the +** current operation is writing content to an index. If possible, it +** queries the target index b-tree for the largest key already written to +** it, then composes and returns an expression that can be used in a WHERE +** clause to select the remaining required rows from the source table. +** It is only possible to return such an expression if: +** +** * The index contains no DESC columns, and +** * The last key written to the index before the operation was +** suspended does not contain any NULL values. +** +** The expression is of the form: +** +** (index-field1, index-field2, ...) > (?, ?, ...) +** +** except that the "?" placeholders are replaced with literal values. +** +** If the expression cannot be created, NULL is returned. In this case, +** the caller has to use an OFFSET clause to extract only the required +** rows from the sourct table, just as it does for an RBU update operation. +*/ +static char *rbuVacuumIndexStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter /* RBU iterator object */ +){ + char *zOrder = 0; + char *zLhs = 0; + char *zSelect = 0; + char *zVector = 0; + char *zRet = 0; + int bFailed = 0; + const char *zSep = ""; + int iCol = 0; + sqlite3_stmt *pXInfo = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + if( sqlite3_column_int(pXInfo, 3) ){ + bFailed = 1; + break; + } + + if( iCid<0 ){ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "_rowid_"; + } + }else{ + zCol = pIter->azTblCol[iCid]; + } + + zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q", + zLhs, zSep, zCol, zCollate + ); + zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC", + zOrder, zSep, iCol, zCol, zCollate + ); + zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")", + zSelect, zSep, iCol, zCol + ); + zSep = ", "; + iCol++; + } + rbuFinalize(p, pXInfo); + if( bFailed ) goto index_start_out; + + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pSel = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg, + sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1", + zSelect, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){ + zSep = ""; + for(iCol=0; iColnCol; iCol++){ + const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol); + if( zQuoted==0 ){ + p->rc = SQLITE_NOMEM; + }else if( zQuoted[0]=='N' ){ + bFailed = 1; + break; + } + zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted); + zSep = ", "; + } + + if( !bFailed ){ + zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector); + } + } + rbuFinalize(p, pSel); + } + + index_start_out: + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zVector); + sqlite3_free(zLhs); + return zRet; +} + +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used +** to obtain the required information. +** +** If the index is of the following form: +** +** CREATE INDEX i1 ON t1(c, b COLLATE nocase); +** +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** "ipk", the returned string is: +** +** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" +** +** As well as the returned string, three other malloc'd strings are +** returned via output parameters. As follows: +** +** pzImposterCols: ... +** pzImposterPk: ... +** pzWhere: ... +*/ +static char *rbuObjIterGetIndexCols( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + char **pzImposterCols, /* OUT: Columns for imposter table */ + char **pzImposterPk, /* OUT: Imposter PK clause */ + char **pzWhere, /* OUT: WHERE clause */ + int *pnBind /* OUT: Trbul number of columns */ +){ + int rc = p->rc; /* Error code */ + int rc2; /* sqlite3_finalize() return code */ + char *zRet = 0; /* String to return */ + char *zImpCols = 0; /* String to return via *pzImposterCols */ + char *zImpPK = 0; /* String to return via *pzImposterPK */ + char *zWhere = 0; /* String to return via *pzWhere */ + int nBind = 0; /* Value to return via *pnBind */ + const char *zCom = ""; /* Set to ", " later on */ + const char *zAnd = ""; /* Set to " AND " later on */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = ? */ + + if( rc==SQLITE_OK ){ + assert( p->zErrmsg==0 ); + rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + } + + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol = 0; + const char *zType; + + if( iCid==-2 ){ + int iSeq = sqlite3_column_int(pXInfo, 0); + zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom, + pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate + ); + zType = ""; + }else { + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; + }else{ + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; + } + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate); + } + + if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ + const char *zOrder = (bDesc ? " DESC" : ""); + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK, zCom, nBind, zCol, zOrder + ); + } + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols, zCom, nBind, zCol, zType, zCollate + ); + zWhere = sqlite3_mprintf( + "%z%s\"rbu_imp_%d%w\" IS ?", zWhere, zAnd, nBind, zCol + ); + if( zRet==0 || zImpPK==0 || zImpCols==0 || zWhere==0 ) rc = SQLITE_NOMEM; + zCom = ", "; + zAnd = " AND "; + nBind++; + } + + rc2 = sqlite3_finalize(pXInfo); + if( rc==SQLITE_OK ) rc = rc2; + + if( rc!=SQLITE_OK ){ + sqlite3_free(zRet); + sqlite3_free(zImpCols); + sqlite3_free(zImpPK); + sqlite3_free(zWhere); + zRet = 0; + zImpCols = 0; + zImpPK = 0; + zWhere = 0; + p->rc = rc; + } + + *pzImposterCols = zImpCols; + *pzImposterPk = zImpPK; + *pzWhere = zWhere; + *pnBind = nBind; + return zRet; +} + +/* +** Assuming the current table columns are "a", "b" and "c", and the zObj +** paramter is passed "old", return a string of the form: +** +** "old.a, old.b, old.b" +** +** With the column names escaped. +** +** For tables with implicit rowids - RBU_PK_EXTERNAL and RBU_PK_NONE, append +** the text ", old._rowid_" to the returned value. +*/ +static char *rbuObjIterGetOldlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zObj +){ + char *zList = 0; + if( p->rc==SQLITE_OK && pIter->abIndexed ){ + const char *zS = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abIndexed[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = sqlite3_mprintf("%z%s%s.\"%w\"", zList, zS, zObj, zCol); + }else{ + zList = sqlite3_mprintf("%z%sNULL", zList, zS); + } + zS = ", "; + if( zList==0 ){ + p->rc = SQLITE_NOMEM; + break; + } + } + + /* For a table with implicit rowids, append "old._rowid_" to the list. */ + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "%z, %s._rowid_", zList, zObj); + } + } + return zList; +} + +/* +** Return an expression that can be used in a WHERE clause to match the +** primary key of the current table. For example, if the table is: +** +** CREATE TABLE t1(a, b, c, PRIMARY KEY(b, c)); +** +** Return the string: +** +** "b = ?1 AND c = ?2" +*/ +static char *rbuObjIterGetWhere( + sqlite3rbu *p, + RbuObjIter *pIter +){ + char *zList = 0; + if( pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE ){ + zList = rbuMPrintf(p, "_rowid_ = ?%d", pIter->nTblCol+1); + }else if( pIter->eType==RBU_PK_EXTERNAL ){ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + zList = rbuMPrintf(p, "%z%sc%d=?%d", zList, zSep, i, i+1); + zSep = " AND "; + } + } + zList = rbuMPrintf(p, + "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList + ); + + }else{ + const char *zSep = ""; + int i; + for(i=0; inTblCol; i++){ + if( pIter->abTblPk[i] ){ + const char *zCol = pIter->azTblCol[i]; + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, zCol, i+1); + zSep = " AND "; + } + } + } + return zList; +} + +/* +** The SELECT statement iterating through the keys for the current object +** (p->objiter.pSelect) currently points to a valid row. However, there +** is something wrong with the rbu_control value in the rbu_control value +** stored in the (p->nCol+1)'th column. Set the error code and error message +** of the RBU handle to something reflecting this. +*/ +static void rbuBadControlError(sqlite3rbu *p){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid rbu_control value"); +} + + +/* +** Return a nul-terminated string containing the comma separated list of +** assignments that should be included following the "SET" keyword of +** an UPDATE statement used to update the table object that the iterator +** passed as the second argument currently points to if the rbu_control +** column of the data_xxx table entry is set to zMask. +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetSetlist( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zMask +){ + char *zList = 0; + if( p->rc==SQLITE_OK ){ + int i; + + if( (int)strlen(zMask)!=pIter->nTblCol ){ + rbuBadControlError(p); + }else{ + const char *zSep = ""; + for(i=0; inTblCol; i++){ + char c = zMask[pIter->aiSrcOrder[i]]; + if( c=='x' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList, zSep, pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='d' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + else if( c=='f' ){ + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 + ); + zSep = ", "; + } + } + } + } + return zList; +} + +/* +** Return a nul-terminated string consisting of nByte comma separated +** "?" expressions. For example, if nByte is 3, return a pointer to +** a buffer containing the string "?,?,?". +** +** The memory for the returned string is obtained from sqlite3_malloc(). +** It is the responsibility of the caller to eventually free it using +** sqlite3_free(). +** +** If an OOM error is encountered when allocating space for the new +** string, an error code is left in the rbu handle passed as the first +** argument and NULL is returned. Or, if an error has already occurred +** when this function is called, NULL is returned immediately, without +** attempting the allocation or modifying the stored error code. +*/ +static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ + char *zRet = 0; + sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1; + + zRet = (char*)rbuMalloc(p, nByte); + if( zRet ){ + int i; + for(i=0; izIdx==0 ); + if( p->rc==SQLITE_OK ){ + const char *zSep = "PRIMARY KEY("; + sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXList) ){ + const char *zOrig = (const char*)sqlite3_column_text(pXList,3); + if( zOrig && strcmp(zOrig, "pk")==0 ){ + const char *zIdx = (const char*)sqlite3_column_text(pXList,1); + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + break; + } + } + rbuFinalize(p, pXList); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + if( sqlite3_column_int(pXInfo, 5) ){ + /* int iCid = sqlite3_column_int(pXInfo, 0); */ + const char *zCol = (const char*)sqlite3_column_text(pXInfo, 2); + const char *zDesc = sqlite3_column_int(pXInfo, 3) ? " DESC" : ""; + z = rbuMPrintf(p, "%z%s\"%w\"%s", z, zSep, zCol, zDesc); + zSep = ", "; + } + } + z = rbuMPrintf(p, "%z)", z); + rbuFinalize(p, pXInfo); + } + return z; +} + +/* +** This function creates the second imposter table used when writing to +** a table b-tree where the table has an external primary key. If the +** iterator passed as the second argument does not currently point to +** a table (not index) with an external primary key, this function is a +** no-op. +** +** Assuming the iterator does point to a table with an external PK, this +** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" +** used to access that PK index. For example, if the target table is +** declared as follows: +** +** CREATE TABLE t1(a, b TEXT, c REAL, PRIMARY KEY(b, c)); +** +** then the imposter table schema is: +** +** CREATE TABLE rbu_imposter2(c1 TEXT, c2 REAL, id INTEGER) WITHOUT ROWID; +** +*/ +static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType==RBU_PK_EXTERNAL ){ + int tnum = pIter->iPkTnum; /* Root page of PK index */ + sqlite3_stmt *pQuery = 0; /* SELECT name ... WHERE rootpage = $tnum */ + const char *zIdx = 0; /* Name of PK index */ + sqlite3_stmt *pXInfo = 0; /* PRAGMA main.index_xinfo = $zIdx */ + const char *zComma = ""; + char *zCols = 0; /* Used to build up list of table cols */ + char *zPk = 0; /* Used to build up table PK declaration */ + + /* Figure out the name of the primary key index for the current table. + ** This is needed for the argument to "PRAGMA index_xinfo". Set + ** zIdx to point to a nul-terminated string containing this name. */ + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_schema WHERE rootpage = ?" + ); + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(pQuery, 1, tnum); + if( SQLITE_ROW==sqlite3_step(pQuery) ){ + zIdx = (const char*)sqlite3_column_text(pQuery, 0); + } + } + if( zIdx ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) + ); + } + rbuFinalize(p, pQuery); + + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int bKey = sqlite3_column_int(pXInfo, 5); + if( bKey ){ + int iCid = sqlite3_column_int(pXInfo, 1); + int bDesc = sqlite3_column_int(pXInfo, 3); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, + iCid, pIter->azTblType[iCid], zCollate + ); + zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); + zComma = ", "; + } + } + zCols = rbuMPrintf(p, "%z, id INTEGER", zCols); + rbuFinalize(p, pXInfo); + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + zCols, zPk + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** If an error has already occurred when this function is called, it +** immediately returns zero (without doing any work). Or, if an error +** occurs during the execution of this function, it sets the error code +** in the sqlite3rbu object indicated by the first argument and returns +** zero. +** +** The iterator passed as the second argument is guaranteed to point to +** a table (not an index) when this function is called. This function +** attempts to create any imposter table required to write to the main +** table b-tree of the table before returning. Non-zero is returned if +** an imposter table are created, or zero otherwise. +** +** An imposter table is required in all cases except RBU_PK_VTAB. Only +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, +** collation sequences. For tables that do not have an external PRIMARY +** KEY, it also means the same PRIMARY KEY declaration. +*/ +static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ + if( p->rc==SQLITE_OK && pIter->eType!=RBU_PK_VTAB ){ + int tnum = pIter->iTnum; + const char *zComma = ""; + char *zSql = 0; + int iCol; + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + + for(iCol=0; p->rc==SQLITE_OK && iColnTblCol; iCol++){ + const char *zPk = ""; + const char *zCol = pIter->azTblCol[iCol]; + const char *zColl = 0; + + p->rc = sqlite3_table_column_metadata( + p->dbMain, "main", pIter->zTbl, zCol, 0, &zColl, 0, 0, 0 + ); + + if( pIter->eType==RBU_PK_IPK && pIter->abTblPk[iCol] ){ + /* If the target table column is an "INTEGER PRIMARY KEY", add + ** "PRIMARY KEY" to the imposter table column declaration. */ + zPk = "PRIMARY KEY "; + } + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", + zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, + (pIter->abNotNull[iCol] ? " NOT NULL" : "") + ); + zComma = ", "; + } + + if( pIter->eType==RBU_PK_WITHOUT_ROWID ){ + char *zPk = rbuWithoutRowidPK(p, pIter); + if( zPk ){ + zSql = rbuMPrintf(p, "%z, %z", zSql, zPk); + } + } + + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, + (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + } +} + +/* +** Prepare a statement used to insert rows into the "rbu_tmp_xxx" table. +** Specifically a statement of the form: +** +** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); +** +** The number of bound variables is equal to the number of columns in +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or +** virtual table. +*/ +static void rbuObjIterPrepareTmpInsert( + sqlite3rbu *p, + RbuObjIter *pIter, + const char *zCollist, + const char *zRbuRowid +){ + int bRbuRowid = (pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE); + char *zBind = rbuObjIterGetBindlist(p, pIter->nTblCol + 1 + bRbuRowid); + if( zBind ){ + assert( pIter->pTmpInsert==0 ); + p->rc = prepareFreeAndCollectError( + p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind + )); + } +} + +static void rbuTmpInsertFunc( + sqlite3_context *pCtx, + int nVal, + sqlite3_value **apVal +){ + sqlite3rbu *p = sqlite3_user_data(pCtx); + int rc = SQLITE_OK; + int i; + + assert( sqlite3_value_int(apVal[0])!=0 + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE + ); + if( sqlite3_value_int(apVal[0])!=0 ){ + p->nPhaseOneStep += p->objiter.nIndex; + } + + for(i=0; rc==SQLITE_OK && iobjiter.pTmpInsert, i+1, apVal[i]); + } + if( rc==SQLITE_OK ){ + sqlite3_step(p->objiter.pTmpInsert); + rc = sqlite3_reset(p->objiter.pTmpInsert); + } + + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + } +} + +static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pStmt = 0; + int rc = p->rc; + char *zRet = 0; + + assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + "SELECT trim(sql) FROM sqlite_schema WHERE type='index' AND name=?" + ); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zSql = (char*)sqlite3_column_text(pStmt, 0); + if( zSql ){ + pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc); + } + if( zSql ){ + int nParen = 0; /* Number of open parenthesis */ + int i; + int iIdxCol = 0; + int nIdxAlloc = 0; + for(i=0; zSql[i]; i++){ + char c = zSql[i]; + + /* If necessary, grow the pIter->aIdxCol[] array */ + if( iIdxCol==nIdxAlloc ){ + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( + pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + ); + if( aIdxCol==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->aIdxCol = aIdxCol; + nIdxAlloc += 16; + } + + if( c=='(' ){ + if( nParen==0 ){ + assert( iIdxCol==0 ); + pIter->aIdxCol[0].zSpan = &zSql[i+1]; + } + nParen++; + } + else if( c==')' ){ + nParen--; + if( nParen==0 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + i++; + break; + } + }else if( c==',' && nParen==1 ){ + int nSpan = &zSql[i] - pIter->aIdxCol[iIdxCol].zSpan; + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; + }else if( c=='"' || c=='\'' || c=='`' ){ + for(i++; 1; i++){ + if( zSql[i]==c ){ + if( zSql[i+1]!=c ) break; + i++; + } + } + }else if( c=='[' ){ + for(i++; 1; i++){ + if( zSql[i]==']' ) break; + } + }else if( c=='-' && zSql[i+1]=='-' ){ + for(i=i+2; zSql[i] && zSql[i]!='\n'; i++); + if( zSql[i]=='\0' ) break; + }else if( c=='/' && zSql[i+1]=='*' ){ + for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++); + if( zSql[i]=='\0' ) break; + i++; + } + } + if( zSql[i] ){ + zRet = rbuStrndup(&zSql[i], &rc); + } + pIter->nIdxCol = iIdxCol; + } + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + p->rc = rc; + return zRet; +} + +/* +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed +** as the second argument are available. +*/ +static int rbuObjIterPrepareAll( + sqlite3rbu *p, + RbuObjIter *pIter, + int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ +){ + assert( pIter->bCleanup==0 ); + if( pIter->pSelect==0 && rbuObjIterCacheTableInfo(p, pIter)==SQLITE_OK ){ + const int tnum = pIter->iTnum; + char *zCollist = 0; /* List of indexed columns */ + char **pz = &p->zErrmsg; + const char *zIdx = pIter->zIdx; + char *zLimit = 0; + + if( nOffset ){ + zLimit = sqlite3_mprintf(" LIMIT -1 OFFSET %d", nOffset); + if( !zLimit ) p->rc = SQLITE_NOMEM; + } + + if( zIdx ){ + const char *zTbl = pIter->zTbl; + char *zImposterCols = 0; /* Columns for imposter table */ + char *zImposterPK = 0; /* Primary key declaration for imposter */ + char *zWhere = 0; /* WHERE clause on PK columns */ + char *zBind = 0; + char *zPart = 0; + int nBind = 0; + + assert( pIter->eType!=RBU_PK_VTAB ); + zPart = rbuObjIterGetIndexWhere(p, pIter); + zCollist = rbuObjIterGetIndexCols( + p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind + ); + zBind = rbuObjIterGetBindlist(p, nBind); + + /* Create the imposter table used to write to this index. */ + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 1); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1,tnum); + rbuMPrintfExec(p, p->dbMain, + "CREATE TABLE \"rbu_imp_%w\"( %s, PRIMARY KEY( %s ) ) WITHOUT ROWID", + zTbl, zImposterCols, zImposterPK + ); + sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); + + /* Create the statement to insert index entries */ + pIter->nCol = nBind; + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pInsert, &p->zErrmsg, + sqlite3_mprintf("INSERT INTO \"rbu_imp_%w\" VALUES(%s)", zTbl, zBind) + ); + } + + /* And to delete index entries */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError( + p->dbMain, &pIter->pDelete, &p->zErrmsg, + sqlite3_mprintf("DELETE FROM \"rbu_imp_%w\" WHERE %s", zTbl, zWhere) + ); + } + + /* Create the SELECT statement to read keys in sorted order */ + if( p->rc==SQLITE_OK ){ + char *zSql; + if( rbuIsVacuum(p) ){ + char *zStart = 0; + if( nOffset ){ + zStart = rbuVacuumIndexStart(p, pIter); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + + zSql = sqlite3_mprintf( + "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s", + zCollist, + pIter->zDataTbl, + zPart, + (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart, + zCollist, zLimit + ); + sqlite3_free(zStart); + }else + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, + zPart, zCollist, zLimit + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s " + "UNION ALL " + "SELECT %s, rbu_control FROM '%q' " + "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 " + "ORDER BY %s%s", + zCollist, p->zStateDb, pIter->zDataTbl, zPart, + zCollist, pIter->zDataTbl, + zPart, + (zPart ? "AND" : "WHERE"), + zCollist, zLimit + ); + } + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql); + }else{ + sqlite3_free(zSql); + } + } + + sqlite3_free(zImposterCols); + sqlite3_free(zImposterPK); + sqlite3_free(zWhere); + sqlite3_free(zBind); + sqlite3_free(zPart); + }else{ + int bRbuRowid = (pIter->eType==RBU_PK_VTAB) + ||(pIter->eType==RBU_PK_NONE) + ||(pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)); + const char *zTbl = pIter->zTbl; /* Table this step applies to */ + const char *zWrite; /* Imposter table name */ + + char *zBindings = rbuObjIterGetBindlist(p, pIter->nTblCol + bRbuRowid); + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zOldlist = rbuObjIterGetOldlist(p, pIter, "old"); + char *zNewlist = rbuObjIterGetOldlist(p, pIter, "new"); + + zCollist = rbuObjIterGetCollist(p, pIter); + pIter->nCol = pIter->nTblCol; + + /* Create the imposter table or tables (if required). */ + rbuCreateImposterTable(p, pIter); + rbuCreateImposterTable2(p, pIter); + zWrite = (pIter->eType==RBU_PK_VTAB ? "" : "rbu_imp_"); + + /* Create the INSERT statement to write to the target PK b-tree */ + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, + sqlite3_mprintf( + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings + ) + ); + } + + /* Create the DELETE statement to write to the target PK b-tree. + ** Because it only performs INSERT operations, this is not required for + ** an rbu vacuum handle. */ + if( rbuIsVacuum(p)==0 && p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pDelete, pz, + sqlite3_mprintf( + "DELETE FROM \"%s%w\" WHERE %s", zWrite, zTbl, zWhere + ) + ); + } + + if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ + const char *zRbuRowid = ""; + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + zRbuRowid = ", rbu_rowid"; + } + + /* Create the rbu_tmp_xxx table and the triggers to populate it. */ + rbuMPrintfExec(p, p->dbRbu, + "CREATE TABLE IF NOT EXISTS %s.'rbu_tmp_%q' AS " + "SELECT *%s FROM '%q' WHERE 0;" + , p->zStateDb, pIter->zDataTbl + , (pIter->eType==RBU_PK_EXTERNAL ? ", 0 AS rbu_rowid" : "") + , pIter->zDataTbl + ); + + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_delete_tr BEFORE DELETE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update1_tr BEFORE UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(3, %s);" + "END;" + + "CREATE TEMP TRIGGER rbu_update2_tr AFTER UPDATE ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(4, %s);" + "END;", + zWrite, zTbl, zOldlist, + zWrite, zTbl, zOldlist, + zWrite, zTbl, zNewlist + ); + + if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ + rbuMPrintfExec(p, p->dbMain, + "CREATE TEMP TRIGGER rbu_insert_tr AFTER INSERT ON \"%s%w\" " + "BEGIN " + " SELECT rbu_tmp_insert(0, %s);" + "END;", + zWrite, zTbl, zNewlist + ); + } + + rbuObjIterPrepareTmpInsert(p, pIter, zCollist, zRbuRowid); + } + + /* Create the SELECT statement to read keys from data_xxx */ + if( p->rc==SQLITE_OK ){ + const char *zRbuRowid = ""; + char *zStart = 0; + char *zOrder = 0; + if( bRbuRowid ){ + zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; + } + + if( rbuIsVacuum(p) ){ + if( nOffset ){ + zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + if( bRbuRowid ){ + zOrder = rbuMPrintf(p, "_rowid_"); + }else{ + zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, + pIter->zDataTbl, (zStart ? zStart : ""), + (zOrder ? "ORDER BY" : ""), zOrder, + zLimit + ) + ); + } + sqlite3_free(zStart); + sqlite3_free(zOrder); + } + + sqlite3_free(zWhere); + sqlite3_free(zOldlist); + sqlite3_free(zNewlist); + sqlite3_free(zBindings); + } + sqlite3_free(zCollist); + sqlite3_free(zLimit); + } + + return p->rc; +} + +/* +** Set output variable *ppStmt to point to an UPDATE statement that may +** be used to update the imposter table for the main table b-tree of the +** table object that pIter currently points to, assuming that the +** rbu_control column of the data_xyz table contains zMask. +** +** If the zMask string does not specify any columns to update, then this +** is not an error. Output variable *ppStmt is set to NULL in this case. +*/ +static int rbuGetUpdateStmt( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* Object iterator */ + const char *zMask, /* rbu_control value ('x.x.') */ + sqlite3_stmt **ppStmt /* OUT: UPDATE statement handle */ +){ + RbuUpdateStmt **pp; + RbuUpdateStmt *pUp = 0; + int nUp = 0; + + /* In case an error occurs */ + *ppStmt = 0; + + /* Search for an existing statement. If one is found, shift it to the front + ** of the LRU queue and return immediately. Otherwise, leave nUp pointing + ** to the number of statements currently in the cache and pUp to the + ** last object in the list. */ + for(pp=&pIter->pRbuUpdate; *pp; pp=&((*pp)->pNext)){ + pUp = *pp; + if( strcmp(pUp->zMask, zMask)==0 ){ + *pp = pUp->pNext; + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + *ppStmt = pUp->pUpdate; + return SQLITE_OK; + } + nUp++; + } + assert( pUp==0 || pUp->pNext==0 ); + + if( nUp>=SQLITE_RBU_UPDATE_CACHESIZE ){ + for(pp=&pIter->pRbuUpdate; *pp!=pUp; pp=&((*pp)->pNext)); + *pp = 0; + sqlite3_finalize(pUp->pUpdate); + pUp->pUpdate = 0; + }else{ + pUp = (RbuUpdateStmt*)rbuMalloc(p, sizeof(RbuUpdateStmt)+pIter->nTblCol+1); + } + + if( pUp ){ + char *zWhere = rbuObjIterGetWhere(p, pIter); + char *zSet = rbuObjIterGetSetlist(p, pIter, zMask); + char *zUpdate = 0; + + pUp->zMask = (char*)&pUp[1]; + memcpy(pUp->zMask, zMask, pIter->nTblCol); + pUp->pNext = pIter->pRbuUpdate; + pIter->pRbuUpdate = pUp; + + if( zSet ){ + const char *zPrefix = ""; + + if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zPrefix, pIter->zTbl, zSet, zWhere + ); + p->rc = prepareFreeAndCollectError( + p->dbMain, &pUp->pUpdate, &p->zErrmsg, zUpdate + ); + *ppStmt = pUp->pUpdate; + } + sqlite3_free(zWhere); + sqlite3_free(zSet); + } + + return p->rc; +} + +static sqlite3 *rbuOpenDbhandle( + sqlite3rbu *p, + const char *zName, + int bUseVfs +){ + sqlite3 *db = 0; + if( p->rc==SQLITE_OK ){ + const int flags = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE|SQLITE_OPEN_URI; + p->rc = sqlite3_open_v2(zName, &db, flags, bUseVfs ? p->zVfsName : 0); + if( p->rc ){ + p->zErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + sqlite3_close(db); + db = 0; + } + } + return db; +} + +/* +** Free an RbuState object allocated by rbuLoadState(). +*/ +static void rbuFreeState(RbuState *p){ + if( p ){ + sqlite3_free(p->zTbl); + sqlite3_free(p->zDataTbl); + sqlite3_free(p->zIdx); + sqlite3_free(p); + } +} + +/* +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the +** responsibility of the caller to eventually free the object using +** sqlite3_free(). +** +** If an error occurs, leave an error code and message in the rbu handle +** and return NULL. +*/ +static RbuState *rbuLoadState(sqlite3rbu *p){ + RbuState *pRet = 0; + sqlite3_stmt *pStmt = 0; + int rc; + int rc2; + + pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); + if( pRet==0 ) return 0; + + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) + ); + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + switch( sqlite3_column_int(pStmt, 0) ){ + case RBU_STATE_STAGE: + pRet->eStage = sqlite3_column_int(pStmt, 1); + if( pRet->eStage!=RBU_STAGE_OAL + && pRet->eStage!=RBU_STAGE_MOVE + && pRet->eStage!=RBU_STAGE_CKPT + ){ + p->rc = SQLITE_CORRUPT; + } + break; + + case RBU_STATE_TBL: + pRet->zTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_IDX: + pRet->zIdx = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + case RBU_STATE_ROW: + pRet->nRow = sqlite3_column_int(pStmt, 1); + break; + + case RBU_STATE_PROGRESS: + pRet->nProgress = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_CKPT: + pRet->iWalCksum = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_COOKIE: + pRet->iCookie = (u32)sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_OALSZ: + pRet->iOalSz = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_PHASEONESTEP: + pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); + break; + + case RBU_STATE_DATATBL: + pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + + default: + rc = SQLITE_CORRUPT; + break; + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + + p->rc = rc; + return pRet; +} + + +/* +** Open the database handle and attach the RBU database as "rbu". If an +** error occurs, leave an error code and message in the RBU handle. +** +** If argument dbMain is not NULL, then it is a database handle already +** open on the target database. Use this handle instead of opening a new +** one. +*/ +static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); + assert( dbMain==0 || rbuIsVacuum(p)==0 ); + + /* Open the RBU database */ + p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + p->dbMain = dbMain; + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile); + } + } + + /* If using separate RBU and state databases, attach the state database to + ** the RBU db handle now. */ + if( p->zState ){ + rbuMPrintfExec(p, p->dbRbu, "ATTACH %Q AS stat", p->zState); + memcpy(p->zStateDb, "stat", 4); + }else{ + memcpy(p->zStateDb, "main", 4); + } + +#if 0 + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, 0); + } +#endif + + /* If it has not already been created, create the rbu_state table */ + rbuMPrintfExec(p, p->dbRbu, RBU_CREATE_STATE, p->zStateDb); + +#if 0 + if( rbuIsVacuum(p) ){ + if( p->rc==SQLITE_OK ){ + int rc2; + int bOk = 0; + sqlite3_stmt *pCnt = 0; + p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, + "SELECT count(*) FROM stat.sqlite_schema" + ); + if( p->rc==SQLITE_OK + && sqlite3_step(pCnt)==SQLITE_ROW + && 1==sqlite3_column_int(pCnt, 0) + ){ + bOk = 1; + } + rc2 = sqlite3_finalize(pCnt); + if( p->rc==SQLITE_OK ) p->rc = rc2; + + if( p->rc==SQLITE_OK && bOk==0 ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("invalid state database"); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); + } + } + } +#endif + + if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ + int bOpen = 0; + int rc; + p->nRbu = 0; + p->pRbuFd = 0; + rc = sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( rc!=SQLITE_NOTFOUND ) p->rc = rc; + if( p->eStage>=RBU_STAGE_MOVE ){ + bOpen = 1; + }else{ + RbuState *pState = rbuLoadState(p); + if( pState ){ + bOpen = (pState->eStage>=RBU_STAGE_MOVE); + rbuFreeState(pState); + } + } + if( bOpen ) p->dbMain = rbuOpenDbhandle(p, p->zRbu, p->nRbu<=1); + } + + p->eStage = 0; + if( p->rc==SQLITE_OK && p->dbMain==0 ){ + if( !rbuIsVacuum(p) ){ + p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); + }else{ + char *zTarget; + char *zExtra = 0; + if( strlen(p->zRbu)>=5 && 0==memcmp("file:", p->zRbu, 5) ){ + zExtra = &p->zRbu[5]; + while( *zExtra ){ + if( *zExtra++=='?' ) break; + } + if( *zExtra=='\0' ) zExtra = 0; + } + + zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", + sqlite3_db_filename(p->dbRbu, "main"), + (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) + ); + + if( zTarget==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + p->dbMain = rbuOpenDbhandle(p, zTarget, p->nRbu<=1); + sqlite3_free(zTarget); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbMain, + "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_create_function(p->dbRbu, + "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 + ); + } + + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_schema"); + + /* Mark the database file just opened as an RBU target database. If + ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. + ** This is an error. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); + } + + if( p->rc==SQLITE_NOTFOUND ){ + p->rc = SQLITE_ERROR; + p->zErrmsg = sqlite3_mprintf("rbu vfs not found"); + } +} + +/* +** This routine is a copy of the sqlite3FileSuffix3() routine from the core. +** It is a no-op unless SQLITE_ENABLE_8_3_NAMES is defined. +** +** If SQLITE_ENABLE_8_3_NAMES is set at compile-time and if the database +** filename in zBaseFilename is a URI with the "8_3_names=1" parameter and +** if filename in z[] has a suffix (a.k.a. "extension") that is longer than +** three characters, then shorten the suffix on z[] to be the last three +** characters of the original suffix. +** +** If SQLITE_ENABLE_8_3_NAMES is set to 2 at compile-time, then always +** do the suffix shortening regardless of URI parameter. +** +** Examples: +** +** test.db-journal => test.nal +** test.db-wal => test.wal +** test.db-shm => test.shm +** test.db-mj7f3319fa => test.9fa +*/ +static void rbuFileSuffix3(const char *zBase, char *z){ +#ifdef SQLITE_ENABLE_8_3_NAMES +#if SQLITE_ENABLE_8_3_NAMES<2 + if( sqlite3_uri_boolean(zBase, "8_3_names", 0) ) +#endif + { + int i, sz; + sz = (int)strlen(z)&0xffffff; + for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); + } +#endif +} + +/* +** Return the current wal-index header checksum for the target database +** as a 64-bit integer. +** +** The checksum is store in the first page of xShmMap memory as an 8-byte +** blob starting at byte offset 40. +*/ +static i64 rbuShmChecksum(sqlite3rbu *p){ + i64 iRet = 0; + if( p->rc==SQLITE_OK ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + u32 volatile *ptr; + p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); + if( p->rc==SQLITE_OK ){ + iRet = ((i64)ptr[10] << 32) + ptr[11]; + } + } + return iRet; +} + +/* +** This function is called as part of initializing or reinitializing an +** incremental checkpoint. +** +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely +** perform the copy operations directly on the file-system. +** +** If argument pState is not NULL, then the incremental checkpoint is +** being resumed. In this case, if the checksum of the wal-index-header +** following recovery is not the same as the checksum saved in the RbuState +** object, then the rbu handle is set to DONE state. This occurs if some +** other client appends a transaction to the wal file in the middle of +** an incremental checkpoint. +*/ +static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ + + /* If pState is NULL, then the wal file may not have been opened and + ** recovered. Running a read-statement here to ensure that doing so + ** does not interfere with the "capture" process below. */ + if( pState==0 ){ + p->eStage = 0; + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_schema", 0, 0, 0); + } + } + + /* Assuming no error has occurred, run a "restart" checkpoint with the + ** sqlite3rbu.eStage variable set to CAPTURE. This turns on the following + ** special behaviour in the rbu VFS: + ** + ** * If the exclusive shm WRITER or READ0 lock cannot be obtained, + ** the checkpoint fails with SQLITE_BUSY (normally SQLite would + ** proceed with running a passive checkpoint instead of failing). + ** + ** * Attempts to read from the *-wal file or write to the database file + ** do not perform any IO. Instead, the frame/page combinations that + ** would be read/written are recorded in the sqlite3rbu.aFrame[] + ** array. + ** + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** READ0 and CHECKPOINT locks taken as part of the checkpoint are + ** no-ops. These locks will not be released until the connection + ** is closed. + ** + ** * Attempting to xSync() the database file causes an SQLITE_NOTICE + ** error. + ** + ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the + ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the + ** contents of aFrame[]. + */ + if( p->rc==SQLITE_OK ){ + int rc2; + p->eStage = RBU_STAGE_CAPTURE; + rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); + if( rc2!=SQLITE_NOTICE ) p->rc = rc2; + } + + if( p->rc==SQLITE_OK && p->nFrame>0 ){ + p->eStage = RBU_STAGE_CKPT; + p->nStep = (pState ? pState->nRow : 0); + p->aBuf = rbuMalloc(p, p->pgsz); + p->iWalCksum = rbuShmChecksum(p); + } + + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); + } + } +} + +/* +** Called when iAmt bytes are read from offset iOff of the wal file while +** the rbu object is in capture mode. Record the frame number of the frame +** being read in the aFrame[] array. +*/ +static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ + const u32 mReq = (1<mLock!=mReq ){ + pRbu->rc = SQLITE_BUSY; + return SQLITE_NOTICE_RBU; + } + + pRbu->pgsz = iAmt; + if( pRbu->nFrame==pRbu->nFrameAlloc ){ + int nNew = (pRbu->nFrameAlloc ? pRbu->nFrameAlloc : 64) * 2; + RbuFrame *aNew; + aNew = (RbuFrame*)sqlite3_realloc64(pRbu->aFrame, nNew * sizeof(RbuFrame)); + if( aNew==0 ) return SQLITE_NOMEM; + pRbu->aFrame = aNew; + pRbu->nFrameAlloc = nNew; + } + + iFrame = (u32)((iOff-32) / (i64)(iAmt+24)) + 1; + if( pRbu->iMaxFrame