From 63847496f14c813a5d80efd5b7de0f1294ffe1e3 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 13 Apr 2024 16:07:11 +0200 Subject: Adding upstream version 3.45.1. Signed-off-by: Daniel Baumann --- ext/misc/README.md | 60 + ext/misc/amatch.c | 1502 +++++++++++++++++++++++ ext/misc/anycollseq.c | 58 + ext/misc/appendvfs.c | 672 +++++++++++ ext/misc/base64.c | 292 +++++ ext/misc/base85.c | 450 +++++++ ext/misc/basexx.c | 89 ++ ext/misc/blobio.c | 152 +++ ext/misc/btreeinfo.c | 430 +++++++ ext/misc/carray.c | 561 +++++++++ ext/misc/carray.h | 50 + ext/misc/cksumvfs.c | 880 ++++++++++++++ ext/misc/closure.c | 966 +++++++++++++++ ext/misc/completion.c | 502 ++++++++ ext/misc/compress.c | 131 ++ ext/misc/csv.c | 974 +++++++++++++++ ext/misc/dbdump.c | 724 +++++++++++ ext/misc/decimal.c | 884 ++++++++++++++ ext/misc/eval.c | 125 ++ ext/misc/explain.c | 323 +++++ ext/misc/fileio.c | 1031 ++++++++++++++++ ext/misc/fossildelta.c | 1093 +++++++++++++++++ ext/misc/fuzzer.c | 1192 ++++++++++++++++++ ext/misc/ieee754.c | 324 +++++ ext/misc/memstat.c | 429 +++++++ ext/misc/memtrace.c | 108 ++ ext/misc/memvfs.c | 575 +++++++++ ext/misc/mmapwarm.c | 108 ++ ext/misc/nextchar.c | 314 +++++ ext/misc/noop.c | 68 ++ ext/misc/normalize.c | 716 +++++++++++ ext/misc/pcachetrace.c | 179 +++ ext/misc/percentile.c | 220 ++++ ext/misc/prefixes.c | 321 +++++ ext/misc/qpvtab.c | 462 +++++++ ext/misc/randomjson.c | 240 ++++ ext/misc/regexp.c | 881 ++++++++++++++ ext/misc/remember.c | 72 ++ ext/misc/rot13.c | 115 ++ ext/misc/scrub.c | 610 ++++++++++ ext/misc/series.c | 585 +++++++++ ext/misc/sha1.c | 393 ++++++ ext/misc/shathree.c | 725 +++++++++++ ext/misc/showauth.c | 103 ++ ext/misc/spellfix.c | 3076 +++++++++++++++++++++++++++++++++++++++++++++++ ext/misc/sqlar.c | 125 ++ ext/misc/stmt.c | 347 ++++++ ext/misc/templatevtab.c | 269 +++++ ext/misc/totype.c | 528 ++++++++ ext/misc/uint.c | 92 ++ ext/misc/unionvtab.c | 1383 +++++++++++++++++++++ ext/misc/urifuncs.c | 209 ++++ ext/misc/uuid.c | 233 ++++ ext/misc/vfslog.c | 760 ++++++++++++ ext/misc/vfsstat.c | 825 +++++++++++++ ext/misc/vtablog.c | 511 ++++++++ ext/misc/vtshim.c | 553 +++++++++ ext/misc/wholenumber.c | 280 +++++ ext/misc/zipfile.c | 2236 ++++++++++++++++++++++++++++++++++ ext/misc/zorder.c | 102 ++ 60 files changed, 32218 insertions(+) create mode 100644 ext/misc/README.md create mode 100644 ext/misc/amatch.c create mode 100644 ext/misc/anycollseq.c create mode 100644 ext/misc/appendvfs.c create mode 100644 ext/misc/base64.c create mode 100644 ext/misc/base85.c create mode 100644 ext/misc/basexx.c create mode 100644 ext/misc/blobio.c create mode 100644 ext/misc/btreeinfo.c create mode 100644 ext/misc/carray.c create mode 100644 ext/misc/carray.h create mode 100644 ext/misc/cksumvfs.c create mode 100644 ext/misc/closure.c create mode 100644 ext/misc/completion.c create mode 100644 ext/misc/compress.c create mode 100644 ext/misc/csv.c create mode 100644 ext/misc/dbdump.c create mode 100644 ext/misc/decimal.c create mode 100644 ext/misc/eval.c create mode 100644 ext/misc/explain.c create mode 100644 ext/misc/fileio.c create mode 100644 ext/misc/fossildelta.c create mode 100644 ext/misc/fuzzer.c create mode 100644 ext/misc/ieee754.c create mode 100644 ext/misc/memstat.c create mode 100644 ext/misc/memtrace.c create mode 100644 ext/misc/memvfs.c create mode 100644 ext/misc/mmapwarm.c create mode 100644 ext/misc/nextchar.c create mode 100644 ext/misc/noop.c create mode 100644 ext/misc/normalize.c create mode 100644 ext/misc/pcachetrace.c create mode 100644 ext/misc/percentile.c create mode 100644 ext/misc/prefixes.c create mode 100644 ext/misc/qpvtab.c create mode 100644 ext/misc/randomjson.c create mode 100644 ext/misc/regexp.c create mode 100644 ext/misc/remember.c create mode 100644 ext/misc/rot13.c create mode 100644 ext/misc/scrub.c create mode 100644 ext/misc/series.c create mode 100644 ext/misc/sha1.c create mode 100644 ext/misc/shathree.c create mode 100644 ext/misc/showauth.c create mode 100644 ext/misc/spellfix.c create mode 100644 ext/misc/sqlar.c create mode 100644 ext/misc/stmt.c create mode 100644 ext/misc/templatevtab.c create mode 100644 ext/misc/totype.c create mode 100644 ext/misc/uint.c create mode 100644 ext/misc/unionvtab.c create mode 100644 ext/misc/urifuncs.c create mode 100644 ext/misc/uuid.c create mode 100644 ext/misc/vfslog.c create mode 100644 ext/misc/vfsstat.c create mode 100644 ext/misc/vtablog.c create mode 100644 ext/misc/vtshim.c create mode 100644 ext/misc/wholenumber.c create mode 100644 ext/misc/zipfile.c create mode 100644 ext/misc/zorder.c (limited to 'ext/misc') diff --git a/ext/misc/README.md b/ext/misc/README.md new file mode 100644 index 0000000..69cb230 --- /dev/null +++ b/ext/misc/README.md @@ -0,0 +1,60 @@ +## Miscellaneous Extensions + +This folder contains a collection of smaller loadable extensions. +See for instructions on how +to compile and use loadable extensions. +Each extension in this folder is implemented in a single file of C code. + +Each source file contains a description in its header comment. See the +header comments for details about each extension. Additional notes are +as follows: + + * **carray.c** — This module implements the + [carray](https://www.sqlite.org/carray.html) table-valued function. + It is a good example of how to go about implementing a custom + [table-valued function](https://www.sqlite.org/vtab.html#tabfunc2). + + * **csv.c** — A [virtual table](https://sqlite.org/vtab.html) + for reading + [Comma-Separated-Value (CSV) files](https://en.wikipedia.org/wiki/Comma-separated_values). + + * **dbdump.c** — This is not actually a loadable extension, but + rather a library that implements an approximate equivalent to the + ".dump" command of the + [command-line shell](https://www.sqlite.org/cli.html). + + * **json1.c** — Various SQL functions and table-valued functions + for processing JSON. This extension is already built into the + [SQLite amalgamation](https://sqlite.org/amalgamation.html). See + for additional information. + + * **memvfs.c** — This file implements a custom + [VFS](https://www.sqlite.org/vfs.html) that stores an entire database + file in a single block of RAM. It serves as a good example of how + to implement a simple custom VFS. + + * **rot13.c** — This file implements the very simple rot13() + substitution function. This file makes a good template for implementing + new custom SQL functions for SQLite. + + * **series.c** — This is an implementation of the + "generate_series" [virtual table](https://www.sqlite.org/vtab.html). + It can make a good template for new custom virtual table implementations. + + * **shathree.c** — An implementation of the sha3() and + sha3_query() SQL functions. The file is named "shathree.c" instead + of "sha3.c" because the default entry point names in SQLite are based + on the source filename with digits removed, so if we used the name + "sha3.c" then the entry point would conflict with the prior "sha1.c" + extension. + + * **unionvtab.c** — Implementation of the unionvtab and + [swarmvtab](https://sqlite.org/swarmvtab.html) virtual tables. + These virtual tables allow a single + large table to be spread out across multiple database files. In the + case of swarmvtab, the individual database files can be attached on + demand. + + * **zipfile.c** — A [virtual table](https://sqlite.org/vtab.html) + that can read and write a + [ZIP archive](https://en.wikipedia.org/wiki/Zip_%28file_format%29). diff --git a/ext/misc/amatch.c b/ext/misc/amatch.c new file mode 100644 index 0000000..dd9bee5 --- /dev/null +++ b/ext/misc/amatch.c @@ -0,0 +1,1502 @@ +/* +** 2013-03-14 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a demonstration virtual table that finds +** "approximate matches" - strings from a finite set that are nearly the +** same as a single input string. The virtual table is called "amatch". +** +** A amatch virtual table is created like this: +** +** CREATE VIRTUAL TABLE f USING approximate_match( +** vocabulary_table=, -- V +** vocabulary_word=, -- W +** vocabulary_language=, -- L +** edit_distances= +** ); +** +** When it is created, the new amatch table must be supplied with the +** the name of a table V and columns V.W and V.L such that +** +** SELECT W FROM V WHERE L=$language +** +** returns the allowed vocabulary for the match. If the "vocabulary_language" +** or L columnname is left unspecified or is an empty string, then no +** filtering of the vocabulary by language is performed. +** +** For efficiency, it is essential that the vocabulary table be indexed: +** +** CREATE vocab_index ON V(W) +** +** A separate edit-cost-table provides scoring information that defines +** what it means for one string to be "close" to another. +** +** The edit-cost-table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. +** +** Each row in the edit-cost-table represents a single character +** transformation going from user input to the vocabulary. The leftmost +** column of the row (column 0) contains an integer identifier of the +** language to which the transformation rule belongs (see "MULTIPLE LANGUAGES" +** below). The second column of the row (column 1) contains the input +** character or characters - the characters of user input. The third +** column contains characters as they appear in the vocabulary table. +** And the fourth column contains the integer cost of making the +** transformation. For example: +** +** CREATE TABLE f_data(iLang, cFrom, cTo, Cost); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); +** +** The first row inserted into the edit-cost-table by the SQL script +** above indicates that the cost of having an extra 'a' in the vocabulary +** table that is missing in the user input 100. (All costs are integers. +** Overall cost must not exceed 16777216.) The second INSERT statement +** creates a rule saying that the cost of having a single letter 'b' in +** user input which is missing in the vocabulary table is 87. The third +** INSERT statement mean that the cost of matching an 'o' in user input +** against an 'oe' in the vocabulary table is 38. And so forth. +** +** The following rules are special: +** +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '', 97); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '', '?', 98); +** INSERT INTO f_data(iLang, cFrom, cTo, Cost) VALUES(0, '?', '?', 99); +** +** The '?' to '' rule is the cost of having any single character in the input +** that is not found in the vocabular. The '' to '?' rule is the cost of +** having a character in the vocabulary table that is missing from input. +** And the '?' to '?' rule is the cost of doing an arbitrary character +** substitution. These three generic rules apply across all languages. +** In other words, the iLang field is ignored for the generic substitution +** rules. If more than one cost is given for a generic substitution rule, +** then the lowest cost is used. +** +** Once it has been created, the amatch virtual table can be queried +** as follows: +** +** SELECT word, distance FROM f +** WHERE word MATCH 'abcdefg' +** AND distance<200; +** +** This query outputs the strings contained in the T(F) field that +** are close to "abcdefg" and in order of increasing distance. No string +** is output more than once. If there are multiple ways to transform the +** target string ("abcdefg") into a string in the vocabulary table then +** the lowest cost transform is the one that is returned. In this example, +** the search is limited to strings with a total distance of less than 200. +** +** For efficiency, it is important to put tight bounds on the distance. +** The time and memory space needed to perform this query is exponential +** in the maximum distance. A good rule of thumb is to limit the distance +** to no more than 1.5 or 2 times the maximum cost of any rule in the +** edit-cost-table. +** +** The amatch is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a amatch table will throw an error. +** +** It is important to put some kind of a limit on the amatch output. This +** can be either in the form of a LIMIT clause at the end of the query, +** or better, a "distance +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct amatch_vtab amatch_vtab; +typedef struct amatch_cursor amatch_cursor; +typedef struct amatch_rule amatch_rule; +typedef struct amatch_word amatch_word; +typedef struct amatch_avl amatch_avl; + + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct amatch_avl { + amatch_word *pWord; /* Points to the object being stored in the tree */ + char *zKey; /* Key. zero-terminated string. Must be unique */ + amatch_avl *pBefore; /* Other elements less than zKey */ + amatch_avl *pAfter; /* Other elements greater than zKey */ + amatch_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the amatch_avl.height and amatch_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void amatchAvlRecomputeHeight(amatch_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static amatch_avl *amatchAvlRotateBefore(amatch_avl *pP){ + amatch_avl *pB = pP->pBefore; + amatch_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static amatch_avl *amatchAvlRotateAfter(amatch_avl *pP){ + amatch_avl *pA = pP->pAfter; + amatch_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + amatchAvlRecomputeHeight(pP); + amatchAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static amatch_avl **amatchAvlFromPtr(amatch_avl *p, amatch_avl **pp){ + amatch_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static amatch_avl *amatchAvlBalance(amatch_avl *p){ + amatch_avl *pTop = p; + amatch_avl **pp; + while( p ){ + amatchAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + amatch_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = amatchAvlRotateAfter(pB); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + amatch_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = amatchAvlRotateBefore(pA); + pp = amatchAvlFromPtr(p,&p); + p = *pp = amatchAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with zKey. Return a pointer +** to the entry or return NULL. +*/ +static amatch_avl *amatchAvlSearch(amatch_avl *p, const char *zKey){ + int c; + while( p && (c = strcmp(zKey, p->zKey))!=0 ){ + p = (c<0) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static amatch_avl *amatchAvlFirst(amatch_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +#if 0 /* NOT USED */ +/* Return the node with the next larger key after p. +*/ +static amatch_avl *amatchAvlNext(amatch_avl *p){ + amatch_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = amatchAvlFirst(p->pAfter); + } + return p; +} +#endif + +#if 0 /* NOT USED */ +/* Verify AVL tree integrity +*/ +static int amatchAvlIntegrity(amatch_avl *pHead){ + amatch_avl *p; + if( pHead==0 ) return 1; + if( (p = pHead->pBefore)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)<0 ); + while( p->pAfter ) p = p->pAfter; + assert( strcmp(p->zKey, pHead->zKey)<0 ); + } + if( (p = pHead->pAfter)!=0 ){ + assert( p->pUp==pHead ); + assert( amatchAvlIntegrity(p) ); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + p = amatchAvlFirst(p); + assert( strcmp(p->zKey, pHead->zKey)>0 ); + } + return 1; +} +static int amatchAvlIntegrity2(amatch_avl *pHead){ + amatch_avl *p, *pNext; + for(p=amatchAvlFirst(pHead); p; p=pNext){ + pNext = amatchAvlNext(p); + if( pNext==0 ) break; + assert( strcmp(p->zKey, pNext->zKey)<0 ); + } + return 1; +} +#endif + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static amatch_avl *amatchAvlInsert(amatch_avl **ppHead, amatch_avl *pNew){ + int c; + amatch_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + c = strcmp(pNew->zKey, p->zKey); + if( c<0 ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( c>0 ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = amatchAvlBalance(p); + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ + return 0; +} + +/* Remove node pOld from the tree. pOld must be an element of the tree or +** the AVL tree will become corrupt. +*/ +static void amatchAvlRemove(amatch_avl **ppHead, amatch_avl *pOld){ + amatch_avl **ppParent; + amatch_avl *pBalance = 0; + /* assert( amatchAvlSearch(*ppHead, pOld->zKey)==pOld ); */ + ppParent = amatchAvlFromPtr(pOld, ppHead); + if( pOld->pBefore==0 && pOld->pAfter==0 ){ + *ppParent = 0; + pBalance = pOld->pUp; + }else if( pOld->pBefore && pOld->pAfter ){ + amatch_avl *pX, *pY; + pX = amatchAvlFirst(pOld->pAfter); + *amatchAvlFromPtr(pX, 0) = pX->pAfter; + if( pX->pAfter ) pX->pAfter->pUp = pX->pUp; + pBalance = pX->pUp; + pX->pAfter = pOld->pAfter; + if( pX->pAfter ){ + pX->pAfter->pUp = pX; + }else{ + assert( pBalance==pOld ); + pBalance = pX; + } + pX->pBefore = pY = pOld->pBefore; + if( pY ) pY->pUp = pX; + pX->pUp = pOld->pUp; + *ppParent = pX; + }else if( pOld->pBefore==0 ){ + *ppParent = pBalance = pOld->pAfter; + pBalance->pUp = pOld->pUp; + }else if( pOld->pAfter==0 ){ + *ppParent = pBalance = pOld->pBefore; + pBalance->pUp = pOld->pUp; + } + *ppHead = amatchAvlBalance(pBalance); + pOld->pUp = 0; + pOld->pBefore = 0; + pOld->pAfter = 0; + /* assert( amatchAvlIntegrity(*ppHead) ); */ + /* assert( amatchAvlIntegrity2(*ppHead) ); */ +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + + +/* +** Various types. +** +** amatch_cost is the "cost" of an edit operation. +** +** amatch_len is the length of a matching string. +** +** amatch_langid is an ruleset identifier. +*/ +typedef int amatch_cost; +typedef signed char amatch_len; +typedef int amatch_langid; + +/* +** Limits +*/ +#define AMATCH_MX_LENGTH 50 /* Maximum length of a rule string */ +#define AMATCH_MX_LANGID 2147483647 /* Maximum rule ID */ +#define AMATCH_MX_COST 1000 /* Maximum single-rule cost */ + +/* +** A match or partial match +*/ +struct amatch_word { + amatch_word *pNext; /* Next on a list of all amatch_words */ + amatch_avl sCost; /* Linkage of this node into the cost tree */ + amatch_avl sWord; /* Linkage of this node into the word tree */ + amatch_cost rCost; /* Cost of the match so far */ + int iSeq; /* Sequence number */ + char zCost[10]; /* Cost key (text rendering of rCost) */ + short int nMatch; /* Input characters matched */ + char zWord[4]; /* Text of the word. Extra space appended as needed */ +}; + +/* +** Each transformation rule is stored as an instance of this object. +** All rules are kept on a linked list sorted by rCost. +*/ +struct amatch_rule { + amatch_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from (a string from user input) */ + amatch_cost rCost; /* Cost of this transformation */ + amatch_langid iLang; /* The langauge to which this rule belongs */ + amatch_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + char zTo[4]; /* Tranform to V.W value (extra space appended) */ +}; + +/* +** A amatch virtual-table object +*/ +struct amatch_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zClassName; /* Name of this class. Default: "amatch" */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zCostTab; /* Name of edit-cost-table */ + char *zVocabTab; /* Name of vocabulary table */ + char *zVocabWord; /* Name of vocabulary table word column */ + char *zVocabLang; /* Name of vocabulary table language column */ + amatch_rule *pRule; /* All active rules in this amatch */ + amatch_cost rIns; /* Generic insertion cost '' -> ? */ + amatch_cost rDel; /* Generic deletion cost ? -> '' */ + amatch_cost rSub; /* Generic substitution cost ? -> ? */ + sqlite3 *db; /* The database connection */ + sqlite3_stmt *pVCheck; /* Query to check zVocabTab */ + int nCursor; /* Number of active cursors */ +}; + +/* A amatch cursor object */ +struct amatch_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid of the current word */ + amatch_langid iLang; /* Use this language ID */ + amatch_cost rLimit; /* Maximum cost of any term */ + int nBuf; /* Space allocated for zBuf */ + int oomErr; /* True following an OOM error */ + int nWord; /* Number of amatch_word objects */ + char *zBuf; /* Temp-use buffer space */ + char *zInput; /* Input word to match against */ + amatch_vtab *pVtab; /* The virtual table this cursor belongs to */ + amatch_word *pAllWords; /* List of all amatch_word objects */ + amatch_word *pCurrent; /* Most recent solution */ + amatch_avl *pCost; /* amatch_word objects keyed by iCost */ + amatch_avl *pWord; /* amatch_word objects keyed by zWord */ +}; + +/* +** The two input rule lists are both sorted in order of increasing +** cost. Merge them together into a single list, sorted by cost, and +** return a pointer to the head of that list. +*/ +static amatch_rule *amatchMergeRules(amatch_rule *pA, amatch_rule *pB){ + amatch_rule head; + amatch_rule *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCost<=pB->rCost ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Statement pStmt currently points to a row in the amatch data table. This +** function allocates and populates a amatch_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int amatchLoadOneRule( + amatch_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + amatch_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iLang = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + amatch_cost rCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + amatch_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + if( zFrom[0]=='?' && zFrom[1]==0 ){ + if( p->rSub==0 || p->rSub>rCost ) p->rSub = rCost; + } + *ppRule = 0; + return SQLITE_OK; + } + + if( rCost<=0 || rCost>AMATCH_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, AMATCH_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>AMATCH_MX_LENGTH || nTo>AMATCH_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, AMATCH_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iLang<0 || iLang>AMATCH_MX_LANGID ){ + *pzErr = sqlite3_mprintf("%s: iLang must be between 0 and %d", + p->zClassName, AMATCH_MX_LANGID + ); + rc = SQLITE_ERROR; + }else + if( strcmp(zFrom,"")==0 && strcmp(zTo,"?")==0 ){ + if( p->rIns==0 || p->rIns>rCost ) p->rIns = rCost; + }else + if( strcmp(zFrom,"?")==0 && strcmp(zTo,"")==0 ){ + if( p->rDel==0 || p->rDel>rCost ) p->rDel = rCost; + }else + { + pRule = sqlite3_malloc64( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = &pRule->zTo[nTo+1]; + pRule->nFrom = (amatch_len)nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = (amatch_len)nTo; + pRule->rCost = rCost; + pRule->iLang = (int)iLang; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Free all the content in the edit-cost-table +*/ +static void amatchFreeRules(amatch_vtab *p){ + while( p->pRule ){ + amatch_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + p->pRule = 0; +} + +/* +** Load the content of the amatch data table into memory. +*/ +static int amatchLoadRules( + sqlite3 *db, /* Database handle */ + amatch_vtab *p, /* Virtual amatch table to configure */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + amatch_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", p->zDb, p->zCostTab); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, p->zCostTab, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + amatch_rule *pRule = 0; + rc = amatchLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets amatch_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + amatch_rule *pX; + amatch_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = amatchMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *amatchDequote(const char *zIn){ + sqlite3_int64 nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc64(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, (size_t)(nIn+1)); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInpVCheck ){ + sqlite3_finalize(p->pVCheck); + p->pVCheck = 0; + } +} + +/* +** Deallocate an amatch_vtab object +*/ +static void amatchFree(amatch_vtab *p){ + if( p ){ + amatchFreeRules(p); + amatchVCheckClear(p); + sqlite3_free(p->zClassName); + sqlite3_free(p->zDb); + sqlite3_free(p->zCostTab); + sqlite3_free(p->zVocabTab); + sqlite3_free(p->zVocabWord); + sqlite3_free(p->zVocabLang); + sqlite3_free(p->zSelf); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the amatch module. +*/ +static int amatchDisconnect(sqlite3_vtab *pVtab){ + amatch_vtab *p = (amatch_vtab*)pVtab; + assert( p->nCursor==0 ); + amatchFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *amatchValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("approximate_match") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int amatchConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + amatch_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zClassName = sqlite3_mprintf("%s", zModule); + if( pNew->zClassName==0 ) goto amatchConnectError; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto amatchConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto amatchConnectError; + for(i=3; izVocabTab); + pNew->zVocabTab = amatchDequote(zVal); + if( pNew->zVocabTab==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_word", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabWord); + pNew->zVocabWord = amatchDequote(zVal); + if( pNew->zVocabWord==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("vocabulary_language", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zVocabLang); + pNew->zVocabLang = amatchDequote(zVal); + if( pNew->zVocabLang==0 ) goto amatchConnectError; + continue; + } + zVal = amatchValueOfKey("edit_distances", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zCostTab); + pNew->zCostTab = amatchDequote(zVal); + if( pNew->zCostTab==0 ) goto amatchConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + amatchFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = SQLITE_OK; + if( pNew->zCostTab==0 ){ + *pzErr = sqlite3_mprintf("no edit_distances table specified"); + rc = SQLITE_ERROR; + }else{ + rc = amatchLoadRules(db, pNew, pzErr); + } + if( rc==SQLITE_OK ){ + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(word,distance,language," + "command HIDDEN,nword HIDDEN)" + ); +#define AMATCH_COL_WORD 0 +#define AMATCH_COL_DISTANCE 1 +#define AMATCH_COL_LANGUAGE 2 +#define AMATCH_COL_COMMAND 3 +#define AMATCH_COL_NWORD 4 + } + if( rc!=SQLITE_OK ){ + amatchFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +amatchConnectError: + amatchFree(pNew); + return rc; +} + +/* +** Open a new amatch cursor. +*/ +static int amatchOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + amatch_vtab *p = (amatch_vtab*)pVTab; + amatch_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void amatchClearCursor(amatch_cursor *pCur){ + amatch_word *pWord, *pNextWord; + for(pWord=pCur->pAllWords; pWord; pWord=pNextWord){ + pNextWord = pWord->pNext; + sqlite3_free(pWord); + } + pCur->pAllWords = 0; + sqlite3_free(pCur->zInput); + pCur->zInput = 0; + sqlite3_free(pCur->zBuf); + pCur->zBuf = 0; + pCur->nBuf = 0; + pCur->pCost = 0; + pCur->pWord = 0; + pCur->pCurrent = 0; + pCur->rLimit = 1000000; + pCur->iLang = 0; + pCur->nWord = 0; +} + +/* +** Close a amatch cursor. +*/ +static int amatchClose(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor *)cur; + amatchClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Render a 24-bit unsigned integer as a 4-byte base-64 number. +*/ +static void amatchEncodeInt(int x, char *z){ + static const char a[] = + "0123456789" + "ABCDEFGHIJ" + "KLMNOPQRST" + "UVWXYZ^abc" + "defghijklm" + "nopqrstuvw" + "xyz~"; + z[0] = a[(x>>18)&0x3f]; + z[1] = a[(x>>12)&0x3f]; + z[2] = a[(x>>6)&0x3f]; + z[3] = a[x&0x3f]; +} + +/* +** Write the zCost[] field for a amatch_word object +*/ +static void amatchWriteCost(amatch_word *pWord){ + amatchEncodeInt(pWord->rCost, pWord->zCost); + amatchEncodeInt(pWord->iSeq, pWord->zCost+4); + pWord->zCost[8] = 0; +} + +/* Circumvent compiler warnings about the use of strcpy() by supplying +** our own implementation. +*/ +static void amatchStrcpy(char *dest, const char *src){ + while( (*(dest++) = *(src++))!=0 ){} +} +static void amatchStrcat(char *dest, const char *src){ + while( *dest ) dest++; + amatchStrcpy(dest, src); +} + +/* +** Add a new amatch_word object to the queue. +** +** If a prior amatch_word object with the same zWord, and nMatch +** already exists, update its rCost (if the new rCost is less) but +** otherwise leave it unchanged. Do not add a duplicate. +** +** Do nothing if the cost exceeds threshold. +*/ +static void amatchAddWord( + amatch_cursor *pCur, + amatch_cost rCost, + int nMatch, + const char *zWordBase, + const char *zWordTail +){ + amatch_word *pWord; + amatch_avl *pNode; + amatch_avl *pOther; + int nBase, nTail; + char zBuf[4]; + + if( rCost>pCur->rLimit ){ + return; + } + nBase = (int)strlen(zWordBase); + nTail = (int)strlen(zWordTail); + if( nBase+nTail+3>pCur->nBuf ){ + pCur->nBuf = nBase+nTail+100; + pCur->zBuf = sqlite3_realloc(pCur->zBuf, pCur->nBuf); + if( pCur->zBuf==0 ){ + pCur->nBuf = 0; + return; + } + } + amatchEncodeInt(nMatch, zBuf); + memcpy(pCur->zBuf, zBuf+2, 2); + memcpy(pCur->zBuf+2, zWordBase, nBase); + memcpy(pCur->zBuf+2+nBase, zWordTail, nTail+1); + pNode = amatchAvlSearch(pCur->pWord, pCur->zBuf); + if( pNode ){ + pWord = pNode->pWord; + if( pWord->rCost>rCost ){ +#ifdef AMATCH_TRACE_1 + printf("UPDATE [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + pWord->rCost = rCost; + amatchWriteCost(pWord); +#ifdef AMATCH_TRACE_1 + printf(" ---> %d (\"%s\" \"%s\")\n", + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + } + return; + } + pWord = sqlite3_malloc64( sizeof(*pWord) + nBase + nTail - 1 ); + if( pWord==0 ) return; + memset(pWord, 0, sizeof(*pWord)); + pWord->rCost = rCost; + pWord->iSeq = pCur->nWord++; + amatchWriteCost(pWord); + pWord->nMatch = (short)nMatch; + pWord->pNext = pCur->pAllWords; + pCur->pAllWords = pWord; + pWord->sCost.zKey = pWord->zCost; + pWord->sCost.pWord = pWord; + pOther = amatchAvlInsert(&pCur->pCost, &pWord->sCost); + assert( pOther==0 ); (void)pOther; + pWord->sWord.zKey = pWord->zWord; + pWord->sWord.pWord = pWord; + amatchStrcpy(pWord->zWord, pCur->zBuf); + pOther = amatchAvlInsert(&pCur->pWord, &pWord->sWord); + assert( pOther==0 ); (void)pOther; +#ifdef AMATCH_TRACE_1 + printf("INSERT [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", pWord->zWord+2, + pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, rCost, + pWord->zWord, pWord->zCost); +#endif +} + + +/* +** Advance a cursor to its next row of output +*/ +static int amatchNext(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + amatch_word *pWord = 0; + amatch_avl *pNode; + int isMatch = 0; + amatch_vtab *p = pCur->pVtab; + int nWord; + int rc; + int i; + const char *zW; + amatch_rule *pRule; + char *zBuf = 0; + char nBuf = 0; + char zNext[8]; + char zNextIn[8]; + int nNextIn; + + if( p->pVCheck==0 ){ + char *zSql; + if( p->zVocabLang && p->zVocabLang[0] ){ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"", + " WHERE \"%w\">=?1 AND \"%w\"=?2" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord, p->zVocabLang + ); + }else{ + zSql = sqlite3_mprintf( + "SELECT \"%w\" FROM \"%w\"" + " WHERE \"%w\">=?1" + " ORDER BY 1", + p->zVocabWord, p->zVocabTab, + p->zVocabWord + ); + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &p->pVCheck, 0); + sqlite3_free(zSql); + if( rc ) return rc; + } + sqlite3_bind_int(p->pVCheck, 2, pCur->iLang); + + do{ + pNode = amatchAvlFirst(pCur->pCost); + if( pNode==0 ){ + pWord = 0; + break; + } + pWord = pNode->pWord; + amatchAvlRemove(&pCur->pCost, &pWord->sCost); + +#ifdef AMATCH_TRACE_1 + printf("PROCESS [%s][%.*s^%s] %d (\"%s\" \"%s\")\n", + pWord->zWord+2, pWord->nMatch, pCur->zInput, pCur->zInput+pWord->nMatch, + pWord->rCost, pWord->zWord, pWord->zCost); +#endif + nWord = (int)strlen(pWord->zWord+2); + if( nWord+20>nBuf ){ + nBuf = (char)(nWord+100); + zBuf = sqlite3_realloc(zBuf, nBuf); + if( zBuf==0 ) return SQLITE_NOMEM; + } + amatchStrcpy(zBuf, pWord->zWord+2); + zNext[0] = 0; + zNextIn[0] = pCur->zInput[pWord->nMatch]; + if( zNextIn[0] ){ + for(i=1; i<=4 && (pCur->zInput[pWord->nMatch+i]&0xc0)==0x80; i++){ + zNextIn[i] = pCur->zInput[pWord->nMatch+i]; + } + zNextIn[i] = 0; + nNextIn = i; + }else{ + nNextIn = 0; + } + + if( zNextIn[0] && zNextIn[0]!='*' ){ + sqlite3_reset(p->pVCheck); + amatchStrcat(zBuf, zNextIn); + sqlite3_bind_text(p->pVCheck, 1, zBuf, nWord+nNextIn, SQLITE_STATIC); + rc = sqlite3_step(p->pVCheck); + if( rc==SQLITE_ROW ){ + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + if( strncmp(zBuf, zW, nWord+nNextIn)==0 ){ + amatchAddWord(pCur, pWord->rCost, pWord->nMatch+nNextIn, zBuf, ""); + } + } + zBuf[nWord] = 0; + } + + while( 1 ){ + amatchStrcpy(zBuf+nWord, zNext); + sqlite3_reset(p->pVCheck); + sqlite3_bind_text(p->pVCheck, 1, zBuf, -1, SQLITE_TRANSIENT); + rc = sqlite3_step(p->pVCheck); + if( rc!=SQLITE_ROW ) break; + zW = (const char*)sqlite3_column_text(p->pVCheck, 0); + amatchStrcpy(zBuf+nWord, zNext); + if( strncmp(zW, zBuf, nWord)!=0 ) break; + if( (zNextIn[0]=='*' && zNextIn[1]==0) + || (zNextIn[0]==0 && zW[nWord]==0) + ){ + isMatch = 1; + zNextIn[0] = 0; + nNextIn = 0; + break; + } + zNext[0] = zW[nWord]; + for(i=1; i<=4 && (zW[nWord+i]&0xc0)==0x80; i++){ + zNext[i] = zW[nWord+i]; + } + zNext[i] = 0; + zBuf[nWord] = 0; + if( p->rIns>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rIns, pWord->nMatch, + zBuf, zNext); + } + if( p->rSub>0 ){ + amatchAddWord(pCur, pWord->rCost+p->rSub, pWord->nMatch+nNextIn, + zBuf, zNext); + } + if( p->rIns<0 && p->rSub<0 ) break; + zNext[i-1]++; /* FIX ME */ + } + sqlite3_reset(p->pVCheck); + + if( p->rDel>0 ){ + zBuf[nWord] = 0; + amatchAddWord(pCur, pWord->rCost+p->rDel, pWord->nMatch+nNextIn, + zBuf, ""); + } + + for(pRule=p->pRule; pRule; pRule=pRule->pNext){ + if( pRule->iLang!=pCur->iLang ) continue; + if( strncmp(pRule->zFrom, pCur->zInput+pWord->nMatch, pRule->nFrom)==0 ){ + amatchAddWord(pCur, pWord->rCost+pRule->rCost, + pWord->nMatch+pRule->nFrom, pWord->zWord+2, pRule->zTo); + } + } + }while( !isMatch ); + pCur->pCurrent = pWord; + sqlite3_free(zBuf); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any amatchColumn, amatchRowid, or amatchEof call. +*/ +static int amatchFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + amatch_cursor *pCur = (amatch_cursor *)pVtabCursor; + const char *zWord = "*"; + int idx; + + amatchClearCursor(pCur); + idx = 0; + if( idxNum & 1 ){ + zWord = (const char*)sqlite3_value_text(argv[0]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iLang = (amatch_cost)sqlite3_value_int(argv[idx]); + idx++; + } + pCur->zInput = sqlite3_mprintf("%s", zWord); + if( pCur->zInput==0 ) return SQLITE_NOMEM; + amatchAddWord(pCur, 0, 0, "", ""); + amatchNext(pVtabCursor); + + return SQLITE_OK; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int amatchColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + amatch_cursor *pCur = (amatch_cursor*)cur; + switch( i ){ + case AMATCH_COL_WORD: { + sqlite3_result_text(ctx, pCur->pCurrent->zWord+2, -1, SQLITE_STATIC); + break; + } + case AMATCH_COL_DISTANCE: { + sqlite3_result_int(ctx, pCur->pCurrent->rCost); + break; + } + case AMATCH_COL_LANGUAGE: { + sqlite3_result_int(ctx, pCur->iLang); + break; + } + case AMATCH_COL_NWORD: { + sqlite3_result_int(ctx, pCur->nWord); + break; + } + default: { + sqlite3_result_null(ctx); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int amatchRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + amatch_cursor *pCur = (amatch_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int amatchEof(sqlite3_vtab_cursor *cur){ + amatch_cursor *pCur = (amatch_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) language == $language +** +** The distance< and distance<= are both treated as distance<=. +** The query plan number is a bit vector: +** +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. +*/ +static int amatchBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int iPlan = 0; + int iDistTerm = -1; + int iLangTerm = -1; + int i; + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 2)==0 + && pConstraint->iColumn==1 + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) + ){ + iPlan |= 2; + iDistTerm = i; + } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iLangTerm = i; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iLangTerm].argvIndex = idx; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==1 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = (double)10000; + + return SQLITE_OK; +} + +/* +** The xUpdate() method. +** +** This implementation disallows DELETE and UPDATE. The only thing +** allowed is INSERT into the "command" column. +*/ +static int amatchUpdate( + sqlite3_vtab *pVTab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + amatch_vtab *p = (amatch_vtab*)pVTab; + const unsigned char *zCmd; + (void)pRowid; + if( argc==1 ){ + pVTab->zErrMsg = sqlite3_mprintf("DELETE from %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pVTab->zErrMsg = sqlite3_mprintf("UPDATE of %s is not allowed", + p->zSelf); + return SQLITE_ERROR; + } + if( sqlite3_value_type(argv[2+AMATCH_COL_WORD])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_DISTANCE])!=SQLITE_NULL + || sqlite3_value_type(argv[2+AMATCH_COL_LANGUAGE])!=SQLITE_NULL + ){ + pVTab->zErrMsg = sqlite3_mprintf( + "INSERT INTO %s allowed for column [command] only", p->zSelf); + return SQLITE_ERROR; + } + zCmd = sqlite3_value_text(argv[2+AMATCH_COL_COMMAND]); + if( zCmd==0 ) return SQLITE_OK; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "approximate_match". +*/ +static sqlite3_module amatchModule = { + 0, /* iVersion */ + amatchConnect, /* xCreate */ + amatchConnect, /* xConnect */ + amatchBestIndex, /* xBestIndex */ + amatchDisconnect, /* xDisconnect */ + amatchDisconnect, /* xDestroy */ + amatchOpen, /* xOpen - open a cursor */ + amatchClose, /* xClose - close a cursor */ + amatchFilter, /* xFilter - configure scan constraints */ + amatchNext, /* xNext - advance a cursor */ + amatchEof, /* xEof - check for end of scan */ + amatchColumn, /* xColumn - read data */ + amatchRowid, /* xRowid - read data */ + amatchUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Register the amatch virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_amatch_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Not used */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "approximate_match", &amatchModule, 0); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + return rc; +} diff --git a/ext/misc/anycollseq.c b/ext/misc/anycollseq.c new file mode 100644 index 0000000..27b7049 --- /dev/null +++ b/ext/misc/anycollseq.c @@ -0,0 +1,58 @@ +/* +** 2017-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a run-time loadable extension to SQLite that +** registers a sqlite3_collation_needed() callback to register a fake +** collating function for any unknown collating sequence. The fake +** collating function works like BINARY. +** +** This extension can be used to load schemas that contain one or more +** unknown collating sequences. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +static int anyCollFunc( + void *NotUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + int rc, n; + n = nKey1 +#include + +/* The append mark at the end of the database is: +** +** Start-Of-SQLite3-NNNNNNNN +** 123456789 123456789 12345 +** +** The NNNNNNNN represents a 64-bit big-endian unsigned integer which is +** the offset to page 1, and also the length of the prefix content. +*/ +#define APND_MARK_PREFIX "Start-Of-SQLite3-" +#define APND_MARK_PREFIX_SZ 17 +#define APND_MARK_FOS_SZ 8 +#define APND_MARK_SIZE (APND_MARK_PREFIX_SZ+APND_MARK_FOS_SZ) + +/* +** Maximum size of the combined prefix + database + append-mark. This +** must be less than 0x40000000 to avoid locking issues on Windows. +*/ +#define APND_MAX_SIZE (0x40000000) + +/* +** Try to align the database to an even multiple of APND_ROUNDUP bytes. +*/ +#ifndef APND_ROUNDUP +#define APND_ROUNDUP 4096 +#endif +#define APND_ALIGN_MASK ((sqlite3_int64)(APND_ROUNDUP-1)) +#define APND_START_ROUNDUP(fsz) (((fsz)+APND_ALIGN_MASK) & ~APND_ALIGN_MASK) + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs ApndVfs; +typedef struct ApndFile ApndFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((ApndFile*)(p))+1)) + +/* An open appendvfs file +** +** An instance of this structure describes the appended database file. +** A separate sqlite3_file object is always appended. The appended +** sqlite3_file object (which can be accessed using ORIGFILE()) describes +** the entire file, including the prefix, the database, and the +** append-mark. +** +** The structure of an AppendVFS database is like this: +** +** +-------------+---------+----------+-------------+ +** | prefix-file | padding | database | append-mark | +** +-------------+---------+----------+-------------+ +** ^ ^ +** | | +** iPgOne iMark +** +** +** "prefix file" - file onto which the database has been appended. +** "padding" - zero or more bytes inserted so that "database" +** starts on an APND_ROUNDUP boundary +** "database" - The SQLite database file +** "append-mark" - The 25-byte "Start-Of-SQLite3-NNNNNNNN" that indicates +** the offset from the start of prefix-file to the start +** of "database". +** +** The size of the database is iMark - iPgOne. +** +** The NNNNNNNN in the "Start-Of-SQLite3-NNNNNNNN" suffix is the value +** of iPgOne stored as a big-ending 64-bit integer. +** +** iMark will be the size of the underlying file minus 25 (APND_MARKSIZE). +** Or, iMark is -1 to indicate that it has not yet been written. +*/ +struct ApndFile { + sqlite3_file base; /* Subclass. MUST BE FIRST! */ + sqlite3_int64 iPgOne; /* Offset to the start of the database */ + sqlite3_int64 iMark; /* Offset of the append mark. -1 if unwritten */ + /* Always followed by another sqlite3_file that describes the whole file */ +}; + +/* +** Methods for ApndFile +*/ +static int apndClose(sqlite3_file*); +static int apndRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int apndWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int apndTruncate(sqlite3_file*, sqlite3_int64 size); +static int apndSync(sqlite3_file*, int flags); +static int apndFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int apndLock(sqlite3_file*, int); +static int apndUnlock(sqlite3_file*, int); +static int apndCheckReservedLock(sqlite3_file*, int *pResOut); +static int apndFileControl(sqlite3_file*, int op, void *pArg); +static int apndSectorSize(sqlite3_file*); +static int apndDeviceCharacteristics(sqlite3_file*); +static int apndShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int apndShmLock(sqlite3_file*, int offset, int n, int flags); +static void apndShmBarrier(sqlite3_file*); +static int apndShmUnmap(sqlite3_file*, int deleteFlag); +static int apndFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int apndUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for ApndVfs +*/ +static int apndOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int apndDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int apndAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int apndFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *apndDlOpen(sqlite3_vfs*, const char *zFilename); +static void apndDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void apndDlClose(sqlite3_vfs*, void*); +static int apndRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int apndSleep(sqlite3_vfs*, int microseconds); +static int apndCurrentTime(sqlite3_vfs*, double*); +static int apndGetLastError(sqlite3_vfs*, int, char *); +static int apndCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int apndSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr apndGetSystemCall(sqlite3_vfs*, const char *z); +static const char *apndNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs apnd_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "apndvfs", /* zName */ + 0, /* pAppData (set when registered) */ + apndOpen, /* xOpen */ + apndDelete, /* xDelete */ + apndAccess, /* xAccess */ + apndFullPathname, /* xFullPathname */ + apndDlOpen, /* xDlOpen */ + apndDlError, /* xDlError */ + apndDlSym, /* xDlSym */ + apndDlClose, /* xDlClose */ + apndRandomness, /* xRandomness */ + apndSleep, /* xSleep */ + apndCurrentTime, /* xCurrentTime */ + apndGetLastError, /* xGetLastError */ + apndCurrentTimeInt64, /* xCurrentTimeInt64 */ + apndSetSystemCall, /* xSetSystemCall */ + apndGetSystemCall, /* xGetSystemCall */ + apndNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods apnd_io_methods = { + 3, /* iVersion */ + apndClose, /* xClose */ + apndRead, /* xRead */ + apndWrite, /* xWrite */ + apndTruncate, /* xTruncate */ + apndSync, /* xSync */ + apndFileSize, /* xFileSize */ + apndLock, /* xLock */ + apndUnlock, /* xUnlock */ + apndCheckReservedLock, /* xCheckReservedLock */ + apndFileControl, /* xFileControl */ + apndSectorSize, /* xSectorSize */ + apndDeviceCharacteristics, /* xDeviceCharacteristics */ + apndShmMap, /* xShmMap */ + apndShmLock, /* xShmLock */ + apndShmBarrier, /* xShmBarrier */ + apndShmUnmap, /* xShmUnmap */ + apndFetch, /* xFetch */ + apndUnfetch /* xUnfetch */ +}; + +/* +** Close an apnd-file. +*/ +static int apndClose(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Read data from an apnd-file. +*/ +static int apndRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xRead(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Add the append-mark onto what should become the end of the file. +* If and only if this succeeds, internal ApndFile.iMark is updated. +* Parameter iWriteEnd is the appendvfs-relative offset of the new mark. +*/ +static int apndWriteMark( + ApndFile *paf, + sqlite3_file *pFile, + sqlite_int64 iWriteEnd +){ + sqlite_int64 iPgOne = paf->iPgOne; + unsigned char a[APND_MARK_SIZE]; + int i = APND_MARK_FOS_SZ; + int rc; + assert(pFile == ORIGFILE(paf)); + memcpy(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ); + while( --i >= 0 ){ + a[APND_MARK_PREFIX_SZ+i] = (unsigned char)(iPgOne & 0xff); + iPgOne >>= 8; + } + iWriteEnd += paf->iPgOne; + if( SQLITE_OK==(rc = pFile->pMethods->xWrite + (pFile, a, APND_MARK_SIZE, iWriteEnd)) ){ + paf->iMark = iWriteEnd; + } + return rc; +} + +/* +** Write data to an apnd-file. +*/ +static int apndWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + ApndFile *paf = (ApndFile *)pFile; + sqlite_int64 iWriteEnd = iOfst + iAmt; + if( iWriteEnd>=APND_MAX_SIZE ) return SQLITE_FULL; + pFile = ORIGFILE(pFile); + /* If append-mark is absent or will be overwritten, write it. */ + if( paf->iMark < 0 || paf->iPgOne + iWriteEnd > paf->iMark ){ + int rc = apndWriteMark(paf, pFile, iWriteEnd); + if( SQLITE_OK!=rc ) return rc; + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, paf->iPgOne+iOfst); +} + +/* +** Truncate an apnd-file. +*/ +static int apndTruncate(sqlite3_file *pFile, sqlite_int64 size){ + ApndFile *paf = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + /* The append mark goes out first so truncate failure does not lose it. */ + if( SQLITE_OK!=apndWriteMark(paf, pFile, size) ) return SQLITE_IOERR; + /* Truncate underlying file just past append mark */ + return pFile->pMethods->xTruncate(pFile, paf->iMark+APND_MARK_SIZE); +} + +/* +** Sync an apnd-file. +*/ +static int apndSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of an apnd-file. +** If the append mark is not yet there, the file-size is 0. +*/ +static int apndFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + ApndFile *paf = (ApndFile *)pFile; + *pSize = ( paf->iMark >= 0 )? (paf->iMark - paf->iPgOne) : 0; + return SQLITE_OK; +} + +/* +** Lock an apnd-file. +*/ +static int apndLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock an apnd-file. +*/ +static int apndUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on an apnd-file. +*/ +static int apndCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on an apnd-file. +*/ +static int apndFileControl(sqlite3_file *pFile, int op, void *pArg){ + ApndFile *paf = (ApndFile *)pFile; + int rc; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_SIZE_HINT ) *(sqlite3_int64*)pArg += paf->iPgOne; + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("apnd(%lld)/%z", paf->iPgOne,*(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for an apnd-file. +*/ +static int apndSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by an apnd-file. +*/ +static int apndDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int apndShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int apndShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void apndShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int apndShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int apndFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + ApndFile *p = (ApndFile *)pFile; + if( p->iMark < 0 || iOfst+iAmt > p->iMark ){ + return SQLITE_IOERR; /* Cannot read what is not yet there. */ + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xFetch(pFile, iOfst+p->iPgOne, iAmt, pp); +} + +/* Release a memory-mapped page */ +static int apndUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + ApndFile *p = (ApndFile *)pFile; + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnfetch(pFile, iOfst+p->iPgOne, pPage); +} + +/* +** Try to read the append-mark off the end of a file. Return the +** start of the appended database if the append-mark is present. +** If there is no valid append-mark, return -1; +** +** An append-mark is only valid if the NNNNNNNN start-of-database offset +** indicates that the appended database contains at least one page. The +** start-of-database value must be a multiple of 512. +*/ +static sqlite3_int64 apndReadMark(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc, i; + sqlite3_int64 iMark; + int msbs = 8 * (APND_MARK_FOS_SZ-1); + unsigned char a[APND_MARK_SIZE]; + + if( APND_MARK_SIZE!=(sz & 0x1ff) ) return -1; + rc = pFile->pMethods->xRead(pFile, a, APND_MARK_SIZE, sz-APND_MARK_SIZE); + if( rc ) return -1; + if( memcmp(a, APND_MARK_PREFIX, APND_MARK_PREFIX_SZ)!=0 ) return -1; + iMark = ((sqlite3_int64)(a[APND_MARK_PREFIX_SZ] & 0x7f)) << msbs; + for(i=1; i<8; i++){ + msbs -= 8; + iMark |= (sqlite3_int64)a[APND_MARK_PREFIX_SZ+i]< (sz - APND_MARK_SIZE - 512) ) return -1; + if( iMark & 0x1ff ) return -1; + return iMark; +} + +static const char apvfsSqliteHdr[] = "SQLite format 3"; +/* +** Check to see if the file is an appendvfs SQLite database file. +** Return true iff it is such. Parameter sz is the file's size. +*/ +static int apndIsAppendvfsDatabase(sqlite3_int64 sz, sqlite3_file *pFile){ + int rc; + char zHdr[16]; + sqlite3_int64 iMark = apndReadMark(sz, pFile); + if( iMark>=0 ){ + /* If file has the correct end-marker, the expected odd size, and the + ** SQLite DB type marker where the end-marker puts it, then it + ** is an appendvfs database. + */ + rc = pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), iMark); + if( SQLITE_OK==rc + && memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))==0 + && (sz & 0x1ff) == APND_MARK_SIZE + && sz>=512+APND_MARK_SIZE + ){ + return 1; /* It's an appendvfs database */ + } + } + return 0; +} + +/* +** Check to see if the file is an ordinary SQLite database file. +** Return true iff so. Parameter sz is the file's size. +*/ +static int apndIsOrdinaryDatabaseFile(sqlite3_int64 sz, sqlite3_file *pFile){ + char zHdr[16]; + if( apndIsAppendvfsDatabase(sz, pFile) /* rule 2 */ + || (sz & 0x1ff) != 0 + || SQLITE_OK!=pFile->pMethods->xRead(pFile, zHdr, sizeof(zHdr), 0) + || memcmp(zHdr, apvfsSqliteHdr, sizeof(zHdr))!=0 + ){ + return 0; + }else{ + return 1; + } +} + +/* +** Open an apnd file handle. +*/ +static int apndOpen( + sqlite3_vfs *pApndVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + ApndFile *pApndFile = (ApndFile*)pFile; + sqlite3_file *pBaseFile = ORIGFILE(pFile); + sqlite3_vfs *pBaseVfs = ORIGVFS(pApndVfs); + int rc; + sqlite3_int64 sz = 0; + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ){ + /* The appendvfs is not to be used for transient or temporary databases. + ** Just use the base VFS open to initialize the given file object and + ** open the underlying file. (Appendvfs is then unused for this file.) + */ + return pBaseVfs->xOpen(pBaseVfs, zName, pFile, flags, pOutFlags); + } + memset(pApndFile, 0, sizeof(ApndFile)); + pFile->pMethods = &apnd_io_methods; + pApndFile->iMark = -1; /* Append mark not yet written */ + + rc = pBaseVfs->xOpen(pBaseVfs, zName, pBaseFile, flags, pOutFlags); + if( rc==SQLITE_OK ){ + rc = pBaseFile->pMethods->xFileSize(pBaseFile, &sz); + if( rc ){ + pBaseFile->pMethods->xClose(pBaseFile); + } + } + if( rc ){ + pFile->pMethods = 0; + return rc; + } + if( apndIsOrdinaryDatabaseFile(sz, pBaseFile) ){ + /* The file being opened appears to be just an ordinary DB. Copy + ** the base dispatch-table so this instance mimics the base VFS. + */ + memmove(pApndFile, pBaseFile, pBaseVfs->szOsFile); + return SQLITE_OK; + } + pApndFile->iPgOne = apndReadMark(sz, pFile); + if( pApndFile->iPgOne>=0 ){ + pApndFile->iMark = sz - APND_MARK_SIZE; /* Append mark found */ + return SQLITE_OK; + } + if( (flags & SQLITE_OPEN_CREATE)==0 ){ + pBaseFile->pMethods->xClose(pBaseFile); + rc = SQLITE_CANTOPEN; + pFile->pMethods = 0; + }else{ + /* Round newly added appendvfs location to #define'd page boundary. + ** Note that nothing has yet been written to the underlying file. + ** The append mark will be written along with first content write. + ** Until then, paf->iMark value indicates it is not yet written. + */ + pApndFile->iPgOne = APND_START_ROUNDUP(sz); + } + return rc; +} + +/* +** Delete an apnd file. +** For an appendvfs, this could mean delete the appendvfs portion, +** leaving the appendee as it was before it gained an appendvfs. +** For now, this code deletes the underlying file too. +*/ +static int apndDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int apndAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int apndFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *apndDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void apndDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*apndDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void apndDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int apndRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int apndSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int apndCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int apndGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int apndCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} +static int apndSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr apndGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *apndNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_appendvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; + (void)db; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + apnd_vfs.iVersion = pOrig->iVersion; + apnd_vfs.pAppData = pOrig; + apnd_vfs.szOsFile = pOrig->szOsFile + sizeof(ApndFile); + rc = sqlite3_vfs_register(&apnd_vfs, 0); +#ifdef APPENDVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))apndvfsRegister); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/base64.c b/ext/misc/base64.c new file mode 100644 index 0000000..bc7a976 --- /dev/null +++ b/ext/misc/base64.c @@ -0,0 +1,292 @@ +/* +** 2022-11-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a SQLite extension for converting in either direction +** between a (binary) blob and base64 text. Base64 can transit a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals, +** and can be used unmodified in XML-like documents. +** +** This is an independent implementation of conversions specified in +** RFC 4648, done on the above date by the author (Larry Brasfield) +** who thereby has the right to put this into the public domain. +** +** The conversions meet RFC 4648 requirements, provided that this +** C source specifies that line-feeds are included in the encoded +** data to limit visible line lengths to 72 characters and to +** terminate any encoded blob having non-zero length. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Generated base64 sequences, with their line-feeds included, +** can be concatenated; the result converted back to binary will +** be the concatenation of the represented binary sequences. +** +** This SQLite3 extension creates a function, base64(x), which +** either: converts text x containing base64 to a returned blob; +** or converts a blob x to returned text containing base64. An +** error will be thrown for other input argument types. +** +** This code relies on UTF-8 encoding only with respect to the +** meaning of the first 128 (7-bit) codes matching that of USASCII. +** It will fail miserably if somehow made to try to convert EBCDIC. +** Because it is table-driven, it could be enhanced to handle that, +** but the world and SQLite have moved on from that anachronism. +** +** To build the extension: +** Set shell variable SQDIR= +** *Nix: gcc -O2 -shared -I$SQDIR -fPIC -o base64.so base64.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR -o base64.dylib base64.c +** Win32: gcc -O2 -shared -I%SQDIR% -o base64.dll base64.c +** Win32: cl /Os -I%SQDIR% base64.c -link -dll -out:base64.dll +*/ + +#include + +#include "sqlite3ext.h" + +#ifndef deliberate_fall_through +/* Quiet some compilers about some of our intentional code. */ +# if GCC_VERSION>=7000000 +# define deliberate_fall_through __attribute__((fallthrough)); +# else +# define deliberate_fall_through +# endif +#endif + +SQLITE_EXTENSION_INIT1; + +#define PC 0x80 /* pad character */ +#define WS 0x81 /* whitespace */ +#define ND 0x82 /* Not above or digit-value */ +#define PAD_CHAR '=' + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +/* Decoding table, ASCII (7-bit) value to base 64 digit value or other */ +static const u8 b64DigitValues[128] = { + /* HT LF VT FF CR */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,WS,WS,WS, WS,WS,ND,ND, + /* US */ + ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,ND, + /*sp + / */ + WS,ND,ND,ND, ND,ND,ND,ND, ND,ND,ND,62, ND,ND,ND,63, + /* 0 1 5 9 = */ + 52,53,54,55, 56,57,58,59, 60,61,ND,ND, ND,PC,ND,ND, + /* A O */ + ND, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10, 11,12,13,14, + /* P Z */ + 15,16,17,18, 19,20,21,22, 23,24,25,ND, ND,ND,ND,ND, + /* a o */ + ND,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40, + /* p z */ + 41,42,43,44, 45,46,47,48, 49,50,51,ND, ND,ND,ND,ND +}; + +static const char b64Numerals[64+1] += "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +#define BX_DV_PROTO(c) \ + ((((u8)(c))<0x80)? (u8)(b64DigitValues[(u8)(c)]) : 0x80) +#define IS_BX_DIGIT(bdp) (((u8)(bdp))<0x80) +#define IS_BX_WS(bdp) ((bdp)==WS) +#define IS_BX_PAD(bdp) ((bdp)==PC) +#define BX_NUMERAL(dv) (b64Numerals[(u8)(dv)]) +/* Width of base64 lines. Should be an integer multiple of 4. */ +#define B64_DARK_MAX 72 + +/* Encode a byte buffer into base64 text with linefeeds appended to limit +** encoded group lengths to B64_DARK_MAX or to terminate the last group. +*/ +static char* toBase64( u8 *pIn, int nbIn, char *pOut ){ + int nCol = 0; + while( nbIn >= 3 ){ + /* Do the bit-shuffle, exploiting unsigned input to avoid masking. */ + pOut[0] = BX_NUMERAL(pIn[0]>>2); + pOut[1] = BX_NUMERAL(((pIn[0]<<4)|(pIn[1]>>4))&0x3f); + pOut[2] = BX_NUMERAL(((pIn[1]&0xf)<<2)|(pIn[2]>>6)); + pOut[3] = BX_NUMERAL(pIn[2]&0x3f); + pOut += 4; + nbIn -= 3; + pIn += 3; + if( (nCol += 4)>=B64_DARK_MAX || nbIn<=0 ){ + *pOut++ = '\n'; + nCol = 0; + } + } + if( nbIn > 0 ){ + signed char nco = nbIn+1; + int nbe; + unsigned long qv = *pIn++; + for( nbe=1; nbe<3; ++nbe ){ + qv <<= 8; + if( nbe=0; --nbe ){ + char ce = (nbe>= 6; + pOut[nbe] = ce; + } + pOut += 4; + *pOut++ = '\n'; + } + *pOut = 0; + return pOut; +} + +/* Skip over text which is not base64 numeral(s). */ +static char * skipNonB64( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_BX_DIGIT(BX_DV_PROTO(c)) ) ++s; + return s; +} + +/* Decode base64 text into a byte buffer. */ +static u8* fromBase64( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 && *pIn!=PAD_CHAR ){ + static signed char nboi[] = { 0, 0, 1, 2, 3 }; + char *pUse = skipNonB64(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo, nac; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>4)? 4 : ncIn; + ncIn -= nti; + nbo = nboi[nti]; + if( nbo==0 ) break; + for( nac=0; nac<4; ++nac ){ + char c = (nac>8) & 0xff; + case 1: + pOut[0] = (qv>>16) & 0xff; + } + pOut += nbo; + } + return pOut; +} + +/* This function does the work for the SQLite base64(x) UDF. */ +static void base64(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + nc = 4*(nv+2/3); /* quads needed */ + nc += (nc+(B64_DARK_MAX-1))/B64_DARK_MAX + 1; /* LFs and a 0-terminator */ + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base64 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase64(bBuf, nb, cBuf) - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 3*((nv+3)/4); /* may overestimate due to LF and padding */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base64 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase64(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base64 accepts only blob or text", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base64 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init +#else +static int sqlite3_base64_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; + return sqlite3_create_function + (db, "base64", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base64, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +#define BASE64_INIT(db) sqlite3_base64_init(db, 0, 0) +#define BASE64_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/base85.c b/ext/misc/base85.c new file mode 100644 index 0000000..e7ef0a0 --- /dev/null +++ b/ext/misc/base85.c @@ -0,0 +1,450 @@ +/* +** 2022-11-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility for converting binary to base85 or vice-versa. +** It can be built as a standalone program or an SQLite3 extension. +** +** Much like base64 representations, base85 can be sent through a +** sane USASCII channel unmolested. It also plays nicely in CSV or +** written as TCL brace-enclosed literals or SQL string literals. +** It is not suited for unmodified use in XML-like documents. +** +** The encoding used resembles Ascii85, but was devised by the author +** (Larry Brasfield) before Mozilla, Adobe, ZMODEM or other Ascii85 +** variant sources existed, in the 1984 timeframe on a VAX mainframe. +** Further, this is an independent implementation of a base85 system. +** Hence, the author has rightfully put this into the public domain. +** +** Base85 numerals are taken from the set of 7-bit USASCII codes, +** excluding control characters and Space ! " ' ( ) { | } ~ Del +** in code order representing digit values 0 to 84 (base 10.) +** +** Groups of 4 bytes, interpreted as big-endian 32-bit values, +** are represented as 5-digit base85 numbers with MS to LS digit +** order. Groups of 1-3 bytes are represented with 2-4 digits, +** still big-endian but 8-24 bit values. (Using big-endian yields +** the simplest transition to byte groups smaller than 4 bytes. +** These byte groups can also be considered base-256 numbers.) +** Groups of 0 bytes are represented with 0 digits and vice-versa. +** No pad characters are used; Encoded base85 numeral sequence +** (aka "group") length maps 1-to-1 to the decoded binary length. +** +** Any character not in the base85 numeral set delimits groups. +** When base85 is streamed or stored in containers of indefinite +** size, newline is used to separate it into sub-sequences of no +** more than 80 digits so that fgets() can be used to read it. +** +** Length limitations are not imposed except that the runtime +** SQLite string or blob length limits are respected. Otherwise, +** any length binary sequence can be represented and recovered. +** Base85 sequences can be concatenated by separating them with +** a non-base85 character; the conversion to binary will then +** be the concatenation of the represented binary sequences. + +** The standalone program either converts base85 on stdin to create +** a binary file or converts a binary file to base85 on stdout. +** Read or make it blurt its help for invocation details. +** +** The SQLite3 extension creates a function, base85(x), which will +** either convert text base85 to a blob or a blob to text base85 +** and return the result (or throw an error for other types.) +** Unless built with OMIT_BASE85_CHECKER defined, it also creates a +** function, is_base85(t), which returns 1 iff the text t contains +** nothing other than base85 numerals and whitespace, or 0 otherwise. +** +** To build the extension: +** Set shell variable SQDIR= +** and variable OPTS to -DOMIT_BASE85_CHECKER if is_base85() unwanted. +** *Nix: gcc -O2 -shared -I$SQDIR $OPTS -fPIC -o base85.so base85.c +** OSX: gcc -O2 -dynamiclib -fPIC -I$SQDIR $OPTS -o base85.dylib base85.c +** Win32: gcc -O2 -shared -I%SQDIR% %OPTS% -o base85.dll base85.c +** Win32: cl /Os -I%SQDIR% %OPTS% base85.c -link -dll -out:base85.dll +** +** To build the standalone program, define PP symbol BASE85_STANDALONE. Eg. +** *Nix or OSX: gcc -O2 -DBASE85_STANDALONE base85.c -o base85 +** Win32: gcc -O2 -DBASE85_STANDALONE -o base85.exe base85.c +** Win32: cl /Os /MD -DBASE85_STANDALONE base85.c +*/ + +#include +#include +#include +#include +#ifndef OMIT_BASE85_CHECKER +# include +#endif + +#ifndef BASE85_STANDALONE + +# include "sqlite3ext.h" + +SQLITE_EXTENSION_INIT1; + +#else + +# ifdef _WIN32 +# include +# include +# else +# define setmode(fd,m) +# endif + +static char *zHelp = + "Usage: base85 \n" + " is either -r to read or -w to write ,\n" + " content to be converted to/from base85 on stdout/stdin.\n" + " names a binary file to be rendered or created.\n" + " Or, the name '-' refers to the stdin or stdout stream.\n" + ; + +static void sayHelp(){ + printf("%s", zHelp); +} +#endif + +#ifndef U8_TYPEDEF +typedef unsigned char u8; +#define U8_TYPEDEF +#endif + +/* Classify c according to interval within USASCII set w.r.t. base85 + * Values of 1 and 3 are base85 numerals. Values of 0, 2, or 4 are not. + */ +#define B85_CLASS( c ) (((c)>='#')+((c)>'&')+((c)>='*')+((c)>'z')) + +/* Provide digitValue to b85Numeral offset as a function of above class. */ +static u8 b85_cOffset[] = { 0, '#', 0, '*'-4, 0 }; +#define B85_DNOS( c ) b85_cOffset[B85_CLASS(c)] + +/* Say whether c is a base85 numeral. */ +#define IS_B85( c ) (B85_CLASS(c) & 1) + +#if 0 /* Not used, */ +static u8 base85DigitValue( char c ){ + u8 dv = (u8)(c - '#'); + if( dv>87 ) return 0xff; + return (dv > 3)? dv-3 : dv; +} +#endif + +/* Width of base64 lines. Should be an integer multiple of 5. */ +#define B85_DARK_MAX 80 + + +static char * skipNonB85( char *s, int nc ){ + char c; + while( nc-- > 0 && (c = *s) && !IS_B85(c) ) ++s; + return s; +} + +/* Convert small integer, known to be in 0..84 inclusive, to base85 numeral. + * Do not use the macro form with argument expression having a side-effect.*/ +#if 0 +static char base85Numeral( u8 b ){ + return (b < 4)? (char)(b + '#') : (char)(b - 4 + '*'); +} +#else +# define base85Numeral( dn )\ + ((char)(((dn) < 4)? (char)((dn) + '#') : (char)((dn) - 4 + '*'))) +#endif + +static char *putcs(char *pc, char *s){ + char c; + while( (c = *s++)!=0 ) *pc++ = c; + return pc; +} + +/* Encode a byte buffer into base85 text. If pSep!=0, it's a C string +** to be appended to encoded groups to limit their length to B85_DARK_MAX +** or to terminate the last group (to aid concatenation.) +*/ +static char* toBase85( u8 *pIn, int nbIn, char *pOut, char *pSep ){ + int nCol = 0; + while( nbIn >= 4 ){ + int nco = 5; + unsigned long qbv = (((unsigned long)pIn[0])<<24) | + (pIn[1]<<16) | (pIn[2]<<8) | pIn[3]; + while( nco > 0 ){ + unsigned nqv = (unsigned)(qbv/85UL); + unsigned char dv = qbv - 85UL*nqv; + qbv = nqv; + pOut[--nco] = base85Numeral(dv); + } + nbIn -= 4; + pIn += 4; + pOut += 5; + if( pSep && (nCol += 5)>=B85_DARK_MAX ){ + pOut = putcs(pOut, pSep); + nCol = 0; + } + } + if( nbIn > 0 ){ + int nco = nbIn + 1; + unsigned long qv = *pIn++; + int nbe = 1; + while( nbe++ < nbIn ){ + qv = (qv<<8) | *pIn++; + } + nCol += nco; + while( nco > 0 ){ + u8 dv = (u8)(qv % 85); + qv /= 85; + pOut[--nco] = base85Numeral(dv); + } + pOut += (nbIn+1); + } + if( pSep && nCol>0 ) pOut = putcs(pOut, pSep); + *pOut = 0; + return pOut; +} + +/* Decode base85 text into a byte buffer. */ +static u8* fromBase85( char *pIn, int ncIn, u8 *pOut ){ + if( ncIn>0 && pIn[ncIn-1]=='\n' ) --ncIn; + while( ncIn>0 ){ + static signed char nboi[] = { 0, 0, 1, 2, 3, 4 }; + char *pUse = skipNonB85(pIn, ncIn); + unsigned long qv = 0L; + int nti, nbo; + ncIn -= (pUse - pIn); + pIn = pUse; + nti = (ncIn>5)? 5 : ncIn; + nbo = nboi[nti]; + if( nbo==0 ) break; + while( nti>0 ){ + char c = *pIn++; + u8 cdo = B85_DNOS(c); + --ncIn; + if( cdo==0 ) break; + qv = 85 * qv + (c - cdo); + --nti; + } + nbo -= nti; /* Adjust for early (non-digit) end of group. */ + switch( nbo ){ + case 4: + *pOut++ = (qv >> 24)&0xff; + case 3: + *pOut++ = (qv >> 16)&0xff; + case 2: + *pOut++ = (qv >> 8)&0xff; + case 1: + *pOut++ = qv&0xff; + case 0: + break; + } + } + return pOut; +} + +#ifndef OMIT_BASE85_CHECKER +/* Say whether input char sequence is all (base85 and/or whitespace).*/ +static int allBase85( char *p, int len ){ + char c; + while( len-- > 0 && (c = *p++) != 0 ){ + if( !IS_B85(c) && !isspace(c) ) return 0; + } + return 1; +} +#endif + +#ifndef BASE85_STANDALONE + +# ifndef OMIT_BASE85_CHECKER +/* This function does the work for the SQLite is_base85(t) UDF. */ +static void is_base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_TEXT: + { + int rv = allBase85( (char *)sqlite3_value_text(av[0]), + sqlite3_value_bytes(av[0]) ); + sqlite3_result_int(context, rv); + } + break; + case SQLITE_NULL: + sqlite3_result_null(context); + break; + default: + sqlite3_result_error(context, "is_base85 accepts only text or NULL", -1); + return; + } +} +# endif + +/* This function does the work for the SQLite base85(x) UDF. */ +static void base85(sqlite3_context *context, int na, sqlite3_value *av[]){ + int nb, nc, nv = sqlite3_value_bytes(av[0]); + int nvMax = sqlite3_limit(sqlite3_context_db_handle(context), + SQLITE_LIMIT_LENGTH, -1); + char *cBuf; + u8 *bBuf; + assert(na==1); + switch( sqlite3_value_type(av[0]) ){ + case SQLITE_BLOB: + nb = nv; + /* ulongs tail newlines tailenc+nul*/ + nc = 5*(nv/4) + nv%4 + nv/64+1 + 2; + if( nvMax < nc ){ + sqlite3_result_error(context, "blob expanded to base85 too big", -1); + return; + } + bBuf = (u8*)sqlite3_value_blob(av[0]); + if( !bBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_text(context,"",-1,SQLITE_STATIC); + break; + } + cBuf = sqlite3_malloc(nc); + if( !cBuf ) goto memFail; + nc = (int)(toBase85(bBuf, nb, cBuf, "\n") - cBuf); + sqlite3_result_text(context, cBuf, nc, sqlite3_free); + break; + case SQLITE_TEXT: + nc = nv; + nb = 4*(nv/5) + nv%5; /* may overestimate */ + if( nvMax < nb ){ + sqlite3_result_error(context, "blob from base85 may be too big", -1); + return; + }else if( nb<1 ){ + nb = 1; + } + cBuf = (char *)sqlite3_value_text(av[0]); + if( !cBuf ){ + if( SQLITE_NOMEM==sqlite3_errcode(sqlite3_context_db_handle(context)) ){ + goto memFail; + } + sqlite3_result_zeroblob(context, 0); + break; + } + bBuf = sqlite3_malloc(nb); + if( !bBuf ) goto memFail; + nb = (int)(fromBase85(cBuf, nc, bBuf) - bBuf); + sqlite3_result_blob(context, bBuf, nb, sqlite3_free); + break; + default: + sqlite3_result_error(context, "base85 accepts only blob or text.", -1); + return; + } + return; + memFail: + sqlite3_result_error(context, "base85 OOM", -1); +} + +/* +** Establish linkage to running SQLite library. +*/ +#ifndef SQLITE_SHELL_EXTFUNCS +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_base_init +#else +static int sqlite3_base85_init +#endif +(sqlite3 *db, char **pzErr, const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErr; +# ifndef OMIT_BASE85_CHECKER + { + int rc = sqlite3_create_function + (db, "is_base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_UTF8, + 0, is_base85, 0, 0); + if( rc!=SQLITE_OK ) return rc; + } +# endif + return sqlite3_create_function + (db, "base85", 1, + SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS|SQLITE_DIRECTONLY|SQLITE_UTF8, + 0, base85, 0, 0); +} + +/* +** Define some macros to allow this extension to be built into the shell +** conveniently, in conjunction with use of SQLITE_SHELL_EXTFUNCS. This +** allows shell.c, as distributed, to have this extension built in. +*/ +# define BASE85_INIT(db) sqlite3_base85_init(db, 0, 0) +# define BASE85_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ + +#else /* standalone program */ + +int main(int na, char *av[]){ + int cin; + int rc = 0; + u8 bBuf[4*(B85_DARK_MAX/5)]; + char cBuf[5*(sizeof(bBuf)/4)+2]; + size_t nio; +# ifndef OMIT_BASE85_CHECKER + int b85Clean = 1; +# endif + char rw; + FILE *fb = 0, *foc = 0; + char fmode[3] = "xb"; + if( na < 3 || av[1][0]!='-' || (rw = av[1][1])==0 || (rw!='r' && rw!='w') ){ + sayHelp(); + return 0; + } + fmode[0] = rw; + if( av[2][0]=='-' && av[2][1]==0 ){ + switch( rw ){ + case 'r': + fb = stdin; + setmode(fileno(stdin), O_BINARY); + break; + case 'w': + fb = stdout; + setmode(fileno(stdout), O_BINARY); + break; + } + }else{ + fb = fopen(av[2], fmode); + foc = fb; + } + if( !fb ){ + fprintf(stderr, "Cannot open %s for %c\n", av[2], rw); + rc = 1; + }else{ + switch( rw ){ + case 'r': + while( (nio = fread( bBuf, 1, sizeof(bBuf), fb))>0 ){ + toBase85( bBuf, (int)nio, cBuf, 0 ); + fprintf(stdout, "%s\n", cBuf); + } + break; + case 'w': + while( 0 != fgets(cBuf, sizeof(cBuf), stdin) ){ + int nc = strlen(cBuf); + size_t nbo = fromBase85( cBuf, nc, bBuf ) - bBuf; + if( 1 != fwrite(bBuf, nbo, 1, fb) ) rc = 1; +# ifndef OMIT_BASE85_CHECKER + b85Clean &= allBase85( cBuf, nc ); +# endif + } + break; + default: + sayHelp(); + rc = 1; + } + if( foc ) fclose(foc); + } +# ifndef OMIT_BASE85_CHECKER + if( !b85Clean ){ + fprintf(stderr, "Base85 input had non-base85 dark or control content.\n"); + } +# endif + return rc; +} + +#endif diff --git a/ext/misc/basexx.c b/ext/misc/basexx.c new file mode 100644 index 0000000..0dcde54 --- /dev/null +++ b/ext/misc/basexx.c @@ -0,0 +1,89 @@ +/* +** 2022-11-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This source allows multiple SQLite extensions to be either: combined +** into a single runtime-loadable library; or built into the SQLite shell +** using a preprocessing convention set by src/shell.c.in (and shell.c). +** +** Presently, it combines the base64.c and base85.c extensions. However, +** it can be used as a template for other combinations. +** +** Example usages: +** +** - Build a runtime-loadable extension from SQLite checkout directory: +** *Nix, OSX: gcc -O2 -shared -I. -fPIC -o basexx.so ext/misc/basexx.c +** Win32: cl /Os -I. ext/misc/basexx.c -link -dll -out:basexx.dll +** +** - Incorporate as built-in in sqlite3 shell: +** *Nix, OSX with gcc on a like platform: +** export mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** export mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** make sqlite3 "OPTS=$mop1 $mop2" +** Win32 with Microsoft toolset on Windows: +** set mop1=-DSQLITE_SHELL_EXTSRC=ext/misc/basexx.c +** set mop2=-DSQLITE_SHELL_EXTFUNCS=BASEXX +** set mops="OPTS=%mop1% %mop2%" +** nmake -f Makefile.msc sqlite3.exe %mops% +*/ + +#ifndef SQLITE_SHELL_EXTFUNCS /* Guard for #include as built-in extension. */ +# include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1; +#endif + +static void init_api_ptr(const sqlite3_api_routines *pApi){ + SQLITE_EXTENSION_INIT2(pApi); +} + +#undef SQLITE_EXTENSION_INIT1 +#define SQLITE_EXTENSION_INIT1 /* */ +#undef SQLITE_EXTENSION_INIT2 +#define SQLITE_EXTENSION_INIT2(v) (void)v + +typedef unsigned char u8; +#define U8_TYPEDEF + +/* These next 2 undef's are only needed because the entry point names + * collide when formulated per the rules stated for loadable extension + * entry point names that will be deduced from the file basenames. + */ +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base64_init +#include "base64.c" + +#undef sqlite3_base_init +#define sqlite3_base_init sqlite3_base85_init +#include "base85.c" + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_basexx_init(sqlite3 *db, char **pzErr, + const sqlite3_api_routines *pApi){ + int rc1; + int rc2; + + init_api_ptr(pApi); + rc1 = BASE64_INIT(db); + rc2 = BASE85_INIT(db); + + if( rc1==SQLITE_OK && rc2==SQLITE_OK ){ + BASE64_EXPOSE(db, pzErr); + BASE64_EXPOSE(db, pzErr); + return SQLITE_OK; + }else{ + return SQLITE_ERROR; + } +} + +# define BASEXX_INIT(db) sqlite3_basexx_init(db, 0, 0) +# define BASEXX_EXPOSE(db, pzErr) /* Not needed, ..._init() does this. */ diff --git a/ext/misc/blobio.c b/ext/misc/blobio.c new file mode 100644 index 0000000..3a1ee84 --- /dev/null +++ b/ext/misc/blobio.c @@ -0,0 +1,152 @@ +/* +** 2019-03-30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** An SQL function that uses the incremental BLOB I/O mechanism of SQLite +** to read or write part of a blob. This is intended for debugging use +** in the CLI. +** +** readblob(SCHEMA,TABLE,COLUMN,ROWID,OFFSET,N) +** +** Returns N bytes of the blob starting at OFFSET. +** +** writeblob(SCHEMA,TABLE,COLUMN,ROWID,OFFSET,NEWDATA) +** +** NEWDATA must be a blob. The content of NEWDATA overwrites the +** existing BLOB data at SCHEMA.TABLE.COLUMN for row ROWID beginning +** at OFFSET bytes into the blob. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +static void readblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_blob *pBlob = 0; + const char *zSchema; + const char *zTable; + const char *zColumn; + sqlite3_int64 iRowid; + int iOfst; + unsigned char *aData; + int nData; + sqlite3 *db; + int rc; + + zSchema = (const char*)sqlite3_value_text(argv[0]); + zTable = (const char*)sqlite3_value_text(argv[1]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad table name", -1); + return; + } + zColumn = (const char*)sqlite3_value_text(argv[2]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad column name", -1); + return; + } + iRowid = sqlite3_value_int64(argv[3]); + iOfst = sqlite3_value_int(argv[4]); + nData = sqlite3_value_int(argv[5]); + if( nData<=0 ) return; + aData = sqlite3_malloc64( nData+1 ); + if( aData==0 ){ + sqlite3_result_error_nomem(context); + return; + } + db = sqlite3_context_db_handle(context); + rc = sqlite3_blob_open(db, zSchema, zTable, zColumn, iRowid, 0, &pBlob); + if( rc ){ + sqlite3_free(aData); + sqlite3_result_error(context, "cannot open BLOB pointer", -1); + return; + } + rc = sqlite3_blob_read(pBlob, aData, nData, iOfst); + sqlite3_blob_close(pBlob); + if( rc ){ + sqlite3_free(aData); + sqlite3_result_error(context, "BLOB read failed", -1); + }else{ + sqlite3_result_blob(context, aData, nData, sqlite3_free); + } +} + +static void writeblobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + sqlite3_blob *pBlob = 0; + const char *zSchema; + const char *zTable; + const char *zColumn; + sqlite3_int64 iRowid; + int iOfst; + unsigned char *aData; + int nData; + sqlite3 *db; + int rc; + + zSchema = (const char*)sqlite3_value_text(argv[0]); + zTable = (const char*)sqlite3_value_text(argv[1]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad table name", -1); + return; + } + zColumn = (const char*)sqlite3_value_text(argv[2]); + if( zTable==0 ){ + sqlite3_result_error(context, "bad column name", -1); + return; + } + iRowid = sqlite3_value_int64(argv[3]); + iOfst = sqlite3_value_int(argv[4]); + if( sqlite3_value_type(argv[5])!=SQLITE_BLOB ){ + sqlite3_result_error(context, "6th argument must be a BLOB", -1); + return; + } + nData = sqlite3_value_bytes(argv[5]); + aData = (unsigned char *)sqlite3_value_blob(argv[5]); + db = sqlite3_context_db_handle(context); + rc = sqlite3_blob_open(db, zSchema, zTable, zColumn, iRowid, 1, &pBlob); + if( rc ){ + sqlite3_result_error(context, "cannot open BLOB pointer", -1); + return; + } + rc = sqlite3_blob_write(pBlob, aData, nData, iOfst); + sqlite3_blob_close(pBlob); + if( rc ){ + sqlite3_result_error(context, "BLOB write failed", -1); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_blobio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readblob", 6, SQLITE_UTF8, 0, + readblobFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writeblob", 6, SQLITE_UTF8, 0, + writeblobFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/btreeinfo.c b/ext/misc/btreeinfo.c new file mode 100644 index 0000000..02f8c03 --- /dev/null +++ b/ext/misc/btreeinfo.c @@ -0,0 +1,430 @@ +/* +** 2017-10-24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_btreeinfo" virtual table. +** +** The sqlite_btreeinfo virtual table is a read-only eponymous-only virtual +** table that shows information about all btrees in an SQLite database file. +** The schema is like this: +** +** CREATE TABLE sqlite_btreeinfo( +** type TEXT, -- "table" or "index" +** name TEXT, -- Name of table or index for this btree. +** tbl_name TEXT, -- Associated table +** rootpage INT, -- The root page of the btree +** sql TEXT, -- SQL for this btree - from sqlite_schema +** hasRowid BOOLEAN, -- True if the btree has a rowid +** nEntry INT, -- Estimated number of entries +** nPage INT, -- Estimated number of pages +** depth INT, -- Depth of the btree +** szPage INT, -- Size of each page in bytes +** zSchema TEXT HIDDEN -- The schema to which this btree belongs +** ); +** +** The first 5 fields are taken directly from the sqlite_schema table. +** Considering only the first 5 fields, the only difference between +** this virtual table and the sqlite_schema table is that this virtual +** table omits all entries that have a 0 or NULL rowid - in other words +** it omits triggers and views. +** +** The value added by this table comes in the next 5 fields. +** +** Note that nEntry and nPage are *estimated*. They are computed doing +** a single search from the root to a leaf, counting the number of cells +** at each level, and assuming that unvisited pages have a similar number +** of cells. +** +** The sqlite_dbpage virtual table must be available for this virtual table +** to operate. +** +** USAGE EXAMPLES: +** +** Show the table btrees in a schema order with the tables with the most +** rows occuring first: +** +** SELECT name, nEntry +** FROM sqlite_btreeinfo +** WHERE type='table' +** ORDER BY nEntry DESC, name; +** +** Show the names of all WITHOUT ROWID tables: +** +** SELECT name FROM sqlite_btreeinfo +** WHERE type='table' AND NOT hasRowid; +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Columns available in this virtual table */ +#define BINFO_COLUMN_TYPE 0 +#define BINFO_COLUMN_NAME 1 +#define BINFO_COLUMN_TBL_NAME 2 +#define BINFO_COLUMN_ROOTPAGE 3 +#define BINFO_COLUMN_SQL 4 +#define BINFO_COLUMN_HASROWID 5 +#define BINFO_COLUMN_NENTRY 6 +#define BINFO_COLUMN_NPAGE 7 +#define BINFO_COLUMN_DEPTH 8 +#define BINFO_COLUMN_SZPAGE 9 +#define BINFO_COLUMN_SCHEMA 10 + +/* Forward declarations */ +typedef struct BinfoTable BinfoTable; +typedef struct BinfoCursor BinfoCursor; + +/* A cursor for the sqlite_btreeinfo table */ +struct BinfoCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pStmt; /* Query against sqlite_schema */ + int rc; /* Result of previous sqlite_step() call */ + int hasRowid; /* hasRowid value. Negative if unknown. */ + sqlite3_int64 nEntry; /* nEntry value */ + int nPage; /* nPage value */ + int depth; /* depth value */ + int szPage; /* size of a btree page. 0 if unknown */ + char *zSchema; /* Schema being interrogated */ +}; + +/* The sqlite_btreeinfo table */ +struct BinfoTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The databse connection */ +}; + +/* +** Connect to the sqlite_btreeinfo virtual table. +*/ +static int binfoConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + BinfoTable *pTab = 0; + int rc = SQLITE_OK; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(\n" + " type TEXT,\n" + " name TEXT,\n" + " tbl_name TEXT,\n" + " rootpage INT,\n" + " sql TEXT,\n" + " hasRowid BOOLEAN,\n" + " nEntry INT,\n" + " nPage INT,\n" + " depth INT,\n" + " szPage INT,\n" + " zSchema TEXT HIDDEN\n" + ")"); + if( rc==SQLITE_OK ){ + pTab = (BinfoTable *)sqlite3_malloc64(sizeof(BinfoTable)); + if( pTab==0 ) rc = SQLITE_NOMEM; + } + assert( rc==SQLITE_OK || pTab==0 ); + if( pTab ){ + pTab->db = db; + } + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a btreeinfo virtual table. +*/ +static int binfoDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 Use "main" for the schema +** 1 Schema identified by parameter ?1 +*/ +static int binfoBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + pIdxInfo->estimatedCost = 10000.0; /* Cost estimate */ + pIdxInfo->estimatedRows = 100; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable + && p->iColumn==BINFO_COLUMN_SCHEMA + && p->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + pIdxInfo->estimatedCost = 1000.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + } + return SQLITE_OK; +} + +/* +** Open a new btreeinfo cursor. +*/ +static int binfoOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + BinfoCursor *pCsr; + + pCsr = (BinfoCursor *)sqlite3_malloc64(sizeof(BinfoCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM; + }else{ + memset(pCsr, 0, sizeof(BinfoCursor)); + pCsr->base.pVtab = pVTab; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a btreeinfo cursor. +*/ +static int binfoClose(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + sqlite3_finalize(pCsr->pStmt); + sqlite3_free(pCsr->zSchema); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a btreeinfo cursor to the next entry in the file. +*/ +static int binfoNext(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + pCsr->rc = sqlite3_step(pCsr->pStmt); + pCsr->hasRowid = -1; + return pCsr->rc==SQLITE_ERROR ? SQLITE_ERROR : SQLITE_OK; +} + +/* We have reached EOF if previous sqlite3_step() returned +** anything other than SQLITE_ROW; +*/ +static int binfoEof(sqlite3_vtab_cursor *pCursor){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + return pCsr->rc!=SQLITE_ROW; +} + +/* Position a cursor back to the beginning. +*/ +static int binfoFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + BinfoTable *pTab = (BinfoTable *)pCursor->pVtab; + char *zSql; + int rc; + + sqlite3_free(pCsr->zSchema); + if( idxNum==1 && sqlite3_value_type(argv[0])!=SQLITE_NULL ){ + pCsr->zSchema = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + }else{ + pCsr->zSchema = sqlite3_mprintf("main"); + } + zSql = sqlite3_mprintf( + "SELECT 0, 'table','sqlite_schema','sqlite_schema',1,NULL " + "UNION ALL " + "SELECT rowid, type, name, tbl_name, rootpage, sql" + " FROM \"%w\".sqlite_schema WHERE rootpage>=1", + pCsr->zSchema); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + pCsr->hasRowid = -1; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + rc = binfoNext(pCursor); + } + return rc; +} + +/* Decode big-endian integers */ +static unsigned int get_uint16(unsigned char *a){ + return (a[0]<<8)|a[1]; +} +static unsigned int get_uint32(unsigned char *a){ + return (a[0]<<24)|(a[1]<<16)|(a[2]<<8)|a[3]; +} + +/* Examine the b-tree rooted at pgno and estimate its size. +** Return non-zero if anything goes wrong. +*/ +static int binfoCompute(sqlite3 *db, int pgno, BinfoCursor *pCsr){ + sqlite3_int64 nEntry = 1; + int nPage = 1; + unsigned char *aData; + sqlite3_stmt *pStmt = 0; + int rc = SQLITE_OK; + int pgsz = 0; + int nCell; + int iCell; + + rc = sqlite3_prepare_v2(db, + "SELECT data FROM sqlite_dbpage('main') WHERE pgno=?1", -1, + &pStmt, 0); + if( rc ) return rc; + pCsr->depth = 1; + while(1){ + sqlite3_bind_int(pStmt, 1, pgno); + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + rc = SQLITE_ERROR; + break; + } + pCsr->szPage = pgsz = sqlite3_column_bytes(pStmt, 0); + aData = (unsigned char*)sqlite3_column_blob(pStmt, 0); + if( aData==0 ){ + rc = SQLITE_NOMEM; + break; + } + if( pgno==1 ){ + aData += 100; + pgsz -= 100; + } + pCsr->hasRowid = aData[0]!=2 && aData[0]!=10; + nCell = get_uint16(aData+3); + nEntry *= (nCell+1); + if( aData[0]==10 || aData[0]==13 ) break; + nPage *= (nCell+1); + if( nCell<=1 ){ + pgno = get_uint32(aData+8); + }else{ + iCell = get_uint16(aData+12+2*(nCell/2)); + if( pgno==1 ) iCell -= 100; + if( iCell<=12 || iCell>=pgsz-4 ){ + rc = SQLITE_CORRUPT; + break; + } + pgno = get_uint32(aData+iCell); + } + pCsr->depth++; + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + pCsr->nPage = nPage; + pCsr->nEntry = nEntry; + if( rc==SQLITE_ROW ) rc = SQLITE_OK; + return rc; +} + +/* Return a column for the sqlite_btreeinfo table */ +static int binfoColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + if( i>=BINFO_COLUMN_HASROWID && i<=BINFO_COLUMN_SZPAGE && pCsr->hasRowid<0 ){ + int pgno = sqlite3_column_int(pCsr->pStmt, BINFO_COLUMN_ROOTPAGE+1); + sqlite3 *db = sqlite3_context_db_handle(ctx); + int rc = binfoCompute(db, pgno, pCsr); + if( rc ){ + pCursor->pVtab->zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + return SQLITE_ERROR; + } + } + switch( i ){ + case BINFO_COLUMN_NAME: + case BINFO_COLUMN_TYPE: + case BINFO_COLUMN_TBL_NAME: + case BINFO_COLUMN_ROOTPAGE: + case BINFO_COLUMN_SQL: { + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pStmt, i+1)); + break; + } + case BINFO_COLUMN_HASROWID: { + sqlite3_result_int(ctx, pCsr->hasRowid); + break; + } + case BINFO_COLUMN_NENTRY: { + sqlite3_result_int64(ctx, pCsr->nEntry); + break; + } + case BINFO_COLUMN_NPAGE: { + sqlite3_result_int(ctx, pCsr->nPage); + break; + } + case BINFO_COLUMN_DEPTH: { + sqlite3_result_int(ctx, pCsr->depth); + break; + } + case BINFO_COLUMN_SCHEMA: { + sqlite3_result_text(ctx, pCsr->zSchema, -1, SQLITE_STATIC); + break; + } + } + return SQLITE_OK; +} + +/* Return the ROWID for the sqlite_btreeinfo table */ +static int binfoRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + BinfoCursor *pCsr = (BinfoCursor *)pCursor; + *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "sqlite_btreeinfo" virtual table module +*/ +int sqlite3BinfoRegister(sqlite3 *db){ + static sqlite3_module binfo_module = { + 0, /* iVersion */ + 0, /* xCreate */ + binfoConnect, /* xConnect */ + binfoBestIndex, /* xBestIndex */ + binfoDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + binfoOpen, /* xOpen - open a cursor */ + binfoClose, /* xClose - close a cursor */ + binfoFilter, /* xFilter - configure scan constraints */ + binfoNext, /* xNext - advance a cursor */ + binfoEof, /* xEof - check for end of scan */ + binfoColumn, /* xColumn - read data */ + binfoRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + return sqlite3_create_module(db, "sqlite_btreeinfo", &binfo_module, 0); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_btreeinfo_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + SQLITE_EXTENSION_INIT2(pApi); + return sqlite3BinfoRegister(db); +} diff --git a/ext/misc/carray.c b/ext/misc/carray.c new file mode 100644 index 0000000..b1caa98 --- /dev/null +++ b/ext/misc/carray.c @@ -0,0 +1,561 @@ +/* +** 2016-06-29 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates how to create a table-valued-function that +** returns the values in a C-language array. +** Examples: +** +** SELECT * FROM carray($ptr,5) +** +** The query above returns 5 integers contained in a C-language array +** at the address $ptr. $ptr is a pointer to the array of integers. +** The pointer value must be assigned to $ptr using the +** sqlite3_bind_pointer() interface with a pointer type of "carray". +** For example: +** +** static int aX[] = { 53, 9, 17, 2231, 4, 99 }; +** int i = sqlite3_bind_parameter_index(pStmt, "$ptr"); +** sqlite3_bind_pointer(pStmt, i, aX, "carray", 0); +** +** There is an optional third parameter to determine the datatype of +** the C-language array. Allowed values of the third parameter are +** 'int32', 'int64', 'double', 'char*', 'struct iovec'. Example: +** +** SELECT * FROM carray($ptr,10,'char*'); +** +** The default value of the third parameter is 'int32'. +** +** HOW IT WORKS +** +** The carray "function" is really a virtual table with the +** following schema: +** +** CREATE TABLE carray( +** value, +** pointer HIDDEN, +** count HIDDEN, +** ctype TEXT HIDDEN +** ); +** +** If the hidden columns "pointer" and "count" are unconstrained, then +** the virtual table has no rows. Otherwise, the virtual table interprets +** the integer value of "pointer" as a pointer to the array and "count" +** as the number of elements in the array. The virtual table steps through +** the array, element by element. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#ifdef _WIN32 + struct iovec { + void *iov_base; + size_t iov_len; + }; +#else +# include +#endif + +/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). +** Must exactly match the definitions in carray.h. +*/ +#ifndef CARRAY_INT32 +# define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +# define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +# define CARRAY_DOUBLE 2 /* Data is doubles */ +# define CARRAY_TEXT 3 /* Data is char* */ +# define CARRAY_BLOB 4 /* Data is struct iovec* */ +#endif + +#ifndef SQLITE_API +# ifdef _WIN32 +# define SQLITE_API __declspec(dllexport) +# else +# define SQLITE_API +# endif +#endif + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Names of allowed datatypes +*/ +static const char *azType[] = { "int32", "int64", "double", "char*", + "struct iovec" }; + +/* +** Structure used to hold the sqlite3_carray_bind() information +*/ +typedef struct carray_bind carray_bind; +struct carray_bind { + void *aData; /* The data */ + int nData; /* Number of elements */ + int mFlags; /* Control flags */ + void (*xDel)(void*); /* Destructor for aData */ +}; + + +/* carray_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct carray_cursor carray_cursor; +struct carray_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + void *pPtr; /* Pointer to the array of values */ + sqlite3_int64 iCnt; /* Number of integers in the array */ + unsigned char eType; /* One of the CARRAY_type values */ +}; + +/* +** The carrayConnect() method is invoked to create a new +** carray_vtab that describes the carray virtual table. +** +** Think of this routine as the constructor for carray_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the carray_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against carray will look like. +*/ +static int carrayConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + sqlite3_vtab *pNew; + int rc; + +/* Column numbers */ +#define CARRAY_COLUMN_VALUE 0 +#define CARRAY_COLUMN_POINTER 1 +#define CARRAY_COLUMN_COUNT 2 +#define CARRAY_COLUMN_CTYPE 3 + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(value,pointer hidden,count hidden,ctype hidden)"); + if( rc==SQLITE_OK ){ + pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + } + return rc; +} + +/* +** This method is the destructor for carray_cursor objects. +*/ +static int carrayDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new carray_cursor object. +*/ +static int carrayOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + carray_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a carray_cursor. +*/ +static int carrayClose(sqlite3_vtab_cursor *cur){ + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a carray_cursor to its next row of output. +*/ +static int carrayNext(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the carray_cursor +** is currently pointing. +*/ +static int carrayColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + carray_cursor *pCur = (carray_cursor*)cur; + sqlite3_int64 x = 0; + switch( i ){ + case CARRAY_COLUMN_POINTER: return SQLITE_OK; + case CARRAY_COLUMN_COUNT: x = pCur->iCnt; break; + case CARRAY_COLUMN_CTYPE: { + sqlite3_result_text(ctx, azType[pCur->eType], -1, SQLITE_STATIC); + return SQLITE_OK; + } + default: { + switch( pCur->eType ){ + case CARRAY_INT32: { + int *p = (int*)pCur->pPtr; + sqlite3_result_int(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_INT64: { + sqlite3_int64 *p = (sqlite3_int64*)pCur->pPtr; + sqlite3_result_int64(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_DOUBLE: { + double *p = (double*)pCur->pPtr; + sqlite3_result_double(ctx, p[pCur->iRowid-1]); + return SQLITE_OK; + } + case CARRAY_TEXT: { + const char **p = (const char**)pCur->pPtr; + sqlite3_result_text(ctx, p[pCur->iRowid-1], -1, SQLITE_TRANSIENT); + return SQLITE_OK; + } + case CARRAY_BLOB: { + const struct iovec *p = (struct iovec*)pCur->pPtr; + sqlite3_result_blob(ctx, p[pCur->iRowid-1].iov_base, + (int)p[pCur->iRowid-1].iov_len, SQLITE_TRANSIENT); + return SQLITE_OK; + } + } + } + } + sqlite3_result_int64(ctx, x); + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int carrayRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + carray_cursor *pCur = (carray_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int carrayEof(sqlite3_vtab_cursor *cur){ + carray_cursor *pCur = (carray_cursor*)cur; + return pCur->iRowid>pCur->iCnt; +} + +/* +** This method is called to "rewind" the carray_cursor object back +** to the first row of output. +*/ +static int carrayFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + carray_cursor *pCur = (carray_cursor *)pVtabCursor; + pCur->pPtr = 0; + pCur->iCnt = 0; + switch( idxNum ){ + case 1: { + carray_bind *pBind = sqlite3_value_pointer(argv[0], "carray-bind"); + if( pBind==0 ) break; + pCur->pPtr = pBind->aData; + pCur->iCnt = pBind->nData; + pCur->eType = pBind->mFlags & 0x07; + break; + } + case 2: + case 3: { + pCur->pPtr = sqlite3_value_pointer(argv[0], "carray"); + pCur->iCnt = pCur->pPtr ? sqlite3_value_int64(argv[1]) : 0; + if( idxNum<3 ){ + pCur->eType = CARRAY_INT32; + }else{ + unsigned char i; + const char *zType = (const char*)sqlite3_value_text(argv[2]); + for(i=0; i=sizeof(azType)/sizeof(azType[0]) ){ + pVtabCursor->pVtab->zErrMsg = sqlite3_mprintf( + "unknown datatype: %Q", zType); + return SQLITE_ERROR; + }else{ + pCur->eType = i; + } + } + break; + } + } + pCur->iRowid = 1; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the carray virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** idxNum is: +** +** 1 If only the pointer= constraint exists. In this case, the +** parameter must be bound using sqlite3_carray_bind(). +** +** 2 if the pointer= and count= constraints exist. +** +** 3 if the ctype= constraint also exists. +** +** idxNum is 0 otherwise and carray becomes an empty table. +*/ +static int carrayBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int ptrIdx = -1; /* Index of the pointer= constraint, or -1 if none */ + int cntIdx = -1; /* Index of the count= constraint, or -1 if none */ + int ctypeIdx = -1; /* Index of the ctype= constraint, or -1 if none */ + + const struct sqlite3_index_constraint *pConstraint; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case CARRAY_COLUMN_POINTER: + ptrIdx = i; + break; + case CARRAY_COLUMN_COUNT: + cntIdx = i; + break; + case CARRAY_COLUMN_CTYPE: + ctypeIdx = i; + break; + } + } + if( ptrIdx>=0 ){ + pIdxInfo->aConstraintUsage[ptrIdx].argvIndex = 1; + pIdxInfo->aConstraintUsage[ptrIdx].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 1; + if( cntIdx>=0 ){ + pIdxInfo->aConstraintUsage[cntIdx].argvIndex = 2; + pIdxInfo->aConstraintUsage[cntIdx].omit = 1; + pIdxInfo->idxNum = 2; + if( ctypeIdx>=0 ){ + pIdxInfo->aConstraintUsage[ctypeIdx].argvIndex = 3; + pIdxInfo->aConstraintUsage[ctypeIdx].omit = 1; + pIdxInfo->idxNum = 3; + } + } + }else{ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + pIdxInfo->idxNum = 0; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** carray virtual table. +*/ +static sqlite3_module carrayModule = { + 0, /* iVersion */ + 0, /* xCreate */ + carrayConnect, /* xConnect */ + carrayBestIndex, /* xBestIndex */ + carrayDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + carrayOpen, /* xOpen - open a cursor */ + carrayClose, /* xClose - close a cursor */ + carrayFilter, /* xFilter - configure scan constraints */ + carrayNext, /* xNext - advance a cursor */ + carrayEof, /* xEof - check for end of scan */ + carrayColumn, /* xColumn - read data */ + carrayRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadow */ + 0 /* xIntegrity */ +}; + +/* +** Destructor for the carray_bind object +*/ +static void carrayBindDel(void *pPtr){ + carray_bind *p = (carray_bind*)pPtr; + if( p->xDel!=SQLITE_STATIC ){ + p->xDel(p->aData); + } + sqlite3_free(p); +} + +/* +** Invoke this interface in order to bind to the single-argument +** version of CARRAY(). +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, + int idx, + void *aData, + int nData, + int mFlags, + void (*xDestroy)(void*) +){ + carray_bind *pNew; + int i; + pNew = sqlite3_malloc64(sizeof(*pNew)); + if( pNew==0 ){ + if( xDestroy!=SQLITE_STATIC && xDestroy!=SQLITE_TRANSIENT ){ + xDestroy(aData); + } + return SQLITE_NOMEM; + } + pNew->nData = nData; + pNew->mFlags = mFlags; + if( xDestroy==SQLITE_TRANSIENT ){ + sqlite3_int64 sz = nData; + switch( mFlags & 0x07 ){ + case CARRAY_INT32: sz *= 4; break; + case CARRAY_INT64: sz *= 8; break; + case CARRAY_DOUBLE: sz *= 8; break; + case CARRAY_TEXT: sz *= sizeof(char*); break; + case CARRAY_BLOB: sz *= sizeof(struct iovec); break; + } + if( (mFlags & 0x07)==CARRAY_TEXT ){ + for(i=0; iaData = sqlite3_malloc64( sz ); + if( pNew->aData==0 ){ + sqlite3_free(pNew); + return SQLITE_NOMEM; + } + if( (mFlags & 0x07)==CARRAY_TEXT ){ + char **az = (char**)pNew->aData; + char *z = (char*)&az[nData]; + for(i=0; iaData; + unsigned char *z = (unsigned char*)&p[nData]; + for(i=0; iaData, aData, sz); + } + pNew->xDel = sqlite3_free; + }else{ + pNew->aData = aData; + pNew->xDel = xDestroy; + } + return sqlite3_bind_pointer(pStmt, idx, pNew, "carray-bind", carrayBindDel); +} + + +/* +** For testing purpose in the TCL test harness, we need a method for +** setting the pointer value. The inttoptr(X) SQL function accomplishes +** this. Tcl script will bind an integer to X and the inttoptr() SQL +** function will use sqlite3_result_pointer() to convert that integer into +** a pointer. +** +** This is for testing on TCL only. +*/ +#ifdef SQLITE_TEST +static void inttoptrFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + void *p; + sqlite3_int64 i64; + i64 = sqlite3_value_int64(argv[0]); + if( sizeof(i64)==sizeof(p) ){ + memcpy(&p, &i64, sizeof(p)); + }else{ + int i32 = i64 & 0xffffffff; + memcpy(&p, &i32, sizeof(p)); + } + sqlite3_result_pointer(context, p, "carray", 0); +} +#endif /* SQLITE_TEST */ + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +SQLITE_API int sqlite3_carray_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "carray", &carrayModule, 0); +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "inttoptr", 1, SQLITE_UTF8, 0, + inttoptrFunc, 0, 0); + } +#endif /* SQLITE_TEST */ +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + return rc; +} diff --git a/ext/misc/carray.h b/ext/misc/carray.h new file mode 100644 index 0000000..ae0a9e8 --- /dev/null +++ b/ext/misc/carray.h @@ -0,0 +1,50 @@ +/* +** 2020-11-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Interface definitions for the CARRAY table-valued function +** extension. +*/ + +#ifndef _CARRAY_H +#define _CARRAY_H + +#include "sqlite3.h" /* Required for error code definitions */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* Use this interface to bind an array to the single-argument version +** of CARRAY(). +*/ +SQLITE_API int sqlite3_carray_bind( + sqlite3_stmt *pStmt, /* Statement to be bound */ + int i, /* Parameter index */ + void *aData, /* Pointer to array data */ + int nData, /* Number of data elements */ + int mFlags, /* CARRAY flags */ + void (*xDel)(void*) /* Destructgor for aData*/ +); + +/* Allowed values for the mFlags parameter to sqlite3_carray_bind(). +*/ +#define CARRAY_INT32 0 /* Data is 32-bit signed integers */ +#define CARRAY_INT64 1 /* Data is 64-bit signed integers */ +#define CARRAY_DOUBLE 2 /* Data is doubles */ +#define CARRAY_TEXT 3 /* Data is char* */ +#define CARRAY_BLOB 4 /* Data is struct iovec */ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _CARRAY_H */ diff --git a/ext/misc/cksumvfs.c b/ext/misc/cksumvfs.c new file mode 100644 index 0000000..e7c2c9d --- /dev/null +++ b/ext/misc/cksumvfs.c @@ -0,0 +1,880 @@ +/* +** 2020-04-20 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a VFS shim that writes a checksum on each page +** of an SQLite database file. When reading pages, the checksum is verified +** and an error is raised if the checksum is incorrect. +** +** COMPILING +** +** This extension requires SQLite 3.32.0 or later. It uses the +** sqlite3_database_file_object() interface which was added in +** version 3.32.0, so it will not link with an earlier version of +** SQLite. +** +** To build this extension as a separately loaded shared library or +** DLL, use compiler command-lines similar to the following: +** +** (linux) gcc -fPIC -shared cksumvfs.c -o cksumvfs.so +** (mac) clang -fPIC -dynamiclib cksumvfs.c -o cksumvfs.dylib +** (windows) cl cksumvfs.c -link -dll -out:cksumvfs.dll +** +** You may want to add additional compiler options, of course, +** according to the needs of your project. +** +** If you want to statically link this extension with your product, +** then compile it like any other C-language module but add the +** "-DSQLITE_CKSUMVFS_STATIC" option so that this module knows that +** it is being statically linked rather than dynamically linked +** +** LOADING +** +** To load this extension as a shared library, you first have to +** bring up a dummy SQLite database connection to use as the argument +** to the sqlite3_load_extension() API call. Then you invoke the +** sqlite3_load_extension() API and shutdown the dummy database +** connection. All subsequent database connections that are opened +** will include this extension. For example: +** +** sqlite3 *db; +** sqlite3_open(":memory:", &db); +** sqlite3_load_extension(db, "./cksumvfs"); +** sqlite3_close(db); +** +** If this extension is compiled with -DSQLITE_CKSUMVFS_STATIC and +** statically linked against the application, initialize it using +** a single API call as follows: +** +** sqlite3_register_cksumvfs(); +** +** Cksumvfs is a VFS Shim. When loaded, "cksmvfs" becomes the new +** default VFS and it uses the prior default VFS as the next VFS +** down in the stack. This is normally what you want. However, in +** complex situations where multiple VFS shims are being loaded, +** it might be important to ensure that cksumvfs is loaded in the +** correct order so that it sequences itself into the default VFS +** Shim stack in the right order. +** +** USING +** +** Open database connections using the sqlite3_open() or +** sqlite3_open_v2() interfaces, as normal. Ordinary database files +** (without a checksum) will operate normally. Databases with +** checksums will return an SQLITE_IOERR_DATA error if a page is +** encountered that contains an invalid checksum. +** +** Checksumming only works on databases that have a reserve-bytes +** value of exactly 8. The default value for reserve-bytes is 0. +** Hence, newly created database files will omit the checksum by +** default. To create a database that includes a checksum, change +** the reserve-bytes value to 8 by runing: +** +** int n = 8; +** sqlite3_file_control(db, 0, SQLITE_FCNTL_RESERVE_BYTES, &n); +** +** If you do this immediately after creating a new database file, +** before anything else has been written into the file, then that +** might be all that you need to do. Otherwise, the API call +** above should be followed by: +** +** sqlite3_exec(db, "VACUUM", 0, 0, 0); +** +** It never hurts to run the VACUUM, even if you don't need it. +** If the database is in WAL mode, you should shutdown and +** reopen all database connections before continuing. +** +** From the CLI, use the ".filectrl reserve_bytes 8" command, +** followed by "VACUUM;". +** +** Note that SQLite allows the number of reserve-bytes to be +** increased but not decreased. So if a database file already +** has a reserve-bytes value greater than 8, there is no way to +** activate checksumming on that database, other than to dump +** and restore the database file. Note also that other extensions +** might also make use of the reserve-bytes. Checksumming will +** be incompatible with those other extensions. +** +** VERIFICATION OF CHECKSUMS +** +** If any checksum is incorrect, the "PRAGMA quick_check" command +** will find it. To verify that checksums are actually enabled +** and running, use the following query: +** +** SELECT count(*), verify_checksum(data) +** FROM sqlite_dbpage +** GROUP BY 2; +** +** There are three possible outputs form the verify_checksum() +** function: 1, 0, and NULL. 1 is returned if the checksum is +** correct. 0 is returned if the checksum is incorrect. NULL +** is returned if the page is unreadable. If checksumming is +** enabled, the read will fail if the checksum is wrong, so the +** usual result from verify_checksum() on a bad checksum is NULL. +** +** If everything is OK, the query above should return a single +** row where the second column is 1. Any other result indicates +** either that there is a checksum error, or checksum validation +** is disabled. +** +** CONTROLLING CHECKSUM VERIFICATION +** +** The cksumvfs extension implements a new PRAGMA statement that can +** be used to disable, re-enable, or query the status of checksum +** verification: +** +** PRAGMA checksum_verification; -- query status +** PRAGMA checksum_verification=OFF; -- disable verification +** PRAGMA checksum_verification=ON; -- re-enable verification +** +** The "checksum_verification" pragma will return "1" (true) or "0" +** (false) if checksum verification is enabled or disabled, respectively. +** "Verification" in this context means the feature that causes +** SQLITE_IOERR_DATA errors if a checksum mismatch is detected while +** reading. Checksums are always kept up-to-date as long as the +** reserve-bytes value of the database is 8, regardless of the setting +** of this pragma. Checksum verification can be disabled (for example) +** to do forensic analysis of a database that has previously reported +** a checksum error. +** +** The "checksum_verification" pragma will always respond with "0" if +** the database file does not have a reserve-bytes value of 8. The +** pragma will return no rows at all if the cksumvfs extension is +** not loaded. +** +** IMPLEMENTATION NOTES +** +** The checksum is stored in the last 8 bytes of each page. This +** module only operates if the "bytes of reserved space on each page" +** value at offset 20 the SQLite database header is exactly 8. If +** the reserved-space value is not 8, this module is a no-op. +*/ +#if defined(SQLITE_AMALGAMATION) && !defined(SQLITE_CKSUMVFS_STATIC) +# define SQLITE_CKSUMVFS_STATIC +#endif +#ifdef SQLITE_CKSUMVFS_STATIC +# include "sqlite3.h" +#else +# include "sqlite3ext.h" + SQLITE_EXTENSION_INIT1 +#endif +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs CksmVfs; +typedef struct CksmFile CksmFile; + +/* +** Useful datatype abbreviations +*/ +#if !defined(SQLITE_AMALGAMATION) + typedef unsigned char u8; + typedef unsigned int u32; +#endif + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) +#define ORIGFILE(p) ((sqlite3_file*)(((CksmFile*)(p))+1)) + +/* An open file */ +struct CksmFile { + sqlite3_file base; /* IO methods */ + const char *zFName; /* Original name of the file */ + char computeCksm; /* True to compute checksums. + ** Always true if reserve size is 8. */ + char verifyCksm; /* True to verify checksums */ + char isWal; /* True if processing a WAL file */ + char inCkpt; /* Currently doing a checkpoint */ + CksmFile *pPartner; /* Ptr from WAL to main-db, or from main-db to WAL */ +}; + +/* +** Methods for CksmFile +*/ +static int cksmClose(sqlite3_file*); +static int cksmRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int cksmWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int cksmTruncate(sqlite3_file*, sqlite3_int64 size); +static int cksmSync(sqlite3_file*, int flags); +static int cksmFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int cksmLock(sqlite3_file*, int); +static int cksmUnlock(sqlite3_file*, int); +static int cksmCheckReservedLock(sqlite3_file*, int *pResOut); +static int cksmFileControl(sqlite3_file*, int op, void *pArg); +static int cksmSectorSize(sqlite3_file*); +static int cksmDeviceCharacteristics(sqlite3_file*); +static int cksmShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int cksmShmLock(sqlite3_file*, int offset, int n, int flags); +static void cksmShmBarrier(sqlite3_file*); +static int cksmShmUnmap(sqlite3_file*, int deleteFlag); +static int cksmFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int cksmUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for CksmVfs +*/ +static int cksmOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int cksmDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int cksmAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int cksmFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *cksmDlOpen(sqlite3_vfs*, const char *zFilename); +static void cksmDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void cksmDlClose(sqlite3_vfs*, void*); +static int cksmRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int cksmSleep(sqlite3_vfs*, int microseconds); +static int cksmCurrentTime(sqlite3_vfs*, double*); +static int cksmGetLastError(sqlite3_vfs*, int, char *); +static int cksmCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); +static int cksmSetSystemCall(sqlite3_vfs*, const char*,sqlite3_syscall_ptr); +static sqlite3_syscall_ptr cksmGetSystemCall(sqlite3_vfs*, const char *z); +static const char *cksmNextSystemCall(sqlite3_vfs*, const char *zName); + +static sqlite3_vfs cksm_vfs = { + 3, /* iVersion (set when registered) */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "cksmvfs", /* zName */ + 0, /* pAppData (set when registered) */ + cksmOpen, /* xOpen */ + cksmDelete, /* xDelete */ + cksmAccess, /* xAccess */ + cksmFullPathname, /* xFullPathname */ + cksmDlOpen, /* xDlOpen */ + cksmDlError, /* xDlError */ + cksmDlSym, /* xDlSym */ + cksmDlClose, /* xDlClose */ + cksmRandomness, /* xRandomness */ + cksmSleep, /* xSleep */ + cksmCurrentTime, /* xCurrentTime */ + cksmGetLastError, /* xGetLastError */ + cksmCurrentTimeInt64, /* xCurrentTimeInt64 */ + cksmSetSystemCall, /* xSetSystemCall */ + cksmGetSystemCall, /* xGetSystemCall */ + cksmNextSystemCall /* xNextSystemCall */ +}; + +static const sqlite3_io_methods cksm_io_methods = { + 3, /* iVersion */ + cksmClose, /* xClose */ + cksmRead, /* xRead */ + cksmWrite, /* xWrite */ + cksmTruncate, /* xTruncate */ + cksmSync, /* xSync */ + cksmFileSize, /* xFileSize */ + cksmLock, /* xLock */ + cksmUnlock, /* xUnlock */ + cksmCheckReservedLock, /* xCheckReservedLock */ + cksmFileControl, /* xFileControl */ + cksmSectorSize, /* xSectorSize */ + cksmDeviceCharacteristics, /* xDeviceCharacteristics */ + cksmShmMap, /* xShmMap */ + cksmShmLock, /* xShmLock */ + cksmShmBarrier, /* xShmBarrier */ + cksmShmUnmap, /* xShmUnmap */ + cksmFetch, /* xFetch */ + cksmUnfetch /* xUnfetch */ +}; + +/* Do byte swapping on a unsigned 32-bit integer */ +#define BYTESWAP32(x) ( \ + (((x)&0x000000FF)<<24) + (((x)&0x0000FF00)<<8) \ + + (((x)&0x00FF0000)>>8) + (((x)&0xFF000000)>>24) \ +) + +/* Compute a checksum on a buffer */ +static void cksmCompute( + u8 *a, /* Content to be checksummed */ + int nByte, /* Bytes of content in a[]. Must be a multiple of 8. */ + u8 *aOut /* OUT: Final 8-byte checksum value output */ +){ + u32 s1 = 0, s2 = 0; + u32 *aData = (u32*)a; + u32 *aEnd = (u32*)&a[nByte]; + u32 x = 1; + + assert( nByte>=8 ); + assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + + if( 1 == *(u8*)&x ){ + /* Little-endian */ + do { + s1 += *aData++ + s2; + s2 += *aData++ + s1; + }while( aData65536 || (nByte & (nByte-1))!=0 ) return; + cksmCompute(data, nByte-8, cksum); + sqlite3_result_int(context, memcmp(data+nByte-8,cksum,8)==0); +} + +#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME +/* +** SQL function: initialize_cksumvfs(SCHEMANAME) +** +** This SQL functions (whose name is actually determined at compile-time +** by the value of the SQLITE_CKSUMVFS_INIT_FUNCNAME macro) invokes: +** +** sqlite3_file_control(db, SCHEMANAME, SQLITE_FCNTL_RESERVE_BYTE, &n); +** +** In order to set the reserve bytes value to 8, so that cksumvfs will +** operation. This feature is provided (if and only if the +** SQLITE_CKSUMVFS_INIT_FUNCNAME compile-time option is set to a string +** which is the name of the SQL function) so as to provide the ability +** to invoke the file-control in programming languages that lack +** direct access to the sqlite3_file_control() interface (ex: Java). +** +** This interface is undocumented, apart from this comment. Usage +** example: +** +** 1. Compile with -DSQLITE_CKSUMVFS_INIT_FUNCNAME="ckvfs_init" +** 2. Run: "SELECT cksum_init('main'); VACUUM;" +*/ +static void cksmInitFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nByte = 8; + const char *zSchemaName = (const char*)sqlite3_value_text(argv[0]); + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_file_control(db, zSchemaName, SQLITE_FCNTL_RESERVE_BYTES, &nByte); + /* Return NULL */ +} +#endif /* SQLITE_CKSUMBFS_INIT_FUNCNAME */ + +/* +** Close a cksm-file. +*/ +static int cksmClose(sqlite3_file *pFile){ + CksmFile *p = (CksmFile *)pFile; + if( p->pPartner ){ + assert( p->pPartner->pPartner==p ); + p->pPartner->pPartner = 0; + p->pPartner = 0; + } + pFile = ORIGFILE(pFile); + return pFile->pMethods->xClose(pFile); +} + +/* +** Set the computeCkSm and verifyCksm flags, if they need to be +** changed. +*/ +static void cksmSetFlags(CksmFile *p, int hasCorrectReserveSize){ + if( hasCorrectReserveSize!=p->computeCksm ){ + p->computeCksm = p->verifyCksm = hasCorrectReserveSize; + if( p->pPartner ){ + p->pPartner->verifyCksm = hasCorrectReserveSize; + p->pPartner->computeCksm = hasCorrectReserveSize; + } + } +} + +/* +** Read data from a cksm-file. +*/ +static int cksmRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + int rc; + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + rc = pFile->pMethods->xRead(pFile, zBuf, iAmt, iOfst); + if( rc==SQLITE_OK ){ + if( iOfst==0 && iAmt>=100 && ( + memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 + )){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + cksmSetFlags(p, hasCorrectReserveSize); + } + /* Verify the checksum if + ** (1) the size indicates that we are dealing with a complete + ** database page + ** (2) checksum verification is enabled + ** (3) we are not in the middle of checkpoint + */ + if( iAmt>=512 /* (1) */ + && p->verifyCksm /* (2) */ + && !p->inCkpt /* (3) */ + ){ + u8 cksum[8]; + cksmCompute((u8*)zBuf, iAmt-8, cksum); + if( memcmp((u8*)zBuf+iAmt-8, cksum, 8)!=0 ){ + sqlite3_log(SQLITE_IOERR_DATA, + "checksum fault offset %lld of \"%s\"", + iOfst, p->zFName); + rc = SQLITE_IOERR_DATA; + } + } + } + return rc; +} + +/* +** Write data to a cksm-file. +*/ +static int cksmWrite( + sqlite3_file *pFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(pFile); + if( iOfst==0 && iAmt>=100 && ( + memcmp(zBuf,"SQLite format 3",16)==0 || memcmp(zBuf,"ZV-",3)==0 + )){ + u8 *d = (u8*)zBuf; + char hasCorrectReserveSize = (d[20]==8); + cksmSetFlags(p, hasCorrectReserveSize); + } + /* If the write size is appropriate for a database page and if + ** checksums where ever enabled, then it will be safe to compute + ** the checksums. The reserve byte size might have increased, but + ** it will never decrease. And because it cannot decrease, the + ** checksum will not overwrite anything. + */ + if( iAmt>=512 + && p->computeCksm + && !p->inCkpt + ){ + cksmCompute((u8*)zBuf, iAmt-8, ((u8*)zBuf)+iAmt-8); + } + return pFile->pMethods->xWrite(pFile, zBuf, iAmt, iOfst); +} + +/* +** Truncate a cksm-file. +*/ +static int cksmTruncate(sqlite3_file *pFile, sqlite_int64 size){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xTruncate(pFile, size); +} + +/* +** Sync a cksm-file. +*/ +static int cksmSync(sqlite3_file *pFile, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSync(pFile, flags); +} + +/* +** Return the current file-size of a cksm-file. +*/ +static int cksmFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + CksmFile *p = (CksmFile *)pFile; + pFile = ORIGFILE(p); + return pFile->pMethods->xFileSize(pFile, pSize); +} + +/* +** Lock a cksm-file. +*/ +static int cksmLock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xLock(pFile, eLock); +} + +/* +** Unlock a cksm-file. +*/ +static int cksmUnlock(sqlite3_file *pFile, int eLock){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xUnlock(pFile, eLock); +} + +/* +** Check if another file-handle holds a RESERVED lock on a cksm-file. +*/ +static int cksmCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xCheckReservedLock(pFile, pResOut); +} + +/* +** File control method. For custom operations on a cksm-file. +*/ +static int cksmFileControl(sqlite3_file *pFile, int op, void *pArg){ + int rc; + CksmFile *p = (CksmFile*)pFile; + pFile = ORIGFILE(pFile); + if( op==SQLITE_FCNTL_PRAGMA ){ + char **azArg = (char**)pArg; + assert( azArg[1]!=0 ); + if( sqlite3_stricmp(azArg[1],"checksum_verification")==0 ){ + char *zArg = azArg[2]; + if( zArg!=0 ){ + if( (zArg[0]>='1' && zArg[0]<='9') + || sqlite3_strlike("enable%",zArg,0)==0 + || sqlite3_stricmp("yes",zArg)==0 + || sqlite3_stricmp("on",zArg)==0 + ){ + p->verifyCksm = p->computeCksm; + }else{ + p->verifyCksm = 0; + } + if( p->pPartner ) p->pPartner->verifyCksm = p->verifyCksm; + } + azArg[0] = sqlite3_mprintf("%d",p->verifyCksm); + return SQLITE_OK; + }else if( p->computeCksm && azArg[2]!=0 + && sqlite3_stricmp(azArg[1], "page_size")==0 ){ + /* Do not allow page size changes on a checksum database */ + return SQLITE_OK; + } + }else if( op==SQLITE_FCNTL_CKPT_START || op==SQLITE_FCNTL_CKPT_DONE ){ + p->inCkpt = op==SQLITE_FCNTL_CKPT_START; + if( p->pPartner ) p->pPartner->inCkpt = p->inCkpt; + }else if( op==SQLITE_FCNTL_CKSM_FILE ){ + /* This VFS needs to obtain a pointer to the corresponding database + ** file handle from within xOpen() calls to open wal files. To do this, + ** it uses the sqlite3_database_file_object() API to obtain a pointer + ** to the file-handle used by SQLite to access the db file. This is + ** fine if cksmvfs happens to be the top-level VFS, but not if there + ** are one or more wrapper VFS. To handle this case, this file-control + ** is used to extract the cksmvfs file-handle from any wrapper file + ** handle. */ + sqlite3_file **ppFile = (sqlite3_file**)pArg; + *ppFile = (sqlite3_file*)p; + return SQLITE_OK; + } + rc = pFile->pMethods->xFileControl(pFile, op, pArg); + if( rc==SQLITE_OK && op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("cksm/%z", *(char**)pArg); + } + return rc; +} + +/* +** Return the sector-size in bytes for a cksm-file. +*/ +static int cksmSectorSize(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xSectorSize(pFile); +} + +/* +** Return the device characteristic flags supported by a cksm-file. +*/ +static int cksmDeviceCharacteristics(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xDeviceCharacteristics(pFile); +} + +/* Create a shared memory file mapping */ +static int cksmShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmMap(pFile,iPg,pgsz,bExtend,pp); +} + +/* Perform locking on a shared-memory segment */ +static int cksmShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmLock(pFile,offset,n,flags); +} + +/* Memory barrier operation on shared memory */ +static void cksmShmBarrier(sqlite3_file *pFile){ + pFile = ORIGFILE(pFile); + pFile->pMethods->xShmBarrier(pFile); +} + +/* Unmap a shared memory segment */ +static int cksmShmUnmap(sqlite3_file *pFile, int deleteFlag){ + pFile = ORIGFILE(pFile); + return pFile->pMethods->xShmUnmap(pFile,deleteFlag); +} + +/* Fetch a page of a memory-mapped file */ +static int cksmFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + CksmFile *p = (CksmFile *)pFile; + if( p->computeCksm ){ + *pp = 0; + return SQLITE_OK; + } + pFile = ORIGFILE(pFile); + if( pFile->pMethods->iVersion>2 && pFile->pMethods->xFetch ){ + return pFile->pMethods->xFetch(pFile, iOfst, iAmt, pp); + } + *pp = 0; + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int cksmUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + pFile = ORIGFILE(pFile); + if( pFile->pMethods->iVersion>2 && pFile->pMethods->xUnfetch ){ + return pFile->pMethods->xUnfetch(pFile, iOfst, pPage); + } + return SQLITE_OK; +} + +/* +** Open a cksm file handle. +*/ +static int cksmOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + CksmFile *p; + sqlite3_file *pSubFile; + sqlite3_vfs *pSubVfs; + int rc; + pSubVfs = ORIGVFS(pVfs); + if( (flags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_WAL))==0 ){ + return pSubVfs->xOpen(pSubVfs, zName, pFile, flags, pOutFlags); + } + p = (CksmFile*)pFile; + memset(p, 0, sizeof(*p)); + pSubFile = ORIGFILE(pFile); + pFile->pMethods = &cksm_io_methods; + rc = pSubVfs->xOpen(pSubVfs, zName, pSubFile, flags, pOutFlags); + if( rc ) goto cksm_open_done; + if( flags & SQLITE_OPEN_WAL ){ + sqlite3_file *pDb = sqlite3_database_file_object(zName); + rc = pDb->pMethods->xFileControl(pDb, SQLITE_FCNTL_CKSM_FILE, (void*)&pDb); + assert( rc==SQLITE_OK ); + p->pPartner = (CksmFile*)pDb; + assert( p->pPartner->pPartner==0 ); + p->pPartner->pPartner = p; + p->isWal = 1; + p->computeCksm = p->pPartner->computeCksm; + }else{ + p->isWal = 0; + p->computeCksm = 0; + } + p->zFName = zName; +cksm_open_done: + if( rc ) pFile->pMethods = 0; + return rc; +} + +/* +** All other VFS methods are pass-thrus. +*/ +static int cksmDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return ORIGVFS(pVfs)->xDelete(ORIGVFS(pVfs), zPath, dirSync); +} +static int cksmAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + return ORIGVFS(pVfs)->xAccess(ORIGVFS(pVfs), zPath, flags, pResOut); +} +static int cksmFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + return ORIGVFS(pVfs)->xFullPathname(ORIGVFS(pVfs),zPath,nOut,zOut); +} +static void *cksmDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} +static void cksmDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} +static void (*cksmDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} +static void cksmDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} +static int cksmRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} +static int cksmSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} +static int cksmCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +static int cksmGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int cksmCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + sqlite3_vfs *pOrig = ORIGVFS(pVfs); + int rc; + assert( pOrig->iVersion>=2 ); + if( pOrig->xCurrentTimeInt64 ){ + rc = pOrig->xCurrentTimeInt64(pOrig, p); + }else{ + double r; + rc = pOrig->xCurrentTime(pOrig, &r); + *p = (sqlite3_int64)(r*86400000.0); + } + return rc; +} +static int cksmSetSystemCall( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_syscall_ptr pCall +){ + return ORIGVFS(pVfs)->xSetSystemCall(ORIGVFS(pVfs),zName,pCall); +} +static sqlite3_syscall_ptr cksmGetSystemCall( + sqlite3_vfs *pVfs, + const char *zName +){ + return ORIGVFS(pVfs)->xGetSystemCall(ORIGVFS(pVfs),zName); +} +static const char *cksmNextSystemCall(sqlite3_vfs *pVfs, const char *zName){ + return ORIGVFS(pVfs)->xNextSystemCall(ORIGVFS(pVfs), zName); +} + +/* Register the verify_checksum() SQL function. +*/ +static int cksmRegisterFunc( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + if( db==0 ) return SQLITE_OK; + rc = sqlite3_create_function(db, "verify_checksum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + 0, cksmVerifyFunc, 0, 0); +#ifdef SQLITE_CKSUMVFS_INIT_FUNCNAME + (void)sqlite3_create_function(db, SQLITE_CKSUMVFS_INIT_FUNCNAME, 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, + 0, cksmInitFunc, 0, 0); +#endif + return rc; +} + +/* +** Register the cksum VFS as the default VFS for the system. +** Also make arrangements to automatically register the "verify_checksum()" +** SQL function on each new database connection. +*/ +static int cksmRegisterVfs(void){ + int rc = SQLITE_OK; + sqlite3_vfs *pOrig; + if( sqlite3_vfs_find("cksmvfs")!=0 ) return SQLITE_OK; + pOrig = sqlite3_vfs_find(0); + if( pOrig==0 ) return SQLITE_ERROR; + cksm_vfs.iVersion = pOrig->iVersion; + cksm_vfs.pAppData = pOrig; + cksm_vfs.szOsFile = pOrig->szOsFile + sizeof(CksmFile); + rc = sqlite3_vfs_register(&cksm_vfs, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))cksmRegisterFunc); + } + return rc; +} + +#if defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer runs when the extension is +** statically linked. +*/ +int sqlite3_register_cksumvfs(const char *NotUsed){ + (void)NotUsed; + return cksmRegisterVfs(); +} +int sqlite3_unregister_cksumvfs(void){ + if( sqlite3_vfs_find("cksmvfs") ){ + sqlite3_vfs_unregister(&cksm_vfs); + sqlite3_cancel_auto_extension((void(*)(void))cksmRegisterFunc); + } + return SQLITE_OK; +} +#endif /* defined(SQLITE_CKSUMVFS_STATIC */ + +#if !defined(SQLITE_CKSUMVFS_STATIC) +/* This variant of the initializer function is used when the +** extension is shared library to be loaded at run-time. +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called by sqlite3_load_extension() when the +** extension is first loaded. +***/ +int sqlite3_cksumvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* not used */ + rc = cksmRegisterFunc(db, 0, 0); + if( rc==SQLITE_OK ){ + rc = cksmRegisterVfs(); + } + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} +#endif /* !defined(SQLITE_CKSUMVFS_STATIC) */ diff --git a/ext/misc/closure.c b/ext/misc/closure.c new file mode 100644 index 0000000..79a5a21 --- /dev/null +++ b/ext/misc/closure.c @@ -0,0 +1,966 @@ +/* +** 2013-04-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file contains code for a virtual table that finds the transitive +** closure of a parent/child relationship in a real table. The virtual +** table is called "transitive_closure". +** +** A transitive_closure virtual table is created like this: +** +** CREATE VIRTUAL TABLE x USING transitive_closure( +** tablename=, -- T +** idcolumn=, -- X +** parentcolumn= -- P +** ); +** +** When it is created, the new transitive_closure table may be supplied +** with default values for the name of a table T and columns T.X and T.P. +** The T.X and T.P columns must contain integers. The ideal case is for +** T.X to be the INTEGER PRIMARY KEY. The T.P column should reference +** the T.X column. The row referenced by T.P is the parent of the current row. +** +** The tablename, idcolumn, and parentcolumn supplied by the CREATE VIRTUAL +** TABLE statement may be overridden in individual queries by including +** terms like tablename='newtable', idcolumn='id2', or +** parentcolumn='parent3' in the WHERE clause of the query. +** +** For efficiency, it is essential that there be an index on the P column: +** +** CREATE Tidx1 ON T(P) +** +** Suppose a specific instance of the closure table is as follows: +** +** CREATE VIRTUAL TABLE ct1 USING transitive_closure( +** tablename='group', +** idcolumn='groupId', +** parentcolumn='parentId' +** ); +** +** Such an instance of the transitive_closure virtual table would be +** appropriate for walking a tree defined using a table like this, for example: +** +** CREATE TABLE group( +** groupId INTEGER PRIMARY KEY, +** parentId INTEGER REFERENCES group +** ); +** CREATE INDEX group_idx1 ON group(parentId); +** +** The group table above would presumably have other application-specific +** fields. The key point here is that rows of the group table form a +** tree. The purpose of the ct1 virtual table is to easily extract +** branches of that tree. +** +** Once it has been created, the ct1 virtual table can be queried +** as follows: +** +** SELECT * FROM element +** WHERE element.groupId IN (SELECT id FROM ct1 WHERE root=?1); +** +** The above query will return all elements that are part of group ?1 +** or children of group ?1 or grand-children of ?1 and so forth for all +** descendents of group ?1. The same query can be formulated as a join: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1; +** +** The depth of the transitive_closure (the number of generations of +** parent/child relations to follow) can be limited by setting "depth" +** column in the WHERE clause. So, for example, the following query +** finds only children and grandchildren but no further descendents: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.depth<=2; +** +** The "ct1.depth<=2" term could be a strict equality "ct1.depth=2" in +** order to find only the grandchildren of ?1, not ?1 itself or the +** children of ?1. +** +** The root=?1 term must be supplied in WHERE clause or else the query +** of the ct1 virtual table will return an empty set. The tablename, +** idcolumn, and parentcolumn attributes can be overridden in the WHERE +** clause if desired. So, for example, the ct1 table could be repurposed +** to find ancestors rather than descendents by inverting the roles of +** the idcolumn and parentcolumn: +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupid=ct1.id +** AND ct1.root=?1 +** AND ct1.idcolumn='parentId' +** AND ct1.parentcolumn='groupId'; +** +** Multiple calls to ct1 could be combined. For example, the following +** query finds all elements that "cousins" of groupId ?1. That is to say +** elements where the groupId is a grandchild of the grandparent of ?1. +** (This definition of "cousins" also includes siblings and self.) +** +** SELECT element.* FROM element, ct1 +** WHERE element.groupId=ct1.id +** AND ct1.depth=2 +** AND ct1.root IN (SELECT id FROM ct1 +** WHERE root=?1 +** AND depth=2 +** AND idcolumn='parentId' +** AND parentcolumn='groupId'); +** +** In our example, the group.groupId column is unique and thus the +** subquery will return exactly one row. For that reason, the IN +** operator could be replaced by "=" to get the same result. But +** in the general case where the idcolumn is not unique, an IN operator +** would be required for this kind of query. +** +** Note that because the tablename, idcolumn, and parentcolumn can +** all be specified in the query, it is possible for an application +** to define a single transitive_closure virtual table for use on lots +** of different hierarchy tables. One might say: +** +** CREATE VIRTUAL TABLE temp.closure USING transitive_closure; +** +** As each database connection is being opened. Then the application +** would always have a "closure" virtual table handy to use for querying. +** +** SELECT element.* FROM element, closure +** WHERE element.groupid=ct1.id +** AND closure.root=?1 +** AND closure.tablename='group' +** AND closure.idname='groupId' +** AND closure.parentname='parentId'; +** +** See the documentation at http://www.sqlite.org/loadext.html for information +** on how to compile and use loadable extensions such as this one. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct closure_vtab closure_vtab; +typedef struct closure_cursor closure_cursor; +typedef struct closure_queue closure_queue; +typedef struct closure_avl closure_avl; + +/***************************************************************************** +** AVL Tree implementation +*/ +/* +** Objects that want to be members of the AVL tree should embedded an +** instance of this structure. +*/ +struct closure_avl { + sqlite3_int64 id; /* Id of this entry in the table */ + int iGeneration; /* Which generation is this entry part of */ + closure_avl *pList; /* A linked list of nodes */ + closure_avl *pBefore; /* Other elements less than id */ + closure_avl *pAfter; /* Other elements greater than id */ + closure_avl *pUp; /* Parent element */ + short int height; /* Height of this node. Leaf==1 */ + short int imbalance; /* Height difference between pBefore and pAfter */ +}; + +/* Recompute the closure_avl.height and closure_avl.imbalance fields for p. +** Assume that the children of p have correct heights. +*/ +static void closureAvlRecomputeHeight(closure_avl *p){ + short int hBefore = p->pBefore ? p->pBefore->height : 0; + short int hAfter = p->pAfter ? p->pAfter->height : 0; + p->imbalance = hBefore - hAfter; /* -: pAfter higher. +: pBefore higher */ + p->height = (hBefore>hAfter ? hBefore : hAfter)+1; +} + +/* +** P B +** / \ / \ +** B Z ==> X P +** / \ / \ +** X Y Y Z +** +*/ +static closure_avl *closureAvlRotateBefore(closure_avl *pP){ + closure_avl *pB = pP->pBefore; + closure_avl *pY = pB->pAfter; + pB->pUp = pP->pUp; + pB->pAfter = pP; + pP->pUp = pB; + pP->pBefore = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pB); + return pB; +} + +/* +** P A +** / \ / \ +** X A ==> P Z +** / \ / \ +** Y Z X Y +** +*/ +static closure_avl *closureAvlRotateAfter(closure_avl *pP){ + closure_avl *pA = pP->pAfter; + closure_avl *pY = pA->pBefore; + pA->pUp = pP->pUp; + pA->pBefore = pP; + pP->pUp = pA; + pP->pAfter = pY; + if( pY ) pY->pUp = pP; + closureAvlRecomputeHeight(pP); + closureAvlRecomputeHeight(pA); + return pA; +} + +/* +** Return a pointer to the pBefore or pAfter pointer in the parent +** of p that points to p. Or if p is the root node, return pp. +*/ +static closure_avl **closureAvlFromPtr(closure_avl *p, closure_avl **pp){ + closure_avl *pUp = p->pUp; + if( pUp==0 ) return pp; + if( pUp->pAfter==p ) return &pUp->pAfter; + return &pUp->pBefore; +} + +/* +** Rebalance all nodes starting with p and working up to the root. +** Return the new root. +*/ +static closure_avl *closureAvlBalance(closure_avl *p){ + closure_avl *pTop = p; + closure_avl **pp; + while( p ){ + closureAvlRecomputeHeight(p); + if( p->imbalance>=2 ){ + closure_avl *pB = p->pBefore; + if( pB->imbalance<0 ) p->pBefore = closureAvlRotateAfter(pB); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateBefore(p); + }else if( p->imbalance<=(-2) ){ + closure_avl *pA = p->pAfter; + if( pA->imbalance>0 ) p->pAfter = closureAvlRotateBefore(pA); + pp = closureAvlFromPtr(p,&p); + p = *pp = closureAvlRotateAfter(p); + } + pTop = p; + p = p->pUp; + } + return pTop; +} + +/* Search the tree rooted at p for an entry with id. Return a pointer +** to the entry or return NULL. +*/ +static closure_avl *closureAvlSearch(closure_avl *p, sqlite3_int64 id){ + while( p && id!=p->id ){ + p = (idid) ? p->pBefore : p->pAfter; + } + return p; +} + +/* Find the first node (the one with the smallest key). +*/ +static closure_avl *closureAvlFirst(closure_avl *p){ + if( p ) while( p->pBefore ) p = p->pBefore; + return p; +} + +/* Return the node with the next larger key after p. +*/ +closure_avl *closureAvlNext(closure_avl *p){ + closure_avl *pPrev = 0; + while( p && p->pAfter==pPrev ){ + pPrev = p; + p = p->pUp; + } + if( p && pPrev==0 ){ + p = closureAvlFirst(p->pAfter); + } + return p; +} + +/* Insert a new node pNew. Return NULL on success. If the key is not +** unique, then do not perform the insert but instead leave pNew unchanged +** and return a pointer to an existing node with the same key. +*/ +static closure_avl *closureAvlInsert( + closure_avl **ppHead, /* Head of the tree */ + closure_avl *pNew /* New node to be inserted */ +){ + closure_avl *p = *ppHead; + if( p==0 ){ + p = pNew; + pNew->pUp = 0; + }else{ + while( p ){ + if( pNew->idid ){ + if( p->pBefore ){ + p = p->pBefore; + }else{ + p->pBefore = pNew; + pNew->pUp = p; + break; + } + }else if( pNew->id>p->id ){ + if( p->pAfter ){ + p = p->pAfter; + }else{ + p->pAfter = pNew; + pNew->pUp = p; + break; + } + }else{ + return p; + } + } + } + pNew->pBefore = 0; + pNew->pAfter = 0; + pNew->height = 1; + pNew->imbalance = 0; + *ppHead = closureAvlBalance(p); + return 0; +} + +/* Walk the tree can call xDestroy on each node +*/ +static void closureAvlDestroy(closure_avl *p, void (*xDestroy)(closure_avl*)){ + if( p ){ + closureAvlDestroy(p->pBefore, xDestroy); + closureAvlDestroy(p->pAfter, xDestroy); + xDestroy(p); + } +} +/* +** End of the AVL Tree implementation +******************************************************************************/ + +/* +** A closure virtual-table object +*/ +struct closure_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zDb; /* Name of database. (ex: "main") */ + char *zSelf; /* Name of this virtual table */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + sqlite3 *db; /* The database connection */ + int nCursor; /* Number of pending cursors */ +}; + +/* A closure cursor object */ +struct closure_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + closure_vtab *pVtab; /* The virtual table this cursor belongs to */ + char *zTableName; /* Name of table holding parent/child relation */ + char *zIdColumn; /* Name of ID column of zTableName */ + char *zParentColumn; /* Name of PARENT column in zTableName */ + closure_avl *pCurrent; /* Current element of output */ + closure_avl *pClosure; /* The complete closure tree */ +}; + +/* A queue of AVL nodes */ +struct closure_queue { + closure_avl *pFirst; /* Oldest node on the queue */ + closure_avl *pLast; /* Youngest node on the queue */ +}; + +/* +** Add a node to the end of the queue +*/ +static void queuePush(closure_queue *pQueue, closure_avl *pNode){ + pNode->pList = 0; + if( pQueue->pLast ){ + pQueue->pLast->pList = pNode; + }else{ + pQueue->pFirst = pNode; + } + pQueue->pLast = pNode; +} + +/* +** Extract the oldest element (the front element) from the queue. +*/ +static closure_avl *queuePull(closure_queue *pQueue){ + closure_avl *p = pQueue->pFirst; + if( p ){ + pQueue->pFirst = p->pList; + if( pQueue->pFirst==0 ) pQueue->pLast = 0; + } + return p; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *closureDequote(const char *zIn){ + sqlite3_int64 nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc64(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, (size_t)(nIn+1)); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInzDb); + sqlite3_free(p->zSelf); + sqlite3_free(p->zTableName); + sqlite3_free(p->zIdColumn); + sqlite3_free(p->zParentColumn); + memset(p, 0, sizeof(*p)); + sqlite3_free(p); + } +} + +/* +** xDisconnect/xDestroy method for the closure module. +*/ +static int closureDisconnect(sqlite3_vtab *pVtab){ + closure_vtab *p = (closure_vtab*)pVtab; + assert( p->nCursor==0 ); + closureFree(p); + return SQLITE_OK; +} + +/* +** Check to see if the argument is of the form: +** +** KEY = VALUE +** +** If it is, return a pointer to the first character of VALUE. +** If not, return NULL. Spaces around the = are ignored. +*/ +static const char *closureValueOfKey(const char *zKey, const char *zStr){ + int nKey = (int)strlen(zKey); + int nStr = (int)strlen(zStr); + int i; + if( nStr module name ("transitive_closure") +** argv[1] -> database name +** argv[2] -> table name +** argv[3...] -> arguments +*/ +static int closureConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + closure_vtab *pNew = 0; /* New virtual table */ + const char *zDb = argv[1]; + const char *zVal; + int i; + + (void)pAux; + *ppVtab = 0; + pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + rc = SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->zDb = sqlite3_mprintf("%s", zDb); + if( pNew->zDb==0 ) goto closureConnectError; + pNew->zSelf = sqlite3_mprintf("%s", argv[2]); + if( pNew->zSelf==0 ) goto closureConnectError; + for(i=3; izTableName); + pNew->zTableName = closureDequote(zVal); + if( pNew->zTableName==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("idcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zIdColumn); + pNew->zIdColumn = closureDequote(zVal); + if( pNew->zIdColumn==0 ) goto closureConnectError; + continue; + } + zVal = closureValueOfKey("parentcolumn", argv[i]); + if( zVal ){ + sqlite3_free(pNew->zParentColumn); + pNew->zParentColumn = closureDequote(zVal); + if( pNew->zParentColumn==0 ) goto closureConnectError; + continue; + } + *pzErr = sqlite3_mprintf("unrecognized argument: [%s]\n", argv[i]); + closureFree(pNew); + *ppVtab = 0; + return SQLITE_ERROR; + } + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(id,depth,root HIDDEN,tablename HIDDEN," + "idcolumn HIDDEN,parentcolumn HIDDEN)" + ); +#define CLOSURE_COL_ID 0 +#define CLOSURE_COL_DEPTH 1 +#define CLOSURE_COL_ROOT 2 +#define CLOSURE_COL_TABLENAME 3 +#define CLOSURE_COL_IDCOLUMN 4 +#define CLOSURE_COL_PARENTCOLUMN 5 + if( rc!=SQLITE_OK ){ + closureFree(pNew); + } + *ppVtab = &pNew->base; + return rc; + +closureConnectError: + closureFree(pNew); + return rc; +} + +/* +** Open a new closure cursor. +*/ +static int closureOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + closure_vtab *p = (closure_vtab*)pVTab; + closure_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void closureClearCursor(closure_cursor *pCur){ + closureAvlDestroy(pCur->pClosure, (void(*)(closure_avl*))sqlite3_free); + sqlite3_free(pCur->zTableName); + sqlite3_free(pCur->zIdColumn); + sqlite3_free(pCur->zParentColumn); + pCur->zTableName = 0; + pCur->zIdColumn = 0; + pCur->zParentColumn = 0; + pCur->pCurrent = 0; + pCur->pClosure = 0; +} + +/* +** Close a closure cursor. +*/ +static int closureClose(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor *)cur; + closureClearCursor(pCur); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Advance a cursor to its next row of output +*/ +static int closureNext(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + pCur->pCurrent = closureAvlNext(pCur->pCurrent); + return SQLITE_OK; +} + +/* +** Allocate and insert a node +*/ +static int closureInsertNode( + closure_queue *pQueue, /* Add new node to this queue */ + closure_cursor *pCur, /* The cursor into which to add the node */ + sqlite3_int64 id, /* The node ID */ + int iGeneration /* The generation number for this node */ +){ + closure_avl *pNew = sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->id = id; + pNew->iGeneration = iGeneration; + closureAvlInsert(&pCur->pClosure, pNew); + queuePush(pQueue, pNew); + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any closureColumn, closureRowid, or closureEof call. +** +** This routine actually computes the closure. +** +** See the comment at the beginning of closureBestIndex() for a +** description of the meaning of idxNum. The idxStr parameter is +** not used. +*/ +static int closureFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + closure_cursor *pCur = (closure_cursor *)pVtabCursor; + closure_vtab *pVtab = pCur->pVtab; + sqlite3_int64 iRoot; + int mxGen = 999999999; + char *zSql; + sqlite3_stmt *pStmt; + closure_avl *pAvl; + int rc = SQLITE_OK; + const char *zTableName = pVtab->zTableName; + const char *zIdColumn = pVtab->zIdColumn; + const char *zParentColumn = pVtab->zParentColumn; + closure_queue sQueue; + + (void)idxStr; /* Unused parameter */ + (void)argc; /* Unused parameter */ + closureClearCursor(pCur); + memset(&sQueue, 0, sizeof(sQueue)); + if( (idxNum & 1)==0 ){ + /* No root=$root in the WHERE clause. Return an empty set */ + return SQLITE_OK; + } + iRoot = sqlite3_value_int64(argv[0]); + if( (idxNum & 0x000f0)!=0 ){ + mxGen = sqlite3_value_int(argv[(idxNum>>4)&0x0f]); + if( (idxNum & 0x00002)!=0 ) mxGen--; + } + if( (idxNum & 0x00f00)!=0 ){ + zTableName = (const char*)sqlite3_value_text(argv[(idxNum>>8)&0x0f]); + pCur->zTableName = sqlite3_mprintf("%s", zTableName); + } + if( (idxNum & 0x0f000)!=0 ){ + zIdColumn = (const char*)sqlite3_value_text(argv[(idxNum>>12)&0x0f]); + pCur->zIdColumn = sqlite3_mprintf("%s", zIdColumn); + } + if( (idxNum & 0x0f0000)!=0 ){ + zParentColumn = (const char*)sqlite3_value_text(argv[(idxNum>>16)&0x0f]); + pCur->zParentColumn = sqlite3_mprintf("%s", zParentColumn); + } + + zSql = sqlite3_mprintf( + "SELECT \"%w\".\"%w\" FROM \"%w\" WHERE \"%w\".\"%w\"=?1", + zTableName, zIdColumn, zTableName, zTableName, zParentColumn); + if( zSql==0 ){ + return SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVtab->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_free(pVtab->base.zErrMsg); + pVtab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pVtab->db)); + return rc; + } + } + if( rc==SQLITE_OK ){ + rc = closureInsertNode(&sQueue, pCur, iRoot, 0); + } + while( (pAvl = queuePull(&sQueue))!=0 ){ + if( pAvl->iGeneration>=mxGen ) continue; + sqlite3_bind_int64(pStmt, 1, pAvl->id); + while( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){ + if( sqlite3_column_type(pStmt,0)==SQLITE_INTEGER ){ + sqlite3_int64 iNew = sqlite3_column_int64(pStmt, 0); + if( closureAvlSearch(pCur->pClosure, iNew)==0 ){ + rc = closureInsertNode(&sQueue, pCur, iNew, pAvl->iGeneration+1); + } + } + } + sqlite3_reset(pStmt); + } + sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ){ + pCur->pCurrent = closureAvlFirst(pCur->pClosure); + } + + return rc; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int closureColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + closure_cursor *pCur = (closure_cursor*)cur; + switch( i ){ + case CLOSURE_COL_ID: { + sqlite3_result_int64(ctx, pCur->pCurrent->id); + break; + } + case CLOSURE_COL_DEPTH: { + sqlite3_result_int(ctx, pCur->pCurrent->iGeneration); + break; + } + case CLOSURE_COL_ROOT: { + sqlite3_result_null(ctx); + break; + } + case CLOSURE_COL_TABLENAME: { + sqlite3_result_text(ctx, + pCur->zTableName ? pCur->zTableName : pCur->pVtab->zTableName, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_IDCOLUMN: { + sqlite3_result_text(ctx, + pCur->zIdColumn ? pCur->zIdColumn : pCur->pVtab->zIdColumn, + -1, SQLITE_TRANSIENT); + break; + } + case CLOSURE_COL_PARENTCOLUMN: { + sqlite3_result_text(ctx, + pCur->zParentColumn ? pCur->zParentColumn : pCur->pVtab->zParentColumn, + -1, SQLITE_TRANSIENT); + break; + } + } + return SQLITE_OK; +} + +/* +** The rowid. For the closure table, this is the same as the "id" column. +*/ +static int closureRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + closure_cursor *pCur = (closure_cursor*)cur; + *pRowid = pCur->pCurrent->id; + return SQLITE_OK; +} + +/* +** EOF indicator +*/ +static int closureEof(sqlite3_vtab_cursor *cur){ + closure_cursor *pCur = (closure_cursor*)cur; + return pCur->pCurrent==0; +} + +/* +** Search for terms of these forms: +** +** (A) root = $root +** (B1) depth < $depth +** (B2) depth <= $depth +** (B3) depth = $depth +** (C) tablename = $tablename +** (D) idcolumn = $idcolumn +** (E) parentcolumn = $parentcolumn +** +** +** +** idxNum meaning +** ---------- ------------------------------------------------------ +** 0x00000001 Term of the form (A) found +** 0x00000002 The term of bit-2 is like (B1) +** 0x000000f0 Index in filter.argv[] of $depth. 0 if not used. +** 0x00000f00 Index in filter.argv[] of $tablename. 0 if not used. +** 0x0000f000 Index in filter.argv[] of $idcolumn. 0 if not used +** 0x000f0000 Index in filter.argv[] of $parentcolumn. 0 if not used. +** +** There must be a term of type (A). If there is not, then the index type +** is 0 and the query will return an empty set. +*/ +static int closureBestIndex( + sqlite3_vtab *pTab, /* The virtual table */ + sqlite3_index_info *pIdxInfo /* Information about the query */ +){ + int iPlan = 0; + int i; + int idx = 1; + const struct sqlite3_index_constraint *pConstraint; + closure_vtab *pVtab = (closure_vtab*)pTab; + double rCost = 10000000.0; + + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==CLOSURE_COL_ROOT + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 100.0; + } + if( (iPlan & 0x0000f0)==0 + && pConstraint->iColumn==CLOSURE_COL_DEPTH + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ) + ){ + iPlan |= idx<<4; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT ) iPlan |= 0x000002; + rCost /= 5.0; + } + if( (iPlan & 0x000f00)==0 + && pConstraint->iColumn==CLOSURE_COL_TABLENAME + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<8; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 5.0; + } + if( (iPlan & 0x00f000)==0 + && pConstraint->iColumn==CLOSURE_COL_IDCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<12; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + if( (iPlan & 0x0f0000)==0 + && pConstraint->iColumn==CLOSURE_COL_PARENTCOLUMN + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= idx<<16; + pIdxInfo->aConstraintUsage[i].argvIndex = ++idx; + pIdxInfo->aConstraintUsage[i].omit = 1; + } + } + if( (pVtab->zTableName==0 && (iPlan & 0x000f00)==0) + || (pVtab->zIdColumn==0 && (iPlan & 0x00f000)==0) + || (pVtab->zParentColumn==0 && (iPlan & 0x0f0000)==0) + ){ + /* All of tablename, idcolumn, and parentcolumn must be specified + ** in either the CREATE VIRTUAL TABLE or in the WHERE clause constraints + ** or else the result is an empty set. */ + iPlan = 0; + } + if( (iPlan&1)==0 ){ + /* If there is no usable "root=?" term, then set the index-type to 0. + ** Also clear any argvIndex variables already set. This is necessary + ** to prevent the core from throwing an "xBestIndex malfunction error" + ** error (because the argvIndex values are not contiguously assigned + ** starting from 1). */ + rCost *= 1e30; + for(i=0; inConstraint; i++, pConstraint++){ + pIdxInfo->aConstraintUsage[i].argvIndex = 0; + } + iPlan = 0; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==CLOSURE_COL_ID + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + pIdxInfo->estimatedCost = rCost; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "transitive_closure". +*/ +static sqlite3_module closureModule = { + 0, /* iVersion */ + closureConnect, /* xCreate */ + closureConnect, /* xConnect */ + closureBestIndex, /* xBestIndex */ + closureDisconnect, /* xDisconnect */ + closureDisconnect, /* xDestroy */ + closureOpen, /* xOpen - open a cursor */ + closureClose, /* xClose - close a cursor */ + closureFilter, /* xFilter - configure scan constraints */ + closureNext, /* xNext - advance a cursor */ + closureEof, /* xEof - check for end of scan */ + closureColumn, /* xColumn - read data */ + closureRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +/* +** Register the closure virtual table +*/ +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_closure_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "transitive_closure", &closureModule, 0); +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + return rc; +} diff --git a/ext/misc/completion.c b/ext/misc/completion.c new file mode 100644 index 0000000..987595a --- /dev/null +++ b/ext/misc/completion.c @@ -0,0 +1,502 @@ +/* +** 2017-07-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an eponymous virtual table that returns suggested +** completions for a partial SQL input. +** +** Suggested usage: +** +** SELECT DISTINCT candidate COLLATE nocase +** FROM completion($prefix,$wholeline) +** ORDER BY 1; +** +** The two query parameters are optional. $prefix is the text of the +** current word being typed and that is to be completed. $wholeline is +** the complete input line, used for context. +** +** The raw completion() table might return the same candidate multiple +** times, for example if the same column name is used to two or more +** tables. And the candidates are returned in an arbitrary order. Hence, +** the DISTINCT and ORDER BY are recommended. +** +** This virtual table operates at the speed of human typing, and so there +** is no attempt to make it fast. Even a slow implementation will be much +** faster than any human can type. +** +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* completion_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a completion virtual table +*/ +typedef struct completion_vtab completion_vtab; +struct completion_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this completion vtab */ +}; + +/* completion_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct completion_cursor completion_cursor; +struct completion_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + int nPrefix, nLine; /* Number of bytes in zPrefix and zLine */ + char *zPrefix; /* The prefix for the word we want to complete */ + char *zLine; /* The whole that we want to complete */ + const char *zCurrentRow; /* Current output row */ + int szRow; /* Length of the zCurrentRow string */ + sqlite3_stmt *pStmt; /* Current statement */ + sqlite3_int64 iRowid; /* The rowid */ + int ePhase; /* Current phase */ + int j; /* inter-phase counter */ +}; + +/* Values for ePhase: +*/ +#define COMPLETION_FIRST_PHASE 1 +#define COMPLETION_KEYWORDS 1 +#define COMPLETION_PRAGMAS 2 +#define COMPLETION_FUNCTIONS 3 +#define COMPLETION_COLLATIONS 4 +#define COMPLETION_INDEXES 5 +#define COMPLETION_TRIGGERS 6 +#define COMPLETION_DATABASES 7 +#define COMPLETION_TABLES 8 /* Also VIEWs and TRIGGERs */ +#define COMPLETION_COLUMNS 9 +#define COMPLETION_MODULES 10 +#define COMPLETION_EOF 11 + +/* +** The completionConnect() method is invoked to create a new +** completion_vtab that describes the completion virtual table. +** +** Think of this routine as the constructor for completion_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the completion_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against completion will look like. +*/ +static int completionConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + completion_vtab *pNew; + int rc; + + (void)(pAux); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + (void)(argv); /* Unused parameter */ + (void)(pzErr); /* Unused parameter */ + +/* Column numbers */ +#define COMPLETION_COLUMN_CANDIDATE 0 /* Suggested completion of the input */ +#define COMPLETION_COLUMN_PREFIX 1 /* Prefix of the word to be completed */ +#define COMPLETION_COLUMN_WHOLELINE 2 /* Entire line seen so far */ +#define COMPLETION_COLUMN_PHASE 3 /* ePhase - used for debugging only */ + + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(" + " candidate TEXT," + " prefix TEXT HIDDEN," + " wholeline TEXT HIDDEN," + " phase INT HIDDEN" /* Used for debugging only */ + ")"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for completion_cursor objects. +*/ +static int completionDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new completion_cursor object. +*/ +static int completionOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + completion_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((completion_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset the completion_cursor. +*/ +static void completionCursorReset(completion_cursor *pCur){ + sqlite3_free(pCur->zPrefix); pCur->zPrefix = 0; pCur->nPrefix = 0; + sqlite3_free(pCur->zLine); pCur->zLine = 0; pCur->nLine = 0; + sqlite3_finalize(pCur->pStmt); pCur->pStmt = 0; + pCur->j = 0; +} + +/* +** Destructor for a completion_cursor. +*/ +static int completionClose(sqlite3_vtab_cursor *cur){ + completionCursorReset((completion_cursor*)cur); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Advance a completion_cursor to its next row of output. +** +** The ->ePhase, ->j, and ->pStmt fields of the completion_cursor object +** record the current state of the scan. This routine sets ->zCurrentRow +** to the current row of output and then returns. If no more rows remain, +** then ->ePhase is set to COMPLETION_EOF which will signal the virtual +** table that has reached the end of its scan. +** +** The current implementation just lists potential identifiers and +** keywords and filters them by zPrefix. Future enhancements should +** take zLine into account to try to restrict the set of identifiers and +** keywords based on what would be legal at the current point of input. +*/ +static int completionNext(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + int eNextPhase = 0; /* Next phase to try if current phase reaches end */ + int iCol = -1; /* If >=0, step pCur->pStmt and use the i-th column */ + pCur->iRowid++; + while( pCur->ePhase!=COMPLETION_EOF ){ + switch( pCur->ePhase ){ + case COMPLETION_KEYWORDS: { + if( pCur->j >= sqlite3_keyword_count() ){ + pCur->zCurrentRow = 0; + pCur->ePhase = COMPLETION_DATABASES; + }else{ + sqlite3_keyword_name(pCur->j++, &pCur->zCurrentRow, &pCur->szRow); + } + iCol = -1; + break; + } + case COMPLETION_DATABASES: { + if( pCur->pStmt==0 ){ + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, + &pCur->pStmt, 0); + } + iCol = 1; + eNextPhase = COMPLETION_TABLES; + break; + } + case COMPLETION_TABLES: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT name FROM \"%w\".sqlite_schema", + zSql, zSep, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_COLUMNS; + break; + } + case COMPLETION_COLUMNS: { + if( pCur->pStmt==0 ){ + sqlite3_stmt *pS2; + char *zSql = 0; + const char *zSep = ""; + sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pS2, 0); + while( sqlite3_step(pS2)==SQLITE_ROW ){ + const char *zDb = (const char*)sqlite3_column_text(pS2, 1); + zSql = sqlite3_mprintf( + "%z%s" + "SELECT pti.name FROM \"%w\".sqlite_schema AS sm" + " JOIN pragma_table_info(sm.name,%Q) AS pti" + " WHERE sm.type='table'", + zSql, zSep, zDb, zDb + ); + if( zSql==0 ) return SQLITE_NOMEM; + zSep = " UNION "; + } + sqlite3_finalize(pS2); + sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pStmt, 0); + sqlite3_free(zSql); + } + iCol = 0; + eNextPhase = COMPLETION_EOF; + break; + } + } + if( iCol<0 ){ + /* This case is when the phase presets zCurrentRow */ + if( pCur->zCurrentRow==0 ) continue; + }else{ + if( sqlite3_step(pCur->pStmt)==SQLITE_ROW ){ + /* Extract the next row of content */ + pCur->zCurrentRow = (const char*)sqlite3_column_text(pCur->pStmt, iCol); + pCur->szRow = sqlite3_column_bytes(pCur->pStmt, iCol); + }else{ + /* When all rows are finished, advance to the next phase */ + sqlite3_finalize(pCur->pStmt); + pCur->pStmt = 0; + pCur->ePhase = eNextPhase; + continue; + } + } + if( pCur->nPrefix==0 ) break; + if( pCur->nPrefix<=pCur->szRow + && sqlite3_strnicmp(pCur->zPrefix, pCur->zCurrentRow, pCur->nPrefix)==0 + ){ + break; + } + } + + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the completion_cursor +** is currently pointing. +*/ +static int completionColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + completion_cursor *pCur = (completion_cursor*)cur; + switch( i ){ + case COMPLETION_COLUMN_CANDIDATE: { + sqlite3_result_text(ctx, pCur->zCurrentRow, pCur->szRow,SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PREFIX: { + sqlite3_result_text(ctx, pCur->zPrefix, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_WHOLELINE: { + sqlite3_result_text(ctx, pCur->zLine, -1, SQLITE_TRANSIENT); + break; + } + case COMPLETION_COLUMN_PHASE: { + sqlite3_result_int(ctx, pCur->ePhase); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int completionRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + completion_cursor *pCur = (completion_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int completionEof(sqlite3_vtab_cursor *cur){ + completion_cursor *pCur = (completion_cursor*)cur; + return pCur->ePhase >= COMPLETION_EOF; +} + +/* +** This method is called to "rewind" the completion_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to completionColumn() or completionRowid() or +** completionEof(). +*/ +static int completionFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + completion_cursor *pCur = (completion_cursor *)pVtabCursor; + int iArg = 0; + (void)(idxStr); /* Unused parameter */ + (void)(argc); /* Unused parameter */ + completionCursorReset(pCur); + if( idxNum & 1 ){ + pCur->nPrefix = sqlite3_value_bytes(argv[iArg]); + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + iArg = 1; + } + if( idxNum & 2 ){ + pCur->nLine = sqlite3_value_bytes(argv[iArg]); + if( pCur->nLine>0 ){ + pCur->zLine = sqlite3_mprintf("%s", sqlite3_value_text(argv[iArg])); + if( pCur->zLine==0 ) return SQLITE_NOMEM; + } + } + if( pCur->zLine!=0 && pCur->zPrefix==0 ){ + int i = pCur->nLine; + while( i>0 && (isalnum(pCur->zLine[i-1]) || pCur->zLine[i-1]=='_') ){ + i--; + } + pCur->nPrefix = pCur->nLine - i; + if( pCur->nPrefix>0 ){ + pCur->zPrefix = sqlite3_mprintf("%.*s", pCur->nPrefix, pCur->zLine + i); + if( pCur->zPrefix==0 ) return SQLITE_NOMEM; + } + } + pCur->iRowid = 0; + pCur->ePhase = COMPLETION_FIRST_PHASE; + return completionNext(pVtabCursor); +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the completion virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** There are two hidden parameters that act as arguments to the table-valued +** function: "prefix" and "wholeline". Bit 0 of idxNum is set if "prefix" +** is available and bit 1 is set if "wholeline" is available. +*/ +static int completionBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxNum = 0; /* The query plan bitmask */ + int prefixIdx = -1; /* Index of the start= constraint, or -1 if none */ + int wholelineIdx = -1; /* Index of the stop= constraint, or -1 if none */ + int nArg = 0; /* Number of arguments that completeFilter() expects */ + const struct sqlite3_index_constraint *pConstraint; + + (void)(tab); /* Unused parameter */ + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->usable==0 ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case COMPLETION_COLUMN_PREFIX: + prefixIdx = i; + idxNum |= 1; + break; + case COMPLETION_COLUMN_WHOLELINE: + wholelineIdx = i; + idxNum |= 2; + break; + } + } + if( prefixIdx>=0 ){ + pIdxInfo->aConstraintUsage[prefixIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[prefixIdx].omit = 1; + } + if( wholelineIdx>=0 ){ + pIdxInfo->aConstraintUsage[wholelineIdx].argvIndex = ++nArg; + pIdxInfo->aConstraintUsage[wholelineIdx].omit = 1; + } + pIdxInfo->idxNum = idxNum; + pIdxInfo->estimatedCost = (double)5000 - 1000*nArg; + pIdxInfo->estimatedRows = 500 - 100*nArg; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** completion virtual table. +*/ +static sqlite3_module completionModule = { + 0, /* iVersion */ + 0, /* xCreate */ + completionConnect, /* xConnect */ + completionBestIndex, /* xBestIndex */ + completionDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + completionOpen, /* xOpen - open a cursor */ + completionClose, /* xClose - close a cursor */ + completionFilter, /* xFilter - configure scan constraints */ + completionNext, /* xNext - advance a cursor */ + completionEof, /* xEof - check for end of scan */ + completionColumn, /* xColumn - read data */ + completionRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3CompletionVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "completion", &completionModule, 0); +#endif + return rc; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_completion_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)(pzErrMsg); /* Unused parameter */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3CompletionVtabInit(db); +#endif + return rc; +} diff --git a/ext/misc/compress.c b/ext/misc/compress.c new file mode 100644 index 0000000..2e7a316 --- /dev/null +++ b/ext/misc/compress.c @@ -0,0 +1,131 @@ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL compression functions +** compress() and uncompress() using ZLIB. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Implementation of the "compress(X)" SQL function. The input X is +** compressed using zLib and the output is returned. +** +** The output is a BLOB that begins with a variable-length integer that +** is the input size in bytes (the size of X before compression). The +** variable-length integer is implemented as 1 to 5 bytes. There are +** seven bits per integer stored in the lower seven bits of each byte. +** More significant bits occur first. The most significant bit (0x80) +** is a flag to indicate the end of the integer. +** +** This function, SQLAR, and ZIP all use the same "deflate" compression +** algorithm, but each is subtly different: +** +** * ZIP uses raw deflate. +** +** * SQLAR uses the "zlib format" which is raw deflate with a two-byte +** algorithm-identification header and a four-byte checksum at the end. +** +** * This utility uses the "zlib format" like SQLAR, but adds the variable- +** length integer uncompressed size value at the beginning. +** +** This function might be extended in the future to support compression +** formats other than deflate, by providing a different algorithm-id +** mark following the variable-length integer size parameter. +*/ +static void compressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pIn; + unsigned char *pOut; + unsigned int nIn; + unsigned long int nOut; + unsigned char x[8]; + int rc; + int i, j; + + pIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + nOut = 13 + nIn + (nIn+999)/1000; + pOut = sqlite3_malloc( nOut+5 ); + for(i=4; i>=0; i--){ + x[i] = (nIn >> (7*(4-i)))&0x7f; + } + for(i=0; i<4 && x[i]==0; i++){} + for(j=0; i<=4; i++, j++) pOut[j] = x[i]; + pOut[j-1] |= 0x80; + rc = compress(&pOut[j], &nOut, pIn, nIn); + if( rc==Z_OK ){ + sqlite3_result_blob(context, pOut, nOut+j, sqlite3_free); + }else{ + sqlite3_free(pOut); + } +} + +/* +** Implementation of the "uncompress(X)" SQL function. The argument X +** is a blob which was obtained from compress(Y). The output will be +** the value Y. +*/ +static void uncompressFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const unsigned char *pIn; + unsigned char *pOut; + unsigned int nIn; + unsigned long int nOut; + int rc; + int i; + + pIn = sqlite3_value_blob(argv[0]); + nIn = sqlite3_value_bytes(argv[0]); + nOut = 0; + for(i=0; i +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** A macro to hint to the compiler that a function should not be +** inlined. +*/ +#if defined(__GNUC__) +# define CSV_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) && _MSC_VER>=1310 +# define CSV_NOINLINE __declspec(noinline) +#else +# define CSV_NOINLINE +#endif + + +/* Max size of the error message in a CsvReader */ +#define CSV_MXERR 200 + +/* Size of the CsvReader input buffer */ +#define CSV_INBUFSZ 1024 + +/* A context object used when read a CSV file. */ +typedef struct CsvReader CsvReader; +struct CsvReader { + FILE *in; /* Read the CSV text from this input stream */ + char *z; /* Accumulated text for a field */ + int n; /* Number of bytes in z */ + int nAlloc; /* Space allocated for z[] */ + int nLine; /* Current line number */ + int bNotFirst; /* True if prior text has been seen */ + int cTerm; /* Character that terminated the most recent field */ + size_t iIn; /* Next unread character in the input buffer */ + size_t nIn; /* Number of characters in the input buffer */ + char *zIn; /* The input buffer */ + char zErr[CSV_MXERR]; /* Error message */ +}; + +/* Initialize a CsvReader object */ +static void csv_reader_init(CsvReader *p){ + p->in = 0; + p->z = 0; + p->n = 0; + p->nAlloc = 0; + p->nLine = 0; + p->bNotFirst = 0; + p->nIn = 0; + p->zIn = 0; + p->zErr[0] = 0; +} + +/* Close and reset a CsvReader object */ +static void csv_reader_reset(CsvReader *p){ + if( p->in ){ + fclose(p->in); + sqlite3_free(p->zIn); + } + sqlite3_free(p->z); + csv_reader_init(p); +} + +/* Report an error on a CsvReader */ +static void csv_errmsg(CsvReader *p, const char *zFormat, ...){ + va_list ap; + va_start(ap, zFormat); + sqlite3_vsnprintf(CSV_MXERR, p->zErr, zFormat, ap); + va_end(ap); +} + +/* Open the file associated with a CsvReader +** Return the number of errors. +*/ +static int csv_reader_open( + CsvReader *p, /* The reader to open */ + const char *zFilename, /* Read from this filename */ + const char *zData /* ... or use this data */ +){ + if( zFilename ){ + p->zIn = sqlite3_malloc( CSV_INBUFSZ ); + if( p->zIn==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + p->in = fopen(zFilename, "rb"); + if( p->in==0 ){ + sqlite3_free(p->zIn); + csv_reader_reset(p); + csv_errmsg(p, "cannot open '%s' for reading", zFilename); + return 1; + } + }else{ + assert( p->in==0 ); + p->zIn = (char*)zData; + p->nIn = strlen(zData); + } + return 0; +} + +/* The input buffer has overflowed. Refill the input buffer, then +** return the next character +*/ +static CSV_NOINLINE int csv_getc_refill(CsvReader *p){ + size_t got; + + assert( p->iIn>=p->nIn ); /* Only called on an empty input buffer */ + assert( p->in!=0 ); /* Only called if reading froma file */ + + got = fread(p->zIn, 1, CSV_INBUFSZ, p->in); + if( got==0 ) return EOF; + p->nIn = got; + p->iIn = 1; + return p->zIn[0]; +} + +/* Return the next character of input. Return EOF at end of input. */ +static int csv_getc(CsvReader *p){ + if( p->iIn >= p->nIn ){ + if( p->in!=0 ) return csv_getc_refill(p); + return EOF; + } + return ((unsigned char*)p->zIn)[p->iIn++]; +} + +/* Increase the size of p->z and append character c to the end. +** Return 0 on success and non-zero if there is an OOM error */ +static CSV_NOINLINE int csv_resize_and_append(CsvReader *p, char c){ + char *zNew; + int nNew = p->nAlloc*2 + 100; + zNew = sqlite3_realloc64(p->z, nNew); + if( zNew ){ + p->z = zNew; + p->nAlloc = nNew; + p->z[p->n++] = c; + return 0; + }else{ + csv_errmsg(p, "out of memory"); + return 1; + } +} + +/* Append a single character to the CsvReader.z[] array. +** Return 0 on success and non-zero if there is an OOM error */ +static int csv_append(CsvReader *p, char c){ + if( p->n>=p->nAlloc-1 ) return csv_resize_and_append(p, c); + p->z[p->n++] = c; + return 0; +} + +/* Read a single field of CSV text. Compatible with rfc4180 and extended +** with the option of having a separator other than ",". +** +** + Input comes from p->in. +** + Store results in p->z of length p->n. Space to hold p->z comes +** from sqlite3_malloc64(). +** + Keep track of the line number in p->nLine. +** + Store the character that terminates the field in p->cTerm. Store +** EOF on end-of-file. +** +** Return 0 at EOF or on OOM. On EOF, the p->cTerm character will have +** been set to EOF. +*/ +static char *csv_read_one_field(CsvReader *p){ + int c; + p->n = 0; + c = csv_getc(p); + if( c==EOF ){ + p->cTerm = EOF; + return 0; + } + if( c=='"' ){ + int pc, ppc; + int startLine = p->nLine; + pc = ppc = 0; + while( 1 ){ + c = csv_getc(p); + if( c<='"' || pc=='"' ){ + if( c=='\n' ) p->nLine++; + if( c=='"' ){ + if( pc=='"' ){ + pc = 0; + continue; + } + } + if( (c==',' && pc=='"') + || (c=='\n' && pc=='"') + || (c=='\n' && pc=='\r' && ppc=='"') + || (c==EOF && pc=='"') + ){ + do{ p->n--; }while( p->z[p->n]!='"' ); + p->cTerm = (char)c; + break; + } + if( pc=='"' && c!='\r' ){ + csv_errmsg(p, "line %d: unescaped %c character", p->nLine, '"'); + break; + } + if( c==EOF ){ + csv_errmsg(p, "line %d: unterminated %c-quoted field\n", + startLine, '"'); + p->cTerm = (char)c; + break; + } + } + if( csv_append(p, (char)c) ) return 0; + ppc = pc; + pc = c; + } + }else{ + /* If this is the first field being parsed and it begins with the + ** UTF-8 BOM (0xEF BB BF) then skip the BOM */ + if( (c&0xff)==0xef && p->bNotFirst==0 ){ + csv_append(p, (char)c); + c = csv_getc(p); + if( (c&0xff)==0xbb ){ + csv_append(p, (char)c); + c = csv_getc(p); + if( (c&0xff)==0xbf ){ + p->bNotFirst = 1; + p->n = 0; + return csv_read_one_field(p); + } + } + } + while( c>',' || (c!=EOF && c!=',' && c!='\n') ){ + if( csv_append(p, (char)c) ) return 0; + c = csv_getc(p); + } + if( c=='\n' ){ + p->nLine++; + if( p->n>0 && p->z[p->n-1]=='\r' ) p->n--; + } + p->cTerm = (char)c; + } + assert( p->z==0 || p->nnAlloc ); + if( p->z ) p->z[p->n] = 0; + p->bNotFirst = 1; + return p->z; +} + + +/* Forward references to the various virtual table methods implemented +** in this file. */ +static int csvtabCreate(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabConnect(sqlite3*, void*, int, const char*const*, + sqlite3_vtab**,char**); +static int csvtabBestIndex(sqlite3_vtab*,sqlite3_index_info*); +static int csvtabDisconnect(sqlite3_vtab*); +static int csvtabOpen(sqlite3_vtab*, sqlite3_vtab_cursor**); +static int csvtabClose(sqlite3_vtab_cursor*); +static int csvtabFilter(sqlite3_vtab_cursor*, int idxNum, const char *idxStr, + int argc, sqlite3_value **argv); +static int csvtabNext(sqlite3_vtab_cursor*); +static int csvtabEof(sqlite3_vtab_cursor*); +static int csvtabColumn(sqlite3_vtab_cursor*,sqlite3_context*,int); +static int csvtabRowid(sqlite3_vtab_cursor*,sqlite3_int64*); + +/* An instance of the CSV virtual table */ +typedef struct CsvTable { + sqlite3_vtab base; /* Base class. Must be first */ + char *zFilename; /* Name of the CSV file */ + char *zData; /* Raw CSV data in lieu of zFilename */ + long iStart; /* Offset to start of data in zFilename */ + int nCol; /* Number of columns in the CSV file */ + unsigned int tstFlags; /* Bit values used for testing */ +} CsvTable; + +/* Allowed values for tstFlags */ +#define CSVTEST_FIDX 0x0001 /* Pretend that constrained searchs cost less*/ + +/* A cursor for the CSV virtual table */ +typedef struct CsvCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + CsvReader rdr; /* The CsvReader object */ + char **azVal; /* Value of the current row */ + int *aLen; /* Length of each entry */ + sqlite3_int64 iRowid; /* The current rowid. Negative for EOF */ +} CsvCursor; + +/* Transfer error message text from a reader into a CsvTable */ +static void csv_xfer_error(CsvTable *pTab, CsvReader *pRdr){ + sqlite3_free(pTab->base.zErrMsg); + pTab->base.zErrMsg = sqlite3_mprintf("%s", pRdr->zErr); +} + +/* +** This method is the destructor fo a CsvTable object. +*/ +static int csvtabDisconnect(sqlite3_vtab *pVtab){ + CsvTable *p = (CsvTable*)pVtab; + sqlite3_free(p->zFilename); + sqlite3_free(p->zData); + sqlite3_free(p); + return SQLITE_OK; +} + +/* Skip leading whitespace. Return a pointer to the first non-whitespace +** character, or to the zero terminator if the string has only whitespace */ +static const char *csv_skip_whitespace(const char *z){ + while( isspace((unsigned char)z[0]) ) z++; + return z; +} + +/* Remove trailing whitespace from the end of string z[] */ +static void csv_trim_whitespace(char *z){ + size_t n = strlen(z); + while( n>0 && isspace((unsigned char)z[n]) ) n--; + z[n] = 0; +} + +/* Dequote the string */ +static void csv_dequote(char *z){ + int j; + char cQuote = z[0]; + size_t i, n; + + if( cQuote!='\'' && cQuote!='"' ) return; + n = strlen(z); + if( n<2 || z[n-1]!=z[0] ) return; + for(i=1, j=0; izErr. If there are no errors, p->zErr[0]==0. +*/ +static int csv_string_parameter( + CsvReader *p, /* Leave the error message here, if there is one */ + const char *zParam, /* Parameter we are checking for */ + const char *zArg, /* Raw text of the virtual table argment */ + char **pzVal /* Write the dequoted string value here */ +){ + const char *zValue; + zValue = csv_parameter(zParam,(int)strlen(zParam),zArg); + if( zValue==0 ) return 0; + p->zErr[0] = 0; + if( *pzVal ){ + csv_errmsg(p, "more than one '%s' parameter", zParam); + return 1; + } + *pzVal = sqlite3_mprintf("%s", zValue); + if( *pzVal==0 ){ + csv_errmsg(p, "out of memory"); + return 1; + } + csv_trim_whitespace(*pzVal); + csv_dequote(*pzVal); + return 1; +} + + +/* Return 0 if the argument is false and 1 if it is true. Return -1 if +** we cannot really tell. +*/ +static int csv_boolean(const char *z){ + if( sqlite3_stricmp("yes",z)==0 + || sqlite3_stricmp("on",z)==0 + || sqlite3_stricmp("true",z)==0 + || (z[0]=='1' && z[1]==0) + ){ + return 1; + } + if( sqlite3_stricmp("no",z)==0 + || sqlite3_stricmp("off",z)==0 + || sqlite3_stricmp("false",z)==0 + || (z[0]=='0' && z[1]==0) + ){ + return 0; + } + return -1; +} + +/* Check to see if the string is of the form: "TAG = BOOLEAN" or just "TAG". +** If it is, set *pValue to be the value of the boolean ("true" if there is +** not "= BOOLEAN" component) and return non-zero. If the input string +** does not begin with TAG, return zero. +*/ +static int csv_boolean_parameter( + const char *zTag, /* Tag we are looking for */ + int nTag, /* Size of the tag in bytes */ + const char *z, /* Input parameter */ + int *pValue /* Write boolean value here */ +){ + int b; + z = csv_skip_whitespace(z); + if( strncmp(zTag, z, nTag)!=0 ) return 0; + z = csv_skip_whitespace(z + nTag); + if( z[0]==0 ){ + *pValue = 1; + return 1; + } + if( z[0]!='=' ) return 0; + z = csv_skip_whitespace(z+1); + b = csv_boolean(z); + if( b>=0 ){ + *pValue = b; + return 1; + } + return 0; +} + +/* +** Parameters: +** filename=FILENAME Name of file containing CSV content +** data=TEXT Direct CSV content. +** schema=SCHEMA Alternative CSV schema. +** header=YES|NO First row of CSV defines the names of +** columns if "yes". Default "no". +** columns=N Assume the CSV file contains N columns. +** +** Only available if compiled with SQLITE_TEST: +** +** testflags=N Bitmask of test flags. Optional +** +** If schema= is omitted, then the columns are named "c0", "c1", "c2", +** and so forth. If columns=N is omitted, then the file is opened and +** the number of columns in the first row is counted to determine the +** column count. If header=YES, then the first row is skipped. +*/ +static int csvtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + CsvTable *pNew = 0; /* The CsvTable object to construct */ + int bHeader = -1; /* header= flags. -1 means not seen yet */ + int rc = SQLITE_OK; /* Result code from this routine */ + int i, j; /* Loop counters */ +#ifdef SQLITE_TEST + int tstFlags = 0; /* Value for testflags=N parameter */ +#endif + int b; /* Value of a boolean parameter */ + int nCol = -99; /* Value of the columns= parameter */ + CsvReader sRdr; /* A CSV file reader used to store an error + ** message and/or to count the number of columns */ + static const char *azParam[] = { + "filename", "data", "schema", + }; + char *azPValue[3]; /* Parameter values */ +# define CSV_FILENAME (azPValue[0]) +# define CSV_DATA (azPValue[1]) +# define CSV_SCHEMA (azPValue[2]) + + + assert( sizeof(azPValue)==sizeof(azParam) ); + memset(&sRdr, 0, sizeof(sRdr)); + memset(azPValue, 0, sizeof(azPValue)); + for(i=3; i=0 ){ + csv_errmsg(&sRdr, "more than one 'header' parameter"); + goto csvtab_connect_error; + } + bHeader = b; + }else +#ifdef SQLITE_TEST + if( (zValue = csv_parameter("testflags",9,z))!=0 ){ + tstFlags = (unsigned int)atoi(zValue); + }else +#endif + if( (zValue = csv_parameter("columns",7,z))!=0 ){ + if( nCol>0 ){ + csv_errmsg(&sRdr, "more than one 'columns' parameter"); + goto csvtab_connect_error; + } + nCol = atoi(zValue); + if( nCol<=0 ){ + csv_errmsg(&sRdr, "column= value must be positive"); + goto csvtab_connect_error; + } + }else + { + csv_errmsg(&sRdr, "bad parameter: '%s'", z); + goto csvtab_connect_error; + } + } + if( (CSV_FILENAME==0)==(CSV_DATA==0) ){ + csv_errmsg(&sRdr, "must specify either filename= or data= but not both"); + goto csvtab_connect_error; + } + + if( (nCol<=0 || bHeader==1) + && csv_reader_open(&sRdr, CSV_FILENAME, CSV_DATA) + ){ + goto csvtab_connect_error; + } + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) goto csvtab_connect_oom; + memset(pNew, 0, sizeof(*pNew)); + if( CSV_SCHEMA==0 ){ + sqlite3_str *pStr = sqlite3_str_new(0); + char *zSep = ""; + int iCol = 0; + sqlite3_str_appendf(pStr, "CREATE TABLE x("); + if( nCol<0 && bHeader<1 ){ + nCol = 0; + do{ + csv_read_one_field(&sRdr); + nCol++; + }while( sRdr.cTerm==',' ); + } + if( nCol>0 && bHeader<1 ){ + for(iCol=0; iCol0 && iColnCol = nCol; + sqlite3_str_appendf(pStr, ")"); + CSV_SCHEMA = sqlite3_str_finish(pStr); + if( CSV_SCHEMA==0 ) goto csvtab_connect_oom; + }else if( nCol<0 ){ + do{ + csv_read_one_field(&sRdr); + pNew->nCol++; + }while( sRdr.cTerm==',' ); + }else{ + pNew->nCol = nCol; + } + pNew->zFilename = CSV_FILENAME; CSV_FILENAME = 0; + pNew->zData = CSV_DATA; CSV_DATA = 0; +#ifdef SQLITE_TEST + pNew->tstFlags = tstFlags; +#endif + if( bHeader!=1 ){ + pNew->iStart = 0; + }else if( pNew->zData ){ + pNew->iStart = (int)sRdr.iIn; + }else{ + pNew->iStart = (int)(ftell(sRdr.in) - sRdr.nIn + sRdr.iIn); + } + csv_reader_reset(&sRdr); + rc = sqlite3_declare_vtab(db, CSV_SCHEMA); + if( rc ){ + csv_errmsg(&sRdr, "bad schema: '%s' - %s", CSV_SCHEMA, sqlite3_errmsg(db)); + goto csvtab_connect_error; + } + for(i=0; ibase); + for(i=0; ibase.pVtab; + int i; + for(i=0; inCol; i++){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + } +} + +/* +** The xConnect and xCreate methods do the same thing, but they must be +** different so that the virtual table is not an eponymous virtual table. +*/ +static int csvtabCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return csvtabConnect(db, pAux, argc, argv, ppVtab, pzErr); +} + +/* +** Destructor for a CsvCursor. +*/ +static int csvtabClose(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + csvtabCursorRowReset(pCur); + csv_reader_reset(&pCur->rdr); + sqlite3_free(cur); + return SQLITE_OK; +} + +/* +** Constructor for a new CsvTable cursor object. +*/ +static int csvtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + CsvTable *pTab = (CsvTable*)p; + CsvCursor *pCur; + size_t nByte; + nByte = sizeof(*pCur) + (sizeof(char*)+sizeof(int))*pTab->nCol; + pCur = sqlite3_malloc64( nByte ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, nByte); + pCur->azVal = (char**)&pCur[1]; + pCur->aLen = (int*)&pCur->azVal[pTab->nCol]; + *ppCursor = &pCur->base; + if( csv_reader_open(&pCur->rdr, pTab->zFilename, pTab->zData) ){ + csv_xfer_error(pTab, &pCur->rdr); + return SQLITE_ERROR; + } + return SQLITE_OK; +} + + +/* +** Advance a CsvCursor to its next row of input. +** Set the EOF marker if we reach the end of input. +*/ +static int csvtabNext(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + int i = 0; + char *z; + do{ + z = csv_read_one_field(&pCur->rdr); + if( z==0 ){ + break; + } + if( inCol ){ + if( pCur->aLen[i] < pCur->rdr.n+1 ){ + char *zNew = sqlite3_realloc64(pCur->azVal[i], pCur->rdr.n+1); + if( zNew==0 ){ + csv_errmsg(&pCur->rdr, "out of memory"); + csv_xfer_error(pTab, &pCur->rdr); + break; + } + pCur->azVal[i] = zNew; + pCur->aLen[i] = pCur->rdr.n+1; + } + memcpy(pCur->azVal[i], z, pCur->rdr.n+1); + i++; + } + }while( pCur->rdr.cTerm==',' ); + if( z==0 && i==0 ){ + pCur->iRowid = -1; + }else{ + pCur->iRowid++; + while( inCol ){ + sqlite3_free(pCur->azVal[i]); + pCur->azVal[i] = 0; + pCur->aLen[i] = 0; + i++; + } + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the CsvCursor +** is currently pointing. +*/ +static int csvtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + CsvCursor *pCur = (CsvCursor*)cur; + CsvTable *pTab = (CsvTable*)cur->pVtab; + if( i>=0 && inCol && pCur->azVal[i]!=0 ){ + sqlite3_result_text(ctx, pCur->azVal[i], -1, SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. +*/ +static int csvtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + CsvCursor *pCur = (CsvCursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int csvtabEof(sqlite3_vtab_cursor *cur){ + CsvCursor *pCur = (CsvCursor*)cur; + return pCur->iRowid<0; +} + +/* +** Only a full table scan is supported. So xFilter simply rewinds to +** the beginning. +*/ +static int csvtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + CsvCursor *pCur = (CsvCursor*)pVtabCursor; + CsvTable *pTab = (CsvTable*)pVtabCursor->pVtab; + pCur->iRowid = 0; + + /* Ensure the field buffer is always allocated. Otherwise, if the + ** first field is zero bytes in size, this may be mistaken for an OOM + ** error in csvtabNext(). */ + if( csv_append(&pCur->rdr, 0) ) return SQLITE_NOMEM; + + if( pCur->rdr.in==0 ){ + assert( pCur->rdr.zIn==pTab->zData ); + assert( pTab->iStart>=0 ); + assert( (size_t)pTab->iStart<=pCur->rdr.nIn ); + pCur->rdr.iIn = pTab->iStart; + }else{ + fseek(pCur->rdr.in, pTab->iStart, SEEK_SET); + pCur->rdr.iIn = 0; + pCur->rdr.nIn = 0; + } + return csvtabNext(pVtabCursor); +} + +/* +** Only a forward full table scan is supported. xBestIndex is mostly +** a no-op. If CSVTEST_FIDX is set, then the presence of equality +** constraints lowers the estimated cost, which is fiction, but is useful +** for testing certain kinds of virtual table behavior. +*/ +static int csvtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + pIdxInfo->estimatedCost = 1000000; +#ifdef SQLITE_TEST + if( (((CsvTable*)tab)->tstFlags & CSVTEST_FIDX)!=0 ){ + /* The usual (and sensible) case is to always do a full table scan. + ** The code in this branch only runs when testflags=1. This code + ** generates an artifical and unrealistic plan which is useful + ** for testing virtual table logic but is not helpful to real applications. + ** + ** Any ==, LIKE, or GLOB constraint is marked as usable by the virtual + ** table (even though it is not) and the cost of running the virtual table + ** is reduced from 1 million to just 10. The constraints are *not* marked + ** as omittable, however, so the query planner should still generate a + ** plan that gives a correct answer, even if they plan is not optimal. + */ + int i; + int nConst = 0; + for(i=0; inConstraint; i++){ + unsigned char op; + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + op = pIdxInfo->aConstraint[i].op; + if( op==SQLITE_INDEX_CONSTRAINT_EQ + || op==SQLITE_INDEX_CONSTRAINT_LIKE + || op==SQLITE_INDEX_CONSTRAINT_GLOB + ){ + pIdxInfo->estimatedCost = 10; + pIdxInfo->aConstraintUsage[nConst].argvIndex = nConst+1; + nConst++; + } + } + } +#endif + return SQLITE_OK; +} + + +static sqlite3_module CsvModule = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#ifdef SQLITE_TEST +/* +** For virtual table testing, make a version of the CSV virtual table +** available that has an xUpdate function. But the xUpdate always returns +** SQLITE_READONLY since the CSV file is not really writable. +*/ +static int csvtabUpdate(sqlite3_vtab *p,int n,sqlite3_value**v,sqlite3_int64*x){ + return SQLITE_READONLY; +} +static sqlite3_module CsvModuleFauxWrite = { + 0, /* iVersion */ + csvtabCreate, /* xCreate */ + csvtabConnect, /* xConnect */ + csvtabBestIndex, /* xBestIndex */ + csvtabDisconnect, /* xDisconnect */ + csvtabDisconnect, /* xDestroy */ + csvtabOpen, /* xOpen - open a cursor */ + csvtabClose, /* xClose - close a cursor */ + csvtabFilter, /* xFilter - configure scan constraints */ + csvtabNext, /* xNext - advance a cursor */ + csvtabEof, /* xEof - check for end of scan */ + csvtabColumn, /* xColumn - read data */ + csvtabRowid, /* xRowid - read data */ + csvtabUpdate, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; +#endif /* SQLITE_TEST */ + +#endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. The new +** CSV virtual table module is registered with the calling database +** connection. +*/ +int sqlite3_csv_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; + SQLITE_EXTENSION_INIT2(pApi); + rc = sqlite3_create_module(db, "csv", &CsvModule, 0); +#ifdef SQLITE_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "csv_wr", &CsvModuleFauxWrite, 0); + } +#endif + return rc; +#else + return SQLITE_OK; +#endif +} diff --git a/ext/misc/dbdump.c b/ext/misc/dbdump.c new file mode 100644 index 0000000..ecf7d81 --- /dev/null +++ b/ext/misc/dbdump.c @@ -0,0 +1,724 @@ +/* +** 2016-03-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements a C-language subroutine that converts the content +** of an SQLite database into UTF-8 text SQL statements that can be used +** to exactly recreate the original database. ROWID values are preserved. +** +** A prototype of the implemented subroutine is this: +** +** int sqlite3_db_dump( +** sqlite3 *db, +** const char *zSchema, +** const char *zTable, +** void (*xCallback)(void*, const char*), +** void *pArg +** ); +** +** The db parameter is the database connection. zSchema is the schema within +** that database which is to be dumped. Usually the zSchema is "main" but +** can also be "temp" or any ATTACH-ed database. If zTable is not NULL, then +** only the content of that one table is dumped. If zTable is NULL, then all +** tables are dumped. +** +** The generate text is passed to xCallback() in multiple calls. The second +** argument to xCallback() is a copy of the pArg parameter. The first +** argument is some of the output text that this routine generates. The +** signature to xCallback() is designed to make it compatible with fputs(). +** +** The sqlite3_db_dump() subroutine returns SQLITE_OK on success or some error +** code if it encounters a problem. +** +** If this file is compiled with -DDBDUMP_STANDALONE then a "main()" routine +** is included so that this routine becomes a command-line utility. The +** command-line utility takes two or three arguments which are the name +** of the database file, the schema, and optionally the table, forming the +** first three arguments of a single call to the library routine. +*/ +#include "sqlite3.h" +#include +#include +#include + +/* +** The state of the dump process. +*/ +typedef struct DState DState; +struct DState { + sqlite3 *db; /* The database connection */ + int nErr; /* Number of errors seen so far */ + int rc; /* Error code */ + int writableSchema; /* True if in writable_schema mode */ + int (*xCallback)(const char*,void*); /* Send output here */ + void *pArg; /* Argument to xCallback() */ +}; + +/* +** A variable length string to which one can append text. +*/ +typedef struct DText DText; +struct DText { + char *z; /* The text */ + int n; /* Number of bytes of content in z[] */ + int nAlloc; /* Number of bytes allocated to z[] */ +}; + +/* +** Initialize and destroy a DText object +*/ +static void initText(DText *p){ + memset(p, 0, sizeof(*p)); +} +static void freeText(DText *p){ + sqlite3_free(p->z); + initText(p); +} + +/* zIn is either a pointer to a NULL-terminated string in memory obtained +** from malloc(), or a NULL pointer. The string pointed to by zAppend is +** added to zIn, and the result returned in memory obtained from malloc(). +** zIn, if it was not NULL, is freed. +** +** If the third argument, quote, is not '\0', then it is used as a +** quote character for zAppend. +*/ +static void appendText(DText *p, char const *zAppend, char quote){ + int len; + int i; + int nAppend = (int)(strlen(zAppend) & 0x3fffffff); + + len = nAppend+p->n+1; + if( quote ){ + len += 2; + for(i=0; in+len>=p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + len + 20; + zNew = sqlite3_realloc(p->z, p->nAlloc); + if( zNew==0 ){ + freeText(p); + return; + } + p->z = zNew; + } + + if( quote ){ + char *zCsr = p->z+p->n; + *zCsr++ = quote; + for(i=0; in = (int)(zCsr - p->z); + *zCsr = '\0'; + }else{ + memcpy(p->z+p->n, zAppend, nAppend); + p->n += nAppend; + p->z[p->n] = '\0'; + } +} + +/* +** Attempt to determine if identifier zName needs to be quoted, either +** because it contains non-alphanumeric characters, or because it is an +** SQLite keyword. Be conservative in this estimate: When in doubt assume +** that quoting is required. +** +** Return '"' if quoting is required. Return 0 if no quoting is required. +*/ +static char quoteChar(const char *zName){ + int i; + if( !isalpha((unsigned char)zName[0]) && zName[0]!='_' ) return '"'; + for(i=0; zName[i]; i++){ + if( !isalnum((unsigned char)zName[i]) && zName[i]!='_' ) return '"'; + } + return sqlite3_keyword_check(zName, i) ? '"' : 0; +} + + +/* +** Release memory previously allocated by tableColumnList(). +*/ +static void freeColumnList(char **azCol){ + int i; + for(i=1; azCol[i]; i++){ + sqlite3_free(azCol[i]); + } + /* azCol[0] is a static string */ + sqlite3_free(azCol); +} + +/* +** Return a list of pointers to strings which are the names of all +** columns in table zTab. The memory to hold the names is dynamically +** allocated and must be released by the caller using a subsequent call +** to freeColumnList(). +** +** The azCol[0] entry is usually NULL. However, if zTab contains a rowid +** value that needs to be preserved, then azCol[0] is filled in with the +** name of the rowid column. +** +** The first regular column in the table is azCol[1]. The list is terminated +** by an entry with azCol[i]==0. +*/ +static char **tableColumnList(DState *p, const char *zTab){ + char **azCol = 0; + sqlite3_stmt *pStmt = 0; + char *zSql; + int nCol = 0; + int nAlloc = 0; + int nPK = 0; /* Number of PRIMARY KEY columns seen */ + int isIPK = 0; /* True if one PRIMARY KEY column of type INTEGER */ + int preserveRowid = 1; + int rc; + + zSql = sqlite3_mprintf("PRAGMA table_info=%Q", zTab); + if( zSql==0 ) return 0; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ) return 0; + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + if( nCol>=nAlloc-2 ){ + char **azNew; + nAlloc = nAlloc*2 + nCol + 10; + azNew = sqlite3_realloc64(azCol, nAlloc*sizeof(azCol[0])); + if( azNew==0 ) goto col_oom; + azCol = azNew; + azCol[0] = 0; + } + azCol[++nCol] = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( azCol[nCol]==0 ) goto col_oom; + if( sqlite3_column_int(pStmt, 5) ){ + nPK++; + if( nPK==1 + && sqlite3_stricmp((const char*)sqlite3_column_text(pStmt,2), + "INTEGER")==0 + ){ + isIPK = 1; + }else{ + isIPK = 0; + } + } + } + sqlite3_finalize(pStmt); + pStmt = 0; + azCol[nCol+1] = 0; + + /* The decision of whether or not a rowid really needs to be preserved + ** is tricky. We never need to preserve a rowid for a WITHOUT ROWID table + ** or a table with an INTEGER PRIMARY KEY. We are unable to preserve + ** rowids on tables where the rowid is inaccessible because there are other + ** columns in the table named "rowid", "_rowid_", and "oid". + */ + if( isIPK ){ + /* If a single PRIMARY KEY column with type INTEGER was seen, then it + ** might be an alise for the ROWID. But it might also be a WITHOUT ROWID + ** table or a INTEGER PRIMARY KEY DESC column, neither of which are + ** ROWID aliases. To distinguish these cases, check to see if + ** there is a "pk" entry in "PRAGMA index_list". There will be + ** no "pk" index if the PRIMARY KEY really is an alias for the ROWID. + */ + zSql = sqlite3_mprintf("SELECT 1 FROM pragma_index_list(%Q)" + " WHERE origin='pk'", zTab); + if( zSql==0 ) goto col_oom; + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + freeColumnList(azCol); + return 0; + } + rc = sqlite3_step(pStmt); + sqlite3_finalize(pStmt); + pStmt = 0; + preserveRowid = rc==SQLITE_ROW; + } + if( preserveRowid ){ + /* Only preserve the rowid if we can find a name to use for the + ** rowid */ + static char *azRowid[] = { "rowid", "_rowid_", "oid" }; + int i, j; + for(j=0; j<3; j++){ + for(i=1; i<=nCol; i++){ + if( sqlite3_stricmp(azRowid[j],azCol[i])==0 ) break; + } + if( i>nCol ){ + /* At this point, we know that azRowid[j] is not the name of any + ** ordinary column in the table. Verify that azRowid[j] is a valid + ** name for the rowid before adding it to azCol[0]. WITHOUT ROWID + ** tables will fail this last check */ + rc = sqlite3_table_column_metadata(p->db,0,zTab,azRowid[j],0,0,0,0,0); + if( rc==SQLITE_OK ) azCol[0] = azRowid[j]; + break; + } + } + } + return azCol; + +col_oom: + sqlite3_finalize(pStmt); + freeColumnList(azCol); + p->nErr++; + p->rc = SQLITE_NOMEM; + return 0; +} + +/* +** Send mprintf-formatted content to the output callback. +*/ +static void output_formatted(DState *p, const char *zFormat, ...){ + va_list ap; + char *z; + va_start(ap, zFormat); + z = sqlite3_vmprintf(zFormat, ap); + va_end(ap); + p->xCallback(z, p->pArg); + sqlite3_free(z); +} + +/* +** Find a string that is not found anywhere in z[]. Return a pointer +** to that string. +** +** Try to use zA and zB first. If both of those are already found in z[] +** then make up some string and store it in the buffer zBuf. +*/ +static const char *unused_string( + const char *z, /* Result must not appear anywhere in z */ + const char *zA, const char *zB, /* Try these first */ + char *zBuf /* Space to store a generated string */ +){ + unsigned i = 0; + if( strstr(z, zA)==0 ) return zA; + if( strstr(z, zB)==0 ) return zB; + do{ + sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++); + }while( strstr(z,zBuf)!=0 ); + return zBuf; +} + +/* +** Output the given string as a quoted string using SQL quoting conventions. +** Additionallly , escape the "\n" and "\r" characters so that they do not +** get corrupted by end-of-line translation facilities in some operating +** systems. +*/ +static void output_quoted_escaped_string(DState *p, const char *z){ + int i; + char c; + for(i=0; (c = z[i])!=0 && c!='\'' && c!='\n' && c!='\r'; i++){} + if( c==0 ){ + output_formatted(p,"'%s'",z); + }else{ + const char *zNL = 0; + const char *zCR = 0; + int nNL = 0; + int nCR = 0; + char zBuf1[20], zBuf2[20]; + for(i=0; z[i]; i++){ + if( z[i]=='\n' ) nNL++; + if( z[i]=='\r' ) nCR++; + } + if( nNL ){ + p->xCallback("replace(", p->pArg); + zNL = unused_string(z, "\\n", "\\012", zBuf1); + } + if( nCR ){ + p->xCallback("replace(", p->pArg); + zCR = unused_string(z, "\\r", "\\015", zBuf2); + } + p->xCallback("'", p->pArg); + while( *z ){ + for(i=0; (c = z[i])!=0 && c!='\n' && c!='\r' && c!='\''; i++){} + if( c=='\'' ) i++; + if( i ){ + output_formatted(p, "%.*s", i, z); + z += i; + } + if( c=='\'' ){ + p->xCallback("'", p->pArg); + continue; + } + if( c==0 ){ + break; + } + z++; + if( c=='\n' ){ + p->xCallback(zNL, p->pArg); + continue; + } + p->xCallback(zCR, p->pArg); + } + p->xCallback("'", p->pArg); + if( nCR ){ + output_formatted(p, ",'%s',char(13))", zCR); + } + if( nNL ){ + output_formatted(p, ",'%s',char(10))", zNL); + } + } +} + +/* +** This is an sqlite3_exec callback routine used for dumping the database. +** Each row received by this callback consists of a table name, +** the table type ("index" or "table") and SQL to create the table. +** This routine should print text sufficient to recreate the table. +*/ +static int dump_callback(void *pArg, int nArg, char **azArg, char **azCol){ + int rc; + const char *zTable; + const char *zType; + const char *zSql; + DState *p = (DState*)pArg; + sqlite3_stmt *pStmt; + + (void)azCol; + if( nArg!=3 ) return 1; + zTable = azArg[0]; + zType = azArg[1]; + zSql = azArg[2]; + + if( strcmp(zTable, "sqlite_sequence")==0 ){ + p->xCallback("DELETE FROM sqlite_sequence;\n", p->pArg); + }else if( sqlite3_strglob("sqlite_stat?", zTable)==0 ){ + p->xCallback("ANALYZE sqlite_schema;\n", p->pArg); + }else if( strncmp(zTable, "sqlite_", 7)==0 ){ + return 0; + }else if( strncmp(zSql, "CREATE VIRTUAL TABLE", 20)==0 ){ + if( !p->writableSchema ){ + p->xCallback("PRAGMA writable_schema=ON;\n", p->pArg); + p->writableSchema = 1; + } + output_formatted(p, + "INSERT INTO sqlite_schema(type,name,tbl_name,rootpage,sql)" + "VALUES('table','%q','%q',0,'%q');", + zTable, zTable, zSql); + return 0; + }else{ + if( sqlite3_strglob("CREATE TABLE ['\"]*", zSql)==0 ){ + p->xCallback("CREATE TABLE IF NOT EXISTS ", p->pArg); + p->xCallback(zSql+13, p->pArg); + }else{ + p->xCallback(zSql, p->pArg); + } + p->xCallback(";\n", p->pArg); + } + + if( strcmp(zType, "table")==0 ){ + DText sSelect; + DText sTable; + char **azTCol; + int i; + int nCol; + + azTCol = tableColumnList(p, zTable); + if( azTCol==0 ) return 0; + + initText(&sTable); + appendText(&sTable, "INSERT INTO ", 0); + + /* Always quote the table name, even if it appears to be pure ascii, + ** in case it is a keyword. Ex: INSERT INTO "table" ... */ + appendText(&sTable, zTable, quoteChar(zTable)); + + /* If preserving the rowid, add a column list after the table name. + ** In other words: "INSERT INTO tab(rowid,a,b,c,...) VALUES(...)" + ** instead of the usual "INSERT INTO tab VALUES(...)". + */ + if( azTCol[0] ){ + appendText(&sTable, "(", 0); + appendText(&sTable, azTCol[0], 0); + for(i=1; azTCol[i]; i++){ + appendText(&sTable, ",", 0); + appendText(&sTable, azTCol[i], quoteChar(azTCol[i])); + } + appendText(&sTable, ")", 0); + } + appendText(&sTable, " VALUES(", 0); + + /* Build an appropriate SELECT statement */ + initText(&sSelect); + appendText(&sSelect, "SELECT ", 0); + if( azTCol[0] ){ + appendText(&sSelect, azTCol[0], 0); + appendText(&sSelect, ",", 0); + } + for(i=1; azTCol[i]; i++){ + appendText(&sSelect, azTCol[i], quoteChar(azTCol[i])); + if( azTCol[i+1] ){ + appendText(&sSelect, ",", 0); + } + } + nCol = i; + if( azTCol[0]==0 ) nCol--; + freeColumnList(azTCol); + appendText(&sSelect, " FROM ", 0); + appendText(&sSelect, zTable, quoteChar(zTable)); + + rc = sqlite3_prepare_v2(p->db, sSelect.z, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + p->nErr++; + if( p->rc==SQLITE_OK ) p->rc = rc; + }else{ + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + p->xCallback(sTable.z, p->pArg); + for(i=0; ixCallback(",", p->pArg); + switch( sqlite3_column_type(pStmt,i) ){ + case SQLITE_INTEGER: { + output_formatted(p, "%lld", sqlite3_column_int64(pStmt,i)); + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_column_double(pStmt,i); + sqlite3_uint64 ur; + memcpy(&ur,&r,sizeof(r)); + if( ur==0x7ff0000000000000LL ){ + p->xCallback("1e999", p->pArg); + }else if( ur==0xfff0000000000000LL ){ + p->xCallback("-1e999", p->pArg); + }else{ + output_formatted(p, "%!.20g", r); + } + break; + } + case SQLITE_NULL: { + p->xCallback("NULL", p->pArg); + break; + } + case SQLITE_TEXT: { + output_quoted_escaped_string(p, + (const char*)sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_BLOB: { + int nByte = sqlite3_column_bytes(pStmt,i); + unsigned char *a = (unsigned char*)sqlite3_column_blob(pStmt,i); + int j; + p->xCallback("x'", p->pArg); + for(j=0; j>4)&15]; + zWord[1] = "0123456789abcdef"[a[j]&15]; + zWord[2] = 0; + p->xCallback(zWord, p->pArg); + } + p->xCallback("'", p->pArg); + break; + } + } + } + p->xCallback(");\n", p->pArg); + } + } + sqlite3_finalize(pStmt); + freeText(&sTable); + freeText(&sSelect); + } + return 0; +} + + +/* +** Execute a query statement that will generate SQL output. Print +** the result columns, comma-separated, on a line and then add a +** semicolon terminator to the end of that line. +** +** If the number of columns is 1 and that column contains text "--" +** then write the semicolon on a separate line. That way, if a +** "--" comment occurs at the end of the statement, the comment +** won't consume the semicolon terminator. +*/ +static void output_sql_from_query( + DState *p, /* Query context */ + const char *zSelect, /* SELECT statement to extract content */ + ... +){ + sqlite3_stmt *pSelect; + int rc; + int nResult; + int i; + const char *z; + char *zSql; + va_list ap; + va_start(ap, zSelect); + zSql = sqlite3_vmprintf(zSelect, ap); + va_end(ap); + if( zSql==0 ){ + p->rc = SQLITE_NOMEM; + p->nErr++; + return; + } + rc = sqlite3_prepare_v2(p->db, zSql, -1, &pSelect, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK || !pSelect ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + p->nErr++; + return; + } + rc = sqlite3_step(pSelect); + nResult = sqlite3_column_count(pSelect); + while( rc==SQLITE_ROW ){ + z = (const char*)sqlite3_column_text(pSelect, 0); + p->xCallback(z, p->pArg); + for(i=1; ixCallback(",", p->pArg); + p->xCallback((const char*)sqlite3_column_text(pSelect,i), p->pArg); + } + if( z==0 ) z = ""; + while( z[0] && (z[0]!='-' || z[1]!='-') ) z++; + if( z[0] ){ + p->xCallback("\n;\n", p->pArg); + }else{ + p->xCallback(";\n", p->pArg); + } + rc = sqlite3_step(pSelect); + } + rc = sqlite3_finalize(pSelect); + if( rc!=SQLITE_OK ){ + output_formatted(p, "/**** ERROR: (%d) %s *****/\n", rc, + sqlite3_errmsg(p->db)); + if( (rc&0xff)!=SQLITE_CORRUPT ) p->nErr++; + } +} + +/* +** Run zQuery. Use dump_callback() as the callback routine so that +** the contents of the query are output as SQL statements. +** +** If we get a SQLITE_CORRUPT error, rerun the query after appending +** "ORDER BY rowid DESC" to the end. +*/ +static void run_schema_dump_query( + DState *p, + const char *zQuery, + ... +){ + char *zErr = 0; + char *z; + va_list ap; + va_start(ap, zQuery); + z = sqlite3_vmprintf(zQuery, ap); + va_end(ap); + sqlite3_exec(p->db, z, dump_callback, p, &zErr); + sqlite3_free(z); + if( zErr ){ + output_formatted(p, "/****** %s ******/\n", zErr); + sqlite3_free(zErr); + p->nErr++; + zErr = 0; + } +} + +/* +** Convert an SQLite database into SQL statements that will recreate that +** database. +*/ +int sqlite3_db_dump( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which schema to dump. Usually "main". */ + const char *zTable, /* Which table to dump. NULL means everything. */ + int (*xCallback)(const char*,void*), /* Output sent to this callback */ + void *pArg /* Second argument of the callback */ +){ + DState x; + memset(&x, 0, sizeof(x)); + x.rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + if( x.rc ) return x.rc; + x.db = db; + x.xCallback = xCallback; + x.pArg = pArg; + xCallback("PRAGMA foreign_keys=OFF;\nBEGIN TRANSACTION;\n", pArg); + if( zTable==0 ){ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_schema " + "WHERE sql NOT NULL AND type=='table' AND name!='sqlite_sequence'", + zSchema + ); + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_schema " + "WHERE name=='sqlite_sequence'", zSchema + ); + output_sql_from_query(&x, + "SELECT sql FROM sqlite_schema " + "WHERE sql NOT NULL AND type IN ('index','trigger','view')", 0 + ); + }else{ + run_schema_dump_query(&x, + "SELECT name, type, sql FROM \"%w\".sqlite_schema " + "WHERE tbl_name=%Q COLLATE nocase AND type=='table'" + " AND sql NOT NULL", + zSchema, zTable + ); + output_sql_from_query(&x, + "SELECT sql FROM \"%w\".sqlite_schema " + "WHERE sql NOT NULL" + " AND type IN ('index','trigger','view')" + " AND tbl_name=%Q COLLATE nocase", + zSchema, zTable + ); + } + if( x.writableSchema ){ + xCallback("PRAGMA writable_schema=OFF;\n", pArg); + } + xCallback(x.nErr ? "ROLLBACK; -- due to errors\n" : "COMMIT;\n", pArg); + sqlite3_exec(db, "COMMIT", 0, 0, 0); + return x.rc; +} + + + +/* The generic subroutine is above. The code the follows implements +** the command-line interface. +*/ +#ifdef DBDUMP_STANDALONE +#include + +/* +** Command-line interface +*/ +int main(int argc, char **argv){ + sqlite3 *db; + const char *zDb; + const char *zSchema; + const char *zTable = 0; + int rc; + + if( argc<2 || argc>4 ){ + fprintf(stderr, "Usage: %s DATABASE ?SCHEMA? ?TABLE?\n", argv[0]); + return 1; + } + zDb = argv[1]; + zSchema = argc>=3 ? argv[2] : "main"; + zTable = argc==4 ? argv[3] : 0; + + rc = sqlite3_open(zDb, &db); + if( rc ){ + fprintf(stderr, "Cannot open \"%s\": %s\n", zDb, sqlite3_errmsg(db)); + sqlite3_close(db); + return 1; + } + rc = sqlite3_db_dump(db, zSchema, zTable, + (int(*)(const char*,void*))fputs, (void*)stdout); + if( rc ){ + fprintf(stderr, "Error: sqlite3_db_dump() returns %d\n", rc); + } + sqlite3_close(db); + return rc!=SQLITE_OK; +} +#endif /* DBDUMP_STANDALONE */ diff --git a/ext/misc/decimal.c b/ext/misc/decimal.c new file mode 100644 index 0000000..9365ae6 --- /dev/null +++ b/ext/misc/decimal.c @@ -0,0 +1,884 @@ +/* +** 2020-06-22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Routines to implement arbitrary-precision decimal math. +** +** The focus here is on simplicity and correctness, not performance. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + + +/* A decimal object */ +typedef struct Decimal Decimal; +struct Decimal { + char sign; /* 0 for positive, 1 for negative */ + char oom; /* True if an OOM is encountered */ + char isNull; /* True if holds a NULL rather than a number */ + char isInit; /* True upon initialization */ + int nDigit; /* Total number of digits */ + int nFrac; /* Number of digits to the right of the decimal point */ + signed char *a; /* Array of digits. Most significant first. */ +}; + +/* +** Release memory held by a Decimal, but do not free the object itself. +*/ +static void decimal_clear(Decimal *p){ + sqlite3_free(p->a); +} + +/* +** Destroy a Decimal object +*/ +static void decimal_free(Decimal *p){ + if( p ){ + decimal_clear(p); + sqlite3_free(p); + } +} + +/* +** Allocate a new Decimal object initialized to the text in zIn[]. +** Return NULL if any kind of error occurs. +*/ +static Decimal *decimalNewFromText(const char *zIn, int n){ + Decimal *p = 0; + int i; + int iExp = 0; + + p = sqlite3_malloc( sizeof(*p) ); + if( p==0 ) goto new_from_text_failed; + p->sign = 0; + p->oom = 0; + p->isInit = 1; + p->isNull = 0; + p->nDigit = 0; + p->nFrac = 0; + p->a = sqlite3_malloc64( n+1 ); + if( p->a==0 ) goto new_from_text_failed; + for(i=0; isspace(zIn[i]); i++){} + if( zIn[i]=='-' ){ + p->sign = 1; + i++; + }else if( zIn[i]=='+' ){ + i++; + } + while( i='0' && c<='9' ){ + p->a[p->nDigit++] = c - '0'; + }else if( c=='.' ){ + p->nFrac = p->nDigit + 1; + }else if( c=='e' || c=='E' ){ + int j = i+1; + int neg = 0; + if( j>=n ) break; + if( zIn[j]=='-' ){ + neg = 1; + j++; + }else if( zIn[j]=='+' ){ + j++; + } + while( j='0' && zIn[j]<='9' ){ + iExp = iExp*10 + zIn[j] - '0'; + } + j++; + } + if( neg ) iExp = -iExp; + break; + } + i++; + } + if( p->nFrac ){ + p->nFrac = p->nDigit - (p->nFrac - 1); + } + if( iExp>0 ){ + if( p->nFrac>0 ){ + if( iExp<=p->nFrac ){ + p->nFrac -= iExp; + iExp = 0; + }else{ + iExp -= p->nFrac; + p->nFrac = 0; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memset(p->a+p->nDigit, 0, iExp); + p->nDigit += iExp; + } + }else if( iExp<0 ){ + int nExtra; + iExp = -iExp; + nExtra = p->nDigit - p->nFrac - 1; + if( nExtra ){ + if( nExtra>=iExp ){ + p->nFrac += iExp; + iExp = 0; + }else{ + iExp -= nExtra; + p->nFrac = p->nDigit - 1; + } + } + if( iExp>0 ){ + p->a = sqlite3_realloc64(p->a, p->nDigit + iExp + 1 ); + if( p->a==0 ) goto new_from_text_failed; + memmove(p->a+iExp, p->a, p->nDigit); + memset(p->a, 0, iExp); + p->nDigit += iExp; + p->nFrac += iExp; + } + } + return p; + +new_from_text_failed: + if( p ){ + if( p->a ) sqlite3_free(p->a); + sqlite3_free(p); + } + return 0; +} + +/* Forward reference */ +static Decimal *decimalFromDouble(double); + +/* +** Allocate a new Decimal object from an sqlite3_value. Return a pointer +** to the new object, or NULL if there is an error. If the pCtx argument +** is not NULL, then errors are reported on it as well. +** +** If the pIn argument is SQLITE_TEXT or SQLITE_INTEGER, it is converted +** directly into a Decimal. For SQLITE_FLOAT or for SQLITE_BLOB of length +** 8 bytes, the resulting double value is expanded into its decimal equivalent. +** If pIn is NULL or if it is a BLOB that is not exactly 8 bytes in length, +** then NULL is returned. +*/ +static Decimal *decimal_new( + sqlite3_context *pCtx, /* Report error here, if not null */ + sqlite3_value *pIn, /* Construct the decimal object from this */ + int bTextOnly /* Always interpret pIn as text if true */ +){ + Decimal *p = 0; + int eType = sqlite3_value_type(pIn); + if( bTextOnly && (eType==SQLITE_FLOAT || eType==SQLITE_BLOB) ){ + eType = SQLITE_TEXT; + } + switch( eType ){ + case SQLITE_TEXT: + case SQLITE_INTEGER: { + const char *zIn = (const char*)sqlite3_value_text(pIn); + int n = sqlite3_value_bytes(pIn); + p = decimalNewFromText(zIn, n); + if( p==0 ) goto new_failed; + break; + } + + case SQLITE_FLOAT: { + p = decimalFromDouble(sqlite3_value_double(pIn)); + break; + } + + case SQLITE_BLOB: { + const unsigned char *x; + unsigned int i; + sqlite3_uint64 v = 0; + double r; + + if( sqlite3_value_bytes(pIn)!=sizeof(r) ) break; + x = sqlite3_value_blob(pIn); + for(i=0; ioom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + z = sqlite3_malloc( p->nDigit+4 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + i = 0; + if( p->nDigit==0 || (p->nDigit==1 && p->a[0]==0) ){ + p->sign = 0; + } + if( p->sign ){ + z[0] = '-'; + i = 1; + } + n = p->nDigit - p->nFrac; + if( n<=0 ){ + z[i++] = '0'; + } + j = 0; + while( n>1 && p->a[j]==0 ){ + j++; + n--; + } + while( n>0 ){ + z[i++] = p->a[j] + '0'; + j++; + n--; + } + if( p->nFrac ){ + z[i++] = '.'; + do{ + z[i++] = p->a[j] + '0'; + j++; + }while( jnDigit ); + } + z[i] = 0; + sqlite3_result_text(pCtx, z, i, sqlite3_free); +} + +/* +** Make the given Decimal the result in an format similar to '%+#e'. +** In other words, show exponential notation with leading and trailing +** zeros omitted. +*/ +static void decimal_result_sci(sqlite3_context *pCtx, Decimal *p){ + char *z; /* The output buffer */ + int i; /* Loop counter */ + int nZero; /* Number of leading zeros */ + int nDigit; /* Number of digits not counting trailing zeros */ + int nFrac; /* Digits to the right of the decimal point */ + int exp; /* Exponent value */ + signed char zero; /* Zero value */ + signed char *a; /* Array of digits */ + + if( p==0 || p->oom ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( p->isNull ){ + sqlite3_result_null(pCtx); + return; + } + for(nDigit=p->nDigit; nDigit>0 && p->a[nDigit-1]==0; nDigit--){} + for(nZero=0; nZeroa[nZero]==0; nZero++){} + nFrac = p->nFrac + (nDigit - p->nDigit); + nDigit -= nZero; + z = sqlite3_malloc( nDigit+20 ); + if( z==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + if( nDigit==0 ){ + zero = 0; + a = &zero; + nDigit = 1; + nFrac = 0; + }else{ + a = &p->a[nZero]; + } + if( p->sign && nDigit>0 ){ + z[0] = '-'; + }else{ + z[0] = '+'; + } + z[1] = a[0]+'0'; + z[2] = '.'; + if( nDigit==1 ){ + z[3] = '0'; + i = 4; + }else{ + for(i=1; iisNull==0 +** pB!=0 +** pB->isNull==0 +*/ +static int decimal_cmp(const Decimal *pA, const Decimal *pB){ + int nASig, nBSig, rc, n; + if( pA->sign!=pB->sign ){ + return pA->sign ? -1 : +1; + } + if( pA->sign ){ + const Decimal *pTemp = pA; + pA = pB; + pB = pTemp; + } + nASig = pA->nDigit - pA->nFrac; + nBSig = pB->nDigit - pB->nFrac; + if( nASig!=nBSig ){ + return nASig - nBSig; + } + n = pA->nDigit; + if( n>pB->nDigit ) n = pB->nDigit; + rc = memcmp(pA->a, pB->a, n); + if( rc==0 ){ + rc = pA->nDigit - pB->nDigit; + } + return rc; +} + +/* +** SQL Function: decimal_cmp(X, Y) +** +** Return negative, zero, or positive if X is less then, equal to, or +** greater than Y. +*/ +static void decimalCmpFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = 0, *pB = 0; + int rc; + + UNUSED_PARAMETER(argc); + pA = decimal_new(context, argv[0], 1); + if( pA==0 || pA->isNull ) goto cmp_done; + pB = decimal_new(context, argv[1], 1); + if( pB==0 || pB->isNull ) goto cmp_done; + rc = decimal_cmp(pA, pB); + if( rc<0 ) rc = -1; + else if( rc>0 ) rc = +1; + sqlite3_result_int(context, rc); +cmp_done: + decimal_free(pA); + decimal_free(pB); +} + +/* +** Expand the Decimal so that it has a least nDigit digits and nFrac +** digits to the right of the decimal point. +*/ +static void decimal_expand(Decimal *p, int nDigit, int nFrac){ + int nAddSig; + int nAddFrac; + if( p==0 ) return; + nAddFrac = nFrac - p->nFrac; + nAddSig = (nDigit - p->nDigit) - nAddFrac; + if( nAddFrac==0 && nAddSig==0 ) return; + p->a = sqlite3_realloc64(p->a, nDigit+1); + if( p->a==0 ){ + p->oom = 1; + return; + } + if( nAddSig ){ + memmove(p->a+nAddSig, p->a, p->nDigit); + memset(p->a, 0, nAddSig); + p->nDigit += nAddSig; + } + if( nAddFrac ){ + memset(p->a+p->nDigit, 0, nAddFrac); + p->nDigit += nAddFrac; + p->nFrac += nAddFrac; + } +} + +/* +** Add the value pB into pA. A := A + B. +** +** Both pA and pB might become denormalized by this routine. +*/ +static void decimal_add(Decimal *pA, Decimal *pB){ + int nSig, nFrac, nDigit; + int i, rc; + if( pA==0 ){ + return; + } + if( pA->oom || pB==0 || pB->oom ){ + pA->oom = 1; + return; + } + if( pA->isNull || pB->isNull ){ + pA->isNull = 1; + return; + } + nSig = pA->nDigit - pA->nFrac; + if( nSig && pA->a[0]==0 ) nSig--; + if( nSignDigit-pB->nFrac ){ + nSig = pB->nDigit - pB->nFrac; + } + nFrac = pA->nFrac; + if( nFracnFrac ) nFrac = pB->nFrac; + nDigit = nSig + nFrac + 1; + decimal_expand(pA, nDigit, nFrac); + decimal_expand(pB, nDigit, nFrac); + if( pA->oom || pB->oom ){ + pA->oom = 1; + }else{ + if( pA->sign==pB->sign ){ + int carry = 0; + for(i=nDigit-1; i>=0; i--){ + int x = pA->a[i] + pB->a[i] + carry; + if( x>=10 ){ + carry = 1; + pA->a[i] = x - 10; + }else{ + carry = 0; + pA->a[i] = x; + } + } + }else{ + signed char *aA, *aB; + int borrow = 0; + rc = memcmp(pA->a, pB->a, nDigit); + if( rc<0 ){ + aA = pB->a; + aB = pA->a; + pA->sign = !pA->sign; + }else{ + aA = pA->a; + aB = pB->a; + } + for(i=nDigit-1; i>=0; i--){ + int x = aA[i] - aB[i] - borrow; + if( x<0 ){ + pA->a[i] = x+10; + borrow = 1; + }else{ + pA->a[i] = x; + borrow = 0; + } + } + } + } +} + +/* +** Multiply A by B. A := A * B +** +** All significant digits after the decimal point are retained. +** Trailing zeros after the decimal point are omitted as long as +** the number of digits after the decimal point is no less than +** either the number of digits in either input. +*/ +static void decimalMul(Decimal *pA, Decimal *pB){ + signed char *acc = 0; + int i, j, k; + int minFrac; + + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + acc = sqlite3_malloc64( pA->nDigit + pB->nDigit + 2 ); + if( acc==0 ){ + pA->oom = 1; + goto mul_end; + } + memset(acc, 0, pA->nDigit + pB->nDigit + 2); + minFrac = pA->nFrac; + if( pB->nFracnFrac; + for(i=pA->nDigit-1; i>=0; i--){ + signed char f = pA->a[i]; + int carry = 0, x; + for(j=pB->nDigit-1, k=i+j+3; j>=0; j--, k--){ + x = acc[k] + f*pB->a[j] + carry; + acc[k] = x%10; + carry = x/10; + } + x = acc[k] + carry; + acc[k] = x%10; + acc[k-1] += x/10; + } + sqlite3_free(pA->a); + pA->a = acc; + acc = 0; + pA->nDigit += pB->nDigit + 2; + pA->nFrac += pB->nFrac; + pA->sign ^= pB->sign; + while( pA->nFrac>minFrac && pA->a[pA->nDigit-1]==0 ){ + pA->nFrac--; + pA->nDigit--; + } + +mul_end: + sqlite3_free(acc); +} + +/* +** Create a new Decimal object that contains an integer power of 2. +*/ +static Decimal *decimalPow2(int N){ + Decimal *pA = 0; /* The result to be returned */ + Decimal *pX = 0; /* Multiplier */ + if( N<-20000 || N>20000 ) goto pow2_fault; + pA = decimalNewFromText("1.0", 3); + if( pA==0 || pA->oom ) goto pow2_fault; + if( N==0 ) return pA; + if( N>0 ){ + pX = decimalNewFromText("2.0", 3); + }else{ + N = -N; + pX = decimalNewFromText("0.5", 3); + } + if( pX==0 || pX->oom ) goto pow2_fault; + while( 1 /* Exit by break */ ){ + if( N & 1 ){ + decimalMul(pA, pX); + if( pA->oom ) goto pow2_fault; + } + N >>= 1; + if( N==0 ) break; + decimalMul(pX, pX); + } + decimal_free(pX); + return pA; + +pow2_fault: + decimal_free(pA); + decimal_free(pX); + return 0; +} + +/* +** Use an IEEE754 binary64 ("double") to generate a new Decimal object. +*/ +static Decimal *decimalFromDouble(double r){ + sqlite3_int64 m, a; + int e; + int isNeg; + Decimal *pA; + Decimal *pX; + char zNum[100]; + if( r<0.0 ){ + isNeg = 1; + r = -r; + }else{ + isNeg = 0; + } + memcpy(&a,&r,sizeof(a)); + if( a==0 ){ + e = 0; + m = 0; + }else{ + e = a>>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + e = e - 1075; + if( e>971 ){ + return 0; /* A NaN or an Infinity */ + } + } + + /* At this point m is the integer significand and e is the exponent */ + sqlite3_snprintf(sizeof(zNum), zNum, "%lld", m); + pA = decimalNewFromText(zNum, (int)strlen(zNum)); + pX = decimalPow2(e); + decimalMul(pA, pX); + decimal_free(pX); + return pA; +} + +/* +** SQL Function: decimal(X) +** OR: decimal_exp(X) +** +** Convert input X into decimal and then back into text. +** +** If X is originally a float, then a full decimal expansion of that floating +** point value is done. Or if X is an 8-byte blob, it is interpreted +** as a float and similarly expanded. +** +** The decimal_exp(X) function returns the result in exponential notation. +** decimal(X) returns a complete decimal, without the e+NNN at the end. +*/ +static void decimalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p = decimal_new(context, argv[0], 0); + UNUSED_PARAMETER(argc); + if( p ){ + if( sqlite3_user_data(context)!=0 ){ + decimal_result_sci(context, p); + }else{ + decimal_result(context, p); + } + decimal_free(p); + } +} + +/* +** Compare text in decimal order. +*/ +static int decimalCollFunc( + void *notUsed, + int nKey1, const void *pKey1, + int nKey2, const void *pKey2 +){ + const unsigned char *zA = (const unsigned char*)pKey1; + const unsigned char *zB = (const unsigned char*)pKey2; + Decimal *pA = decimalNewFromText((const char*)zA, nKey1); + Decimal *pB = decimalNewFromText((const char*)zB, nKey2); + int rc; + UNUSED_PARAMETER(notUsed); + if( pA==0 || pB==0 ){ + rc = 0; + }else{ + rc = decimal_cmp(pA, pB); + } + decimal_free(pA); + decimal_free(pB); + return rc; +} + + +/* +** SQL Function: decimal_add(X, Y) +** decimal_sub(X, Y) +** +** Return the sum or difference of X and Y. +*/ +static void decimalAddFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + decimal_add(pA, pB); + decimal_result(context, pA); + decimal_free(pA); + decimal_free(pB); +} +static void decimalSubFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pB ){ + pB->sign = !pB->sign; + decimal_add(pA, pB); + decimal_result(context, pA); + } + decimal_free(pA); + decimal_free(pB); +} + +/* Aggregate funcion: decimal_sum(X) +** +** Works like sum() except that it uses decimal arithmetic for unlimited +** precision. +*/ +static void decimalSumStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( !p->isInit ){ + p->isInit = 1; + p->a = sqlite3_malloc(2); + if( p->a==0 ){ + p->oom = 1; + }else{ + p->a[0] = 0; + } + p->nDigit = 1; + p->nFrac = 0; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *p; + Decimal *pArg; + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + if( p==0 ) return; + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pArg = decimal_new(context, argv[0], 1); + if( pArg ) pArg->sign = !pArg->sign; + decimal_add(p, pArg); + decimal_free(pArg); +} +static void decimalSumValue(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); +} +static void decimalSumFinalize(sqlite3_context *context){ + Decimal *p = sqlite3_aggregate_context(context, 0); + if( p==0 ) return; + decimal_result(context, p); + decimal_clear(p); +} + +/* +** SQL Function: decimal_mul(X, Y) +** +** Return the product of X and Y. +*/ +static void decimalMulFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + Decimal *pA = decimal_new(context, argv[0], 1); + Decimal *pB = decimal_new(context, argv[1], 1); + UNUSED_PARAMETER(argc); + if( pA==0 || pA->oom || pA->isNull + || pB==0 || pB->oom || pB->isNull + ){ + goto mul_end; + } + decimalMul(pA, pB); + if( pA->oom ){ + goto mul_end; + } + decimal_result(context, pA); + +mul_end: + decimal_free(pA); + decimal_free(pB); +} + +/* +** SQL Function: decimal_pow2(N) +** +** Return the N-th power of 2. N must be an integer. +*/ +static void decimalPow2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_INTEGER ){ + Decimal *pA = decimalPow2(sqlite3_value_int(argv[0])); + decimal_result_sci(context, pA); + decimal_free(pA); + } +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_decimal_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + static const struct { + const char *zFuncName; + int nArg; + int iArg; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "decimal", 1, 0, decimalFunc }, + { "decimal_exp", 1, 1, decimalFunc }, + { "decimal_cmp", 2, 0, decimalCmpFunc }, + { "decimal_add", 2, 0, decimalAddFunc }, + { "decimal_sub", 2, 0, decimalSubFunc }, + { "decimal_mul", 2, 0, decimalMulFunc }, + { "decimal_pow2", 1, 0, decimalPow2Func }, + }; + unsigned int i; + (void)pzErrMsg; /* Unused parameter */ + + SQLITE_EXTENSION_INIT2(pApi); + + for(i=0; i<(int)(sizeof(aFunc)/sizeof(aFunc[0])) && rc==SQLITE_OK; i++){ + rc = sqlite3_create_function(db, aFunc[i].zFuncName, aFunc[i].nArg, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, + aFunc[i].iArg ? db : 0, aFunc[i].xFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_window_function(db, "decimal_sum", 1, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_DETERMINISTIC, 0, + decimalSumStep, decimalSumFinalize, + decimalSumValue, decimalSumInverse, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_collation(db, "decimal", SQLITE_UTF8, + 0, decimalCollFunc); + } + return rc; +} diff --git a/ext/misc/eval.c b/ext/misc/eval.c new file mode 100644 index 0000000..d3849d6 --- /dev/null +++ b/ext/misc/eval.c @@ -0,0 +1,125 @@ +/* +** 2014-11-10 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL function eval() which runs +** SQL statements recursively. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** Structure used to accumulate the output +*/ +struct EvalResult { + char *z; /* Accumulated output */ + const char *zSep; /* Separator */ + int szSep; /* Size of the separator string */ + sqlite3_int64 nAlloc; /* Number of bytes allocated for z[] */ + sqlite3_int64 nUsed; /* Number of bytes of z[] actually used */ +}; + +/* +** Callback from sqlite_exec() for the eval() function. +*/ +static int callback(void *pCtx, int argc, char **argv, char **colnames){ + struct EvalResult *p = (struct EvalResult*)pCtx; + int i; + if( argv==0 ) return 0; + for(i=0; inUsed+p->szSep+1 > p->nAlloc ){ + char *zNew; + p->nAlloc = p->nAlloc*2 + sz + p->szSep + 1; + /* Using sqlite3_realloc64() would be better, but it is a recent + ** addition and will cause a segfault if loaded by an older version + ** of SQLite. */ + zNew = p->nAlloc<=0x7fffffff ? sqlite3_realloc64(p->z, p->nAlloc) : 0; + if( zNew==0 ){ + sqlite3_free(p->z); + memset(p, 0, sizeof(*p)); + return 1; + } + p->z = zNew; + } + if( p->nUsed>0 ){ + memcpy(&p->z[p->nUsed], p->zSep, p->szSep); + p->nUsed += p->szSep; + } + memcpy(&p->z[p->nUsed], z, sz); + p->nUsed += sz; + } + return 0; +} + +/* +** Implementation of the eval(X) and eval(X,Y) SQL functions. +** +** Evaluate the SQL text in X. Return the results, using string +** Y as the separator. If Y is omitted, use a single space character. +*/ +static void sqlEvalFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zSql; + sqlite3 *db; + char *zErr = 0; + int rc; + struct EvalResult x; + + memset(&x, 0, sizeof(x)); + x.zSep = " "; + zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ) return; + if( argc>1 ){ + x.zSep = (const char*)sqlite3_value_text(argv[1]); + if( x.zSep==0 ) return; + } + x.szSep = (int)strlen(x.zSep); + db = sqlite3_context_db_handle(context); + rc = sqlite3_exec(db, zSql, callback, &x, &zErr); + if( rc!=SQLITE_OK ){ + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + }else if( x.zSep==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(x.z); + }else{ + sqlite3_result_text(context, x.z, (int)x.nUsed, sqlite3_free); + } +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_eval_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "eval", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + sqlEvalFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "eval", 2, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + sqlEvalFunc, 0, 0); + } + return rc; +} diff --git a/ext/misc/explain.c b/ext/misc/explain.c new file mode 100644 index 0000000..726af76 --- /dev/null +++ b/ext/misc/explain.c @@ -0,0 +1,323 @@ +/* +** 2018-09-16 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates an eponymous virtual table that returns the +** EXPLAIN output from an SQL statement. +** +** Usage example: +** +** .load ./explain +** SELECT p2 FROM explain('SELECT * FROM sqlite_schema') +** WHERE opcode='OpenRead'; +** +** This module was originally written to help simplify SQLite testing, +** by providing an easier means of verifying certain patterns in the +** generated bytecode. +*/ +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* explain_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a explain virtual table +*/ +typedef struct explain_vtab explain_vtab; +struct explain_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this explain vtab */ +}; + +/* explain_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result from an EXPLAIN operation. +*/ +typedef struct explain_cursor explain_cursor; +struct explain_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + char *zSql; /* Value for the EXPLN_COLUMN_SQL column */ + sqlite3_stmt *pExplain; /* Statement being explained */ + int rc; /* Result of last sqlite3_step() on pExplain */ +}; + +/* +** The explainConnect() method is invoked to create a new +** explain_vtab that describes the explain virtual table. +** +** Think of this routine as the constructor for explain_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the explain_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against explain will look like. +*/ +static int explainConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + explain_vtab *pNew; + int rc; + +/* Column numbers */ +#define EXPLN_COLUMN_ADDR 0 /* Instruction address */ +#define EXPLN_COLUMN_OPCODE 1 /* Opcode */ +#define EXPLN_COLUMN_P1 2 /* Operand 1 */ +#define EXPLN_COLUMN_P2 3 /* Operand 2 */ +#define EXPLN_COLUMN_P3 4 /* Operand 3 */ +#define EXPLN_COLUMN_P4 5 /* Operand 4 */ +#define EXPLN_COLUMN_P5 6 /* Operand 5 */ +#define EXPLN_COLUMN_COMMENT 7 /* Comment */ +#define EXPLN_COLUMN_SQL 8 /* SQL that is being explained */ + + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(addr,opcode,p1,p2,p3,p4,p5,comment,sql HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for explain_cursor objects. +*/ +static int explainDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new explain_cursor object. +*/ +static int explainOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + explain_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((explain_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a explain_cursor. +*/ +static int explainClose(sqlite3_vtab_cursor *cur){ + explain_cursor *pCur = (explain_cursor*)cur; + sqlite3_finalize(pCur->pExplain); + sqlite3_free(pCur->zSql); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a explain_cursor to its next row of output. +*/ +static int explainNext(sqlite3_vtab_cursor *cur){ + explain_cursor *pCur = (explain_cursor*)cur; + pCur->rc = sqlite3_step(pCur->pExplain); + if( pCur->rc!=SQLITE_DONE && pCur->rc!=SQLITE_ROW ) return pCur->rc; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the explain_cursor +** is currently pointing. +*/ +static int explainColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + explain_cursor *pCur = (explain_cursor*)cur; + if( i==EXPLN_COLUMN_SQL ){ + sqlite3_result_text(ctx, pCur->zSql, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_value(ctx, sqlite3_column_value(pCur->pExplain, i)); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int explainRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + explain_cursor *pCur = (explain_cursor*)cur; + *pRowid = sqlite3_column_int64(pCur->pExplain, 0); + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int explainEof(sqlite3_vtab_cursor *cur){ + explain_cursor *pCur = (explain_cursor*)cur; + return pCur->rc!=SQLITE_ROW; +} + +/* +** This method is called to "rewind" the explain_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to explainColumn() or explainRowid() or +** explainEof(). +** +** The argv[0] is the SQL statement that is to be explained. +*/ +static int explainFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + explain_cursor *pCur = (explain_cursor *)pVtabCursor; + char *zSql = 0; + int rc; + sqlite3_finalize(pCur->pExplain); + pCur->pExplain = 0; + if( sqlite3_value_type(argv[0])!=SQLITE_TEXT ){ + pCur->rc = SQLITE_DONE; + return SQLITE_OK; + } + sqlite3_free(pCur->zSql); + pCur->zSql = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + if( pCur->zSql ){ + zSql = sqlite3_mprintf("EXPLAIN %s", pCur->zSql); + } + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pCur->db, zSql, -1, &pCur->pExplain, 0); + sqlite3_free(zSql); + } + if( rc ){ + sqlite3_finalize(pCur->pExplain); + pCur->pExplain = 0; + sqlite3_free(pCur->zSql); + pCur->zSql = 0; + }else{ + pCur->rc = sqlite3_step(pCur->pExplain); + rc = (pCur->rc==SQLITE_DONE || pCur->rc==SQLITE_ROW) ? SQLITE_OK : pCur->rc; + } + return rc; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the explain virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int explainBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop counter */ + int idx = -1; /* Index of a usable == constraint against SQL */ + int unusable = 0; /* True if there are unusable constraints on SQL */ + + pIdxInfo->estimatedRows = 500; + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->iColumn!=EXPLN_COLUMN_SQL ) continue; + if( !p->usable ){ + unusable = 1; + }else if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + idx = i; + } + } + if( idx>=0 ){ + /* There exists a usable == constraint against the SQL column */ + pIdxInfo->estimatedCost = 10.0; + pIdxInfo->idxNum = 1; + pIdxInfo->aConstraintUsage[idx].argvIndex = 1; + pIdxInfo->aConstraintUsage[idx].omit = 1; + }else if( unusable ){ + /* There are unusable constraints against the SQL column. Do not allow + ** this plan to continue forward. */ + return SQLITE_CONSTRAINT; + } + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** explain virtual table. +*/ +static sqlite3_module explainModule = { + 0, /* iVersion */ + 0, /* xCreate */ + explainConnect, /* xConnect */ + explainBestIndex, /* xBestIndex */ + explainDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + explainOpen, /* xOpen - open a cursor */ + explainClose, /* xClose - close a cursor */ + explainFilter, /* xFilter - configure scan constraints */ + explainNext, /* xNext - advance a cursor */ + explainEof, /* xEof - check for end of scan */ + explainColumn, /* xColumn - read data */ + explainRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3ExplainVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "explain", &explainModule, 0); +#endif + return rc; +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_explain_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3ExplainVtabInit(db); +#endif + return rc; +} diff --git a/ext/misc/fileio.c b/ext/misc/fileio.c new file mode 100644 index 0000000..70546ad --- /dev/null +++ b/ext/misc/fileio.c @@ -0,0 +1,1031 @@ +/* +** 2014-06-13 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements SQL functions readfile() and +** writefile(), and eponymous virtual type "fsdir". +** +** WRITEFILE(FILE, DATA [, MODE [, MTIME]]): +** +** If neither of the optional arguments is present, then this UDF +** function writes blob DATA to file FILE. If successful, the number +** of bytes written is returned. If an error occurs, NULL is returned. +** +** If the first option argument - MODE - is present, then it must +** be passed an integer value that corresponds to a POSIX mode +** value (file type + permissions, as returned in the stat.st_mode +** field by the stat() system call). Three types of files may +** be written/created: +** +** regular files: (mode & 0170000)==0100000 +** symbolic links: (mode & 0170000)==0120000 +** directories: (mode & 0170000)==0040000 +** +** For a directory, the DATA is ignored. For a symbolic link, it is +** interpreted as text and used as the target of the link. For a +** regular file, it is interpreted as a blob and written into the +** named file. Regardless of the type of file, its permissions are +** set to (mode & 0777) before returning. +** +** If the optional MTIME argument is present, then it is interpreted +** as an integer - the number of seconds since the unix epoch. The +** modification-time of the target file is set to this value before +** returning. +** +** If three or more arguments are passed to this function and an +** error is encountered, an exception is raised. +** +** READFILE(FILE): +** +** Read and return the contents of file FILE (type blob) from disk. +** +** FSDIR: +** +** Used as follows: +** +** SELECT * FROM fsdir($path [, $dir]); +** +** Parameter $path is an absolute or relative pathname. If the file that it +** refers to does not exist, it is an error. If the path refers to a regular +** file or symbolic link, it returns a single row. Or, if the path refers +** to a directory, it returns one row for the directory, and one row for each +** file within the hierarchy rooted at $path. +** +** Each row has the following columns: +** +** name: Path to file or directory (text value). +** mode: Value of stat.st_mode for directory entry (an integer). +** mtime: Value of stat.st_mtime for directory entry (an integer). +** data: For a regular file, a blob containing the file data. For a +** symlink, a text value containing the text of the link. For a +** directory, NULL. +** +** If a non-NULL value is specified for the optional $dir parameter and +** $path is a relative path, then $path is interpreted relative to $dir. +** And the paths returned in the "name" column of the table are also +** relative to directory $dir. +** +** Notes on building this extension for Windows: +** Unless linked statically with the SQLite library, a preprocessor +** symbol, FILEIO_WIN32_DLL, must be #define'd to create a stand-alone +** DLL form of this extension for WIN32. See its use below for details. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +#include +#include +#include +#if !defined(_WIN32) && !defined(WIN32) +# include +# include +# include +# include +#else +# include "windows.h" +# include +# include +# include "test_windirent.h" +# define dirent DIRENT +# ifndef chmod +# define chmod _chmod +# endif +# ifndef stat +# define stat _stat +# endif +# define mkdir(path,mode) _mkdir(path) +# define lstat(path,buf) stat(path,buf) +#endif +#include +#include + + +/* +** Structure of the fsdir() table-valued function +*/ + /* 0 1 2 3 4 5 */ +#define FSDIR_SCHEMA "(name,mode,mtime,data,path HIDDEN,dir HIDDEN)" +#define FSDIR_COLUMN_NAME 0 /* Name of the file */ +#define FSDIR_COLUMN_MODE 1 /* Access mode */ +#define FSDIR_COLUMN_MTIME 2 /* Last modification time */ +#define FSDIR_COLUMN_DATA 3 /* File content */ +#define FSDIR_COLUMN_PATH 4 /* Path to top of search */ +#define FSDIR_COLUMN_DIR 5 /* Path is relative to this directory */ + + +/* +** Set the result stored by context ctx to a blob containing the +** contents of file zName. Or, leave the result unchanged (NULL) +** if the file does not exist or is unreadable. +** +** If the file exceeds the SQLite blob size limit, through an +** SQLITE_TOOBIG error. +** +** Throw an SQLITE_IOERR if there are difficulties pulling the file +** off of disk. +*/ +static void readFileContents(sqlite3_context *ctx, const char *zName){ + FILE *in; + sqlite3_int64 nIn; + void *pBuf; + sqlite3 *db; + int mxBlob; + + in = fopen(zName, "rb"); + if( in==0 ){ + /* File does not exist or is unreadable. Leave the result set to NULL. */ + return; + } + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + db = sqlite3_context_db_handle(ctx); + mxBlob = sqlite3_limit(db, SQLITE_LIMIT_LENGTH, -1); + if( nIn>mxBlob ){ + sqlite3_result_error_code(ctx, SQLITE_TOOBIG); + fclose(in); + return; + } + pBuf = sqlite3_malloc64( nIn ? nIn : 1 ); + if( pBuf==0 ){ + sqlite3_result_error_nomem(ctx); + fclose(in); + return; + } + if( nIn==(sqlite3_int64)fread(pBuf, 1, (size_t)nIn, in) ){ + sqlite3_result_blob64(ctx, pBuf, nIn, sqlite3_free); + }else{ + sqlite3_result_error_code(ctx, SQLITE_IOERR); + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + (void)(argc); /* Unused parameter */ + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + readFileContents(context, zName); +} + +/* +** Set the error message contained in context ctx to the results of +** vprintf(zFmt, ...). +*/ +static void ctxErrorMsg(sqlite3_context *ctx, const char *zFmt, ...){ + char *zMsg = 0; + va_list ap; + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + va_end(ap); +} + +#if defined(_WIN32) +/* +** This function is designed to convert a Win32 FILETIME structure into the +** number of seconds since the Unix Epoch (1970-01-01 00:00:00 UTC). +*/ +static sqlite3_uint64 fileTimeToUnixTime( + LPFILETIME pFileTime +){ + SYSTEMTIME epochSystemTime; + ULARGE_INTEGER epochIntervals; + FILETIME epochFileTime; + ULARGE_INTEGER fileIntervals; + + memset(&epochSystemTime, 0, sizeof(SYSTEMTIME)); + epochSystemTime.wYear = 1970; + epochSystemTime.wMonth = 1; + epochSystemTime.wDay = 1; + SystemTimeToFileTime(&epochSystemTime, &epochFileTime); + epochIntervals.LowPart = epochFileTime.dwLowDateTime; + epochIntervals.HighPart = epochFileTime.dwHighDateTime; + + fileIntervals.LowPart = pFileTime->dwLowDateTime; + fileIntervals.HighPart = pFileTime->dwHighDateTime; + + return (fileIntervals.QuadPart - epochIntervals.QuadPart) / 10000000; +} + + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +# /* To allow a standalone DLL, use this next replacement function: */ +# undef sqlite3_win32_utf8_to_unicode +# define sqlite3_win32_utf8_to_unicode utf8_to_utf16 +# +LPWSTR utf8_to_utf16(const char *z){ + int nAllot = MultiByteToWideChar(CP_UTF8, 0, z, -1, NULL, 0); + LPWSTR rv = sqlite3_malloc(nAllot * sizeof(WCHAR)); + if( rv!=0 && 0 < MultiByteToWideChar(CP_UTF8, 0, z, -1, rv, nAllot) ) + return rv; + sqlite3_free(rv); + return 0; +} +#endif + +/* +** This function attempts to normalize the time values found in the stat() +** buffer to UTC. This is necessary on Win32, where the runtime library +** appears to return these values as local times. +*/ +static void statTimesToUtc( + const char *zPath, + struct stat *pStatBuf +){ + HANDLE hFindFile; + WIN32_FIND_DATAW fd; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + zUnicodeName = sqlite3_win32_utf8_to_unicode(zPath); + if( zUnicodeName ){ + memset(&fd, 0, sizeof(WIN32_FIND_DATAW)); + hFindFile = FindFirstFileW(zUnicodeName, &fd); + if( hFindFile!=NULL ){ + pStatBuf->st_ctime = (time_t)fileTimeToUnixTime(&fd.ftCreationTime); + pStatBuf->st_atime = (time_t)fileTimeToUnixTime(&fd.ftLastAccessTime); + pStatBuf->st_mtime = (time_t)fileTimeToUnixTime(&fd.ftLastWriteTime); + FindClose(hFindFile); + } + sqlite3_free(zUnicodeName); + } +} +#endif + +/* +** This function is used in place of stat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls stat(). +*/ +static int fileStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = stat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return stat(zPath, pStatBuf); +#endif +} + +/* +** This function is used in place of lstat(). On Windows, special handling +** is required in order for the included time to be returned as UTC. On all +** other systems, this function simply calls lstat(). +*/ +static int fileLinkStat( + const char *zPath, + struct stat *pStatBuf +){ +#if defined(_WIN32) + int rc = lstat(zPath, pStatBuf); + if( rc==0 ) statTimesToUtc(zPath, pStatBuf); + return rc; +#else + return lstat(zPath, pStatBuf); +#endif +} + +/* +** Argument zFile is the name of a file that will be created and/or written +** by SQL function writefile(). This function ensures that the directory +** zFile will be written to exists, creating it if required. The permissions +** for any path components created by this function are set in accordance +** with the current umask. +** +** If an OOM condition is encountered, SQLITE_NOMEM is returned. Otherwise, +** SQLITE_OK is returned if the directory is successfully created, or +** SQLITE_ERROR otherwise. +*/ +static int makeDirectory( + const char *zFile +){ + char *zCopy = sqlite3_mprintf("%s", zFile); + int rc = SQLITE_OK; + + if( zCopy==0 ){ + rc = SQLITE_NOMEM; + }else{ + int nCopy = (int)strlen(zCopy); + int i = 1; + + while( rc==SQLITE_OK ){ + struct stat sStat; + int rc2; + + for(; zCopy[i]!='/' && i=0 ){ +#if defined(_WIN32) +#if !SQLITE_OS_WINRT + /* Windows */ + FILETIME lastAccess; + FILETIME lastWrite; + SYSTEMTIME currentTime; + LONGLONG intervals; + HANDLE hFile; + LPWSTR zUnicodeName; + extern LPWSTR sqlite3_win32_utf8_to_unicode(const char*); + + GetSystemTime(¤tTime); + SystemTimeToFileTime(¤tTime, &lastAccess); + intervals = Int32x32To64(mtime, 10000000) + 116444736000000000; + lastWrite.dwLowDateTime = (DWORD)intervals; + lastWrite.dwHighDateTime = intervals >> 32; + zUnicodeName = sqlite3_win32_utf8_to_unicode(zFile); + if( zUnicodeName==0 ){ + return 1; + } + hFile = CreateFileW( + zUnicodeName, FILE_WRITE_ATTRIBUTES, 0, NULL, OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, NULL + ); + sqlite3_free(zUnicodeName); + if( hFile!=INVALID_HANDLE_VALUE ){ + BOOL bResult = SetFileTime(hFile, NULL, &lastAccess, &lastWrite); + CloseHandle(hFile); + return !bResult; + }else{ + return 1; + } +#endif +#elif defined(AT_FDCWD) && 0 /* utimensat() is not universally available */ + /* Recent unix */ + struct timespec times[2]; + times[0].tv_nsec = times[1].tv_nsec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimensat(AT_FDCWD, zFile, times, AT_SYMLINK_NOFOLLOW) ){ + return 1; + } +#else + /* Legacy unix */ + struct timeval times[2]; + times[0].tv_usec = times[1].tv_usec = 0; + times[0].tv_sec = time(0); + times[1].tv_sec = mtime; + if( utimes(zFile, times) ){ + return 1; + } +#endif + } + + return 0; +} + +/* +** Implementation of the "writefile(W,X[,Y[,Z]]])" SQL function. +** Refer to header comments at the top of this file for details. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zFile; + mode_t mode = 0; + int res; + sqlite3_int64 mtime = -1; + + if( argc<2 || argc>4 ){ + sqlite3_result_error(context, + "wrong number of arguments to function writefile()", -1 + ); + return; + } + + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + if( argc>=3 ){ + mode = (mode_t)sqlite3_value_int(argv[2]); + } + if( argc==4 ){ + mtime = sqlite3_value_int64(argv[3]); + } + + res = writeFile(context, zFile, argv[1], mode, mtime); + if( res==1 && errno==ENOENT ){ + if( makeDirectory(zFile)==SQLITE_OK ){ + res = writeFile(context, zFile, argv[1], mode, mtime); + } + } + + if( argc>2 && res!=0 ){ + if( S_ISLNK(mode) ){ + ctxErrorMsg(context, "failed to create symlink: %s", zFile); + }else if( S_ISDIR(mode) ){ + ctxErrorMsg(context, "failed to create directory: %s", zFile); + }else{ + ctxErrorMsg(context, "failed to write file: %s", zFile); + } + } +} + +/* +** SQL function: lsmode(MODE) +** +** Given a numberic st_mode from stat(), convert it into a human-readable +** text string in the style of "ls -l". +*/ +static void lsModeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int i; + int iMode = sqlite3_value_int(argv[0]); + char z[16]; + (void)argc; + if( S_ISLNK(iMode) ){ + z[0] = 'l'; + }else if( S_ISREG(iMode) ){ + z[0] = '-'; + }else if( S_ISDIR(iMode) ){ + z[0] = 'd'; + }else{ + z[0] = '?'; + } + for(i=0; i<3; i++){ + int m = (iMode >> ((2-i)*3)); + char *a = &z[1 + i*3]; + a[0] = (m & 0x4) ? 'r' : '-'; + a[1] = (m & 0x2) ? 'w' : '-'; + a[2] = (m & 0x1) ? 'x' : '-'; + } + z[10] = '\0'; + sqlite3_result_text(context, z, -1, SQLITE_TRANSIENT); +} + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Cursor type for recursively iterating through a directory structure. +*/ +typedef struct fsdir_cursor fsdir_cursor; +typedef struct FsdirLevel FsdirLevel; + +struct FsdirLevel { + DIR *pDir; /* From opendir() */ + char *zDir; /* Name of directory (nul-terminated) */ +}; + +struct fsdir_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + + int nLvl; /* Number of entries in aLvl[] array */ + int iLvl; /* Index of current entry */ + FsdirLevel *aLvl; /* Hierarchy of directories being traversed */ + + const char *zBase; + int nBase; + + struct stat sStat; /* Current lstat() results */ + char *zPath; /* Path to current entry */ + sqlite3_int64 iRowid; /* Current rowid */ +}; + +typedef struct fsdir_tab fsdir_tab; +struct fsdir_tab { + sqlite3_vtab base; /* Base class - must be first */ +}; + +/* +** Construct a new fsdir virtual table object. +*/ +static int fsdirConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + fsdir_tab *pNew = 0; + int rc; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; + rc = sqlite3_declare_vtab(db, "CREATE TABLE x" FSDIR_SCHEMA); + if( rc==SQLITE_OK ){ + pNew = (fsdir_tab*)sqlite3_malloc( sizeof(*pNew) ); + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + } + *ppVtab = (sqlite3_vtab*)pNew; + return rc; +} + +/* +** This method is the destructor for fsdir vtab objects. +*/ +static int fsdirDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new fsdir_cursor object. +*/ +static int fsdirOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + fsdir_cursor *pCur; + (void)p; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->iLvl = -1; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Reset a cursor back to the state it was in when first returned +** by fsdirOpen(). +*/ +static void fsdirResetCursor(fsdir_cursor *pCur){ + int i; + for(i=0; i<=pCur->iLvl; i++){ + FsdirLevel *pLvl = &pCur->aLvl[i]; + if( pLvl->pDir ) closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + } + sqlite3_free(pCur->zPath); + sqlite3_free(pCur->aLvl); + pCur->aLvl = 0; + pCur->zPath = 0; + pCur->zBase = 0; + pCur->nBase = 0; + pCur->nLvl = 0; + pCur->iLvl = -1; + pCur->iRowid = 1; +} + +/* +** Destructor for an fsdir_cursor. +*/ +static int fsdirClose(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + + fsdirResetCursor(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Set the error message for the virtual table associated with cursor +** pCur to the results of vprintf(zFmt, ...). +*/ +static void fsdirSetErrmsg(fsdir_cursor *pCur, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + pCur->base.pVtab->zErrMsg = sqlite3_vmprintf(zFmt, ap); + va_end(ap); +} + + +/* +** Advance an fsdir_cursor to its next row of output. +*/ +static int fsdirNext(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + mode_t m = pCur->sStat.st_mode; + + pCur->iRowid++; + if( S_ISDIR(m) ){ + /* Descend into this directory */ + int iNew = pCur->iLvl + 1; + FsdirLevel *pLvl; + if( iNew>=pCur->nLvl ){ + int nNew = iNew+1; + sqlite3_int64 nByte = nNew*sizeof(FsdirLevel); + FsdirLevel *aNew = (FsdirLevel*)sqlite3_realloc64(pCur->aLvl, nByte); + if( aNew==0 ) return SQLITE_NOMEM; + memset(&aNew[pCur->nLvl], 0, sizeof(FsdirLevel)*(nNew-pCur->nLvl)); + pCur->aLvl = aNew; + pCur->nLvl = nNew; + } + pCur->iLvl = iNew; + pLvl = &pCur->aLvl[iNew]; + + pLvl->zDir = pCur->zPath; + pCur->zPath = 0; + pLvl->pDir = opendir(pLvl->zDir); + if( pLvl->pDir==0 ){ + fsdirSetErrmsg(pCur, "cannot read directory: %s", pCur->zPath); + return SQLITE_ERROR; + } + } + + while( pCur->iLvl>=0 ){ + FsdirLevel *pLvl = &pCur->aLvl[pCur->iLvl]; + struct dirent *pEntry = readdir(pLvl->pDir); + if( pEntry ){ + if( pEntry->d_name[0]=='.' ){ + if( pEntry->d_name[1]=='.' && pEntry->d_name[2]=='\0' ) continue; + if( pEntry->d_name[1]=='\0' ) continue; + } + sqlite3_free(pCur->zPath); + pCur->zPath = sqlite3_mprintf("%s/%s", pLvl->zDir, pEntry->d_name); + if( pCur->zPath==0 ) return SQLITE_NOMEM; + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + return SQLITE_OK; + } + closedir(pLvl->pDir); + sqlite3_free(pLvl->zDir); + pLvl->pDir = 0; + pLvl->zDir = 0; + pCur->iLvl--; + } + + /* EOF */ + sqlite3_free(pCur->zPath); + pCur->zPath = 0; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the series_cursor +** is currently pointing. +*/ +static int fsdirColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + switch( i ){ + case FSDIR_COLUMN_NAME: { + sqlite3_result_text(ctx, &pCur->zPath[pCur->nBase], -1, SQLITE_TRANSIENT); + break; + } + + case FSDIR_COLUMN_MODE: + sqlite3_result_int64(ctx, pCur->sStat.st_mode); + break; + + case FSDIR_COLUMN_MTIME: + sqlite3_result_int64(ctx, pCur->sStat.st_mtime); + break; + + case FSDIR_COLUMN_DATA: { + mode_t m = pCur->sStat.st_mode; + if( S_ISDIR(m) ){ + sqlite3_result_null(ctx); +#if !defined(_WIN32) && !defined(WIN32) + }else if( S_ISLNK(m) ){ + char aStatic[64]; + char *aBuf = aStatic; + sqlite3_int64 nBuf = 64; + int n; + + while( 1 ){ + n = readlink(pCur->zPath, aBuf, nBuf); + if( nzPath); + } + } + case FSDIR_COLUMN_PATH: + default: { + /* The FSDIR_COLUMN_PATH and FSDIR_COLUMN_DIR are input parameters. + ** always return their values as NULL */ + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** first row returned is assigned rowid value 1, and each subsequent +** row a value 1 more than that of the previous. +*/ +static int fsdirRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fsdirEof(sqlite3_vtab_cursor *cur){ + fsdir_cursor *pCur = (fsdir_cursor*)cur; + return (pCur->zPath==0); +} + +/* +** xFilter callback. +** +** idxNum==1 PATH parameter only +** idxNum==2 Both PATH and DIR supplied +*/ +static int fsdirFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + const char *zDir = 0; + fsdir_cursor *pCur = (fsdir_cursor*)cur; + (void)idxStr; + fsdirResetCursor(pCur); + + if( idxNum==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires an argument"); + return SQLITE_ERROR; + } + + assert( argc==idxNum && (argc==1 || argc==2) ); + zDir = (const char*)sqlite3_value_text(argv[0]); + if( zDir==0 ){ + fsdirSetErrmsg(pCur, "table function fsdir requires a non-NULL argument"); + return SQLITE_ERROR; + } + if( argc==2 ){ + pCur->zBase = (const char*)sqlite3_value_text(argv[1]); + } + if( pCur->zBase ){ + pCur->nBase = (int)strlen(pCur->zBase)+1; + pCur->zPath = sqlite3_mprintf("%s/%s", pCur->zBase, zDir); + }else{ + pCur->zPath = sqlite3_mprintf("%s", zDir); + } + + if( pCur->zPath==0 ){ + return SQLITE_NOMEM; + } + if( fileLinkStat(pCur->zPath, &pCur->sStat) ){ + fsdirSetErrmsg(pCur, "cannot stat file: %s", pCur->zPath); + return SQLITE_ERROR; + } + + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the generate_series virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +** +** In this implementation idxNum is used to represent the +** query plan. idxStr is unused. +** +** The query plan is represented by values of idxNum: +** +** (1) The path value is supplied by argv[0] +** (2) Path is in argv[0] and dir is in argv[1] +*/ +static int fsdirBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop over constraints */ + int idxPath = -1; /* Index in pIdxInfo->aConstraint of PATH= */ + int idxDir = -1; /* Index in pIdxInfo->aConstraint of DIR= */ + int seenPath = 0; /* True if an unusable PATH= constraint is seen */ + int seenDir = 0; /* True if an unusable DIR= constraint is seen */ + const struct sqlite3_index_constraint *pConstraint; + + (void)tab; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + switch( pConstraint->iColumn ){ + case FSDIR_COLUMN_PATH: { + if( pConstraint->usable ){ + idxPath = i; + seenPath = 0; + }else if( idxPath<0 ){ + seenPath = 1; + } + break; + } + case FSDIR_COLUMN_DIR: { + if( pConstraint->usable ){ + idxDir = i; + seenDir = 0; + }else if( idxDir<0 ){ + seenDir = 1; + } + break; + } + } + } + if( seenPath || seenDir ){ + /* If input parameters are unusable, disallow this plan */ + return SQLITE_CONSTRAINT; + } + + if( idxPath<0 ){ + pIdxInfo->idxNum = 0; + /* The pIdxInfo->estimatedCost should have been initialized to a huge + ** number. Leave it unchanged. */ + pIdxInfo->estimatedRows = 0x7fffffff; + }else{ + pIdxInfo->aConstraintUsage[idxPath].omit = 1; + pIdxInfo->aConstraintUsage[idxPath].argvIndex = 1; + if( idxDir>=0 ){ + pIdxInfo->aConstraintUsage[idxDir].omit = 1; + pIdxInfo->aConstraintUsage[idxDir].argvIndex = 2; + pIdxInfo->idxNum = 2; + pIdxInfo->estimatedCost = 10.0; + }else{ + pIdxInfo->idxNum = 1; + pIdxInfo->estimatedCost = 100.0; + } + } + + return SQLITE_OK; +} + +/* +** Register the "fsdir" virtual table. +*/ +static int fsdirRegister(sqlite3 *db){ + static sqlite3_module fsdirModule = { + 0, /* iVersion */ + 0, /* xCreate */ + fsdirConnect, /* xConnect */ + fsdirBestIndex, /* xBestIndex */ + fsdirDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + fsdirOpen, /* xOpen - open a cursor */ + fsdirClose, /* xClose - close a cursor */ + fsdirFilter, /* xFilter - configure scan constraints */ + fsdirNext, /* xNext - advance a cursor */ + fsdirEof, /* xEof - check for end of scan */ + fsdirColumn, /* xColumn - read data */ + fsdirRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + + int rc = sqlite3_create_module(db, "fsdir", &fsdirModule, 0); + return rc; +} +#else /* SQLITE_OMIT_VIRTUALTABLE */ +# define fsdirRegister(x) SQLITE_OK +#endif + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fileio_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "readfile", 1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + readfileFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "writefile", -1, + SQLITE_UTF8|SQLITE_DIRECTONLY, 0, + writefileFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "lsmode", 1, SQLITE_UTF8, 0, + lsModeFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = fsdirRegister(db); + } + return rc; +} + +#if defined(FILEIO_WIN32_DLL) && (defined(_WIN32) || defined(WIN32)) +/* To allow a standalone DLL, make test_windirent.c use the same + * redefined SQLite API calls as the above extension code does. + * Just pull in this .c to accomplish this. As a beneficial side + * effect, this extension becomes a single translation unit. */ +# include "test_windirent.c" +#endif diff --git a/ext/misc/fossildelta.c b/ext/misc/fossildelta.c new file mode 100644 index 0000000..e638737 --- /dev/null +++ b/ext/misc/fossildelta.c @@ -0,0 +1,1093 @@ +/* +** 2019-02-19 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements the delta functions used by the RBU +** extension. Three scalar functions and one table-valued function are +** implemented here: +** +** delta_apply(X,D) -- apply delta D to file X and return the result +** delta_create(X,Y) -- compute and return a delta that carries X into Y +** delta_output_size(D) -- blob size in bytes output from applying delta D +** delta_parse(D) -- returns rows describing delta D +** +** The delta format is the Fossil delta format, described in a comment +** on the delete_create() function implementation below, and also at +** +** https://www.fossil-scm.org/fossil/doc/trunk/www/delta_format.wiki +** +** This delta format is used by the RBU extension, which is the main +** reason that these routines are included in the extension library. +** RBU does not use this extension directly. Rather, this extension is +** provided as a convenience to developers who want to analyze RBU files +** that contain deltas. +*/ +#include +#include +#include +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 + +#ifndef SQLITE_AMALGAMATION +/* +** The "u32" type must be an unsigned 32-bit integer. Adjust this +*/ +typedef unsigned int u32; + +/* +** Must be a 16-bit value +*/ +typedef short int s16; +typedef unsigned short int u16; + +#endif /* SQLITE_AMALGAMATION */ + + +/* +** The width of a hash window in bytes. The algorithm only works if this +** is a power of 2. +*/ +#define NHASH 16 + +/* +** The current state of the rolling hash. +** +** z[] holds the values that have been hashed. z[] is a circular buffer. +** z[i] is the first entry and z[(i+NHASH-1)%NHASH] is the last entry of +** the window. +** +** Hash.a is the sum of all elements of hash.z[]. Hash.b is a weighted +** sum. Hash.b is z[i]*NHASH + z[i+1]*(NHASH-1) + ... + z[i+NHASH-1]*1. +** (Each index for z[] should be module NHASH, of course. The %NHASH operator +** is omitted in the prior expression for brevity.) +*/ +typedef struct hash hash; +struct hash { + u16 a, b; /* Hash values */ + u16 i; /* Start of the hash window */ + char z[NHASH]; /* The values that have been hashed */ +}; + +/* +** Initialize the rolling hash using the first NHASH characters of z[] +*/ +static void hash_init(hash *pHash, const char *z){ + u16 a, b, i; + a = b = z[0]; + for(i=1; iz, z, NHASH); + pHash->a = a & 0xffff; + pHash->b = b & 0xffff; + pHash->i = 0; +} + +/* +** Advance the rolling hash by a single character "c" +*/ +static void hash_next(hash *pHash, int c){ + u16 old = pHash->z[pHash->i]; + pHash->z[pHash->i] = c; + pHash->i = (pHash->i+1)&(NHASH-1); + pHash->a = pHash->a - old + c; + pHash->b = pHash->b - NHASH*old + pHash->a; +} + +/* +** Return a 32-bit hash value +*/ +static u32 hash_32bit(hash *pHash){ + return (pHash->a & 0xffff) | (((u32)(pHash->b & 0xffff))<<16); +} + +/* +** Compute a hash on NHASH bytes. +** +** This routine is intended to be equivalent to: +** hash h; +** hash_init(&h, zInput); +** return hash_32bit(&h); +*/ +static u32 hash_once(const char *z){ + u16 a, b, i; + a = b = z[0]; + for(i=1; i0; i++, v>>=6){ + zBuf[i] = zDigits[v&0x3f]; + } + for(j=i-1; j>=0; j--){ + *(*pz)++ = zBuf[j]; + } +} + +/* +** Read bytes from *pz and convert them into a positive integer. When +** finished, leave *pz pointing to the first character past the end of +** the integer. The *pLen parameter holds the length of the string +** in *pz and is decremented once for each character in the integer. +*/ +static unsigned int deltaGetInt(const char **pz, int *pLen){ + static const signed char zValue[] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, + 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, -1, -1, -1, -1, 36, + -1, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, -1, -1, -1, 63, -1, + }; + unsigned int v = 0; + int c; + unsigned char *z = (unsigned char*)*pz; + unsigned char *zStart = z; + while( (c = zValue[0x7f&*(z++)])>=0 ){ + v = (v<<6) + c; + } + z--; + *pLen -= z - zStart; + *pz = (char*)z; + return v; +} + +/* +** Return the number digits in the base-64 representation of a positive integer +*/ +static int digit_count(int v){ + unsigned int i, x; + for(i=1, x=64; v>=x; i++, x <<= 6){} + return i; +} + +#ifdef __GNUC__ +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif + +/* +** Compute a 32-bit big-endian checksum on the N-byte buffer. If the +** buffer is not a multiple of 4 bytes length, compute the sum that would +** have occurred if the buffer was padded with zeros to the next multiple +** of four bytes. +*/ +static unsigned int checksum(const char *zIn, size_t N){ + static const int byteOrderTest = 1; + const unsigned char *z = (const unsigned char *)zIn; + const unsigned char *zEnd = (const unsigned char*)&zIn[N&~3]; + unsigned sum = 0; + assert( (z - (const unsigned char*)0)%4==0 ); /* Four-byte alignment */ + if( 0==*(char*)&byteOrderTest ){ + /* This is a big-endian machine */ + while( z=4003000 + while( z=1300 + while( z= 16){ + sum0 += ((unsigned)z[0] + z[4] + z[8] + z[12]); + sum1 += ((unsigned)z[1] + z[5] + z[9] + z[13]); + sum2 += ((unsigned)z[2] + z[6] + z[10]+ z[14]); + sum += ((unsigned)z[3] + z[7] + z[11]+ z[15]); + z += 16; + N -= 16; + } + while(N >= 4){ + sum0 += z[0]; + sum1 += z[1]; + sum2 += z[2]; + sum += z[3]; + z += 4; + N -= 4; + } + sum += (sum2 << 8) + (sum1 << 16) + (sum0 << 24); +#endif + } + switch(N&3){ + case 3: sum += (z[2] << 8); + case 2: sum += (z[1] << 16); + case 1: sum += (z[0] << 24); + default: ; + } + return sum; +} + +/* +** Create a new delta. +** +** The delta is written into a preallocated buffer, zDelta, which +** should be at least 60 bytes longer than the target file, zOut. +** The delta string will be NUL-terminated, but it might also contain +** embedded NUL characters if either the zSrc or zOut files are +** binary. This function returns the length of the delta string +** in bytes, excluding the final NUL terminator character. +** +** Output Format: +** +** The delta begins with a base64 number followed by a newline. This +** number is the number of bytes in the TARGET file. Thus, given a +** delta file z, a program can compute the size of the output file +** simply by reading the first line and decoding the base-64 number +** found there. The delta_output_size() routine does exactly this. +** +** After the initial size number, the delta consists of a series of +** literal text segments and commands to copy from the SOURCE file. +** A copy command looks like this: +** +** NNN@MMM, +** +** where NNN is the number of bytes to be copied and MMM is the offset +** into the source file of the first byte (both base-64). If NNN is 0 +** it means copy the rest of the input file. Literal text is like this: +** +** NNN:TTTTT +** +** where NNN is the number of bytes of text (base-64) and TTTTT is the text. +** +** The last term is of the form +** +** NNN; +** +** In this case, NNN is a 32-bit bigendian checksum of the output file +** that can be used to verify that the delta applied correctly. All +** numbers are in base-64. +** +** Pure text files generate a pure text delta. Binary files generate a +** delta that may contain some binary data. +** +** Algorithm: +** +** The encoder first builds a hash table to help it find matching +** patterns in the source file. 16-byte chunks of the source file +** sampled at evenly spaced intervals are used to populate the hash +** table. +** +** Next we begin scanning the target file using a sliding 16-byte +** window. The hash of the 16-byte window in the target is used to +** search for a matching section in the source file. When a match +** is found, a copy command is added to the delta. An effort is +** made to extend the matching section to regions that come before +** and after the 16-byte hash window. A copy command is only issued +** if the result would use less space that just quoting the text +** literally. Literal text is added to the delta for sections that +** do not match or which can not be encoded efficiently using copy +** commands. +*/ +static int delta_create( + const char *zSrc, /* The source or pattern file */ + unsigned int lenSrc, /* Length of the source file */ + const char *zOut, /* The target file */ + unsigned int lenOut, /* Length of the target file */ + char *zDelta /* Write the delta into this buffer */ +){ + int i, base; + char *zOrigDelta = zDelta; + hash h; + int nHash; /* Number of hash table entries */ + int *landmark; /* Primary hash table */ + int *collide; /* Collision chain */ + int lastRead = -1; /* Last byte of zSrc read by a COPY command */ + + /* Add the target file size to the beginning of the delta + */ + putInt(lenOut, &zDelta); + *(zDelta++) = '\n'; + + /* If the source file is very small, it means that we have no + ** chance of ever doing a copy command. Just output a single + ** literal segment for the entire target and exit. + */ + if( lenSrc<=NHASH ){ + putInt(lenOut, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, zOut, lenOut); + zDelta += lenOut; + putInt(checksum(zOut, lenOut), &zDelta); + *(zDelta++) = ';'; + return zDelta - zOrigDelta; + } + + /* Compute the hash table used to locate matching sections in the + ** source file. + */ + nHash = lenSrc/NHASH; + collide = sqlite3_malloc64( (sqlite3_int64)nHash*2*sizeof(int) ); + memset(collide, -1, nHash*2*sizeof(int)); + landmark = &collide[nHash]; + for(i=0; i=0 && (limit--)>0 ){ + /* + ** The hash window has identified a potential match against + ** landmark block iBlock. But we need to investigate further. + ** + ** Look for a region in zOut that matches zSrc. Anchor the search + ** at zSrc[iSrc] and zOut[base+i]. Do not include anything prior to + ** zOut[base] or after zOut[outLen] nor anything after zSrc[srcLen]. + ** + ** Set cnt equal to the length of the match and set ofst so that + ** zSrc[ofst] is the first element of the match. litsz is the number + ** of characters between zOut[base] and the beginning of the match. + ** sz will be the overhead (in bytes) needed to encode the copy + ** command. Only generate copy command if the overhead of the + ** copy command is less than the amount of literal text to be copied. + */ + int cnt, ofst, litsz; + int j, k, x, y; + int sz; + int limitX; + + /* Beginning at iSrc, match forwards as far as we can. j counts + ** the number of characters that match */ + iSrc = iBlock*NHASH; + y = base+i; + limitX = ( lenSrc-iSrc <= lenOut-y ) ? lenSrc : iSrc + lenOut - y; + for(x=iSrc; x=sz && cnt>bestCnt ){ + /* Remember this match only if it is the best so far and it + ** does not increase the file size */ + bestCnt = cnt; + bestOfst = iSrc-k; + bestLitsz = litsz; + } + + /* Check the next matching block */ + iBlock = collide[iBlock]; + } + + /* We have a copy command that does not cause the delta to be larger + ** than a literal insert. So add the copy command to the delta. + */ + if( bestCnt>0 ){ + if( bestLitsz>0 ){ + /* Add an insert command before the copy */ + putInt(bestLitsz,&zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], bestLitsz); + zDelta += bestLitsz; + base += bestLitsz; + } + base += bestCnt; + putInt(bestCnt, &zDelta); + *(zDelta++) = '@'; + putInt(bestOfst, &zDelta); + *(zDelta++) = ','; + if( bestOfst + bestCnt -1 > lastRead ){ + lastRead = bestOfst + bestCnt - 1; + } + bestCnt = 0; + break; + } + + /* If we reach this point, it means no match is found so far */ + if( base+i+NHASH>=lenOut ){ + /* We have reached the end of the file and have not found any + ** matches. Do an "insert" for everything that does not match */ + putInt(lenOut-base, &zDelta); + *(zDelta++) = ':'; + memcpy(zDelta, &zOut[base], lenOut-base); + zDelta += lenOut-base; + base = lenOut; + break; + } + + /* Advance the hash by one character. Keep looking for a match */ + hash_next(&h, zOut[base+i+NHASH]); + i++; + } + } + /* Output a final "insert" record to get all the text at the end of + ** the file that does not match anything in the source file. + */ + if( base0 ){ + unsigned int cnt, ofst; + cnt = deltaGetInt(&zDelta, &lenDelta); + switch( zDelta[0] ){ + case '@': { + zDelta++; lenDelta--; + ofst = deltaGetInt(&zDelta, &lenDelta); + if( lenDelta>0 && zDelta[0]!=',' ){ + /* ERROR: copy command not terminated by ',' */ + return -1; + } + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: copy exceeds output file size */ + return -1; + } + if( ofst+cnt > lenSrc ){ + /* ERROR: copy extends past end of input */ + return -1; + } + memcpy(zOut, &zSrc[ofst], cnt); + zOut += cnt; + break; + } + case ':': { + zDelta++; lenDelta--; + total += cnt; + if( total>limit ){ + /* ERROR: insert command gives an output larger than predicted */ + return -1; + } + if( cnt>lenDelta ){ + /* ERROR: insert count exceeds size of delta */ + return -1; + } + memcpy(zOut, zDelta, cnt); + zOut += cnt; + zDelta += cnt; + lenDelta -= cnt; + break; + } + case ';': { + zDelta++; lenDelta--; + zOut[0] = 0; +#ifdef FOSSIL_ENABLE_DELTA_CKSUM_TEST + if( cnt!=checksum(zOrigOut, total) ){ + /* ERROR: bad checksum */ + return -1; + } +#endif + if( total!=limit ){ + /* ERROR: generated size does not match predicted size */ + return -1; + } + return total; + } + default: { + /* ERROR: unknown delta operator */ + return -1; + } + } + } + /* ERROR: unterminated delta */ + return -1; +} + +/* +** SQL functions: delta_create(X,Y) +** +** Return a delta for carrying X into Y. +*/ +static void deltaCreateFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aOrig; int nOrig; /* old blob */ + const char *aNew; int nNew; /* new blob */ + char *aOut; int nOut; /* output delta */ + + assert( argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nNew = sqlite3_value_bytes(argv[1]); + aNew = (const char*)sqlite3_value_blob(argv[1]); + aOut = sqlite3_malloc64(nNew+70); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut = delta_create(aOrig, nOrig, aNew, nNew, aOut); + if( nOut<0 ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "cannot create fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + +/* +** SQL functions: delta_apply(X,D) +** +** Return the result of applying delta D to input X. +*/ +static void deltaApplyFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aOrig; int nOrig; /* The X input */ + const char *aDelta; int nDelta; /* The input delta (D) */ + char *aOut; int nOut, nOut2; /* The output */ + + assert( argc==2 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + if( sqlite3_value_type(argv[1])==SQLITE_NULL ) return; + nOrig = sqlite3_value_bytes(argv[0]); + aOrig = (const char*)sqlite3_value_blob(argv[0]); + nDelta = sqlite3_value_bytes(argv[1]); + aDelta = (const char*)sqlite3_value_blob(argv[1]); + + /* Figure out the size of the output */ + nOut = delta_output_size(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + } + aOut = sqlite3_malloc64((sqlite3_int64)nOut+1); + if( aOut==0 ){ + sqlite3_result_error_nomem(context); + }else{ + nOut2 = delta_apply(aOrig, nOrig, aDelta, nDelta, aOut); + if( nOut2!=nOut ){ + sqlite3_free(aOut); + sqlite3_result_error(context, "corrupt fossil delta", -1); + }else{ + sqlite3_result_blob(context, aOut, nOut, sqlite3_free); + } + } +} + + +/* +** SQL functions: delta_output_size(D) +** +** Return the size of the output that results from applying delta D. +*/ +static void deltaOutputSizeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *aDelta; int nDelta; /* The input delta (D) */ + int nOut; /* Size of output */ + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + nDelta = sqlite3_value_bytes(argv[0]); + aDelta = (const char*)sqlite3_value_blob(argv[0]); + + /* Figure out the size of the output */ + nOut = delta_output_size(aDelta, nDelta); + if( nOut<0 ){ + sqlite3_result_error(context, "corrupt fossil delta", -1); + return; + }else{ + sqlite3_result_int(context, nOut); + } +} + +/***************************************************************************** +** Table-valued SQL function: delta_parse(DELTA) +** +** Schema: +** +** CREATE TABLE delta_parse( +** op TEXT, +** a1 INT, +** a2 ANY, +** delta HIDDEN BLOB +** ); +** +** Given an input DELTA, this function parses the delta and returns +** rows for each entry in the delta. The op column has one of the +** values SIZE, COPY, INSERT, CHECKSUM, ERROR. +** +** Assuming no errors, the first row has op='SIZE'. a1 is the size of +** the output in bytes and a2 is NULL. +** +** After the initial SIZE row, there are zero or more 'COPY' and/or 'INSERT' +** rows. A COPY row means content is copied from the source into the +** output. Column a1 is the number of bytes to copy and a2 is the offset +** into source from which to begin copying. An INSERT row means to +** insert text into the output stream. Column a1 is the number of bytes +** to insert and column is a BLOB that contains the text to be inserted. +** +** The last row of a well-formed delta will have an op value of 'CHECKSUM'. +** The a1 column will be the value of the checksum and a2 will be NULL. +** +** If the input delta is not well-formed, then a row with an op value +** of 'ERROR' is returned. The a1 value of the ERROR row is the offset +** into the delta where the error was encountered and a2 is NULL. +*/ +typedef struct deltaparsevtab_vtab deltaparsevtab_vtab; +typedef struct deltaparsevtab_cursor deltaparsevtab_cursor; +struct deltaparsevtab_vtab { + sqlite3_vtab base; /* Base class - must be first */ + /* No additional information needed */ +}; +struct deltaparsevtab_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + char *aDelta; /* The delta being parsed */ + int nDelta; /* Number of bytes in the delta */ + int iCursor; /* Current cursor location */ + int eOp; /* Name of current operator */ + unsigned int a1, a2; /* Arguments to current operator */ + int iNext; /* Next cursor value */ +}; + +/* Operator names: +*/ +static const char *azOp[] = { + "SIZE", "COPY", "INSERT", "CHECKSUM", "ERROR", "EOF" +}; +#define DELTAPARSE_OP_SIZE 0 +#define DELTAPARSE_OP_COPY 1 +#define DELTAPARSE_OP_INSERT 2 +#define DELTAPARSE_OP_CHECKSUM 3 +#define DELTAPARSE_OP_ERROR 4 +#define DELTAPARSE_OP_EOF 5 + +/* +** The deltaparsevtabConnect() method is invoked to create a new +** deltaparse virtual table. +** +** Think of this routine as the constructor for deltaparsevtab_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the deltaparsevtab_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against the virtual table will look like. +*/ +static int deltaparsevtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + deltaparsevtab_vtab *pNew; + int rc; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(op,a1,a2,delta HIDDEN)" + ); + /* For convenience, define symbolic names for the index to each column. */ +#define DELTAPARSEVTAB_OP 0 +#define DELTAPARSEVTAB_A1 1 +#define DELTAPARSEVTAB_A2 2 +#define DELTAPARSEVTAB_DELTA 3 + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc64( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for deltaparsevtab_vtab objects. +*/ +static int deltaparsevtabDisconnect(sqlite3_vtab *pVtab){ + deltaparsevtab_vtab *p = (deltaparsevtab_vtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new deltaparsevtab_cursor object. +*/ +static int deltaparsevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + deltaparsevtab_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a deltaparsevtab_cursor. +*/ +static int deltaparsevtabClose(sqlite3_vtab_cursor *cur){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + sqlite3_free(pCur->aDelta); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a deltaparsevtab_cursor to its next row of output. +*/ +static int deltaparsevtabNext(sqlite3_vtab_cursor *cur){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + const char *z; + int i = 0; + + pCur->iCursor = pCur->iNext; + z = pCur->aDelta + pCur->iCursor; + pCur->a1 = deltaGetInt(&z, &i); + switch( z[0] ){ + case '@': { + z++; + pCur->a2 = deltaGetInt(&z, &i); + pCur->eOp = DELTAPARSE_OP_COPY; + pCur->iNext = (int)(&z[1] - pCur->aDelta); + break; + } + case ':': { + z++; + pCur->a2 = (unsigned int)(z - pCur->aDelta); + pCur->eOp = DELTAPARSE_OP_INSERT; + pCur->iNext = (int)(&z[pCur->a1] - pCur->aDelta); + break; + } + case ';': { + pCur->eOp = DELTAPARSE_OP_CHECKSUM; + pCur->iNext = pCur->nDelta; + break; + } + default: { + if( pCur->iNext==pCur->nDelta ){ + pCur->eOp = DELTAPARSE_OP_EOF; + }else{ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->iNext = pCur->nDelta; + } + break; + } + } + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the deltaparsevtab_cursor +** is currently pointing. +*/ +static int deltaparsevtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + switch( i ){ + case DELTAPARSEVTAB_OP: { + sqlite3_result_text(ctx, azOp[pCur->eOp], -1, SQLITE_STATIC); + break; + } + case DELTAPARSEVTAB_A1: { + sqlite3_result_int(ctx, pCur->a1); + break; + } + case DELTAPARSEVTAB_A2: { + if( pCur->eOp==DELTAPARSE_OP_COPY ){ + sqlite3_result_int(ctx, pCur->a2); + }else if( pCur->eOp==DELTAPARSE_OP_INSERT ){ + sqlite3_result_blob(ctx, pCur->aDelta+pCur->a2, pCur->a1, + SQLITE_TRANSIENT); + } + break; + } + case DELTAPARSEVTAB_DELTA: { + sqlite3_result_blob(ctx, pCur->aDelta, pCur->nDelta, SQLITE_TRANSIENT); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int deltaparsevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + *pRowid = pCur->iCursor; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int deltaparsevtabEof(sqlite3_vtab_cursor *cur){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor*)cur; + return pCur->eOp==DELTAPARSE_OP_EOF; +} + +/* +** This method is called to "rewind" the deltaparsevtab_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to deltaparsevtabColumn() or deltaparsevtabRowid() or +** deltaparsevtabEof(). +*/ +static int deltaparsevtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + deltaparsevtab_cursor *pCur = (deltaparsevtab_cursor *)pVtabCursor; + const char *a; + int i = 0; + pCur->eOp = DELTAPARSE_OP_ERROR; + if( idxNum!=1 ){ + return SQLITE_OK; + } + pCur->nDelta = sqlite3_value_bytes(argv[0]); + a = (const char*)sqlite3_value_blob(argv[0]); + if( pCur->nDelta==0 || a==0 ){ + return SQLITE_OK; + } + pCur->aDelta = sqlite3_malloc64( pCur->nDelta+1 ); + if( pCur->aDelta==0 ){ + pCur->nDelta = 0; + return SQLITE_NOMEM; + } + memcpy(pCur->aDelta, a, pCur->nDelta); + pCur->aDelta[pCur->nDelta] = 0; + a = pCur->aDelta; + pCur->eOp = DELTAPARSE_OP_SIZE; + pCur->a1 = deltaGetInt(&a, &i); + if( a[0]!='\n' ){ + pCur->eOp = DELTAPARSE_OP_ERROR; + pCur->a1 = pCur->a2 = 0; + pCur->iNext = pCur->nDelta; + return SQLITE_OK; + } + a++; + pCur->iNext = (unsigned int)(a - pCur->aDelta); + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int deltaparsevtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + for(i=0; inConstraint; i++){ + if( pIdxInfo->aConstraint[i].iColumn != DELTAPARSEVTAB_DELTA ) continue; + if( pIdxInfo->aConstraint[i].usable==0 ) continue; + if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->estimatedCost = (double)1; + pIdxInfo->estimatedRows = 10; + pIdxInfo->idxNum = 1; + return SQLITE_OK; + } + pIdxInfo->idxNum = 0; + pIdxInfo->estimatedCost = (double)0x7fffffff; + pIdxInfo->estimatedRows = 0x7fffffff; + return SQLITE_CONSTRAINT; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module deltaparsevtabModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ deltaparsevtabConnect, + /* xBestIndex */ deltaparsevtabBestIndex, + /* xDisconnect */ deltaparsevtabDisconnect, + /* xDestroy */ 0, + /* xOpen */ deltaparsevtabOpen, + /* xClose */ deltaparsevtabClose, + /* xFilter */ deltaparsevtabFilter, + /* xNext */ deltaparsevtabNext, + /* xEof */ deltaparsevtabEof, + /* xColumn */ deltaparsevtabColumn, + /* xRowid */ deltaparsevtabRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 +}; + + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fossildelta_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const int enc = SQLITE_UTF8|SQLITE_INNOCUOUS; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "delta_create", 2, enc, 0, + deltaCreateFunc, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "delta_apply", 2, enc, 0, + deltaApplyFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function(db, "delta_output_size", 1, enc, 0, + deltaOutputSizeFunc, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "delta_parse", &deltaparsevtabModule, 0); + } + return rc; +} diff --git a/ext/misc/fuzzer.c b/ext/misc/fuzzer.c new file mode 100644 index 0000000..92b7c0d --- /dev/null +++ b/ext/misc/fuzzer.c @@ -0,0 +1,1192 @@ +/* +** 2011 March 24 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** Code for a demonstration virtual table that generates variations +** on an input word at increasing edit distances from the original. +** +** A fuzzer virtual table is created like this: +** +** CREATE VIRTUAL TABLE f USING fuzzer(); +** +** When it is created, the new fuzzer table must be supplied with the +** name of a "fuzzer data table", which must reside in the same database +** file as the new fuzzer table. The fuzzer data table contains the various +** transformations and their costs that the fuzzer logic uses to generate +** variations. +** +** The fuzzer data table must contain exactly four columns (more precisely, +** the statement "SELECT * FROM " must return records +** that consist of four columns). It does not matter what the columns are +** named. +** +** Each row in the fuzzer data table represents a single character +** transformation. The left most column of the row (column 0) contains an +** integer value - the identifier of the ruleset to which the transformation +** rule belongs (see "MULTIPLE RULE SETS" below). The second column of the +** row (column 0) contains the input character or characters. The third +** column contains the output character or characters. And the fourth column +** contains the integer cost of making the transformation. For example: +** +** CREATE TABLE f_data(ruleset, cFrom, cTo, Cost); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, '', 'a', 100); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'b', '', 87); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'o', 'oe', 38); +** INSERT INTO f_data(ruleset, cFrom, cTo, Cost) VALUES(0, 'oe', 'o', 40); +** +** The first row inserted into the fuzzer data table by the SQL script +** above indicates that the cost of inserting a letter 'a' is 100. (All +** costs are integers. We recommend that costs be scaled so that the +** average cost is around 100.) The second INSERT statement creates a rule +** saying that the cost of deleting a single letter 'b' is 87. The third +** and fourth INSERT statements mean that the cost of transforming a +** single letter "o" into the two-letter sequence "oe" is 38 and that the +** cost of transforming "oe" back into "o" is 40. +** +** The contents of the fuzzer data table are loaded into main memory when +** a fuzzer table is first created, and may be internally reloaded by the +** system at any subsequent time. Therefore, the fuzzer data table should be +** populated before the fuzzer table is created and not modified thereafter. +** If you do need to modify the contents of the fuzzer data table, it is +** recommended that the associated fuzzer table be dropped, the fuzzer data +** table edited, and the fuzzer table recreated within a single transaction. +** Alternatively, the fuzzer data table can be edited then the database +** connection can be closed and reopened. +** +** Once it has been created, the fuzzer table can be queried as follows: +** +** SELECT word, distance FROM f +** WHERE word MATCH 'abcdefg' +** AND distance<200; +** +** This first query outputs the string "abcdefg" and all strings that +** can be derived from that string by appling the specified transformations. +** The strings are output together with their total transformation cost +** (called "distance") and appear in order of increasing cost. No string +** is output more than once. If there are multiple ways to transform the +** target string into the output string then the lowest cost transform is +** the one that is returned. In the example, the search is limited to +** strings with a total distance of less than 200. +** +** The fuzzer is a read-only table. Any attempt to DELETE, INSERT, or +** UPDATE on a fuzzer table will throw an error. +** +** It is important to put some kind of a limit on the fuzzer output. This +** can be either in the form of a LIMIT clause at the end of the query, +** or better, a "distance +#include +#include +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* +** Forward declaration of objects used by this implementation +*/ +typedef struct fuzzer_vtab fuzzer_vtab; +typedef struct fuzzer_cursor fuzzer_cursor; +typedef struct fuzzer_rule fuzzer_rule; +typedef struct fuzzer_seen fuzzer_seen; +typedef struct fuzzer_stem fuzzer_stem; + +/* +** Various types. +** +** fuzzer_cost is the "cost" of an edit operation. +** +** fuzzer_len is the length of a matching string. +** +** fuzzer_ruleid is an ruleset identifier. +*/ +typedef int fuzzer_cost; +typedef signed char fuzzer_len; +typedef int fuzzer_ruleid; + +/* +** Limits +*/ +#define FUZZER_MX_LENGTH 50 /* Maximum length of a rule string */ +#define FUZZER_MX_RULEID 2147483647 /* Maximum rule ID */ +#define FUZZER_MX_COST 1000 /* Maximum single-rule cost */ +#define FUZZER_MX_OUTPUT_LENGTH 100 /* Maximum length of an output string */ + + +/* +** Each transformation rule is stored as an instance of this object. +** All rules are kept on a linked list sorted by rCost. +*/ +struct fuzzer_rule { + fuzzer_rule *pNext; /* Next rule in order of increasing rCost */ + char *zFrom; /* Transform from */ + fuzzer_cost rCost; /* Cost of this transformation */ + fuzzer_len nFrom, nTo; /* Length of the zFrom and zTo strings */ + fuzzer_ruleid iRuleset; /* The rule set to which this rule belongs */ + char zTo[4]; /* Transform to (extra space appended) */ +}; + +/* +** A stem object is used to generate variants. It is also used to record +** previously generated outputs. +** +** Every stem is added to a hash table as it is output. Generation of +** duplicate stems is suppressed. +** +** Active stems (those that might generate new outputs) are kepts on a linked +** list sorted by increasing cost. The cost is the sum of rBaseCost and +** pRule->rCost. +*/ +struct fuzzer_stem { + char *zBasis; /* Word being fuzzed */ + const fuzzer_rule *pRule; /* Current rule to apply */ + fuzzer_stem *pNext; /* Next stem in rCost order */ + fuzzer_stem *pHash; /* Next stem with same hash on zBasis */ + fuzzer_cost rBaseCost; /* Base cost of getting to zBasis */ + fuzzer_cost rCostX; /* Precomputed rBaseCost + pRule->rCost */ + fuzzer_len nBasis; /* Length of the zBasis string */ + fuzzer_len n; /* Apply pRule at this character offset */ +}; + +/* +** A fuzzer virtual-table object +*/ +struct fuzzer_vtab { + sqlite3_vtab base; /* Base class - must be first */ + char *zClassName; /* Name of this class. Default: "fuzzer" */ + fuzzer_rule *pRule; /* All active rules in this fuzzer */ + int nCursor; /* Number of active cursors */ +}; + +#define FUZZER_HASH 4001 /* Hash table size */ +#define FUZZER_NQUEUE 20 /* Number of slots on the stem queue */ + +/* A fuzzer cursor object */ +struct fuzzer_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid of the current word */ + fuzzer_vtab *pVtab; /* The virtual table this cursor belongs to */ + fuzzer_cost rLimit; /* Maximum cost of any term */ + fuzzer_stem *pStem; /* Stem with smallest rCostX */ + fuzzer_stem *pDone; /* Stems already processed to completion */ + fuzzer_stem *aQueue[FUZZER_NQUEUE]; /* Queue of stems with higher rCostX */ + int mxQueue; /* Largest used index in aQueue[] */ + char *zBuf; /* Temporary use buffer */ + int nBuf; /* Bytes allocated for zBuf */ + int nStem; /* Number of stems allocated */ + int iRuleset; /* Only process rules from this ruleset */ + fuzzer_rule nullRule; /* Null rule used first */ + fuzzer_stem *apHash[FUZZER_HASH]; /* Hash of previously generated terms */ +}; + +/* +** The two input rule lists are both sorted in order of increasing +** cost. Merge them together into a single list, sorted by cost, and +** return a pointer to the head of that list. +*/ +static fuzzer_rule *fuzzerMergeRules(fuzzer_rule *pA, fuzzer_rule *pB){ + fuzzer_rule head; + fuzzer_rule *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCost<=pB->rCost ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Statement pStmt currently points to a row in the fuzzer data table. This +** function allocates and populates a fuzzer_rule structure according to +** the content of the row. +** +** If successful, *ppRule is set to point to the new object and SQLITE_OK +** is returned. Otherwise, *ppRule is zeroed, *pzErr may be set to point +** to an error message and an SQLite error code returned. +*/ +static int fuzzerLoadOneRule( + fuzzer_vtab *p, /* Fuzzer virtual table handle */ + sqlite3_stmt *pStmt, /* Base rule on statements current row */ + fuzzer_rule **ppRule, /* OUT: New rule object */ + char **pzErr /* OUT: Error message */ +){ + sqlite3_int64 iRuleset = sqlite3_column_int64(pStmt, 0); + const char *zFrom = (const char *)sqlite3_column_text(pStmt, 1); + const char *zTo = (const char *)sqlite3_column_text(pStmt, 2); + int nCost = sqlite3_column_int(pStmt, 3); + + int rc = SQLITE_OK; /* Return code */ + int nFrom; /* Size of string zFrom, in bytes */ + int nTo; /* Size of string zTo, in bytes */ + fuzzer_rule *pRule = 0; /* New rule object to return */ + + if( zFrom==0 ) zFrom = ""; + if( zTo==0 ) zTo = ""; + nFrom = (int)strlen(zFrom); + nTo = (int)strlen(zTo); + + /* Silently ignore null transformations */ + if( strcmp(zFrom, zTo)==0 ){ + *ppRule = 0; + return SQLITE_OK; + } + + if( nCost<=0 || nCost>FUZZER_MX_COST ){ + *pzErr = sqlite3_mprintf("%s: cost must be between 1 and %d", + p->zClassName, FUZZER_MX_COST + ); + rc = SQLITE_ERROR; + }else + if( nFrom>FUZZER_MX_LENGTH || nTo>FUZZER_MX_LENGTH ){ + *pzErr = sqlite3_mprintf("%s: maximum string length is %d", + p->zClassName, FUZZER_MX_LENGTH + ); + rc = SQLITE_ERROR; + }else + if( iRuleset<0 || iRuleset>FUZZER_MX_RULEID ){ + *pzErr = sqlite3_mprintf("%s: ruleset must be between 0 and %d", + p->zClassName, FUZZER_MX_RULEID + ); + rc = SQLITE_ERROR; + }else{ + + pRule = sqlite3_malloc64( sizeof(*pRule) + nFrom + nTo ); + if( pRule==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pRule, 0, sizeof(*pRule)); + pRule->zFrom = pRule->zTo; + pRule->zFrom += nTo + 1; + pRule->nFrom = (fuzzer_len)nFrom; + memcpy(pRule->zFrom, zFrom, nFrom+1); + memcpy(pRule->zTo, zTo, nTo+1); + pRule->nTo = (fuzzer_len)nTo; + pRule->rCost = nCost; + pRule->iRuleset = (int)iRuleset; + } + } + + *ppRule = pRule; + return rc; +} + +/* +** Load the content of the fuzzer data table into memory. +*/ +static int fuzzerLoadRules( + sqlite3 *db, /* Database handle */ + fuzzer_vtab *p, /* Virtual fuzzer table to configure */ + const char *zDb, /* Database containing rules data */ + const char *zData, /* Table containing rules data */ + char **pzErr /* OUT: Error message */ +){ + int rc = SQLITE_OK; /* Return code */ + char *zSql; /* SELECT used to read from rules table */ + fuzzer_rule *pHead = 0; + + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zData); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + int rc2; /* finalize() return code */ + sqlite3_stmt *pStmt = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc!=SQLITE_OK ){ + *pzErr = sqlite3_mprintf("%s: %s", p->zClassName, sqlite3_errmsg(db)); + }else if( sqlite3_column_count(pStmt)!=4 ){ + *pzErr = sqlite3_mprintf("%s: %s has %d columns, expected 4", + p->zClassName, zData, sqlite3_column_count(pStmt) + ); + rc = SQLITE_ERROR; + }else{ + while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + fuzzer_rule *pRule = 0; + rc = fuzzerLoadOneRule(p, pStmt, &pRule, pzErr); + if( pRule ){ + pRule->pNext = pHead; + pHead = pRule; + } + } + } + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + sqlite3_free(zSql); + + /* All rules are now in a singly linked list starting at pHead. This + ** block sorts them by cost and then sets fuzzer_vtab.pRule to point to + ** point to the head of the sorted list. + */ + if( rc==SQLITE_OK ){ + unsigned int i; + fuzzer_rule *pX; + fuzzer_rule *a[15]; + for(i=0; ipNext; + pX->pNext = 0; + for(i=0; a[i] && ipRule = fuzzerMergeRules(p->pRule, pX); + }else{ + /* An error has occurred. Setting p->pRule to point to the head of the + ** allocated list ensures that the list will be cleaned up in this case. + */ + assert( p->pRule==0 ); + p->pRule = pHead; + } + + return rc; +} + +/* +** This function converts an SQL quoted string into an unquoted string +** and returns a pointer to a buffer allocated using sqlite3_malloc() +** containing the result. The caller should eventually free this buffer +** using sqlite3_free. +** +** Examples: +** +** "abc" becomes abc +** 'xyz' becomes xyz +** [pqr] becomes pqr +** `mno` becomes mno +*/ +static char *fuzzerDequote(const char *zIn){ + sqlite3_int64 nIn; /* Size of input string, in bytes */ + char *zOut; /* Output (dequoted) string */ + + nIn = strlen(zIn); + zOut = sqlite3_malloc64(nIn+1); + if( zOut ){ + char q = zIn[0]; /* Quote character (if any ) */ + + if( q!='[' && q!= '\'' && q!='"' && q!='`' ){ + memcpy(zOut, zIn, (size_t)(nIn+1)); + }else{ + int iOut = 0; /* Index of next byte to write to output */ + int iIn; /* Index of next byte to read from input */ + + if( q=='[' ) q = ']'; + for(iIn=1; iInnCursor==0 ); + while( p->pRule ){ + fuzzer_rule *pRule = p->pRule; + p->pRule = pRule->pNext; + sqlite3_free(pRule); + } + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** xConnect/xCreate method for the fuzzer module. Arguments are: +** +** argv[0] -> module name ("fuzzer") +** argv[1] -> database name +** argv[2] -> table name +** argv[3] -> fuzzer rule table name +*/ +static int fuzzerConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + int rc = SQLITE_OK; /* Return code */ + fuzzer_vtab *pNew = 0; /* New virtual table */ + const char *zModule = argv[0]; + const char *zDb = argv[1]; + + if( argc!=4 ){ + *pzErr = sqlite3_mprintf( + "%s: wrong number of CREATE VIRTUAL TABLE arguments", zModule + ); + rc = SQLITE_ERROR; + }else{ + sqlite3_int64 nModule; /* Length of zModule, in bytes */ + + nModule = strlen(zModule); + pNew = sqlite3_malloc64( sizeof(*pNew) + nModule + 1); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + char *zTab; /* Dequoted name of fuzzer data table */ + + memset(pNew, 0, sizeof(*pNew)); + pNew->zClassName = (char*)&pNew[1]; + memcpy(pNew->zClassName, zModule, (size_t)(nModule+1)); + + zTab = fuzzerDequote(argv[3]); + if( zTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = fuzzerLoadRules(db, pNew, zDb, zTab, pzErr); + sqlite3_free(zTab); + } + + if( rc==SQLITE_OK ){ + rc = sqlite3_declare_vtab(db, "CREATE TABLE x(word,distance,ruleset)"); + } + if( rc!=SQLITE_OK ){ + fuzzerDisconnect((sqlite3_vtab *)pNew); + pNew = 0; + }else{ + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + } + } + + *ppVtab = (sqlite3_vtab *)pNew; + return rc; +} + +/* +** Open a new fuzzer cursor. +*/ +static int fuzzerOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + fuzzer_vtab *p = (fuzzer_vtab*)pVTab; + fuzzer_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->pVtab = p; + *ppCursor = &pCur->base; + p->nCursor++; + return SQLITE_OK; +} + +/* +** Free all stems in a list. +*/ +static void fuzzerClearStemList(fuzzer_stem *pStem){ + while( pStem ){ + fuzzer_stem *pNext = pStem->pNext; + sqlite3_free(pStem); + pStem = pNext; + } +} + +/* +** Free up all the memory allocated by a cursor. Set it rLimit to 0 +** to indicate that it is at EOF. +*/ +static void fuzzerClearCursor(fuzzer_cursor *pCur, int clearHash){ + int i; + fuzzerClearStemList(pCur->pStem); + fuzzerClearStemList(pCur->pDone); + for(i=0; iaQueue[i]); + pCur->rLimit = (fuzzer_cost)0; + if( clearHash && pCur->nStem ){ + pCur->mxQueue = 0; + pCur->pStem = 0; + pCur->pDone = 0; + memset(pCur->aQueue, 0, sizeof(pCur->aQueue)); + memset(pCur->apHash, 0, sizeof(pCur->apHash)); + } + pCur->nStem = 0; +} + +/* +** Close a fuzzer cursor. +*/ +static int fuzzerClose(sqlite3_vtab_cursor *cur){ + fuzzer_cursor *pCur = (fuzzer_cursor *)cur; + fuzzerClearCursor(pCur, 0); + sqlite3_free(pCur->zBuf); + pCur->pVtab->nCursor--; + sqlite3_free(pCur); + return SQLITE_OK; +} + +/* +** Compute the current output term for a fuzzer_stem. +*/ +static int fuzzerRender( + fuzzer_stem *pStem, /* The stem to be rendered */ + char **pzBuf, /* Write results into this buffer. realloc if needed */ + int *pnBuf /* Size of the buffer */ +){ + const fuzzer_rule *pRule = pStem->pRule; + int n; /* Size of output term without nul-term */ + char *z; /* Buffer to assemble output term in */ + + n = pStem->nBasis + pRule->nTo - pRule->nFrom; + if( (*pnBuf)n; + z = *pzBuf; + if( n<0 ){ + memcpy(z, pStem->zBasis, pStem->nBasis+1); + }else{ + memcpy(z, pStem->zBasis, n); + memcpy(&z[n], pRule->zTo, pRule->nTo); + memcpy(&z[n+pRule->nTo], &pStem->zBasis[n+pRule->nFrom], + pStem->nBasis-n-pRule->nFrom+1); + } + + assert( z[pStem->nBasis + pRule->nTo - pRule->nFrom]==0 ); + return SQLITE_OK; +} + +/* +** Compute a hash on zBasis. +*/ +static unsigned int fuzzerHash(const char *z){ + unsigned int h = 0; + while( *z ){ h = (h<<3) ^ (h>>29) ^ *(z++); } + return h % FUZZER_HASH; +} + +/* +** Current cost of a stem +*/ +static fuzzer_cost fuzzerCost(fuzzer_stem *pStem){ + return pStem->rCostX = pStem->rBaseCost + pStem->pRule->rCost; +} + +#if 0 +/* +** Print a description of a fuzzer_stem on stderr. +*/ +static void fuzzerStemPrint( + const char *zPrefix, + fuzzer_stem *pStem, + const char *zSuffix +){ + if( pStem->n<0 ){ + fprintf(stderr, "%s[%s](%d)-->self%s", + zPrefix, + pStem->zBasis, pStem->rBaseCost, + zSuffix + ); + }else{ + char *zBuf = 0; + int nBuf = 0; + if( fuzzerRender(pStem, &zBuf, &nBuf)!=SQLITE_OK ) return; + fprintf(stderr, "%s[%s](%d)-->{%s}(%d)%s", + zPrefix, + pStem->zBasis, pStem->rBaseCost, zBuf, pStem->, + zSuffix + ); + sqlite3_free(zBuf); + } +} +#endif + +/* +** Return 1 if the string to which the cursor is point has already +** been emitted. Return 0 if not. Return -1 on a memory allocation +** failures. +*/ +static int fuzzerSeen(fuzzer_cursor *pCur, fuzzer_stem *pStem){ + unsigned int h; + fuzzer_stem *pLookup; + + if( fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ + return -1; + } + h = fuzzerHash(pCur->zBuf); + pLookup = pCur->apHash[h]; + while( pLookup && strcmp(pLookup->zBasis, pCur->zBuf)!=0 ){ + pLookup = pLookup->pHash; + } + return pLookup!=0; +} + +/* +** If argument pRule is NULL, this function returns false. +** +** Otherwise, it returns true if rule pRule should be skipped. A rule +** should be skipped if it does not belong to rule-set iRuleset, or if +** applying it to stem pStem would create a string longer than +** FUZZER_MX_OUTPUT_LENGTH bytes. +*/ +static int fuzzerSkipRule( + const fuzzer_rule *pRule, /* Determine whether or not to skip this */ + fuzzer_stem *pStem, /* Stem rule may be applied to */ + int iRuleset /* Rule-set used by the current query */ +){ + return pRule && ( + (pRule->iRuleset!=iRuleset) + || (pStem->nBasis + pRule->nTo - pRule->nFrom)>FUZZER_MX_OUTPUT_LENGTH + ); +} + +/* +** Advance a fuzzer_stem to its next value. Return 0 if there are +** no more values that can be generated by this fuzzer_stem. Return +** -1 on a memory allocation failure. +*/ +static int fuzzerAdvance(fuzzer_cursor *pCur, fuzzer_stem *pStem){ + const fuzzer_rule *pRule; + while( (pRule = pStem->pRule)!=0 ){ + assert( pRule==&pCur->nullRule || pRule->iRuleset==pCur->iRuleset ); + while( pStem->n < pStem->nBasis - pRule->nFrom ){ + pStem->n++; + if( pRule->nFrom==0 + || memcmp(&pStem->zBasis[pStem->n], pRule->zFrom, pRule->nFrom)==0 + ){ + /* Found a rewrite case. Make sure it is not a duplicate */ + int rc = fuzzerSeen(pCur, pStem); + if( rc<0 ) return -1; + if( rc==0 ){ + fuzzerCost(pStem); + return 1; + } + } + } + pStem->n = -1; + do{ + pRule = pRule->pNext; + }while( fuzzerSkipRule(pRule, pStem, pCur->iRuleset) ); + pStem->pRule = pRule; + if( pRule && fuzzerCost(pStem)>pCur->rLimit ) pStem->pRule = 0; + } + return 0; +} + +/* +** The two input stem lists are both sorted in order of increasing +** rCostX. Merge them together into a single list, sorted by rCostX, and +** return a pointer to the head of that new list. +*/ +static fuzzer_stem *fuzzerMergeStems(fuzzer_stem *pA, fuzzer_stem *pB){ + fuzzer_stem head; + fuzzer_stem *pTail; + + pTail = &head; + while( pA && pB ){ + if( pA->rCostX<=pB->rCostX ){ + pTail->pNext = pA; + pTail = pA; + pA = pA->pNext; + }else{ + pTail->pNext = pB; + pTail = pB; + pB = pB->pNext; + } + } + if( pA==0 ){ + pTail->pNext = pB; + }else{ + pTail->pNext = pA; + } + return head.pNext; +} + +/* +** Load pCur->pStem with the lowest-cost stem. Return a pointer +** to the lowest-cost stem. +*/ +static fuzzer_stem *fuzzerLowestCostStem(fuzzer_cursor *pCur){ + fuzzer_stem *pBest, *pX; + int iBest; + int i; + + if( pCur->pStem==0 ){ + iBest = -1; + pBest = 0; + for(i=0; i<=pCur->mxQueue; i++){ + pX = pCur->aQueue[i]; + if( pX==0 ) continue; + if( pBest==0 || pBest->rCostX>pX->rCostX ){ + pBest = pX; + iBest = i; + } + } + if( pBest ){ + pCur->aQueue[iBest] = pBest->pNext; + pBest->pNext = 0; + pCur->pStem = pBest; + } + } + return pCur->pStem; +} + +/* +** Insert pNew into queue of pending stems. Then find the stem +** with the lowest rCostX and move it into pCur->pStem. +** list. The insert is done such the pNew is in the correct order +** according to fuzzer_stem.zBaseCost+fuzzer_stem.pRule->rCost. +*/ +static fuzzer_stem *fuzzerInsert(fuzzer_cursor *pCur, fuzzer_stem *pNew){ + fuzzer_stem *pX; + int i; + + /* If pCur->pStem exists and is greater than pNew, then make pNew + ** the new pCur->pStem and insert the old pCur->pStem instead. + */ + if( (pX = pCur->pStem)!=0 && pX->rCostX>pNew->rCostX ){ + pNew->pNext = 0; + pCur->pStem = pNew; + pNew = pX; + } + + /* Insert the new value */ + pNew->pNext = 0; + pX = pNew; + for(i=0; i<=pCur->mxQueue; i++){ + if( pCur->aQueue[i] ){ + pX = fuzzerMergeStems(pX, pCur->aQueue[i]); + pCur->aQueue[i] = 0; + }else{ + pCur->aQueue[i] = pX; + break; + } + } + if( i>pCur->mxQueue ){ + if( imxQueue = i; + pCur->aQueue[i] = pX; + }else{ + assert( pCur->mxQueue==FUZZER_NQUEUE-1 ); + pX = fuzzerMergeStems(pX, pCur->aQueue[FUZZER_NQUEUE-1]); + pCur->aQueue[FUZZER_NQUEUE-1] = pX; + } + } + + return fuzzerLowestCostStem(pCur); +} + +/* +** Allocate a new fuzzer_stem. Add it to the hash table but do not +** link it into either the pCur->pStem or pCur->pDone lists. +*/ +static fuzzer_stem *fuzzerNewStem( + fuzzer_cursor *pCur, + const char *zWord, + fuzzer_cost rBaseCost +){ + fuzzer_stem *pNew; + fuzzer_rule *pRule; + unsigned int h; + + pNew = sqlite3_malloc64( sizeof(*pNew) + strlen(zWord) + 1 ); + if( pNew==0 ) return 0; + memset(pNew, 0, sizeof(*pNew)); + pNew->zBasis = (char*)&pNew[1]; + pNew->nBasis = (fuzzer_len)strlen(zWord); + memcpy(pNew->zBasis, zWord, pNew->nBasis+1); + pRule = pCur->pVtab->pRule; + while( fuzzerSkipRule(pRule, pNew, pCur->iRuleset) ){ + pRule = pRule->pNext; + } + pNew->pRule = pRule; + pNew->n = -1; + pNew->rBaseCost = pNew->rCostX = rBaseCost; + h = fuzzerHash(pNew->zBasis); + pNew->pHash = pCur->apHash[h]; + pCur->apHash[h] = pNew; + pCur->nStem++; + return pNew; +} + + +/* +** Advance a cursor to its next row of output +*/ +static int fuzzerNext(sqlite3_vtab_cursor *cur){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + int rc; + fuzzer_stem *pStem, *pNew; + + pCur->iRowid++; + + /* Use the element the cursor is currently point to to create + ** a new stem and insert the new stem into the priority queue. + */ + pStem = pCur->pStem; + if( pStem->rCostX>0 ){ + rc = fuzzerRender(pStem, &pCur->zBuf, &pCur->nBuf); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + pNew = fuzzerNewStem(pCur, pCur->zBuf, pStem->rCostX); + if( pNew ){ + if( fuzzerAdvance(pCur, pNew)==0 ){ + pNew->pNext = pCur->pDone; + pCur->pDone = pNew; + }else{ + if( fuzzerInsert(pCur, pNew)==pNew ){ + return SQLITE_OK; + } + } + }else{ + return SQLITE_NOMEM; + } + } + + /* Adjust the priority queue so that the first element of the + ** stem list is the next lowest cost word. + */ + while( (pStem = pCur->pStem)!=0 ){ + int res = fuzzerAdvance(pCur, pStem); + if( res<0 ){ + return SQLITE_NOMEM; + }else if( res>0 ){ + pCur->pStem = 0; + pStem = fuzzerInsert(pCur, pStem); + if( (rc = fuzzerSeen(pCur, pStem))!=0 ){ + if( rc<0 ) return SQLITE_NOMEM; + continue; + } + return SQLITE_OK; /* New word found */ + } + pCur->pStem = 0; + pStem->pNext = pCur->pDone; + pCur->pDone = pStem; + if( fuzzerLowestCostStem(pCur) ){ + rc = fuzzerSeen(pCur, pCur->pStem); + if( rc<0 ) return SQLITE_NOMEM; + if( rc==0 ){ + return SQLITE_OK; + } + } + } + + /* Reach this point only if queue has been exhausted and there is + ** nothing left to be output. */ + pCur->rLimit = (fuzzer_cost)0; + return SQLITE_OK; +} + +/* +** Called to "rewind" a cursor back to the beginning so that +** it starts its output over again. Always called at least once +** prior to any fuzzerColumn, fuzzerRowid, or fuzzerEof call. +*/ +static int fuzzerFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + fuzzer_cursor *pCur = (fuzzer_cursor *)pVtabCursor; + const char *zWord = ""; + fuzzer_stem *pStem; + int idx; + + fuzzerClearCursor(pCur, 1); + pCur->rLimit = 2147483647; + idx = 0; + if( idxNum & 1 ){ + zWord = (const char*)sqlite3_value_text(argv[0]); + idx++; + } + if( idxNum & 2 ){ + pCur->rLimit = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + if( idxNum & 4 ){ + pCur->iRuleset = (fuzzer_cost)sqlite3_value_int(argv[idx]); + idx++; + } + pCur->nullRule.pNext = pCur->pVtab->pRule; + pCur->nullRule.rCost = 0; + pCur->nullRule.nFrom = 0; + pCur->nullRule.nTo = 0; + pCur->nullRule.zFrom = ""; + pCur->iRowid = 1; + assert( pCur->pStem==0 ); + + /* If the query term is longer than FUZZER_MX_OUTPUT_LENGTH bytes, this + ** query will return zero rows. */ + if( (int)strlen(zWord)pStem = pStem = fuzzerNewStem(pCur, zWord, (fuzzer_cost)0); + if( pStem==0 ) return SQLITE_NOMEM; + pStem->pRule = &pCur->nullRule; + pStem->n = pStem->nBasis; + }else{ + pCur->rLimit = 0; + } + + return SQLITE_OK; +} + +/* +** Only the word and distance columns have values. All other columns +** return NULL +*/ +static int fuzzerColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + if( i==0 ){ + /* the "word" column */ + if( fuzzerRender(pCur->pStem, &pCur->zBuf, &pCur->nBuf)==SQLITE_NOMEM ){ + return SQLITE_NOMEM; + } + sqlite3_result_text(ctx, pCur->zBuf, -1, SQLITE_TRANSIENT); + }else if( i==1 ){ + /* the "distance" column */ + sqlite3_result_int(ctx, pCur->pStem->rCostX); + }else{ + /* All other columns are NULL */ + sqlite3_result_null(ctx); + } + return SQLITE_OK; +} + +/* +** The rowid. +*/ +static int fuzzerRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** When the fuzzer_cursor.rLimit value is 0 or less, that is a signal +** that the cursor has nothing more to output. +*/ +static int fuzzerEof(sqlite3_vtab_cursor *cur){ + fuzzer_cursor *pCur = (fuzzer_cursor*)cur; + return pCur->rLimit<=(fuzzer_cost)0; +} + +/* +** Search for terms of these forms: +** +** (A) word MATCH $str +** (B1) distance < $value +** (B2) distance <= $value +** (C) ruleid == $ruleid +** +** The distance< and distance<= are both treated as distance<=. +** The query plan number is a bit vector: +** +** bit 1: Term of the form (A) found +** bit 2: Term like (B1) or (B2) found +** bit 3: Term like (C) found +** +** If bit-1 is set, $str is always in filter.argv[0]. If bit-2 is set +** then $value is in filter.argv[0] if bit-1 is clear and is in +** filter.argv[1] if bit-1 is set. If bit-3 is set, then $ruleid is +** in filter.argv[0] if bit-1 and bit-2 are both zero, is in +** filter.argv[1] if exactly one of bit-1 and bit-2 are set, and is in +** filter.argv[2] if both bit-1 and bit-2 are set. +*/ +static int fuzzerBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int iPlan = 0; + int iDistTerm = -1; + int iRulesetTerm = -1; + int i; + int seenMatch = 0; + const struct sqlite3_index_constraint *pConstraint; + double rCost = 1e12; + + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH ){ + seenMatch = 1; + } + if( pConstraint->usable==0 ) continue; + if( (iPlan & 1)==0 + && pConstraint->iColumn==0 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_MATCH + ){ + iPlan |= 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + rCost /= 1e6; + } + if( (iPlan & 2)==0 + && pConstraint->iColumn==1 + && (pConstraint->op==SQLITE_INDEX_CONSTRAINT_LT + || pConstraint->op==SQLITE_INDEX_CONSTRAINT_LE) + ){ + iPlan |= 2; + iDistTerm = i; + rCost /= 10.0; + } + if( (iPlan & 4)==0 + && pConstraint->iColumn==2 + && pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ + ){ + iPlan |= 4; + pIdxInfo->aConstraintUsage[i].omit = 1; + iRulesetTerm = i; + rCost /= 10.0; + } + } + if( iPlan & 2 ){ + pIdxInfo->aConstraintUsage[iDistTerm].argvIndex = 1+((iPlan&1)!=0); + } + if( iPlan & 4 ){ + int idx = 1; + if( iPlan & 1 ) idx++; + if( iPlan & 2 ) idx++; + pIdxInfo->aConstraintUsage[iRulesetTerm].argvIndex = idx; + } + pIdxInfo->idxNum = iPlan; + if( pIdxInfo->nOrderBy==1 + && pIdxInfo->aOrderBy[0].iColumn==1 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + if( seenMatch && (iPlan&1)==0 ) rCost = 1e99; + pIdxInfo->estimatedCost = rCost; + + return SQLITE_OK; +} + +/* +** A virtual table module that implements the "fuzzer". +*/ +static sqlite3_module fuzzerModule = { + 0, /* iVersion */ + fuzzerConnect, + fuzzerConnect, + fuzzerBestIndex, + fuzzerDisconnect, + fuzzerDisconnect, + fuzzerOpen, /* xOpen - open a cursor */ + fuzzerClose, /* xClose - close a cursor */ + fuzzerFilter, /* xFilter - configure scan constraints */ + fuzzerNext, /* xNext - advance a cursor */ + fuzzerEof, /* xEof - check for end of scan */ + fuzzerColumn, /* xColumn - read data */ + fuzzerRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_fuzzer_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "fuzzer", &fuzzerModule, 0); +#endif + return rc; +} diff --git a/ext/misc/ieee754.c b/ext/misc/ieee754.c new file mode 100644 index 0000000..99489fe --- /dev/null +++ b/ext/misc/ieee754.c @@ -0,0 +1,324 @@ +/* +** 2013-04-17 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This SQLite extension implements functions for the exact display +** and input of IEEE754 Binary64 floating-point numbers. +** +** ieee754(X) +** ieee754(Y,Z) +** +** In the first form, the value X should be a floating-point number. +** The function will return a string of the form 'ieee754(Y,Z)' where +** Y and Z are integers such that X==Y*pow(2,Z). +** +** In the second form, Y and Z are integers which are the mantissa and +** base-2 exponent of a new floating point number. The function returns +** a floating-point value equal to Y*pow(2,Z). +** +** Examples: +** +** ieee754(2.0) -> 'ieee754(2,0)' +** ieee754(45.25) -> 'ieee754(181,-2)' +** ieee754(2, 0) -> 2.0 +** ieee754(181, -2) -> 45.25 +** +** Two additional functions break apart the one-argument ieee754() +** result into separate integer values: +** +** ieee754_mantissa(45.25) -> 181 +** ieee754_exponent(45.25) -> -2 +** +** These functions convert binary64 numbers into blobs and back again. +** +** ieee754_from_blob(x'3ff0000000000000') -> 1.0 +** ieee754_to_blob(1.0) -> x'3ff0000000000000' +** +** In all single-argument functions, if the argument is an 8-byte blob +** then that blob is interpreted as a big-endian binary64 value. +** +** +** EXACT DECIMAL REPRESENTATION OF BINARY64 VALUES +** ----------------------------------------------- +** +** This extension in combination with the separate 'decimal' extension +** can be used to compute the exact decimal representation of binary64 +** values. To begin, first compute a table of exponent values: +** +** CREATE TABLE pow2(x INTEGER PRIMARY KEY, v TEXT); +** WITH RECURSIVE c(x,v) AS ( +** VALUES(0,'1') +** UNION ALL +** SELECT x+1, decimal_mul(v,'2') FROM c WHERE x+1<=971 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** WITH RECURSIVE c(x,v) AS ( +** VALUES(-1,'0.5') +** UNION ALL +** SELECT x-1, decimal_mul(v,'0.5') FROM c WHERE x-1>=-1075 +** ) INSERT INTO pow2(x,v) SELECT x, v FROM c; +** +** Then, to compute the exact decimal representation of a floating +** point value (the value 47.49 is used in the example) do: +** +** WITH c(n) AS (VALUES(47.49)) +** ---------------^^^^^---- Replace with whatever you want +** SELECT decimal_mul(ieee754_mantissa(c.n),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.n); +** +** Here is a query to show various boundry values for the binary64 +** number format: +** +** WITH c(name,bin) AS (VALUES +** ('minimum positive value', x'0000000000000001'), +** ('maximum subnormal value', x'000fffffffffffff'), +** ('mininum positive nornal value', x'0010000000000000'), +** ('maximum value', x'7fefffffffffffff')) +** SELECT c.name, decimal_mul(ieee754_mantissa(c.bin),pow2.v) +** FROM pow2, c WHERE pow2.x=ieee754_exponent(c.bin); +** +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include + +/* Mark a function parameter as unused, to suppress nuisance compiler +** warnings. */ +#ifndef UNUSED_PARAMETER +# define UNUSED_PARAMETER(X) (void)(X) +#endif + +/* +** Implementation of the ieee754() function +*/ +static void ieee754func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + if( argc==1 ){ + sqlite3_int64 m, a; + double r; + int e; + int isNeg; + char zResult[100]; + assert( sizeof(m)==sizeof(r) ); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(r) + ){ + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>52; + m = a & ((((sqlite3_int64)1)<<52)-1); + if( e==0 ){ + m <<= 1; + }else{ + m |= ((sqlite3_int64)1)<<52; + } + while( e<1075 && m>0 && (m&1)==0 ){ + m >>= 1; + e++; + } + if( isNeg ) m = -m; + } + switch( *(int*)sqlite3_user_data(context) ){ + case 0: + sqlite3_snprintf(sizeof(zResult), zResult, "ieee754(%lld,%d)", + m, e-1075); + sqlite3_result_text(context, zResult, -1, SQLITE_TRANSIENT); + break; + case 1: + sqlite3_result_int64(context, m); + break; + case 2: + sqlite3_result_int(context, e-1075); + break; + } + }else{ + sqlite3_int64 m, e, a; + double r; + int isNeg = 0; + m = sqlite3_value_int64(argv[0]); + e = sqlite3_value_int64(argv[1]); + + /* Limit the range of e. Ticket 22dea1cfdb9151e4 2021-03-02 */ + if( e>10000 ){ + e = 10000; + }else if( e<-10000 ){ + e = -10000; + } + + if( m<0 ){ + isNeg = 1; + m = -m; + if( m<0 ) return; + }else if( m==0 && e>-1000 && e<1000 ){ + sqlite3_result_double(context, 0.0); + return; + } + while( (m>>32)&0xffe00000 ){ + m >>= 1; + e++; + } + while( m!=0 && ((m>>32)&0xfff00000)==0 ){ + m <<= 1; + e--; + } + e += 1075; + if( e<=0 ){ + /* Subnormal */ + if( 1-e >= 64 ){ + m = 0; + }else{ + m >>= 1-e; + } + e = 0; + }else if( e>0x7ff ){ + e = 0x7ff; + } + a = m & ((((sqlite3_int64)1)<<52)-1); + a |= e<<52; + if( isNeg ) a |= ((sqlite3_uint64)1)<<63; + memcpy(&r, &a, sizeof(r)); + sqlite3_result_double(context, r); + } +} + +/* +** Functions to convert between blobs and floats. +*/ +static void ieee754func_from_blob( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + UNUSED_PARAMETER(argc); + if( sqlite3_value_type(argv[0])==SQLITE_BLOB + && sqlite3_value_bytes(argv[0])==sizeof(double) + ){ + double r; + const unsigned char *x = sqlite3_value_blob(argv[0]); + unsigned int i; + sqlite3_uint64 v = 0; + for(i=0; i>= 8; + } + sqlite3_result_blob(context, a, sizeof(r), SQLITE_TRANSIENT); + } +} + +/* +** SQL Function: ieee754_inc(r,N) +** +** Move the floating point value r by N quantums and return the new +** values. +** +** Behind the scenes: this routine merely casts r into a 64-bit unsigned +** integer, adds N, then casts the value back into float. +** +** Example: To find the smallest positive number: +** +** SELECT ieee754_inc(0.0,+1); +*/ +static void ieee754inc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double r; + sqlite3_int64 N; + sqlite3_uint64 m1, m2; + double r2; + UNUSED_PARAMETER(argc); + r = sqlite3_value_double(argv[0]); + N = sqlite3_value_int64(argv[1]); + memcpy(&m1, &r, 8); + m2 = m1 + N; + memcpy(&r2, &m2, 8); + sqlite3_result_double(context, r2); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_ieee_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + static const struct { + char *zFName; + int nArg; + int iAux; + void (*xFunc)(sqlite3_context*,int,sqlite3_value**); + } aFunc[] = { + { "ieee754", 1, 0, ieee754func }, + { "ieee754", 2, 0, ieee754func }, + { "ieee754_mantissa", 1, 1, ieee754func }, + { "ieee754_exponent", 1, 2, ieee754func }, + { "ieee754_to_blob", 1, 0, ieee754func_to_blob }, + { "ieee754_from_blob", 1, 0, ieee754func_from_blob }, + { "ieee754_inc", 2, 0, ieee754inc }, + }; + unsigned int i; + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + for(i=0; i +#include + +#ifndef SQLITE_OMIT_VIRTUALTABLE + +/* memstat_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a memstat virtual table +*/ +typedef struct memstat_vtab memstat_vtab; +struct memstat_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this memstat vtab */ +}; + +/* memstat_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct memstat_cursor memstat_cursor; +struct memstat_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + int iRowid; /* Current row in aMemstatColumn[] */ + int iDb; /* Which schema we are looking at */ + int nDb; /* Number of schemas */ + char **azDb; /* Names of all schemas */ + sqlite3_int64 aVal[2]; /* Result values */ +}; + +/* +** The memstatConnect() method is invoked to create a new +** memstat_vtab that describes the memstat virtual table. +** +** Think of this routine as the constructor for memstat_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the memstat_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against memstat will look like. +*/ +static int memstatConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + memstat_vtab *pNew; + int rc; + +/* Column numbers */ +#define MSV_COLUMN_NAME 0 /* Name of quantity being measured */ +#define MSV_COLUMN_SCHEMA 1 /* schema name */ +#define MSV_COLUMN_VALUE 2 /* Current value */ +#define MSV_COLUMN_HIWTR 3 /* Highwater mark */ + + rc = sqlite3_declare_vtab(db,"CREATE TABLE x(name,schema,value,hiwtr)"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for memstat_cursor objects. +*/ +static int memstatDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new memstat_cursor object. +*/ +static int memstatOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + memstat_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((memstat_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Clear all the schema names from a cursor +*/ +static void memstatClearSchema(memstat_cursor *pCur){ + int i; + if( pCur->azDb==0 ) return; + for(i=0; inDb; i++){ + sqlite3_free(pCur->azDb[i]); + } + sqlite3_free(pCur->azDb); + pCur->azDb = 0; + pCur->nDb = 0; +} + +/* +** Fill in the azDb[] array for the cursor. +*/ +static int memstatFindSchemas(memstat_cursor *pCur){ + sqlite3_stmt *pStmt = 0; + int rc; + if( pCur->nDb ) return SQLITE_OK; + rc = sqlite3_prepare_v2(pCur->db, "PRAGMA database_list", -1, &pStmt, 0); + if( rc ){ + sqlite3_finalize(pStmt); + return rc; + } + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + char **az, *z; + az = sqlite3_realloc64(pCur->azDb, sizeof(char*)*(pCur->nDb+1)); + if( az==0 ){ + memstatClearSchema(pCur); + return SQLITE_NOMEM; + } + pCur->azDb = az; + z = sqlite3_mprintf("%s", sqlite3_column_text(pStmt, 1)); + if( z==0 ){ + memstatClearSchema(pCur); + return SQLITE_NOMEM; + } + pCur->azDb[pCur->nDb] = z; + pCur->nDb++; + } + sqlite3_finalize(pStmt); + return SQLITE_OK; +} + + +/* +** Destructor for a memstat_cursor. +*/ +static int memstatClose(sqlite3_vtab_cursor *cur){ + memstat_cursor *pCur = (memstat_cursor*)cur; + memstatClearSchema(pCur); + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Allowed values for aMemstatColumn[].eType +*/ +#define MSV_GSTAT 0 /* sqlite3_status64() information */ +#define MSV_DB 1 /* sqlite3_db_status() information */ +#define MSV_ZIPVFS 2 /* ZIPVFS file-control with 64-bit return */ + +/* +** An array of quantities that can be measured and reported by +** this virtual table +*/ +static const struct MemstatColumns { + const char *zName; /* Symbolic name */ + unsigned char eType; /* Type of interface */ + unsigned char mNull; /* Bitmask of which columns are NULL */ + /* 2: dbname, 4: current, 8: hiwtr */ + int eOp; /* Opcode */ +} aMemstatColumn[] = { + {"MEMORY_USED", MSV_GSTAT, 2, SQLITE_STATUS_MEMORY_USED }, + {"MALLOC_SIZE", MSV_GSTAT, 6, SQLITE_STATUS_MALLOC_SIZE }, + {"MALLOC_COUNT", MSV_GSTAT, 2, SQLITE_STATUS_MALLOC_COUNT }, + {"PAGECACHE_USED", MSV_GSTAT, 2, SQLITE_STATUS_PAGECACHE_USED }, + {"PAGECACHE_OVERFLOW", MSV_GSTAT, 2, SQLITE_STATUS_PAGECACHE_OVERFLOW }, + {"PAGECACHE_SIZE", MSV_GSTAT, 6, SQLITE_STATUS_PAGECACHE_SIZE }, + {"PARSER_STACK", MSV_GSTAT, 6, SQLITE_STATUS_PARSER_STACK }, + {"DB_LOOKASIDE_USED", MSV_DB, 2, SQLITE_DBSTATUS_LOOKASIDE_USED }, + {"DB_LOOKASIDE_HIT", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_HIT }, + {"DB_LOOKASIDE_MISS_SIZE", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE}, + {"DB_LOOKASIDE_MISS_FULL", MSV_DB, 6, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL}, + {"DB_CACHE_USED", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_USED }, +#if SQLITE_VERSION_NUMBER >= 3140000 + {"DB_CACHE_USED_SHARED", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_USED_SHARED }, +#endif + {"DB_SCHEMA_USED", MSV_DB, 10, SQLITE_DBSTATUS_SCHEMA_USED }, + {"DB_STMT_USED", MSV_DB, 10, SQLITE_DBSTATUS_STMT_USED }, + {"DB_CACHE_HIT", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_HIT }, + {"DB_CACHE_MISS", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_MISS }, + {"DB_CACHE_WRITE", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_WRITE }, +#if SQLITE_VERSION_NUMBER >= 3230000 + {"DB_CACHE_SPILL", MSV_DB, 10, SQLITE_DBSTATUS_CACHE_SPILL }, +#endif + {"DB_DEFERRED_FKS", MSV_DB, 10, SQLITE_DBSTATUS_DEFERRED_FKS }, +#ifdef SQLITE_ENABLE_ZIPVFS + {"ZIPVFS_CACHE_USED", MSV_ZIPVFS, 8, 231454 }, + {"ZIPVFS_CACHE_HIT", MSV_ZIPVFS, 8, 231455 }, + {"ZIPVFS_CACHE_MISS", MSV_ZIPVFS, 8, 231456 }, + {"ZIPVFS_CACHE_WRITE", MSV_ZIPVFS, 8, 231457 }, + {"ZIPVFS_DIRECT_READ", MSV_ZIPVFS, 8, 231458 }, + {"ZIPVFS_DIRECT_BYTES", MSV_ZIPVFS, 8, 231459 }, +#endif /* SQLITE_ENABLE_ZIPVFS */ +}; +#define MSV_NROW (sizeof(aMemstatColumn)/sizeof(aMemstatColumn[0])) + +/* +** Advance a memstat_cursor to its next row of output. +*/ +static int memstatNext(sqlite3_vtab_cursor *cur){ + memstat_cursor *pCur = (memstat_cursor*)cur; + int i; + assert( pCur->iRowid<=MSV_NROW ); + while(1){ + i = (int)pCur->iRowid - 1; + if( i<0 || (aMemstatColumn[i].mNull & 2)!=0 || (++pCur->iDb)>=pCur->nDb ){ + pCur->iRowid++; + if( pCur->iRowid>MSV_NROW ) return SQLITE_OK; /* End of the table */ + pCur->iDb = 0; + i++; + } + pCur->aVal[0] = 0; + pCur->aVal[1] = 0; + switch( aMemstatColumn[i].eType ){ + case MSV_GSTAT: { + if( sqlite3_libversion_number()>=3010000 ){ + sqlite3_status64(aMemstatColumn[i].eOp, + &pCur->aVal[0], &pCur->aVal[1],0); + }else{ + int xCur, xHiwtr; + sqlite3_status(aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0); + pCur->aVal[0] = xCur; + pCur->aVal[1] = xHiwtr; + } + break; + } + case MSV_DB: { + int xCur, xHiwtr; + sqlite3_db_status(pCur->db, aMemstatColumn[i].eOp, &xCur, &xHiwtr, 0); + pCur->aVal[0] = xCur; + pCur->aVal[1] = xHiwtr; + break; + } + case MSV_ZIPVFS: { + int rc; + rc = sqlite3_file_control(pCur->db, pCur->azDb[pCur->iDb], + aMemstatColumn[i].eOp, (void*)&pCur->aVal[0]); + if( rc!=SQLITE_OK ) continue; + break; + } + } + break; + } + return SQLITE_OK; +} + + +/* +** Return values of columns for the row at which the memstat_cursor +** is currently pointing. +*/ +static int memstatColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int iCol /* Which column to return */ +){ + memstat_cursor *pCur = (memstat_cursor*)cur; + int i; + assert( pCur->iRowid>0 && pCur->iRowid<=MSV_NROW ); + i = (int)pCur->iRowid - 1; + if( (aMemstatColumn[i].mNull & (1<azDb[pCur->iDb], -1, 0); + break; + } + case MSV_COLUMN_VALUE: { + sqlite3_result_int64(ctx, pCur->aVal[0]); + break; + } + case MSV_COLUMN_HIWTR: { + sqlite3_result_int64(ctx, pCur->aVal[1]); + break; + } + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int memstatRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + memstat_cursor *pCur = (memstat_cursor*)cur; + *pRowid = pCur->iRowid*1000 + pCur->iDb; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int memstatEof(sqlite3_vtab_cursor *cur){ + memstat_cursor *pCur = (memstat_cursor*)cur; + return pCur->iRowid>MSV_NROW; +} + +/* +** This method is called to "rewind" the memstat_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to memstatColumn() or memstatRowid() or +** memstatEof(). +*/ +static int memstatFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + memstat_cursor *pCur = (memstat_cursor *)pVtabCursor; + int rc = memstatFindSchemas(pCur); + if( rc ) return rc; + pCur->iRowid = 0; + pCur->iDb = 0; + return memstatNext(pVtabCursor); +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the memstat virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int memstatBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + pIdxInfo->estimatedCost = (double)500; + pIdxInfo->estimatedRows = 500; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** memstat virtual table. +*/ +static sqlite3_module memstatModule = { + 0, /* iVersion */ + 0, /* xCreate */ + memstatConnect, /* xConnect */ + memstatBestIndex, /* xBestIndex */ + memstatDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + memstatOpen, /* xOpen - open a cursor */ + memstatClose, /* xClose - close a cursor */ + memstatFilter, /* xFilter - configure scan constraints */ + memstatNext, /* xNext - advance a cursor */ + memstatEof, /* xEof - check for end of scan */ + memstatColumn, /* xColumn - read data */ + memstatRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +int sqlite3MemstatVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "sqlite_memstat", &memstatModule, 0); +#endif + return rc; +} + +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_memstat_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3MemstatVtabInit(db); +#endif + return rc; +} +#endif /* SQLITE_CORE */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_MEMSTATVTAB) */ diff --git a/ext/misc/memtrace.c b/ext/misc/memtrace.c new file mode 100644 index 0000000..19ff4c4 --- /dev/null +++ b/ext/misc/memtrace.c @@ -0,0 +1,108 @@ +/* +** 2019-01-21 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements an extension that uses the SQLITE_CONFIG_MALLOC +** mechanism to add a tracing layer on top of SQLite. If this extension +** is registered prior to sqlite3_initialize(), it will cause all memory +** allocation activities to be logged on standard output, or to some other +** FILE specified by the initializer. +** +** This file needs to be compiled into the application that uses it. +** +** This extension is used to implement the --memtrace option of the +** command-line shell. +*/ +#include +#include +#include + +/* The original memory allocation routines */ +static sqlite3_mem_methods memtraceBase; +static FILE *memtraceOut; + +/* Methods that trace memory allocations */ +static void *memtraceMalloc(int n){ + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: allocate %d bytes\n", + memtraceBase.xRoundup(n)); + } + return memtraceBase.xMalloc(n); +} +static void memtraceFree(void *p){ + if( p==0 ) return; + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: free %d bytes\n", memtraceBase.xSize(p)); + } + memtraceBase.xFree(p); +} +static void *memtraceRealloc(void *p, int n){ + if( p==0 ) return memtraceMalloc(n); + if( n==0 ){ + memtraceFree(p); + return 0; + } + if( memtraceOut ){ + fprintf(memtraceOut, "MEMTRACE: resize %d -> %d bytes\n", + memtraceBase.xSize(p), memtraceBase.xRoundup(n)); + } + return memtraceBase.xRealloc(p, n); +} +static int memtraceSize(void *p){ + return memtraceBase.xSize(p); +} +static int memtraceRoundup(int n){ + return memtraceBase.xRoundup(n); +} +static int memtraceInit(void *p){ + return memtraceBase.xInit(p); +} +static void memtraceShutdown(void *p){ + memtraceBase.xShutdown(p); +} + +/* The substitute memory allocator */ +static sqlite3_mem_methods ersaztMethods = { + memtraceMalloc, + memtraceFree, + memtraceRealloc, + memtraceSize, + memtraceRoundup, + memtraceInit, + memtraceShutdown, + 0 +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3MemTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETMALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &ersaztMethods); + } + } + memtraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3MemTraceDeactivate(void){ + int rc = SQLITE_OK; + if( memtraceBase.xMalloc!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_MALLOC, &memtraceBase); + if( rc==SQLITE_OK ){ + memset(&memtraceBase, 0, sizeof(memtraceBase)); + } + } + memtraceOut = 0; + return rc; +} diff --git a/ext/misc/memvfs.c b/ext/misc/memvfs.c new file mode 100644 index 0000000..83fc946 --- /dev/null +++ b/ext/misc/memvfs.c @@ -0,0 +1,575 @@ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This is an in-memory VFS implementation. The application supplies +** a chunk of memory to hold the database file. +** +** Because there is place to store a rollback or wal journal, the database +** must use one of journal_mode=MEMORY or journal_mode=NONE. +** +** USAGE: +** +** sqlite3_open_v2("file:/whatever?ptr=0xf05538&sz=14336&max=65536", &db, +** SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI, +** "memvfs"); +** +** These are the query parameters: +** +** ptr= The address of the memory buffer that holds the database. +** +** sz= The current size the database file +** +** maxsz= The maximum size of the database. In other words, the +** amount of space allocated for the ptr= buffer. +** +** freeonclose= If true, then sqlite3_free() is called on the ptr= +** value when the connection closes. +** +** The ptr= and sz= query parameters are required. If maxsz= is omitted, +** then it defaults to the sz= value. Parameter values can be in either +** decimal or hexadecimal. The filename in the URI is ignored. +*/ +#include +SQLITE_EXTENSION_INIT1 +#include +#include + + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szMax; /* Space allocated to aData */ + unsigned char *aData; /* content of the file */ + int bFreeOnClose; /* Invoke sqlite3_free() on aData at close */ +}; + +/* +** Methods for MemFile +*/ +static int memClose(sqlite3_file*); +static int memRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memTruncate(sqlite3_file*, sqlite3_int64 size); +static int memSync(sqlite3_file*, int flags); +static int memFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memLock(sqlite3_file*, int); +static int memUnlock(sqlite3_file*, int); +static int memCheckReservedLock(sqlite3_file*, int *pResOut); +static int memFileControl(sqlite3_file*, int op, void *pArg); +static int memSectorSize(sqlite3_file*); +static int memDeviceCharacteristics(sqlite3_file*); +static int memShmMap(sqlite3_file*, int iPg, int pgsz, int, void volatile**); +static int memShmLock(sqlite3_file*, int offset, int n, int flags); +static void memShmBarrier(sqlite3_file*); +static int memShmUnmap(sqlite3_file*, int deleteFlag); +static int memFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int memDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int memAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memDlOpen(sqlite3_vfs*, const char *zFilename); +static void memDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memDlClose(sqlite3_vfs*, void*); +static int memRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memSleep(sqlite3_vfs*, int microseconds); +static int memCurrentTime(sqlite3_vfs*, double*); +static int memGetLastError(sqlite3_vfs*, int, char *); +static int memCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs mem_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memvfs", /* zName */ + 0, /* pAppData (set when registered) */ + memOpen, /* xOpen */ + memDelete, /* xDelete */ + memAccess, /* xAccess */ + memFullPathname, /* xFullPathname */ + memDlOpen, /* xDlOpen */ + memDlError, /* xDlError */ + memDlSym, /* xDlSym */ + memDlClose, /* xDlClose */ + memRandomness, /* xRandomness */ + memSleep, /* xSleep */ + memCurrentTime, /* xCurrentTime */ + memGetLastError, /* xGetLastError */ + memCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +static const sqlite3_io_methods mem_io_methods = { + 3, /* iVersion */ + memClose, /* xClose */ + memRead, /* xRead */ + memWrite, /* xWrite */ + memTruncate, /* xTruncate */ + memSync, /* xSync */ + memFileSize, /* xFileSize */ + memLock, /* xLock */ + memUnlock, /* xUnlock */ + memCheckReservedLock, /* xCheckReservedLock */ + memFileControl, /* xFileControl */ + memSectorSize, /* xSectorSize */ + memDeviceCharacteristics, /* xDeviceCharacteristics */ + memShmMap, /* xShmMap */ + memShmLock, /* xShmLock */ + memShmBarrier, /* xShmBarrier */ + memShmUnmap, /* xShmUnmap */ + memFetch, /* xFetch */ + memUnfetch /* xUnfetch */ +}; + + + +/* +** Close an mem-file. +** +** The pData pointer is owned by the application, so there is nothing +** to free. +*/ +static int memClose(sqlite3_file *pFile){ + MemFile *p = (MemFile *)pFile; + if( p->bFreeOnClose ) sqlite3_free(p->aData); + return SQLITE_OK; +} + +/* +** Read data from an mem-file. +*/ +static int memRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + memcpy(zBuf, p->aData+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Write data to an mem-file. +*/ +static int memWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + MemFile *p = (MemFile *)pFile; + if( iOfst+iAmt>p->sz ){ + if( iOfst+iAmt>p->szMax ) return SQLITE_FULL; + if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); + p->sz = iOfst+iAmt; + } + memcpy(p->aData+iOfst, z, iAmt); + return SQLITE_OK; +} + +/* +** Truncate an mem-file. +*/ +static int memTruncate(sqlite3_file *pFile, sqlite_int64 size){ + MemFile *p = (MemFile *)pFile; + if( size>p->sz ){ + if( size>p->szMax ) return SQLITE_FULL; + memset(p->aData+p->sz, 0, size-p->sz); + } + p->sz = size; + return SQLITE_OK; +} + +/* +** Sync an mem-file. +*/ +static int memSync(sqlite3_file *pFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an mem-file. +*/ +static int memFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemFile *p = (MemFile *)pFile; + *pSize = p->sz; + return SQLITE_OK; +} + +/* +** Lock an mem-file. +*/ +static int memLock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Unlock an mem-file. +*/ +static int memUnlock(sqlite3_file *pFile, int eLock){ + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an mem-file. +*/ +static int memCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an mem-file. +*/ +static int memFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemFile *p = (MemFile *)pFile; + int rc = SQLITE_NOTFOUND; + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("mem(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + return rc; +} + +/* +** Return the sector-size in bytes for an mem-file. +*/ +static int memSectorSize(sqlite3_file *pFile){ + return 1024; +} + +/* +** Return the device characteristic flags supported by an mem-file. +*/ +static int memDeviceCharacteristics(sqlite3_file *pFile){ + return SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL; +} + +/* Create a shared memory file mapping */ +static int memShmMap( + sqlite3_file *pFile, + int iPg, + int pgsz, + int bExtend, + void volatile **pp +){ + return SQLITE_IOERR_SHMMAP; +} + +/* Perform locking on a shared-memory segment */ +static int memShmLock(sqlite3_file *pFile, int offset, int n, int flags){ + return SQLITE_IOERR_SHMLOCK; +} + +/* Memory barrier operation on shared memory */ +static void memShmBarrier(sqlite3_file *pFile){ + return; +} + +/* Unmap a shared memory segment */ +static int memShmUnmap(sqlite3_file *pFile, int deleteFlag){ + return SQLITE_OK; +} + +/* Fetch a page of a memory-mapped file */ +static int memFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemFile *p = (MemFile *)pFile; + *pp = (void*)(p->aData + iOfst); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFile, + int flags, + int *pOutFlags +){ + MemFile *p = (MemFile*)pFile; + memset(p, 0, sizeof(*p)); + if( (flags & SQLITE_OPEN_MAIN_DB)==0 ) return SQLITE_CANTOPEN; + p->aData = (unsigned char*)sqlite3_uri_int64(zName,"ptr",0); + if( p->aData==0 ) return SQLITE_CANTOPEN; + p->sz = sqlite3_uri_int64(zName,"sz",0); + if( p->sz<0 ) return SQLITE_CANTOPEN; + p->szMax = sqlite3_uri_int64(zName,"max",p->sz); + if( p->szMaxsz ) return SQLITE_CANTOPEN; + p->bFreeOnClose = sqlite3_uri_boolean(zName,"freeonclose",0); + pFile->pMethods = &mem_io_methods; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int memAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} + +static int memGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +#ifdef MEMVFS_TEST +/* +** memvfs_from_file(FILENAME, MAXSIZE) +** +** This an SQL function used to help in testing the memvfs VFS. The +** function reads the content of a file into memory and then returns +** a URI that can be handed to ATTACH to attach the memory buffer as +** a database. Example: +** +** ATTACH memvfs_from_file('test.db',1048576) AS inmem; +** +** The optional MAXSIZE argument gives the size of the memory allocation +** used to hold the database. If omitted, it defaults to the size of the +** file on disk. +*/ +#include +static void memvfsFromFileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + unsigned char *p; + sqlite3_int64 sz; + sqlite3_int64 szMax; + FILE *in; + const char *zFilename = (const char*)sqlite3_value_text(argv[0]); + char *zUri; + + if( zFilename==0 ) return; + in = fopen(zFilename, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + szMax = sz = ftell(in); + rewind(in); + if( argc>=2 ){ + szMax = sqlite3_value_int64(argv[1]); + if( szMaxzName,"memvfs")!=0 ) return; + rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); + if( rc ) return; + fwrite(p->aData, 1, (size_t)p->sz, out); + fclose(out); +} +#endif /* MEMVFS_TEST */ + +#ifdef MEMVFS_TEST +/* Called for each new database connection */ +static int memvfsRegister( + sqlite3 *db, + char **pzErrMsg, + const struct sqlite3_api_routines *pThunk +){ + sqlite3_create_function(db, "memvfs_from_file", 1, SQLITE_UTF8, 0, + memvfsFromFileFunc, 0, 0); + sqlite3_create_function(db, "memvfs_from_file", 2, SQLITE_UTF8, 0, + memvfsFromFileFunc, 0, 0); + sqlite3_create_function(db, "memvfs_to_file", 2, SQLITE_UTF8, 0, + memvfsToFileFunc, 0, 0); + return SQLITE_OK; +} +#endif /* MEMVFS_TEST */ + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +int sqlite3_memvfs_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + mem_vfs.pAppData = sqlite3_vfs_find(0); + if( mem_vfs.pAppData==0 ) return SQLITE_ERROR; + mem_vfs.szOsFile = sizeof(MemFile); + rc = sqlite3_vfs_register(&mem_vfs, 1); +#ifdef MEMVFS_TEST + if( rc==SQLITE_OK ){ + rc = sqlite3_auto_extension((void(*)(void))memvfsRegister); + } + if( rc==SQLITE_OK ){ + rc = memvfsRegister(db, pzErrMsg, pApi); + } +#endif + if( rc==SQLITE_OK ) rc = SQLITE_OK_LOAD_PERMANENTLY; + return rc; +} diff --git a/ext/misc/mmapwarm.c b/ext/misc/mmapwarm.c new file mode 100644 index 0000000..851f8b0 --- /dev/null +++ b/ext/misc/mmapwarm.c @@ -0,0 +1,108 @@ +/* +** 2017-09-18 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +*/ + +#include "sqlite3.h" + + +/* +** This function is used to touch each page of a mapping of a memory +** mapped SQLite database. Assuming that the system has sufficient free +** memory and supports sufficiently large mappings, this causes the OS +** to cache the entire database in main memory, making subsequent +** database accesses faster. +** +** If the second parameter to this function is not NULL, it is the name of +** the specific database to operate on (i.e. "main" or the name of an +** attached database). +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if the file is not memory-mapped, or if +** the mapping does not span the entire file. If an error does occur, a +** transaction may be left open on the database file. +** +** It is illegal to call this function when the database handle has an +** open transaction. SQLITE_MISUSE is returned in this case. +*/ +int sqlite3_mmap_warm(sqlite3 *db, const char *zDb){ + int rc = SQLITE_OK; + char *zSql = 0; + int pgsz = 0; + unsigned int nTotal = 0; + + if( 0==sqlite3_get_autocommit(db) ) return SQLITE_MISUSE; + + /* Open a read-only transaction on the file in question */ + zSql = sqlite3_mprintf("BEGIN; SELECT * FROM %s%q%ssqlite_schema", + (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "") + ); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + + /* Find the SQLite page size of the file */ + if( rc==SQLITE_OK ){ + zSql = sqlite3_mprintf("PRAGMA %s%q%spage_size", + (zDb ? "'" : ""), (zDb ? zDb : ""), (zDb ? "'." : "") + ); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_stmt *pPgsz = 0; + rc = sqlite3_prepare_v2(db, zSql, -1, &pPgsz, 0); + sqlite3_free(zSql); + if( rc==SQLITE_OK ){ + if( sqlite3_step(pPgsz)==SQLITE_ROW ){ + pgsz = sqlite3_column_int(pPgsz, 0); + } + rc = sqlite3_finalize(pPgsz); + } + if( rc==SQLITE_OK && pgsz==0 ){ + rc = SQLITE_ERROR; + } + } + } + + /* Touch each mmap'd page of the file */ + if( rc==SQLITE_OK ){ + int rc2; + sqlite3_file *pFd = 0; + rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_FILE_POINTER, &pFd); + if( rc==SQLITE_OK && pFd->pMethods->iVersion>=3 ){ + sqlite3_int64 iPg = 1; + sqlite3_io_methods const *p = pFd->pMethods; + while( 1 ){ + unsigned char *pMap; + rc = p->xFetch(pFd, pgsz*iPg, pgsz, (void**)&pMap); + if( rc!=SQLITE_OK || pMap==0 ) break; + + nTotal += (unsigned int)pMap[0]; + nTotal += (unsigned int)pMap[pgsz-1]; + + rc = p->xUnfetch(pFd, pgsz*iPg, (void*)pMap); + if( rc!=SQLITE_OK ) break; + iPg++; + } + sqlite3_log(SQLITE_OK, + "sqlite3_mmap_warm_cache: Warmed up %d pages of %s", iPg==1?0:iPg, + sqlite3_db_filename(db, zDb) + ); + } + + rc2 = sqlite3_exec(db, "END", 0, 0, 0); + if( rc==SQLITE_OK ) rc = rc2; + } + + (void)nTotal; + return rc; +} diff --git a/ext/misc/nextchar.c b/ext/misc/nextchar.c new file mode 100644 index 0000000..60fa3db --- /dev/null +++ b/ext/misc/nextchar.c @@ -0,0 +1,314 @@ +/* +** 2013-02-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the next_char(A,T,F,W,C) SQL function. +** +** The next_char(A,T,F,W,C) function finds all valid "next" characters for +** string A given the vocabulary in T.F. If the W value exists and is a +** non-empty string, then it is an SQL expression that limits the entries +** in T.F that will be considered. If C exists and is a non-empty string, +** then it is the name of the collating sequence to use for comparison. If +** +** Only the first three arguments are required. If the C parameter is +** omitted or is NULL or is an empty string, then the default collating +** sequence of T.F is used for comparision. If the W parameter is omitted +** or is NULL or is an empty string, then no filtering of the output is +** done. +** +** The T.F column should be indexed using collation C or else this routine +** will be quite slow. +** +** For example, suppose an application has a dictionary like this: +** +** CREATE TABLE dictionary(word TEXT UNIQUE); +** +** Further suppose that for user keypad entry, it is desired to disable +** (gray out) keys that are not valid as the next character. If the +** the user has previously entered (say) 'cha' then to find all allowed +** next characters (and thereby determine when keys should not be grayed +** out) run the following query: +** +** SELECT next_char('cha','dictionary','word'); +** +** IMPLEMENTATION NOTES: +** +** The next_char function is implemented using recursive SQL that makes +** use of the table name and column name as part of a query. If either +** the table name or column name are keywords or contain special characters, +** then they should be escaped. For example: +** +** SELECT next_char('cha','[dictionary]','[word]'); +** +** This also means that the table name can be a subquery: +** +** SELECT next_char('cha','(SELECT word AS w FROM dictionary)','w'); +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include + +/* +** A structure to hold context of the next_char() computation across +** nested function calls. +*/ +typedef struct nextCharContext nextCharContext; +struct nextCharContext { + sqlite3 *db; /* Database connection */ + sqlite3_stmt *pStmt; /* Prepared statement used to query */ + const unsigned char *zPrefix; /* Prefix to scan */ + int nPrefix; /* Size of zPrefix in bytes */ + int nAlloc; /* Space allocated to aResult */ + int nUsed; /* Space used in aResult */ + unsigned int *aResult; /* Array of next characters */ + int mallocFailed; /* True if malloc fails */ + int otherError; /* True for any other failure */ +}; + +/* +** Append a result character if the character is not already in the +** result. +*/ +static void nextCharAppend(nextCharContext *p, unsigned c){ + int i; + for(i=0; inUsed; i++){ + if( p->aResult[i]==c ) return; + } + if( p->nUsed+1 > p->nAlloc ){ + unsigned int *aNew; + int n = p->nAlloc*2 + 30; + aNew = sqlite3_realloc64(p->aResult, n*sizeof(unsigned int)); + if( aNew==0 ){ + p->mallocFailed = 1; + return; + }else{ + p->aResult = aNew; + p->nAlloc = n; + } + } + p->aResult[p->nUsed++] = c; +} + +/* +** Write a character into z[] as UTF8. Return the number of bytes needed +** to hold the character +*/ +static int writeUtf8(unsigned char *z, unsigned c){ + if( c<0x00080 ){ + z[0] = (unsigned char)(c&0xff); + return 1; + } + if( c<0x00800 ){ + z[0] = 0xC0 + (unsigned char)((c>>6)&0x1F); + z[1] = 0x80 + (unsigned char)(c & 0x3F); + return 2; + } + if( c<0x10000 ){ + z[0] = 0xE0 + (unsigned char)((c>>12)&0x0F); + z[1] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[2] = 0x80 + (unsigned char)(c & 0x3F); + return 3; + } + z[0] = 0xF0 + (unsigned char)((c>>18) & 0x07); + z[1] = 0x80 + (unsigned char)((c>>12) & 0x3F); + z[2] = 0x80 + (unsigned char)((c>>6) & 0x3F); + z[3] = 0x80 + (unsigned char)(c & 0x3F); + return 4; +} + +/* +** Read a UTF8 character out of z[] and write it into *pOut. Return +** the number of bytes in z[] that were used to construct the character. +*/ +static int readUtf8(const unsigned char *z, unsigned *pOut){ + static const unsigned char validBits[] = { + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, + 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e, 0x1f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x00, 0x01, 0x02, 0x03, 0x00, 0x01, 0x00, 0x00, + }; + unsigned c = z[0]; + if( c<0xc0 ){ + *pOut = c; + return 1; + }else{ + int n = 1; + c = validBits[c-0xc0]; + while( (z[n] & 0xc0)==0x80 ){ + c = (c<<6) + (0x3f & z[n++]); + } + if( c<0x80 || (c&0xFFFFF800)==0xD800 || (c&0xFFFFFFFE)==0xFFFE ){ + c = 0xFFFD; + } + *pOut = c; + return n; + } +} + +/* +** The nextCharContext structure has been set up. Add all "next" characters +** to the result set. +*/ +static void findNextChars(nextCharContext *p){ + unsigned cPrev = 0; + unsigned char zPrev[8]; + int n, rc; + + for(;;){ + sqlite3_bind_text(p->pStmt, 1, (char*)p->zPrefix, p->nPrefix, + SQLITE_STATIC); + n = writeUtf8(zPrev, cPrev+1); + sqlite3_bind_text(p->pStmt, 2, (char*)zPrev, n, SQLITE_STATIC); + rc = sqlite3_step(p->pStmt); + if( rc==SQLITE_DONE ){ + sqlite3_reset(p->pStmt); + return; + }else if( rc!=SQLITE_ROW ){ + p->otherError = rc; + return; + }else{ + const unsigned char *zOut = sqlite3_column_text(p->pStmt, 0); + unsigned cNext; + n = readUtf8(zOut+p->nPrefix, &cNext); + sqlite3_reset(p->pStmt); + nextCharAppend(p, cNext); + cPrev = cNext; + if( p->mallocFailed ) return; + } + } +} + + +/* +** next_character(A,T,F,W) +** +** Return a string composted of all next possible characters after +** A for elements of T.F. If W is supplied, then it is an SQL expression +** that limits the elements in T.F that are considered. +*/ +static void nextCharFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + nextCharContext c; + const unsigned char *zTable = sqlite3_value_text(argv[1]); + const unsigned char *zField = sqlite3_value_text(argv[2]); + const unsigned char *zWhere; + const unsigned char *zCollName; + char *zWhereClause = 0; + char *zColl = 0; + char *zSql; + int rc; + + memset(&c, 0, sizeof(c)); + c.db = sqlite3_context_db_handle(context); + c.zPrefix = sqlite3_value_text(argv[0]); + c.nPrefix = sqlite3_value_bytes(argv[0]); + if( zTable==0 || zField==0 || c.zPrefix==0 ) return; + if( argc>=4 + && (zWhere = sqlite3_value_text(argv[3]))!=0 + && zWhere[0]!=0 + ){ + zWhereClause = sqlite3_mprintf("AND (%s)", zWhere); + if( zWhereClause==0 ){ + sqlite3_result_error_nomem(context); + return; + } + }else{ + zWhereClause = ""; + } + if( argc>=5 + && (zCollName = sqlite3_value_text(argv[4]))!=0 + && zCollName[0]!=0 + ){ + zColl = sqlite3_mprintf("collate \"%w\"", zCollName); + if( zColl==0 ){ + sqlite3_result_error_nomem(context); + if( zWhereClause[0] ) sqlite3_free(zWhereClause); + return; + } + }else{ + zColl = ""; + } + zSql = sqlite3_mprintf( + "SELECT %s FROM %s" + " WHERE %s>=(?1 || ?2) %s" + " AND %s<=(?1 || char(1114111)) %s" /* 1114111 == 0x10ffff */ + " %s" + " ORDER BY 1 %s ASC LIMIT 1", + zField, zTable, zField, zColl, zField, zColl, zWhereClause, zColl + ); + if( zWhereClause[0] ) sqlite3_free(zWhereClause); + if( zColl[0] ) sqlite3_free(zColl); + if( zSql==0 ){ + sqlite3_result_error_nomem(context); + return; + } + + rc = sqlite3_prepare_v2(c.db, zSql, -1, &c.pStmt, 0); + sqlite3_free(zSql); + if( rc ){ + sqlite3_result_error(context, sqlite3_errmsg(c.db), -1); + return; + } + findNextChars(&c); + if( c.mallocFailed ){ + sqlite3_result_error_nomem(context); + }else{ + unsigned char *pRes; + pRes = sqlite3_malloc64( c.nUsed*4 + 1 ); + if( pRes==0 ){ + sqlite3_result_error_nomem(context); + }else{ + int i; + int n = 0; + for(i=0; i +#include + +/* +** Implementation of the noop() function. +** +** The function returns its argument, unchanged. +*/ +static void noopfunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + sqlite3_result_value(context, argv[0]); +} + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_noop_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "noop", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "noop_i", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_INNOCUOUS, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "noop_do", 1, + SQLITE_UTF8 | SQLITE_DETERMINISTIC | SQLITE_DIRECTONLY, + 0, noopfunc, 0, 0); + if( rc ) return rc; + rc = sqlite3_create_function(db, "noop_nd", 1, + SQLITE_UTF8, + 0, noopfunc, 0, 0); + return rc; +} diff --git a/ext/misc/normalize.c b/ext/misc/normalize.c new file mode 100644 index 0000000..08d7733 --- /dev/null +++ b/ext/misc/normalize.c @@ -0,0 +1,716 @@ +/* +** 2018-01-08 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the sqlite3_normalize() function. +** +** char *sqlite3_normalize(const char *zSql); +** +** This function takes an SQL string as input and returns a "normalized" +** version of that string in memory obtained from sqlite3_malloc64(). The +** caller is responsible for ensuring that the returned memory is freed. +** +** If a memory allocation error occurs, this routine returns NULL. +** +** The normalization consists of the following transformations: +** +** (1) Convert every literal (string, blob literal, numeric constant, +** or "NULL" constant) into a ? +** +** (2) Remove all superfluous whitespace, including comments. Change +** all required whitespace to a single space character. +** +** (3) Lowercase all ASCII characters. +** +** (4) If an IN or NOT IN operator is followed by a list of 1 or more +** values, convert that list into "(?,?,?)". +** +** The purpose of normalization is two-fold: +** +** (1) Sanitize queries by removing potentially private or sensitive +** information contained in literals. +** +** (2) Identify structurally identical queries by comparing their +** normalized forms. +** +** Command-Line Utility +** -------------------- +** +** This file also contains code for a command-line utility that converts +** SQL queries in text files into their normalized forms. To build the +** command-line program, compile this file with -DSQLITE_NORMALIZE_CLI +** and link it against the SQLite library. +*/ +#include +#include + +/* +** Implementation note: +** +** Much of the tokenizer logic is copied out of the tokenize.c source file +** of SQLite. That logic could be simplified for this particular application, +** but that would impose a risk of introducing subtle errors. It is best to +** keep the code as close to the original as possible. +** +** The tokenize code is in sync with the SQLite core as of 2018-01-08. +** Any future changes to the core tokenizer might require corresponding +** adjustments to the tokenizer logic in this module. +*/ + + +/* Character classes for tokenizing +** +** In the sqlite3GetToken() function, a switch() on aiClass[c] is implemented +** using a lookup table, whereas a switch() directly on c uses a binary search. +** The lookup table is much faster. To maximize speed, and to ensure that +** a lookup table is used, all of the classes need to be small integers and +** all of them need to be used within the switch. +*/ +#define CC_X 0 /* The letter 'x', or start of BLOB literal */ +#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ +#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_DIGIT 3 /* Digits */ +#define CC_DOLLAR 4 /* '$' */ +#define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ +#define CC_VARNUM 6 /* '?'. Numeric SQL variables */ +#define CC_SPACE 7 /* Space characters */ +#define CC_QUOTE 8 /* '"', '\'', or '`'. String literals, quoted ids */ +#define CC_QUOTE2 9 /* '['. [...] style quoted ids */ +#define CC_PIPE 10 /* '|'. Bitwise OR or concatenate */ +#define CC_MINUS 11 /* '-'. Minus or SQL-style comment */ +#define CC_LT 12 /* '<'. Part of < or <= or <> */ +#define CC_GT 13 /* '>'. Part of > or >= */ +#define CC_EQ 14 /* '='. Part of = or == */ +#define CC_BANG 15 /* '!'. Part of != */ +#define CC_SLASH 16 /* '/'. / or c-style comment */ +#define CC_LP 17 /* '(' */ +#define CC_RP 18 /* ')' */ +#define CC_SEMI 19 /* ';' */ +#define CC_PLUS 20 /* '+' */ +#define CC_STAR 21 /* '*' */ +#define CC_PERCENT 22 /* '%' */ +#define CC_COMMA 23 /* ',' */ +#define CC_AND 24 /* '&' */ +#define CC_TILDA 25 /* '~' */ +#define CC_DOT 26 /* '.' */ +#define CC_ILLEGAL 27 /* Illegal character */ + +static const unsigned char aiClass[] = { +/* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ +/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, +/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, +/* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, +/* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, +/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, +/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +}; + +/* An array to map all upper-case characters into their corresponding +** lower-case character. +** +** SQLite only considers US-ASCII (or EBCDIC) characters. We do not +** handle case conversions for the UTF character set since the tables +** involved are nearly as big or bigger than SQLite itself. +*/ +static const unsigned char sqlite3UpperToLower[] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, + 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, + 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, + 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 97, 98, 99,100,101,102,103, + 104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121, + 122, 91, 92, 93, 94, 95, 96, 97, 98, 99,100,101,102,103,104,105,106,107, + 108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125, + 126,127,128,129,130,131,132,133,134,135,136,137,138,139,140,141,142,143, + 144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161, + 162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179, + 180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197, + 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, + 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, + 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, + 252,253,254,255 +}; + +/* +** The following 256 byte lookup table is used to support SQLites built-in +** equivalents to the following standard library functions: +** +** isspace() 0x01 +** isalpha() 0x02 +** isdigit() 0x04 +** isalnum() 0x06 +** isxdigit() 0x08 +** toupper() 0x20 +** SQLite identifier character 0x40 +** Quote character 0x80 +** +** Bit 0x20 is set if the mapped character requires translation to upper +** case. i.e. if the character is a lower-case ASCII character. +** If x is a lower-case ASCII character, then its upper-case equivalent +** is (x - 0x20). Therefore toupper() can be implemented as: +** +** (x & ~(map[x]&0x20)) +** +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] +** array. tolower() is used more often than toupper() by SQLite. +** +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an +** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any +** non-ASCII UTF character. Hence the test for whether or not a character is +** part of an identifier is 0x46. +*/ +static const unsigned char sqlite3CtypeMap[256] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ + 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 10..17 ........ */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 18..1f ........ */ + 0x01, 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x80, /* 20..27 !"#$%&' */ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 28..2f ()*+,-./ */ + 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, 0x0c, /* 30..37 01234567 */ + 0x0c, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 38..3f 89:;<=>? */ + + 0x00, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x0a, 0x02, /* 40..47 @ABCDEFG */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 48..4f HIJKLMNO */ + 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02, /* 50..57 PQRSTUVW */ + 0x02, 0x02, 0x02, 0x80, 0x00, 0x00, 0x00, 0x40, /* 58..5f XYZ[\]^_ */ + 0x80, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x2a, 0x22, /* 60..67 `abcdefg */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 68..6f hijklmno */ + 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, 0x22, /* 70..77 pqrstuvw */ + 0x22, 0x22, 0x22, 0x00, 0x00, 0x00, 0x00, 0x00, /* 78..7f xyz{|}~. */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 80..87 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 88..8f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 90..97 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* 98..9f ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a0..a7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* a8..af ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b0..b7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* b8..bf ........ */ + + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c0..c7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* c8..cf ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d0..d7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* d8..df ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e0..e7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* e8..ef ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ + 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ +}; +#define sqlite3Toupper(x) ((x)&~(sqlite3CtypeMap[(unsigned char)(x)]&0x20)) +#define sqlite3Isspace(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x01) +#define sqlite3Isalnum(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x06) +#define sqlite3Isalpha(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x02) +#define sqlite3Isdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x04) +#define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) +#define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) +#define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) + + +/* +** If X is a character that can be used in an identifier then +** IdChar(X) will be true. Otherwise it is false. +** +** For ASCII, any character with the high-order bit set is +** allowed in an identifier. For 7-bit characters, +** sqlite3IsIdChar[X] must be 1. +** +** For EBCDIC, the rules are more complex but have the same +** end result. +** +** Ticket #1066. the SQL standard does not allow '$' in the +** middle of identifiers. But many SQL implementations do. +** SQLite will allow '$' in identifiers for compatibility. +** But the feature is undocumented. +*/ +#define IdChar(C) ((sqlite3CtypeMap[(unsigned char)C]&0x46)!=0) + +/* +** Ignore testcase() macros +*/ +#define testcase(X) + +/* +** Token values +*/ +#define TK_SPACE 0 +#define TK_NAME 1 +#define TK_LITERAL 2 +#define TK_PUNCT 3 +#define TK_ERROR 4 + +#define TK_MINUS TK_PUNCT +#define TK_LP TK_PUNCT +#define TK_RP TK_PUNCT +#define TK_SEMI TK_PUNCT +#define TK_PLUS TK_PUNCT +#define TK_STAR TK_PUNCT +#define TK_SLASH TK_PUNCT +#define TK_REM TK_PUNCT +#define TK_EQ TK_PUNCT +#define TK_LE TK_PUNCT +#define TK_NE TK_PUNCT +#define TK_LSHIFT TK_PUNCT +#define TK_LT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_RSHIFT TK_PUNCT +#define TK_GT TK_PUNCT +#define TK_GE TK_PUNCT +#define TK_BITOR TK_PUNCT +#define TK_CONCAT TK_PUNCT +#define TK_COMMA TK_PUNCT +#define TK_BITAND TK_PUNCT +#define TK_BITNOT TK_PUNCT +#define TK_STRING TK_LITERAL +#define TK_ID TK_NAME +#define TK_ILLEGAL TK_ERROR +#define TK_DOT TK_PUNCT +#define TK_INTEGER TK_LITERAL +#define TK_FLOAT TK_LITERAL +#define TK_VARIABLE TK_LITERAL +#define TK_BLOB TK_LITERAL + +/* Disable nuisence warnings about case fall-through */ +#if !defined(deliberate_fall_through) && defined(__GCC__) && __GCC__>=7 +# define deliberate_fall_through __attribute__((fallthrough)); +#else +# define deliberate_fall_through +#endif + +/* +** Return the length (in bytes) of the token that begins at z[0]. +** Store the token type in *tokenType before returning. +*/ +static int sqlite3GetToken(const unsigned char *z, int *tokenType){ + int i, c; + switch( aiClass[*z] ){ /* Switch on the character-class of the first byte + ** of the token. See the comment on the CC_ defines + ** above. */ + case CC_SPACE: { + for(i=1; sqlite3Isspace(z[i]); i++){} + *tokenType = TK_SPACE; + return i; + } + case CC_MINUS: { + if( z[1]=='-' ){ + for(i=2; (c=z[i])!=0 && c!='\n'; i++){} + *tokenType = TK_SPACE; + return i; + } + *tokenType = TK_MINUS; + return 1; + } + case CC_LP: { + *tokenType = TK_LP; + return 1; + } + case CC_RP: { + *tokenType = TK_RP; + return 1; + } + case CC_SEMI: { + *tokenType = TK_SEMI; + return 1; + } + case CC_PLUS: { + *tokenType = TK_PLUS; + return 1; + } + case CC_STAR: { + *tokenType = TK_STAR; + return 1; + } + case CC_SLASH: { + if( z[1]!='*' || z[2]==0 ){ + *tokenType = TK_SLASH; + return 1; + } + for(i=3, c=z[2]; (c!='*' || z[i]!='/') && (c=z[i])!=0; i++){} + if( c ) i++; + *tokenType = TK_SPACE; + return i; + } + case CC_PERCENT: { + *tokenType = TK_REM; + return 1; + } + case CC_EQ: { + *tokenType = TK_EQ; + return 1 + (z[1]=='='); + } + case CC_LT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_LE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_NE; + return 2; + }else if( c=='<' ){ + *tokenType = TK_LSHIFT; + return 2; + }else{ + *tokenType = TK_LT; + return 1; + } + } + case CC_GT: { + if( (c=z[1])=='=' ){ + *tokenType = TK_GE; + return 2; + }else if( c=='>' ){ + *tokenType = TK_RSHIFT; + return 2; + }else{ + *tokenType = TK_GT; + return 1; + } + } + case CC_BANG: { + if( z[1]!='=' ){ + *tokenType = TK_ILLEGAL; + return 1; + }else{ + *tokenType = TK_NE; + return 2; + } + } + case CC_PIPE: { + if( z[1]!='|' ){ + *tokenType = TK_BITOR; + return 1; + }else{ + *tokenType = TK_CONCAT; + return 2; + } + } + case CC_COMMA: { + *tokenType = TK_COMMA; + return 1; + } + case CC_AND: { + *tokenType = TK_BITAND; + return 1; + } + case CC_TILDA: { + *tokenType = TK_BITNOT; + return 1; + } + case CC_QUOTE: { + int delim = z[0]; + testcase( delim=='`' ); + testcase( delim=='\'' ); + testcase( delim=='"' ); + for(i=1; (c=z[i])!=0; i++){ + if( c==delim ){ + if( z[i+1]==delim ){ + i++; + }else{ + break; + } + } + } + if( c=='\'' ){ + *tokenType = TK_STRING; + return i+1; + }else if( c!=0 ){ + *tokenType = TK_ID; + return i+1; + }else{ + *tokenType = TK_ILLEGAL; + return i; + } + } + case CC_DOT: { + if( !sqlite3Isdigit(z[1]) ){ + *tokenType = TK_DOT; + return 1; + } + /* If the next character is a digit, this is a floating point + ** number that begins with ".". Fall thru into the next case */ + /* no break */ deliberate_fall_through + } + case CC_DIGIT: { + *tokenType = TK_INTEGER; + if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ + for(i=3; sqlite3Isxdigit(z[i]); i++){} + return i; + } + for(i=0; sqlite3Isdigit(z[i]); i++){} + if( z[i]=='.' ){ + i++; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + i += 2; + while( sqlite3Isdigit(z[i]) ){ i++; } + *tokenType = TK_FLOAT; + } + while( IdChar(z[i]) ){ + *tokenType = TK_ILLEGAL; + i++; + } + return i; + } + case CC_QUOTE2: { + for(i=1, c=z[0]; c!=']' && (c=z[i])!=0; i++){} + *tokenType = c==']' ? TK_ID : TK_ILLEGAL; + return i; + } + case CC_VARNUM: { + *tokenType = TK_VARIABLE; + for(i=1; sqlite3Isdigit(z[i]); i++){} + return i; + } + case CC_DOLLAR: + case CC_VARALPHA: { + int n = 0; + testcase( z[0]=='$' ); testcase( z[0]=='@' ); + testcase( z[0]==':' ); testcase( z[0]=='#' ); + *tokenType = TK_VARIABLE; + for(i=1; (c=z[i])!=0; i++){ + if( IdChar(c) ){ + n++; + }else if( c=='(' && n>0 ){ + do{ + i++; + }while( (c=z[i])!=0 && !sqlite3Isspace(c) && c!=')' ); + if( c==')' ){ + i++; + }else{ + *tokenType = TK_ILLEGAL; + } + break; + }else if( c==':' && z[i+1]==':' ){ + i++; + }else{ + break; + } + } + if( n==0 ) *tokenType = TK_ILLEGAL; + return i; + } + case CC_KYWD: { + for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + if( IdChar(z[i]) ){ + /* This token started out using characters that can appear in keywords, + ** but z[i] is a character not allowed within keywords, so this must + ** be an identifier instead */ + i++; + break; + } + *tokenType = TK_ID; + return i; + } + case CC_X: { + testcase( z[0]=='x' ); testcase( z[0]=='X' ); + if( z[1]=='\'' ){ + *tokenType = TK_BLOB; + for(i=2; sqlite3Isxdigit(z[i]); i++){} + if( z[i]!='\'' || i%2 ){ + *tokenType = TK_ILLEGAL; + while( z[i] && z[i]!='\'' ){ i++; } + } + if( z[i] ) i++; + return i; + } + /* If it is not a BLOB literal, then it must be an ID, since no + ** SQL keywords start with the letter 'x'. Fall through */ + /* no break */ deliberate_fall_through + } + case CC_ID: { + i = 1; + break; + } + default: { + *tokenType = TK_ILLEGAL; + return 1; + } + } + while( IdChar(z[i]) ){ i++; } + *tokenType = TK_ID; + return i; +} + +char *sqlite3_normalize(const char *zSql){ + char *z; /* The output string */ + sqlite3_int64 nZ; /* Size of the output string in bytes */ + sqlite3_int64 nSql; /* Size of the input string in bytes */ + int i; /* Next character to read from zSql[] */ + int j; /* Next slot to fill in on z[] */ + int tokenType; /* Type of the next token */ + int n; /* Size of the next token */ + int k; /* Loop counter */ + + nSql = strlen(zSql); + nZ = nSql; + z = sqlite3_malloc64( nZ+2 ); + if( z==0 ) return 0; + for(i=j=0; zSql[i]; i += n){ + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_ERROR: { + sqlite3_free(z); + return 0; + } + case TK_LITERAL: { + z[j++] = '?'; + break; + } + case TK_PUNCT: + case TK_NAME: { + if( n==4 && sqlite3_strnicmp(zSql+i,"NULL",4)==0 ){ + if( (j>=3 && strncmp(z+j-2,"is",2)==0 && !IdChar(z[j-3])) + || (j>=4 && strncmp(z+j-3,"not",3)==0 && !IdChar(z[j-4])) + ){ + /* NULL is a keyword in this case, not a literal value */ + }else{ + /* Here the NULL is a literal value */ + z[j++] = '?'; + break; + } + } + if( j>0 && IdChar(z[j-1]) && IdChar(zSql[i]) ) z[j++] = ' '; + for(k=0; k0 && z[j-1]==' ' ){ j--; } + if( j>0 && z[j-1]!=';' ){ z[j++] = ';'; } + z[j] = 0; + + /* Make a second pass converting "in(...)" where the "..." is not a + ** SELECT statement into "in(?,?,?)" */ + for(i=0; i5 ){ + memmove(z+n+5, z+n+k, j-(n+k)); + } + j = j-k+5; + z[j] = 0; + memcpy(z+n, "?,?,?", 5); + } + return z; +} + +/* +** For testing purposes, or to build a stand-alone SQL normalizer program, +** compile this one source file with the -DSQLITE_NORMALIZE_CLI and link +** it against any SQLite library. The resulting command-line program will +** run sqlite3_normalize() over the text of all files named on the command- +** line and show the result on standard output. +*/ +#ifdef SQLITE_NORMALIZE_CLI +#include +#include + +/* +** Break zIn up into separate SQL statements and run sqlite3_normalize() +** on each one. Print the result of each run. +*/ +static void normalizeFile(char *zIn){ + int i; + if( zIn==0 ) return; + for(i=0; zIn[i]; i++){ + char cSaved; + if( zIn[i]!=';' ) continue; + cSaved = zIn[i+1]; + zIn[i+1] = 0; + if( sqlite3_complete(zIn) ){ + char *zOut = sqlite3_normalize(zIn); + if( zOut ){ + printf("%s\n", zOut); + sqlite3_free(zOut); + }else{ + fprintf(stderr, "ERROR: %s\n", zIn); + } + zIn[i+1] = cSaved; + zIn += i+1; + i = -1; + }else{ + zIn[i+1] = cSaved; + } + } +} + +/* +** The main routine for "sql_normalize". Read files named on the +** command-line and run the text of each through sqlite3_normalize(). +*/ +int main(int argc, char **argv){ + int i; + FILE *in; + char *zBuf = 0; + sqlite3_int64 sz, got; + + for(i=1; i +#include +#include + +/* The original page cache routines */ +static sqlite3_pcache_methods2 pcacheBase; +static FILE *pcachetraceOut; + +/* Methods that trace pcache activity */ +static int pcachetraceInit(void *pArg){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p)\n", pArg); + } + nRes = pcacheBase.xInit(pArg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xInit(%p) -> %d\n", pArg, nRes); + } + return nRes; +} +static void pcachetraceShutdown(void *pArg){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShutdown(%p)\n", pArg); + } + pcacheBase.xShutdown(pArg); +} +static sqlite3_pcache *pcachetraceCreate(int szPage, int szExtra, int bPurge){ + sqlite3_pcache *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d)\n", + szPage, szExtra, bPurge); + } + pRes = pcacheBase.xCreate(szPage, szExtra, bPurge); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCreate(%d,%d,%d) -> %p\n", + szPage, szExtra, bPurge, pRes); + } + return pRes; +} +static void pcachetraceCachesize(sqlite3_pcache *p, int nCachesize){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xCachesize(%p, %d)\n", p, nCachesize); + } + pcacheBase.xCachesize(p, nCachesize); +} +static int pcachetracePagecount(sqlite3_pcache *p){ + int nRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p)\n", p); + } + nRes = pcacheBase.xPagecount(p); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xPagecount(%p) -> %d\n", p, nRes); + } + return nRes; +} +static sqlite3_pcache_page *pcachetraceFetch( + sqlite3_pcache *p, + unsigned key, + int crFg +){ + sqlite3_pcache_page *pRes; + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d)\n", p, key, crFg); + } + pRes = pcacheBase.xFetch(p, key, crFg); + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xFetch(%p,%u,%d) -> %p\n", + p, key, crFg, pRes); + } + return pRes; +} +static void pcachetraceUnpin( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + int bDiscard +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xUnpin(%p, %p, %d)\n", + p, pPg, bDiscard); + } + pcacheBase.xUnpin(p, pPg, bDiscard); +} +static void pcachetraceRekey( + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, + unsigned oldKey, + unsigned newKey +){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xRekey(%p, %p, %u, %u)\n", + p, pPg, oldKey, newKey); + } + pcacheBase.xRekey(p, pPg, oldKey, newKey); +} +static void pcachetraceTruncate(sqlite3_pcache *p, unsigned n){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xTruncate(%p, %u)\n", p, n); + } + pcacheBase.xTruncate(p, n); +} +static void pcachetraceDestroy(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xDestroy(%p)\n", p); + } + pcacheBase.xDestroy(p); +} +static void pcachetraceShrink(sqlite3_pcache *p){ + if( pcachetraceOut ){ + fprintf(pcachetraceOut, "PCACHETRACE: xShrink(%p)\n", p); + } + pcacheBase.xShrink(p); +} + +/* The substitute pcache methods */ +static sqlite3_pcache_methods2 ersaztPcacheMethods = { + 0, + 0, + pcachetraceInit, + pcachetraceShutdown, + pcachetraceCreate, + pcachetraceCachesize, + pcachetracePagecount, + pcachetraceFetch, + pcachetraceUnpin, + pcachetraceRekey, + pcachetraceTruncate, + pcachetraceDestroy, + pcachetraceShrink +}; + +/* Begin tracing memory allocations to out. */ +int sqlite3PcacheTraceActivate(FILE *out){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch==0 ){ + rc = sqlite3_config(SQLITE_CONFIG_GETPCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &ersaztPcacheMethods); + } + } + pcachetraceOut = out; + return rc; +} + +/* Deactivate memory tracing */ +int sqlite3PcacheTraceDeactivate(void){ + int rc = SQLITE_OK; + if( pcacheBase.xFetch!=0 ){ + rc = sqlite3_config(SQLITE_CONFIG_PCACHE2, &pcacheBase); + if( rc==SQLITE_OK ){ + memset(&pcacheBase, 0, sizeof(pcacheBase)); + } + } + pcachetraceOut = 0; + return rc; +} diff --git a/ext/misc/percentile.c b/ext/misc/percentile.c new file mode 100644 index 0000000..d83bc5b --- /dev/null +++ b/ext/misc/percentile.c @@ -0,0 +1,220 @@ +/* +** 2013-05-28 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains code to implement the percentile(Y,P) SQL function +** as described below: +** +** (1) The percentile(Y,P) function is an aggregate function taking +** exactly two arguments. +** +** (2) If the P argument to percentile(Y,P) is not the same for every +** row in the aggregate then an error is thrown. The word "same" +** in the previous sentence means that the value differ by less +** than 0.001. +** +** (3) If the P argument to percentile(Y,P) evaluates to anything other +** than a number in the range of 0.0 to 100.0 inclusive then an +** error is thrown. +** +** (4) If any Y argument to percentile(Y,P) evaluates to a value that +** is not NULL and is not numeric then an error is thrown. +** +** (5) If any Y argument to percentile(Y,P) evaluates to plus or minus +** infinity then an error is thrown. (SQLite always interprets NaN +** values as NULL.) +** +** (6) Both Y and P in percentile(Y,P) can be arbitrary expressions, +** including CASE WHEN expressions. +** +** (7) The percentile(Y,P) aggregate is able to handle inputs of at least +** one million (1,000,000) rows. +** +** (8) If there are no non-NULL values for Y, then percentile(Y,P) +** returns NULL. +** +** (9) If there is exactly one non-NULL value for Y, the percentile(Y,P) +** returns the one Y value. +** +** (10) If there N non-NULL values of Y where N is two or more and +** the Y values are ordered from least to greatest and a graph is +** drawn from 0 to N-1 such that the height of the graph at J is +** the J-th Y value and such that straight lines are drawn between +** adjacent Y values, then the percentile(Y,P) function returns +** the height of the graph at P*(N-1)/100. +** +** (11) The percentile(Y,P) function always returns either a floating +** point number or NULL. +** +** (12) The percentile(Y,P) is implemented as a single C99 source-code +** file that compiles into a shared-library or DLL that can be loaded +** into SQLite using the sqlite3_load_extension() interface. +*/ +#include "sqlite3ext.h" +SQLITE_EXTENSION_INIT1 +#include +#include +#include + +/* The following object is the session context for a single percentile() +** function. We have to remember all input Y values until the very end. +** Those values are accumulated in the Percentile.a[] array. +*/ +typedef struct Percentile Percentile; +struct Percentile { + unsigned nAlloc; /* Number of slots allocated for a[] */ + unsigned nUsed; /* Number of slots actually used in a[] */ + double rPct; /* 1.0 more than the value for P */ + double *a; /* Array of Y values */ +}; + +/* +** Return TRUE if the input floating-point number is an infinity. +*/ +static int isInfinity(double r){ + sqlite3_uint64 u; + assert( sizeof(u)==sizeof(r) ); + memcpy(&u, &r, sizeof(u)); + return ((u>>52)&0x7ff)==0x7ff; +} + +/* +** Return TRUE if two doubles differ by 0.001 or less +*/ +static int sameValue(double a, double b){ + a -= b; + return a>=-0.001 && a<=0.001; +} + +/* +** The "step" function for percentile(Y,P) is called once for each +** input row. +*/ +static void percentStep(sqlite3_context *pCtx, int argc, sqlite3_value **argv){ + Percentile *p; + double rPct; + int eType; + double y; + assert( argc==2 ); + + /* Requirement 3: P must be a number between 0 and 100 */ + eType = sqlite3_value_numeric_type(argv[1]); + rPct = sqlite3_value_double(argv[1]); + if( (eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT) + || rPct<0.0 || rPct>100.0 ){ + sqlite3_result_error(pCtx, "2nd argument to percentile() is not " + "a number between 0.0 and 100.0", -1); + return; + } + + /* Allocate the session context. */ + p = (Percentile*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p==0 ) return; + + /* Remember the P value. Throw an error if the P value is different + ** from any prior row, per Requirement (2). */ + if( p->rPct==0.0 ){ + p->rPct = rPct+1.0; + }else if( !sameValue(p->rPct,rPct+1.0) ){ + sqlite3_result_error(pCtx, "2nd argument to percentile() is not the " + "same for all input rows", -1); + return; + } + + /* Ignore rows for which Y is NULL */ + eType = sqlite3_value_type(argv[0]); + if( eType==SQLITE_NULL ) return; + + /* If not NULL, then Y must be numeric. Otherwise throw an error. + ** Requirement 4 */ + if( eType!=SQLITE_INTEGER && eType!=SQLITE_FLOAT ){ + sqlite3_result_error(pCtx, "1st argument to percentile() is not " + "numeric", -1); + return; + } + + /* Throw an error if the Y value is infinity or NaN */ + y = sqlite3_value_double(argv[0]); + if( isInfinity(y) ){ + sqlite3_result_error(pCtx, "Inf input to percentile()", -1); + return; + } + + /* Allocate and store the Y */ + if( p->nUsed>=p->nAlloc ){ + unsigned n = p->nAlloc*2 + 250; + double *a = sqlite3_realloc64(p->a, sizeof(double)*n); + if( a==0 ){ + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); + sqlite3_result_error_nomem(pCtx); + return; + } + p->nAlloc = n; + p->a = a; + } + p->a[p->nUsed++] = y; +} + +/* +** Compare to doubles for sorting using qsort() +*/ +static int SQLITE_CDECL doubleCmp(const void *pA, const void *pB){ + double a = *(double*)pA; + double b = *(double*)pB; + if( a==b ) return 0; + if( aa==0 ) return; + if( p->nUsed ){ + qsort(p->a, p->nUsed, sizeof(double), doubleCmp); + ix = (p->rPct-1.0)*(p->nUsed-1)*0.01; + i1 = (unsigned)ix; + i2 = ix==(double)i1 || i1==p->nUsed-1 ? i1 : i1+1; + v1 = p->a[i1]; + v2 = p->a[i2]; + vx = v1 + (v2-v1)*(ix-i1); + sqlite3_result_double(pCtx, vx); + } + sqlite3_free(p->a); + memset(p, 0, sizeof(*p)); +} + + +#ifdef _WIN32 +__declspec(dllexport) +#endif +int sqlite3_percentile_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); + (void)pzErrMsg; /* Unused parameter */ + rc = sqlite3_create_function(db, "percentile", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS, 0, + 0, percentStep, percentFinal); + return rc; +} diff --git a/ext/misc/prefixes.c b/ext/misc/prefixes.c new file mode 100644 index 0000000..e6517e7 --- /dev/null +++ b/ext/misc/prefixes.c @@ -0,0 +1,321 @@ +/* +** 2018-04-19 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements a table-valued function: +** +** prefixes('abcdefg') +** +** The function has a single (non-HIDDEN) column named prefix that takes +** on all prefixes of the string in its argument, including an empty string +** and the input string itself. The order of prefixes is from longest +** to shortest. +*/ +#if !defined(SQLITE_CORE) || !defined(SQLITE_OMIT_VIRTUALTABLE) +#if !defined(SQLITEINT_H) +#include "sqlite3ext.h" +#endif +SQLITE_EXTENSION_INIT1 +#include +#include + +/* prefixes_vtab is a subclass of sqlite3_vtab which is +** underlying representation of the virtual table +*/ +typedef struct prefixes_vtab prefixes_vtab; +struct prefixes_vtab { + sqlite3_vtab base; /* Base class - must be first */ + /* No additional fields are necessary */ +}; + +/* prefixes_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct prefixes_cursor prefixes_cursor; +struct prefixes_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_int64 iRowid; /* The rowid */ + char *zStr; /* Original string to be prefixed */ + int nStr; /* Length of the string in bytes */ +}; + +/* +** The prefixesConnect() method is invoked to create a new +** template virtual table. +** +** Think of this routine as the constructor for prefixes_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the prefixes_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against the virtual table will look like. +*/ +static int prefixesConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + prefixes_vtab *pNew; + int rc; + + rc = sqlite3_declare_vtab(db, + "CREATE TABLE prefixes(prefix TEXT, original_string TEXT HIDDEN)" + ); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + return rc; +} + +/* +** This method is the destructor for prefixes_vtab objects. +*/ +static int prefixesDisconnect(sqlite3_vtab *pVtab){ + prefixes_vtab *p = (prefixes_vtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new prefixes_cursor object. +*/ +static int prefixesOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + prefixes_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Destructor for a prefixes_cursor. +*/ +static int prefixesClose(sqlite3_vtab_cursor *cur){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + sqlite3_free(pCur->zStr); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a prefixes_cursor to its next row of output. +*/ +static int prefixesNext(sqlite3_vtab_cursor *cur){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + pCur->iRowid++; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the prefixes_cursor +** is currently pointing. +*/ +static int prefixesColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + switch( i ){ + case 0: + sqlite3_result_text(ctx, pCur->zStr, pCur->nStr - (int)pCur->iRowid, + 0); + break; + default: + sqlite3_result_text(ctx, pCur->zStr, pCur->nStr, 0); + break; + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int prefixesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int prefixesEof(sqlite3_vtab_cursor *cur){ + prefixes_cursor *pCur = (prefixes_cursor*)cur; + return pCur->iRowid>pCur->nStr; +} + +/* +** This method is called to "rewind" the prefixes_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to prefixesColumn() or prefixesRowid() or +** prefixesEof(). +*/ +static int prefixesFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + prefixes_cursor *pCur = (prefixes_cursor *)pVtabCursor; + sqlite3_free(pCur->zStr); + if( argc>0 ){ + pCur->zStr = sqlite3_mprintf("%s", sqlite3_value_text(argv[0])); + pCur->nStr = pCur->zStr ? (int)strlen(pCur->zStr) : 0; + }else{ + pCur->zStr = 0; + pCur->nStr = 0; + } + pCur->iRowid = 0; + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int prefixesBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + /* Search for a usable equality constraint against column 1 + ** (original_string) and use it if at all possible */ + int i; + const struct sqlite3_index_constraint *p; + + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->iColumn!=1 ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( !p->usable ) continue; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->estimatedCost = (double)10; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->estimatedCost = (double)1000000000; + pIdxInfo->estimatedRows = 1000000000; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module prefixesModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ prefixesConnect, + /* xBestIndex */ prefixesBestIndex, + /* xDisconnect */ prefixesDisconnect, + /* xDestroy */ 0, + /* xOpen */ prefixesOpen, + /* xClose */ prefixesClose, + /* xFilter */ prefixesFilter, + /* xNext */ prefixesNext, + /* xEof */ prefixesEof, + /* xColumn */ prefixesColumn, + /* xRowid */ prefixesRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 +}; + +/* +** This is a copy of the SQLITE_SKIP_UTF8(zIn) macro in sqliteInt.h. +** +** Assuming zIn points to the first byte of a UTF-8 character, +** advance zIn to point to the first byte of the next UTF-8 character. +*/ +#define PREFIX_SKIP_UTF8(zIn) { \ + if( (*(zIn++))>=0xc0 ){ \ + while( (*zIn & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + +/* +** Implementation of function prefix_length(). This function accepts two +** strings as arguments and returns the length in characters (not bytes), +** of the longest prefix shared by the two strings. For example: +** +** prefix_length('abcdxxx', 'abcyy') == 3 +** prefix_length('abcdxxx', 'bcyyy') == 0 +** prefix_length('abcdxxx', 'ab') == 2 +** prefix_length('ab', 'abcd') == 2 +** +** This function assumes the input is well-formed utf-8. If it is not, +** it is possible for this function to return -1. +*/ +static void prefixLengthFunc( + sqlite3_context *ctx, + int nVal, + sqlite3_value **apVal +){ + int nByte; /* Number of bytes to compare */ + int nRet = 0; /* Return value */ + const unsigned char *zL = sqlite3_value_text(apVal[0]); + const unsigned char *zR = sqlite3_value_text(apVal[1]); + int nL = sqlite3_value_bytes(apVal[0]); + int nR = sqlite3_value_bytes(apVal[1]); + int i; + + nByte = (nL > nR ? nL : nR); + for(i=0; i