diff options
Diffstat (limited to 'test/fuzzcheck.c')
| -rw-r--r-- | test/fuzzcheck.c | 2471 |
1 files changed, 2471 insertions, 0 deletions
diff --git a/test/fuzzcheck.c b/test/fuzzcheck.c new file mode 100644 index 0000000..a0bcc3b --- /dev/null +++ b/test/fuzzcheck.c @@ -0,0 +1,2471 @@ +/* +** 2015-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This is a utility program designed to aid running regressions tests on +** the SQLite library using data from external fuzzers. +** +** This program reads content from an SQLite database file with the following +** schema: +** +** CREATE TABLE db( +** dbid INTEGER PRIMARY KEY, -- database id +** dbcontent BLOB -- database disk file image +** ); +** CREATE TABLE xsql( +** sqlid INTEGER PRIMARY KEY, -- SQL script id +** sqltext TEXT -- Text of SQL statements to run +** ); +** CREATE TABLE IF NOT EXISTS readme( +** msg TEXT -- Human-readable description of this test collection +** ); +** +** For each database file in the DB table, the SQL text in the XSQL table +** is run against that database. All README.MSG values are printed prior +** to the start of the test (unless the --quiet option is used). If the +** DB table is empty, then all entries in XSQL are run against an empty +** in-memory database. +** +** This program is looking for crashes, assertion faults, and/or memory leaks. +** No attempt is made to verify the output. The assumption is that either all +** of the database files or all of the SQL statements are malformed inputs, +** generated by a fuzzer, that need to be checked to make sure they do not +** present a security risk. +** +** This program also includes some command-line options to help with +** creation and maintenance of the source content database. The command +** +** ./fuzzcheck database.db --load-sql FILE... +** +** Loads all FILE... arguments into the XSQL table. The --load-db option +** works the same but loads the files into the DB table. The -m option can +** be used to initialize the README table. The "database.db" file is created +** if it does not previously exist. Example: +** +** ./fuzzcheck new.db --load-sql *.sql +** ./fuzzcheck new.db --load-db *.db +** ./fuzzcheck new.db -m 'New test cases' +** +** The three commands above will create the "new.db" file and initialize all +** tables. Then do "./fuzzcheck new.db" to run the tests. +** +** DEBUGGING HINTS: +** +** If fuzzcheck does crash, it can be run in the debugger and the content +** of the global variable g.zTextName[] will identify the specific XSQL and +** DB values that were running when the crash occurred. +** +** DBSQLFUZZ: (Added 2020-02-25) +** +** The dbsqlfuzz fuzzer includes both a database file and SQL to run against +** that database in its input. This utility can now process dbsqlfuzz +** input files. Load such files using the "--load-dbsql FILE ..." command-line +** option. +** +** Dbsqlfuzz inputs are ordinary text. The first part of the file is text +** that describes the content of the database (using a lot of hexadecimal), +** then there is a divider line followed by the SQL to run against the +** database. Because they are ordinary text, dbsqlfuzz inputs are stored +** in the XSQL table, as if they were ordinary SQL inputs. The isDbSql() +** function can look at a text string and determine whether or not it is +** a valid dbsqlfuzz input. +*/ +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <stdarg.h> +#include <ctype.h> +#include <assert.h> +#include "sqlite3.h" +#include "sqlite3recover.h" +#define ISSPACE(X) isspace((unsigned char)(X)) +#define ISDIGIT(X) isdigit((unsigned char)(X)) + + +#ifdef __unix__ +# include <signal.h> +# include <unistd.h> +#endif + +#include <stddef.h> +#if !defined(_MSC_VER) +# include <stdint.h> +#endif + +#if defined(_MSC_VER) +typedef unsigned char uint8_t; +#endif + +/* +** Files in the virtual file system. +*/ +typedef struct VFile VFile; +struct VFile { + char *zFilename; /* Filename. NULL for delete-on-close. From malloc() */ + int sz; /* Size of the file in bytes */ + int nRef; /* Number of references to this file */ + unsigned char *a; /* Content of the file. From malloc() */ +}; +typedef struct VHandle VHandle; +struct VHandle { + sqlite3_file base; /* Base class. Must be first */ + VFile *pVFile; /* The underlying file */ +}; + +/* +** The value of a database file template, or of an SQL script +*/ +typedef struct Blob Blob; +struct Blob { + Blob *pNext; /* Next in a list */ + int id; /* Id of this Blob */ + int seq; /* Sequence number */ + int sz; /* Size of this Blob in bytes */ + unsigned char a[1]; /* Blob content. Extra space allocated as needed. */ +}; + +/* +** Maximum number of files in the in-memory virtual filesystem. +*/ +#define MX_FILE 10 + +/* +** Maximum allowed file size +*/ +#define MX_FILE_SZ 10000000 + +/* +** All global variables are gathered into the "g" singleton. +*/ +static struct GlobalVars { + const char *zArgv0; /* Name of program */ + const char *zDbFile; /* Name of database file */ + VFile aFile[MX_FILE]; /* The virtual filesystem */ + int nDb; /* Number of template databases */ + Blob *pFirstDb; /* Content of first template database */ + int nSql; /* Number of SQL scripts */ + Blob *pFirstSql; /* First SQL script */ + unsigned int uRandom; /* Seed for the SQLite PRNG */ + unsigned int nInvariant; /* Number of invariant checks run */ + char zTestName[100]; /* Name of current test */ +} g; + +/* +** Include the external vt02.c module. +*/ +extern int sqlite3_vt02_init(sqlite3*,char***,void*); + + +/* +** Print an error message and quit. +*/ +static void fatalError(const char *zFormat, ...){ + va_list ap; + fprintf(stderr, "%s", g.zArgv0); + if( g.zDbFile ) fprintf(stderr, " %s", g.zDbFile); + if( g.zTestName[0] ) fprintf(stderr, " (%s)", g.zTestName); + fprintf(stderr, ": "); + va_start(ap, zFormat); + vfprintf(stderr, zFormat, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(1); +} + +/* +** signal handler +*/ +#ifdef __unix__ +static void signalHandler(int signum){ + const char *zSig; + if( signum==SIGABRT ){ + zSig = "abort"; + }else if( signum==SIGALRM ){ + zSig = "timeout"; + }else if( signum==SIGSEGV ){ + zSig = "segfault"; + }else{ + zSig = "signal"; + } + fatalError(zSig); +} +#endif + +/* +** Set the an alarm to go off after N seconds. Disable the alarm +** if N==0 +*/ +static void setAlarm(int N){ +#ifdef __unix__ + alarm(N); +#else + (void)N; +#endif +} + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK +/* +** This an SQL progress handler. After an SQL statement has run for +** many steps, we want to interrupt it. This guards against infinite +** loops from recursive common table expressions. +** +** *pVdbeLimitFlag is true if the --limit-vdbe command-line option is used. +** In that case, hitting the progress handler is a fatal error. +*/ +static int progressHandler(void *pVdbeLimitFlag){ + if( *(int*)pVdbeLimitFlag ) fatalError("too many VDBE cycles"); + return 1; +} +#endif + +/* +** Reallocate memory. Show an error and quit if unable. +*/ +static void *safe_realloc(void *pOld, int szNew){ + void *pNew = realloc(pOld, szNew<=0 ? 1 : szNew); + if( pNew==0 ) fatalError("unable to realloc for %d bytes", szNew); + return pNew; +} + +/* +** Initialize the virtual file system. +*/ +static void formatVfs(void){ + int i; + for(i=0; i<MX_FILE; i++){ + g.aFile[i].sz = -1; + g.aFile[i].zFilename = 0; + g.aFile[i].a = 0; + g.aFile[i].nRef = 0; + } +} + + +/* +** Erase all information in the virtual file system. +*/ +static void reformatVfs(void){ + int i; + for(i=0; i<MX_FILE; i++){ + if( g.aFile[i].sz<0 ) continue; + if( g.aFile[i].zFilename ){ + free(g.aFile[i].zFilename); + g.aFile[i].zFilename = 0; + } + if( g.aFile[i].nRef>0 ){ + fatalError("file %d still open. nRef=%d", i, g.aFile[i].nRef); + } + g.aFile[i].sz = -1; + free(g.aFile[i].a); + g.aFile[i].a = 0; + g.aFile[i].nRef = 0; + } +} + +/* +** Find a VFile by name +*/ +static VFile *findVFile(const char *zName){ + int i; + if( zName==0 ) return 0; + for(i=0; i<MX_FILE; i++){ + if( g.aFile[i].zFilename==0 ) continue; + if( strcmp(g.aFile[i].zFilename, zName)==0 ) return &g.aFile[i]; + } + return 0; +} + +/* +** Find a VFile by name. Create it if it does not already exist and +** initialize it to the size and content given. +** +** Return NULL only if the filesystem is full. +*/ +static VFile *createVFile(const char *zName, int sz, unsigned char *pData){ + VFile *pNew = findVFile(zName); + int i; + if( pNew ) return pNew; + for(i=0; i<MX_FILE && g.aFile[i].sz>=0; i++){} + if( i>=MX_FILE ) return 0; + pNew = &g.aFile[i]; + if( zName ){ + int nName = (int)strlen(zName)+1; + pNew->zFilename = safe_realloc(0, nName); + memcpy(pNew->zFilename, zName, nName); + }else{ + pNew->zFilename = 0; + } + pNew->nRef = 0; + pNew->sz = sz; + pNew->a = safe_realloc(0, sz); + if( sz>0 ) memcpy(pNew->a, pData, sz); + return pNew; +} + +/* Return true if the line is all zeros */ +static int allZero(unsigned char *aLine){ + int i; + for(i=0; i<16 && aLine[i]==0; i++){} + return i==16; +} + +/* +** Render a database and query as text that can be input into +** the CLI. +*/ +static void renderDbSqlForCLI( + FILE *out, /* Write to this file */ + const char *zFile, /* Name of the database file */ + unsigned char *aDb, /* Database content */ + int nDb, /* Number of bytes in aDb[] */ + unsigned char *zSql, /* SQL content */ + int nSql /* Bytes of SQL */ +){ + fprintf(out, ".print ******* %s *******\n", zFile); + if( nDb>100 ){ + int i, j; /* Loop counters */ + int pgsz; /* Size of each page */ + int lastPage = 0; /* Last page number shown */ + int iPage; /* Current page number */ + unsigned char *aLine; /* Single line to display */ + unsigned char buf[16]; /* Fake line */ + unsigned char bShow[256]; /* Characters ok to display */ + + memset(bShow, '.', sizeof(bShow)); + for(i=' '; i<='~'; i++){ + if( i!='{' && i!='}' && i!='"' && i!='\\' ) bShow[i] = i; + } + pgsz = (aDb[16]<<8) | aDb[17]; + if( pgsz==0 ) pgsz = 65536; + if( pgsz<512 || (pgsz&(pgsz-1))!=0 ) pgsz = 4096; + fprintf(out,".open --hexdb\n"); + fprintf(out,"| size %d pagesize %d filename %s\n",nDb,pgsz,zFile); + for(i=0; i<nDb; i += 16){ + if( i+16>nDb ){ + memset(buf, 0, sizeof(buf)); + memcpy(buf, aDb+i, nDb-i); + aLine = buf; + }else{ + aLine = aDb + i; + } + if( allZero(aLine) ) continue; + iPage = i/pgsz + 1; + if( lastPage!=iPage ){ + fprintf(out,"| page %d offset %d\n", iPage, (iPage-1)*pgsz); + lastPage = iPage; + } + fprintf(out,"| %5d:", i-(iPage-1)*pgsz); + for(j=0; j<16; j++) fprintf(out," %02x", aLine[j]); + fprintf(out," "); + for(j=0; j<16; j++){ + unsigned char c = (unsigned char)aLine[j]; + fputc( bShow[c], stdout); + } + fputc('\n', stdout); + } + fprintf(out,"| end %s\n", zFile); + }else{ + fprintf(out,".open :memory:\n"); + } + fprintf(out,".testctrl prng_seed 1 db\n"); + fprintf(out,".testctrl internal_functions\n"); + fprintf(out,"%.*s", nSql, zSql); + if( nSql>0 && zSql[nSql-1]!='\n' ) fprintf(out, "\n"); +} + +/* +** Read the complete content of a file into memory. Add a 0x00 terminator +** and return a pointer to the result. +** +** The file content is held in memory obtained from sqlite_malloc64() which +** should be freed by the caller. +*/ +static char *readFile(const char *zFilename, long *sz){ + FILE *in; + long nIn; + unsigned char *pBuf; + + *sz = 0; + if( zFilename==0 ) return 0; + in = fopen(zFilename, "rb"); + if( in==0 ) return 0; + fseek(in, 0, SEEK_END); + *sz = nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc64( nIn+1 ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + pBuf[nIn] = 0; + fclose(in); + return (char*)pBuf; + } + sqlite3_free(pBuf); + *sz = 0; + fclose(in); + return 0; +} + + +/* +** Implementation of the "readfile(X)" SQL function. The entire content +** of the file named X is read and returned as a BLOB. NULL is returned +** if the file does not exist or is unreadable. +*/ +static void readfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + long nIn; + void *pBuf; + const char *zName = (const char*)sqlite3_value_text(argv[0]); + + if( zName==0 ) return; + pBuf = readFile(zName, &nIn); + if( pBuf ){ + sqlite3_result_blob(context, pBuf, nIn, sqlite3_free); + } +} + +/* +** Implementation of the "readtextfile(X)" SQL function. The text content +** of the file named X through the end of the file or to the first \000 +** character, whichever comes first, is read and returned as TEXT. NULL +** is returned if the file does not exist or is unreadable. +*/ +static void readtextfileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + const char *zName; + FILE *in; + long nIn; + char *pBuf; + + zName = (const char*)sqlite3_value_text(argv[0]); + if( zName==0 ) return; + in = fopen(zName, "rb"); + if( in==0 ) return; + fseek(in, 0, SEEK_END); + nIn = ftell(in); + rewind(in); + pBuf = sqlite3_malloc64( nIn+1 ); + if( pBuf && 1==fread(pBuf, nIn, 1, in) ){ + pBuf[nIn] = 0; + sqlite3_result_text(context, pBuf, -1, sqlite3_free); + }else{ + sqlite3_free(pBuf); + } + fclose(in); +} + +/* +** Implementation of the "writefile(X,Y)" SQL function. The argument Y +** is written into file X. The number of bytes written is returned. Or +** NULL is returned if something goes wrong, such as being unable to open +** file X for writing. +*/ +static void writefileFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FILE *out; + const char *z; + sqlite3_int64 rc; + const char *zFile; + + (void)argc; + zFile = (const char*)sqlite3_value_text(argv[0]); + if( zFile==0 ) return; + out = fopen(zFile, "wb"); + if( out==0 ) return; + z = (const char*)sqlite3_value_blob(argv[1]); + if( z==0 ){ + rc = 0; + }else{ + rc = fwrite(z, 1, sqlite3_value_bytes(argv[1]), out); + } + fclose(out); + sqlite3_result_int64(context, rc); +} + + +/* +** Load a list of Blob objects from the database +*/ +static void blobListLoadFromDb( + sqlite3 *db, /* Read from this database */ + const char *zSql, /* Query used to extract the blobs */ + int onlyId, /* Only load where id is this value */ + int *pN, /* OUT: Write number of blobs loaded here */ + Blob **ppList /* OUT: Write the head of the blob list here */ +){ + Blob head; + Blob *p; + sqlite3_stmt *pStmt; + int n = 0; + int rc; + char *z2; + + if( onlyId>0 ){ + z2 = sqlite3_mprintf("%s WHERE rowid=%d", zSql, onlyId); + }else{ + z2 = sqlite3_mprintf("%s", zSql); + } + rc = sqlite3_prepare_v2(db, z2, -1, &pStmt, 0); + sqlite3_free(z2); + if( rc ) fatalError("%s", sqlite3_errmsg(db)); + head.pNext = 0; + p = &head; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int sz = sqlite3_column_bytes(pStmt, 1); + Blob *pNew = safe_realloc(0, sizeof(*pNew)+sz ); + pNew->id = sqlite3_column_int(pStmt, 0); + pNew->sz = sz; + pNew->seq = n++; + pNew->pNext = 0; + memcpy(pNew->a, sqlite3_column_blob(pStmt,1), sz); + pNew->a[sz] = 0; + p->pNext = pNew; + p = pNew; + } + sqlite3_finalize(pStmt); + *pN = n; + *ppList = head.pNext; +} + +/* +** Free a list of Blob objects +*/ +static void blobListFree(Blob *p){ + Blob *pNext; + while( p ){ + pNext = p->pNext; + free(p); + p = pNext; + } +} + +/* Return the current wall-clock time +** +** The number of milliseconds since the julian epoch. +** 1907-01-01 00:00:00 -> 210866716800000 +** 2021-01-01 00:00:00 -> 212476176000000 +*/ +static sqlite3_int64 timeOfDay(void){ + static sqlite3_vfs *clockVfs = 0; + sqlite3_int64 t; + if( clockVfs==0 ){ + clockVfs = sqlite3_vfs_find(0); + if( clockVfs==0 ) return 0; + } + if( clockVfs->iVersion>=1 && clockVfs->xCurrentTimeInt64!=0 ){ + clockVfs->xCurrentTimeInt64(clockVfs, &t); + }else{ + double r; + clockVfs->xCurrentTime(clockVfs, &r); + t = (sqlite3_int64)(r*86400000.0); + } + return t; +} + +/*************************************************************************** +** Code to process combined database+SQL scripts generated by the +** dbsqlfuzz fuzzer. +*/ + +/* An instance of the following object is passed by pointer as the +** client data to various callbacks. +*/ +typedef struct FuzzCtx { + sqlite3 *db; /* The database connection */ + sqlite3_int64 iCutoffTime; /* Stop processing at this time. */ + sqlite3_int64 iLastCb; /* Time recorded for previous progress callback */ + sqlite3_int64 mxInterval; /* Longest interval between two progress calls */ + unsigned nCb; /* Number of progress callbacks */ + unsigned mxCb; /* Maximum number of progress callbacks allowed */ + unsigned execCnt; /* Number of calls to the sqlite3_exec callback */ + int timeoutHit; /* True when reaching a timeout */ +} FuzzCtx; + +/* Verbosity level for the dbsqlfuzz test runner */ +static int eVerbosity = 0; + +/* True to activate PRAGMA vdbe_debug=on */ +static int bVdbeDebug = 0; + +/* Timeout for each fuzzing attempt, in milliseconds */ +static int giTimeout = 10000; /* Defaults to 10 seconds */ + +/* Maximum number of progress handler callbacks */ +static unsigned int mxProgressCb = 2000; + +/* Maximum string length in SQLite */ +static int lengthLimit = 1000000; + +/* Maximum expression depth */ +static int depthLimit = 500; + +/* Limit on the amount of heap memory that can be used */ +static sqlite3_int64 heapLimit = 100000000; + +/* Maximum byte-code program length in SQLite */ +static int vdbeOpLimit = 25000; + +/* Maximum size of the in-memory database */ +static sqlite3_int64 maxDbSize = 104857600; +/* OOM simulation parameters */ +static unsigned int oomCounter = 0; /* Simulate OOM when equals 1 */ +static unsigned int oomRepeat = 0; /* Number of OOMs in a row */ +static void*(*defaultMalloc)(int) = 0; /* The low-level malloc routine */ + +/* Enable recovery */ +static int bNoRecover = 0; + +/* This routine is called when a simulated OOM occurs. It is broken +** out as a separate routine to make it easy to set a breakpoint on +** the OOM +*/ +void oomFault(void){ + if( eVerbosity ){ + printf("Simulated OOM fault\n"); + } + if( oomRepeat>0 ){ + oomRepeat--; + }else{ + oomCounter--; + } +} + +/* This routine is a replacement malloc() that is used to simulate +** Out-Of-Memory (OOM) errors for testing purposes. +*/ +static void *oomMalloc(int nByte){ + if( oomCounter ){ + if( oomCounter==1 ){ + oomFault(); + return 0; + }else{ + oomCounter--; + } + } + return defaultMalloc(nByte); +} + +/* Register the OOM simulator. This must occur before any memory +** allocations */ +static void registerOomSimulator(void){ + sqlite3_mem_methods mem; + sqlite3_shutdown(); + sqlite3_config(SQLITE_CONFIG_GETMALLOC, &mem); + defaultMalloc = mem.xMalloc; + mem.xMalloc = oomMalloc; + sqlite3_config(SQLITE_CONFIG_MALLOC, &mem); +} + +/* Turn off any pending OOM simulation */ +static void disableOom(void){ + oomCounter = 0; + oomRepeat = 0; +} + +/* +** Translate a single byte of Hex into an integer. +** This routine only works if h really is a valid hexadecimal +** character: 0..9a..fA..F +*/ +static unsigned char hexToInt(unsigned int h){ +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); /* EBCDIC */ +#else + h += 9*(1&(h>>6)); /* ASCII */ +#endif + return h & 0xf; +} + +/* +** The first character of buffer zIn[0..nIn-1] is a '['. This routine +** checked to see if the buffer holds "[NNNN]" or "[+NNNN]" and if it +** does it makes corresponding changes to the *pK value and *pI value +** and returns true. If the input buffer does not match the patterns, +** no changes are made to either *pK or *pI and this routine returns false. +*/ +static int isOffset( + const unsigned char *zIn, /* Text input */ + int nIn, /* Bytes of input */ + unsigned int *pK, /* half-byte cursor to adjust */ + unsigned int *pI /* Input index to adjust */ +){ + int i; + unsigned int k = 0; + unsigned char c; + for(i=1; i<nIn && (c = zIn[i])!=']'; i++){ + if( !isxdigit(c) ) return 0; + k = k*16 + hexToInt(c); + } + if( i==nIn ) return 0; + *pK = 2*k; + *pI += i; + return 1; +} + +/* +** Decode the text starting at zIn into a binary database file. +** The maximum length of zIn is nIn bytes. Store the binary database +** file in space obtained from sqlite3_malloc(). +** +** Return the number of bytes of zIn consumed. Or return -1 if there +** is an error. One potential error is that the recipe specifies a +** database file larger than MX_FILE_SZ bytes. +** +** Abort on an OOM. +*/ +static int decodeDatabase( + const unsigned char *zIn, /* Input text to be decoded */ + int nIn, /* Bytes of input text */ + unsigned char **paDecode, /* OUT: decoded database file */ + int *pnDecode /* OUT: Size of decoded database */ +){ + unsigned char *a, *aNew; /* Database under construction */ + int mx = 0; /* Current size of the database */ + sqlite3_uint64 nAlloc = 4096; /* Space allocated in a[] */ + unsigned int i; /* Next byte of zIn[] to read */ + unsigned int j; /* Temporary integer */ + unsigned int k; /* half-byte cursor index for output */ + unsigned int n; /* Number of bytes of input */ + unsigned char b = 0; + if( nIn<4 ) return -1; + n = (unsigned int)nIn; + a = sqlite3_malloc64( nAlloc ); + if( a==0 ){ + fprintf(stderr, "Out of memory!\n"); + exit(1); + } + memset(a, 0, (size_t)nAlloc); + for(i=k=0; i<n; i++){ + unsigned char c = (unsigned char)zIn[i]; + if( isxdigit(c) ){ + k++; + if( k & 1 ){ + b = hexToInt(c)*16; + }else{ + b += hexToInt(c); + j = k/2 - 1; + if( j>=nAlloc ){ + sqlite3_uint64 newSize; + if( nAlloc==MX_FILE_SZ || j>=MX_FILE_SZ ){ + if( eVerbosity ){ + fprintf(stderr, "Input database too big: max %d bytes\n", + MX_FILE_SZ); + } + sqlite3_free(a); + return -1; + } + newSize = nAlloc*2; + if( newSize<=j ){ + newSize = (j+4096)&~4095; + } + if( newSize>MX_FILE_SZ ){ + if( j>=MX_FILE_SZ ){ + sqlite3_free(a); + return -1; + } + newSize = MX_FILE_SZ; + } + aNew = sqlite3_realloc64( a, newSize ); + if( aNew==0 ){ + sqlite3_free(a); + return -1; + } + a = aNew; + assert( newSize > nAlloc ); + memset(a+nAlloc, 0, (size_t)(newSize - nAlloc)); + nAlloc = newSize; + } + if( j>=(unsigned)mx ){ + mx = (j + 4095)&~4095; + if( mx>MX_FILE_SZ ) mx = MX_FILE_SZ; + } + assert( j<nAlloc ); + a[j] = b; + } + }else if( zIn[i]=='[' && i<n-3 && isOffset(zIn+i, nIn-i, &k, &i) ){ + continue; + }else if( zIn[i]=='\n' && i<n-4 && memcmp(zIn+i,"\n--\n",4)==0 ){ + i += 4; + break; + } + } + *pnDecode = mx; + *paDecode = a; + return i; +} + +/* +** Progress handler callback. +** +** The argument is the cutoff-time after which all processing should +** stop. So return non-zero if the cut-off time is exceeded. +*/ +static int progress_handler(void *pClientData) { + FuzzCtx *p = (FuzzCtx*)pClientData; + sqlite3_int64 iNow = timeOfDay(); + int rc = iNow>=p->iCutoffTime; + sqlite3_int64 iDiff = iNow - p->iLastCb; + /* printf("time-remaining: %lld\n", p->iCutoffTime - iNow); */ + if( iDiff > p->mxInterval ) p->mxInterval = iDiff; + p->nCb++; + if( rc==0 && p->mxCb>0 && p->mxCb<=p->nCb ) rc = 1; + if( rc && !p->timeoutHit && eVerbosity>=2 ){ + printf("Timeout on progress callback %d\n", p->nCb); + fflush(stdout); + p->timeoutHit = 1; + } + return rc; +} + +/* +** Flag bits set by block_troublesome_sql() +*/ +#define BTS_SELECT 0x000001 +#define BTS_NONSELECT 0x000002 +#define BTS_BADFUNC 0x000004 +#define BTS_BADPRAGMA 0x000008 /* Sticky for rest of the script */ + +/* +** Disallow debugging pragmas such as "PRAGMA vdbe_debug" and +** "PRAGMA parser_trace" since they can dramatically increase the +** amount of output without actually testing anything useful. +** +** Also block ATTACH if attaching a file from the filesystem. +*/ +static int block_troublesome_sql( + void *pClientData, + int eCode, + const char *zArg1, + const char *zArg2, + const char *zArg3, + const char *zArg4 +){ + unsigned int *pBtsFlags = (unsigned int*)pClientData; + + (void)zArg3; + (void)zArg4; + switch( eCode ){ + case SQLITE_PRAGMA: { + if( sqlite3_stricmp("busy_timeout",zArg1)==0 + && (zArg2==0 || strtoll(zArg2,0,0)>100 || strtoll(zArg2,0,10)>100) + ){ + return SQLITE_DENY; + }else if( sqlite3_stricmp("hard_heap_limit", zArg1)==0 + || sqlite3_stricmp("reverse_unordered_selects", zArg1)==0 + ){ + /* BTS_BADPRAGMA is sticky. A hard_heap_limit or + ** revert_unordered_selects should inhibit all future attempts + ** at verifying query invariants */ + *pBtsFlags |= BTS_BADPRAGMA; + }else if( eVerbosity==0 ){ + if( sqlite3_strnicmp("vdbe_", zArg1, 5)==0 + || sqlite3_stricmp("parser_trace", zArg1)==0 + || sqlite3_stricmp("temp_store_directory", zArg1)==0 + ){ + return SQLITE_DENY; + } + }else if( sqlite3_stricmp("oom",zArg1)==0 + && zArg2!=0 && zArg2[0]!=0 ){ + oomCounter = atoi(zArg2); + } + *pBtsFlags |= BTS_NONSELECT; + break; + } + case SQLITE_ATTACH: { + /* Deny the ATTACH if it is attaching anything other than an in-memory + ** database. */ + *pBtsFlags |= BTS_NONSELECT; + if( zArg1==0 ) return SQLITE_DENY; + if( strcmp(zArg1,":memory:")==0 ) return SQLITE_OK; + if( sqlite3_strglob("file:*[?]vfs=memdb", zArg1)==0 + && sqlite3_strglob("file:*[^/a-zA-Z0-9_.]*[?]vfs=memdb", zArg1)!=0 + ){ + return SQLITE_OK; + } + return SQLITE_DENY; + } + case SQLITE_SELECT: { + *pBtsFlags |= BTS_SELECT; + break; + } + case SQLITE_FUNCTION: { + static const char *azBadFuncs[] = { + "avg", + "count", + "cume_dist", + "current_date", + "current_time", + "current_timestamp", + "date", + "datetime", + "decimal_sum", + "dense_rank", + "first_value", + "geopoly_group_bbox", + "group_concat", + "implies_nonnull_row", + "json_group_array", + "json_group_object", + "julianday", + "lag", + "last_value", + "lead", + "max", + "min", + "nth_value", + "ntile", + "percent_rank", + "random", + "randomblob", + "rank", + "row_number", + "sqlite_offset", + "strftime", + "sum", + "time", + "total", + "unixepoch", + }; + int first, last; + first = 0; + last = sizeof(azBadFuncs)/sizeof(azBadFuncs[0]) - 1; + do{ + int mid = (first+last)/2; + int c = sqlite3_stricmp(azBadFuncs[mid], zArg2); + if( c<0 ){ + first = mid+1; + }else if( c>0 ){ + last = mid-1; + }else{ + *pBtsFlags |= BTS_BADFUNC; + break; + } + }while( first<=last ); + break; + } + case SQLITE_READ: { + /* Benign */ + break; + } + default: { + *pBtsFlags |= BTS_NONSELECT; + } + } + return SQLITE_OK; +} + +/* Implementation found in fuzzinvariant.c */ +extern int fuzz_invariant( + sqlite3 *db, /* The database connection */ + sqlite3_stmt *pStmt, /* Test statement stopped on an SQLITE_ROW */ + int iCnt, /* Invariant sequence number, starting at 0 */ + int iRow, /* The row number for pStmt */ + int nRow, /* Total number of output rows */ + int *pbCorrupt, /* IN/OUT: Flag indicating a corrupt database file */ + int eVerbosity /* How much debugging output */ +); + +/* Implementation of sqlite_dbdata and sqlite_dbptr */ +extern int sqlite3_dbdata_init(sqlite3*,const char**,void*); + + +/* +** This function is used as a callback by the recover extension. Simply +** print the supplied SQL statement to stdout. +*/ +static int recoverSqlCb(void *pCtx, const char *zSql){ + if( eVerbosity>=2 ){ + printf("%s\n", zSql); + } + return SQLITE_OK; +} + +/* +** This function is called to recover data from the database. +*/ +static int recoverDatabase(sqlite3 *db){ + int rc; /* Return code from this routine */ + const char *zLAF = "lost_and_found"; /* Name of "lost_and_found" table */ + int bFreelist = 1; /* True to scan the freelist */ + int bRowids = 1; /* True to restore ROWID values */ + sqlite3_recover *p; /* The recovery object */ + + p = sqlite3_recover_init_sql(db, "main", recoverSqlCb, 0); + sqlite3_recover_config(p, SQLITE_RECOVER_LOST_AND_FOUND, (void*)zLAF); + sqlite3_recover_config(p, SQLITE_RECOVER_ROWIDS, (void*)&bRowids); + sqlite3_recover_config(p, SQLITE_RECOVER_FREELIST_CORRUPT,(void*)&bFreelist); + sqlite3_recover_run(p); + if( sqlite3_recover_errcode(p)!=SQLITE_OK ){ + const char *zErr = sqlite3_recover_errmsg(p); + int errCode = sqlite3_recover_errcode(p); + if( eVerbosity>0 ){ + printf("recovery error: %s (%d)\n", zErr, errCode); + } + } + rc = sqlite3_recover_finish(p); + if( eVerbosity>0 && rc ){ + printf("recovery returns error code %d\n", rc); + } + return rc; +} + +/* +** Run the SQL text +*/ +static int runDbSql(sqlite3 *db, const char *zSql, unsigned int *pBtsFlags){ + int rc; + sqlite3_stmt *pStmt; + int bCorrupt = 0; + while( isspace(zSql[0]&0x7f) ) zSql++; + if( zSql[0]==0 ) return SQLITE_OK; + if( eVerbosity>=4 ){ + printf("RUNNING-SQL: [%s]\n", zSql); + fflush(stdout); + } + (*pBtsFlags) &= ~BTS_BADPRAGMA; + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + if( rc==SQLITE_OK ){ + int nRow = 0; + while( (rc = sqlite3_step(pStmt))==SQLITE_ROW ){ + nRow++; + if( eVerbosity>=4 ){ + int j; + for(j=0; j<sqlite3_column_count(pStmt); j++){ + if( j ) printf(","); + switch( sqlite3_column_type(pStmt, j) ){ + case SQLITE_NULL: { + printf("NULL"); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + printf("%s", sqlite3_column_text(pStmt, j)); + break; + } + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pStmt, j); + int i; + const unsigned char *a; + a = (const unsigned char*)sqlite3_column_blob(pStmt, j); + printf("x'"); + for(i=0; i<n; i++){ + printf("%02x", a[i]); + } + printf("'"); + break; + } + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pStmt, j); + int i; + const unsigned char *a; + a = (const unsigned char*)sqlite3_column_blob(pStmt, j); + printf("'"); + for(i=0; i<n; i++){ + if( a[i]=='\'' ){ + printf("''"); + }else{ + putchar(a[i]); + } + } + printf("'"); + break; + } + } /* End switch() */ + } /* End for() */ + printf("\n"); + fflush(stdout); + } /* End if( eVerbosity>=5 ) */ + } /* End while( SQLITE_ROW */ + if( rc==SQLITE_DONE ){ + if( (*pBtsFlags)==BTS_SELECT + && !sqlite3_stmt_isexplain(pStmt) + && nRow>0 + ){ + int iRow = 0; + sqlite3_reset(pStmt); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + int iCnt = 0; + iRow++; + for(iCnt=0; iCnt<99999; iCnt++){ + rc = fuzz_invariant(db, pStmt, iCnt, iRow, nRow, + &bCorrupt, eVerbosity); + if( rc==SQLITE_DONE ) break; + if( rc!=SQLITE_ERROR ) g.nInvariant++; + if( eVerbosity>0 ){ + if( rc==SQLITE_OK ){ + printf("invariant-check: ok\n"); + }else if( rc==SQLITE_CORRUPT ){ + printf("invariant-check: failed due to database corruption\n"); + } + } + } + } + } + }else if( eVerbosity>=4 ){ + printf("SQL-ERROR: (%d) %s\n", rc, sqlite3_errmsg(db)); + fflush(stdout); + } + }else if( eVerbosity>=4 ){ + printf("SQL-ERROR (%d): %s\n", rc, sqlite3_errmsg(db)); + fflush(stdout); + } /* End if( SQLITE_OK ) */ + return sqlite3_finalize(pStmt); +} + +/* Invoke this routine to run a single test case */ +int runCombinedDbSqlInput( + const uint8_t *aData, /* Combined DB+SQL content */ + size_t nByte, /* Size of aData in bytes */ + int iTimeout, /* Use this timeout */ + int bScript, /* If true, just render CLI output */ + int iSqlId /* SQL identifier */ +){ + int rc; /* SQLite API return value */ + int iSql; /* Index in aData[] of start of SQL */ + unsigned char *aDb = 0; /* Decoded database content */ + int nDb = 0; /* Size of the decoded database */ + int i; /* Loop counter */ + int j; /* Start of current SQL statement */ + char *zSql = 0; /* SQL text to run */ + int nSql; /* Bytes of SQL text */ + FuzzCtx cx; /* Fuzzing context */ + unsigned int btsFlags = 0; /* Parsing flags */ + + if( nByte<10 ) return 0; + if( sqlite3_initialize() ) return 0; + if( sqlite3_memory_used()!=0 ){ + int nAlloc = 0; + int nNotUsed = 0; + sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &nAlloc, &nNotUsed, 0); + fprintf(stderr,"memory leak prior to test start:" + " %lld bytes in %d allocations\n", + sqlite3_memory_used(), nAlloc); + exit(1); + } + memset(&cx, 0, sizeof(cx)); + iSql = decodeDatabase((unsigned char*)aData, (int)nByte, &aDb, &nDb); + if( iSql<0 ) return 0; + nSql = (int)(nByte - iSql); + if( bScript ){ + char zName[100]; + sqlite3_snprintf(sizeof(zName),zName,"dbsql%06d.db",iSqlId); + renderDbSqlForCLI(stdout, zName, aDb, nDb, + (unsigned char*)(aData+iSql), nSql); + sqlite3_free(aDb); + return 0; + } + if( eVerbosity>=3 ){ + printf( + "****** %d-byte input, %d-byte database, %d-byte script " + "******\n", (int)nByte, nDb, nSql); + fflush(stdout); + } + rc = sqlite3_open(0, &cx.db); + if( rc ){ + sqlite3_free(aDb); + return 1; + } + if( bVdbeDebug ){ + sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON", 0, 0, 0); + } + + /* Invoke the progress handler frequently to check to see if we + ** are taking too long. The progress handler will return true + ** (which will block further processing) if more than giTimeout seconds have + ** elapsed since the start of the test. + */ + cx.iLastCb = timeOfDay(); + cx.iCutoffTime = cx.iLastCb + (iTimeout<giTimeout ? iTimeout : giTimeout); + cx.mxCb = mxProgressCb; +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + sqlite3_progress_handler(cx.db, 10, progress_handler, (void*)&cx); +#endif + + /* Set a limit on the maximum size of a prepared statement, and the + ** maximum length of a string or blob */ + if( vdbeOpLimit>0 ){ + sqlite3_limit(cx.db, SQLITE_LIMIT_VDBE_OP, vdbeOpLimit); + } + if( lengthLimit>0 ){ + sqlite3_limit(cx.db, SQLITE_LIMIT_LENGTH, lengthLimit); + } + if( depthLimit>0 ){ + sqlite3_limit(cx.db, SQLITE_LIMIT_EXPR_DEPTH, depthLimit); + } + sqlite3_limit(cx.db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 100); + sqlite3_hard_heap_limit64(heapLimit); + + if( nDb>=20 && aDb[18]==2 && aDb[19]==2 ){ + aDb[18] = aDb[19] = 1; + } + rc = sqlite3_deserialize(cx.db, "main", aDb, nDb, nDb, + SQLITE_DESERIALIZE_RESIZEABLE | + SQLITE_DESERIALIZE_FREEONCLOSE); + if( rc ){ + fprintf(stderr, "sqlite3_deserialize() failed with %d\n", rc); + goto testrun_finished; + } + if( maxDbSize>0 ){ + sqlite3_int64 x = maxDbSize; + sqlite3_file_control(cx.db, "main", SQLITE_FCNTL_SIZE_LIMIT, &x); + } + + /* For high debugging levels, turn on debug mode */ + if( eVerbosity>=5 ){ + sqlite3_exec(cx.db, "PRAGMA vdbe_debug=ON;", 0, 0, 0); + } + + /* Block debug pragmas and ATTACH/DETACH. But wait until after + ** deserialize to do this because deserialize depends on ATTACH */ + sqlite3_set_authorizer(cx.db, block_troublesome_sql, &btsFlags); + + /* Add the vt02 virtual table */ + sqlite3_vt02_init(cx.db, 0, 0); + + /* Add support for sqlite_dbdata and sqlite_dbptr virtual tables used + ** by the recovery API */ + sqlite3_dbdata_init(cx.db, 0, 0); + + /* Consistent PRNG seed */ +#ifdef SQLITE_TESTCTRL_PRNG_SEED + sqlite3_table_column_metadata(cx.db, 0, "x", 0, 0, 0, 0, 0, 0); + sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, 1, cx.db); +#else + sqlite3_randomness(0,0); +#endif + + /* Run recovery on the initial database, just to make sure recovery + ** works. */ + if( !bNoRecover ){ + recoverDatabase(cx.db); + } + + zSql = sqlite3_malloc( nSql + 1 ); + if( zSql==0 ){ + fprintf(stderr, "Out of memory!\n"); + }else{ + memcpy(zSql, aData+iSql, nSql); + zSql[nSql] = 0; + for(i=j=0; zSql[i]; i++){ + if( zSql[i]==';' ){ + char cSaved = zSql[i+1]; + zSql[i+1] = 0; + if( sqlite3_complete(zSql+j) ){ + rc = runDbSql(cx.db, zSql+j, &btsFlags); + j = i+1; + } + zSql[i+1] = cSaved; + if( rc==SQLITE_INTERRUPT || progress_handler(&cx) ){ + goto testrun_finished; + } + } + } + if( j<i ){ + runDbSql(cx.db, zSql+j, &btsFlags); + } + } +testrun_finished: + sqlite3_free(zSql); + rc = sqlite3_close(cx.db); + if( rc!=SQLITE_OK ){ + fprintf(stdout, "sqlite3_close() returns %d\n", rc); + } + if( eVerbosity>=2 && !bScript ){ + fprintf(stdout, "Peak memory usages: %f MB\n", + sqlite3_memory_highwater(1) / 1000000.0); + } + if( sqlite3_memory_used()!=0 ){ + int nAlloc = 0; + int nNotUsed = 0; + sqlite3_status(SQLITE_STATUS_MALLOC_COUNT, &nAlloc, &nNotUsed, 0); + fprintf(stderr,"Memory leak: %lld bytes in %d allocations\n", + sqlite3_memory_used(), nAlloc); + exit(1); + } + sqlite3_hard_heap_limit64(0); + sqlite3_soft_heap_limit64(0); + return 0; +} + +/* +** END of the dbsqlfuzz code +***************************************************************************/ + +/* Look at a SQL text and try to determine if it begins with a database +** description, such as would be found in a dbsqlfuzz test case. Return +** true if this does appear to be a dbsqlfuzz test case and false otherwise. +*/ +static int isDbSql(unsigned char *a, int n){ + unsigned char buf[12]; + int i; + if( n>4 && memcmp(a,"\n--\n",4)==0 ) return 1; + while( n>0 && isspace(a[0]) ){ a++; n--; } + for(i=0; n>0 && i<8; n--, a++){ + if( isxdigit(a[0]) ) buf[i++] = a[0]; + } + if( i==8 && memcmp(buf,"53514c69",8)==0 ) return 1; + return 0; +} + +/* Implementation of the isdbsql(TEXT) SQL function. +*/ +static void isDbSqlFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int n = sqlite3_value_bytes(argv[0]); + unsigned char *a = (unsigned char*)sqlite3_value_blob(argv[0]); + sqlite3_result_int(context, a!=0 && n>0 && isDbSql(a,n)); +} + +/* Methods for the VHandle object +*/ +static int inmemClose(sqlite3_file *pFile){ + VHandle *p = (VHandle*)pFile; + VFile *pVFile = p->pVFile; + pVFile->nRef--; + if( pVFile->nRef==0 && pVFile->zFilename==0 ){ + pVFile->sz = -1; + free(pVFile->a); + pVFile->a = 0; + } + return SQLITE_OK; +} +static int inmemRead( + sqlite3_file *pFile, /* Read from this open file */ + void *pData, /* Store content in this buffer */ + int iAmt, /* Bytes of content */ + sqlite3_int64 iOfst /* Start reading here */ +){ + VHandle *pHandle = (VHandle*)pFile; + VFile *pVFile = pHandle->pVFile; + if( iOfst<0 || iOfst>=pVFile->sz ){ + memset(pData, 0, iAmt); + return SQLITE_IOERR_SHORT_READ; + } + if( iOfst+iAmt>pVFile->sz ){ + memset(pData, 0, iAmt); + iAmt = (int)(pVFile->sz - iOfst); + memcpy(pData, pVFile->a + iOfst, iAmt); + return SQLITE_IOERR_SHORT_READ; + } + memcpy(pData, pVFile->a + iOfst, iAmt); + return SQLITE_OK; +} +static int inmemWrite( + sqlite3_file *pFile, /* Write to this file */ + const void *pData, /* Content to write */ + int iAmt, /* bytes to write */ + sqlite3_int64 iOfst /* Start writing here */ +){ + VHandle *pHandle = (VHandle*)pFile; + VFile *pVFile = pHandle->pVFile; + if( iOfst+iAmt > pVFile->sz ){ + if( iOfst+iAmt >= MX_FILE_SZ ){ + return SQLITE_FULL; + } + pVFile->a = safe_realloc(pVFile->a, (int)(iOfst+iAmt)); + if( iOfst > pVFile->sz ){ + memset(pVFile->a + pVFile->sz, 0, (int)(iOfst - pVFile->sz)); + } + pVFile->sz = (int)(iOfst + iAmt); + } + memcpy(pVFile->a + iOfst, pData, iAmt); + return SQLITE_OK; +} +static int inmemTruncate(sqlite3_file *pFile, sqlite3_int64 iSize){ + VHandle *pHandle = (VHandle*)pFile; + VFile *pVFile = pHandle->pVFile; + if( pVFile->sz>iSize && iSize>=0 ) pVFile->sz = (int)iSize; + return SQLITE_OK; +} +static int inmemSync(sqlite3_file *pFile, int flags){ + return SQLITE_OK; +} +static int inmemFileSize(sqlite3_file *pFile, sqlite3_int64 *pSize){ + *pSize = ((VHandle*)pFile)->pVFile->sz; + return SQLITE_OK; +} +static int inmemLock(sqlite3_file *pFile, int type){ + return SQLITE_OK; +} +static int inmemUnlock(sqlite3_file *pFile, int type){ + return SQLITE_OK; +} +static int inmemCheckReservedLock(sqlite3_file *pFile, int *pOut){ + *pOut = 0; + return SQLITE_OK; +} +static int inmemFileControl(sqlite3_file *pFile, int op, void *pArg){ + return SQLITE_NOTFOUND; +} +static int inmemSectorSize(sqlite3_file *pFile){ + return 512; +} +static int inmemDeviceCharacteristics(sqlite3_file *pFile){ + return + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + SQLITE_IOCAP_POWERSAFE_OVERWRITE; +} + + +/* Method table for VHandle +*/ +static sqlite3_io_methods VHandleMethods = { + /* iVersion */ 1, + /* xClose */ inmemClose, + /* xRead */ inmemRead, + /* xWrite */ inmemWrite, + /* xTruncate */ inmemTruncate, + /* xSync */ inmemSync, + /* xFileSize */ inmemFileSize, + /* xLock */ inmemLock, + /* xUnlock */ inmemUnlock, + /* xCheck... */ inmemCheckReservedLock, + /* xFileCtrl */ inmemFileControl, + /* xSectorSz */ inmemSectorSize, + /* xDevchar */ inmemDeviceCharacteristics, + /* xShmMap */ 0, + /* xShmLock */ 0, + /* xShmBarrier */ 0, + /* xShmUnmap */ 0, + /* xFetch */ 0, + /* xUnfetch */ 0 +}; + +/* +** Open a new file in the inmem VFS. All files are anonymous and are +** delete-on-close. +*/ +static int inmemOpen( + sqlite3_vfs *pVfs, + const char *zFilename, + sqlite3_file *pFile, + int openFlags, + int *pOutFlags +){ + VFile *pVFile = createVFile(zFilename, 0, (unsigned char*)""); + VHandle *pHandle = (VHandle*)pFile; + if( pVFile==0 ){ + return SQLITE_FULL; + } + pHandle->pVFile = pVFile; + pVFile->nRef++; + pFile->pMethods = &VHandleMethods; + if( pOutFlags ) *pOutFlags = openFlags; + return SQLITE_OK; +} + +/* +** Delete a file by name +*/ +static int inmemDelete( + sqlite3_vfs *pVfs, + const char *zFilename, + int syncdir +){ + VFile *pVFile = findVFile(zFilename); + if( pVFile==0 ) return SQLITE_OK; + if( pVFile->nRef==0 ){ + free(pVFile->zFilename); + pVFile->zFilename = 0; + pVFile->sz = -1; + free(pVFile->a); + pVFile->a = 0; + return SQLITE_OK; + } + return SQLITE_IOERR_DELETE; +} + +/* Check for the existance of a file +*/ +static int inmemAccess( + sqlite3_vfs *pVfs, + const char *zFilename, + int flags, + int *pResOut +){ + VFile *pVFile = findVFile(zFilename); + *pResOut = pVFile!=0; + return SQLITE_OK; +} + +/* Get the canonical pathname for a file +*/ +static int inmemFullPathname( + sqlite3_vfs *pVfs, + const char *zFilename, + int nOut, + char *zOut +){ + sqlite3_snprintf(nOut, zOut, "%s", zFilename); + return SQLITE_OK; +} + +/* Always use the same random see, for repeatability. +*/ +static int inmemRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ + memset(zBuf, 0, nBuf); + memcpy(zBuf, &g.uRandom, nBuf<sizeof(g.uRandom) ? nBuf : sizeof(g.uRandom)); + return nBuf; +} + +/* +** Register the VFS that reads from the g.aFile[] set of files. +*/ +static void inmemVfsRegister(int makeDefault){ + static sqlite3_vfs inmemVfs; + sqlite3_vfs *pDefault = sqlite3_vfs_find(0); + inmemVfs.iVersion = 3; + inmemVfs.szOsFile = sizeof(VHandle); + inmemVfs.mxPathname = 200; + inmemVfs.zName = "inmem"; + inmemVfs.xOpen = inmemOpen; + inmemVfs.xDelete = inmemDelete; + inmemVfs.xAccess = inmemAccess; + inmemVfs.xFullPathname = inmemFullPathname; + inmemVfs.xRandomness = inmemRandomness; + inmemVfs.xSleep = pDefault->xSleep; + inmemVfs.xCurrentTimeInt64 = pDefault->xCurrentTimeInt64; + sqlite3_vfs_register(&inmemVfs, makeDefault); +}; + +/* +** Allowed values for the runFlags parameter to runSql() +*/ +#define SQL_TRACE 0x0001 /* Print each SQL statement as it is prepared */ +#define SQL_OUTPUT 0x0002 /* Show the SQL output */ + +/* +** Run multiple commands of SQL. Similar to sqlite3_exec(), but does not +** stop if an error is encountered. +*/ +static void runSql(sqlite3 *db, const char *zSql, unsigned runFlags){ + const char *zMore; + sqlite3_stmt *pStmt; + + while( zSql && zSql[0] ){ + zMore = 0; + pStmt = 0; + sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zMore); + if( zMore==zSql ) break; + if( runFlags & SQL_TRACE ){ + const char *z = zSql; + int n; + while( z<zMore && ISSPACE(z[0]) ) z++; + n = (int)(zMore - z); + while( n>0 && ISSPACE(z[n-1]) ) n--; + if( n==0 ) break; + if( pStmt==0 ){ + printf("TRACE: %.*s (error: %s)\n", n, z, sqlite3_errmsg(db)); + }else{ + printf("TRACE: %.*s\n", n, z); + } + } + zSql = zMore; + if( pStmt ){ + if( (runFlags & SQL_OUTPUT)==0 ){ + while( SQLITE_ROW==sqlite3_step(pStmt) ){} + }else{ + int nCol = -1; + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + int i; + if( nCol<0 ){ + nCol = sqlite3_column_count(pStmt); + }else if( nCol>0 ){ + printf("--------------------------------------------\n"); + } + for(i=0; i<nCol; i++){ + int eType = sqlite3_column_type(pStmt,i); + printf("%s = ", sqlite3_column_name(pStmt,i)); + switch( eType ){ + case SQLITE_NULL: { + printf("NULL\n"); + break; + } + case SQLITE_INTEGER: { + printf("INT %s\n", sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_FLOAT: { + printf("FLOAT %s\n", sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_TEXT: { + printf("TEXT [%s]\n", sqlite3_column_text(pStmt,i)); + break; + } + case SQLITE_BLOB: { + printf("BLOB (%d bytes)\n", sqlite3_column_bytes(pStmt,i)); + break; + } + } + } + } + } + sqlite3_finalize(pStmt); + } + } +} + +/* +** Rebuild the database file. +** +** (1) Remove duplicate entries +** (2) Put all entries in order +** (3) Vacuum +*/ +static void rebuild_database(sqlite3 *db, int dbSqlOnly){ + int rc; + char *zSql; + zSql = sqlite3_mprintf( + "BEGIN;\n" + "CREATE TEMP TABLE dbx AS SELECT DISTINCT dbcontent FROM db;\n" + "DELETE FROM db;\n" + "INSERT INTO db(dbid, dbcontent) " + " SELECT NULL, dbcontent FROM dbx ORDER BY 2;\n" + "DROP TABLE dbx;\n" + "CREATE TEMP TABLE sx AS SELECT DISTINCT sqltext FROM xsql %s;\n" + "DELETE FROM xsql;\n" + "INSERT INTO xsql(sqlid,sqltext) " + " SELECT NULL, sqltext FROM sx ORDER BY 2;\n" + "DROP TABLE sx;\n" + "COMMIT;\n" + "PRAGMA page_size=1024;\n" + "VACUUM;\n", + dbSqlOnly ? " WHERE isdbsql(sqltext)" : "" + ); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc ) fatalError("cannot rebuild: %s", sqlite3_errmsg(db)); +} + +/* +** Return the value of a hexadecimal digit. Return -1 if the input +** is not a hex digit. +*/ +static int hexDigitValue(char c){ + if( c>='0' && c<='9' ) return c - '0'; + if( c>='a' && c<='f' ) return c - 'a' + 10; + if( c>='A' && c<='F' ) return c - 'A' + 10; + return -1; +} + +/* +** Interpret zArg as an integer value, possibly with suffixes. +*/ +static int integerValue(const char *zArg){ + sqlite3_int64 v = 0; + static const struct { char *zSuffix; int iMult; } aMult[] = { + { "KiB", 1024 }, + { "MiB", 1024*1024 }, + { "GiB", 1024*1024*1024 }, + { "KB", 1000 }, + { "MB", 1000000 }, + { "GB", 1000000000 }, + { "K", 1000 }, + { "M", 1000000 }, + { "G", 1000000000 }, + }; + int i; + int isNeg = 0; + if( zArg[0]=='-' ){ + isNeg = 1; + zArg++; + }else if( zArg[0]=='+' ){ + zArg++; + } + if( zArg[0]=='0' && zArg[1]=='x' ){ + int x; + zArg += 2; + while( (x = hexDigitValue(zArg[0]))>=0 ){ + v = (v<<4) + x; + zArg++; + } + }else{ + while( ISDIGIT(zArg[0]) ){ + v = v*10 + zArg[0] - '0'; + zArg++; + } + } + for(i=0; i<sizeof(aMult)/sizeof(aMult[0]); i++){ + if( sqlite3_stricmp(aMult[i].zSuffix, zArg)==0 ){ + v *= aMult[i].iMult; + break; + } + } + if( v>0x7fffffff ) fatalError("parameter too large - max 2147483648"); + return (int)(isNeg? -v : v); +} + +/* +** Return the number of "v" characters in a string. Return 0 if there +** are any characters in the string other than "v". +*/ +static int numberOfVChar(const char *z){ + int N = 0; + while( z[0] && z[0]=='v' ){ + z++; + N++; + } + return z[0]==0 ? N : 0; +} + +/* +** Print sketchy documentation for this utility program +*/ +static void showHelp(void){ + printf("Usage: %s [options] SOURCE-DB ?ARGS...?\n", g.zArgv0); + printf( +"Read databases and SQL scripts from SOURCE-DB and execute each script against\n" +"each database, checking for crashes and memory leaks.\n" +"Options:\n" +" --cell-size-check Set the PRAGMA cell_size_check=ON\n" +" --dbid N Use only the database where dbid=N\n" +" --export-db DIR Write databases to files(s) in DIR. Works with --dbid\n" +" --export-sql DIR Write SQL to file(s) in DIR. Also works with --sqlid\n" +" --help Show this help text\n" +" --info Show information about SOURCE-DB w/o running tests\n" +" --limit-depth N Limit expression depth to N. Default: 500\n" +" --limit-heap N Limit heap memory to N. Default: 100M\n" +" --limit-mem N Limit memory used by test SQLite instance to N bytes\n" +" --limit-vdbe Panic if any test runs for more than 100,000 cycles\n" +" --load-sql FILE.. Load SQL scripts fron files into SOURCE-DB\n" +" --load-db FILE.. Load template databases from files into SOURCE_DB\n" +" --load-dbsql FILE.. Load dbsqlfuzz outputs into the xsql table\n" +" ^^^^------ Use \"-\" for FILE to read filenames from stdin\n" +" -m TEXT Add a description to the database\n" +" --native-vfs Use the native VFS for initially empty database files\n" +" --native-malloc Turn off MEMSYS3/5 and Lookaside\n" +" --no-recover Do not run recovery on dbsqlfuzz databases\n" +" --oss-fuzz Enable OSS-FUZZ testing\n" +" --prng-seed N Seed value for the PRGN inside of SQLite\n" +" -q|--quiet Reduced output\n" +" --rebuild Rebuild and vacuum the database file\n" +" --result-trace Show the results of each SQL command\n" +" --script Output CLI script instead of running tests\n" +" --skip N Skip the first N test cases\n" +" --spinner Use a spinner to show progress\n" +" --sqlid N Use only SQL where sqlid=N\n" +" --timeout N Maximum time for any one test in N millseconds\n" +" -v|--verbose Increased output. Repeat for more output.\n" +" --vdbe-debug Activate VDBE debugging.\n" + ); +} + +int main(int argc, char **argv){ + sqlite3_int64 iBegin; /* Start time of this program */ + int quietFlag = 0; /* True if --quiet or -q */ + int verboseFlag = 0; /* True if --verbose or -v */ + char *zInsSql = 0; /* SQL statement for --load-db or --load-sql */ + int iFirstInsArg = 0; /* First argv[] for --load-db or --load-sql */ + sqlite3 *db = 0; /* The open database connection */ + sqlite3_stmt *pStmt; /* A prepared statement */ + int rc; /* Result code from SQLite interface calls */ + Blob *pSql; /* For looping over SQL scripts */ + Blob *pDb; /* For looping over template databases */ + int i; /* Loop index for the argv[] loop */ + int dbSqlOnly = 0; /* Only use scripts that are dbsqlfuzz */ + int onlySqlid = -1; /* --sqlid */ + int onlyDbid = -1; /* --dbid */ + int nativeFlag = 0; /* --native-vfs */ + int rebuildFlag = 0; /* --rebuild */ + int vdbeLimitFlag = 0; /* --limit-vdbe */ + int infoFlag = 0; /* --info */ + int nSkip = 0; /* --skip */ + int bScript = 0; /* --script */ + int bSpinner = 0; /* True for --spinner */ + int timeoutTest = 0; /* undocumented --timeout-test flag */ + int runFlags = 0; /* Flags sent to runSql() */ + char *zMsg = 0; /* Add this message */ + int nSrcDb = 0; /* Number of source databases */ + char **azSrcDb = 0; /* Array of source database names */ + int iSrcDb; /* Loop over all source databases */ + int nTest = 0; /* Total number of tests performed */ + char *zDbName = ""; /* Appreviated name of a source database */ + const char *zFailCode = 0; /* Value of the TEST_FAILURE env variable */ + int cellSzCkFlag = 0; /* --cell-size-check */ + int sqlFuzz = 0; /* True for SQL fuzz. False for DB fuzz */ + int iTimeout = 120000; /* Default 120-second timeout */ + int nMem = 0; /* Memory limit override */ + int nMemThisDb = 0; /* Memory limit set by the CONFIG table */ + char *zExpDb = 0; /* Write Databases to files in this directory */ + char *zExpSql = 0; /* Write SQL to files in this directory */ + void *pHeap = 0; /* Heap for use by SQLite */ + int ossFuzz = 0; /* enable OSS-FUZZ testing */ + int ossFuzzThisDb = 0; /* ossFuzz value for this particular database */ + int nativeMalloc = 0; /* Turn off MEMSYS3/5 and lookaside if true */ + sqlite3_vfs *pDfltVfs; /* The default VFS */ + int openFlags4Data; /* Flags for sqlite3_open_v2() */ + int bTimer = 0; /* Show elapse time for each test */ + int nV; /* How much to increase verbosity with -vvvv */ + sqlite3_int64 tmStart; /* Start of each test */ + + sqlite3_config(SQLITE_CONFIG_URI,1); + registerOomSimulator(); + sqlite3_initialize(); + iBegin = timeOfDay(); +#ifdef __unix__ + signal(SIGALRM, signalHandler); + signal(SIGSEGV, signalHandler); + signal(SIGABRT, signalHandler); +#endif + g.zArgv0 = argv[0]; + openFlags4Data = SQLITE_OPEN_READONLY; + zFailCode = getenv("TEST_FAILURE"); + pDfltVfs = sqlite3_vfs_find(0); + inmemVfsRegister(1); + for(i=1; i<argc; i++){ + const char *z = argv[i]; + if( z[0]=='-' ){ + z++; + if( z[0]=='-' ) z++; + if( strcmp(z,"cell-size-check")==0 ){ + cellSzCkFlag = 1; + }else + if( strcmp(z,"dbid")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + onlyDbid = integerValue(argv[++i]); + }else + if( strcmp(z,"export-db")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + zExpDb = argv[++i]; + }else + if( strcmp(z,"export-sql")==0 || strcmp(z,"export-dbsql")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + zExpSql = argv[++i]; + }else + if( strcmp(z,"help")==0 ){ + showHelp(); + return 0; + }else + if( strcmp(z,"info")==0 ){ + infoFlag = 1; + }else + if( strcmp(z,"limit-depth")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + depthLimit = integerValue(argv[++i]); + }else + if( strcmp(z,"limit-heap")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + heapLimit = integerValue(argv[++i]); + }else + if( strcmp(z,"limit-mem")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + nMem = integerValue(argv[++i]); + }else + if( strcmp(z,"limit-vdbe")==0 ){ + vdbeLimitFlag = 1; + }else + if( strcmp(z,"load-sql")==0 ){ + zInsSql = "INSERT INTO xsql(sqltext)" + "VALUES(CAST(readtextfile(?1) AS text))"; + iFirstInsArg = i+1; + openFlags4Data = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + break; + }else + if( strcmp(z,"load-db")==0 ){ + zInsSql = "INSERT INTO db(dbcontent) VALUES(readfile(?1))"; + iFirstInsArg = i+1; + openFlags4Data = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + break; + }else + if( strcmp(z,"load-dbsql")==0 ){ + zInsSql = "INSERT INTO xsql(sqltext)" + "VALUES(readfile(?1))"; + iFirstInsArg = i+1; + openFlags4Data = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + dbSqlOnly = 1; + break; + }else + if( strcmp(z,"m")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + zMsg = argv[++i]; + openFlags4Data = SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE; + }else + if( strcmp(z,"native-malloc")==0 ){ + nativeMalloc = 1; + }else + if( strcmp(z,"native-vfs")==0 ){ + nativeFlag = 1; + }else + if( strcmp(z,"no-recover")==0 ){ + bNoRecover = 1; + }else + if( strcmp(z,"oss-fuzz")==0 ){ + ossFuzz = 1; + }else + if( strcmp(z,"prng-seed")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + g.uRandom = atoi(argv[++i]); + }else + if( strcmp(z,"quiet")==0 || strcmp(z,"q")==0 ){ + quietFlag = 1; + verboseFlag = 0; + eVerbosity = 0; + }else + if( strcmp(z,"rebuild")==0 ){ + rebuildFlag = 1; + openFlags4Data = SQLITE_OPEN_READWRITE; + }else + if( strcmp(z,"result-trace")==0 ){ + runFlags |= SQL_OUTPUT; + }else + if( strcmp(z,"script")==0 ){ + bScript = 1; + }else + if( strcmp(z,"skip")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + nSkip = atoi(argv[++i]); + }else + if( strcmp(z,"spinner")==0 ){ + bSpinner = 1; + }else + if( strcmp(z,"timer")==0 ){ + bTimer = 1; + }else + if( strcmp(z,"sqlid")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + onlySqlid = integerValue(argv[++i]); + }else + if( strcmp(z,"timeout")==0 ){ + if( i>=argc-1 ) fatalError("missing arguments on %s", argv[i]); + iTimeout = integerValue(argv[++i]); + }else + if( strcmp(z,"timeout-test")==0 ){ + timeoutTest = 1; +#ifndef __unix__ + fatalError("timeout is not available on non-unix systems"); +#endif + }else + if( strcmp(z,"vdbe-debug")==0 ){ + bVdbeDebug = 1; + }else + if( strcmp(z,"verbose")==0 ){ + quietFlag = 0; + verboseFlag++; + eVerbosity++; + if( verboseFlag>1 ) runFlags |= SQL_TRACE; + }else + if( (nV = numberOfVChar(z))>=1 ){ + quietFlag = 0; + verboseFlag += nV; + eVerbosity += nV; + if( verboseFlag>1 ) runFlags |= SQL_TRACE; + }else + if( strcmp(z,"version")==0 ){ + int ii; + const char *zz; + printf("SQLite %s %s\n", sqlite3_libversion(), sqlite3_sourceid()); + for(ii=0; (zz = sqlite3_compileoption_get(ii))!=0; ii++){ + printf("%s\n", zz); + } + return 0; + }else + if( strcmp(z,"is-dbsql")==0 ){ + i++; + for(i++; i<argc; i++){ + long nData; + char *aData = readFile(argv[i], &nData); + printf("%d %s\n", isDbSql((unsigned char*)aData,nData), argv[i]); + sqlite3_free(aData); + } + exit(0); + }else + { + fatalError("unknown option: %s", argv[i]); + } + }else{ + nSrcDb++; + azSrcDb = safe_realloc(azSrcDb, nSrcDb*sizeof(azSrcDb[0])); + azSrcDb[nSrcDb-1] = argv[i]; + } + } + if( nSrcDb==0 ) fatalError("no source database specified"); + if( nSrcDb>1 ){ + if( zMsg ){ + fatalError("cannot change the description of more than one database"); + } + if( zInsSql ){ + fatalError("cannot import into more than one database"); + } + } + + /* Process each source database separately */ + for(iSrcDb=0; iSrcDb<nSrcDb; iSrcDb++){ + char *zRawData = 0; + long nRawData = 0; + g.zDbFile = azSrcDb[iSrcDb]; + rc = sqlite3_open_v2(azSrcDb[iSrcDb], &db, + openFlags4Data, pDfltVfs->zName); + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "SELECT count(*) FROM sqlite_schema", 0, 0, 0); + } + if( rc ){ + sqlite3_close(db); + zRawData = readFile(azSrcDb[iSrcDb], &nRawData); + if( zRawData==0 ){ + fatalError("input file \"%s\" is not recognized\n", azSrcDb[iSrcDb]); + } + sqlite3_open(":memory:", &db); + } + + /* Print the description, if there is one */ + if( infoFlag ){ + int n; + zDbName = azSrcDb[iSrcDb]; + i = (int)strlen(zDbName) - 1; + while( i>0 && zDbName[i-1]!='/' && zDbName[i-1]!='\\' ){ i--; } + zDbName += i; + sqlite3_prepare_v2(db, "SELECT msg FROM readme", -1, &pStmt, 0); + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%s: %s", zDbName, sqlite3_column_text(pStmt,0)); + }else{ + printf("%s: (empty \"readme\")", zDbName); + } + sqlite3_finalize(pStmt); + sqlite3_prepare_v2(db, "SELECT count(*) FROM db", -1, &pStmt, 0); + if( pStmt + && sqlite3_step(pStmt)==SQLITE_ROW + && (n = sqlite3_column_int(pStmt,0))>0 + ){ + printf(" - %d DBs", n); + } + sqlite3_finalize(pStmt); + sqlite3_prepare_v2(db, "SELECT count(*) FROM xsql", -1, &pStmt, 0); + if( pStmt + && sqlite3_step(pStmt)==SQLITE_ROW + && (n = sqlite3_column_int(pStmt,0))>0 + ){ + printf(" - %d scripts", n); + } + sqlite3_finalize(pStmt); + printf("\n"); + sqlite3_close(db); + sqlite3_free(zRawData); + continue; + } + + rc = sqlite3_exec(db, + "CREATE TABLE IF NOT EXISTS db(\n" + " dbid INTEGER PRIMARY KEY, -- database id\n" + " dbcontent BLOB -- database disk file image\n" + ");\n" + "CREATE TABLE IF NOT EXISTS xsql(\n" + " sqlid INTEGER PRIMARY KEY, -- SQL script id\n" + " sqltext TEXT -- Text of SQL statements to run\n" + ");" + "CREATE TABLE IF NOT EXISTS readme(\n" + " msg TEXT -- Human-readable description of this file\n" + ");", 0, 0, 0); + if( rc ) fatalError("cannot create schema: %s", sqlite3_errmsg(db)); + if( zMsg ){ + char *zSql; + zSql = sqlite3_mprintf( + "DELETE FROM readme; INSERT INTO readme(msg) VALUES(%Q)", zMsg); + rc = sqlite3_exec(db, zSql, 0, 0, 0); + sqlite3_free(zSql); + if( rc ) fatalError("cannot change description: %s", sqlite3_errmsg(db)); + } + if( zRawData ){ + zInsSql = "INSERT INTO xsql(sqltext) VALUES(?1)"; + rc = sqlite3_prepare_v2(db, zInsSql, -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare statement [%s]: %s", + zInsSql, sqlite3_errmsg(db)); + sqlite3_bind_text(pStmt, 1, zRawData, nRawData, SQLITE_STATIC); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc ) fatalError("insert failed for %s", argv[i]); + sqlite3_finalize(pStmt); + rebuild_database(db, dbSqlOnly); + zInsSql = 0; + sqlite3_free(zRawData); + zRawData = 0; + } + ossFuzzThisDb = ossFuzz; + + /* If the CONFIG(name,value) table exists, read db-specific settings + ** from that table */ + if( sqlite3_table_column_metadata(db,0,"config",0,0,0,0,0,0)==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, "SELECT name, value FROM config", + -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare query of CONFIG table: %s", + sqlite3_errmsg(db)); + while( SQLITE_ROW==sqlite3_step(pStmt) ){ + const char *zName = (const char *)sqlite3_column_text(pStmt,0); + if( zName==0 ) continue; + if( strcmp(zName, "oss-fuzz")==0 ){ + ossFuzzThisDb = sqlite3_column_int(pStmt,1); + if( verboseFlag ) printf("Config: oss-fuzz=%d\n", ossFuzzThisDb); + } + if( strcmp(zName, "limit-mem")==0 ){ + nMemThisDb = sqlite3_column_int(pStmt,1); + if( verboseFlag ) printf("Config: limit-mem=%d\n", nMemThisDb); + } + } + sqlite3_finalize(pStmt); + } + + if( zInsSql ){ + sqlite3_create_function(db, "readfile", 1, SQLITE_UTF8, 0, + readfileFunc, 0, 0); + sqlite3_create_function(db, "readtextfile", 1, SQLITE_UTF8, 0, + readtextfileFunc, 0, 0); + sqlite3_create_function(db, "isdbsql", 1, SQLITE_UTF8, 0, + isDbSqlFunc, 0, 0); + rc = sqlite3_prepare_v2(db, zInsSql, -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare statement [%s]: %s", + zInsSql, sqlite3_errmsg(db)); + rc = sqlite3_exec(db, "BEGIN", 0, 0, 0); + if( rc ) fatalError("cannot start a transaction"); + for(i=iFirstInsArg; i<argc; i++){ + if( strcmp(argv[i],"-")==0 ){ + /* A filename of "-" means read multiple filenames from stdin */ + char zLine[2000]; + while( rc==0 && fgets(zLine,sizeof(zLine),stdin)!=0 ){ + size_t kk = strlen(zLine); + while( kk>0 && zLine[kk-1]<=' ' ) kk--; + sqlite3_bind_text(pStmt, 1, zLine, (int)kk, SQLITE_STATIC); + if( verboseFlag ) printf("loading %.*s\n", (int)kk, zLine); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc ) fatalError("insert failed for %s", zLine); + } + }else{ + sqlite3_bind_text(pStmt, 1, argv[i], -1, SQLITE_STATIC); + if( verboseFlag ) printf("loading %s\n", argv[i]); + sqlite3_step(pStmt); + rc = sqlite3_reset(pStmt); + if( rc ) fatalError("insert failed for %s", argv[i]); + } + } + sqlite3_finalize(pStmt); + rc = sqlite3_exec(db, "COMMIT", 0, 0, 0); + if( rc ) fatalError("cannot commit the transaction: %s", + sqlite3_errmsg(db)); + rebuild_database(db, dbSqlOnly); + sqlite3_close(db); + return 0; + } + rc = sqlite3_exec(db, "PRAGMA query_only=1;", 0, 0, 0); + if( rc ) fatalError("cannot set database to query-only"); + if( zExpDb!=0 || zExpSql!=0 ){ + sqlite3_create_function(db, "writefile", 2, SQLITE_UTF8, 0, + writefileFunc, 0, 0); + if( zExpDb!=0 ){ + const char *zExDb = + "SELECT writefile(printf('%s/db%06d.db',?1,dbid),dbcontent)," + " dbid, printf('%s/db%06d.db',?1,dbid), length(dbcontent)" + " FROM db WHERE ?2<0 OR dbid=?2;"; + rc = sqlite3_prepare_v2(db, zExDb, -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare statement [%s]: %s", + zExDb, sqlite3_errmsg(db)); + sqlite3_bind_text64(pStmt, 1, zExpDb, strlen(zExpDb), + SQLITE_STATIC, SQLITE_UTF8); + sqlite3_bind_int(pStmt, 2, onlyDbid); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("write db-%d (%d bytes) into %s\n", + sqlite3_column_int(pStmt,1), + sqlite3_column_int(pStmt,3), + sqlite3_column_text(pStmt,2)); + } + sqlite3_finalize(pStmt); + } + if( zExpSql!=0 ){ + const char *zExSql = + "SELECT writefile(printf('%s/sql%06d.txt',?1,sqlid),sqltext)," + " sqlid, printf('%s/sql%06d.txt',?1,sqlid), length(sqltext)" + " FROM xsql WHERE ?2<0 OR sqlid=?2;"; + rc = sqlite3_prepare_v2(db, zExSql, -1, &pStmt, 0); + if( rc ) fatalError("cannot prepare statement [%s]: %s", + zExSql, sqlite3_errmsg(db)); + sqlite3_bind_text64(pStmt, 1, zExpSql, strlen(zExpSql), + SQLITE_STATIC, SQLITE_UTF8); + sqlite3_bind_int(pStmt, 2, onlySqlid); + while( sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("write sql-%d (%d bytes) into %s\n", + sqlite3_column_int(pStmt,1), + sqlite3_column_int(pStmt,3), + sqlite3_column_text(pStmt,2)); + } + sqlite3_finalize(pStmt); + } + sqlite3_close(db); + return 0; + } + + /* Load all SQL script content and all initial database images from the + ** source db + */ + blobListLoadFromDb(db, "SELECT sqlid, sqltext FROM xsql", onlySqlid, + &g.nSql, &g.pFirstSql); + if( g.nSql==0 ) fatalError("need at least one SQL script"); + blobListLoadFromDb(db, "SELECT dbid, dbcontent FROM db", onlyDbid, + &g.nDb, &g.pFirstDb); + if( g.nDb==0 ){ + g.pFirstDb = safe_realloc(0, sizeof(Blob)); + memset(g.pFirstDb, 0, sizeof(Blob)); + g.pFirstDb->id = 1; + g.pFirstDb->seq = 0; + g.nDb = 1; + sqlFuzz = 1; + } + + /* Print the description, if there is one */ + if( !quietFlag && !bScript ){ + zDbName = azSrcDb[iSrcDb]; + i = (int)strlen(zDbName) - 1; + while( i>0 && zDbName[i-1]!='/' && zDbName[i-1]!='\\' ){ i--; } + zDbName += i; + sqlite3_prepare_v2(db, "SELECT msg FROM readme", -1, &pStmt, 0); + if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ + printf("%s: %s\n", zDbName, sqlite3_column_text(pStmt,0)); + } + sqlite3_finalize(pStmt); + } + + /* Rebuild the database, if requested */ + if( rebuildFlag ){ + if( !quietFlag ){ + printf("%s: rebuilding... ", zDbName); + fflush(stdout); + } + rebuild_database(db, 0); + if( !quietFlag ) printf("done\n"); + } + + /* Close the source database. Verify that no SQLite memory allocations are + ** outstanding. + */ + sqlite3_close(db); + if( sqlite3_memory_used()>0 ){ + fatalError("SQLite has memory in use before the start of testing"); + } + + /* Limit available memory, if requested */ + sqlite3_shutdown(); + + if( nMemThisDb>0 && nMem==0 ){ + if( !nativeMalloc ){ + pHeap = realloc(pHeap, nMemThisDb); + if( pHeap==0 ){ + fatalError("failed to allocate %d bytes of heap memory", nMem); + } + sqlite3_config(SQLITE_CONFIG_HEAP, pHeap, nMemThisDb, 128); + }else{ + sqlite3_hard_heap_limit64((sqlite3_int64)nMemThisDb); + } + }else{ + sqlite3_hard_heap_limit64(0); + } + + /* Disable lookaside with the --native-malloc option */ + if( nativeMalloc ){ + sqlite3_config(SQLITE_CONFIG_LOOKASIDE, 0, 0); + } + + /* Reset the in-memory virtual filesystem */ + formatVfs(); + + /* Run a test using each SQL script against each database. + */ + if( !verboseFlag && !quietFlag && !bSpinner && !bScript ){ + printf("%s:", zDbName); + } + for(pSql=g.pFirstSql; pSql; pSql=pSql->pNext){ + tmStart = timeOfDay(); + if( isDbSql(pSql->a, pSql->sz) ){ + sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d",pSql->id); + if( bScript ){ + /* No progress output */ + }else if( bSpinner ){ + int nTotal =g.nSql; + int idx = pSql->seq; + printf("\r%s: %d/%d ", zDbName, idx, nTotal); + fflush(stdout); + }else if( verboseFlag ){ + printf("%s\n", g.zTestName); + fflush(stdout); + }else if( !quietFlag ){ + static int prevAmt = -1; + int idx = pSql->seq; + int amt = idx*10/(g.nSql); + if( amt!=prevAmt ){ + printf(" %d%%", amt*10); + fflush(stdout); + prevAmt = amt; + } + } + if( nSkip>0 ){ + nSkip--; + }else{ + runCombinedDbSqlInput(pSql->a, pSql->sz, iTimeout, bScript, pSql->id); + } + nTest++; + if( bTimer && !bScript ){ + sqlite3_int64 tmEnd = timeOfDay(); + printf("%lld %s\n", tmEnd - tmStart, g.zTestName); + } + g.zTestName[0] = 0; + disableOom(); + continue; + } + for(pDb=g.pFirstDb; pDb; pDb=pDb->pNext){ + int openFlags; + const char *zVfs = "inmem"; + sqlite3_snprintf(sizeof(g.zTestName), g.zTestName, "sqlid=%d,dbid=%d", + pSql->id, pDb->id); + if( bScript ){ + /* No progress output */ + }else if( bSpinner ){ + int nTotal = g.nDb*g.nSql; + int idx = pSql->seq*g.nDb + pDb->id - 1; + printf("\r%s: %d/%d ", zDbName, idx, nTotal); + fflush(stdout); + }else if( verboseFlag ){ + printf("%s\n", g.zTestName); + fflush(stdout); + }else if( !quietFlag ){ + static int prevAmt = -1; + int idx = pSql->seq*g.nDb + pDb->id - 1; + int amt = idx*10/(g.nDb*g.nSql); + if( amt!=prevAmt ){ + printf(" %d%%", amt*10); + fflush(stdout); + prevAmt = amt; + } + } + if( nSkip>0 ){ + nSkip--; + continue; + } + if( bScript ){ + char zName[100]; + sqlite3_snprintf(sizeof(zName), zName, "db%06d.db", + pDb->id>1 ? pDb->id : pSql->id); + renderDbSqlForCLI(stdout, zName, + pDb->a, pDb->sz, pSql->a, pSql->sz); + continue; + } + createVFile("main.db", pDb->sz, pDb->a); + sqlite3_randomness(0,0); + if( ossFuzzThisDb ){ +#ifndef SQLITE_OSS_FUZZ + fatalError("--oss-fuzz not supported: recompile" + " with -DSQLITE_OSS_FUZZ"); +#else + extern int LLVMFuzzerTestOneInput(const uint8_t*, size_t); + LLVMFuzzerTestOneInput((const uint8_t*)pSql->a, (size_t)pSql->sz); +#endif + }else{ + openFlags = SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE; + if( nativeFlag && pDb->sz==0 ){ + openFlags |= SQLITE_OPEN_MEMORY; + zVfs = 0; + } + rc = sqlite3_open_v2("main.db", &db, openFlags, zVfs); + if( rc ) fatalError("cannot open inmem database"); + sqlite3_limit(db, SQLITE_LIMIT_LENGTH, 100000000); + sqlite3_limit(db, SQLITE_LIMIT_LIKE_PATTERN_LENGTH, 50); + if( cellSzCkFlag ) runSql(db, "PRAGMA cell_size_check=ON", runFlags); + setAlarm((iTimeout+999)/1000); + /* Enable test functions */ + sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, db); +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( sqlFuzz || vdbeLimitFlag ){ + sqlite3_progress_handler(db, 100000, progressHandler, + &vdbeLimitFlag); + } +#endif +#ifdef SQLITE_TESTCTRL_PRNG_SEED + sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, 1, db); +#endif + if( bVdbeDebug ){ + sqlite3_exec(db, "PRAGMA vdbe_debug=ON", 0, 0, 0); + } + do{ + runSql(db, (char*)pSql->a, runFlags); + }while( timeoutTest ); + setAlarm(0); + sqlite3_exec(db, "PRAGMA temp_store_directory=''", 0, 0, 0); + sqlite3_close(db); + } + if( sqlite3_memory_used()>0 ){ + fatalError("memory leak: %lld bytes outstanding", + sqlite3_memory_used()); + } + reformatVfs(); + nTest++; + if( bTimer ){ + sqlite3_int64 tmEnd = timeOfDay(); + printf("%lld %s\n", tmEnd - tmStart, g.zTestName); + } + g.zTestName[0] = 0; + + /* Simulate an error if the TEST_FAILURE environment variable is "5". + ** This is used to verify that automated test script really do spot + ** errors that occur in this test program. + */ + if( zFailCode ){ + if( zFailCode[0]=='5' && zFailCode[1]==0 ){ + fatalError("simulated failure"); + }else if( zFailCode[0]!=0 ){ + /* If TEST_FAILURE is something other than 5, just exit the test + ** early */ + printf("\nExit early due to TEST_FAILURE being set\n"); + iSrcDb = nSrcDb-1; + goto sourcedb_cleanup; + } + } + } + } + if( bScript ){ + /* No progress output */ + }else if( bSpinner ){ + int nTotal = g.nDb*g.nSql; + printf("\r%s: %d/%d \n", zDbName, nTotal, nTotal); + }else if( !quietFlag && !verboseFlag ){ + printf(" 100%% - %d tests\n", g.nDb*g.nSql); + } + + /* Clean up at the end of processing a single source database + */ + sourcedb_cleanup: + blobListFree(g.pFirstSql); + blobListFree(g.pFirstDb); + reformatVfs(); + + } /* End loop over all source databases */ + + if( !quietFlag && !bScript ){ + sqlite3_int64 iElapse = timeOfDay() - iBegin; + if( g.nInvariant ){ + printf("fuzzcheck: %u query invariants checked\n", g.nInvariant); + } + printf("fuzzcheck: 0 errors out of %d tests in %d.%03d seconds\n" + "SQLite %s %s\n", + nTest, (int)(iElapse/1000), (int)(iElapse%1000), + sqlite3_libversion(), sqlite3_sourceid()); + } + free(azSrcDb); + free(pHeap); + return 0; +} |