1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
|
<!DOCTYPE html>
<html><head>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="content-type" content="text/html; charset=UTF-8">
<link href="sqlite.css" rel="stylesheet">
<title>TH3</title>
<!-- path= -->
</head>
<body>
<div class=nosearch>
<a href="index.html">
<img class="logo" src="images/sqlite370_banner.gif" alt="SQLite" border="0">
</a>
<div><!-- IE hack to prevent disappearing logo --></div>
<div class="tagline desktoponly">
Small. Fast. Reliable.<br>Choose any three.
</div>
<div class="menu mainmenu">
<ul>
<li><a href="index.html">Home</a>
<li class='mobileonly'><a href="javascript:void(0)" onclick='toggle_div("submenu")'>Menu</a>
<li class='wideonly'><a href='about.html'>About</a>
<li class='desktoponly'><a href="docs.html">Documentation</a>
<li class='desktoponly'><a href="download.html">Download</a>
<li class='wideonly'><a href='copyright.html'>License</a>
<li class='desktoponly'><a href="support.html">Support</a>
<li class='desktoponly'><a href="prosupport.html">Purchase</a>
<li class='search' id='search_menubutton'>
<a href="javascript:void(0)" onclick='toggle_search()'>Search</a>
</ul>
</div>
<div class="menu submenu" id="submenu">
<ul>
<li><a href='about.html'>About</a>
<li><a href='docs.html'>Documentation</a>
<li><a href='download.html'>Download</a>
<li><a href='support.html'>Support</a>
<li><a href='prosupport.html'>Purchase</a>
</ul>
</div>
<div class="searchmenu" id="searchmenu">
<form method="GET" action="search">
<select name="s" id="searchtype">
<option value="d">Search Documentation</option>
<option value="c">Search Changelog</option>
</select>
<input type="text" name="q" id="searchbox" value="">
<input type="submit" value="Go">
</form>
</div>
</div>
<script>
function toggle_div(nm) {
var w = document.getElementById(nm);
if( w.style.display=="block" ){
w.style.display = "none";
}else{
w.style.display = "block";
}
}
function toggle_search() {
var w = document.getElementById("searchmenu");
if( w.style.display=="block" ){
w.style.display = "none";
} else {
w.style.display = "block";
setTimeout(function(){
document.getElementById("searchbox").focus()
}, 30);
}
}
function div_off(nm){document.getElementById(nm).style.display="none";}
window.onbeforeunload = function(e){div_off("submenu");}
/* Disable the Search feature if we are not operating from CGI, since */
/* Search is accomplished using CGI and will not work without it. */
if( !location.origin || !location.origin.match || !location.origin.match(/http/) ){
document.getElementById("search_menubutton").style.display = "none";
}
/* Used by the Hide/Show button beside syntax diagrams, to toggle the */
function hideorshow(btn,obj){
var x = document.getElementById(obj);
var b = document.getElementById(btn);
if( x.style.display!='none' ){
x.style.display = 'none';
b.innerHTML='show';
}else{
x.style.display = '';
b.innerHTML='hide';
}
return false;
}
var antiRobot = 0;
function antiRobotGo(){
if( antiRobot!=3 ) return;
antiRobot = 7;
var j = document.getElementById("mtimelink");
if(j && j.hasAttribute("data-href")) j.href=j.getAttribute("data-href");
}
function antiRobotDefense(){
document.body.onmousedown=function(){
antiRobot |= 2;
antiRobotGo();
document.body.onmousedown=null;
}
document.body.onmousemove=function(){
antiRobot |= 2;
antiRobotGo();
document.body.onmousemove=null;
}
setTimeout(function(){
antiRobot |= 1;
antiRobotGo();
}, 100)
antiRobotGo();
}
antiRobotDefense();
</script>
<div class=fancy>
<div class=nosearch>
<div class="fancy_title">
TH3
</div>
<div class="fancy_toc">
<a onclick="toggle_toc()">
<span class="fancy_toc_mark" id="toc_mk">►</span>
Table Of Contents
</a>
<div id="toc_sub"><div class="fancy-toc1"><a href="#overview">1. Overview</a></div>
<div class="fancy-toc2"><a href="#history">1.1. History</a></div>
<div class="fancy-toc1"><a href="#operation">2. Operation</a></div>
<div class="fancy-toc1"><a href="#generating_a_test_program">3. Generating A Test Program</a></div>
<div class="fancy-toc2"><a href="#test_automation_scripts">3.1. Test Automation Scripts</a></div>
<div class="fancy-toc1"><a href="#test_coverage">4. Test Coverage</a></div>
<div class="fancy-toc1"><a href="#mutation_testing">5. Mutation Testing</a></div>
<div class="fancy-toc1"><a href="#th3_license">6. TH3 License</a></div>
</div>
</div>
<script>
function toggle_toc(){
var sub = document.getElementById("toc_sub")
var mk = document.getElementById("toc_mk")
if( sub.style.display!="block" ){
sub.style.display = "block";
mk.innerHTML = "▼";
} else {
sub.style.display = "none";
mk.innerHTML = "►";
}
}
</script>
</div>
<h1 id="overview"><span>1. </span>Overview</h1>
<p>SQLite Test Harness #3 (hereafter "TH3") is one of
<a href="testing.html#harnesses">three test harnesses</a> used for testing SQLite.
TH3 meets the following objectives:</p>
<ul>
<li><p> TH3 is able to run on embedded platforms that lack the support
infrastructure of workstations.</p></li>
<li><p> TH3 tests SQLite in an as-deployed configuration using only
published and documented interfaces.
In other words, TH3 tests the compiled object code, not
the source code, thus verifying that no problems were introduced
by compiler bugs. "Test what you fly and fly what you test."</p></li>
<li><p> TH3 checks SQLite's response to out-of-memory errors, disk I/O
errors, and power loss during transaction commit. </p></li>
<li><p> TH3 exercises SQLite in a variety of run-time configurations
(UTF8 vs UTF16, different pages sizes, varying journal modes, etc.)
</p></li>
<li><p> TH3 achieves 100% branch test coverage (and 100%
<a href="https://en.wikipedia.org/wiki/Modified_condition/decision_coverage">MC/DC</a>)
over the SQLite core.
(Test coverage of extensions such as FTS and RTREE is less than 100%).
</p></li>
</ul>
<p>TH3 was originally written for validation testing only, but has
subsequently been used for development testing and debugging
as well, and has proven very helpful in those roles. A full-coverage
test takes less than five minutes on a workstation and hence
serves as a fast regression test during day-to-day maintenance
of the SQLite code base.</p>
<h2 id="history"><span>1.1. </span>History</h2>
<p>TH3 originated from an effort to test SQLite on
<a href="https://en.wikipedia.org/wiki/Symbian">SymbianOS</a>.
Prior to TH3, all SQLite tests were run using the
<a href="http://www.tcl.tk/">TCL</a> script language, but TCL would not (easily)
compile on SymbianOS which made testing difficult. The first attempt
to remedy this problem was the "TH1" (Test Harness #1) scripting
language - a reimplementation of parts of the TCL language in a
more portable form that would compile and run on SymbianOS, and
that was sufficient to run the SQLite tests. TH1
did not survive as a standard testing tool for SQLite,
but it did find continued service as a
scripting language used to customize the
<a href="http://www.fossil-scm.org/">Fossil</a> version control system.
There was also a "Test Harness #2" which was an attempt to
create a simple scripting language using operator prefix notation
to drive tests. TH3 was the third attempt.
</p><p>At about that same time, some avionics manufacturers were
expressing interest in SQLite, which prompted the SQLite developers
to design TH3 to support the rigorous testing standards of
<a href="https://en.wikipedia.org/wiki/DO-178B">DO-178B</a>.
</p><p>The first code for TH3 was laid down on 2008-09-25.
An intense effort over the next 10 months resulted in TH3 achieving
100% MC/DC on 2009-07-25. The TH3 code continues to be improved and
expanded.
</p><p>As of 2018-05-19, the TH3 source tree consists
and well over 500,000 lines of source code in 1709 separate files.
</p><h1 id="operation"><span>2. </span>Operation</h1>
<p>TH3 is a test program generator. The output of TH3 is a program
implemented in C-code and intended to be
linked against the SQLite library under test. The generated test
program is compiled and run on the target platform in order to verify
correct operation of SQLite on that platform.</p>
<p>The inputs to TH3 are test modules written in C or SQL and
small configuration
files that determine how to initialize SQLite. The
TH3 package includes 1,444 test
modules and more than 47 configurations (as of 2018-05-19).
New modules and configurations
can be added to customize TH3 for specialized applications.
Each time TH3 is run, it reads
a subset of the available test modules and configuration files to generate
a custom C program that performs all of the specified tests under all
specified configurations. A complete test of SQLite normally involves running
TH3 multiple times to generate multiple test programs covering different
aspects of SQLite's operation, then linking all test programs against
a common SQLite library and running them separately on the target platform.
</p>
<p>There are no arbitrary limits in TH3. One could generate a
single test program that contained all test modules and all configuration files.
However, such a test program might be too large to deploy on embedded
platforms. (As of 2018-05-19, a full-up TH3 test is over 850,000 lines and
58MB of C code.) TH3 provides the ability to break the library of test
modules up into smaller, more easily digested pieces.</p>
<p>Each individual test module might contain dozens, hundreds, or thousands
of separate tests. The test modules can be written in C or as scripts of
SQL or a mixture of the two. About two-thirds of the existing test modules are
written in pure SQL with the remainder either in pure C or a combination
of C and SQL.
</p>
<p>Each test module file contains a header which describes the circumstances
under which the test is valid. For a particular configuration, only those
modules that are compatible with the configuration are run. </p>
<h1 id="generating_a_test_program"><span>3. </span>Generating A Test Program</h1>
<p>The TH3 program generator is a TCL script named "<tt>mkth3.tcl</tt>".
To generate a test program, one has merely to run this script and supply
the names of files containing test modules and configurations on the
command line. Test modules are files that use the "<tt>.test</tt>" suffix
and configurations are files that use the "<tt>.cfg</tt>" suffix. A
typical invocation of mkth3.tcl might look something like the following:</p>
<div class="codeblock"><pre>tclsh mkth3.tcl *.test *.cfg >testprog1.c
</pre></div>
<p>The output from the mkth3.tcl script is a C program that contains
everything needed to run the tests - everything that is except for
the SQLite library itself. The generated test program contains
implementations for all of the support interfaces used by the test
modules and it contains the <tt>main()</tt> routine that drives the
tests. To convert the test program into a working executable, simply
compile it against SQLite:</p>
<div class="codeblock"><pre>cc -o testprog1 testprog1.c sqlite3.c
</pre></div>
<p>The compilation step shown immediately above is merely representative.
In a working installation, one would normally want
to specify optimization parameters and compile-time switches on the
compiler command line.</p>
<p>For testing on embedded systems, the mkth3.tcl script and the compiler
steps shown above are performed on an ordinary workstation using
a cross-compiler, then the resulting test program is
transferred onto the device to be run.</p>
<p>Once the test program is generated, it is run with no arguments to
perform the tests. Progress information as well as error diagnostics
appear on standard output. (Alternative output arrangements can be made
using a compile-time option for embedded devices that lack a standard
output channel.) The program returns zero if there are no
errors and non-zero if any problems were detected.</p>
<p>Typical output from a single TH3 test program run looks like this:
</p><div class="codeblock"><pre>With SQLite 3.8.11 2015-05-15 04:13:15 56ef98a04765c34c1c2f3ed7a6f03a732f3b886e
-DSQLITE_COVERAGE_TEST
-DSQLITE_NO_SYNC
-DSQLITE_SYSTEM_MALLOC
-DSQLITE_THREADSAFE=1
Config-begin c1.
Begin c1.pager08
End c1.pager08
Begin c1.build33
End c1.build33
Begin c1.orderby01
End c1.orderby01
<i>... 15014 lines of output omitted ....</i>
Begin 64k.syscall01
End 64k.syscall01
Begin 64k.build01
End 64k.build01
Begin 64k.auth01
End 64k.auth01
Config-end 64k. TH3 memory used: 6373738
Config-begin wal1.
Begin wal1.wal37
End wal1.wal37
Config-end wal1. TH3 memory used: 100961
All 226 VDBE coverage points reached
th3: 0 errors out of 1442264 tests in 213.741 seconds. 64-bit little-endian
th3: SQLite 3.8.11 2015-05-15 04:13:15 56ef98a04765c34c1c2f3ed7a6f03a732f3b886e
</pre></div>
<p>The output begins with a report of the <a href="c3ref/c_source_id.html">SQLITE_SOURCE_ID</a>
(cross-checked again <a href="c3ref/libversion.html">sqlite3_sourceid()</a>) for the
SQLite under test and the compile-time options used as reported
by <a href="c3ref/compileoption_get.html">sqlite3_compileoption_get()</a>. The output concludes with a summary
of the test results and a repeat of the <a href="c3ref/c_source_id.html">SQLITE_SOURCE_ID</a>. If any
errors are detected, additional lines detail the problem. The error
reporting lines always begin with a single space character so that they
can be quickly extracted from large output files using:
</p><div class="codeblock"><pre>grep "^ "
</pre></div>
<p>The default output shows the beginning and end of each configuration
and test module combination. In the example above "c1" and "64k" are
configurations and "pager08", "build33", "orderby01", etc. are test modules.
Compile-time and run-time options are available to increase or decrease
the amount of output.
The output can be increased by showing each test case within each
test module. The output can be decreased
by degrees: omitting test modules starts and stops,
omitting configuration starts and stops, and finally by omitting all output.
</p><h2 id="test_automation_scripts"><span>3.1. </span>Test Automation Scripts</h2>
<p>TH3 comes with additional TCL scripts that help automate the testing
process on workstations. The "th3make" script automatically runs "mkth3.tcl"
and "gcc" and then runs the resulting test program and checks the results.
Arguments to th3make include all of the "*.test" test modules and
"*.cfg" configurations that are to be included in the test. Additional
options to th3make can cause the test program to be compiled using different
compilers (GCC, Clang, MSVC), to use different output verbosity levels, to
run the test program under valgrind, to check the output for coverage using
gcov, and so forth. The th3make script also accepts "*.rc" filenames as
arguments. These *.rc files are just collections of other arguments that
are commonly used together for a single purpose. For example, the "quick.rc"
file contains a set of eight arguments to th3make that run a fast (3-minute)
full-coverage test. This allows the operator to type "./th3make quick.rc" as
a short-cut to typing out all of the required command-line options. The
following are a few of the more than 40 available *.rc files:
</p><ul>
<li><b>alignment</b><i>N</i><b>.rc</b> -
These files contain -D options to the compiler that are used by
various notable downstreams.
</li><li><b>cov.rc</b> - Options for measuring test coverage
</li><li><b>extensions.rc</b> - Options to enable <a href="fts3.html#fts4">FTS4</a>, <a href="rtree.html">R-Trees</a>,
and <a href="fileformat2.html#stat4tab">STAT4</a>.
</li><li><b>fast.rc</b> - Run most tests, including those not needed for
coverage, skipping only soak tests, using delivery compiler options
(ex: -O3)
</li><li><b>memdebug.rc</b> - like test.rc but also enable
<a href="compile.html#memdebug">-DSQLITE_MEMDEBUG</a>.
</li><li><b>min.rc</b> - Run the minimum set of tests needed for 100% coverage.
</li><li><b>quick.rc</b> - Run all tests required for 100% coverage tests
using -Os and <a href="compile.html#debug">-DSQLITE_DEBUG</a>.
</li><li><b>test.rc</b> - Run the same tests as in fast.rc but without
compiler optimization and enabling options like
<a href="compile.html#debug">-DSQLITE_DEBUG</a> and
-DSQLITE_ENABLE_EXPENSIVE_ASSERT.
</li><li><b>test-ex.rc</b> - long-running soak tests.
</li></ul>
<p>The TH3 repository also includes the "multitest.tcl" script, another
TCL script used to automate TH3 testing on workstations. Multitest.tcl
automatically compiles SQLite, then
runs ./th3make repeatedly with a variety of alignments, and captures
the output in a succinct summary screen. A typical multitest.tcl run
generates output that looks like this:
</p><div class="codeblock"><pre>./multitest.tcl -q --jobs 3
start-time: 2018-05-19 03:17:12 UTC
file mkdir sqlite3bld
cd sqlite3bld
exec sh /ramdisk/sqlite/configure
file copy -force config.h ../config.h
exec make clean sqlite3.c
file rename sqlite3.c ../sqlite3.c
file rename sqlite3.h ../sqlite3.h
exec make clean sqlite3.c OPTS=-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1
file rename sqlite3.c ../sqlite3udl.c
exec make clean sqlite3.c OPTS=-DSQLITE_SMALL_STACK=1
file rename sqlite3.c ../sqlite3ss.c
cd ..
*******************************************************************************
t01: cov.rc.................................................... Ok (00:03:42)
t02: cov.rc ++STAT4 ++DESERIALIZE -D_HAVE_SQLITE_CONFIG_H...... Ok (00:04:45)
t03: vfs-cov.rc................................................ Ok (00:03:59)
t04: demo.rc................................................... Ok (00:00:05)
t07: test.rc ../th3private/*.test.............................. Ok (00:00:21)
t08: test.rc ../th3private/*.test ++STAT4...................... Ok (00:01:41)
t05: quick.rc.................................................. Ok (00:04:26)
t09: quick.rc ~TEST_REALLOC_STRESS -funsigned-char............. Ok (00:05:39)
t10: quick.rc ~THREADSAFE=0 -DLONGDOUBLE_TYPE=double........... Ok (00:03:24)
t06: quick.rc extensions.rc -D_HAVE_SQLITE_CONFIG_H............ Ok (00:09:03)
t11: quick.rc sqlite3ss.c ~MAX_ATTACHED=125.................... Ok (00:04:39)
t12: quick.rc ~BYTEORDER=0 ++RTREE............................. Ok (00:07:28)
t13: quick.rc ~DISABLE_INTRINSIC ++RTREE....................... Ok (00:07:31)
t16: quick.rc ~TRACE_SIZE_LIMIT=15 cov1/main16.test............ Ok (00:00:22)
t14: quick.rc ~DIRECT_OVERFLOW_READ -fsigned-char.............. Ok (00:04:35)
t15: quick.rc ~UNTESTABLE ~EXTRA_IFNULLROW..................... Ok (00:01:44)
t17: quick.rc ~MAX_MMAP_SIZE=0................................. Ok (00:04:46)
t18: quick.rc ++NULL_TRIM ++OFFSET_SQL_FUNC.................... Ok (00:04:47)
t19: quick.rc ++BATCH_ATOMIC_WRITE ++DESERIALIZE............... Ok (00:05:41)
t20: lean1.rc quick.rc......................................... Ok (00:03:09)
t22: test.rc alignment2.rc sqlite3udl.c........................ Ok (00:44:22)
t21: test.rc alignment1.rc..................................... Ok (01:02:32)
t23: memdebug1.rc extensions.rc................................ Ok (01:49:58)
t25: valgrind1.rc -O3 extensions.rc............................ Ok (00:56:08)
t24: memdebug2.rc extensions.rc................................ Ok (01:43:34)
t27: test-ex1.rc............................................... Ok (00:45:00)
t26: valgrind2.rc -O3 extensions.rc............................ Ok (01:02:52)
t29: test-ex3.rc............................................... Ok (00:31:48)
t28: test-ex2.rc............................................... Ok (01:12:03)
t30: test-ex4.rc............................................... Ok (01:09:47)
t32: test.rc alignment4.rc -m32 CC=clang....................... Ok (00:48:31)
t31: test.rc alignment3.rc sqlite3udl.c........................ Ok (01:22:29)
t34: test.rc alignment6.rc..................................... Ok (00:35:31)
t33: test.rc alignment5.rc extensions.rc....................... Ok (00:59:33)
t35: test.rc alignment7.rc..................................... Ok (00:44:10)
t40: fast.rc alignment2.rc sqlite3udl.c........................ Ok (00:15:46)
t39: fast.rc alignment1.rc extensions.rc -m32.................. Ok (00:33:19)
t36: test.rc ~MUTATION_TEST.................................... Ok (00:35:45)
t42: fast.rc alignment4.rc..................................... Ok (00:13:03)
t43: fast.rc alignment5.rc..................................... Ok (00:13:32)
t44: fast.rc alignment6.rc..................................... Ok (00:11:41)
t41: fast.rc alignment3.rc sqlite3udl.c........................ Ok (00:26:31)
t45: fast.rc alignment7.rc..................................... Ok (00:12:57)
t46: fast.rc -fsanitize=undefined.............................. Ok (00:38:18)
*******************************************************************************
0 failures on 44 th3makes and 198583082 tests in (07:16:01) 3 cores on bella
SQLite 3.24.0 2018-05-18 17:58:33 c6071ac99cfa4b6272ac4d739fc61a85acb544f6c1c2a
</pre></div>
<p>As can be seen above, a single run
of multitest.tcl invokes th3make dozens of times and takes between 12 and 24
CPU hours. The middle section of the output shows the arguments to each
individual th3make run and the result and elapse time for that th3make.
All build products and output for the separate th3make runs are
captures in subdirectories for post-test analysis.
The two-line summary at the bottom shows the total number of errors and tests
over all th3make runs and the total elapse time, together with the
<a href="c3ref/c_source_id.html">SQLITE_SOURCE_ID</a> information for the version of SQLite that was
tested. This summary information is recorded in the
<a href="https://www.sqlite.org/checklists">release
checklist</a> during final testing.
</p><p>Abbreviations are applied in the multitest.tcl output so that
each th3make invocation will fit on a single 80-column output line.
The initial "th3make" verb is omitted.
"~" is shorthand for "-DSQLITE_" and "++" is stands for
"-DSQLITE_ENABLE". Hence, multitest.tcl output line
</p><div class="codeblock"><pre>quick.rc ~DISABLE_INTRINSIC ++RTREE
</pre></div>
<p>Really means
</p><div class="codeblock"><pre>th3make quick.rc -DSQLITE_DISABLE_INTRINSIC -DSQLITE_ENABLE_RTREE
</pre></div>
<h1 id="test_coverage"><span>4. </span>Test Coverage</h1>
<p>Using one particular subset of the available TH3 test modules (the "cov1"
tests) SQLite obtained
<a href="testing.html#coverage">100% branch test coverage</a> and 100% <a href="testing.html#mcdc">MC/DC</a> as measured
by <a href="http://gcc.gnu.org/onlinedocs/gcc/Gcov.html">gcov</a>
on Linux x86 and x86_64 hardware. All releases of SQLite since
<a href="releaselog/3_6_17.html">version 3.6.17</a> (2009-08-10) have been tested to this standard.
The SQLite developers
are committed to maintaining 100% branch coverage and MC/DC for all
future releases of SQLite.</p>
<p>The cov1 test set used to obtain 100% branch test coverage are only a
subset of the tests currently implemented using TH3. New test modules are
added on a regular basis.</p>
<a name="muttest"></a>
<h1 id="mutation_testing"><span>5. </span>Mutation Testing</h1>
<p>The TH3 source tree contains a scripted name
"mutation-test.tcl" that automates the process of
<a href="testing.html#mutationtests">mutation testing</a>.
</p><p>The mutation-test.tcl script takes care of all of the details for
running a mutation test:
</p><ol>
<li> The script compiles the TH3 test harness into machine code ("th3.o") if
necessary.
</li><li> The script compiles the sqlite3.c source file into assembly language
("sqlite3.s") if necessary.
</li><li> The script loops through instructions in the assembly language file
to locate branch operations.
<ol type="a">
<li>The script makes a copy of the original sqlite3.s file.
</li><li>The copy is edited to change the branch instruction into either
a no-op or an unconditional jump.
</li><li>The copy of sqlite3.s is assembled into sqlite3.o then linked
again th3.o to generate the "th3" executable.
</li><li>The "th3" binary is run and the output checked for errors.
</li></ol>
</li><li> The script shows progress for each cycle of the previous step then
displays a summary of "survivors" at the end. A "survivor" is a
mutation that was not detected by TH3.
</li></ol>
<p>Mutation testing can be slow, since each test can take up to 5
minutes on a fast workstation, and there are two tests for each
branch instructions, and over 20,000 branch instructions. Efforts are
made to expedite operation. For example, TH3 is compiled in such a
way that it exits as soon as it finds the first error, and as many
of the mutations are easily detected, many cycles happen in only
a few seconds. Nevertheless, the mutation-test.tcl script includes
command-line options to limit the range of code lines tested so that
mutation testing only needs to be performed on blocks of code that
have recently changed.
</p><h1 id="th3_license"><span>6. </span>TH3 License</h1>
<p>SQLite itself is in the <a href="copyright.html">public domain</a> and
can be used for any purpose. But TH3 is proprietary and requires a license.
</p>
<p>Even though open-source users do not have direct access to TH3, all
users of SQLite benefit from TH3 indirectly since each version of SQLite is
validated running TH3 on multiple platforms (Linux, Windows, WinRT, Mac,
OpenBSD) prior to release. So anyone using an official release
of SQLite can deploy their application with the confidence of knowing that
it has been tested using TH3. They simply cannot rerun those tests
themselves without purchasing a TH3 license.</p>
<p align="center"><small><i>This page last modified on <a href="https://sqlite.org/docsrc/honeypot" id="mtimelink" data-href="https://sqlite.org/docsrc/finfo/pages/th3.in?m=7955ba7a18">2022-01-08 05:02:57</a> UTC </small></i></p>
|
