diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 17:28:19 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-05-05 17:28:19 +0000 |
commit | 18657a960e125336f704ea058e25c27bd3900dcb (patch) | |
tree | 17b438b680ed45a996d7b59951e6aa34023783f2 /ext/wasm/api | |
parent | Initial commit. (diff) | |
download | sqlite3-18657a960e125336f704ea058e25c27bd3900dcb.tar.xz sqlite3-18657a960e125336f704ea058e25c27bd3900dcb.zip |
Adding upstream version 3.40.1.upstream/3.40.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'ext/wasm/api')
-rw-r--r-- | ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api | 94 | ||||
-rw-r--r-- | ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api | 3 | ||||
-rw-r--r-- | ext/wasm/api/README.md | 133 | ||||
-rw-r--r-- | ext/wasm/api/extern-post-js.js | 103 | ||||
-rw-r--r-- | ext/wasm/api/extern-pre-js.js | 7 | ||||
-rw-r--r-- | ext/wasm/api/post-js-footer.js | 4 | ||||
-rw-r--r-- | ext/wasm/api/post-js-header.js | 25 | ||||
-rw-r--r-- | ext/wasm/api/pre-js.js | 100 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-cleanup.js | 70 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-glue.js | 720 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-oo1.js | 1800 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-opfs.js | 1311 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-prologue.js | 1602 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-api-worker1.js | 654 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-license-version-header.js | 25 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-opfs-async-proxy.js | 830 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-wasi.h | 69 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-wasm.c | 1181 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-worker1-promiser.js | 259 | ||||
-rw-r--r-- | ext/wasm/api/sqlite3-worker1.js | 49 |
20 files changed, 9039 insertions, 0 deletions
diff --git a/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api new file mode 100644 index 0000000..b903bed --- /dev/null +++ b/ext/wasm/api/EXPORTED_FUNCTIONS.sqlite3-api @@ -0,0 +1,94 @@ +_sqlite3_aggregate_context +_sqlite3_bind_blob +_sqlite3_bind_double +_sqlite3_bind_int +_sqlite3_bind_int64 +_sqlite3_bind_null +_sqlite3_bind_parameter_count +_sqlite3_bind_parameter_index +_sqlite3_bind_text +_sqlite3_changes +_sqlite3_changes64 +_sqlite3_clear_bindings +_sqlite3_close_v2 +_sqlite3_column_blob +_sqlite3_column_bytes +_sqlite3_column_count +_sqlite3_column_count +_sqlite3_column_double +_sqlite3_column_int +_sqlite3_column_int64 +_sqlite3_column_name +_sqlite3_column_text +_sqlite3_column_type +_sqlite3_compileoption_get +_sqlite3_compileoption_used +_sqlite3_create_function +_sqlite3_create_function_v2 +_sqlite3_create_window_function +_sqlite3_data_count +_sqlite3_db_filename +_sqlite3_db_handle +_sqlite3_db_name +_sqlite3_deserialize +_sqlite3_errmsg +_sqlite3_error_offset +_sqlite3_errstr +_sqlite3_exec +_sqlite3_expanded_sql +_sqlite3_extended_errcode +_sqlite3_extended_result_codes +_sqlite3_file_control +_sqlite3_finalize +_sqlite3_free +_sqlite3_initialize +_sqlite3_libversion +_sqlite3_libversion_number +_sqlite3_malloc +_sqlite3_malloc64 +_sqlite3_msize +_sqlite3_open +_sqlite3_open_v2 +_sqlite3_prepare_v2 +_sqlite3_prepare_v3 +_sqlite3_randomness +_sqlite3_realloc +_sqlite3_realloc64 +_sqlite3_reset +_sqlite3_result_blob +_sqlite3_result_double +_sqlite3_result_error +_sqlite3_result_error_code +_sqlite3_result_error_nomem +_sqlite3_result_error_toobig +_sqlite3_result_int +_sqlite3_result_int64 +_sqlite3_result_null +_sqlite3_result_text +_sqlite3_serialize +_sqlite3_shutdown +_sqlite3_sourceid +_sqlite3_sql +_sqlite3_step +_sqlite3_strglob +_sqlite3_strlike +_sqlite3_total_changes +_sqlite3_total_changes64 +_sqlite3_trace_v2 +_sqlite3_uri_boolean +_sqlite3_uri_int64 +_sqlite3_uri_key +_sqlite3_uri_parameter +_sqlite3_user_data +_sqlite3_value_blob +_sqlite3_value_bytes +_sqlite3_value_double +_sqlite3_value_int +_sqlite3_value_int64 +_sqlite3_value_text +_sqlite3_value_type +_sqlite3_vfs_find +_sqlite3_vfs_register +_sqlite3_vfs_unregister +_malloc +_free diff --git a/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api new file mode 100644 index 0000000..aab1d8b --- /dev/null +++ b/ext/wasm/api/EXPORTED_RUNTIME_METHODS.sqlite3-api @@ -0,0 +1,3 @@ +FS +wasmMemory + diff --git a/ext/wasm/api/README.md b/ext/wasm/api/README.md new file mode 100644 index 0000000..6440eba --- /dev/null +++ b/ext/wasm/api/README.md @@ -0,0 +1,133 @@ +# sqlite3-api.js And Friends + +This is the README for the files `sqlite3-*.js` and +`sqlite3-wasm.c`. This collection of files is used to build a +single-file distribution of the sqlite3 WASM API. It is broken into +multiple JS files because: + +1. To facilitate including or excluding certain components for + specific use cases. e.g. by removing `sqlite3-api-oo1.js` if the + OO#1 API is not needed. + +2. To facilitate modularizing the pieces for use in different WASM + build environments. e.g. the files `post-js-*.js` are for use with + Emscripten's `--post-js` feature, and nowhere else. + +3. Certain components must be in their own standalone files in order + to be loaded as JS Workers. + +Note that the structure described here is the current state of things, +not necessarily the "final" state. + +The overall idea is that the following files get concatenated +together, in the listed order, the resulting file is loaded by a +browser client: + +- `sqlite3-api-prologue.js`\ + Contains the initial bootstrap setup of the sqlite3 API + objects. This is exposed as a function, rather than objects, so that + the next step can pass in a config object which abstracts away parts + of the WASM environment, to facilitate plugging it in to arbitrary + WASM toolchains. +- `../common/whwasmutil.js`\ + A semi-third-party collection of JS/WASM utility code intended to + replace much of the Emscripten glue. The sqlite3 APIs internally use + these APIs instead of their Emscripten counterparts, in order to be + more portable to arbitrary WASM toolchains. This API is + configurable, in principle, for use with arbitrary WASM + toolchains. It is "semi-third-party" in that it was created in order + to support this tree but is standalone and maintained together + with... +- `../jaccwabyt/jaccwabyt.js`\ + Another semi-third-party API which creates bindings between JS + and C structs, such that changes to the struct state from either JS + or C are visible to the other end of the connection. This is also an + independent spinoff project, conceived for the sqlite3 project but + maintained separately. +- `sqlite3-api-glue.js`\ + Invokes functionality exposed by the previous two files to + flesh out low-level parts of `sqlite3-api-prologue.js`. Most of + these pieces related to the `sqlite3.capi.wasm` object. +- `sqlite3-api-build-version.js`\ + Gets created by the build process and populates the + `sqlite3.version` object. This part is not critical, but records the + version of the library against which this module was built. +- `sqlite3-api-oo1.js`\ + Provides a high-level object-oriented wrapper to the lower-level C + API, colloquially known as OO API #1. Its API is similar to other + high-level sqlite3 JS wrappers and should feel relatively familiar + to anyone familiar with such APIs. That said, it is not a "required + component" and can be elided from builds which do not want it. +- `sqlite3-api-worker1.js`\ + A Worker-thread-based API which uses OO API #1 to provide an + interface to a database which can be driven from the main Window + thread via the Worker message-passing interface. Like OO API #1, + this is an optional component, offering one of any number of + potential implementations for such an API. + - `sqlite3-worker1.js`\ + Is not part of the amalgamated sources and is intended to be + loaded by a client Worker thread. It loads the sqlite3 module + and runs the Worker #1 API which is implemented in + `sqlite3-api-worker1.js`. + - `sqlite3-worker1-promiser.js`\ + Is likewise not part of the amalgamated sources and provides + a Promise-based interface into the Worker #1 API. This is + a far user-friendlier way to interface with databases running + in a Worker thread. +- `sqlite3-api-opfs.js`\ + is an sqlite3 VFS implementation which supports Google Chrome's + Origin-Private FileSystem (OPFS) as a storage layer to provide + persistent storage for database files in a browser. It requires... + - `sqlite3-opfs-async-proxy.js`\ + is the asynchronous backend part of the OPFS proxy. It speaks + directly to the (async) OPFS API and channels those results back + to its synchronous counterpart. This file, because it must be + started in its own Worker, is not part of the amalgamation. +- **`api/sqlite3-api-cleanup.js`**\ + The previous files do not immediately extend the library. Instead + they add callback functions to be called during its + bootstrapping. Some also temporarily create global objects in order + to communicate their state to the files which follow them. This file + cleans up any dangling globals and runs the API bootstrapping + process, which is what finally executes the initialization code + installed by the previous files. As of this writing, this code + ensures that the previous files leave no more than a single global + symbol installed. When adapting the API for non-Emscripten + toolchains, this "should" be the only file where changes are needed. + +The build process glues those files together, resulting in +`sqlite3-api.js`, which is everything except for the `post-js-*.js` +files, and `sqlite3.js`, which is the Emscripten-generated amalgamated +output and includes the `post-js-*.js` parts, as well as the +Emscripten-provided module loading pieces. + +The non-JS outlier file is `sqlite3-wasm.c`: it is a proxy for +`sqlite3.c` which `#include`'s that file and adds a couple more +WASM-specific helper functions, at least one of which requires access +to private/static `sqlite3.c` internals. `sqlite3.wasm` is compiled +from this file rather than `sqlite3.c`. + +The following files are part of the build process but are injected +into the build-generated `sqlite3.js` along with `sqlite3-api.js`. + +- `extern-pre-js.js`\ + Emscripten-specific header for Emscripten's `--extern-pre-js` + flag. As of this writing, that file is only used for experimentation + purposes and holds no code relevant to the production deliverables. +- `pre-js.js`\ + Emscripten-specific header for Emscripten's `--pre-js` flag. This + file is intended as a place to override certain Emscripten behavior + before it starts up, but corner-case Emscripten bugs keep that from + being a reality. +- `post-js-header.js`\ + Emscripten-specific header for the `--post-js` input. It opens up + a lexical scope by starting a post-run handler for Emscripten. +- `post-js-footer.js`\ + Emscripten-specific footer for the `--post-js` input. This closes + off the lexical scope opened by `post-js-header.js`. +- `extern-post-js.js`\ + Emscripten-specific header for Emscripten's `--extern-post-js` + flag. This file overwrites the Emscripten-installed + `sqlite3InitModule()` function with one which, after the module is + loaded, also initializes the asynchronous parts of the sqlite3 + module. For example, the OPFS VFS support. diff --git a/ext/wasm/api/extern-post-js.js b/ext/wasm/api/extern-post-js.js new file mode 100644 index 0000000..84b99b5 --- /dev/null +++ b/ext/wasm/api/extern-post-js.js @@ -0,0 +1,103 @@ +/* extern-post-js.js must be appended to the resulting sqlite3.js + file. It gets its name from being used as the value for the + --extern-post-js=... Emscripten flag. Note that this code, unlike + most of the associated JS code, runs outside of the + Emscripten-generated module init scope, in the current + global scope. */ +(function(){ + /** + In order to hide the sqlite3InitModule()'s resulting Emscripten + module from downstream clients (and simplify our documentation by + being able to elide those details), we rewrite + sqlite3InitModule() to return the sqlite3 object. + + Unfortunately, we cannot modify the module-loader/exporter-based + impls which Emscripten installs at some point in the file above + this. + */ + const originalInit = self.sqlite3InitModule; + if(!originalInit){ + throw new Error("Expecting self.sqlite3InitModule to be defined by the Emscripten build."); + } + /** + We need to add some state which our custom Module.locateFile() + can see, but an Emscripten limitation currently prevents us from + attaching it to the sqlite3InitModule function object: + + https://github.com/emscripten-core/emscripten/issues/18071 + + The only(?) current workaround is to temporarily stash this state + into the global scope and delete it when sqlite3InitModule() + is called. + */ + const initModuleState = self.sqlite3InitModuleState = Object.assign(Object.create(null),{ + moduleScript: self?.document?.currentScript, + isWorker: ('undefined' !== typeof WorkerGlobalScope), + location: self.location, + urlParams: new URL(self.location.href).searchParams + }); + initModuleState.debugModule = + (new URL(self.location.href).searchParams).has('sqlite3.debugModule') + ? (...args)=>console.warn('sqlite3.debugModule:',...args) + : ()=>{}; + + if(initModuleState.urlParams.has('sqlite3.dir')){ + initModuleState.sqlite3Dir = initModuleState.urlParams.get('sqlite3.dir') +'/'; + }else if(initModuleState.moduleScript){ + const li = initModuleState.moduleScript.src.split('/'); + li.pop(); + initModuleState.sqlite3Dir = li.join('/') + '/'; + } + + self.sqlite3InitModule = (...args)=>{ + //console.warn("Using replaced sqlite3InitModule()",self.location); + return originalInit(...args).then((EmscriptenModule)=>{ + if(self.window!==self && + (EmscriptenModule['ENVIRONMENT_IS_PTHREAD'] + || EmscriptenModule['_pthread_self'] + || 'function'===typeof threadAlert + || self.location.pathname.endsWith('.worker.js') + )){ + /** Workaround for wasmfs-generated worker, which calls this + routine from each individual thread and requires that its + argument be returned. All of the criteria above are fragile, + based solely on inspection of the offending code, not public + Emscripten details. */ + return EmscriptenModule; + } + EmscriptenModule.sqlite3.scriptInfo = initModuleState; + //console.warn("sqlite3.scriptInfo =",EmscriptenModule.sqlite3.scriptInfo); + const f = EmscriptenModule.sqlite3.asyncPostInit; + delete EmscriptenModule.sqlite3.asyncPostInit; + return f(); + }).catch((e)=>{ + console.error("Exception loading sqlite3 module:",e); + throw e; + }); + }; + self.sqlite3InitModule.ready = originalInit.ready; + + if(self.sqlite3InitModuleState.moduleScript){ + const sim = self.sqlite3InitModuleState; + let src = sim.moduleScript.src.split('/'); + src.pop(); + sim.scriptDir = src.join('/') + '/'; + } + initModuleState.debugModule('sqlite3InitModuleState =',initModuleState); + if(0){ + console.warn("Replaced sqlite3InitModule()"); + console.warn("self.location.href =",self.location.href); + if('undefined' !== typeof document){ + console.warn("document.currentScript.src =", + document?.currentScript?.src); + } + } + /* Replace the various module exports performed by the Emscripten + glue... */ + if (typeof exports === 'object' && typeof module === 'object') + module.exports = sqlite3InitModule; + else if (typeof exports === 'object') + exports["sqlite3InitModule"] = sqlite3InitModule; + /* AMD modules get injected in a way we cannot override, + so we can't handle those here. */ +})(); diff --git a/ext/wasm/api/extern-pre-js.js b/ext/wasm/api/extern-pre-js.js new file mode 100644 index 0000000..7d47d33 --- /dev/null +++ b/ext/wasm/api/extern-pre-js.js @@ -0,0 +1,7 @@ +/* extern-pre-js.js must be prepended to the resulting sqlite3.js + file. This file is currently only used for holding snippets during + test and development. + + It gets its name from being used as the value for the + --extern-pre-js=... Emscripten flag. +*/ diff --git a/ext/wasm/api/post-js-footer.js b/ext/wasm/api/post-js-footer.js new file mode 100644 index 0000000..58882cb --- /dev/null +++ b/ext/wasm/api/post-js-footer.js @@ -0,0 +1,4 @@ +/* The current function scope was opened via post-js-header.js, which + gets prepended to this at build-time. This file closes that + scope. */ +})/*postRun.push(...)*/; diff --git a/ext/wasm/api/post-js-header.js b/ext/wasm/api/post-js-header.js new file mode 100644 index 0000000..82a80e5 --- /dev/null +++ b/ext/wasm/api/post-js-header.js @@ -0,0 +1,25 @@ +/** + post-js-header.js is to be prepended to other code to create + post-js.js for use with Emscripten's --post-js flag. This code + requires that it be running in that context. The Emscripten + environment must have been set up already but it will not have + loaded its WASM when the code in this file is run. The function it + installs will be run after the WASM module is loaded, at which + point the sqlite3 JS API bits will get set up. +*/ +if(!Module.postRun) Module.postRun = []; +Module.postRun.push(function(Module/*the Emscripten-style module object*/){ + 'use strict'; + /* This function will contain at least the following: + + - post-js-header.js (this file) + - sqlite3-api-prologue.js => Bootstrapping bits to attach the rest to + - common/whwasmutil.js => Replacements for much of Emscripten's glue + - jaccwaby/jaccwabyt.js => Jaccwabyt (C/JS struct binding) + - sqlite3-api-glue.js => glues previous parts together + - sqlite3-api-oo.js => SQLite3 OO API #1 + - sqlite3-api-worker1.js => Worker-based API + - sqlite3-api-opfs.js => OPFS VFS + - sqlite3-api-cleanup.js => final API cleanup + - post-js-footer.js => closes this postRun() function + */ diff --git a/ext/wasm/api/pre-js.js b/ext/wasm/api/pre-js.js new file mode 100644 index 0000000..f31dea1 --- /dev/null +++ b/ext/wasm/api/pre-js.js @@ -0,0 +1,100 @@ +/** + BEGIN FILE: api/pre-js.js + + This file is intended to be prepended to the sqlite3.js build using + Emscripten's --pre-js=THIS_FILE flag (or equivalent). +*/ + +// See notes in extern-post-js.js +const sqlite3InitModuleState = self.sqlite3InitModuleState || Object.create(null); +delete self.sqlite3InitModuleState; +sqlite3InitModuleState.debugModule('self.location =',self.location); + +/** + This custom locateFile() tries to figure out where to load `path` + from. The intent is to provide a way for foo/bar/X.js loaded from a + Worker constructor or importScripts() to be able to resolve + foo/bar/X.wasm (in the latter case, with some help): + + 1) If URL param named the same as `path` is set, it is returned. + + 2) If sqlite3InitModuleState.sqlite3Dir is set, then (thatName + path) + is returned (note that it's assumed to end with '/'). + + 3) If this code is running in the main UI thread AND it was loaded + from a SCRIPT tag, the directory part of that URL is used + as the prefix. (This form of resolution unfortunately does not + function for scripts loaded via importScripts().) + + 4) If none of the above apply, (prefix+path) is returned. +*/ +Module['locateFile'] = function(path, prefix) { + let theFile; + const up = this.urlParams; + if(up.has(path)){ + theFile = up.get(path); + }else if(this.sqlite3Dir){ + theFile = this.sqlite3Dir + path; + }else if(this.scriptDir){ + theFile = this.scriptDir + path; + }else{ + theFile = prefix + path; + } + sqlite3InitModuleState.debugModule( + "locateFile(",arguments[0], ',', arguments[1],")", + 'sqlite3InitModuleState.scriptDir =',this.scriptDir, + 'up.entries() =',Array.from(up.entries()), + "result =", theFile + ); + return theFile; +}.bind(sqlite3InitModuleState); + +/** + Bug warning: a custom Module.instantiateWasm() does not work + in WASMFS builds: + + https://github.com/emscripten-core/emscripten/issues/17951 + + In such builds we must disable this. +*/ +const xNameOfInstantiateWasm = true + ? 'instantiateWasm' + : 'emscripten-bug-17951'; +Module[xNameOfInstantiateWasm] = function callee(imports,onSuccess){ + imports.env.foo = function(){}; + const uri = Module.locateFile( + callee.uri, ( + ('undefined'===typeof scriptDirectory/*var defined by Emscripten glue*/) + ? '' : scriptDirectory) + ); + sqlite3InitModuleState.debugModule( + "instantiateWasm() uri =", uri + ); + const wfetch = ()=>fetch(uri, {credentials: 'same-origin'}); + const loadWasm = WebAssembly.instantiateStreaming + ? async ()=>{ + return WebAssembly.instantiateStreaming(wfetch(), imports) + .then((arg)=>onSuccess(arg.instance, arg.module)); + } + : async ()=>{ // Safari < v15 + return wfetch() + .then(response => response.arrayBuffer()) + .then(bytes => WebAssembly.instantiate(bytes, imports)) + .then((arg)=>onSuccess(arg.instance, arg.module)); + }; + loadWasm(); + return {}; +}; +/* + It is literally impossible to reliably get the name of _this_ script + at runtime, so impossible to derive X.wasm from script name + X.js. Thus we need, at build-time, to redefine + Module[xNameOfInstantiateWasm].uri by appending it to a build-specific + copy of this file with the name of the wasm file. This is apparently + why Emscripten hard-codes the name of the wasm file into their glue + scripts. +*/ +Module[xNameOfInstantiateWasm].uri = 'sqlite3.wasm'; +/* END FILE: api/pre-js.js, noting that the build process may add a + line after this one to change the above .uri to a build-specific + one. */ diff --git a/ext/wasm/api/sqlite3-api-cleanup.js b/ext/wasm/api/sqlite3-api-cleanup.js new file mode 100644 index 0000000..bef4d91 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-cleanup.js @@ -0,0 +1,70 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is the tail end of the sqlite3-api.js constellation, + intended to be appended after all other sqlite3-api-*.js files so + that it can finalize any setup and clean up any global symbols + temporarily used for setting up the API's various subsystems. +*/ +'use strict'; +if('undefined' !== typeof Module){ // presumably an Emscripten build + /** + Install a suitable default configuration for sqlite3ApiBootstrap(). + */ + const SABC = Object.assign( + Object.create(null), { + Module: Module /* ==> Currently needs to be exposed here for + test code. NOT part of the public API. */, + exports: Module['asm'], + memory: Module.wasmMemory /* gets set if built with -sIMPORT_MEMORY */ + }, + self.sqlite3ApiConfig || Object.create(null) + ); + + /** + For current (2022-08-22) purposes, automatically call + sqlite3ApiBootstrap(). That decision will be revisited at some + point, as we really want client code to be able to call this to + configure certain parts. Clients may modify + self.sqlite3ApiBootstrap.defaultConfig to tweak the default + configuration used by a no-args call to sqlite3ApiBootstrap(), + but must have first loaded their WASM module in order to be + able to provide the necessary configuration state. + */ + //console.warn("self.sqlite3ApiConfig = ",self.sqlite3ApiConfig); + self.sqlite3ApiConfig = SABC; + let sqlite3; + try{ + sqlite3 = self.sqlite3ApiBootstrap(); + }catch(e){ + console.error("sqlite3ApiBootstrap() error:",e); + throw e; + }finally{ + delete self.sqlite3ApiBootstrap; + delete self.sqlite3ApiConfig; + } + + if(self.location && +self.location.port > 1024){ + console.warn("Installing sqlite3 bits as global S for local dev/test purposes."); + self.S = sqlite3; + } + + /* Clean up temporary references to our APIs... */ + delete sqlite3.util /* arguable, but these are (currently) internal-use APIs */; + Module.sqlite3 = sqlite3 /* Needed for customized sqlite3InitModule() to be able to + pass the sqlite3 object off to the client. */; +}else{ + console.warn("This is not running in an Emscripten module context, so", + "self.sqlite3ApiBootstrap() is _not_ being called due to lack", + "of config info for the WASM environment.", + "It must be called manually."); +} diff --git a/ext/wasm/api/sqlite3-api-glue.js b/ext/wasm/api/sqlite3-api-glue.js new file mode 100644 index 0000000..86aa1d1 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-glue.js @@ -0,0 +1,720 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file glues together disparate pieces of JS which are loaded in + previous steps of the sqlite3-api.js bootstrapping process: + sqlite3-api-prologue.js, whwasmutil.js, and jaccwabyt.js. It + initializes the main API pieces so that the downstream components + (e.g. sqlite3-api-oo1.js) have all that they need. +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + const toss3 = sqlite3.SQLite3Error.toss; + const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; + self.WhWasmUtilInstaller(wasm); + delete self.WhWasmUtilInstaller; + + /** + Install JS<->C struct bindings for the non-opaque struct types we + need... */ + sqlite3.StructBinder = self.Jaccwabyt({ + heap: 0 ? wasm.memory : wasm.heap8u, + alloc: wasm.alloc, + dealloc: wasm.dealloc, + functionTable: wasm.functionTable, + bigIntEnabled: wasm.bigIntEnabled, + memberPrefix: '$' + }); + delete self.Jaccwabyt; + + if(0){ + /* "The problem" is that the following isn't even remotely + type-safe. OTOH, nothing about WASM pointers is. */ + const argPointer = wasm.xWrap.argAdapter('*'); + wasm.xWrap.argAdapter('StructType', (v)=>{ + if(v && v.constructor && v instanceof StructBinder.StructType){ + v = v.pointer; + } + return wasm.isPtr(v) + ? argPointer(v) + : toss("Invalid (object) type for StructType-type argument."); + }); + } + + {/* Convert Arrays and certain TypedArrays to strings for + 'flexible-string'-type arguments */ + const xString = wasm.xWrap.argAdapter('string'); + wasm.xWrap.argAdapter( + 'flexible-string', (v)=>xString(util.flexibleString(v)) + ); + } + + if(1){// WhWasmUtil.xWrap() bindings... + /** + Add some descriptive xWrap() aliases for '*' intended to (A) + initially improve readability/correctness of capi.signatures + and (B) eventually perhaps provide automatic conversion from + higher-level representations, e.g. capi.sqlite3_vfs to + `sqlite3_vfs*` via capi.sqlite3_vfs.pointer. + */ + const aPtr = wasm.xWrap.argAdapter('*'); + wasm.xWrap.argAdapter('sqlite3*', aPtr) + ('sqlite3_stmt*', aPtr) + ('sqlite3_context*', aPtr) + ('sqlite3_value*', aPtr) + ('sqlite3_vfs*', aPtr) + ('void*', aPtr); + wasm.xWrap.resultAdapter('sqlite3*', aPtr) + ('sqlite3_context*', aPtr) + ('sqlite3_stmt*', aPtr) + ('sqlite3_vfs*', aPtr) + ('void*', aPtr); + + /** + Populate api object with sqlite3_...() by binding the "raw" wasm + exports into type-converting proxies using wasm.xWrap(). + */ + for(const e of wasm.bindingSignatures){ + capi[e[0]] = wasm.xWrap.apply(null, e); + } + for(const e of wasm.bindingSignatures.wasm){ + wasm[e[0]] = wasm.xWrap.apply(null, e); + } + + /* For C API functions which cannot work properly unless + wasm.bigIntEnabled is true, install a bogus impl which + throws if called when bigIntEnabled is false. */ + const fI64Disabled = function(fname){ + return ()=>toss(fname+"() disabled due to lack", + "of BigInt support in this build."); + }; + for(const e of wasm.bindingSignatures.int64){ + capi[e[0]] = wasm.bigIntEnabled + ? wasm.xWrap.apply(null, e) + : fI64Disabled(e[0]); + } + + /* There's no(?) need to expose bindingSignatures to clients, + implicitly making it part of the public interface. */ + delete wasm.bindingSignatures; + + if(wasm.exports.sqlite3_wasm_db_error){ + util.sqlite3_wasm_db_error = wasm.xWrap( + 'sqlite3_wasm_db_error', 'int', 'sqlite3*', 'int', 'string' + ); + }else{ + util.sqlite3_wasm_db_error = function(pDb,errCode,msg){ + console.warn("sqlite3_wasm_db_error() is not exported.",arguments); + return errCode; + }; + } + + }/*xWrap() bindings*/; + + /** + When registering a VFS and its related components it may be + necessary to ensure that JS keeps a reference to them to keep + them from getting garbage collected. Simply pass each such value + to this function and a reference will be held to it for the life + of the app. + */ + capi.sqlite3_vfs_register.addReference = function f(...args){ + if(!f._) f._ = []; + f._.push(...args); + }; + + /** + Internal helper to assist in validating call argument counts in + the hand-written sqlite3_xyz() wrappers. We do this only for + consistency with non-special-case wrappings. + */ + const __dbArgcMismatch = (pDb,f,n)=>{ + return sqlite3.util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE, + f+"() requires "+n+" argument"+ + (1===n?"":'s')+"."); + }; + + /** + Helper for flexible-string conversions which require a + byte-length counterpart argument. Passed a value and its + ostensible length, this function returns [V,N], where V + is either v or a transformed copy of v and N is either n, + -1, or the byte length of v (if it's a byte array). + */ + const __flexiString = function(v,n){ + if('string'===typeof v){ + n = -1; + }else if(util.isSQLableTypedArray(v)){ + n = v.byteLength; + v = util.typedArrayToString(v); + }else if(Array.isArray(v)){ + v = v.join(""); + n = -1; + } + return [v, n]; + }; + + if(1){/* Special-case handling of sqlite3_exec() */ + const __exec = wasm.xWrap("sqlite3_exec", "int", + ["sqlite3*", "flexible-string", "*", "*", "**"]); + /* Documented in the api object's initializer. */ + capi.sqlite3_exec = function f(pDb, sql, callback, pVoid, pErrMsg){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_exec",f.length); + }else if('function' !== typeof callback){ + return __exec(pDb, sql, callback, pVoid, pErrMsg); + } + /* Wrap the callback in a WASM-bound function and convert the callback's + `(char**)` arguments to arrays of strings... */ + const cbwrap = function(pVoid, nCols, pColVals, pColNames){ + let rc = capi.SQLITE_ERROR; + try { + let aVals = [], aNames = [], i = 0, offset = 0; + for( ; i < nCols; offset += (wasm.ptrSizeof * ++i) ){ + aVals.push( wasm.cstringToJs(wasm.getPtrValue(pColVals + offset)) ); + aNames.push( wasm.cstringToJs(wasm.getPtrValue(pColNames + offset)) ); + } + rc = callback(pVoid, nCols, aVals, aNames) | 0; + /* The first 2 args of the callback are useless for JS but + we want the JS mapping of the C API to be as close to the + C API as possible. */ + }catch(e){ + /* If we set the db error state here, the higher-level exec() call + replaces it with its own, so we have no way of reporting the + exception message except the console. We must not propagate + exceptions through the C API. */ + } + return rc; + }; + let pFunc, rc; + try{ + pFunc = wasm.installFunction("ipipp", cbwrap); + rc = __exec(pDb, sql, pFunc, pVoid, pErrMsg); + }catch(e){ + rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, + "Error running exec(): "+e.message); + }finally{ + if(pFunc) wasm.uninstallFunction(pFunc); + } + return rc; + }; + }/*sqlite3_exec() proxy*/; + + if(1){/* Special-case handling of sqlite3_create_function_v2() + and sqlite3_create_window_function() */ + const sqlite3CreateFunction = wasm.xWrap( + "sqlite3_create_function_v2", "int", + ["sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + "*"/*xStep*/,"*"/*xFinal*/, "*"/*xValue*/, "*"/*xDestroy*/] + ); + const sqlite3CreateWindowFunction = wasm.xWrap( + "sqlite3_create_window_function", "int", + ["sqlite3*", "string"/*funcName*/, "int"/*nArg*/, + "int"/*eTextRep*/, "*"/*pApp*/, + "*"/*xStep*/,"*"/*xFinal*/, "*"/*xValue*/, + "*"/*xInverse*/, "*"/*xDestroy*/] + ); + + const __udfSetResult = function(pCtx, val){ + //console.warn("udfSetResult",typeof val, val); + switch(typeof val) { + case 'undefined': + /* Assume that the client already called sqlite3_result_xxx(). */ + break; + case 'boolean': + capi.sqlite3_result_int(pCtx, val ? 1 : 0); + break; + case 'bigint': + if(wasm.bigIntEnabled){ + if(util.bigIntFits64(val)) capi.sqlite3_result_int64(pCtx, val); + else toss3("BigInt value",val.toString(),"is too BigInt for int64."); + }else if(util.bigIntFits32(val)){ + capi.sqlite3_result_int(pCtx, Number(val)); + }else if(util.bigIntFitsDouble(val)){ + capi.sqlite3_result_double(pCtx, Number(val)); + }else{ + toss3("BigInt value",val.toString(),"is too BigInt."); + } + break; + case 'number': { + (util.isInt32(val) + ? capi.sqlite3_result_int + : capi.sqlite3_result_double)(pCtx, val); + break; + } + case 'string': + capi.sqlite3_result_text(pCtx, val, -1, capi.SQLITE_TRANSIENT); + break; + case 'object': + if(null===val/*yes, typeof null === 'object'*/) { + capi.sqlite3_result_null(pCtx); + break; + }else if(util.isBindableTypedArray(val)){ + const pBlob = wasm.allocFromTypedArray(val); + capi.sqlite3_result_blob( + pCtx, pBlob, val.byteLength, + wasm.exports[sqlite3.config.deallocExportName] + ); + break; + } + // else fall through + default: + toss3("Don't not how to handle this UDF result value:",(typeof val), val); + }; + }/*__udfSetResult()*/; + + const __udfConvertArgs = function(argc, pArgv){ + let i, pVal, valType, arg; + const tgt = []; + for(i = 0; i < argc; ++i){ + pVal = wasm.getPtrValue(pArgv + (wasm.ptrSizeof * i)); + /** + Curiously: despite ostensibly requiring 8-byte + alignment, the pArgv array is parcelled into chunks of + 4 bytes (1 pointer each). The values those point to + have 8-byte alignment but the individual argv entries + do not. + */ + valType = capi.sqlite3_value_type(pVal); + switch(valType){ + case capi.SQLITE_INTEGER: + if(wasm.bigIntEnabled){ + arg = capi.sqlite3_value_int64(pVal); + if(util.bigIntFitsDouble(arg)) arg = Number(arg); + } + else arg = capi.sqlite3_value_double(pVal)/*yes, double, for larger integers*/; + break; + case capi.SQLITE_FLOAT: + arg = capi.sqlite3_value_double(pVal); + break; + case capi.SQLITE_TEXT: + arg = capi.sqlite3_value_text(pVal); + break; + case capi.SQLITE_BLOB:{ + const n = capi.sqlite3_value_bytes(pVal); + const pBlob = capi.sqlite3_value_blob(pVal); + if(n && !pBlob) sqlite3.WasmAllocError.toss( + "Cannot allocate memory for blob argument of",n,"byte(s)" + ); + arg = n ? wasm.heap8u().slice(pBlob, pBlob + Number(n)) : null; + break; + } + case capi.SQLITE_NULL: + arg = null; break; + default: + toss3("Unhandled sqlite3_value_type()",valType, + "is possibly indicative of incorrect", + "pointer size assumption."); + } + tgt.push(arg); + } + return tgt; + }/*__udfConvertArgs()*/; + + const __udfSetError = (pCtx, e)=>{ + if(e instanceof sqlite3.WasmAllocError){ + capi.sqlite3_result_error_nomem(pCtx); + }else{ + const msg = ('string'===typeof e) ? e : e.message; + capi.sqlite3_result_error(pCtx, msg, -1); + } + }; + + const __xFunc = function(callback){ + return function(pCtx, argc, pArgv){ + try{ __udfSetResult(pCtx, callback(pCtx, ...__udfConvertArgs(argc, pArgv))) } + catch(e){ + //console.error('xFunc() caught:',e); + __udfSetError(pCtx, e); + } + }; + }; + + const __xInverseAndStep = function(callback){ + return function(pCtx, argc, pArgv){ + try{ callback(pCtx, ...__udfConvertArgs(argc, pArgv)) } + catch(e){ __udfSetError(pCtx, e) } + }; + }; + + const __xFinalAndValue = function(callback){ + return function(pCtx){ + try{ __udfSetResult(pCtx, callback(pCtx)) } + catch(e){ __udfSetError(pCtx, e) } + }; + }; + + const __xDestroy = function(callback){ + return function(pVoid){ + try{ callback(pVoid) } + catch(e){ console.error("UDF xDestroy method threw:",e) } + }; + }; + + const __xMap = Object.assign(Object.create(null), { + xFunc: {sig:'v(pip)', f:__xFunc}, + xStep: {sig:'v(pip)', f:__xInverseAndStep}, + xInverse: {sig:'v(pip)', f:__xInverseAndStep}, + xFinal: {sig:'v(p)', f:__xFinalAndValue}, + xValue: {sig:'v(p)', f:__xFinalAndValue}, + xDestroy: {sig:'v(p)', f:__xDestroy} + }); + + const __xWrapFuncs = function(theFuncs, tgtUninst){ + const rc = [] + let k; + for(k in theFuncs){ + let fArg = theFuncs[k]; + if('function'===typeof fArg){ + const w = __xMap[k]; + fArg = wasm.installFunction(w.sig, w.f(fArg)); + tgtUninst.push(fArg); + } + rc.push(fArg); + } + return rc; + }; + + /* Documented in the api object's initializer. */ + capi.sqlite3_create_function_v2 = function f( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, //void (*xFunc)(sqlite3_context*,int,sqlite3_value**) + xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xFinal, //void (*xFinal)(sqlite3_context*) + xDestroy //void (*xDestroy)(void*) + ){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_create_function_v2",f.length); + } + /* Wrap the callbacks in a WASM-bound functions... */ + const uninstall = [/*funcs to uninstall on error*/]; + let rc; + try{ + const funcArgs = __xWrapFuncs({xFunc, xStep, xFinal, xDestroy}, + uninstall); + rc = sqlite3CreateFunction(pDb, funcName, nArg, eTextRep, + pApp, ...funcArgs); + }catch(e){ + console.error("sqlite3_create_function_v2() setup threw:",e); + for(let v of uninstall){ + wasm.uninstallFunction(v); + } + rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, + "Creation of UDF threw: "+e.message); + } + return rc; + }; + + capi.sqlite3_create_function = function f( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, xStep, xFinal + ){ + return (f.length===arguments.length) + ? capi.sqlite3_create_function_v2(pDb, funcName, nArg, eTextRep, + pApp, xFunc, xStep, xFinal, 0) + : __dbArgcMismatch(pDb,"sqlite3_create_function",f.length); + }; + + /* Documented in the api object's initializer. */ + capi.sqlite3_create_window_function = function f( + pDb, funcName, nArg, eTextRep, pApp, + xStep, //void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xFinal, //void (*xFinal)(sqlite3_context*) + xValue, //void (*xFinal)(sqlite3_context*) + xInverse,//void (*xStep)(sqlite3_context*,int,sqlite3_value**) + xDestroy //void (*xDestroy)(void*) + ){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_create_window_function",f.length); + } + /* Wrap the callbacks in a WASM-bound functions... */ + const uninstall = [/*funcs to uninstall on error*/]; + let rc; + try{ + const funcArgs = __xWrapFuncs({xStep, xFinal, xValue, xInverse, xDestroy}, + uninstall); + rc = sqlite3CreateWindowFunction(pDb, funcName, nArg, eTextRep, + pApp, ...funcArgs); + }catch(e){ + console.error("sqlite3_create_window_function() setup threw:",e); + for(let v of uninstall){ + wasm.uninstallFunction(v); + } + rc = util.sqlite3_wasm_db_error(pDb, capi.SQLITE_ERROR, + "Creation of UDF threw: "+e.message); + } + return rc; + }; + /** + A helper for UDFs implemented in JS and bound to WASM by the + client. Given a JS value, udfSetResult(pCtx,X) calls one of the + sqlite3_result_xyz(pCtx,...) routines, depending on X's data + type: + + - `null`: sqlite3_result_null() + - `boolean`: sqlite3_result_int() + - `number`: sqlite3_result_int() or sqlite3_result_double() + - `string`: sqlite3_result_text() + - Uint8Array or Int8Array: sqlite3_result_blob() + - `undefined`: indicates that the UDF called one of the + `sqlite3_result_xyz()` routines on its own, making this + function a no-op. Results are _undefined_ if this function is + passed the `undefined` value but did _not_ call one of the + `sqlite3_result_xyz()` routines. + + Anything else triggers sqlite3_result_error(). + */ + capi.sqlite3_create_function_v2.udfSetResult = + capi.sqlite3_create_function.udfSetResult = + capi.sqlite3_create_window_function.udfSetResult = __udfSetResult; + + /** + A helper for UDFs implemented in JS and bound to WASM by the + client. When passed the + (argc,argv) values from the UDF-related functions which receive + them (xFunc, xStep, xInverse), it creates a JS array + representing those arguments, converting each to JS in a manner + appropriate to its data type: numeric, text, blob + (Uint8Array), or null. + + Results are undefined if it's passed anything other than those + two arguments from those specific contexts. + + Thus an argc of 4 will result in a length-4 array containing + the converted values from the corresponding argv. + + The conversion will throw only on allocation error or an internal + error. + */ + capi.sqlite3_create_function_v2.udfConvertArgs = + capi.sqlite3_create_function.udfConvertArgs = + capi.sqlite3_create_window_function.udfConvertArgs = __udfConvertArgs; + + /** + A helper for UDFs implemented in JS and bound to WASM by the + client. It expects to be a passed `(sqlite3_context*, Error)` + (an exception object or message string). And it sets the + current UDF's result to sqlite3_result_error_nomem() or + sqlite3_result_error(), depending on whether the 2nd argument + is a sqlite3.WasmAllocError object or not. + */ + capi.sqlite3_create_function_v2.udfSetError = + capi.sqlite3_create_function.udfSetError = + capi.sqlite3_create_window_function.udfSetError = __udfSetError; + + }/*sqlite3_create_function_v2() and sqlite3_create_window_function() proxies*/; + + if(1){/* Special-case handling of sqlite3_prepare_v2() and + sqlite3_prepare_v3() */ + /** + Scope-local holder of the two impls of sqlite3_prepare_v2/v3(). + */ + const __prepare = Object.create(null); + /** + This binding expects a JS string as its 2nd argument and + null as its final argument. In order to compile multiple + statements from a single string, the "full" impl (see + below) must be used. + */ + __prepare.basic = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "string", + "int"/*ignored for this impl!*/, + "int", "**", + "**"/*MUST be 0 or null or undefined!*/]); + /** + Impl which requires that the 2nd argument be a pointer + to the SQL string, instead of being converted to a + string. This variant is necessary for cases where we + require a non-NULL value for the final argument + (exec()'ing multiple statements from one input + string). For simpler cases, where only the first + statement in the SQL string is required, the wrapper + named sqlite3_prepare_v2() is sufficient and easier to + use because it doesn't require dealing with pointers. + */ + __prepare.full = wasm.xWrap('sqlite3_prepare_v3', + "int", ["sqlite3*", "*", "int", "int", + "**", "**"]); + + /* Documented in the api object's initializer. */ + capi.sqlite3_prepare_v3 = function f(pDb, sql, sqlLen, prepFlags, ppStmt, pzTail){ + if(f.length!==arguments.length){ + return __dbArgcMismatch(pDb,"sqlite3_prepare_v3",f.length); + } + const [xSql, xSqlLen] = __flexiString(sql, sqlLen); + switch(typeof xSql){ + case 'string': return __prepare.basic(pDb, xSql, xSqlLen, prepFlags, ppStmt, null); + case 'number': return __prepare.full(pDb, xSql, xSqlLen, prepFlags, ppStmt, pzTail); + default: + return util.sqlite3_wasm_db_error( + pDb, capi.SQLITE_MISUSE, + "Invalid SQL argument type for sqlite3_prepare_v2/v3()." + ); + } + }; + + /* Documented in the api object's initializer. */ + capi.sqlite3_prepare_v2 = function f(pDb, sql, sqlLen, ppStmt, pzTail){ + return (f.length===arguments.length) + ? capi.sqlite3_prepare_v3(pDb, sql, sqlLen, 0, ppStmt, pzTail) + : __dbArgcMismatch(pDb,"sqlite3_prepare_v2",f.length); + }; + }/*sqlite3_prepare_v2/v3()*/; + + {/* Import C-level constants and structs... */ + const cJson = wasm.xCall('sqlite3_wasm_enum_json'); + if(!cJson){ + toss("Maintenance required: increase sqlite3_wasm_enum_json()'s", + "static buffer size!"); + } + wasm.ctype = JSON.parse(wasm.cstringToJs(cJson)); + //console.debug('wasm.ctype length =',wasm.cstrlen(cJson)); + for(const t of ['access', 'blobFinalizers', 'dataTypes', + 'encodings', 'fcntl', 'flock', 'ioCap', + 'openFlags', 'prepareFlags', 'resultCodes', + 'serialize', 'syncFlags', 'trace', 'udfFlags', + 'version' + ]){ + for(const e of Object.entries(wasm.ctype[t])){ + // ^^^ [k,v] there triggers a buggy code transormation via one + // of the Emscripten-driven optimizers. + capi[e[0]] = e[1]; + } + } + const __rcMap = Object.create(null); + for(const t of ['resultCodes']){ + for(const e of Object.entries(wasm.ctype[t])){ + __rcMap[e[1]] = e[0]; + } + } + /** + For the given integer, returns the SQLITE_xxx result code as a + string, or undefined if no such mapping is found. + */ + capi.sqlite3_js_rc_str = (rc)=>__rcMap[rc]; + /* Bind all registered C-side structs... */ + const notThese = Object.assign(Object.create(null),{ + // Structs NOT to register + WasmTestStruct: true + }); + if(!util.isUIThread()){ + /* We remove the kvvfs VFS from Worker threads below. */ + notThese.sqlite3_kvvfs_methods = true; + } + for(const s of wasm.ctype.structs){ + if(!notThese[s.name]){ + capi[s.name] = sqlite3.StructBinder(s); + } + } + }/*end C constant imports*/ + + const pKvvfs = capi.sqlite3_vfs_find("kvvfs"); + if( pKvvfs ){/* kvvfs-specific glue */ + if(util.isUIThread()){ + const kvvfsMethods = new capi.sqlite3_kvvfs_methods( + wasm.exports.sqlite3_wasm_kvvfs_methods() + ); + delete capi.sqlite3_kvvfs_methods; + + const kvvfsMakeKey = wasm.exports.sqlite3_wasm_kvvfsMakeKeyOnPstack, + pstack = wasm.pstack, + pAllocRaw = wasm.exports.sqlite3_wasm_pstack_alloc; + + const kvvfsStorage = (zClass)=> + ((115/*=='s'*/===wasm.getMemValue(zClass)) + ? sessionStorage : localStorage); + + /** + Implementations for members of the object referred to by + sqlite3_wasm_kvvfs_methods(). We swap out the native + implementations with these, which use localStorage or + sessionStorage for their backing store. + */ + const kvvfsImpls = { + xRead: (zClass, zKey, zBuf, nBuf)=>{ + const stack = pstack.pointer, + astack = wasm.scopedAllocPush(); + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return -3/*OOM*/; + const jKey = wasm.cstringToJs(zXKey); + const jV = kvvfsStorage(zClass).getItem(jKey); + if(!jV) return -1; + const nV = jV.length /* Note that we are relying 100% on v being + ASCII so that jV.length is equal to the + C-string's byte length. */; + if(nBuf<=0) return nV; + else if(1===nBuf){ + wasm.setMemValue(zBuf, 0); + return nV; + } + const zV = wasm.scopedAllocCString(jV); + if(nBuf > nV + 1) nBuf = nV + 1; + wasm.heap8u().copyWithin(zBuf, zV, zV + nBuf - 1); + wasm.setMemValue(zBuf + nBuf - 1, 0); + return nBuf - 1; + }catch(e){ + console.error("kvstorageRead()",e); + return -2; + }finally{ + pstack.restore(stack); + wasm.scopedAllocPop(astack); + } + }, + xWrite: (zClass, zKey, zData)=>{ + const stack = pstack.pointer; + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + const jKey = wasm.cstringToJs(zXKey); + kvvfsStorage(zClass).setItem(jKey, wasm.cstringToJs(zData)); + return 0; + }catch(e){ + console.error("kvstorageWrite()",e); + return capi.SQLITE_IOERR; + }finally{ + pstack.restore(stack); + } + }, + xDelete: (zClass, zKey)=>{ + const stack = pstack.pointer; + try { + const zXKey = kvvfsMakeKey(zClass,zKey); + if(!zXKey) return 1/*OOM*/; + kvvfsStorage(zClass).removeItem(wasm.cstringToJs(zXKey)); + return 0; + }catch(e){ + console.error("kvstorageDelete()",e); + return capi.SQLITE_IOERR; + }finally{ + pstack.restore(stack); + } + } + }/*kvvfsImpls*/; + for(const k of Object.keys(kvvfsImpls)){ + kvvfsMethods[kvvfsMethods.memberKey(k)] = + wasm.installFunction( + kvvfsMethods.memberSignature(k), + kvvfsImpls[k] + ); + } + }else{ + /* Worker thread: unregister kvvfs to avoid it being used + for anything other than local/sessionStorage. It "can" + be used that way but it's not really intended to be. */ + capi.sqlite3_vfs_unregister(pKvvfs); + } + }/*pKvvfs*/ + +}); diff --git a/ext/wasm/api/sqlite3-api-oo1.js b/ext/wasm/api/sqlite3-api-oo1.js new file mode 100644 index 0000000..02ce9c0 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-oo1.js @@ -0,0 +1,1800 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file contains the so-called OO #1 API wrapper for the sqlite3 + WASM build. It requires that sqlite3-api-glue.js has already run + and it installs its deliverable as self.sqlite3.oo1. +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ + const toss = (...args)=>{throw new Error(args.join(' '))}; + const toss3 = (...args)=>{throw new sqlite3.SQLite3Error(...args)}; + + const capi = sqlite3.capi, wasm = sqlite3.wasm, util = sqlite3.util; + /* What follows is colloquially known as "OO API #1". It is a + binding of the sqlite3 API which is designed to be run within + the same thread (main or worker) as the one in which the + sqlite3 WASM binding was initialized. This wrapper cannot use + the sqlite3 binding if, e.g., the wrapper is in the main thread + and the sqlite3 API is in a worker. */ + + /** + In order to keep clients from manipulating, perhaps + inadvertently, the underlying pointer values of DB and Stmt + instances, we'll gate access to them via the `pointer` property + accessor and store their real values in this map. Keys = DB/Stmt + objects, values = pointer values. This also unifies how those are + accessed, for potential use downstream via custom + wasm.xWrap() function signatures which know how to extract + it. + */ + const __ptrMap = new WeakMap(); + /** + Map of DB instances to objects, each object being a map of Stmt + wasm pointers to Stmt objects. + */ + const __stmtMap = new WeakMap(); + + /** If object opts has _its own_ property named p then that + property's value is returned, else dflt is returned. */ + const getOwnOption = (opts, p, dflt)=>{ + const d = Object.getOwnPropertyDescriptor(opts,p); + return d ? d.value : dflt; + }; + + // Documented in DB.checkRc() + const checkSqlite3Rc = function(dbPtr, sqliteResultCode){ + if(sqliteResultCode){ + if(dbPtr instanceof DB) dbPtr = dbPtr.pointer; + toss3( + "sqlite result code",sqliteResultCode+":", + (dbPtr + ? capi.sqlite3_errmsg(dbPtr) + : capi.sqlite3_errstr(sqliteResultCode)) + ); + } + }; + + /** + sqlite3_trace_v2() callback which gets installed by the DB ctor + if its open-flags contain "t". + */ + const __dbTraceToConsole = + wasm.installFunction('i(ippp)', function(t,c,p,x){ + if(capi.SQLITE_TRACE_STMT===t){ + // x == SQL, p == sqlite3_stmt* + console.log("SQL TRACE #"+(++this.counter), + wasm.cstringToJs(x)); + } + }.bind({counter: 0})); + + /** + A map of sqlite3_vfs pointers to SQL code to run when the DB + constructor opens a database with the given VFS. + */ + const __vfsPostOpenSql = Object.create(null); + + /** + A proxy for DB class constructors. It must be called with the + being-construct DB object as its "this". See the DB constructor + for the argument docs. This is split into a separate function + in order to enable simple creation of special-case DB constructors, + e.g. JsStorageDb and OpfsDb. + + Expects to be passed a configuration object with the following + properties: + + - `.filename`: the db filename. It may be a special name like ":memory:" + or "". + + - `.flags`: as documented in the DB constructor. + + - `.vfs`: as documented in the DB constructor. + + It also accepts those as the first 3 arguments. + */ + const dbCtorHelper = function ctor(...args){ + if(!ctor._name2vfs){ + /** + Map special filenames which we handle here (instead of in C) + to some helpful metadata... + + As of 2022-09-20, the C API supports the names :localStorage: + and :sessionStorage: for kvvfs. However, C code cannot + determine (without embedded JS code, e.g. via Emscripten's + EM_JS()) whether the kvvfs is legal in the current browser + context (namely the main UI thread). In order to help client + code fail early on, instead of it being delayed until they + try to read or write a kvvfs-backed db, we'll check for those + names here and throw if they're not legal in the current + context. + */ + ctor._name2vfs = Object.create(null); + const isWorkerThread = ('function'===typeof importScripts/*===running in worker thread*/) + ? (n)=>toss3("The VFS for",n,"is only available in the main window thread.") + : false; + ctor._name2vfs[':localStorage:'] = { + vfs: 'kvvfs', filename: isWorkerThread || (()=>'local') + }; + ctor._name2vfs[':sessionStorage:'] = { + vfs: 'kvvfs', filename: isWorkerThread || (()=>'session') + }; + } + const opt = ctor.normalizeArgs(...args); + let fn = opt.filename, vfsName = opt.vfs, flagsStr = opt.flags; + if(('string'!==typeof fn && 'number'!==typeof fn) + || 'string'!==typeof flagsStr + || (vfsName && ('string'!==typeof vfsName && 'number'!==typeof vfsName))){ + console.error("Invalid DB ctor args",opt,arguments); + toss3("Invalid arguments for DB constructor."); + } + let fnJs = ('number'===typeof fn) ? wasm.cstringToJs(fn) : fn; + const vfsCheck = ctor._name2vfs[fnJs]; + if(vfsCheck){ + vfsName = vfsCheck.vfs; + fn = fnJs = vfsCheck.filename(fnJs); + } + let pDb, oflags = 0; + if( flagsStr.indexOf('c')>=0 ){ + oflags |= capi.SQLITE_OPEN_CREATE | capi.SQLITE_OPEN_READWRITE; + } + if( flagsStr.indexOf('w')>=0 ) oflags |= capi.SQLITE_OPEN_READWRITE; + if( 0===oflags ) oflags |= capi.SQLITE_OPEN_READONLY; + oflags |= capi.SQLITE_OPEN_EXRESCODE; + const stack = wasm.pstack.pointer; + try { + const pPtr = wasm.pstack.allocPtr() /* output (sqlite3**) arg */; + let rc = capi.sqlite3_open_v2(fn, pPtr, oflags, vfsName || 0); + pDb = wasm.getPtrValue(pPtr); + checkSqlite3Rc(pDb, rc); + if(flagsStr.indexOf('t')>=0){ + capi.sqlite3_trace_v2(pDb, capi.SQLITE_TRACE_STMT, + __dbTraceToConsole, 0); + } + // Check for per-VFS post-open SQL... + const pVfs = capi.sqlite3_js_db_vfs(pDb); + //console.warn("Opened db",fn,"with vfs",vfsName,pVfs); + if(!pVfs) toss3("Internal error: cannot get VFS for new db handle."); + const postInitSql = __vfsPostOpenSql[pVfs]; + if(postInitSql){ + rc = capi.sqlite3_exec(pDb, postInitSql, 0, 0, 0); + checkSqlite3Rc(pDb, rc); + } + }catch( e ){ + if( pDb ) capi.sqlite3_close_v2(pDb); + throw e; + }finally{ + wasm.pstack.restore(stack); + } + this.filename = fnJs; + __ptrMap.set(this, pDb); + __stmtMap.set(this, Object.create(null)); + }; + + /** + Sets SQL which should be exec()'d on a DB instance after it is + opened with the given VFS pointer. This is intended only for use + by DB subclasses or sqlite3_vfs implementations. + */ + dbCtorHelper.setVfsPostOpenSql = function(pVfs, sql){ + __vfsPostOpenSql[pVfs] = sql; + }; + + /** + A helper for DB constructors. It accepts either a single + config-style object or up to 3 arguments (filename, dbOpenFlags, + dbVfsName). It returns a new object containing: + + { filename: ..., flags: ..., vfs: ... } + + If passed an object, any additional properties it has are copied + as-is into the new object. + */ + dbCtorHelper.normalizeArgs = function(filename=':memory:',flags = 'c',vfs = null){ + const arg = {}; + if(1===arguments.length && 'object'===typeof arguments[0]){ + const x = arguments[0]; + Object.keys(x).forEach((k)=>arg[k] = x[k]); + if(undefined===arg.flags) arg.flags = 'c'; + if(undefined===arg.vfs) arg.vfs = null; + if(undefined===arg.filename) arg.filename = ':memory:'; + }else{ + arg.filename = filename; + arg.flags = flags; + arg.vfs = vfs; + } + return arg; + }; + /** + The DB class provides a high-level OO wrapper around an sqlite3 + db handle. + + The given db filename must be resolvable using whatever + filesystem layer (virtual or otherwise) is set up for the default + sqlite3 VFS. + + Note that the special sqlite3 db names ":memory:" and "" + (temporary db) have their normal special meanings here and need + not resolve to real filenames, but "" uses an on-storage + temporary database and requires that the VFS support that. + + The second argument specifies the open/create mode for the + database. It must be string containing a sequence of letters (in + any order, but case sensitive) specifying the mode: + + - "c": create if it does not exist, else fail if it does not + exist. Implies the "w" flag. + + - "w": write. Implies "r": a db cannot be write-only. + + - "r": read-only if neither "w" nor "c" are provided, else it + is ignored. + + - "t": enable tracing of SQL executed on this database handle, + sending it to `console.log()`. To disable it later, call + `sqlite3.capi.sqlite3_trace_v2(thisDb.pointer, 0, 0, 0)`. + + If "w" is not provided, the db is implicitly read-only, noting + that "rc" is meaningless + + Any other letters are currently ignored. The default is + "c". These modes are ignored for the special ":memory:" and "" + names and _may_ be ignored altogether for certain VFSes. + + The final argument is analogous to the final argument of + sqlite3_open_v2(): the name of an sqlite3 VFS. Pass a falsy value, + or none at all, to use the default. If passed a value, it must + be the string name of a VFS. + + The constructor optionally (and preferably) takes its arguments + in the form of a single configuration object with the following + properties: + + - `filename`: database file name + - `flags`: open-mode flags + - `vfs`: the VFS fname + + The `filename` and `vfs` arguments may be either JS strings or + C-strings allocated via WASM. `flags` is required to be a JS + string (because it's specific to this API, which is specific + to JS). + + For purposes of passing a DB instance to C-style sqlite3 + functions, the DB object's read-only `pointer` property holds its + `sqlite3*` pointer value. That property can also be used to check + whether this DB instance is still open. + + In the main window thread, the filenames `":localStorage:"` and + `":sessionStorage:"` are special: they cause the db to use either + localStorage or sessionStorage for storing the database using + the kvvfs. If one of these names are used, they trump + any vfs name set in the arguments. + */ + const DB = function(...args){ + dbCtorHelper.apply(this, args); + }; + DB.dbCtorHelper = dbCtorHelper; + + /** + Internal-use enum for mapping JS types to DB-bindable types. + These do not (and need not) line up with the SQLITE_type + values. All values in this enum must be truthy and distinct + but they need not be numbers. + */ + const BindTypes = { + null: 1, + number: 2, + string: 3, + boolean: 4, + blob: 5 + }; + BindTypes['undefined'] == BindTypes.null; + if(wasm.bigIntEnabled){ + BindTypes.bigint = BindTypes.number; + } + + /** + This class wraps sqlite3_stmt. Calling this constructor + directly will trigger an exception. Use DB.prepare() to create + new instances. + + For purposes of passing a Stmt instance to C-style sqlite3 + functions, its read-only `pointer` property holds its `sqlite3_stmt*` + pointer value. + + Other non-function properties include: + + - `db`: the DB object which created the statement. + + - `columnCount`: the number of result columns in the query, or 0 for + queries which cannot return results. + + - `parameterCount`: the number of bindable paramters in the query. + */ + const Stmt = function(){ + if(BindTypes!==arguments[2]){ + toss3("Do not call the Stmt constructor directly. Use DB.prepare()."); + } + this.db = arguments[0]; + __ptrMap.set(this, arguments[1]); + this.columnCount = capi.sqlite3_column_count(this.pointer); + this.parameterCount = capi.sqlite3_bind_parameter_count(this.pointer); + }; + + /** Throws if the given DB has been closed, else it is returned. */ + const affirmDbOpen = function(db){ + if(!db.pointer) toss3("DB has been closed."); + return db; + }; + + /** Throws if ndx is not an integer or if it is out of range + for stmt.columnCount, else returns stmt. + + Reminder: this will also fail after the statement is finalized + but the resulting error will be about an out-of-bounds column + index rather than a statement-is-finalized error. + */ + const affirmColIndex = function(stmt,ndx){ + if((ndx !== (ndx|0)) || ndx<0 || ndx>=stmt.columnCount){ + toss3("Column index",ndx,"is out of range."); + } + return stmt; + }; + + /** + Expects to be passed the `arguments` object from DB.exec(). Does + the argument processing/validation, throws on error, and returns + a new object on success: + + { sql: the SQL, opt: optionsObj, cbArg: function} + + The opt object is a normalized copy of any passed to this + function. The sql will be converted to a string if it is provided + in one of the supported non-string formats. + + cbArg is only set if the opt.callback or opt.resultRows are set, + in which case it's a function which expects to be passed the + current Stmt and returns the callback argument of the type + indicated by the input arguments. + */ + const parseExecArgs = function(db, args){ + const out = Object.create(null); + out.opt = Object.create(null); + switch(args.length){ + case 1: + if('string'===typeof args[0] || util.isSQLableTypedArray(args[0])){ + out.sql = args[0]; + }else if(Array.isArray(args[0])){ + out.sql = args[0]; + }else if(args[0] && 'object'===typeof args[0]){ + out.opt = args[0]; + out.sql = out.opt.sql; + } + break; + case 2: + out.sql = args[0]; + out.opt = args[1]; + break; + default: toss3("Invalid argument count for exec()."); + }; + out.sql = util.flexibleString(out.sql); + if('string'!==typeof out.sql){ + toss3("Missing SQL argument or unsupported SQL value type."); + } + const opt = out.opt; + switch(opt.returnValue){ + case 'resultRows': + if(!opt.resultRows) opt.resultRows = []; + out.returnVal = ()=>opt.resultRows; + break; + case 'saveSql': + if(!opt.saveSql) opt.saveSql = []; + out.returnVal = ()=>opt.saveSql; + break; + case undefined: + case 'this': + out.returnVal = ()=>db; + break; + default: + toss3("Invalid returnValue value:",opt.returnValue); + } + if(opt.callback || opt.resultRows){ + switch((undefined===opt.rowMode) + ? 'array' : opt.rowMode) { + case 'object': out.cbArg = (stmt)=>stmt.get(Object.create(null)); break; + case 'array': out.cbArg = (stmt)=>stmt.get([]); break; + case 'stmt': + if(Array.isArray(opt.resultRows)){ + toss3("exec(): invalid rowMode for a resultRows array: must", + "be one of 'array', 'object',", + "a result column number, or column name reference."); + } + out.cbArg = (stmt)=>stmt; + break; + default: + if(util.isInt32(opt.rowMode)){ + out.cbArg = (stmt)=>stmt.get(opt.rowMode); + break; + }else if('string'===typeof opt.rowMode && opt.rowMode.length>1){ + /* "$X", ":X", and "@X" fetch column named "X" (case-sensitive!) */ + const prefix = opt.rowMode[0]; + if(':'===prefix || '@'===prefix || '$'===prefix){ + out.cbArg = function(stmt){ + const rc = stmt.get(this.obj)[this.colName]; + return (undefined===rc) ? toss3("exec(): unknown result column:",this.colName) : rc; + }.bind({ + obj:Object.create(null), + colName: opt.rowMode.substr(1) + }); + break; + } + } + toss3("Invalid rowMode:",opt.rowMode); + } + } + return out; + }; + + /** + Internal impl of the DB.selectArray() and + selectObject() methods. + */ + const __selectFirstRow = (db, sql, bind, getArg)=>{ + let stmt, rc; + try { + stmt = db.prepare(sql).bind(bind); + if(stmt.step()) rc = stmt.get(getArg); + }finally{ + if(stmt) stmt.finalize(); + } + return rc; + }; + + /** + Expects to be given a DB instance or an `sqlite3*` pointer (may + be null) and an sqlite3 API result code. If the result code is + not falsy, this function throws an SQLite3Error with an error + message from sqlite3_errmsg(), using dbPtr as the db handle, or + sqlite3_errstr() if dbPtr is falsy. Note that if it's passed a + non-error code like SQLITE_ROW or SQLITE_DONE, it will still + throw but the error string might be "Not an error." The various + non-0 non-error codes need to be checked for in + client code where they are expected. + */ + DB.checkRc = checkSqlite3Rc; + + DB.prototype = { + /** Returns true if this db handle is open, else false. */ + isOpen: function(){ + return !!this.pointer; + }, + /** Throws if this given DB has been closed, else returns `this`. */ + affirmOpen: function(){ + return affirmDbOpen(this); + }, + /** + Finalizes all open statements and closes this database + connection. This is a no-op if the db has already been + closed. After calling close(), `this.pointer` will resolve to + `undefined`, so that can be used to check whether the db + instance is still opened. + + If this.onclose.before is a function then it is called before + any close-related cleanup. + + If this.onclose.after is a function then it is called after the + db is closed but before auxiliary state like this.filename is + cleared. + + Both onclose handlers are passed this object. If this db is not + opened, neither of the handlers are called. Any exceptions the + handlers throw are ignored because "destructors must not + throw." + + Note that garbage collection of a db handle, if it happens at + all, will never trigger close(), so onclose handlers are not a + reliable way to implement close-time cleanup or maintenance of + a db. + */ + close: function(){ + if(this.pointer){ + if(this.onclose && (this.onclose.before instanceof Function)){ + try{this.onclose.before(this)} + catch(e){/*ignore*/} + } + const pDb = this.pointer; + Object.keys(__stmtMap.get(this)).forEach((k,s)=>{ + if(s && s.pointer) s.finalize(); + }); + __ptrMap.delete(this); + __stmtMap.delete(this); + capi.sqlite3_close_v2(pDb); + if(this.onclose && (this.onclose.after instanceof Function)){ + try{this.onclose.after(this)} + catch(e){/*ignore*/} + } + delete this.filename; + } + }, + /** + Returns the number of changes, as per sqlite3_changes() + (if the first argument is false) or sqlite3_total_changes() + (if it's true). If the 2nd argument is true, it uses + sqlite3_changes64() or sqlite3_total_changes64(), which + will trigger an exception if this build does not have + BigInt support enabled. + */ + changes: function(total=false,sixtyFour=false){ + const p = affirmDbOpen(this).pointer; + if(total){ + return sixtyFour + ? capi.sqlite3_total_changes64(p) + : capi.sqlite3_total_changes(p); + }else{ + return sixtyFour + ? capi.sqlite3_changes64(p) + : capi.sqlite3_changes(p); + } + }, + /** + Similar to the this.filename but returns the + sqlite3_db_filename() value for the given database name, + defaulting to "main". The argument may be either a JS string + or a pointer to a WASM-allocated C-string. + */ + dbFilename: function(dbName='main'){ + return capi.sqlite3_db_filename(affirmDbOpen(this).pointer, dbName); + }, + /** + Returns the name of the given 0-based db number, as documented + for sqlite3_db_name(). + */ + dbName: function(dbNumber=0){ + return capi.sqlite3_db_name(affirmDbOpen(this).pointer, dbNumber); + }, + /** + Returns the name of the sqlite3_vfs used by the given database + of this connection (defaulting to 'main'). The argument may be + either a JS string or a WASM C-string. Returns undefined if the + given db name is invalid. Throws if this object has been + close()d. + */ + dbVfsName: function(dbName=0){ + let rc; + const pVfs = capi.sqlite3_js_db_vfs( + affirmDbOpen(this).pointer, dbName + ); + if(pVfs){ + const v = new capi.sqlite3_vfs(pVfs); + try{ rc = wasm.cstringToJs(v.$zName) } + finally { v.dispose() } + } + return rc; + }, + /** + Compiles the given SQL and returns a prepared Stmt. This is + the only way to create new Stmt objects. Throws on error. + + The given SQL must be a string, a Uint8Array holding SQL, a + WASM pointer to memory holding the NUL-terminated SQL string, + or an array of strings. In the latter case, the array is + concatenated together, with no separators, to form the SQL + string (arrays are often a convenient way to formulate long + statements). If the SQL contains no statements, an + SQLite3Error is thrown. + + Design note: the C API permits empty SQL, reporting it as a 0 + result code and a NULL stmt pointer. Supporting that case here + would cause extra work for all clients: any use of the Stmt API + on such a statement will necessarily throw, so clients would be + required to check `stmt.pointer` after calling `prepare()` in + order to determine whether the Stmt instance is empty or not. + Long-time practice (with other sqlite3 script bindings) + suggests that the empty-prepare case is sufficiently rare that + supporting it here would simply hurt overall usability. + */ + prepare: function(sql){ + affirmDbOpen(this); + const stack = wasm.pstack.pointer; + let ppStmt, pStmt; + try{ + ppStmt = wasm.pstack.alloc(8)/* output (sqlite3_stmt**) arg */; + DB.checkRc(this, capi.sqlite3_prepare_v2(this.pointer, sql, -1, ppStmt, null)); + pStmt = wasm.getPtrValue(ppStmt); + } + finally { + wasm.pstack.restore(stack); + } + if(!pStmt) toss3("Cannot prepare empty SQL."); + const stmt = new Stmt(this, pStmt, BindTypes); + __stmtMap.get(this)[pStmt] = stmt; + return stmt; + }, + /** + Executes one or more SQL statements in the form of a single + string. Its arguments must be either (sql,optionsObject) or + (optionsObject). In the latter case, optionsObject.sql must + contain the SQL to execute. By default it returns this object + but that can be changed via the `returnValue` option as + described below. Throws on error. + + If no SQL is provided, or a non-string is provided, an + exception is triggered. Empty SQL, on the other hand, is + simply a no-op. + + The optional options object may contain any of the following + properties: + + - `sql` = the SQL to run (unless it's provided as the first + argument). This must be of type string, Uint8Array, or an array + of strings. In the latter case they're concatenated together + as-is, _with no separator_ between elements, before evaluation. + The array form is often simpler for long hand-written queries. + + - `bind` = a single value valid as an argument for + Stmt.bind(). This is _only_ applied to the _first_ non-empty + statement in the SQL which has any bindable parameters. (Empty + statements are skipped entirely.) + + - `saveSql` = an optional array. If set, the SQL of each + executed statement is appended to this array before the + statement is executed (but after it is prepared - we don't have + the string until after that). Empty SQL statements are elided + but can have odd effects in the output. e.g. SQL of: `"select + 1; -- empty\n; select 2"` will result in an array containing + `["select 1;", "--empty \n; select 2"]`. That's simply how + sqlite3 records the SQL for the 2nd statement. + + ================================================================== + The following options apply _only_ to the _first_ statement + which has a non-zero result column count, regardless of whether + the statement actually produces any result rows. + ================================================================== + + - `columnNames`: if this is an array, the column names of the + result set are stored in this array before the callback (if + any) is triggered (regardless of whether the query produces any + result rows). If no statement has result columns, this value is + unchanged. Achtung: an SQL result may have multiple columns + with identical names. + + - `callback` = a function which gets called for each row of + the result set, but only if that statement has any result + _rows_. The callback's "this" is the options object, noting + that this function synthesizes one if the caller does not pass + one to exec(). The second argument passed to the callback is + always the current Stmt object, as it's needed if the caller + wants to fetch the column names or some such (noting that they + could also be fetched via `this.columnNames`, if the client + provides the `columnNames` option). + + ACHTUNG: The callback MUST NOT modify the Stmt object. Calling + any of the Stmt.get() variants, Stmt.getColumnName(), or + similar, is legal, but calling step() or finalize() is + not. Member methods which are illegal in this context will + trigger an exception. + + The first argument passed to the callback defaults to an array of + values from the current result row but may be changed with ... + + - `rowMode` = specifies the type of he callback's first argument. + It may be any of... + + A) A string describing what type of argument should be passed + as the first argument to the callback: + + A.1) `'array'` (the default) causes the results of + `stmt.get([])` to be passed to the `callback` and/or appended + to `resultRows` + + A.2) `'object'` causes the results of + `stmt.get(Object.create(null))` to be passed to the + `callback` and/or appended to `resultRows`. Achtung: an SQL + result may have multiple columns with identical names. In + that case, the right-most column will be the one set in this + object! + + A.3) `'stmt'` causes the current Stmt to be passed to the + callback, but this mode will trigger an exception if + `resultRows` is an array because appending the statement to + the array would be downright unhelpful. + + B) An integer, indicating a zero-based column in the result + row. Only that one single value will be passed on. + + C) A string with a minimum length of 2 and leading character of + ':', '$', or '@' will fetch the row as an object, extract that + one field, and pass that field's value to the callback. Note + that these keys are case-sensitive so must match the case used + in the SQL. e.g. `"select a A from t"` with a `rowMode` of + `'$A'` would work but `'$a'` would not. A reference to a column + not in the result set will trigger an exception on the first + row (as the check is not performed until rows are fetched). + Note also that `$` is a legal identifier character in JS so + need not be quoted. (Design note: those 3 characters were + chosen because they are the characters support for naming bound + parameters.) + + Any other `rowMode` value triggers an exception. + + - `resultRows`: if this is an array, it functions similarly to + the `callback` option: each row of the result set (if any), + with the exception that the `rowMode` 'stmt' is not legal. It + is legal to use both `resultRows` and `callback`, but + `resultRows` is likely much simpler to use for small data sets + and can be used over a WebWorker-style message interface. + exec() throws if `resultRows` is set and `rowMode` is 'stmt'. + + - `returnValue`: is a string specifying what this function + should return: + + A) The default value is `"this"`, meaning that the + DB object itself should be returned. + + B) `"resultRows"` means to return the value of the + `resultRows` option. If `resultRows` is not set, this + function behaves as if it were set to an empty array. + + C) `"saveSql"` means to return the value of the + `saveSql` option. If `saveSql` is not set, this + function behaves as if it were set to an empty array. + + Potential TODOs: + + - `bind`: permit an array of arrays/objects to bind. The first + sub-array would act on the first statement which has bindable + parameters (as it does now). The 2nd would act on the next such + statement, etc. + + - `callback` and `resultRows`: permit an array entries with + semantics similar to those described for `bind` above. + + */ + exec: function(/*(sql [,obj]) || (obj)*/){ + affirmDbOpen(this); + const arg = parseExecArgs(this, arguments); + if(!arg.sql){ + return toss3("exec() requires an SQL string."); + } + const opt = arg.opt; + const callback = opt.callback; + const resultRows = + Array.isArray(opt.resultRows) ? opt.resultRows : undefined; + let stmt; + let bind = opt.bind; + let evalFirstResult = !!(arg.cbArg || opt.columnNames) /* true to evaluate the first result-returning query */; + const stack = wasm.scopedAllocPush(); + try{ + const isTA = util.isSQLableTypedArray(arg.sql) + /* Optimization: if the SQL is a TypedArray we can save some string + conversion costs. */; + /* Allocate the two output pointers (ppStmt, pzTail) and heap + space for the SQL (pSql). When prepare_v2() returns, pzTail + will point to somewhere in pSql. */ + let sqlByteLen = isTA ? arg.sql.byteLength : wasm.jstrlen(arg.sql); + const ppStmt = wasm.scopedAlloc(/* output (sqlite3_stmt**) arg and pzTail */ + (2 * wasm.ptrSizeof) + + (sqlByteLen + 1/* SQL + NUL */)); + const pzTail = ppStmt + wasm.ptrSizeof /* final arg to sqlite3_prepare_v2() */; + let pSql = pzTail + wasm.ptrSizeof; + const pSqlEnd = pSql + sqlByteLen; + if(isTA) wasm.heap8().set(arg.sql, pSql); + else wasm.jstrcpy(arg.sql, wasm.heap8(), pSql, sqlByteLen, false); + wasm.setMemValue(pSql + sqlByteLen, 0/*NUL terminator*/); + while(pSql && wasm.getMemValue(pSql, 'i8') + /* Maintenance reminder:^^^ _must_ be 'i8' or else we + will very likely cause an endless loop. What that's + doing is checking for a terminating NUL byte. If we + use i32 or similar then we read 4 bytes, read stuff + around the NUL terminator, and get stuck in and + endless loop at the end of the SQL, endlessly + re-preparing an empty statement. */ ){ + wasm.setPtrValue(ppStmt, 0); + wasm.setPtrValue(pzTail, 0); + DB.checkRc(this, capi.sqlite3_prepare_v3( + this.pointer, pSql, sqlByteLen, 0, ppStmt, pzTail + )); + const pStmt = wasm.getPtrValue(ppStmt); + pSql = wasm.getPtrValue(pzTail); + sqlByteLen = pSqlEnd - pSql; + if(!pStmt) continue; + if(Array.isArray(opt.saveSql)){ + opt.saveSql.push(capi.sqlite3_sql(pStmt).trim()); + } + stmt = new Stmt(this, pStmt, BindTypes); + if(bind && stmt.parameterCount){ + stmt.bind(bind); + bind = null; + } + if(evalFirstResult && stmt.columnCount){ + /* Only forward SELECT results for the FIRST query + in the SQL which potentially has them. */ + evalFirstResult = false; + if(Array.isArray(opt.columnNames)){ + stmt.getColumnNames(opt.columnNames); + } + while(!!arg.cbArg && stmt.step()){ + stmt._isLocked = true; + const row = arg.cbArg(stmt); + if(resultRows) resultRows.push(row); + if(callback) callback.call(opt, row, stmt); + stmt._isLocked = false; + } + }else{ + stmt.step(); + } + stmt.finalize(); + stmt = null; + } + }/*catch(e){ + console.warn("DB.exec() is propagating exception",opt,e); + throw e; + }*/finally{ + if(stmt){ + delete stmt._isLocked; + stmt.finalize(); + } + wasm.scopedAllocPop(stack); + } + return arg.returnVal(); + }/*exec()*/, + /** + Creates a new scalar UDF (User-Defined Function) which is + accessible via SQL code. This function may be called in any + of the following forms: + + - (name, function) + - (name, function, optionsObject) + - (name, optionsObject) + - (optionsObject) + + In the final two cases, the function must be defined as the + `callback` property of the options object (optionally called + `xFunc` to align with the C API documentation). In the final + case, the function's name must be the 'name' property. + + The first two call forms can only be used for creating scalar + functions. Creating an aggregate or window function requires + the options-object form (see below for details). + + UDFs cannot currently be removed from a DB handle after they're + added. More correctly, they can be removed as documented for + sqlite3_create_function_v2(), but doing so will "leak" the + JS-created WASM binding of those functions. + + On success, returns this object. Throws on error. + + When called from SQL arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity as + is feasible, triggering an exception if a type conversion + cannot be determined. The docs for sqlite3_create_function_v2() + describe the conversions in more detail. + + The values set in the options object differ for scalar and + aggregate functions: + + - Scalar: set the `xFunc` function-type property to the UDF + function. + + - Aggregate: set the `xStep` and `xFinal` function-type + properties to the "step" and "final" callbacks for the + aggregate. Do not set the `xFunc` property. + + - Window: set the `xStep`, `xFinal`, `xValue`, and `xInverse` + function-type properties. Do not set the `xFunc` property. + + The options object may optionally have an `xDestroy` + function-type property, as per sqlite3_create_function_v2(). + Its argument will be the WASM-pointer-type value of the `pApp` + property, and this function will throw if `pApp` is defined but + is not null, undefined, or a numeric (WASM pointer) + value. i.e. `pApp`, if set, must be value suitable for use as a + WASM pointer argument, noting that `null` or `undefined` will + translate to 0 for that purpose. + + The options object may contain flags to modify how + the function is defined: + + - `arity`: the number of arguments which SQL calls to this + function expect or require. The default value is `xFunc.length` + or `xStep.length` (i.e. the number of declared parameters it + has) **MINUS 1** (see below for why). As a special case, if the + `length` is 0, its arity is also 0 instead of -1. A negative + arity value means that the function is variadic and may accept + any number of arguments, up to sqlite3's compile-time + limits. sqlite3 will enforce the argument count if is zero or + greater. The callback always receives a pointer to an + `sqlite3_context` object as its first argument. Any arguments + after that are from SQL code. The leading context argument does + _not_ count towards the function's arity. See the docs for + sqlite3.capi.sqlite3_create_function_v2() for why that argument + is needed in the interface. + + The following options-object properties correspond to flags + documented at: + + https://sqlite.org/c3ref/create_function.html + + - `deterministic` = sqlite3.capi.SQLITE_DETERMINISTIC + - `directOnly` = sqlite3.capi.SQLITE_DIRECTONLY + - `innocuous` = sqlite3.capi.SQLITE_INNOCUOUS + + Sidebar: the ability to add new WASM-accessible functions to + the runtime requires that the WASM build is compiled with the + equivalent functionality as that provided by Emscripten's + `-sALLOW_TABLE_GROWTH` flag. + */ + createFunction: function f(name, xFunc, opt){ + const isFunc = (f)=>(f instanceof Function); + switch(arguments.length){ + case 1: /* (optionsObject) */ + opt = name; + name = opt.name; + xFunc = opt.xFunc || 0; + break; + case 2: /* (name, callback|optionsObject) */ + if(!isFunc(xFunc)){ + opt = xFunc; + xFunc = opt.xFunc || 0; + } + break; + case 3: /* name, xFunc, opt */ + break; + default: break; + } + if(!opt) opt = {}; + if('string' !== typeof name){ + toss3("Invalid arguments: missing function name."); + } + let xStep = opt.xStep || 0; + let xFinal = opt.xFinal || 0; + const xValue = opt.xValue || 0; + const xInverse = opt.xInverse || 0; + let isWindow = undefined; + if(isFunc(xFunc)){ + isWindow = false; + if(isFunc(xStep) || isFunc(xFinal)){ + toss3("Ambiguous arguments: scalar or aggregate?"); + } + xStep = xFinal = null; + }else if(isFunc(xStep)){ + if(!isFunc(xFinal)){ + toss3("Missing xFinal() callback for aggregate or window UDF."); + } + xFunc = null; + }else if(isFunc(xFinal)){ + toss3("Missing xStep() callback for aggregate or window UDF."); + }else{ + toss3("Missing function-type properties."); + } + if(false === isWindow){ + if(isFunc(xValue) || isFunc(xInverse)){ + toss3("xValue and xInverse are not permitted for non-window UDFs."); + } + }else if(isFunc(xValue)){ + if(!isFunc(xInverse)){ + toss3("xInverse must be provided if xValue is."); + } + isWindow = true; + }else if(isFunc(xInverse)){ + toss3("xValue must be provided if xInverse is."); + } + const pApp = opt.pApp; + if(undefined!==pApp && + null!==pApp && + (('number'!==typeof pApp) || !util.isInt32(pApp))){ + toss3("Invalid value for pApp property. Must be a legal WASM pointer value."); + } + const xDestroy = opt.xDestroy || 0; + if(xDestroy && !isFunc(xDestroy)){ + toss3("xDestroy property must be a function."); + } + let fFlags = 0 /*flags for sqlite3_create_function_v2()*/; + if(getOwnOption(opt, 'deterministic')) fFlags |= capi.SQLITE_DETERMINISTIC; + if(getOwnOption(opt, 'directOnly')) fFlags |= capi.SQLITE_DIRECTONLY; + if(getOwnOption(opt, 'innocuous')) fFlags |= capi.SQLITE_INNOCUOUS; + name = name.toLowerCase(); + const xArity = xFunc || xStep; + const arity = getOwnOption(opt, 'arity'); + const arityArg = ('number'===typeof arity + ? arity + : (xArity.length ? xArity.length-1/*for pCtx arg*/ : 0)); + let rc; + if( isWindow ){ + rc = capi.sqlite3_create_window_function( + this.pointer, name, arityArg, + capi.SQLITE_UTF8 | fFlags, pApp || 0, + xStep, xFinal, xValue, xInverse, xDestroy); + }else{ + rc = capi.sqlite3_create_function_v2( + this.pointer, name, arityArg, + capi.SQLITE_UTF8 | fFlags, pApp || 0, + xFunc, xStep, xFinal, xDestroy); + } + DB.checkRc(this, rc); + return this; + }/*createFunction()*/, + /** + Prepares the given SQL, step()s it one time, and returns + the value of the first result column. If it has no results, + undefined is returned. + + If passed a second argument, it is treated like an argument + to Stmt.bind(), so may be any type supported by that + function. Passing the undefined value is the same as passing + no value, which is useful when... + + If passed a 3rd argument, it is expected to be one of the + SQLITE_{typename} constants. Passing the undefined value is + the same as not passing a value. + + Throws on error (e.g. malformed SQL). + */ + selectValue: function(sql,bind,asType){ + let stmt, rc; + try { + stmt = this.prepare(sql).bind(bind); + if(stmt.step()) rc = stmt.get(0,asType); + }finally{ + if(stmt) stmt.finalize(); + } + return rc; + }, + /** + Prepares the given SQL, step()s it one time, and returns an + array containing the values of the first result row. If it has + no results, `undefined` is returned. + + If passed a second argument other than `undefined`, it is + treated like an argument to Stmt.bind(), so may be any type + supported by that function. + + Throws on error (e.g. malformed SQL). + */ + selectArray: function(sql,bind){ + return __selectFirstRow(this, sql, bind, []); + }, + + /** + Prepares the given SQL, step()s it one time, and returns an + object containing the key/value pairs of the first result + row. If it has no results, `undefined` is returned. + + Note that the order of returned object's keys is not guaranteed + to be the same as the order of the fields in the query string. + + If passed a second argument other than `undefined`, it is + treated like an argument to Stmt.bind(), so may be any type + supported by that function. + + Throws on error (e.g. malformed SQL). + */ + selectObject: function(sql,bind){ + return __selectFirstRow(this, sql, bind, {}); + }, + + /** + Returns the number of currently-opened Stmt handles for this db + handle, or 0 if this DB instance is closed. + */ + openStatementCount: function(){ + return this.pointer ? Object.keys(__stmtMap.get(this)).length : 0; + }, + + /** + Starts a transaction, calls the given callback, and then either + rolls back or commits the savepoint, depending on whether the + callback throws. The callback is passed this db object as its + only argument. On success, returns the result of the + callback. Throws on error. + + Note that transactions may not be nested, so this will throw if + it is called recursively. For nested transactions, use the + savepoint() method or manually manage SAVEPOINTs using exec(). + */ + transaction: function(callback){ + affirmDbOpen(this).exec("BEGIN"); + try { + const rc = callback(this); + this.exec("COMMIT"); + return rc; + }catch(e){ + this.exec("ROLLBACK"); + throw e; + } + }, + + /** + This works similarly to transaction() but uses sqlite3's SAVEPOINT + feature. This function starts a savepoint (with an unspecified name) + and calls the given callback function, passing it this db object. + If the callback returns, the savepoint is released (committed). If + the callback throws, the savepoint is rolled back. If it does not + throw, it returns the result of the callback. + */ + savepoint: function(callback){ + affirmDbOpen(this).exec("SAVEPOINT oo1"); + try { + const rc = callback(this); + this.exec("RELEASE oo1"); + return rc; + }catch(e){ + this.exec("ROLLBACK to SAVEPOINT oo1; RELEASE SAVEPOINT oo1"); + throw e; + } + } + }/*DB.prototype*/; + + + /** Throws if the given Stmt has been finalized, else stmt is + returned. */ + const affirmStmtOpen = function(stmt){ + if(!stmt.pointer) toss3("Stmt has been closed."); + return stmt; + }; + + /** Returns an opaque truthy value from the BindTypes + enum if v's type is a valid bindable type, else + returns a falsy value. As a special case, a value of + undefined is treated as a bind type of null. */ + const isSupportedBindType = function(v){ + let t = BindTypes[(null===v||undefined===v) ? 'null' : typeof v]; + switch(t){ + case BindTypes.boolean: + case BindTypes.null: + case BindTypes.number: + case BindTypes.string: + return t; + case BindTypes.bigint: + if(wasm.bigIntEnabled) return t; + /* else fall through */ + default: + //console.log("isSupportedBindType",t,v); + return util.isBindableTypedArray(v) ? BindTypes.blob : undefined; + } + }; + + /** + If isSupportedBindType(v) returns a truthy value, this + function returns that value, else it throws. + */ + const affirmSupportedBindType = function(v){ + //console.log('affirmSupportedBindType',v); + return isSupportedBindType(v) || toss3("Unsupported bind() argument type:",typeof v); + }; + + /** + If key is a number and within range of stmt's bound parameter + count, key is returned. + + If key is not a number then it is checked against named + parameters. If a match is found, its index is returned. + + Else it throws. + */ + const affirmParamIndex = function(stmt,key){ + const n = ('number'===typeof key) + ? key : capi.sqlite3_bind_parameter_index(stmt.pointer, key); + if(0===n || !util.isInt32(n)){ + toss3("Invalid bind() parameter name: "+key); + } + else if(n<1 || n>stmt.parameterCount) toss3("Bind index",key,"is out of range."); + return n; + }; + + /** + If stmt._isLocked is truthy, this throws an exception + complaining that the 2nd argument (an operation name, + e.g. "bind()") is not legal while the statement is "locked". + Locking happens before an exec()-like callback is passed a + statement, to ensure that the callback does not mutate or + finalize the statement. If it does not throw, it returns stmt. + */ + const affirmUnlocked = function(stmt,currentOpName){ + if(stmt._isLocked){ + toss3("Operation is illegal when statement is locked:",currentOpName); + } + return stmt; + }; + + /** + Binds a single bound parameter value on the given stmt at the + given index (numeric or named) using the given bindType (see + the BindTypes enum) and value. Throws on error. Returns stmt on + success. + */ + const bindOne = function f(stmt,ndx,bindType,val){ + affirmUnlocked(stmt, 'bind()'); + if(!f._){ + f._tooBigInt = (v)=>toss3( + "BigInt value is too big to store without precision loss:", v + ); + /* Reminder: when not in BigInt mode, it's impossible for + JS to represent a number out of the range we can bind, + so we have no range checking. */ + f._ = { + string: function(stmt, ndx, val, asBlob){ + if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = wasm.scopedAllocPush(); + try{ + const n = wasm.jstrlen(val); + const pStr = wasm.scopedAlloc(n); + wasm.jstrcpy(val, wasm.heap8u(), pStr, n, false); + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, n, capi.SQLITE_TRANSIENT); + }finally{ + wasm.scopedAllocPop(stack); + } + }else{ + const bytes = wasm.jstrToUintArray(val,false); + const pStr = wasm.alloc(bytes.length || 1); + wasm.heap8u().set(bytes.length ? bytes : [0], pStr); + try{ + const f = asBlob ? capi.sqlite3_bind_blob : capi.sqlite3_bind_text; + return f(stmt.pointer, ndx, pStr, bytes.length, capi.SQLITE_TRANSIENT); + }finally{ + wasm.dealloc(pStr); + } + } + } + }; + }/* static init */ + affirmSupportedBindType(val); + ndx = affirmParamIndex(stmt,ndx); + let rc = 0; + switch((null===val || undefined===val) ? BindTypes.null : bindType){ + case BindTypes.null: + rc = capi.sqlite3_bind_null(stmt.pointer, ndx); + break; + case BindTypes.string: + rc = f._.string(stmt, ndx, val, false); + break; + case BindTypes.number: { + let m; + if(util.isInt32(val)) m = capi.sqlite3_bind_int; + else if('bigint'===typeof val){ + if(!util.bigIntFits64(val)){ + f._tooBigInt(val); + }else if(wasm.bigIntEnabled){ + m = capi.sqlite3_bind_int64; + }else if(util.bigIntFitsDouble(val)){ + val = Number(val); + m = capi.sqlite3_bind_double; + }else{ + f._tooBigInt(val); + } + }else{ // !int32, !bigint + val = Number(val); + if(wasm.bigIntEnabled && Number.isInteger(val)){ + m = capi.sqlite3_bind_int64; + }else{ + m = capi.sqlite3_bind_double; + } + } + rc = m(stmt.pointer, ndx, val); + break; + } + case BindTypes.boolean: + rc = capi.sqlite3_bind_int(stmt.pointer, ndx, val ? 1 : 0); + break; + case BindTypes.blob: { + if('string'===typeof val){ + rc = f._.string(stmt, ndx, val, true); + }else if(!util.isBindableTypedArray(val)){ + toss3("Binding a value as a blob requires", + "that it be a string, Uint8Array, or Int8Array."); + }else if(1){ + /* _Hypothetically_ more efficient than the impl in the 'else' block. */ + const stack = wasm.scopedAllocPush(); + try{ + const pBlob = wasm.scopedAlloc(val.byteLength || 1); + wasm.heap8().set(val.byteLength ? val : [0], pBlob) + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + }finally{ + wasm.scopedAllocPop(stack); + } + }else{ + const pBlob = wasm.allocFromTypedArray(val); + try{ + rc = capi.sqlite3_bind_blob(stmt.pointer, ndx, pBlob, val.byteLength, + capi.SQLITE_TRANSIENT); + }finally{ + wasm.dealloc(pBlob); + } + } + break; + } + default: + console.warn("Unsupported bind() argument type:",val); + toss3("Unsupported bind() argument type: "+(typeof val)); + } + if(rc) DB.checkRc(stmt.db.pointer, rc); + return stmt; + }; + + Stmt.prototype = { + /** + "Finalizes" this statement. This is a no-op if the + statement has already been finalizes. Returns + undefined. Most methods in this class will throw if called + after this is. + */ + finalize: function(){ + if(this.pointer){ + affirmUnlocked(this,'finalize()'); + delete __stmtMap.get(this.db)[this.pointer]; + capi.sqlite3_finalize(this.pointer); + __ptrMap.delete(this); + delete this._mayGet; + delete this.columnCount; + delete this.parameterCount; + delete this.db; + delete this._isLocked; + } + }, + /** Clears all bound values. Returns this object. + Throws if this statement has been finalized. */ + clearBindings: function(){ + affirmUnlocked(affirmStmtOpen(this), 'clearBindings()') + capi.sqlite3_clear_bindings(this.pointer); + this._mayGet = false; + return this; + }, + /** + Resets this statement so that it may be step()ed again + from the beginning. Returns this object. Throws if this + statement has been finalized. + + If passed a truthy argument then this.clearBindings() is + also called, otherwise any existing bindings, along with + any memory allocated for them, are retained. + */ + reset: function(alsoClearBinds){ + affirmUnlocked(this,'reset()'); + if(alsoClearBinds) this.clearBindings(); + capi.sqlite3_reset(affirmStmtOpen(this).pointer); + this._mayGet = false; + return this; + }, + /** + Binds one or more values to its bindable parameters. It + accepts 1 or 2 arguments: + + If passed a single argument, it must be either an array, an + object, or a value of a bindable type (see below). + + If passed 2 arguments, the first one is the 1-based bind + index or bindable parameter name and the second one must be + a value of a bindable type. + + Bindable value types: + + - null is bound as NULL. + + - undefined as a standalone value is a no-op intended to + simplify certain client-side use cases: passing undefined as + a value to this function will not actually bind anything and + this function will skip confirmation that binding is even + legal. (Those semantics simplify certain client-side uses.) + Conversely, a value of undefined as an array or object + property when binding an array/object (see below) is treated + the same as null. + + - Numbers are bound as either doubles or integers: doubles if + they are larger than 32 bits, else double or int32, depending + on whether they have a fractional part. Booleans are bound as + integer 0 or 1. It is not expected the distinction of binding + doubles which have no fractional parts is integers is + significant for the majority of clients due to sqlite3's data + typing model. If [BigInt] support is enabled then this + routine will bind BigInt values as 64-bit integers if they'll + fit in 64 bits. If that support disabled, it will store the + BigInt as an int32 or a double if it can do so without loss + of precision. If the BigInt is _too BigInt_ then it will + throw. + + - Strings are bound as strings (use bindAsBlob() to force + blob binding). + + - Uint8Array and Int8Array instances are bound as blobs. + (TODO: binding the other TypedArray types.) + + If passed an array, each element of the array is bound at + the parameter index equal to the array index plus 1 + (because arrays are 0-based but binding is 1-based). + + If passed an object, each object key is treated as a + bindable parameter name. The object keys _must_ match any + bindable parameter names, including any `$`, `@`, or `:` + prefix. Because `$` is a legal identifier chararacter in + JavaScript, that is the suggested prefix for bindable + parameters: `stmt.bind({$a: 1, $b: 2})`. + + It returns this object on success and throws on + error. Errors include: + + - Any bind index is out of range, a named bind parameter + does not match, or this statement has no bindable + parameters. + + - Any value to bind is of an unsupported type. + + - Passed no arguments or more than two. + + - The statement has been finalized. + */ + bind: function(/*[ndx,] arg*/){ + affirmStmtOpen(this); + let ndx, arg; + switch(arguments.length){ + case 1: ndx = 1; arg = arguments[0]; break; + case 2: ndx = arguments[0]; arg = arguments[1]; break; + default: toss3("Invalid bind() arguments."); + } + if(undefined===arg){ + /* It might seem intuitive to bind undefined as NULL + but this approach simplifies certain client-side + uses when passing on arguments between 2+ levels of + functions. */ + return this; + }else if(!this.parameterCount){ + toss3("This statement has no bindable parameters."); + } + this._mayGet = false; + if(null===arg){ + /* bind NULL */ + return bindOne(this, ndx, BindTypes.null, arg); + } + else if(Array.isArray(arg)){ + /* bind each entry by index */ + if(1!==arguments.length){ + toss3("When binding an array, an index argument is not permitted."); + } + arg.forEach((v,i)=>bindOne(this, i+1, affirmSupportedBindType(v), v)); + return this; + } + else if('object'===typeof arg/*null was checked above*/ + && !util.isBindableTypedArray(arg)){ + /* Treat each property of arg as a named bound parameter. */ + if(1!==arguments.length){ + toss3("When binding an object, an index argument is not permitted."); + } + Object.keys(arg) + .forEach(k=>bindOne(this, k, + affirmSupportedBindType(arg[k]), + arg[k])); + return this; + }else{ + return bindOne(this, ndx, affirmSupportedBindType(arg), arg); + } + toss3("Should not reach this point."); + }, + /** + Special case of bind() which binds the given value using the + BLOB binding mechanism instead of the default selected one for + the value. The ndx may be a numbered or named bind index. The + value must be of type string, null/undefined (both get treated + as null), or a TypedArray of a type supported by the bind() + API. + + If passed a single argument, a bind index of 1 is assumed and + the first argument is the value. + */ + bindAsBlob: function(ndx,arg){ + affirmStmtOpen(this); + if(1===arguments.length){ + arg = ndx; + ndx = 1; + } + const t = affirmSupportedBindType(arg); + if(BindTypes.string !== t && BindTypes.blob !== t + && BindTypes.null !== t){ + toss3("Invalid value type for bindAsBlob()"); + } + bindOne(this, ndx, BindTypes.blob, arg); + this._mayGet = false; + return this; + }, + /** + Steps the statement one time. If the result indicates that a + row of data is available, a truthy value is returned. + If no row of data is available, a falsy + value is returned. Throws on error. + */ + step: function(){ + affirmUnlocked(this, 'step()'); + const rc = capi.sqlite3_step(affirmStmtOpen(this).pointer); + switch(rc){ + case capi.SQLITE_DONE: return this._mayGet = false; + case capi.SQLITE_ROW: return this._mayGet = true; + default: + this._mayGet = false; + console.warn("sqlite3_step() rc=",rc, + capi.sqlite3_js_rc_str(rc), + "SQL =", capi.sqlite3_sql(this.pointer)); + DB.checkRc(this.db.pointer, rc); + } + }, + /** + Functions exactly like step() except that... + + 1) On success, it calls this.reset() and returns this object. + 2) On error, it throws and does not call reset(). + + This is intended to simplify constructs like: + + ``` + for(...) { + stmt.bind(...).stepReset(); + } + ``` + + Note that the reset() call makes it illegal to call this.get() + after the step. + */ + stepReset: function(){ + this.step(); + return this.reset(); + }, + /** + Functions like step() except that it finalizes this statement + immediately after stepping unless the step cannot be performed + because the statement is locked. Throws on error, but any error + other than the statement-is-locked case will also trigger + finalization of this statement. + + On success, it returns true if the step indicated that a row of + data was available, else it returns false. + + This is intended to simplify use cases such as: + + ``` + aDb.prepare("insert into foo(a) values(?)").bind(123).stepFinalize(); + ``` + */ + stepFinalize: function(){ + const rc = this.step(); + this.finalize(); + return rc; + }, + /** + Fetches the value from the given 0-based column index of + the current data row, throwing if index is out of range. + + Requires that step() has just returned a truthy value, else + an exception is thrown. + + By default it will determine the data type of the result + automatically. If passed a second arugment, it must be one + of the enumeration values for sqlite3 types, which are + defined as members of the sqlite3 module: SQLITE_INTEGER, + SQLITE_FLOAT, SQLITE_TEXT, SQLITE_BLOB. Any other value, + except for undefined, will trigger an exception. Passing + undefined is the same as not passing a value. It is legal + to, e.g., fetch an integer value as a string, in which case + sqlite3 will convert the value to a string. + + If ndx is an array, this function behaves a differently: it + assigns the indexes of the array, from 0 to the number of + result columns, to the values of the corresponding column, + and returns that array. + + If ndx is a plain object, this function behaves even + differentlier: it assigns the properties of the object to + the values of their corresponding result columns. + + Blobs are returned as Uint8Array instances. + + Potential TODO: add type ID SQLITE_JSON, which fetches the + result as a string and passes it (if it's not null) to + JSON.parse(), returning the result of that. Until then, + getJSON() can be used for that. + */ + get: function(ndx,asType){ + if(!affirmStmtOpen(this)._mayGet){ + toss3("Stmt.step() has not (recently) returned true."); + } + if(Array.isArray(ndx)){ + let i = 0; + while(i<this.columnCount){ + ndx[i] = this.get(i++); + } + return ndx; + }else if(ndx && 'object'===typeof ndx){ + let i = 0; + while(i<this.columnCount){ + ndx[capi.sqlite3_column_name(this.pointer,i)] = this.get(i++); + } + return ndx; + } + affirmColIndex(this, ndx); + switch(undefined===asType + ? capi.sqlite3_column_type(this.pointer, ndx) + : asType){ + case capi.SQLITE_NULL: return null; + case capi.SQLITE_INTEGER:{ + if(wasm.bigIntEnabled){ + const rc = capi.sqlite3_column_int64(this.pointer, ndx); + if(rc>=Number.MIN_SAFE_INTEGER && rc<=Number.MAX_SAFE_INTEGER){ + /* Coerce "normal" number ranges to normal number values, + and only return BigInt-type values for numbers out of this + range. */ + return Number(rc).valueOf(); + } + return rc; + }else{ + const rc = capi.sqlite3_column_double(this.pointer, ndx); + if(rc>Number.MAX_SAFE_INTEGER || rc<Number.MIN_SAFE_INTEGER){ + /* Throwing here is arguable but, since we're explicitly + extracting an SQLITE_INTEGER-type value, it seems fair to throw + if the extracted number is out of range for that type. + This policy may be laxened to simply pass on the number and + hope for the best, as the C API would do. */ + toss3("Integer is out of range for JS integer range: "+rc); + } + //console.log("get integer rc=",rc,isInt32(rc)); + return util.isInt32(rc) ? (rc | 0) : rc; + } + } + case capi.SQLITE_FLOAT: + return capi.sqlite3_column_double(this.pointer, ndx); + case capi.SQLITE_TEXT: + return capi.sqlite3_column_text(this.pointer, ndx); + case capi.SQLITE_BLOB: { + const n = capi.sqlite3_column_bytes(this.pointer, ndx), + ptr = capi.sqlite3_column_blob(this.pointer, ndx), + rc = new Uint8Array(n); + //heap = n ? wasm.heap8() : false; + if(n) rc.set(wasm.heap8u().slice(ptr, ptr+n), 0); + //for(let i = 0; i < n; ++i) rc[i] = heap[ptr + i]; + if(n && this.db._blobXfer instanceof Array){ + /* This is an optimization soley for the + Worker-based API. These values will be + transfered to the main thread directly + instead of being copied. */ + this.db._blobXfer.push(rc.buffer); + } + return rc; + } + default: toss3("Don't know how to translate", + "type of result column #"+ndx+"."); + } + toss3("Not reached."); + }, + /** Equivalent to get(ndx) but coerces the result to an + integer. */ + getInt: function(ndx){return this.get(ndx,capi.SQLITE_INTEGER)}, + /** Equivalent to get(ndx) but coerces the result to a + float. */ + getFloat: function(ndx){return this.get(ndx,capi.SQLITE_FLOAT)}, + /** Equivalent to get(ndx) but coerces the result to a + string. */ + getString: function(ndx){return this.get(ndx,capi.SQLITE_TEXT)}, + /** Equivalent to get(ndx) but coerces the result to a + Uint8Array. */ + getBlob: function(ndx){return this.get(ndx,capi.SQLITE_BLOB)}, + /** + A convenience wrapper around get() which fetches the value + as a string and then, if it is not null, passes it to + JSON.parse(), returning that result. Throws if parsing + fails. If the result is null, null is returned. An empty + string, on the other hand, will trigger an exception. + */ + getJSON: function(ndx){ + const s = this.get(ndx, capi.SQLITE_STRING); + return null===s ? s : JSON.parse(s); + }, + // Design note: the only reason most of these getters have a 'get' + // prefix is for consistency with getVALUE_TYPE(). The latter + // arguably really need that prefix for API readability and the + // rest arguably don't, but consistency is a powerful thing. + /** + Returns the result column name of the given index, or + throws if index is out of bounds or this statement has been + finalized. This can be used without having run step() + first. + */ + getColumnName: function(ndx){ + return capi.sqlite3_column_name( + affirmColIndex(affirmStmtOpen(this),ndx).pointer, ndx + ); + }, + /** + If this statement potentially has result columns, this + function returns an array of all such names. If passed an + array, it is used as the target and all names are appended + to it. Returns the target array. Throws if this statement + cannot have result columns. This object's columnCount member + holds the number of columns. + */ + getColumnNames: function(tgt=[]){ + affirmColIndex(affirmStmtOpen(this),0); + for(let i = 0; i < this.columnCount; ++i){ + tgt.push(capi.sqlite3_column_name(this.pointer, i)); + } + return tgt; + }, + /** + If this statement has named bindable parameters and the + given name matches one, its 1-based bind index is + returned. If no match is found, 0 is returned. If it has no + bindable parameters, the undefined value is returned. + */ + getParamIndex: function(name){ + return (affirmStmtOpen(this).parameterCount + ? capi.sqlite3_bind_parameter_index(this.pointer, name) + : undefined); + } + }/*Stmt.prototype*/; + + {/* Add the `pointer` property to DB and Stmt. */ + const prop = { + enumerable: true, + get: function(){return __ptrMap.get(this)}, + set: ()=>toss3("The pointer property is read-only.") + } + Object.defineProperty(Stmt.prototype, 'pointer', prop); + Object.defineProperty(DB.prototype, 'pointer', prop); + } + + /** The OO API's public namespace. */ + sqlite3.oo1 = { + version: { + lib: capi.sqlite3_libversion(), + ooApi: "0.1" + }, + DB, + Stmt + }/*oo1 object*/; + + if(util.isUIThread()){ + /** + Functionally equivalent to DB(storageName,'c','kvvfs') except + that it throws if the given storage name is not one of 'local' + or 'session'. + */ + sqlite3.oo1.JsStorageDb = function(storageName='session'){ + if('session'!==storageName && 'local'!==storageName){ + toss3("JsStorageDb db name must be one of 'session' or 'local'."); + } + dbCtorHelper.call(this, { + filename: storageName, + flags: 'c', + vfs: "kvvfs" + }); + }; + const jdb = sqlite3.oo1.JsStorageDb; + jdb.prototype = Object.create(DB.prototype); + /** Equivalent to sqlite3_js_kvvfs_clear(). */ + jdb.clearStorage = capi.sqlite3_js_kvvfs_clear; + /** + Clears this database instance's storage or throws if this + instance has been closed. Returns the number of + database blocks which were cleaned up. + */ + jdb.prototype.clearStorage = function(){ + return jdb.clearStorage(affirmDbOpen(this).filename); + }; + /** Equivalent to sqlite3_js_kvvfs_size(). */ + jdb.storageSize = capi.sqlite3_js_kvvfs_size; + /** + Returns the _approximate_ number of bytes this database takes + up in its storage or throws if this instance has been closed. + */ + jdb.prototype.storageSize = function(){ + return jdb.storageSize(affirmDbOpen(this).filename); + }; + }/*main-window-only bits*/ + +}); + diff --git a/ext/wasm/api/sqlite3-api-opfs.js b/ext/wasm/api/sqlite3-api-opfs.js new file mode 100644 index 0000000..da5496f --- /dev/null +++ b/ext/wasm/api/sqlite3-api-opfs.js @@ -0,0 +1,1311 @@ +/* + 2022-09-18 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file holds the synchronous half of an sqlite3_vfs + implementation which proxies, in a synchronous fashion, the + asynchronous Origin-Private FileSystem (OPFS) APIs using a second + Worker, implemented in sqlite3-opfs-async-proxy.js. This file is + intended to be appended to the main sqlite3 JS deliverable somewhere + after sqlite3-api-oo1.js and before sqlite3-api-cleanup.js. +*/ +'use strict'; +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +/** + installOpfsVfs() returns a Promise which, on success, installs an + sqlite3_vfs named "opfs", suitable for use with all sqlite3 APIs + which accept a VFS. It is intended to be called via + sqlite3ApiBootstrap.initializersAsync or an equivalent mechanism. + + The installed VFS uses the Origin-Private FileSystem API for + all file storage. On error it is rejected with an exception + explaining the problem. Reasons for rejection include, but are + not limited to: + + - The counterpart Worker (see below) could not be loaded. + + - The environment does not support OPFS. That includes when + this function is called from the main window thread. + + Significant notes and limitations: + + - As of this writing, OPFS is still very much in flux and only + available in bleeding-edge versions of Chrome (v102+, noting that + that number will increase as the OPFS API matures). + + - The OPFS features used here are only available in dedicated Worker + threads. This file tries to detect that case, resulting in a + rejected Promise if those features do not seem to be available. + + - It requires the SharedArrayBuffer and Atomics classes, and the + former is only available if the HTTP server emits the so-called + COOP and COEP response headers. These features are required for + proxying OPFS's synchronous API via the synchronous interface + required by the sqlite3_vfs API. + + - This function may only be called a single time. When called, this + function removes itself from the sqlite3 object. + + All arguments to this function are for internal/development purposes + only. They do not constitute a public API and may change at any + time. + + The argument may optionally be a plain object with the following + configuration options: + + - proxyUri: as described above + + - verbose (=2): an integer 0-3. 0 disables all logging, 1 enables + logging of errors. 2 enables logging of warnings and errors. 3 + additionally enables debugging info. + + - sanityChecks (=false): if true, some basic sanity tests are + run on the OPFS VFS API after it's initialized, before the + returned Promise resolves. + + On success, the Promise resolves to the top-most sqlite3 namespace + object and that object gets a new object installed in its + `opfs` property, containing several OPFS-specific utilities. +*/ +const installOpfsVfs = function callee(options){ + if(!self.SharedArrayBuffer || + !self.Atomics || + !self.FileSystemHandle || + !self.FileSystemDirectoryHandle || + !self.FileSystemFileHandle || + !self.FileSystemFileHandle.prototype.createSyncAccessHandle || + !navigator.storage.getDirectory){ + return Promise.reject( + new Error("This environment does not have OPFS support.") + ); + } + if(!options || 'object'!==typeof options){ + options = Object.create(null); + } + const urlParams = new URL(self.location.href).searchParams; + if(undefined===options.verbose){ + options.verbose = urlParams.has('opfs-verbose') ? 3 : 2; + } + if(undefined===options.sanityChecks){ + options.sanityChecks = urlParams.has('opfs-sanity-check'); + } + if(undefined===options.proxyUri){ + options.proxyUri = callee.defaultProxyUri; + } + + if('function' === typeof options.proxyUri){ + options.proxyUri = options.proxyUri(); + } + const thePromise = new Promise(function(promiseResolve, promiseReject_){ + const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) + }; + const logImpl = (level,...args)=>{ + if(options.verbose>level) loggers[level]("OPFS syncer:",...args); + }; + const log = (...args)=>logImpl(2, ...args); + const warn = (...args)=>logImpl(1, ...args); + const error = (...args)=>logImpl(0, ...args); + const toss = function(...args){throw new Error(args.join(' '))}; + const capi = sqlite3.capi; + const wasm = sqlite3.wasm; + const sqlite3_vfs = capi.sqlite3_vfs; + const sqlite3_file = capi.sqlite3_file; + const sqlite3_io_methods = capi.sqlite3_io_methods; + /** + Generic utilities for working with OPFS. This will get filled out + by the Promise setup and, on success, installed as sqlite3.opfs. + */ + const opfsUtil = Object.create(null); + /** + Not part of the public API. Solely for internal/development + use. + */ + opfsUtil.metrics = { + dump: function(){ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + m.avgWait = (m.count && m.wait) ? (m.wait / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":",metrics, + "\nTotal of",n,"op(s) for",t, + "ms (incl. "+w+" ms of waiting on the async side)"); + console.log("Serialization metrics:",metrics.s11n); + W.postMessage({type:'opfs-async-metrics'}); + }, + reset: function(){ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; + } + }/*metrics*/; + const promiseReject = function(err){ + opfsVfs.dispose(); + return promiseReject_(err); + }; + const W = new Worker(options.proxyUri); + W._originalOnError = W.onerror /* will be restored later */; + W.onerror = function(err){ + // The error object doesn't contain any useful info when the + // failure is, e.g., that the remote script is 404. + error("Error initializing OPFS asyncer:",err); + promiseReject(new Error("Loading OPFS async Worker failed for unknown reasons.")); + }; + const pDVfs = capi.sqlite3_vfs_find(null)/*pointer to default VFS*/; + const dVfs = pDVfs + ? new sqlite3_vfs(pDVfs) + : null /* dVfs will be null when sqlite3 is built with + SQLITE_OS_OTHER. Though we cannot currently handle + that case, the hope is to eventually be able to. */; + const opfsVfs = new sqlite3_vfs(); + const opfsIoMethods = new sqlite3_io_methods(); + opfsVfs.$iVersion = 2/*yes, two*/; + opfsVfs.$szOsFile = capi.sqlite3_file.structInfo.sizeof; + opfsVfs.$mxPathname = 1024/*sure, why not?*/; + opfsVfs.$zName = wasm.allocCString("opfs"); + // All C-side memory of opfsVfs is zeroed out, but just to be explicit: + opfsVfs.$xDlOpen = opfsVfs.$xDlError = opfsVfs.$xDlSym = opfsVfs.$xDlClose = null; + opfsVfs.ondispose = [ + '$zName', opfsVfs.$zName, + 'cleanup default VFS wrapper', ()=>(dVfs ? dVfs.dispose() : null), + 'cleanup opfsIoMethods', ()=>opfsIoMethods.dispose() + ]; + /** + Pedantic sidebar about opfsVfs.ondispose: the entries in that array + are items to clean up when opfsVfs.dispose() is called, but in this + environment it will never be called. The VFS instance simply + hangs around until the WASM module instance is cleaned up. We + "could" _hypothetically_ clean it up by "importing" an + sqlite3_os_end() impl into the wasm build, but the shutdown order + of the wasm engine and the JS one are undefined so there is no + guaranty that the opfsVfs instance would be available in one + environment or the other when sqlite3_os_end() is called (_if_ it + gets called at all in a wasm build, which is undefined). + */ + /** + State which we send to the async-api Worker or share with it. + This object must initially contain only cloneable or sharable + objects. After the worker's "inited" message arrives, other types + of data may be added to it. + + For purposes of Atomics.wait() and Atomics.notify(), we use a + SharedArrayBuffer with one slot reserved for each of the API + proxy's methods. The sync side of the API uses Atomics.wait() + on the corresponding slot and the async side uses + Atomics.notify() on that slot. + + The approach of using a single SAB to serialize comms for all + instances might(?) lead to deadlock situations in multi-db + cases. We should probably have one SAB here with a single slot + for locking a per-file initialization step and then allocate a + separate SAB like the above one for each file. That will + require a bit of acrobatics but should be feasible. The most + problematic part is that xOpen() would have to use + postMessage() to communicate its SharedArrayBuffer, and mixing + that approach with Atomics.wait/notify() gets a bit messy. + */ + const state = Object.create(null); + state.verbose = options.verbose; + state.littleEndian = (()=>{ + const buffer = new ArrayBuffer(2); + new DataView(buffer).setInt16(0, 256, true /* ==>littleEndian */); + // Int16Array uses the platform's endianness. + return new Int16Array(buffer)[0] === 256; + })(); + /** + Whether the async counterpart should log exceptions to + the serialization channel. That produces a great deal of + noise for seemingly innocuous things like xAccess() checks + for missing files, so this option may have one of 3 values: + + 0 = no exception logging + + 1 = only log exceptions for "significant" ops like xOpen(), + xRead(), and xWrite(). + + 2 = log all exceptions. + */ + state.asyncS11nExceptions = 1; + /* Size of file I/O buffer block. 64k = max sqlite3 page size, and + xRead/xWrite() will never deal in blocks larger than that. */ + state.fileBufferSize = 1024 * 64; + state.sabS11nOffset = state.fileBufferSize; + /** + The size of the block in our SAB for serializing arguments and + result values. Needs to be large enough to hold serialized + values of any of the proxied APIs. Filenames are the largest + part but are limited to opfsVfs.$mxPathname bytes. + */ + state.sabS11nSize = opfsVfs.$mxPathname * 2; + /** + The SAB used for all data I/O between the synchronous and + async halves (file i/o and arg/result s11n). + */ + state.sabIO = new SharedArrayBuffer( + state.fileBufferSize/* file i/o block */ + + state.sabS11nSize/* argument/result serialization block */ + ); + state.opIds = Object.create(null); + const metrics = Object.create(null); + { + /* Indexes for use in our SharedArrayBuffer... */ + let i = 0; + /* SAB slot used to communicate which operation is desired + between both workers. This worker writes to it and the other + listens for changes. */ + state.opIds.whichOp = i++; + /* Slot for storing return values. This worker listens to that + slot and the other worker writes to it. */ + state.opIds.rc = i++; + /* Each function gets an ID which this worker writes to + the whichOp slot. The async-api worker uses Atomic.wait() + on the whichOp slot to figure out which operation to run + next. */ + state.opIds.xAccess = i++; + state.opIds.xClose = i++; + state.opIds.xDelete = i++; + state.opIds.xDeleteNoWait = i++; + state.opIds.xFileControl = i++; + state.opIds.xFileSize = i++; + state.opIds.xLock = i++; + state.opIds.xOpen = i++; + state.opIds.xRead = i++; + state.opIds.xSleep = i++; + state.opIds.xSync = i++; + state.opIds.xTruncate = i++; + state.opIds.xUnlock = i++; + state.opIds.xWrite = i++; + state.opIds.mkdir = i++; + state.opIds['opfs-async-metrics'] = i++; + state.opIds['opfs-async-shutdown'] = i++; + /* The retry slot is used by the async part for wait-and-retry + semantics. Though we could hypothetically use the xSleep slot + for that, doing so might lead to undesired side effects. */ + state.opIds.retry = i++; + state.sabOP = new SharedArrayBuffer( + i * 4/* ==sizeof int32, noting that Atomics.wait() and friends + can only function on Int32Array views of an SAB. */); + opfsUtil.metrics.reset(); + } + /** + SQLITE_xxx constants to export to the async worker + counterpart... + */ + state.sq3Codes = Object.create(null); + [ + 'SQLITE_ACCESS_EXISTS', + 'SQLITE_ACCESS_READWRITE', + 'SQLITE_ERROR', + 'SQLITE_IOERR', + 'SQLITE_IOERR_ACCESS', + 'SQLITE_IOERR_CLOSE', + 'SQLITE_IOERR_DELETE', + 'SQLITE_IOERR_FSYNC', + 'SQLITE_IOERR_LOCK', + 'SQLITE_IOERR_READ', + 'SQLITE_IOERR_SHORT_READ', + 'SQLITE_IOERR_TRUNCATE', + 'SQLITE_IOERR_UNLOCK', + 'SQLITE_IOERR_WRITE', + 'SQLITE_LOCK_EXCLUSIVE', + 'SQLITE_LOCK_NONE', + 'SQLITE_LOCK_PENDING', + 'SQLITE_LOCK_RESERVED', + 'SQLITE_LOCK_SHARED', + 'SQLITE_MISUSE', + 'SQLITE_NOTFOUND', + 'SQLITE_OPEN_CREATE', + 'SQLITE_OPEN_DELETEONCLOSE', + 'SQLITE_OPEN_READONLY' + ].forEach((k)=>{ + if(undefined === (state.sq3Codes[k] = capi[k])){ + toss("Maintenance required: not found:",k); + } + }); + + /** + Runs the given operation (by name) in the async worker + counterpart, waits for its response, and returns the result + which the async worker writes to SAB[state.opIds.rc]. The + 2nd and subsequent arguments must be the aruguments for the + async op. + */ + const opRun = (op,...args)=>{ + const opNdx = state.opIds[op] || toss("Invalid op ID:",op); + state.s11n.serialize(...args); + Atomics.store(state.sabOPView, state.opIds.rc, -1); + Atomics.store(state.sabOPView, state.opIds.whichOp, opNdx); + Atomics.notify(state.sabOPView, state.opIds.whichOp) + /* async thread will take over here */; + const t = performance.now(); + Atomics.wait(state.sabOPView, state.opIds.rc, -1) + /* When this wait() call returns, the async half will have + completed the operation and reported its results. */; + const rc = Atomics.load(state.sabOPView, state.opIds.rc); + metrics[op].wait += performance.now() - t; + if(rc && state.asyncS11nExceptions){ + const err = state.s11n.deserialize(); + if(err) error(op+"() async error:",...err); + } + return rc; + }; + + /** + Not part of the public API. Only for test/development use. + */ + opfsUtil.debug = { + asyncShutdown: ()=>{ + warn("Shutting down OPFS async listener. The OPFS VFS will no longer work."); + opRun('opfs-async-shutdown'); + }, + asyncRestart: ()=>{ + warn("Attempting to restart OPFS VFS async listener. Might work, might not."); + W.postMessage({type: 'opfs-async-restart'}); + } + }; + + const initS11n = ()=>{ + /** + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ACHTUNG: this code is 100% duplicated in the other half of + this proxy! The documentation is maintained in the + "synchronous half". + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + This proxy de/serializes cross-thread function arguments and + output-pointer values via the state.sabIO SharedArrayBuffer, + using the region defined by (state.sabS11nOffset, + state.sabS11nOffset]. Only one dataset is recorded at a time. + + This is not a general-purpose format. It only supports the + range of operations, and data sizes, needed by the + sqlite3_vfs and sqlite3_io_methods operations. Serialized + data are transient and this serialization algorithm may + change at any time. + + The data format can be succinctly summarized as: + + Nt...Td...D + + Where: + + - N = number of entries (1 byte) + + - t = type ID of first argument (1 byte) + + - ...T = type IDs of the 2nd and subsequent arguments (1 byte + each). + + - d = raw bytes of first argument (per-type size). + + - ...D = raw bytes of the 2nd and subsequent arguments (per-type + size). + + All types except strings have fixed sizes. Strings are stored + using their TextEncoder/TextDecoder representations. It would + arguably make more sense to store them as Int16Arrays of + their JS character values, but how best/fastest to get that + in and out of string form is an open point. Initial + experimentation with that approach did not gain us any speed. + + Historical note: this impl was initially about 1% this size by + using using JSON.stringify/parse(), but using fit-to-purpose + serialization saves considerable runtime. + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + /* Only arguments and return values of these types may be + serialized. This covers the whole range of types needed by the + sqlite3_vfs API. */ + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + + /** + Returns an array of the deserialized state stored by the most + recent serialize() operation (from from this thread or the + counterpart thread), or null if the serialization buffer is + empty. If passed a truthy argument, the serialization buffer + is cleared after deserialization. + */ + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + + /** + Serializes all arguments to the shared buffer for consumption + by the counterpart thread. + + This routine is only intended for serializing OPFS VFS + arguments and (in at least one special case) result values, + and the buffer is sized to be able to comfortably handle + those. + + If passed no arguments then it zeroes out the serialization + state. + */ + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + return state.s11n; + }/*initS11n()*/; + + /** + Generates a random ASCII string len characters long, intended for + use as a temporary file name. + */ + const randomFilename = function f(len=16){ + if(!f._chars){ + f._chars = "abcdefghijklmnopqrstuvwxyz"+ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ + "012346789"; + f._n = f._chars.length; + } + const a = []; + let i = 0; + for( ; i < len; ++i){ + const ndx = Math.random() * (f._n * 64) % f._n | 0; + a[i] = f._chars[ndx]; + } + return a.join(''); + }; + + /** + Map of sqlite3_file pointers to objects constructed by xOpen(). + */ + const __openFiles = Object.create(null); + + /** + Installs a StructBinder-bound function pointer member of the + given name and function in the given StructType target object. + It creates a WASM proxy for the given function and arranges for + that proxy to be cleaned up when tgt.dispose() is called. Throws + on the slightest hint of error (e.g. tgt is-not-a StructType, + name does not map to a struct-bound member, etc.). + + Returns a proxy for this function which is bound to tgt and takes + 2 args (name,func). That function returns the same thing, + permitting calls to be chained. + + If called with only 1 arg, it has no side effects but returns a + func with the same signature as described above. + */ + const installMethod = function callee(tgt, name, func){ + if(!(tgt instanceof sqlite3.StructBinder.StructType)){ + toss("Usage error: target object is-not-a StructType."); + } + if(1===arguments.length){ + return (n,f)=>callee(tgt,n,f); + } + if(!callee.argcProxy){ + callee.argcProxy = function(func,sig){ + return function(...args){ + if(func.length!==arguments.length){ + toss("Argument mismatch. Native signature is:",sig); + } + return func.apply(this, args); + } + }; + callee.removeFuncList = function(){ + if(this.ondispose.__removeFuncList){ + this.ondispose.__removeFuncList.forEach( + (v,ndx)=>{ + if('number'===typeof v){ + try{wasm.uninstallFunction(v)} + catch(e){/*ignore*/} + } + /* else it's a descriptive label for the next number in + the list. */ + } + ); + delete this.ondispose.__removeFuncList; + } + }; + }/*static init*/ + const sigN = tgt.memberSignature(name); + if(sigN.length<2){ + toss("Member",name," is not a function pointer. Signature =",sigN); + } + const memKey = tgt.memberKey(name); + const fProxy = 0 + /** This middle-man proxy is only for use during development, to + confirm that we always pass the proper number of + arguments. We know that the C-level code will always use the + correct argument count. */ + ? callee.argcProxy(func, sigN) + : func; + const pFunc = wasm.installFunction(fProxy, tgt.memberSignature(name, true)); + tgt[memKey] = pFunc; + if(!tgt.ondispose) tgt.ondispose = []; + if(!tgt.ondispose.__removeFuncList){ + tgt.ondispose.push('ondispose.__removeFuncList handler', + callee.removeFuncList); + tgt.ondispose.__removeFuncList = []; + } + tgt.ondispose.__removeFuncList.push(memKey, pFunc); + return (n,f)=>callee(tgt, n, f); + }/*installMethod*/; + + const opTimer = Object.create(null); + opTimer.op = undefined; + opTimer.start = undefined; + const mTimeStart = (op)=>{ + opTimer.start = performance.now(); + opTimer.op = op; + ++metrics[op].count; + }; + const mTimeEnd = ()=>( + metrics[opTimer.op].time += performance.now() - opTimer.start + ); + + /** + Impls for the sqlite3_io_methods methods. Maintenance reminder: + members are in alphabetical order to simplify finding them. + */ + const ioSyncWrappers = { + xCheckReservedLock: function(pFile,pOut){ + /** + As of late 2022, only a single lock can be held on an OPFS + file. We have no way of checking whether any _other_ db + connection has a lock except by trying to obtain and (on + success) release a sync-handle for it, but doing so would + involve an inherent race condition. For the time being, + pending a better solution, we simply report whether the + given pFile instance has a lock. + */ + const f = __openFiles[pFile]; + wasm.setMemValue(pOut, f.lockMode ? 1 : 0, 'i32'); + return 0; + }, + xClose: function(pFile){ + mTimeStart('xClose'); + let rc = 0; + const f = __openFiles[pFile]; + if(f){ + delete __openFiles[pFile]; + rc = opRun('xClose', pFile); + if(f.sq3File) f.sq3File.dispose(); + } + mTimeEnd(); + return rc; + }, + xDeviceCharacteristics: function(pFile){ + //debug("xDeviceCharacteristics(",pFile,")"); + return capi.SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN; + }, + xFileControl: function(pFile, opId, pArg){ + mTimeStart('xFileControl'); + const rc = (capi.SQLITE_FCNTL_SYNC===opId) + ? opRun('xSync', pFile, 0) + : capi.SQLITE_NOTFOUND; + mTimeEnd(); + return rc; + }, + xFileSize: function(pFile,pSz64){ + mTimeStart('xFileSize'); + const rc = opRun('xFileSize', pFile); + if(0==rc){ + const sz = state.s11n.deserialize()[0]; + wasm.setMemValue(pSz64, sz, 'i64'); + } + mTimeEnd(); + return rc; + }, + xLock: function(pFile,lockType){ + mTimeStart('xLock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === f.lockType ) { + rc = opRun('xLock', pFile, lockType); + if( 0===rc ) f.lockType = lockType; + }else{ + f.lockType = lockType; + } + mTimeEnd(); + return rc; + }, + xRead: function(pFile,pDest,n,offset64){ + mTimeStart('xRead'); + const f = __openFiles[pFile]; + let rc; + try { + rc = opRun('xRead',pFile, n, Number(offset64)); + if(0===rc || capi.SQLITE_IOERR_SHORT_READ===rc){ + /** + Results get written to the SharedArrayBuffer f.sabView. + Because the heap is _not_ a SharedArrayBuffer, we have + to copy the results. TypedArray.set() seems to be the + fastest way to copy this. */ + wasm.heap8u().set(f.sabView.subarray(0, n), pDest); + } + }catch(e){ + error("xRead(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_READ; + } + mTimeEnd(); + return rc; + }, + xSync: function(pFile,flags){ + ++metrics.xSync.count; + return 0; // impl'd in xFileControl() + }, + xTruncate: function(pFile,sz64){ + mTimeStart('xTruncate'); + const rc = opRun('xTruncate', pFile, Number(sz64)); + mTimeEnd(); + return rc; + }, + xUnlock: function(pFile,lockType){ + mTimeStart('xUnlock'); + const f = __openFiles[pFile]; + let rc = 0; + if( capi.SQLITE_LOCK_NONE === lockType + && f.lockType ){ + rc = opRun('xUnlock', pFile, lockType); + } + if( 0===rc ) f.lockType = lockType; + mTimeEnd(); + return rc; + }, + xWrite: function(pFile,pSrc,n,offset64){ + mTimeStart('xWrite'); + const f = __openFiles[pFile]; + let rc; + try { + f.sabView.set(wasm.heap8u().subarray(pSrc, pSrc+n)); + rc = opRun('xWrite', pFile, n, Number(offset64)); + }catch(e){ + error("xWrite(",arguments,") failed:",e,f); + rc = capi.SQLITE_IOERR_WRITE; + } + mTimeEnd(); + return rc; + } + }/*ioSyncWrappers*/; + + /** + Impls for the sqlite3_vfs methods. Maintenance reminder: members + are in alphabetical order to simplify finding them. + */ + const vfsSyncWrappers = { + xAccess: function(pVfs,zName,flags,pOut){ + mTimeStart('xAccess'); + const rc = opRun('xAccess', wasm.cstringToJs(zName)); + wasm.setMemValue( pOut, (rc ? 0 : 1), 'i32' ); + mTimeEnd(); + return 0; + }, + xCurrentTime: function(pVfs,pOut){ + /* If it turns out that we need to adjust for timezone, see: + https://stackoverflow.com/a/11760121/1458521 */ + wasm.setMemValue(pOut, 2440587.5 + (new Date().getTime()/86400000), + 'double'); + return 0; + }, + xCurrentTimeInt64: function(pVfs,pOut){ + // TODO: confirm that this calculation is correct + wasm.setMemValue(pOut, (2440587.5 * 86400000) + new Date().getTime(), + 'i64'); + return 0; + }, + xDelete: function(pVfs, zName, doSyncDir){ + mTimeStart('xDelete'); + opRun('xDelete', wasm.cstringToJs(zName), doSyncDir, false); + /* We're ignoring errors because we cannot yet differentiate + between harmless and non-harmless failures. */ + mTimeEnd(); + return 0; + }, + xFullPathname: function(pVfs,zName,nOut,pOut){ + /* Until/unless we have some notion of "current dir" + in OPFS, simply copy zName to pOut... */ + const i = wasm.cstrncpy(pOut, zName, nOut); + return i<nOut ? 0 : capi.SQLITE_CANTOPEN + /*CANTOPEN is required by the docs but SQLITE_RANGE would be a closer match*/; + }, + xGetLastError: function(pVfs,nOut,pOut){ + /* TODO: store exception.message values from the async + partner in a dedicated SharedArrayBuffer, noting that we'd have + to encode them... TextEncoder can do that for us. */ + warn("OPFS xGetLastError() has nothing sensible to return."); + return 0; + }, + //xSleep is optionally defined below + xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){ + mTimeStart('xOpen'); + if(0===zName){ + zName = randomFilename(); + }else if('number'===typeof zName){ + zName = wasm.cstringToJs(zName); + } + const fh = Object.create(null); + fh.fid = pFile; + fh.filename = zName; + fh.sab = new SharedArrayBuffer(state.fileBufferSize); + fh.flags = flags; + const rc = opRun('xOpen', pFile, zName, flags); + if(!rc){ + /* Recall that sqlite3_vfs::xClose() will be called, even on + error, unless pFile->pMethods is NULL. */ + if(fh.readOnly){ + wasm.setMemValue(pOutFlags, capi.SQLITE_OPEN_READONLY, 'i32'); + } + __openFiles[pFile] = fh; + fh.sabView = state.sabFileBufView; + fh.sq3File = new sqlite3_file(pFile); + fh.sq3File.$pMethods = opfsIoMethods.pointer; + fh.lockType = capi.SQLITE_LOCK_NONE; + } + mTimeEnd(); + return rc; + }/*xOpen()*/ + }/*vfsSyncWrappers*/; + + if(dVfs){ + opfsVfs.$xRandomness = dVfs.$xRandomness; + opfsVfs.$xSleep = dVfs.$xSleep; + } + if(!opfsVfs.$xRandomness){ + /* If the default VFS has no xRandomness(), add a basic JS impl... */ + vfsSyncWrappers.xRandomness = function(pVfs, nOut, pOut){ + const heap = wasm.heap8u(); + let i = 0; + for(; i < nOut; ++i) heap[pOut + i] = (Math.random()*255000) & 0xFF; + return i; + }; + } + if(!opfsVfs.$xSleep){ + /* If we can inherit an xSleep() impl from the default VFS then + assume it's sane and use it, otherwise install a JS-based + one. */ + vfsSyncWrappers.xSleep = function(pVfs,ms){ + Atomics.wait(state.sabOPView, state.opIds.xSleep, 0, ms); + return 0; + }; + } + + /* Install the vfs/io_methods into their C-level shared instances... */ + for(let k of Object.keys(ioSyncWrappers)){ + installMethod(opfsIoMethods, k, ioSyncWrappers[k]); + } + for(let k of Object.keys(vfsSyncWrappers)){ + installMethod(opfsVfs, k, vfsSyncWrappers[k]); + } + + /** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg + is true, the result is returned as an array of path elements, + else an absolute path string is returned. + */ + opfsUtil.getResolvedPath = function(filename,splitIt){ + const p = new URL(filename, "file://irrelevant").pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; + }; + + /** + Takes the absolute path to a filesystem element. Returns an + array of [handleOfContainingDir, filename]. If the 2nd argument + is truthy then each directory element leading to the file is + created along the way. Throws if any creation or resolution + fails. + */ + opfsUtil.getDirForFilename = async function f(absFilename, createDirs = false){ + const path = opfsUtil.getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = opfsUtil.rootDirectory; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; + }; + + /** + Creates the given directory name, recursively, in + the OPFS filesystem. Returns true if it succeeds or the + directory already exists, else false. + */ + opfsUtil.mkdir = async function(absDirName){ + try { + await opfsUtil.getDirForFilename(absDirName+"/filepart", true); + return true; + }catch(e){ + //console.warn("mkdir(",absDirName,") failed:",e); + return false; + } + }; + /** + Checks whether the given OPFS filesystem entry exists, + returning true if it does, false if it doesn't. + */ + opfsUtil.entryExists = async function(fsEntryName){ + try { + const [dh, fn] = await opfsUtil.getDirForFilename(fsEntryName); + await dh.getFileHandle(fn); + return true; + }catch(e){ + return false; + } + }; + + /** + Generates a random ASCII string, intended for use as a + temporary file name. Its argument is the length of the string, + defaulting to 16. + */ + opfsUtil.randomFilename = randomFilename; + + /** + Re-registers the OPFS VFS. This is intended only for odd use + cases which have to call sqlite3_shutdown() as part of their + initialization process, which will unregister the VFS + registered by installOpfsVfs(). If passed a truthy value, the + OPFS VFS is registered as the default VFS, else it is not made + the default. Returns the result of the the + sqlite3_vfs_register() call. + + Design note: the problem of having to re-register things after + a shutdown/initialize pair is more general. How to best plug + that in to the library is unclear. In particular, we cannot + hook in to any C-side calls to sqlite3_initialize(), so we + cannot add an after-initialize callback mechanism. + */ + opfsUtil.registerVfs = (asDefault=false)=>{ + return wasm.exports.sqlite3_vfs_register( + opfsVfs.pointer, asDefault ? 1 : 0 + ); + }; + + /** + Returns a promise which resolves to an object which represents + all files and directories in the OPFS tree. The top-most object + has two properties: `dirs` is an array of directory entries + (described below) and `files` is a list of file names for all + files in that directory. + + Traversal starts at sqlite3.opfs.rootDirectory. + + Each `dirs` entry is an object in this form: + + ``` + { name: directoryName, + dirs: [...subdirs], + files: [...file names] + } + ``` + + The `files` and `subdirs` entries are always set but may be + empty arrays. + + The returned object has the same structure but its `name` is + an empty string. All returned objects are created with + Object.create(null), so have no prototype. + + Design note: the entries do not contain more information, + e.g. file sizes, because getting such info is not only + expensive but is subject to locking-related errors. + */ + opfsUtil.treeList = async function(){ + const doDir = async function callee(dirHandle,tgt){ + tgt.name = dirHandle.name; + tgt.dirs = []; + tgt.files = []; + for await (const handle of dirHandle.values()){ + if('directory' === handle.kind){ + const subDir = Object.create(null); + tgt.dirs.push(subDir); + await callee(handle, subDir); + }else{ + tgt.files.push(handle.name); + } + } + }; + const root = Object.create(null); + await doDir(opfsUtil.rootDirectory, root); + return root; + }; + + /** + Irrevocably deletes _all_ files in the current origin's OPFS. + Obviously, this must be used with great caution. It may throw + an exception if removal of anything fails (e.g. a file is + locked), but the precise conditions under which it will throw + are not documented (so we cannot tell you what they are). + */ + opfsUtil.rmfr = async function(){ + const dir = opfsUtil.rootDirectory, opt = {recurse: true}; + for await (const handle of dir.values()){ + dir.removeEntry(handle.name, opt); + } + }; + + /** + Deletes the given OPFS filesystem entry. As this environment + has no notion of "current directory", the given name must be an + absolute path. If the 2nd argument is truthy, deletion is + recursive (use with caution!). + + The returned Promise resolves to true if the deletion was + successful, else false (but...). The OPFS API reports the + reason for the failure only in human-readable form, not + exceptions which can be type-checked to determine the + failure. Because of that... + + If the final argument is truthy then this function will + propagate any exception on error, rather than returning false. + */ + opfsUtil.unlink = async function(fsEntryName, recursive = false, + throwOnError = false){ + try { + const [hDir, filenamePart] = + await opfsUtil.getDirForFilename(fsEntryName, false); + await hDir.removeEntry(filenamePart, {recursive}); + return true; + }catch(e){ + if(throwOnError){ + throw new Error("unlink(",arguments[0],") failed: "+e.message,{ + cause: e + }); + } + return false; + } + }; + + /** + Traverses the OPFS filesystem, calling a callback for each one. + The argument may be either a callback function or an options object + with any of the following properties: + + - `callback`: function which gets called for each filesystem + entry. It gets passed 3 arguments: 1) the + FileSystemFileHandle or FileSystemDirectoryHandle of each + entry (noting that both are instanceof FileSystemHandle). 2) + the FileSystemDirectoryHandle of the parent directory. 3) the + current depth level, with 0 being at the top of the tree + relative to the starting directory. If the callback returns a + literal false, as opposed to any other falsy value, traversal + stops without an error. Any exceptions it throws are + propagated. Results are undefined if the callback manipulate + the filesystem (e.g. removing or adding entries) because the + how OPFS iterators behave in the face of such changes is + undocumented. + + - `recursive` [bool=true]: specifies whether to recurse into + subdirectories or not. Whether recursion is depth-first or + breadth-first is unspecified! + + - `directory` [FileSystemDirectoryEntry=sqlite3.opfs.rootDirectory] + specifies the starting directory. + + If this function is passed a function, it is assumed to be the + callback. + + Returns a promise because it has to (by virtue of being async) + but that promise has no specific meaning: the traversal it + performs is synchronous. The promise must be used to catch any + exceptions propagated by the callback, however. + + TODO: add an option which specifies whether to traverse + depth-first or breadth-first. We currently do depth-first but + an incremental file browsing widget would benefit more from + breadth-first. + */ + opfsUtil.traverse = async function(opt){ + const defaultOpt = { + recursive: true, + directory: opfsUtil.rootDirectory + }; + if('function'===typeof opt){ + opt = {callback:opt}; + } + opt = Object.assign(defaultOpt, opt||{}); + const doDir = async function callee(dirHandle, depth){ + for await (const handle of dirHandle.values()){ + if(false === opt.callback(handle, dirHandle, depth)) return false; + else if(opt.recursive && 'directory' === handle.kind){ + if(false === await callee(handle, depth + 1)) break; + } + } + }; + doDir(opt.directory, 0); + }; + + //TODO to support fiddle and worker1 db upload: + //opfsUtil.createFile = function(absName, content=undefined){...} + + if(sqlite3.oo1){ + opfsUtil.OpfsDb = function(...args){ + const opt = sqlite3.oo1.DB.dbCtorHelper.normalizeArgs(...args); + opt.vfs = opfsVfs.$zName; + sqlite3.oo1.DB.dbCtorHelper.call(this, opt); + }; + opfsUtil.OpfsDb.prototype = Object.create(sqlite3.oo1.DB.prototype); + sqlite3.oo1.DB.dbCtorHelper.setVfsPostOpenSql( + opfsVfs.pointer, + [ + /* Truncate journal mode is faster than delete or wal for + this vfs, per speedtest1. */ + "pragma journal_mode=truncate;" + /* + This vfs benefits hugely from cache on moderate/large + speedtest1 --size 50 and --size 100 workloads. We currently + rely on setting a non-default cache size when building + sqlite3.wasm. If that policy changes, the cache can + be set here. + */ + //"pragma cache_size=-8388608;" + ].join('') + ); + } + + /** + Potential TODOs: + + - Expose one or both of the Worker objects via opfsUtil and + publish an interface for proxying the higher-level OPFS + features like getting a directory listing. + */ + const sanityCheck = function(){ + const scope = wasm.scopedAllocPush(); + const sq3File = new sqlite3_file(); + try{ + const fid = sq3File.pointer; + const openFlags = capi.SQLITE_OPEN_CREATE + | capi.SQLITE_OPEN_READWRITE + //| capi.SQLITE_OPEN_DELETEONCLOSE + | capi.SQLITE_OPEN_MAIN_DB; + const pOut = wasm.scopedAlloc(8); + const dbFile = "/sanity/check/file"+randomFilename(8); + const zDbFile = wasm.scopedAllocCString(dbFile); + let rc; + state.s11n.serialize("This is ä string."); + rc = state.s11n.deserialize(); + log("deserialize() says:",rc); + if("This is ä string."!==rc[0]) toss("String d13n error."); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + log("xAccess(",dbFile,") exists ?=",rc); + rc = vfsSyncWrappers.xOpen(opfsVfs.pointer, zDbFile, + fid, openFlags, pOut); + log("open rc =",rc,"state.sabOPView[xOpen] =", + state.sabOPView[state.opIds.xOpen]); + if(0!==rc){ + error("open failed with code",rc); + return; + } + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + if(!rc) toss("xAccess() failed to detect file."); + rc = ioSyncWrappers.xSync(sq3File.pointer, 0); + if(rc) toss('sync failed w/ rc',rc); + rc = ioSyncWrappers.xTruncate(sq3File.pointer, 1024); + if(rc) toss('truncate failed w/ rc',rc); + wasm.setMemValue(pOut,0,'i64'); + rc = ioSyncWrappers.xFileSize(sq3File.pointer, pOut); + if(rc) toss('xFileSize failed w/ rc',rc); + log("xFileSize says:",wasm.getMemValue(pOut, 'i64')); + rc = ioSyncWrappers.xWrite(sq3File.pointer, zDbFile, 10, 1); + if(rc) toss("xWrite() failed!"); + const readBuf = wasm.scopedAlloc(16); + rc = ioSyncWrappers.xRead(sq3File.pointer, readBuf, 6, 2); + wasm.setMemValue(readBuf+6,0); + let jRead = wasm.cstringToJs(readBuf); + log("xRead() got:",jRead); + if("sanity"!==jRead) toss("Unexpected xRead() value."); + if(vfsSyncWrappers.xSleep){ + log("xSleep()ing before close()ing..."); + vfsSyncWrappers.xSleep(opfsVfs.pointer,2000); + log("waking up from xSleep()"); + } + rc = ioSyncWrappers.xClose(fid); + log("xClose rc =",rc,"sabOPView =",state.sabOPView); + log("Deleting file:",dbFile); + vfsSyncWrappers.xDelete(opfsVfs.pointer, zDbFile, 0x1234); + vfsSyncWrappers.xAccess(opfsVfs.pointer, zDbFile, 0, pOut); + rc = wasm.getMemValue(pOut,'i32'); + if(rc) toss("Expecting 0 from xAccess(",dbFile,") after xDelete()."); + warn("End of OPFS sanity checks."); + }finally{ + sq3File.dispose(); + wasm.scopedAllocPop(scope); + } + }/*sanityCheck()*/; + + W.onmessage = function({data}){ + //log("Worker.onmessage:",data); + switch(data.type){ + case 'opfs-async-loaded': + /*Arrives as soon as the asyc proxy finishes loading. + Pass our config and shared state on to the async worker.*/ + W.postMessage({type: 'opfs-async-init',args: state}); + break; + case 'opfs-async-inited':{ + /*Indicates that the async partner has received the 'init' + and has finished initializing, so the real work can + begin...*/ + try { + const rc = capi.sqlite3_vfs_register(opfsVfs.pointer, 0); + if(rc){ + toss("sqlite3_vfs_register(OPFS) failed with rc",rc); + } + if(opfsVfs.pointer !== capi.sqlite3_vfs_find("opfs")){ + toss("BUG: sqlite3_vfs_find() failed for just-installed OPFS VFS"); + } + capi.sqlite3_vfs_register.addReference(opfsVfs, opfsIoMethods); + state.sabOPView = new Int32Array(state.sabOP); + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + initS11n(); + if(options.sanityChecks){ + warn("Running sanity checks because of opfs-sanity-check URL arg..."); + sanityCheck(); + } + navigator.storage.getDirectory().then((d)=>{ + W.onerror = W._originalOnError; + delete W._originalOnError; + sqlite3.opfs = opfsUtil; + opfsUtil.rootDirectory = d; + log("End of OPFS sqlite3_vfs setup.", opfsVfs); + promiseResolve(sqlite3); + }); + }catch(e){ + error(e); + promiseReject(e); + } + break; + } + default: + promiseReject(e); + error("Unexpected message from the async worker:",data); + break; + }/*switch(data.type)*/ + }/*W.onmessage()*/; + })/*thePromise*/; + return thePromise; +}/*installOpfsVfs()*/; +installOpfsVfs.defaultProxyUri = + "sqlite3-opfs-async-proxy.js"; +self.sqlite3ApiBootstrap.initializersAsync.push(async (sqlite3)=>{ + if(sqlite3.scriptInfo && !sqlite3.scriptInfo.isWorker){ + return; + } + try{ + let proxyJs = installOpfsVfs.defaultProxyUri; + if(sqlite3.scriptInfo.sqlite3Dir){ + installOpfsVfs.defaultProxyUri = + sqlite3.scriptInfo.sqlite3Dir + proxyJs; + //console.warn("installOpfsVfs.defaultProxyUri =",installOpfsVfs.defaultProxyUri); + } + return installOpfsVfs().catch((e)=>{ + console.warn("Ignoring inability to install OPFS sqlite3_vfs:",e.message); + }); + }catch(e){ + console.error("installOpfsVfs() exception:",e); + throw e; + } +}); +}/*sqlite3ApiBootstrap.initializers.push()*/); diff --git a/ext/wasm/api/sqlite3-api-prologue.js b/ext/wasm/api/sqlite3-api-prologue.js new file mode 100644 index 0000000..fed1c56 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-prologue.js @@ -0,0 +1,1602 @@ +/* + 2022-05-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file is intended to be combined at build-time with other + related code, most notably a header and footer which wraps this whole + file into an Emscripten Module.postRun() handler which has a parameter + named "Module" (the Emscripten Module object). The exact requirements, + conventions, and build process are very much under construction and + will be (re)documented once they've stopped fluctuating so much. + + Project home page: https://sqlite.org + + Documentation home page: https://sqlite.org/wasm + + Specific goals of this subproject: + + - Except where noted in the non-goals, provide a more-or-less + feature-complete wrapper to the sqlite3 C API, insofar as WASM + feature parity with C allows for. In fact, provide at least 4 + APIs... + + 1) 1-to-1 bindings as exported from WASM, with no automatic + type conversions between JS and C. + + 2) A binding of (1) which provides certain JS/C type conversions + to greatly simplify its use. + + 3) A higher-level API, more akin to sql.js and node.js-style + implementations. This one speaks directly to the low-level + API. This API must be used from the same thread as the + low-level API. + + 4) A second higher-level API which speaks to the previous APIs via + worker messages. This one is intended for use in the main + thread, with the lower-level APIs installed in a Worker thread, + and talking to them via Worker messages. Because Workers are + asynchronouns and have only a single message channel, some + acrobatics are needed here to feed async work results back to + the client (as we cannot simply pass around callbacks between + the main and Worker threads). + + - Insofar as possible, support client-side storage using JS + filesystem APIs. As of this writing, such things are still very + much under development. + + Specific non-goals of this project: + + - As WASM is a web-centric technology and UTF-8 is the King of + Encodings in that realm, there are no currently plans to support + the UTF16-related sqlite3 APIs. They would add a complication to + the bindings for no appreciable benefit. Though web-related + implementation details take priority, and the JavaScript + components of the API specifically focus on browser clients, the + lower-level WASM module "should" work in non-web WASM + environments. + + - Supporting old or niche-market platforms. WASM is built for a + modern web and requires modern platforms. + + - Though scalar User-Defined Functions (UDFs) may be created in + JavaScript, there are currently no plans to add support for + aggregate and window functions. + + Attribution: + + This project is endebted to the work of sql.js: + + https://github.com/sql-js/sql.js + + sql.js was an essential stepping stone in this code's development as + it demonstrated how to handle some of the WASM-related voodoo (like + handling pointers-to-pointers and adding JS implementations of + C-bound callback functions). These APIs have a considerably + different shape than sql.js's, however. +*/ + +/** + sqlite3ApiBootstrap() is the only global symbol persistently + exposed by this API. It is intended to be called one time at the + end of the API amalgamation process, passed configuration details + for the current environment, and then optionally be removed from + the global object using `delete self.sqlite3ApiBootstrap`. + + This function expects a configuration object, intended to abstract + away details specific to any given WASM environment, primarily so + that it can be used without any _direct_ dependency on + Emscripten. (Note the default values for the config object!) The + config object is only honored the first time this is + called. Subsequent calls ignore the argument and return the same + (configured) object which gets initialized by the first call. + This function will throw if any of the required config options are + missing. + + The config object properties include: + + - `exports`[^1]: the "exports" object for the current WASM + environment. In an Emscripten-based build, this should be set to + `Module['asm']`. + + - `memory`[^1]: optional WebAssembly.Memory object, defaulting to + `exports.memory`. In Emscripten environments this should be set + to `Module.wasmMemory` if the build uses `-sIMPORT_MEMORY`, or be + left undefined/falsy to default to `exports.memory` when using + WASM-exported memory. + + - `bigIntEnabled`: true if BigInt support is enabled. Defaults to + true if `self.BigInt64Array` is available, else false. Some APIs + will throw exceptions if called without BigInt support, as BigInt + is required for marshalling C-side int64 into and out of JS. + + - `allocExportName`: the name of the function, in `exports`, of the + `malloc(3)`-compatible routine for the WASM environment. Defaults + to `"malloc"`. + + - `deallocExportName`: the name of the function, in `exports`, of + the `free(3)`-compatible routine for the WASM + environment. Defaults to `"free"`. + + - `wasmfsOpfsDir`[^1]: if the environment supports persistent + storage, this directory names the "mount point" for that + directory. It must be prefixed by `/` and may contain only a + single directory-name part. Using the root directory name is not + supported by any current persistent backend. This setting is + only used in WASMFS-enabled builds. + + + [^1] = This property may optionally be a function, in which case this + function re-assigns it to the value returned from that function, + enabling delayed evaluation. + +*/ +'use strict'; +self.sqlite3ApiBootstrap = function sqlite3ApiBootstrap( + apiConfig = (self.sqlite3ApiConfig || sqlite3ApiBootstrap.defaultConfig) +){ + if(sqlite3ApiBootstrap.sqlite3){ /* already initalized */ + console.warn("sqlite3ApiBootstrap() called multiple times.", + "Config and external initializers are ignored on calls after the first."); + return sqlite3ApiBootstrap.sqlite3; + } + const config = Object.assign(Object.create(null),{ + exports: undefined, + memory: undefined, + bigIntEnabled: (()=>{ + if('undefined'!==typeof Module){ + /* Emscripten module will contain HEAPU64 when built with + -sWASM_BIGINT=1, else it will not. */ + return !!Module.HEAPU64; + } + return !!self.BigInt64Array; + })(), + allocExportName: 'malloc', + deallocExportName: 'free', + wasmfsOpfsDir: '/opfs' + }, apiConfig || {}); + + [ + // If any of these config options are functions, replace them with + // the result of calling that function... + 'exports', 'memory', 'wasmfsOpfsDir' + ].forEach((k)=>{ + if('function' === typeof config[k]){ + config[k] = config[k](); + } + }); + + /** + The main sqlite3 binding API gets installed into this object, + mimicking the C API as closely as we can. The numerous members + names with prefixes 'sqlite3_' and 'SQLITE_' behave, insofar as + possible, identically to the C-native counterparts, as documented at: + + https://www.sqlite.org/c3ref/intro.html + + A very few exceptions require an additional level of proxy + function or may otherwise require special attention in the WASM + environment, and all such cases are documented somewhere below + in this file or in sqlite3-api-glue.js. capi members which are + not documented are installed as 1-to-1 proxies for their + C-side counterparts. + */ + const capi = Object.create(null); + /** + Holds state which are specific to the WASM-related + infrastructure and glue code. It is not expected that client + code will normally need these, but they're exposed here in case + it does. These APIs are _not_ to be considered an + official/stable part of the sqlite3 WASM API. They may change + as the developers' experience suggests appropriate changes. + + Note that a number of members of this object are injected + dynamically after the api object is fully constructed, so + not all are documented in this file. + */ + const wasm = Object.create(null); + + /** Internal helper for SQLite3Error ctor. */ + const __rcStr = (rc)=>{ + return (capi.sqlite3_js_rc_str && capi.sqlite3_js_rc_str(rc)) + || ("Unknown result code #"+rc); + }; + + /** Internal helper for SQLite3Error ctor. */ + const __isInt = (n)=>'number'===typeof n && n===(n | 0); + + /** + An Error subclass specifically for reporting DB-level errors and + enabling clients to unambiguously identify such exceptions. + The C-level APIs never throw, but some of the higher-level + C-style APIs do and the object-oriented APIs use exceptions + exclusively to report errors. + */ + class SQLite3Error extends Error { + /** + Constructs this object with a message depending on its arguments: + + - If it's passed only a single integer argument, it is assumed + to be an sqlite3 C API result code. The message becomes the + result of sqlite3.capi.sqlite3_js_rc_str() or (if that returns + falsy) a synthesized string which contains that integer. + + - If passed 2 arguments and the 2nd is a object, it bevaves + like the Error(string,object) constructor except that the first + argument is subject to the is-integer semantics from the + previous point. + + - Else all arguments are concatenated with a space between each + one, using args.join(' '), to create the error message. + */ + constructor(...args){ + if(1===args.length && __isInt(args[0])){ + super(__rcStr(args[0])); + }else if(2===args.length && 'object'===typeof args){ + if(__isInt(args[0])) super(__rcStr(args[0]), args[1]); + else super(...args); + }else{ + super(args.join(' ')); + } + this.name = 'SQLite3Error'; + } + }; + + /** + Functionally equivalent to the SQLite3Error constructor but may + be used as part of an expression, e.g.: + + ``` + return someFunction(x) || SQLite3Error.toss(...); + ``` + */ + SQLite3Error.toss = (...args)=>{ + throw new SQLite3Error(...args); + }; + const toss3 = SQLite3Error.toss; + + if(config.wasmfsOpfsDir && !/^\/[^/]+$/.test(config.wasmfsOpfsDir)){ + toss3("config.wasmfsOpfsDir must be falsy or in the form '/dir-name'."); + } + + /** + Returns true if n is a 32-bit (signed) integer, else + false. This is used for determining when we need to switch to + double-type DB operations for integer values in order to keep + more precision. + */ + const isInt32 = (n)=>{ + return ('bigint'!==typeof n /*TypeError: can't convert BigInt to number*/) + && !!(n===(n|0) && n<=2147483647 && n>=-2147483648); + }; + /** + Returns true if the given BigInt value is small enough to fit + into an int64 value, else false. + */ + const bigIntFits64 = function f(b){ + if(!f._max){ + f._max = BigInt("0x7fffffffffffffff"); + f._min = ~f._max; + } + return b >= f._min && b <= f._max; + }; + + /** + Returns true if the given BigInt value is small enough to fit + into an int32, else false. + */ + const bigIntFits32 = (b)=>(b >= (-0x7fffffffn - 1n) && b <= 0x7fffffffn); + + /** + Returns true if the given BigInt value is small enough to fit + into a double value without loss of precision, else false. + */ + const bigIntFitsDouble = function f(b){ + if(!f._min){ + f._min = Number.MIN_SAFE_INTEGER; + f._max = Number.MAX_SAFE_INTEGER; + } + return b >= f._min && b <= f._max; + }; + + /** Returns v if v appears to be a TypedArray, else false. */ + const isTypedArray = (v)=>{ + return (v && v.constructor && isInt32(v.constructor.BYTES_PER_ELEMENT)) ? v : false; + }; + + + /** Internal helper to use in operations which need to distinguish + between TypedArrays which are backed by a SharedArrayBuffer + from those which are not. */ + const __SAB = ('undefined'===typeof SharedArrayBuffer) + ? function(){} : SharedArrayBuffer; + /** Returns true if the given TypedArray object is backed by a + SharedArrayBuffer, else false. */ + const isSharedTypedArray = (aTypedArray)=>(aTypedArray.buffer instanceof __SAB); + + /** + Returns either aTypedArray.slice(begin,end) (if + aTypedArray.buffer is a SharedArrayBuffer) or + aTypedArray.subarray(begin,end) (if it's not). + + This distinction is important for APIs which don't like to + work on SABs, e.g. TextDecoder, and possibly for our + own APIs which work on memory ranges which "might" be + modified by other threads while they're working. + */ + const typedArrayPart = (aTypedArray, begin, end)=>{ + return isSharedTypedArray(aTypedArray) + ? aTypedArray.slice(begin, end) + : aTypedArray.subarray(begin, end); + }; + + /** + Returns true if v appears to be one of our bind()-able + TypedArray types: Uint8Array or Int8Array. Support for + TypedArrays with element sizes >1 is TODO. + */ + const isBindableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** + Returns true if v appears to be one of the TypedArray types + which is legal for holding SQL code (as opposed to binary blobs). + + Currently this is the same as isBindableTypedArray() but it + seems likely that we'll eventually want to add Uint32Array + and friends to the isBindableTypedArray() list but not to the + isSQLableTypedArray() list. + */ + const isSQLableTypedArray = (v)=>{ + return v && v.constructor && (1===v.constructor.BYTES_PER_ELEMENT); + }; + + /** Returns true if isBindableTypedArray(v) does, else throws with a message + that v is not a supported TypedArray value. */ + const affirmBindableTypedArray = (v)=>{ + return isBindableTypedArray(v) + || toss3("Value is not of a supported TypedArray type."); + }; + + const utf8Decoder = new TextDecoder('utf-8'); + + /** + Uses TextDecoder to decode the given half-open range of the + given TypedArray to a string. This differs from a simple + call to TextDecoder in that it accounts for whether the + first argument is backed by a SharedArrayBuffer or not, + and can work more efficiently if it's not (TextDecoder + refuses to act upon an SAB). + */ + const typedArrayToString = function(typedArray, begin, end){ + return utf8Decoder.decode(typedArrayPart(typedArray, begin,end)); + }; + + /** + If v is-a Array, its join("") result is returned. If + isSQLableTypedArray(v) is true then typedArrayToString(v) is + returned. If it looks like a WASM pointer, wasm.cstringToJs(v) is + returned. Else v is returned as-is. + */ + const flexibleString = function(v){ + if(isSQLableTypedArray(v)) return typedArrayToString(v); + else if(Array.isArray(v)) return v.join(""); + else if(wasm.isPtr(v)) v = wasm.cstringToJs(v); + return v; + }; + + /** + An Error subclass specifically for reporting Wasm-level malloc() + failure and enabling clients to unambiguously identify such + exceptions. + */ + class WasmAllocError extends Error { + /** + If called with 2 arguments and the 2nd one is an object, it + behaves like the Error constructor, else it concatenates all + arguments together with a single space between each to + construct an error message string. As a special case, if + called with no arguments then it uses a default error + message. + */ + constructor(...args){ + if(2===args.length && 'object'===typeof args){ + super(...args); + }else if(args.length){ + super(args.join(' ')); + }else{ + super("Allocation failed."); + } + this.name = 'WasmAllocError'; + } + }; + /** + Functionally equivalent to the WasmAllocError constructor but may + be used as part of an expression, e.g.: + + ``` + return someAllocatingFunction(x) || WasmAllocError.toss(...); + ``` + */ + WasmAllocError.toss = (...args)=>{ + throw new WasmAllocError(...args); + }; + + Object.assign(capi, { + /** + sqlite3_create_function_v2() differs from its native + counterpart only in the following ways: + + 1) The fourth argument (`eTextRep`) argument must not specify + any encoding other than sqlite3.SQLITE_UTF8. The JS API does not + currently support any other encoding and likely never + will. This function does not replace that argument on its own + because it may contain other flags. + + 2) Any of the four final arguments may be either WASM pointers + (assumed to be function pointers) or JS Functions. In the + latter case, each gets bound to WASM using + sqlite3.capi.wasm.installFunction() and that wrapper is passed + on to the native implementation. + + The semantics of JS functions are: + + xFunc: is passed `(pCtx, ...values)`. Its return value becomes + the new SQL function's result. + + xStep: is passed `(pCtx, ...values)`. Its return value is + ignored. + + xFinal: is passed `(pCtx)`. Its return value becomes the new + aggregate SQL function's result. + + xDestroy: is passed `(void*)`. Its return value is ignored. The + pointer passed to it is the one from the 5th argument to + sqlite3_create_function_v2(). + + Note that: + + - `pCtx` in the above descriptions is a `sqlite3_context*`. At + least 99 times out of a hundred, that initial argument will + be irrelevant for JS UDF bindings, but it needs to be there + so that the cases where it _is_ relevant, in particular with + window and aggregate functions, have full access to the + lower-level sqlite3 APIs. + + - When wrapping JS functions, the remaining arguments are passd + to them as positional arguments, not as an array of + arguments, because that allows callback definitions to be + more JS-idiomatic than C-like. For example `(pCtx,a,b)=>a+b` + is more intuitive and legible than + `(pCtx,args)=>args[0]+args[1]`. For cases where an array of + arguments would be more convenient, the callbacks simply need + to be declared like `(pCtx,...args)=>{...}`, in which case + `args` will be an array. + + - If a JS wrapper throws, it gets translated to + sqlite3_result_error() or sqlite3_result_error_nomem(), + depending on whether the exception is an + sqlite3.WasmAllocError object or not. + + - When passing on WASM function pointers, arguments are _not_ + converted or reformulated. They are passed on as-is in raw + pointer form using their native C signatures. Only JS + functions passed in to this routine, and thus wrapped by this + routine, get automatic conversions of arguments and result + values. The routines which perform those conversions are + exposed for client-side use as + sqlite3_create_function_v2.convertUdfArgs() and + sqlite3_create_function_v2.setUdfResult(). sqlite3_create_function() + and sqlite3_create_window_function() have those same methods. + + For xFunc(), xStep(), and xFinal(): + + - When called from SQL, arguments to the UDF, and its result, + will be converted between JS and SQL with as much fidelity as + is feasible, triggering an exception if a type conversion + cannot be determined. Some freedom is afforded to numeric + conversions due to friction between the JS and C worlds: + integers which are larger than 32 bits may be treated as + doubles or BigInts. + + If any JS-side bound functions throw, those exceptions are + intercepted and converted to database-side errors with the + exception of xDestroy(): any exception from it is ignored, + possibly generating a console.error() message. Destructors + must not throw. + + Once installed, there is currently no way to uninstall the + automatically-converted WASM-bound JS functions from WASM. They + can be uninstalled from the database as documented in the C + API, but this wrapper currently has no infrastructure in place + to also free the WASM-bound JS wrappers, effectively resulting + in a memory leak if the client uninstalls the UDF. Improving that + is a potential TODO, but removing client-installed UDFs is rare + in practice. If this factor is relevant for a given client, + they can create WASM-bound JS functions themselves, hold on to their + pointers, and pass the pointers in to here. Later on, they can + free those pointers (using `wasm.uninstallFunction()` or + equivalent). + + C reference: https://www.sqlite.org/c3ref/create_function.html + + Maintenance reminder: the ability to add new + WASM-accessible functions to the runtime requires that the + WASM build is compiled with emcc's `-sALLOW_TABLE_GROWTH` + flag. + */ + sqlite3_create_function_v2: function( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, xStep, xFinal, xDestroy + ){/*installed later*/}, + /** + Equivalent to passing the same arguments to + sqlite3_create_function_v2(), with 0 as the final argument. + */ + sqlite3_create_function:function( + pDb, funcName, nArg, eTextRep, pApp, + xFunc, xStep, xFinal + ){/*installed later*/}, + /** + The sqlite3_create_window_function() JS wrapper differs from + its native implementation in the exact same way that + sqlite3_create_function_v2() does. The additional function, + xInverse(), is treated identically to xStep() by the wrapping + layer. + */ + sqlite3_create_window_function: function( + pDb, funcName, nArg, eTextRep, pApp, + xStep, xFinal, xValue, xInverse, xDestroy + ){/*installed later*/}, + /** + The sqlite3_prepare_v3() binding handles two different uses + with differing JS/WASM semantics: + + 1) sqlite3_prepare_v3(pDb, sqlString, -1, prepFlags, ppStmt , null) + + 2) sqlite3_prepare_v3(pDb, sqlPointer, sqlByteLen, prepFlags, ppStmt, sqlPointerToPointer) + + Note that the SQL length argument (the 3rd argument) must, for + usage (1), always be negative because it must be a byte length + and that value is expensive to calculate from JS (where only + the character length of strings is readily available). It is + retained in this API's interface for code/documentation + compatibility reasons but is currently _always_ ignored. With + usage (2), the 3rd argument is used as-is but is is still + critical that the C-style input string (2nd argument) be + terminated with a 0 byte. + + In usage (1), the 2nd argument must be of type string, + Uint8Array, or Int8Array (either of which is assumed to + hold SQL). If it is, this function assumes case (1) and + calls the underyling C function with the equivalent of: + + (pDb, sqlAsString, -1, prepFlags, ppStmt, null) + + The `pzTail` argument is ignored in this case because its + result is meaningless when a string-type value is passed + through: the string goes through another level of internal + conversion for WASM's sake and the result pointer would refer + to that transient conversion's memory, not the passed-in + string. + + If the sql argument is not a string, it must be a _pointer_ to + a NUL-terminated string which was allocated in the WASM memory + (e.g. using capi.wasm.alloc() or equivalent). In that case, + the final argument may be 0/null/undefined or must be a pointer + to which the "tail" of the compiled SQL is written, as + documented for the C-side sqlite3_prepare_v3(). In case (2), + the underlying C function is called with the equivalent of: + + (pDb, sqlAsPointer, sqlByteLen, prepFlags, ppStmt, pzTail) + + It returns its result and compiled statement as documented in + the C API. Fetching the output pointers (5th and 6th + parameters) requires using `capi.wasm.getMemValue()` (or + equivalent) and the `pzTail` will point to an address relative to + the `sqlAsPointer` value. + + If passed an invalid 2nd argument type, this function will + return SQLITE_MISUSE and sqlite3_errmsg() will contain a string + describing the problem. + + Side-note: if given an empty string, or one which contains only + comments or an empty SQL expression, 0 is returned but the result + output pointer will be NULL. + */ + sqlite3_prepare_v3: (dbPtr, sql, sqlByteLen, prepFlags, + stmtPtrPtr, strPtrPtr)=>{}/*installed later*/, + + /** + Equivalent to calling sqlite3_prapare_v3() with 0 as its 4th argument. + */ + sqlite3_prepare_v2: (dbPtr, sql, sqlByteLen, + stmtPtrPtr,strPtrPtr)=>{}/*installed later*/, + + /** + This binding enables the callback argument to be a JavaScript. + + If the callback is a function, then for the duration of the + sqlite3_exec() call, it installs a WASM-bound function which + acts as a proxy for the given callback. That proxy will also + perform a conversion of the callback's arguments from + `(char**)` to JS arrays of strings. However, for API + consistency's sake it will still honor the C-level callback + parameter order and will call it like: + + `callback(pVoid, colCount, listOfValues, listOfColNames)` + + If the callback is not a JS function then this binding performs + no translation of the callback, but the sql argument is still + converted to a WASM string for the call using the + "flexible-string" argument converter. + */ + sqlite3_exec: (pDb, sql, callback, pVoid, pErrMsg)=>{}/*installed later*/, + + /** + If passed a single argument which appears to be a byte-oriented + TypedArray (Int8Array or Uint8Array), this function treats that + TypedArray as an output target, fetches `theArray.byteLength` + bytes of randomness, and populates the whole array with it. As + a special case, if the array's length is 0, this function + behaves as if it were passed (0,0). When called this way, it + returns its argument, else it returns the `undefined` value. + + If called with any other arguments, they are passed on as-is + to the C API. Results are undefined if passed any incompatible + values. + */ + sqlite3_randomness: (n, outPtr)=>{/*installed later*/}, + }/*capi*/); + + /** + Various internal-use utilities are added here as needed. They + are bound to an object only so that we have access to them in + the differently-scoped steps of the API bootstrapping + process. At the end of the API setup process, this object gets + removed. These are NOT part of the public API. + */ + const util = { + affirmBindableTypedArray, flexibleString, + bigIntFits32, bigIntFits64, bigIntFitsDouble, + isBindableTypedArray, + isInt32, isSQLableTypedArray, isTypedArray, + typedArrayToString, + isUIThread: ()=>'undefined'===typeof WorkerGlobalScope, + isSharedTypedArray, + typedArrayPart + }; + + Object.assign(wasm, { + /** + Emscripten APIs have a deep-seated assumption that all pointers + are 32 bits. We'll remain optimistic that that won't always be + the case and will use this constant in places where we might + otherwise use a hard-coded 4. + */ + ptrSizeof: config.wasmPtrSizeof || 4, + /** + The WASM IR (Intermediate Representation) value for + pointer-type values. It MUST refer to a value type of the + size described by this.ptrSizeof _or_ it may be any value + which ends in '*', which Emscripten's glue code internally + translates to i32. + */ + ptrIR: config.wasmPtrIR || "i32", + /** + True if BigInt support was enabled via (e.g.) the + Emscripten -sWASM_BIGINT flag, else false. When + enabled, certain 64-bit sqlite3 APIs are enabled which + are not otherwise enabled due to JS/WASM int64 + impedence mismatches. + */ + bigIntEnabled: !!config.bigIntEnabled, + /** + The symbols exported by the WASM environment. + */ + exports: config.exports + || toss3("Missing API config.exports (WASM module exports)."), + + /** + When Emscripten compiles with `-sIMPORT_MEMORY`, it + initalizes the heap and imports it into wasm, as opposed to + the other way around. In this case, the memory is not + available via this.exports.memory. + */ + memory: config.memory || config.exports['memory'] + || toss3("API config object requires a WebAssembly.Memory object", + "in either config.exports.memory (exported)", + "or config.memory (imported)."), + + /** + The API's one single point of access to the WASM-side memory + allocator. Works like malloc(3) (and is likely bound to + malloc()) but throws an WasmAllocError if allocation fails. It is + important that any code which might pass through the sqlite3 C + API NOT throw and must instead return SQLITE_NOMEM (or + equivalent, depending on the context). + + Very few cases in the sqlite3 JS APIs can result in + client-defined functions propagating exceptions via the C-style + API. Most notably, this applies to WASM-bound JS functions + which are created directly by clients and passed on _as WASM + function pointers_ to functions such as + sqlite3_create_function_v2(). Such bindings created + transparently by this API will automatically use wrappers which + catch exceptions and convert them to appropriate error codes. + + For cases where non-throwing allocation is required, use + sqlite3.wasm.alloc.impl(), which is direct binding of the + underlying C-level allocator. + + Design note: this function is not named "malloc" primarily + because Emscripten uses that name and we wanted to avoid any + confusion early on in this code's development, when it still + had close ties to Emscripten's glue code. + */ + alloc: undefined/*installed later*/, + + /** + The API's one single point of access to the WASM-side memory + deallocator. Works like free(3) (and is likely bound to + free()). + + Design note: this function is not named "free" for the same + reason that this.alloc() is not called this.malloc(). + */ + dealloc: undefined/*installed later*/ + + /* Many more wasm-related APIs get installed later on. */ + }/*wasm*/); + + /** + wasm.alloc()'s srcTypedArray.byteLength bytes, + populates them with the values from the source + TypedArray, and returns the pointer to that memory. The + returned pointer must eventually be passed to + wasm.dealloc() to clean it up. + + As a special case, to avoid further special cases where + this is used, if srcTypedArray.byteLength is 0, it + allocates a single byte and sets it to the value + 0. Even in such cases, calls must behave as if the + allocated memory has exactly srcTypedArray.byteLength + bytes. + + ACHTUNG: this currently only works for Uint8Array and + Int8Array types and will throw if srcTypedArray is of + any other type. + */ + wasm.allocFromTypedArray = function(srcTypedArray){ + affirmBindableTypedArray(srcTypedArray); + const pRet = wasm.alloc(srcTypedArray.byteLength || 1); + wasm.heapForSize(srcTypedArray.constructor).set( + srcTypedArray.byteLength ? srcTypedArray : [0], pRet + ); + return pRet; + }; + + const keyAlloc = config.allocExportName || 'malloc', + keyDealloc = config.deallocExportName || 'free'; + for(const key of [keyAlloc, keyDealloc]){ + const f = wasm.exports[key]; + if(!(f instanceof Function)) toss3("Missing required exports[",key,"] function."); + } + + wasm.alloc = function f(n){ + const m = f.impl(n); + if(!m) throw new WasmAllocError("Failed to allocate",n," bytes."); + return m; + }; + wasm.alloc.impl = wasm.exports[keyAlloc]; + wasm.dealloc = wasm.exports[keyDealloc]; + + /** + Reports info about compile-time options using + sqlite_compileoption_get() and sqlite3_compileoption_used(). It + has several distinct uses: + + If optName is an array then it is expected to be a list of + compilation options and this function returns an object + which maps each such option to true or false, indicating + whether or not the given option was included in this + build. That object is returned. + + If optName is an object, its keys are expected to be compilation + options and this function sets each entry to true or false, + indicating whether the compilation option was used or not. That + object is returned. + + If passed no arguments then it returns an object mapping + all known compilation options to their compile-time values, + or boolean true if they are defined with no value. This + result, which is relatively expensive to compute, is cached + and returned for future no-argument calls. + + In all other cases it returns true if the given option was + active when when compiling the sqlite3 module, else false. + + Compile-time option names may optionally include their + "SQLITE_" prefix. When it returns an object of all options, + the prefix is elided. + */ + wasm.compileOptionUsed = function f(optName){ + if(!arguments.length){ + if(f._result) return f._result; + else if(!f._opt){ + f._rx = /^([^=]+)=(.+)/; + f._rxInt = /^-?\d+$/; + f._opt = function(opt, rv){ + const m = f._rx.exec(opt); + rv[0] = (m ? m[1] : opt); + rv[1] = m ? (f._rxInt.test(m[2]) ? +m[2] : m[2]) : true; + }; + } + const rc = {}, ov = [0,0]; + let i = 0, k; + while((k = capi.sqlite3_compileoption_get(i++))){ + f._opt(k,ov); + rc[ov[0]] = ov[1]; + } + return f._result = rc; + }else if(Array.isArray(optName)){ + const rc = {}; + optName.forEach((v)=>{ + rc[v] = capi.sqlite3_compileoption_used(v); + }); + return rc; + }else if('object' === typeof optName){ + Object.keys(optName).forEach((k)=> { + optName[k] = capi.sqlite3_compileoption_used(k); + }); + return optName; + } + return ( + 'string'===typeof optName + ) ? !!capi.sqlite3_compileoption_used(optName) : false; + }/*compileOptionUsed()*/; + + /** + Signatures for the WASM-exported C-side functions. Each entry + is an array with 2+ elements: + + [ "c-side name", + "result type" (wasm.xWrap() syntax), + [arg types in xWrap() syntax] + // ^^^ this needn't strictly be an array: it can be subsequent + // elements instead: [x,y,z] is equivalent to x,y,z + ] + + Note that support for the API-specific data types in the + result/argument type strings gets plugged in at a later phase in + the API initialization process. + */ + wasm.bindingSignatures = [ + // Please keep these sorted by function name! + ["sqlite3_aggregate_context","void*", "sqlite3_context*", "int"], + ["sqlite3_bind_blob","int", "sqlite3_stmt*", "int", "*", "int", "*" + /* TODO: we should arguably write a custom wrapper which knows + how to handle Blob, TypedArrays, and JS strings. */ + ], + ["sqlite3_bind_double","int", "sqlite3_stmt*", "int", "f64"], + ["sqlite3_bind_int","int", "sqlite3_stmt*", "int", "int"], + ["sqlite3_bind_null",undefined, "sqlite3_stmt*", "int"], + ["sqlite3_bind_parameter_count", "int", "sqlite3_stmt*"], + ["sqlite3_bind_parameter_index","int", "sqlite3_stmt*", "string"], + ["sqlite3_bind_text","int", "sqlite3_stmt*", "int", "string", "int", "int" + /* We should arguably create a hand-written binding of + bind_text() which does more flexible text conversion, along + the lines of sqlite3_prepare_v3(). The slightly problematic + part is the final argument (text destructor). */ + ], + ["sqlite3_close_v2", "int", "sqlite3*"], + ["sqlite3_changes", "int", "sqlite3*"], + ["sqlite3_clear_bindings","int", "sqlite3_stmt*"], + ["sqlite3_column_blob","*", "sqlite3_stmt*", "int"], + ["sqlite3_column_bytes","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_count", "int", "sqlite3_stmt*"], + ["sqlite3_column_double","f64", "sqlite3_stmt*", "int"], + ["sqlite3_column_int","int", "sqlite3_stmt*", "int"], + ["sqlite3_column_name","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_text","string", "sqlite3_stmt*", "int"], + ["sqlite3_column_type","int", "sqlite3_stmt*", "int"], + ["sqlite3_compileoption_get", "string", "int"], + ["sqlite3_compileoption_used", "int", "string"], + /* sqlite3_create_function(), sqlite3_create_function_v2(), and + sqlite3_create_window_function() use hand-written bindings to + simplify handling of their function-type arguments. */ + ["sqlite3_data_count", "int", "sqlite3_stmt*"], + ["sqlite3_db_filename", "string", "sqlite3*", "string"], + ["sqlite3_db_handle", "sqlite3*", "sqlite3_stmt*"], + ["sqlite3_db_name", "string", "sqlite3*", "int"], + ["sqlite3_deserialize", "int", "sqlite3*", "string", "*", "i64", "i64", "int"] + /* Careful! Short version: de/serialize() are problematic because they + might use a different allocator than the user for managing the + deserialized block. de/serialize() are ONLY safe to use with + sqlite3_malloc(), sqlite3_free(), and its 64-bit variants. */, + ["sqlite3_errmsg", "string", "sqlite3*"], + ["sqlite3_error_offset", "int", "sqlite3*"], + ["sqlite3_errstr", "string", "int"], + /*["sqlite3_exec", "int", "sqlite3*", "string", "*", "*", "**" + Handled seperately to perform translation of the callback + into a WASM-usable one. ],*/ + ["sqlite3_expanded_sql", "string", "sqlite3_stmt*"], + ["sqlite3_extended_errcode", "int", "sqlite3*"], + ["sqlite3_extended_result_codes", "int", "sqlite3*", "int"], + ["sqlite3_file_control", "int", "sqlite3*", "string", "int", "*"], + ["sqlite3_finalize", "int", "sqlite3_stmt*"], + ["sqlite3_free", undefined,"*"], + ["sqlite3_initialize", undefined], + /*["sqlite3_interrupt", undefined, "sqlite3*" + ^^^ we cannot actually currently support this because JS is + single-threaded and we don't have a portable way to access a DB + from 2 SharedWorkers concurrently. ],*/ + ["sqlite3_libversion", "string"], + ["sqlite3_libversion_number", "int"], + ["sqlite3_malloc", "*","int"], + ["sqlite3_open", "int", "string", "*"], + ["sqlite3_open_v2", "int", "string", "*", "int", "string"], + /* sqlite3_prepare_v2() and sqlite3_prepare_v3() are handled + separately due to us requiring two different sets of semantics + for those, depending on how their SQL argument is provided. */ + /* sqlite3_randomness() uses a hand-written wrapper to extend + the range of supported argument types. */ + ["sqlite3_realloc", "*","*","int"], + ["sqlite3_reset", "int", "sqlite3_stmt*"], + ["sqlite3_result_blob",undefined, "*", "*", "int", "*"], + ["sqlite3_result_double",undefined, "*", "f64"], + ["sqlite3_result_error",undefined, "*", "string", "int"], + ["sqlite3_result_error_code", undefined, "*", "int"], + ["sqlite3_result_error_nomem", undefined, "*"], + ["sqlite3_result_error_toobig", undefined, "*"], + ["sqlite3_result_int",undefined, "*", "int"], + ["sqlite3_result_null",undefined, "*"], + ["sqlite3_result_text",undefined, "*", "string", "int", "*"], + ["sqlite3_serialize","*", "sqlite3*", "string", "*", "int"], + ["sqlite3_shutdown", undefined], + ["sqlite3_sourceid", "string"], + ["sqlite3_sql", "string", "sqlite3_stmt*"], + ["sqlite3_step", "int", "sqlite3_stmt*"], + ["sqlite3_strglob", "int", "string","string"], + ["sqlite3_strlike", "int", "string","string","int"], + ["sqlite3_trace_v2", "int", "sqlite3*", "int", "*", "*"], + ["sqlite3_total_changes", "int", "sqlite3*"], + ["sqlite3_uri_boolean", "int", "string", "string", "int"], + ["sqlite3_uri_key", "string", "string", "int"], + ["sqlite3_uri_parameter", "string", "string", "string"], + ["sqlite3_user_data","void*", "sqlite3_context*"], + ["sqlite3_value_blob", "*", "sqlite3_value*"], + ["sqlite3_value_bytes","int", "sqlite3_value*"], + ["sqlite3_value_double","f64", "sqlite3_value*"], + ["sqlite3_value_int","int", "sqlite3_value*"], + ["sqlite3_value_text", "string", "sqlite3_value*"], + ["sqlite3_value_type", "int", "sqlite3_value*"], + ["sqlite3_vfs_find", "*", "string"], + ["sqlite3_vfs_register", "int", "sqlite3_vfs*", "int"], + ["sqlite3_vfs_unregister", "int", "sqlite3_vfs*"] + ]/*wasm.bindingSignatures*/; + + if(false && wasm.compileOptionUsed('SQLITE_ENABLE_NORMALIZE')){ + /* ^^^ "the problem" is that this is an option feature and the + build-time function-export list does not currently take + optional features into account. */ + wasm.bindingSignatures.push(["sqlite3_normalized_sql", "string", "sqlite3_stmt*"]); + } + + /** + Functions which require BigInt (int64) support are separated from + the others because we need to conditionally bind them or apply + dummy impls, depending on the capabilities of the environment. + */ + wasm.bindingSignatures.int64 = [ + ["sqlite3_bind_int64","int", ["sqlite3_stmt*", "int", "i64"]], + ["sqlite3_changes64","i64", ["sqlite3*"]], + ["sqlite3_column_int64","i64", ["sqlite3_stmt*", "int"]], + ["sqlite3_malloc64", "*","i64"], + ["sqlite3_msize", "i64", "*"], + ["sqlite3_realloc64", "*","*", "i64"], + ["sqlite3_result_int64",undefined, "*", "i64"], + ["sqlite3_total_changes64", "i64", ["sqlite3*"]], + ["sqlite3_uri_int64", "i64", ["string", "string", "i64"]], + ["sqlite3_value_int64","i64", "sqlite3_value*"], + ]; + + /** + Functions which are intended solely for API-internal use by the + WASM components, not client code. These get installed into + sqlite3.wasm. + */ + wasm.bindingSignatures.wasm = [ + ["sqlite3_wasm_db_reset", "int", "sqlite3*"], + ["sqlite3_wasm_db_vfs", "sqlite3_vfs*", "sqlite3*","string"], + ["sqlite3_wasm_vfs_create_file", "int", + "sqlite3_vfs*","string","*", "int"], + ["sqlite3_wasm_vfs_unlink", "int", "sqlite3_vfs*","string"] + ]; + + + /** + sqlite3.wasm.pstack (pseudo-stack) holds a special-case + stack-style allocator intended only for use with _small_ data of + not more than (in total) a few kb in size, managed as if it were + stack-based. + + It has only a single intended usage: + + ``` + const stackPos = pstack.pointer; + try{ + const ptr = pstack.alloc(8); + // ==> pstack.pointer === ptr + const otherPtr = pstack.alloc(8); + // ==> pstack.pointer === otherPtr + ... + }finally{ + pstack.restore(stackPos); + // ==> pstack.pointer === stackPos + } + ``` + + This allocator is much faster than a general-purpose one but is + limited to usage patterns like the one shown above. + + It operates from a static range of memory which lives outside of + space managed by Emscripten's stack-management, so does not + collide with Emscripten-provided stack allocation APIs. The + memory lives in the WASM heap and can be used with routines such + as wasm.setMemValue() and any wasm.heap8u().slice(). + */ + wasm.pstack = Object.assign(Object.create(null),{ + /** + Sets the current pstack position to the given pointer. Results + are undefined if the passed-in value did not come from + this.pointer. + */ + restore: wasm.exports.sqlite3_wasm_pstack_restore, + /** + Attempts to allocate the given number of bytes from the + pstack. On success, it zeroes out a block of memory of the + given size, adjusts the pstack pointer, and returns a pointer + to the memory. On error, returns throws a WasmAllocError. The + memory must eventually be released using restore(). + + This method always adjusts the given value to be a multiple + of 8 bytes because failing to do so can lead to incorrect + results when reading and writing 64-bit values from/to the WASM + heap. Similarly, the returned address is always 8-byte aligned. + */ + alloc: (n)=>{ + return wasm.exports.sqlite3_wasm_pstack_alloc(n) + || WasmAllocError.toss("Could not allocate",n, + "bytes from the pstack."); + }, + /** + alloc()'s n chunks, each sz bytes, as a single memory block and + returns the addresses as an array of n element, each holding + the address of one chunk. + + Throws a WasmAllocError if allocation fails. + + Example: + + ``` + const [p1, p2, p3] = wasm.pstack.allocChunks(3,4); + ``` + */ + allocChunks: (n,sz)=>{ + const mem = wasm.pstack.alloc(n * sz); + const rc = []; + let i = 0, offset = 0; + for(; i < n; offset = (sz * ++i)){ + rc.push(mem + offset); + } + return rc; + }, + /** + A convenience wrapper for allocChunks() which sizes each chunk + as either 8 bytes (safePtrSize is truthy) or wasm.ptrSizeof (if + safePtrSize is falsy). + + How it returns its result differs depending on its first + argument: if it's 1, it returns a single pointer value. If it's + more than 1, it returns the same as allocChunks(). + + When a returned pointers will refer to a 64-bit value, e.g. a + double or int64, and that value must be written or fetched, + e.g. using wasm.setMemValue() or wasm.getMemValue(), it is + important that the pointer in question be aligned to an 8-byte + boundary or else it will not be fetched or written properly and + will corrupt or read neighboring memory. + + However, when all pointers involved point to "small" data, it + is safe to pass a falsy value to save a tiny bit of memory. + */ + allocPtr: (n=1,safePtrSize=true)=>{ + return 1===n + ? wasm.pstack.alloc(safePtrSize ? 8 : wasm.ptrSizeof) + : wasm.pstack.allocChunks(n, safePtrSize ? 8 : wasm.ptrSizeof); + } + })/*wasm.pstack*/; + Object.defineProperties(wasm.pstack, { + /** + sqlite3.wasm.pstack.pointer resolves to the current pstack + position pointer. This value is intended _only_ to be saved + for passing to restore(). Writing to this memory, without + first reserving it via wasm.pstack.alloc() and friends, leads + to undefined results. + */ + pointer: { + configurable: false, iterable: true, writeable: false, + get: wasm.exports.sqlite3_wasm_pstack_ptr + //Whether or not a setter as an alternative to restore() is + //clearer or would just lead to confusion is unclear. + //set: wasm.exports.sqlite3_wasm_pstack_restore + }, + /** + sqlite3.wasm.pstack.quota to the total number of bytes + available in the pstack, including any space which is currently + allocated. This value is a compile-time constant. + */ + quota: { + configurable: false, iterable: true, writeable: false, + get: wasm.exports.sqlite3_wasm_pstack_quota + }, + /** + sqlite3.wasm.pstack.remaining resolves to the amount of space + remaining in the pstack. + */ + remaining: { + configurable: false, iterable: true, writeable: false, + get: wasm.exports.sqlite3_wasm_pstack_remaining + } + })/*wasm.pstack properties*/; + + capi.sqlite3_randomness = (...args)=>{ + if(1===args.length && util.isTypedArray(args[0]) + && 1===args[0].BYTES_PER_ELEMENT){ + const ta = args[0]; + if(0===ta.byteLength){ + wasm.exports.sqlite3_randomness(0,0); + return ta; + } + const stack = wasm.pstack.pointer; + try { + let n = ta.byteLength, offset = 0; + const r = wasm.exports.sqlite3_randomness; + const heap = wasm.heap8u(); + const nAlloc = n < 512 ? n : 512; + const ptr = wasm.pstack.alloc(nAlloc); + do{ + const j = (n>nAlloc ? nAlloc : n); + r(j, ptr); + ta.set(typedArrayPart(heap, ptr, ptr+j), offset); + n -= j; + offset += j; + } while(n > 0); + }catch(e){ + console.error("Highly unexpected (and ignored!) "+ + "exception in sqlite3_randomness():",e); + }finally{ + wasm.pstack.restore(stack); + } + return ta; + } + wasm.exports.sqlite3_randomness(...args); + }; + + /** State for sqlite3_wasmfs_opfs_dir(). */ + let __wasmfsOpfsDir = undefined; + /** + If the wasm environment has a WASMFS/OPFS-backed persistent + storage directory, its path is returned by this function. If it + does not then it returns "" (noting that "" is a falsy value). + + The first time this is called, this function inspects the current + environment to determine whether persistence support is available + and, if it is, enables it (if needed). + + This function currently only recognizes the WASMFS/OPFS storage + combination and its path refers to storage rooted in the + Emscripten-managed virtual filesystem. + */ + capi.sqlite3_wasmfs_opfs_dir = function(){ + if(undefined !== __wasmfsOpfsDir) return __wasmfsOpfsDir; + // If we have no OPFS, there is no persistent dir + const pdir = config.wasmfsOpfsDir; + if(!pdir + || !self.FileSystemHandle + || !self.FileSystemDirectoryHandle + || !self.FileSystemFileHandle){ + return __wasmfsOpfsDir = ""; + } + try{ + if(pdir && 0===wasm.xCallWrapped( + 'sqlite3_wasm_init_wasmfs', 'i32', ['string'], pdir + )){ + return __wasmfsOpfsDir = pdir; + }else{ + return __wasmfsOpfsDir = ""; + } + }catch(e){ + // sqlite3_wasm_init_wasmfs() is not available + return __wasmfsOpfsDir = ""; + } + }; + + /** + Experimental and subject to change or removal. + + Returns true if sqlite3.capi.sqlite3_wasmfs_opfs_dir() is a + non-empty string and the given name starts with (that string + + '/'), else returns false. + */ + capi.sqlite3_wasmfs_filename_is_persistent = function(name){ + const p = capi.sqlite3_wasmfs_opfs_dir(); + return (p && name) ? name.startsWith(p+'/') : false; + }; + + // This bit is highly arguable and is incompatible with the fiddle shell. + if(false && 0===wasm.exports.sqlite3_vfs_find(0)){ + /* Assume that sqlite3_initialize() has not yet been called. + This will be the case in an SQLITE_OS_KV build. */ + wasm.exports.sqlite3_initialize(); + } + + /** + Given an `sqlite3*`, an sqlite3_vfs name, and an optional db name + (defaulting to "main"), returns a truthy value (see below) if + that db uses that VFS, else returns false. If pDb is falsy then + the 3rd argument is ignored and this function returns a truthy + value if the default VFS name matches that of the 2nd + argument. Results are undefined if pDb is truthy but refers to an + invalid pointer. The 3rd argument specifies the database name of + the given database connection to check, defaulting to the main + db. + + The 2nd and 3rd arguments may either be a JS string or a WASM + C-string. If the 2nd argument is a NULL WASM pointer, the default + VFS is assumed. If the 3rd is a NULL WASM pointer, "main" is + assumed. + + The truthy value it returns is a pointer to the `sqlite3_vfs` + object. + + To permit safe use of this function from APIs which may be called + via the C stack (like SQL UDFs), this function does not throw: if + bad arguments cause a conversion error when passing into + wasm-space, false is returned. + */ + capi.sqlite3_js_db_uses_vfs = function(pDb,vfsName,dbName=0){ + try{ + const pK = capi.sqlite3_vfs_find(vfsName); + if(!pK) return false; + else if(!pDb){ + return pK===capi.sqlite3_vfs_find(0) ? pK : false; + }else{ + return pK===capi.sqlite3_js_db_vfs(pDb,dbName) ? pK : false; + } + }catch(e){ + /* Ignore - probably bad args to a wasm-bound function. */ + return false; + } + }; + + /** + Returns an array of the names of all currently-registered sqlite3 + VFSes. + */ + capi.sqlite3_js_vfs_list = function(){ + const rc = []; + let pVfs = capi.sqlite3_vfs_find(0); + while(pVfs){ + const oVfs = new capi.sqlite3_vfs(pVfs); + rc.push(wasm.cstringToJs(oVfs.$zName)); + pVfs = oVfs.$pNext; + oVfs.dispose(); + } + return rc; + }; + + /** + Serializes the given `sqlite3*` pointer to a Uint8Array, as per + sqlite3_serialize(). On success it returns a Uint8Array. On + error it throws with a description of the problem. + */ + capi.sqlite3_js_db_export = function(pDb){ + if(!pDb) toss3('Invalid sqlite3* argument.'); + if(!wasm.bigIntEnabled) toss3('BigInt64 support is not enabled.'); + const stack = wasm.pstack.pointer; + let pOut; + try{ + const pSize = wasm.pstack.alloc(8/*i64*/ + wasm.ptrSizeof); + const ppOut = pSize + 8; + /** + Maintenance reminder, since this cost a full hour of grief + and confusion: if the order of pSize/ppOut are reversed in + that memory block, fetching the value of pSize after the + export reads a garbage size because it's not on an 8-byte + memory boundary! + */ + let rc = wasm.exports.sqlite3_wasm_db_serialize( + pDb, ppOut, pSize, 0 + ); + if(rc){ + toss3("Database serialization failed with code", + sqlite3.capi.sqlite3_js_rc_str(rc)); + } + pOut = wasm.getPtrValue(ppOut); + const nOut = wasm.getMemValue(pSize, 'i64'); + rc = nOut + ? wasm.heap8u().slice(pOut, pOut + Number(nOut)) + : new Uint8Array(); + return rc; + }finally{ + if(pOut) wasm.exports.sqlite3_free(pOut); + wasm.pstack.restore(stack); + } + }; + + /** + Given a `sqlite3*` and a database name (JS string or WASM + C-string pointer, which may be 0), returns a pointer to the + sqlite3_vfs responsible for it. If the given db name is null/0, + or not provided, then "main" is assumed. + */ + capi.sqlite3_js_db_vfs = + (dbPointer, dbName=0)=>wasm.sqlite3_wasm_db_vfs(dbPointer, dbName); + + /** + A thin wrapper around capi.sqlite3_aggregate_context() which + behaves the same except that it throws a WasmAllocError if that + function returns 0. As a special case, if n is falsy it does + _not_ throw if that function returns 0. That special case is + intended for use with xFinal() implementations. + */ + capi.sqlite3_js_aggregate_context = (pCtx, n)=>{ + return capi.sqlite3_aggregate_context(pCtx, n) + || (n ? WasmAllocError.toss("Cannot allocate",n, + "bytes for sqlite3_aggregate_context()") + : 0); + }; + + if( util.isUIThread() ){ + /* Features specific to the main window thread... */ + + /** + Internal helper for sqlite3_js_kvvfs_clear() and friends. + Its argument should be one of ('local','session',""). + */ + const __kvvfsInfo = function(which){ + const rc = Object.create(null); + rc.prefix = 'kvvfs-'+which; + rc.stores = []; + if('session'===which || ""===which) rc.stores.push(self.sessionStorage); + if('local'===which || ""===which) rc.stores.push(self.localStorage); + return rc; + }; + + /** + Clears all storage used by the kvvfs DB backend, deleting any + DB(s) stored there. Its argument must be either 'session', + 'local', or "". In the first two cases, only sessionStorage + resp. localStorage is cleared. If it's an empty string (the + default) then both are cleared. Only storage keys which match + the pattern used by kvvfs are cleared: any other client-side + data are retained. + + This function is only available in the main window thread. + + Returns the number of entries cleared. + */ + capi.sqlite3_js_kvvfs_clear = function(which=""){ + let rc = 0; + const kvinfo = __kvvfsInfo(which); + kvinfo.stores.forEach((s)=>{ + const toRm = [] /* keys to remove */; + let i; + for( i = 0; i < s.length; ++i ){ + const k = s.key(i); + if(k.startsWith(kvinfo.prefix)) toRm.push(k); + } + toRm.forEach((kk)=>s.removeItem(kk)); + rc += toRm.length; + }); + return rc; + }; + + /** + This routine guesses the approximate amount of + window.localStorage and/or window.sessionStorage in use by the + kvvfs database backend. Its argument must be one of + ('session', 'local', ""). In the first two cases, only + sessionStorage resp. localStorage is counted. If it's an empty + string (the default) then both are counted. Only storage keys + which match the pattern used by kvvfs are counted. The returned + value is the "length" value of every matching key and value, + noting that JavaScript stores each character in 2 bytes. + + Note that the returned size is not authoritative from the + perspective of how much data can fit into localStorage and + sessionStorage, as the precise algorithms for determining + those limits are unspecified and may include per-entry + overhead invisible to clients. + */ + capi.sqlite3_js_kvvfs_size = function(which=""){ + let sz = 0; + const kvinfo = __kvvfsInfo(which); + kvinfo.stores.forEach((s)=>{ + let i; + for(i = 0; i < s.length; ++i){ + const k = s.key(i); + if(k.startsWith(kvinfo.prefix)){ + sz += k.length; + sz += s.getItem(k).length; + } + } + }); + return sz * 2 /* because JS uses 2-byte char encoding */; + }; + + }/* main-window-only bits */ + + + /* The remainder of the API will be set up in later steps. */ + const sqlite3 = { + WasmAllocError: WasmAllocError, + SQLite3Error: SQLite3Error, + capi, + util, + wasm, + config, + /** + Holds the version info of the sqlite3 source tree from which + the generated sqlite3-api.js gets built. Note that its version + may well differ from that reported by sqlite3_libversion(), but + that should be considered a source file mismatch, as the JS and + WASM files are intended to be built and distributed together. + + This object is initially a placeholder which gets replaced by a + build-generated object. + */ + version: Object.create(null), + /** + Performs any optional asynchronous library-level initialization + which might be required. This function returns a Promise which + resolves to the sqlite3 namespace object. Any error in the + async init will be fatal to the init as a whole, but init + routines are themselves welcome to install dummy catch() + handlers which are not fatal if their failure should be + considered non-fatal. If called more than once, the second and + subsequent calls are no-ops which return a pre-resolved + Promise. + + Ideally this function is called as part of the Promise chain + which handles the loading and bootstrapping of the API. If not + then it must be called by client-level code, which must not use + the library until the returned promise resolves. + + Bug: if called while a prior call is still resolving, the 2nd + call will resolve prematurely, before the 1st call has finished + resolving. The current build setup precludes that possibility, + so it's only a hypothetical problem if/when this function + ever needs to be invoked by clients. + + In Emscripten-based builds, this function is called + automatically and deleted from this object. + */ + asyncPostInit: async function(){ + let lip = sqlite3ApiBootstrap.initializersAsync; + delete sqlite3ApiBootstrap.initializersAsync; + if(!lip || !lip.length) return Promise.resolve(sqlite3); + // Is it okay to resolve these in parallel or do we need them + // to resolve in order? We currently only have 1, so it + // makes no difference. + lip = lip.map((f)=>{ + const p = (f instanceof Promise) ? f : f(sqlite3); + return p.catch((e)=>{ + console.error("an async sqlite3 initializer failed:",e); + throw e; + }); + }); + //let p = lip.shift(); + //while(lip.length) p = p.then(lip.shift()); + //return p.then(()=>sqlite3); + return Promise.all(lip).then(()=>sqlite3); + }, + /** + scriptInfo ideally gets injected into this object by the + infrastructure which assembles the JS/WASM module. It contains + state which must be collected before sqlite3ApiBootstrap() can + be declared. It is not necessarily available to any + sqlite3ApiBootstrap.initializers but "should" be in place (if + it's added at all) by the time that + sqlite3ApiBootstrap.initializersAsync is processed. + + This state is not part of the public API, only intended for use + with the sqlite3 API bootstrapping and wasm-loading process. + */ + scriptInfo: undefined + }; + try{ + sqlite3ApiBootstrap.initializers.forEach((f)=>{ + f(sqlite3); + }); + }catch(e){ + /* If we don't report this here, it can get completely swallowed + up and disappear into the abyss of Promises and Workers. */ + console.error("sqlite3 bootstrap initializer threw:",e); + throw e; + } + delete sqlite3ApiBootstrap.initializers; + sqlite3ApiBootstrap.sqlite3 = sqlite3; + return sqlite3; +}/*sqlite3ApiBootstrap()*/; +/** + self.sqlite3ApiBootstrap.initializers is an internal detail used by + the various pieces of the sqlite3 API's amalgamation process. It + must not be modified by client code except when plugging such code + into the amalgamation process. + + Each component of the amalgamation is expected to append a function + to this array. When sqlite3ApiBootstrap() is called for the first + time, each such function will be called (in their appended order) + and passed the sqlite3 namespace object, into which they can install + their features (noting that most will also require that certain + features alread have been installed). At the end of that process, + this array is deleted. + + Note that the order of insertion into this array is significant for + some pieces. e.g. sqlite3.capi and sqlite3.wasm cannot be fully + utilized until the whwasmutil.js part is plugged in via + sqlite3-api-glue.js. +*/ +self.sqlite3ApiBootstrap.initializers = []; +/** + self.sqlite3ApiBootstrap.initializersAsync is an internal detail + used by the sqlite3 API's amalgamation process. It must not be + modified by client code except when plugging such code into the + amalgamation process. + + The counterpart of self.sqlite3ApiBootstrap.initializers, + specifically for initializers which are asynchronous. All entries in + this list must be either async functions, non-async functions which + return a Promise, or a Promise. Each function in the list is called + with the sqlite3 ojbect as its only argument. + + The resolved value of any Promise is ignored and rejection will kill + the asyncPostInit() process (at an indeterminate point because all + of them are run asynchronously in parallel). + + This list is not processed until the client calls + sqlite3.asyncPostInit(). This means, for example, that intializers + added to self.sqlite3ApiBootstrap.initializers may push entries to + this list. +*/ +self.sqlite3ApiBootstrap.initializersAsync = []; +/** + Client code may assign sqlite3ApiBootstrap.defaultConfig an + object-type value before calling sqlite3ApiBootstrap() (without + arguments) in order to tell that call to use this object as its + default config value. The intention of this is to provide + downstream clients with a reasonably flexible approach for plugging in + an environment-suitable configuration without having to define a new + global-scope symbol. +*/ +self.sqlite3ApiBootstrap.defaultConfig = Object.create(null); +/** + Placeholder: gets installed by the first call to + self.sqlite3ApiBootstrap(). However, it is recommended that the + caller of sqlite3ApiBootstrap() capture its return value and delete + self.sqlite3ApiBootstrap after calling it. It returns the same + value which will be stored here. +*/ +self.sqlite3ApiBootstrap.sqlite3 = undefined; + diff --git a/ext/wasm/api/sqlite3-api-worker1.js b/ext/wasm/api/sqlite3-api-worker1.js new file mode 100644 index 0000000..62e2bb9 --- /dev/null +++ b/ext/wasm/api/sqlite3-api-worker1.js @@ -0,0 +1,654 @@ +/* + 2022-07-22 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements the initializer for the sqlite3 "Worker API + #1", a very basic DB access API intended to be scripted from a main + window thread via Worker-style messages. Because of limitations in + that type of communication, this API is minimalistic and only + capable of serving relatively basic DB requests (e.g. it cannot + process nested query loops concurrently). + + This file requires that the core C-style sqlite3 API and OO API #1 + have been loaded. +*/ + +/** + sqlite3.initWorker1API() implements a Worker-based wrapper around + SQLite3 OO API #1, colloquially known as "Worker API #1". + + In order to permit this API to be loaded in worker threads without + automatically registering onmessage handlers, initializing the + worker API requires calling initWorker1API(). If this function is + called from a non-worker thread then it throws an exception. It + must only be called once per Worker. + + When initialized, it installs message listeners to receive Worker + messages and then it posts a message in the form: + + ``` + {type:'sqlite3-api', result:'worker1-ready'} + ``` + + to let the client know that it has been initialized. Clients may + optionally depend on this function not returning until + initialization is complete, as the initialization is synchronous. + In some contexts, however, listening for the above message is + a better fit. + + Note that the worker-based interface can be slightly quirky because + of its async nature. In particular, any number of messages may be posted + to the worker before it starts handling any of them. If, e.g., an + "open" operation fails, any subsequent messages will fail. The + Promise-based wrapper for this API (`sqlite3-worker1-promiser.js`) + is more comfortable to use in that regard. + + The documentation for the input and output worker messages for + this API follows... + + ==================================================================== + Common message format... + + Each message posted to the worker has an operation-independent + envelope and operation-dependent arguments: + + ``` + { + type: string, // one of: 'open', 'close', 'exec', 'config-get' + + messageId: OPTIONAL arbitrary value. The worker will copy it as-is + into response messages to assist in client-side dispatching. + + dbId: a db identifier string (returned by 'open') which tells the + operation which database instance to work on. If not provided, the + first-opened db is used. This is an "opaque" value, with no + inherently useful syntax or information. Its value is subject to + change with any given build of this API and cannot be used as a + basis for anything useful beyond its one intended purpose. + + args: ...operation-dependent arguments... + + // the framework may add other properties for testing or debugging + // purposes. + + } + ``` + + Response messages, posted back to the main thread, look like: + + ``` + { + type: string. Same as above except for error responses, which have the type + 'error', + + messageId: same value, if any, provided by the inbound message + + dbId: the id of the db which was operated on, if any, as returned + by the corresponding 'open' operation. + + result: ...operation-dependent result... + + } + ``` + + ==================================================================== + Error responses + + Errors are reported messages in an operation-independent format: + + ``` + { + type: "error", + + messageId: ...as above..., + + dbId: ...as above... + + result: { + + operation: type of the triggering operation: 'open', 'close', ... + + message: ...error message text... + + errorClass: string. The ErrorClass.name property from the thrown exception. + + input: the message object which triggered the error. + + stack: _if available_, a stack trace array. + + } + + } + ``` + + + ==================================================================== + "config-get" + + This operation fetches the serializable parts of the sqlite3 API + configuration. + + Message format: + + ``` + { + type: "config-get", + messageId: ...as above..., + args: currently ignored and may be elided. + } + ``` + + Response: + + ``` + { + type: "config-get", + messageId: ...as above..., + result: { + + version: sqlite3.version object + + bigIntEnabled: bool. True if BigInt support is enabled. + + wasmfsOpfsDir: path prefix, if any, _intended_ for use with + WASMFS OPFS persistent storage. + + wasmfsOpfsEnabled: true if persistent storage is enabled in the + current environment. Only files stored under wasmfsOpfsDir + will persist using that mechanism, however. It is legal to use + the non-WASMFS OPFS VFS to open a database via a URI-style + db filename. + + vfsList: result of sqlite3.capi.sqlite3_js_vfs_list() + } + } + ``` + + + ==================================================================== + "open" a database + + Message format: + + ``` + { + type: "open", + messageId: ...as above..., + args:{ + + filename [=":memory:" or "" (unspecified)]: the db filename. + See the sqlite3.oo1.DB constructor for peculiarities and + transformations, + + vfs: sqlite3_vfs name. Ignored if filename is ":memory:" or "". + This may change how the given filename is resolved. + } + } + ``` + + Response: + + ``` + { + type: "open", + messageId: ...as above..., + result: { + filename: db filename, possibly differing from the input. + + dbId: an opaque ID value which must be passed in the message + envelope to other calls in this API to tell them which db to + use. If it is not provided to future calls, they will default to + operating on the least-recently-opened db. This property is, for + API consistency's sake, also part of the containing message + envelope. Only the `open` operation includes it in the `result` + property. + + persistent: true if the given filename resides in the + known-persistent storage, else false. + + vfs: name of the VFS the "main" db is using. + } + } + ``` + + ==================================================================== + "close" a database + + Message format: + + ``` + { + type: "close", + messageId: ...as above... + dbId: ...as above... + args: OPTIONAL {unlink: boolean} + } + ``` + + If the `dbId` does not refer to an opened ID, this is a no-op. If + the `args` object contains a truthy `unlink` value then the database + will be unlinked (deleted) after closing it. The inability to close a + db (because it's not opened) or delete its file does not trigger an + error. + + Response: + + ``` + { + type: "close", + messageId: ...as above..., + result: { + + filename: filename of closed db, or undefined if no db was closed + + } + } + ``` + + ==================================================================== + "exec" SQL + + All SQL execution is processed through the exec operation. It offers + most of the features of the oo1.DB.exec() method, with a few limitations + imposed by the state having to cross thread boundaries. + + Message format: + + ``` + { + type: "exec", + messageId: ...as above... + dbId: ...as above... + args: string (SQL) or {... see below ...} + } + ``` + + Response: + + ``` + { + type: "exec", + messageId: ...as above..., + dbId: ...as above... + result: { + input arguments, possibly modified. See below. + } + } + ``` + + The arguments are in the same form accepted by oo1.DB.exec(), with + the exceptions noted below. + + A function-type args.callback property cannot cross + the window/Worker boundary, so is not useful here. If + args.callback is a string then it is assumed to be a + message type key, in which case a callback function will be + applied which posts each row result via: + + postMessage({type: thatKeyType, + rowNumber: 1-based-#, + row: theRow, + columnNames: anArray + }) + + And, at the end of the result set (whether or not any result rows + were produced), it will post an identical message with + (row=undefined, rowNumber=null) to alert the caller than the result + set is completed. Note that a row value of `null` is a legal row + result for certain arg.rowMode values. + + (Design note: we don't use (row=undefined, rowNumber=undefined) to + indicate end-of-results because fetching those would be + indistinguishable from fetching from an empty object unless the + client used hasOwnProperty() (or similar) to distinguish "missing + property" from "property with the undefined value". Similarly, + `null` is a legal value for `row` in some case , whereas the db + layer won't emit a result value of `undefined`.) + + The callback proxy must not recurse into this interface. An exec() + call will tie up the Worker thread, causing any recursion attempt + to wait until the first exec() is completed. + + The response is the input options object (or a synthesized one if + passed only a string), noting that options.resultRows and + options.columnNames may be populated by the call to db.exec(). + +*/ +self.sqlite3ApiBootstrap.initializers.push(function(sqlite3){ +sqlite3.initWorker1API = function(){ + 'use strict'; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if('function' !== typeof importScripts){ + toss("initWorker1API() must be run from a Worker thread."); + } + const self = this.self; + const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object."); + const DB = sqlite3.oo1.DB; + + /** + Returns the app-wide unique ID for the given db, creating one if + needed. + */ + const getDbId = function(db){ + let id = wState.idMap.get(db); + if(id) return id; + id = 'db#'+(++wState.idSeq)+'@'+db.pointer; + /** ^^^ can't simply use db.pointer b/c closing/opening may re-use + the same address, which could map pending messages to a wrong + instance. */ + wState.idMap.set(db, id); + return id; + }; + + /** + Internal helper for managing Worker-level state. + */ + const wState = { + /** + Each opened DB is added to this.dbList, and the first entry in + that list is the default db. As each db is closed, its entry is + removed from the list. + */ + dbList: [], + /** Sequence number of dbId generation. */ + idSeq: 0, + /** Map of DB instances to dbId. */ + idMap: new WeakMap, + /** Temp holder for "transferable" postMessage() state. */ + xfer: [], + open: function(opt){ + const db = new DB(opt); + this.dbs[getDbId(db)] = db; + if(this.dbList.indexOf(db)<0) this.dbList.push(db); + return db; + }, + close: function(db,alsoUnlink){ + if(db){ + delete this.dbs[getDbId(db)]; + const filename = db.filename; + const pVfs = sqlite3.wasm.sqlite3_wasm_db_vfs(db.pointer, 0); + db.close(); + const ddNdx = this.dbList.indexOf(db); + if(ddNdx>=0) this.dbList.splice(ddNdx, 1); + if(alsoUnlink && filename && pVfs){ + sqlite3.wasm.sqlite3_wasm_vfs_unlink(pVfs, filename); + } + } + }, + /** + Posts the given worker message value. If xferList is provided, + it must be an array, in which case a copy of it passed as + postMessage()'s second argument and xferList.length is set to + 0. + */ + post: function(msg,xferList){ + if(xferList && xferList.length){ + self.postMessage( msg, Array.from(xferList) ); + xferList.length = 0; + }else{ + self.postMessage(msg); + } + }, + /** Map of DB IDs to DBs. */ + dbs: Object.create(null), + /** Fetch the DB for the given id. Throw if require=true and the + id is not valid, else return the db or undefined. */ + getDb: function(id,require=true){ + return this.dbs[id] + || (require ? toss("Unknown (or closed) DB ID:",id) : undefined); + } + }; + + /** Throws if the given db is falsy or not opened, else returns its + argument. */ + const affirmDbOpen = function(db = wState.dbList[0]){ + return (db && db.pointer) ? db : toss("DB is not opened."); + }; + + /** Extract dbId from the given message payload. */ + const getMsgDb = function(msgData,affirmExists=true){ + const db = wState.getDb(msgData.dbId,false) || wState.dbList[0]; + return affirmExists ? affirmDbOpen(db) : db; + }; + + const getDefaultDbId = function(){ + return wState.dbList[0] && getDbId(wState.dbList[0]); + }; + + const guessVfs = function(filename){ + const m = /^file:.+(vfs=(\w+))/.exec(filename); + return sqlite3.capi.sqlite3_vfs_find(m ? m[2] : 0); + }; + + const isSpecialDbFilename = (n)=>{ + return ""===n || ':'===n[0]; + }; + + /** + A level of "organizational abstraction" for the Worker1 + API. Each method in this object must map directly to a Worker1 + message type key. The onmessage() dispatcher attempts to + dispatch all inbound messages to a method of this object, + passing it the event.data part of the inbound event object. All + methods must return a plain Object containing any result + state, which the dispatcher may amend. All methods must throw + on error. + */ + const wMsgHandler = { + open: function(ev){ + const oargs = Object.create(null), args = (ev.args || Object.create(null)); + if(args.simulateError){ // undocumented internal testing option + toss("Throwing because of simulateError flag."); + } + const rc = Object.create(null); + const pDir = sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + let byteArray, pVfs; + oargs.vfs = args.vfs; + if(isSpecialDbFilename(args.filename)){ + oargs.filename = args.filename || ""; + }else{ + oargs.filename = args.filename; + byteArray = args.byteArray; + if(byteArray) pVfs = guessVfs(args.filename); + } + if(pVfs){ + /* 2022-11-02: this feature is as-yet untested except that + sqlite3_wasm_vfs_create_file() has been tested from the + browser dev console. */ + let pMem; + try{ + pMem = sqlite3.wasm.allocFromTypedArray(byteArray); + const rc = sqlite3.wasm.sqlite3_wasm_vfs_create_file( + pVfs, oargs.filename, pMem, byteArray.byteLength + ); + if(rc) sqlite3.SQLite3Error.toss(rc); + }catch(e){ + throw new sqlite3.SQLite3Error( + e.name+' creating '+args.filename+": "+e.message, { + cause: e + } + ); + }finally{ + if(pMem) sqlite3.wasm.dealloc(pMem); + } + } + const db = wState.open(oargs); + rc.filename = db.filename; + rc.persistent = (!!pDir && db.filename.startsWith(pDir+'/')) + || !!sqlite3.capi.sqlite3_js_db_uses_vfs(db.pointer, "opfs"); + rc.dbId = getDbId(db); + rc.vfs = db.dbVfsName(); + return rc; + }, + + close: function(ev){ + const db = getMsgDb(ev,false); + const response = { + filename: db && db.filename + }; + if(db){ + const doUnlink = ((ev.args && 'object'===typeof ev.args) + ? !!ev.args.unlink : false); + wState.close(db, doUnlink); + } + return response; + }, + + exec: function(ev){ + const rc = ( + 'string'===typeof ev.args + ) ? {sql: ev.args} : (ev.args || Object.create(null)); + if('stmt'===rc.rowMode){ + toss("Invalid rowMode for 'exec': stmt mode", + "does not work in the Worker API."); + }else if(!rc.sql){ + toss("'exec' requires input SQL."); + } + const db = getMsgDb(ev); + if(rc.callback || Array.isArray(rc.resultRows)){ + // Part of a copy-avoidance optimization for blobs + db._blobXfer = wState.xfer; + } + const theCallback = rc.callback; + let rowNumber = 0; + const hadColNames = !!rc.columnNames; + if('string' === typeof theCallback){ + if(!hadColNames) rc.columnNames = []; + /* Treat this as a worker message type and post each + row as a message of that type. */ + rc.callback = function(row,stmt){ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: ++rowNumber, + row: row + }, wState.xfer); + } + } + try { + db.exec(rc); + if(rc.callback instanceof Function){ + rc.callback = theCallback; + /* Post a sentinel message to tell the client that the end + of the result set has been reached (possibly with zero + rows). */ + wState.post({ + type: theCallback, + columnNames: rc.columnNames, + rowNumber: null /*null to distinguish from "property not set"*/, + row: undefined /*undefined because null is a legal row value + for some rowType values, but undefined is not*/ + }); + } + }finally{ + delete db._blobXfer; + if(rc.callback) rc.callback = theCallback; + } + return rc; + }/*exec()*/, + + 'config-get': function(){ + const rc = Object.create(null), src = sqlite3.config; + [ + 'wasmfsOpfsDir', 'bigIntEnabled' + ].forEach(function(k){ + if(Object.getOwnPropertyDescriptor(src, k)) rc[k] = src[k]; + }); + rc.wasmfsOpfsEnabled = !!sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + rc.version = sqlite3.version; + rc.vfsList = sqlite3.capi.sqlite3_js_vfs_list(); + rc.opfsEnabled = !!sqlite3.opfs; + return rc; + }, + + /** + Exports the database to a byte array, as per + sqlite3_serialize(). Response is an object: + + { + byteArray: Uint8Array (db file contents), + filename: the current db filename, + mimetype: 'application/x-sqlite3' + } + */ + export: function(ev){ + const db = getMsgDb(ev); + const response = { + byteArray: sqlite3.capi.sqlite3_js_db_export(db.pointer), + filename: db.filename, + mimetype: 'application/x-sqlite3' + }; + wState.xfer.push(response.byteArray.buffer); + return response; + }/*export()*/, + + toss: function(ev){ + toss("Testing worker exception"); + }, + + 'opfs-tree': async function(ev){ + if(!sqlite3.opfs) toss("OPFS support is unavailable."); + const response = await sqlite3.opfs.treeList(); + return response; + } + }/*wMsgHandler*/; + + self.onmessage = async function(ev){ + ev = ev.data; + let result, dbId = ev.dbId, evType = ev.type; + const arrivalTime = performance.now(); + try { + if(wMsgHandler.hasOwnProperty(evType) && + wMsgHandler[evType] instanceof Function){ + result = await wMsgHandler[evType](ev); + }else{ + toss("Unknown db worker message type:",ev.type); + } + }catch(err){ + evType = 'error'; + result = { + operation: ev.type, + message: err.message, + errorClass: err.name, + input: ev + }; + if(err.stack){ + result.stack = ('string'===typeof err.stack) + ? err.stack.split(/\n\s*/) : err.stack; + } + if(0) console.warn("Worker is propagating an exception to main thread.", + "Reporting it _here_ for the stack trace:",err,result); + } + if(!dbId){ + dbId = result.dbId/*from 'open' cmd*/ + || getDefaultDbId(); + } + // Timing info is primarily for use in testing this API. It's not part of + // the public API. arrivalTime = when the worker got the message. + wState.post({ + type: evType, + dbId: dbId, + messageId: ev.messageId, + workerReceivedTime: arrivalTime, + workerRespondTime: performance.now(), + departureTime: ev.departureTime, + // TODO: move the timing bits into... + //timing:{ + // departure: ev.departureTime, + // workerReceived: arrivalTime, + // workerResponse: performance.now(); + //}, + result: result + }, wState.xfer); + }; + self.postMessage({type:'sqlite3-api',result:'worker1-ready'}); +}.bind({self, sqlite3}); +}); diff --git a/ext/wasm/api/sqlite3-license-version-header.js b/ext/wasm/api/sqlite3-license-version-header.js new file mode 100644 index 0000000..f8b3edd --- /dev/null +++ b/ext/wasm/api/sqlite3-license-version-header.js @@ -0,0 +1,25 @@ +/* +** LICENSE for the sqlite3 WebAssembly/JavaScript APIs. +** +** This bundle (typically released as sqlite3.js or sqlite3-wasmfs.js) +** is an amalgamation of JavaScript source code from two projects: +** +** 1) https://emscripten.org: the Emscripten "glue code" is covered by +** the terms of the MIT license and University of Illinois/NCSA +** Open Source License, as described at: +** +** https://emscripten.org/docs/introducing_emscripten/emscripten_license.html +** +** 2) https://sqlite.org: all code and documentation labeled as being +** from this source are released under the same terms as the sqlite3 +** C library: +** +** 2022-10-16 +** +** The author disclaims copyright to this source code. In place of a +** legal notice, here is a blessing: +** +** * May you do good and not evil. +** * May you find forgiveness for yourself and forgive others. +** * May you share freely, never taking more than you give. +*/ diff --git a/ext/wasm/api/sqlite3-opfs-async-proxy.js b/ext/wasm/api/sqlite3-opfs-async-proxy.js new file mode 100644 index 0000000..e465748 --- /dev/null +++ b/ext/wasm/api/sqlite3-opfs-async-proxy.js @@ -0,0 +1,830 @@ +/* + 2022-09-16 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + A Worker which manages asynchronous OPFS handles on behalf of a + synchronous API which controls it via a combination of Worker + messages, SharedArrayBuffer, and Atomics. It is the asynchronous + counterpart of the API defined in sqlite3-api-opfs.js. + + Highly indebted to: + + https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/OriginPrivateFileSystemVFS.js + + for demonstrating how to use the OPFS APIs. + + This file is to be loaded as a Worker. It does not have any direct + access to the sqlite3 JS/WASM bits, so any bits which it needs (most + notably SQLITE_xxx integer codes) have to be imported into it via an + initialization process. + + This file represents an implementation detail of a larger piece of + code, and not a public interface. Its details may change at any time + and are not intended to be used by any client-level code. +*/ +"use strict"; +const toss = function(...args){throw new Error(args.join(' '))}; +if(self.window === self){ + toss("This code cannot run from the main thread.", + "Load it as a Worker from a separate Worker."); +}else if(!navigator.storage.getDirectory){ + toss("This API requires navigator.storage.getDirectory."); +} + +/** + Will hold state copied to this object from the syncronous side of + this API. +*/ +const state = Object.create(null); + +/** + verbose: + + 0 = no logging output + 1 = only errors + 2 = warnings and errors + 3 = debug, warnings, and errors +*/ +state.verbose = 2; + +const loggers = { + 0:console.error.bind(console), + 1:console.warn.bind(console), + 2:console.log.bind(console) +}; +const logImpl = (level,...args)=>{ + if(state.verbose>level) loggers[level]("OPFS asyncer:",...args); +}; +const log = (...args)=>logImpl(2, ...args); +const warn = (...args)=>logImpl(1, ...args); +const error = (...args)=>logImpl(0, ...args); +const metrics = Object.create(null); +metrics.reset = ()=>{ + let k; + const r = (m)=>(m.count = m.time = m.wait = 0); + for(k in state.opIds){ + r(metrics[k] = Object.create(null)); + } + let s = metrics.s11n = Object.create(null); + s = s.serialize = Object.create(null); + s.count = s.time = 0; + s = metrics.s11n.deserialize = Object.create(null); + s.count = s.time = 0; +}; +metrics.dump = ()=>{ + let k, n = 0, t = 0, w = 0; + for(k in state.opIds){ + const m = metrics[k]; + n += m.count; + t += m.time; + w += m.wait; + m.avgTime = (m.count && m.time) ? (m.time / m.count) : 0; + } + console.log(self.location.href, + "metrics for",self.location.href,":\n", + metrics, + "\nTotal of",n,"op(s) for",t,"ms", + "approx",w,"ms spent waiting on OPFS APIs."); + console.log("Serialization metrics:",metrics.s11n); +}; + +/** + __openFiles is a map of sqlite3_file pointers (integers) to + metadata related to a given OPFS file handles. The pointers are, in + this side of the interface, opaque file handle IDs provided by the + synchronous part of this constellation. Each value is an object + with a structure demonstrated in the xOpen() impl. +*/ +const __openFiles = Object.create(null); +/** + __autoLocks is a Set of sqlite3_file pointers (integers) which were + "auto-locked". i.e. those for which we obtained a sync access + handle without an explicit xLock() call. Such locks will be + released during db connection idle time, whereas a sync access + handle obtained via xLock(), or subsequently xLock()'d after + auto-acquisition, will not be released until xUnlock() is called. + + Maintenance reminder: if we relinquish auto-locks at the end of the + operation which acquires them, we pay a massive performance + penalty: speedtest1 benchmarks take up to 4x as long. By delaying + the lock release until idle time, the hit is negligible. +*/ +const __autoLocks = new Set(); + +/** + Expects an OPFS file path. It gets resolved, such that ".." + components are properly expanded, and returned. If the 2nd arg is + true, the result is returned as an array of path elements, else an + absolute path string is returned. +*/ +const getResolvedPath = function(filename,splitIt){ + const p = new URL( + filename, 'file://irrelevant' + ).pathname; + return splitIt ? p.split('/').filter((v)=>!!v) : p; +}; + +/** + Takes the absolute path to a filesystem element. Returns an array + of [handleOfContainingDir, filename]. If the 2nd argument is truthy + then each directory element leading to the file is created along + the way. Throws if any creation or resolution fails. +*/ +const getDirForFilename = async function f(absFilename, createDirs = false){ + const path = getResolvedPath(absFilename, true); + const filename = path.pop(); + let dh = state.rootDir; + for(const dirName of path){ + if(dirName){ + dh = await dh.getDirectoryHandle(dirName, {create: !!createDirs}); + } + } + return [dh, filename]; +}; + +/** + An error class specifically for use with getSyncHandle(), the goal + of which is to eventually be able to distinguish unambiguously + between locking-related failures and other types, noting that we + cannot currently do so because createSyncAccessHandle() does not + define its exceptions in the required level of detail. +*/ +class GetSyncHandleError extends Error { + constructor(errorObject, ...msg){ + super(); + this.error = errorObject; + this.message = [ + ...msg, ': Original exception ['+errorObject.name+']:', + errorObject.message + ].join(' '); + this.name = 'GetSyncHandleError'; + } +}; + +/** + Returns the sync access handle associated with the given file + handle object (which must be a valid handle object, as created by + xOpen()), lazily opening it if needed. + + In order to help alleviate cross-tab contention for a dabase, + if an exception is thrown while acquiring the handle, this routine + will wait briefly and try again, up to 3 times. If acquisition + still fails at that point it will give up and propagate the + exception. +*/ +const getSyncHandle = async (fh)=>{ + if(!fh.syncHandle){ + const t = performance.now(); + log("Acquiring sync handle for",fh.filenameAbs); + const maxTries = 4, msBase = 300; + let i = 1, ms = msBase; + for(; true; ms = msBase * ++i){ + try { + //if(i<3) toss("Just testing getSyncHandle() wait-and-retry."); + //TODO? A config option which tells it to throw here + //randomly every now and then, for testing purposes. + fh.syncHandle = await fh.fileHandle.createSyncAccessHandle(); + break; + }catch(e){ + if(i === maxTries){ + throw new GetSyncHandleError( + e, "Error getting sync handle.",maxTries, + "attempts failed.",fh.filenameAbs + ); + } + warn("Error getting sync handle. Waiting",ms, + "ms and trying again.",fh.filenameAbs,e); + Atomics.wait(state.sabOPView, state.opIds.retry, 0, ms); + } + } + log("Got sync handle for",fh.filenameAbs,'in',performance.now() - t,'ms'); + if(!fh.xLock){ + __autoLocks.add(fh.fid); + log("Auto-locked",fh.fid,fh.filenameAbs); + } + } + return fh.syncHandle; +}; + +/** + If the given file-holding object has a sync handle attached to it, + that handle is remove and asynchronously closed. Though it may + sound sensible to continue work as soon as the close() returns + (noting that it's asynchronous), doing so can cause operations + performed soon afterwards, e.g. a call to getSyncHandle() to fail + because they may happen out of order from the close(). OPFS does + not guaranty that the actual order of operations is retained in + such cases. i.e. always "await" on the result of this function. +*/ +const closeSyncHandle = async (fh)=>{ + if(fh.syncHandle){ + log("Closing sync handle for",fh.filenameAbs); + const h = fh.syncHandle; + delete fh.syncHandle; + delete fh.xLock; + __autoLocks.delete(fh.fid); + return h.close(); + } +}; + +/** + A proxy for closeSyncHandle() which is guaranteed to not throw. + + This function is part of a lock/unlock step in functions which + require a sync access handle but may be called without xLock() + having been called first. Such calls need to release that + handle to avoid locking the file for all of time. This is an + _attempt_ at reducing cross-tab contention but it may prove + to be more of a problem than a solution and may need to be + removed. +*/ +const closeSyncHandleNoThrow = async (fh)=>{ + try{await closeSyncHandle(fh)} + catch(e){ + warn("closeSyncHandleNoThrow() ignoring:",e,fh); + } +}; + +/** + Stores the given value at state.sabOPView[state.opIds.rc] and then + Atomics.notify()'s it. +*/ +const storeAndNotify = (opName, value)=>{ + log(opName+"() => notify(",value,")"); + Atomics.store(state.sabOPView, state.opIds.rc, value); + Atomics.notify(state.sabOPView, state.opIds.rc); +}; + +/** + Throws if fh is a file-holding object which is flagged as read-only. +*/ +const affirmNotRO = function(opName,fh){ + if(fh.readOnly) toss(opName+"(): File is read-only: "+fh.filenameAbs); +}; +const affirmLocked = function(opName,fh){ + //if(!fh.syncHandle) toss(opName+"(): File does not have a lock: "+fh.filenameAbs); + /** + Currently a no-op, as speedtest1 triggers xRead() without a + lock (that seems like a bug but it's currently uninvestigated). + This means, however, that some OPFS VFS routines may trigger + acquisition of a lock but never let it go until xUnlock() is + called (which it likely won't be if xLock() was not called). + */ +}; + +/** + We track 2 different timers: the "metrics" timer records how much + time we spend performing work. The "wait" timer records how much + time we spend waiting on the underlying OPFS timer. See the calls + to mTimeStart(), mTimeEnd(), wTimeStart(), and wTimeEnd() + throughout this file to see how they're used. +*/ +const __mTimer = Object.create(null); +__mTimer.op = undefined; +__mTimer.start = undefined; +const mTimeStart = (op)=>{ + __mTimer.start = performance.now(); + __mTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); + ++metrics[op].count; +}; +const mTimeEnd = ()=>( + metrics[__mTimer.op].time += performance.now() - __mTimer.start +); +const __wTimer = Object.create(null); +__wTimer.op = undefined; +__wTimer.start = undefined; +const wTimeStart = (op)=>{ + __wTimer.start = performance.now(); + __wTimer.op = op; + //metrics[op] || toss("Maintenance required: missing metrics for",op); +}; +const wTimeEnd = ()=>( + metrics[__wTimer.op].wait += performance.now() - __wTimer.start +); + +/** + Gets set to true by the 'opfs-async-shutdown' command to quit the + wait loop. This is only intended for debugging purposes: we cannot + inspect this file's state while the tight waitLoop() is running and + need a way to stop that loop for introspection purposes. +*/ +let flagAsyncShutdown = false; + + +/** + Asynchronous wrappers for sqlite3_vfs and sqlite3_io_methods + methods, as well as helpers like mkdir(). Maintenance reminder: + members are in alphabetical order to simplify finding them. +*/ +const vfsAsyncImpls = { + 'opfs-async-metrics': async ()=>{ + mTimeStart('opfs-async-metrics'); + metrics.dump(); + storeAndNotify('opfs-async-metrics', 0); + mTimeEnd(); + }, + 'opfs-async-shutdown': async ()=>{ + flagAsyncShutdown = true; + storeAndNotify('opfs-async-shutdown', 0); + }, + mkdir: async (dirname)=>{ + mTimeStart('mkdir'); + let rc = 0; + wTimeStart('mkdir'); + try { + await getDirForFilename(dirname+"/filepart", true); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('mkdir', rc); + mTimeEnd(); + }, + xAccess: async (filename)=>{ + mTimeStart('xAccess'); + /* OPFS cannot support the full range of xAccess() queries sqlite3 + calls for. We can essentially just tell if the file is + accessible, but if it is it's automatically writable (unless + it's locked, which we cannot(?) know without trying to open + it). OPFS does not have the notion of read-only. + + The return semantics of this function differ from sqlite3's + xAccess semantics because we are limited in what we can + communicate back to our synchronous communication partner: 0 = + accessible, non-0 means not accessible. + */ + let rc = 0; + wTimeStart('xAccess'); + try{ + const [dh, fn] = await getDirForFilename(filename); + await dh.getFileHandle(fn); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + }finally{ + wTimeEnd(); + } + storeAndNotify('xAccess', rc); + mTimeEnd(); + }, + xClose: async function(fid/*sqlite3_file pointer*/){ + const opName = 'xClose'; + mTimeStart(opName); + __autoLocks.delete(fid); + const fh = __openFiles[fid]; + let rc = 0; + wTimeStart(opName); + if(fh){ + delete __openFiles[fid]; + await closeSyncHandle(fh); + if(fh.deleteOnClose){ + try{ await fh.dirHandle.removeEntry(fh.filenamePart) } + catch(e){ warn("Ignoring dirHandle.removeEntry() failure of",fh,e) } + } + }else{ + state.s11n.serialize(); + rc = state.sq3Codes.SQLITE_NOTFOUND; + } + wTimeEnd(); + storeAndNotify(opName, rc); + mTimeEnd(); + }, + xDelete: async function(...args){ + mTimeStart('xDelete'); + const rc = await vfsAsyncImpls.xDeleteNoWait(...args); + storeAndNotify('xDelete', rc); + mTimeEnd(); + }, + xDeleteNoWait: async function(filename, syncDir = 0, recursive = false){ + /* The syncDir flag is, for purposes of the VFS API's semantics, + ignored here. However, if it has the value 0x1234 then: after + deleting the given file, recursively try to delete any empty + directories left behind in its wake (ignoring any errors and + stopping at the first failure). + + That said: we don't know for sure that removeEntry() fails if + the dir is not empty because the API is not documented. It has, + however, a "recursive" flag which defaults to false, so + presumably it will fail if the dir is not empty and that flag + is false. + */ + let rc = 0; + wTimeStart('xDelete'); + try { + while(filename){ + const [hDir, filenamePart] = await getDirForFilename(filename, false); + if(!filenamePart) break; + await hDir.removeEntry(filenamePart, {recursive}); + if(0x1234 !== syncDir) break; + recursive = false; + filename = getResolvedPath(filename, true); + filename.pop(); + filename = filename.join('/'); + } + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_DELETE; + } + wTimeEnd(); + return rc; + }, + xFileSize: async function(fid/*sqlite3_file pointer*/){ + mTimeStart('xFileSize'); + const fh = __openFiles[fid]; + let rc; + wTimeStart('xFileSize'); + try{ + affirmLocked('xFileSize',fh); + rc = await (await getSyncHandle(fh)).getSize(); + state.s11n.serialize(Number(rc)); + rc = 0; + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR; + } + wTimeEnd(); + storeAndNotify('xFileSize', rc); + mTimeEnd(); + }, + xLock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xLock'); + const fh = __openFiles[fid]; + let rc = 0; + const oldLockType = fh.xLock; + fh.xLock = lockType; + if( !fh.syncHandle ){ + wTimeStart('xLock'); + try { + await getSyncHandle(fh); + __autoLocks.delete(fid); + }catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_LOCK; + fh.xLock = oldLockType; + } + wTimeEnd(); + } + storeAndNotify('xLock',rc); + mTimeEnd(); + }, + xOpen: async function(fid/*sqlite3_file pointer*/, filename, + flags/*SQLITE_OPEN_...*/){ + const opName = 'xOpen'; + mTimeStart(opName); + const deleteOnClose = (state.sq3Codes.SQLITE_OPEN_DELETEONCLOSE & flags); + const create = (state.sq3Codes.SQLITE_OPEN_CREATE & flags); + wTimeStart('xOpen'); + try{ + let hDir, filenamePart; + try { + [hDir, filenamePart] = await getDirForFilename(filename, !!create); + }catch(e){ + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_NOTFOUND); + mTimeEnd(); + wTimeEnd(); + return; + } + const hFile = await hDir.getFileHandle(filenamePart, {create}); + /** + wa-sqlite, at this point, grabs a SyncAccessHandle and + assigns it to the syncHandle prop of the file state + object, but only for certain cases and it's unclear why it + places that limitation on it. + */ + wTimeEnd(); + __openFiles[fid] = Object.assign(Object.create(null),{ + fid: fid, + filenameAbs: filename, + filenamePart: filenamePart, + dirHandle: hDir, + fileHandle: hFile, + sabView: state.sabFileBufView, + readOnly: create + ? false : (state.sq3Codes.SQLITE_OPEN_READONLY & flags), + deleteOnClose: deleteOnClose + }); + storeAndNotify(opName, 0); + }catch(e){ + wTimeEnd(); + error(opName,e); + state.s11n.storeException(1,e); + storeAndNotify(opName, state.sq3Codes.SQLITE_IOERR); + } + mTimeEnd(); + }, + xRead: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xRead'); + let rc = 0, nRead; + const fh = __openFiles[fid]; + try{ + affirmLocked('xRead',fh); + wTimeStart('xRead'); + nRead = (await getSyncHandle(fh)).read( + fh.sabView.subarray(0, n), + {at: Number(offset64)} + ); + wTimeEnd(); + if(nRead < n){/* Zero-fill remaining bytes */ + fh.sabView.fill(0, nRead, n); + rc = state.sq3Codes.SQLITE_IOERR_SHORT_READ; + } + }catch(e){ + if(undefined===nRead) wTimeEnd(); + error("xRead() failed",e,fh); + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_READ; + } + storeAndNotify('xRead',rc); + mTimeEnd(); + }, + xSync: async function(fid/*sqlite3_file pointer*/,flags/*ignored*/){ + mTimeStart('xSync'); + const fh = __openFiles[fid]; + let rc = 0; + if(!fh.readOnly && fh.syncHandle){ + try { + wTimeStart('xSync'); + await fh.syncHandle.flush(); + }catch(e){ + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_FSYNC; + } + wTimeEnd(); + } + storeAndNotify('xSync',rc); + mTimeEnd(); + }, + xTruncate: async function(fid/*sqlite3_file pointer*/,size){ + mTimeStart('xTruncate'); + let rc = 0; + const fh = __openFiles[fid]; + wTimeStart('xTruncate'); + try{ + affirmLocked('xTruncate',fh); + affirmNotRO('xTruncate', fh); + await (await getSyncHandle(fh)).truncate(size); + }catch(e){ + error("xTruncate():",e,fh); + state.s11n.storeException(2,e); + rc = state.sq3Codes.SQLITE_IOERR_TRUNCATE; + } + wTimeEnd(); + storeAndNotify('xTruncate',rc); + mTimeEnd(); + }, + xUnlock: async function(fid/*sqlite3_file pointer*/, + lockType/*SQLITE_LOCK_...*/){ + mTimeStart('xUnlock'); + let rc = 0; + const fh = __openFiles[fid]; + if( state.sq3Codes.SQLITE_LOCK_NONE===lockType + && fh.syncHandle ){ + wTimeStart('xUnlock'); + try { await closeSyncHandle(fh) } + catch(e){ + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_UNLOCK; + } + wTimeEnd(); + } + storeAndNotify('xUnlock',rc); + mTimeEnd(); + }, + xWrite: async function(fid/*sqlite3_file pointer*/,n,offset64){ + mTimeStart('xWrite'); + let rc; + const fh = __openFiles[fid]; + wTimeStart('xWrite'); + try{ + affirmLocked('xWrite',fh); + affirmNotRO('xWrite', fh); + rc = ( + n === (await getSyncHandle(fh)) + .write(fh.sabView.subarray(0, n), + {at: Number(offset64)}) + ) ? 0 : state.sq3Codes.SQLITE_IOERR_WRITE; + }catch(e){ + error("xWrite():",e,fh); + state.s11n.storeException(1,e); + rc = state.sq3Codes.SQLITE_IOERR_WRITE; + } + wTimeEnd(); + storeAndNotify('xWrite',rc); + mTimeEnd(); + } +}/*vfsAsyncImpls*/; + +const initS11n = ()=>{ + /** + ACHTUNG: this code is 100% duplicated in the other half of this + proxy! The documentation is maintained in the "synchronous half". + */ + if(state.s11n) return state.s11n; + const textDecoder = new TextDecoder(), + textEncoder = new TextEncoder('utf-8'), + viewU8 = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize), + viewDV = new DataView(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.s11n = Object.create(null); + const TypeIds = Object.create(null); + TypeIds.number = { id: 1, size: 8, getter: 'getFloat64', setter: 'setFloat64' }; + TypeIds.bigint = { id: 2, size: 8, getter: 'getBigInt64', setter: 'setBigInt64' }; + TypeIds.boolean = { id: 3, size: 4, getter: 'getInt32', setter: 'setInt32' }; + TypeIds.string = { id: 4 }; + const getTypeId = (v)=>( + TypeIds[typeof v] + || toss("Maintenance required: this value type cannot be serialized.",v) + ); + const getTypeIdById = (tid)=>{ + switch(tid){ + case TypeIds.number.id: return TypeIds.number; + case TypeIds.bigint.id: return TypeIds.bigint; + case TypeIds.boolean.id: return TypeIds.boolean; + case TypeIds.string.id: return TypeIds.string; + default: toss("Invalid type ID:",tid); + } + }; + state.s11n.deserialize = function(clear=false){ + ++metrics.s11n.deserialize.count; + const t = performance.now(); + const argc = viewU8[0]; + const rc = argc ? [] : null; + if(argc){ + const typeIds = []; + let offset = 1, i, n, v; + for(i = 0; i < argc; ++i, ++offset){ + typeIds.push(getTypeIdById(viewU8[offset])); + } + for(i = 0; i < argc; ++i){ + const t = typeIds[i]; + if(t.getter){ + v = viewDV[t.getter](offset, state.littleEndian); + offset += t.size; + }else{/*String*/ + n = viewDV.getInt32(offset, state.littleEndian); + offset += 4; + v = textDecoder.decode(viewU8.slice(offset, offset+n)); + offset += n; + } + rc.push(v); + } + } + if(clear) viewU8[0] = 0; + //log("deserialize:",argc, rc); + metrics.s11n.deserialize.time += performance.now() - t; + return rc; + }; + state.s11n.serialize = function(...args){ + const t = performance.now(); + ++metrics.s11n.serialize.count; + if(args.length){ + //log("serialize():",args); + const typeIds = []; + let i = 0, offset = 1; + viewU8[0] = args.length & 0xff /* header = # of args */; + for(; i < args.length; ++i, ++offset){ + /* Write the TypeIds.id value into the next args.length + bytes. */ + typeIds.push(getTypeId(args[i])); + viewU8[offset] = typeIds[i].id; + } + for(i = 0; i < args.length; ++i) { + /* Deserialize the following bytes based on their + corresponding TypeIds.id from the header. */ + const t = typeIds[i]; + if(t.setter){ + viewDV[t.setter](offset, args[i], state.littleEndian); + offset += t.size; + }else{/*String*/ + const s = textEncoder.encode(args[i]); + viewDV.setInt32(offset, s.byteLength, state.littleEndian); + offset += 4; + viewU8.set(s, offset); + offset += s.byteLength; + } + } + //log("serialize() result:",viewU8.slice(0,offset)); + }else{ + viewU8[0] = 0; + } + metrics.s11n.serialize.time += performance.now() - t; + }; + + state.s11n.storeException = state.asyncS11nExceptions + ? ((priority,e)=>{ + if(priority<=state.asyncS11nExceptions){ + state.s11n.serialize([e.name,': ',e.message].join("")); + } + }) + : ()=>{}; + + return state.s11n; +}/*initS11n()*/; + +const waitLoop = async function f(){ + const opHandlers = Object.create(null); + for(let k of Object.keys(state.opIds)){ + const vi = vfsAsyncImpls[k]; + if(!vi) continue; + const o = Object.create(null); + opHandlers[state.opIds[k]] = o; + o.key = k; + o.f = vi; + } + /** + waitTime is how long (ms) to wait for each Atomics.wait(). + We need to wake up periodically to give the thread a chance + to do other things. + */ + const waitTime = 500; + while(!flagAsyncShutdown){ + try { + if('timed-out'===Atomics.wait( + state.sabOPView, state.opIds.whichOp, 0, waitTime + )){ + if(__autoLocks.size){ + /* Release all auto-locks. */ + for(const fid of __autoLocks){ + const fh = __openFiles[fid]; + await closeSyncHandleNoThrow(fh); + log("Auto-unlocked",fid,fh.filenameAbs); + } + } + continue; + } + const opId = Atomics.load(state.sabOPView, state.opIds.whichOp); + Atomics.store(state.sabOPView, state.opIds.whichOp, 0); + const hnd = opHandlers[opId] ?? toss("No waitLoop handler for whichOp #",opId); + const args = state.s11n.deserialize( + true /* clear s11n to keep the caller from confusing this with + an exception string written by the upcoming + operation */ + ) || []; + //warn("waitLoop() whichOp =",opId, hnd, args); + if(hnd.f) await hnd.f(...args); + else error("Missing callback for opId",opId); + }catch(e){ + error('in waitLoop():',e); + } + } +}; + +navigator.storage.getDirectory().then(function(d){ + const wMsg = (type)=>postMessage({type}); + state.rootDir = d; + self.onmessage = function({data}){ + switch(data.type){ + case 'opfs-async-init':{ + /* Receive shared state from synchronous partner */ + const opt = data.args; + state.littleEndian = opt.littleEndian; + state.asyncS11nExceptions = opt.asyncS11nExceptions; + state.verbose = opt.verbose ?? 2; + state.fileBufferSize = opt.fileBufferSize; + state.sabS11nOffset = opt.sabS11nOffset; + state.sabS11nSize = opt.sabS11nSize; + state.sabOP = opt.sabOP; + state.sabOPView = new Int32Array(state.sabOP); + state.sabIO = opt.sabIO; + state.sabFileBufView = new Uint8Array(state.sabIO, 0, state.fileBufferSize); + state.sabS11nView = new Uint8Array(state.sabIO, state.sabS11nOffset, state.sabS11nSize); + state.opIds = opt.opIds; + state.sq3Codes = opt.sq3Codes; + Object.keys(vfsAsyncImpls).forEach((k)=>{ + if(!Number.isFinite(state.opIds[k])){ + toss("Maintenance required: missing state.opIds[",k,"]"); + } + }); + initS11n(); + metrics.reset(); + log("init state",state); + wMsg('opfs-async-inited'); + waitLoop(); + break; + } + case 'opfs-async-restart': + if(flagAsyncShutdown){ + warn("Restarting after opfs-async-shutdown. Might or might not work."); + flagAsyncShutdown = false; + waitLoop(); + } + break; + case 'opfs-async-metrics': + metrics.dump(); + break; + } + }; + wMsg('opfs-async-loaded'); +}).catch((e)=>error("error initializing OPFS asyncer:",e)); diff --git a/ext/wasm/api/sqlite3-wasi.h b/ext/wasm/api/sqlite3-wasi.h new file mode 100644 index 0000000..096f45d --- /dev/null +++ b/ext/wasm/api/sqlite3-wasi.h @@ -0,0 +1,69 @@ +/** + Dummy function stubs to get sqlite3.c compiling with + wasi-sdk. This requires, in addition: + + -D_WASI_EMULATED_MMAN -D_WASI_EMULATED_GETPID + + -lwasi-emulated-getpid +*/ +typedef unsigned mode_t; +int fchmod(int fd, mode_t mode); +int fchmod(int fd, mode_t mode){ + return (fd && mode) ? 0 : 0; +} +typedef unsigned uid_t; +typedef uid_t gid_t; +int fchown(int fd, uid_t owner, gid_t group); +int fchown(int fd, uid_t owner, gid_t group){ + return (fd && owner && group) ? 0 : 0; +} +uid_t geteuid(void); +uid_t geteuid(void){return 0;} +#if !defined(F_WRLCK) +enum { +F_WRLCK, +F_RDLCK, +F_GETLK, +F_SETLK, +F_UNLCK +}; +#endif + +#undef HAVE_PREAD + +#include <wasi/api.h> +#define WASM__KEEP __attribute__((used)) + +#if 0 +/** + wasi-sdk cannot build sqlite3's default VFS without at least the following + functions. They are apparently syscalls which clients have to implement or + otherwise obtain. + + https://github.com/WebAssembly/WASI/blob/main/phases/snapshot/docs.md +*/ +environ_get +environ_sizes_get +clock_time_get +fd_close +fd_fdstat_get +fd_fdstat_set_flags +fd_filestat_get +fd_filestat_set_size +fd_pread +fd_prestat_get +fd_prestat_dir_name +fd_read +fd_seek +fd_sync +fd_write +path_create_directory +path_filestat_get +path_filestat_set_times +path_open +path_readlink +path_remove_directory +path_unlink_file +poll_oneoff +proc_exit +#endif diff --git a/ext/wasm/api/sqlite3-wasm.c b/ext/wasm/api/sqlite3-wasm.c new file mode 100644 index 0000000..af5ed6b --- /dev/null +++ b/ext/wasm/api/sqlite3-wasm.c @@ -0,0 +1,1181 @@ +/* +** This file requires access to sqlite3.c static state in order to +** implement certain WASM-specific features, and thus directly +** includes that file. Unlike the rest of sqlite3.c, this file +** requires compiling with -std=c99 (or equivalent, or a later C +** version) because it makes use of features not available in C89. +** +** At its simplest, to build sqlite3.wasm either place this file +** in the same directory as sqlite3.c/h before compilation or use the +** -I/path flag to tell the compiler where to find both of those +** files, then compile this file. For example: +** +** emcc -o sqlite3.wasm ... -I/path/to/sqlite3-c-and-h sqlite3-wasm.c +*/ +#define SQLITE_WASM +#ifdef SQLITE_WASM_ENABLE_C_TESTS +/* +** Code blocked off by SQLITE_WASM_TESTS is intended solely for use in +** unit/regression testing. They may be safely omitted from +** client-side builds. The main unit test script, tester1.js, will +** skip related tests if it doesn't find the corresponding functions +** in the WASM exports. +*/ +# define SQLITE_WASM_TESTS 1 +#else +# define SQLITE_WASM_TESTS 0 +#endif + +/* +** Threading and file locking: JS is single-threaded. Each Worker +** thread is a separate instance of the JS engine so can never access +** the same db handle as another thread, thus multi-threading support +** is unnecessary in the library. Because the filesystems are virtual +** and local to a given wasm runtime instance, two Workers can never +** access the same db file at once, with the exception of OPFS. As of +** this writing (2022-09-30), OPFS exclusively locks a file when +** opening it, so two Workers can never open the same OPFS-backed file +** at once. That situation will change if and when lower-level locking +** features are added to OPFS (as is currently planned, per folks +** involved with its development). +** +** Summary: except for the case of future OPFS, which supports +** locking, and any similar future filesystems, threading and file +** locking support are unnecessary in the wasm build. +*/ + +/* +** Undefine any SQLITE_... config flags which we specifically do not +** want undefined. Please keep these alphabetized. +*/ +#undef SQLITE_OMIT_DESERIALIZE +#undef SQLITE_OMIT_MEMORYDB + +/* +** Define any SQLITE_... config defaults we want if they aren't +** overridden by the builder. Please keep these alphabetized. +*/ + +/**********************************************************************/ +/* SQLITE_D... */ +#ifndef SQLITE_DEFAULT_CACHE_SIZE +/* +** The OPFS impls benefit tremendously from an increased cache size +** when working on large workloads, e.g. speedtest1 --size 50 or +** higher. On smaller workloads, e.g. speedtest1 --size 25, they +** clearly benefit from having 4mb of cache, but not as much as a +** larger cache benefits the larger workloads. Speed differences +** between 2x and nearly 3x have been measured with ample page cache. +*/ +# define SQLITE_DEFAULT_CACHE_SIZE -16384 +#endif +#if 0 && !defined(SQLITE_DEFAULT_PAGE_SIZE) +/* TODO: experiment with this. */ +# define SQLITE_DEFAULT_PAGE_SIZE 8192 /*4096*/ +#endif +#ifndef SQLITE_DEFAULT_UNIX_VFS +# define SQLITE_DEFAULT_UNIX_VFS "unix-none" +#endif +#undef SQLITE_DQS +#define SQLITE_DQS 0 + +/**********************************************************************/ +/* SQLITE_ENABLE_... */ +#ifndef SQLITE_ENABLE_BYTECODE_VTAB +# define SQLITE_ENABLE_BYTECODE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBPAGE_VTAB +# define SQLITE_ENABLE_DBPAGE_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_DBSTAT_VTAB +# define SQLITE_ENABLE_DBSTAT_VTAB 1 +#endif +#ifndef SQLITE_ENABLE_EXPLAIN_COMMENTS +# define SQLITE_ENABLE_EXPLAIN_COMMENTS 1 +#endif +#ifndef SQLITE_ENABLE_FTS4 +# define SQLITE_ENABLE_FTS4 1 +#endif +#ifndef SQLITE_ENABLE_OFFSET_SQL_FUNC +# define SQLITE_ENABLE_OFFSET_SQL_FUNC 1 +#endif +#ifndef SQLITE_ENABLE_RTREE +# define SQLITE_ENABLE_RTREE 1 +#endif +#ifndef SQLITE_ENABLE_STMTVTAB +# define SQLITE_ENABLE_STMTVTAB 1 +#endif +#ifndef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +# define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +#endif + +/**********************************************************************/ +/* SQLITE_O... */ +#ifndef SQLITE_OMIT_DEPRECATED +# define SQLITE_OMIT_DEPRECATED 1 +#endif +#ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION 1 +#endif +#ifndef SQLITE_OMIT_SHARED_CACHE +# define SQLITE_OMIT_SHARED_CACHE 1 +#endif +#ifndef SQLITE_OMIT_UTF16 +# define SQLITE_OMIT_UTF16 1 +#endif +#ifndef SQLITE_OMIT_WAL +# define SQLITE_OMIT_WAL 1 +#endif +#ifndef SQLITE_OS_KV_OPTIONAL +# define SQLITE_OS_KV_OPTIONAL 1 +#endif + +/**********************************************************************/ +/* SQLITE_T... */ +#ifndef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 +#endif +#ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +#endif + +/**********************************************************************/ +/* SQLITE_USE_... */ +#ifndef SQLITE_USE_URI +# define SQLITE_USE_URI 1 +#endif + +#include <assert.h> +#include "sqlite3.c" /* yes, .c instead of .h. */ + +#if defined(__EMSCRIPTEN__) +# include <emscripten/console.h> +#endif + +/* +** SQLITE_WASM_KEEP is functionally identical to EMSCRIPTEN_KEEPALIVE +** but is not Emscripten-specific. It explicitly marks functions for +** export into the target wasm file without requiring explicit listing +** of those functions in Emscripten's -sEXPORTED_FUNCTIONS=... list +** (or equivalent in other build platforms). Any function with neither +** this attribute nor which is listed as an explicit export will not +** be exported from the wasm file (but may still be used internally +** within the wasm file). +** +** The functions in this file (sqlite3-wasm.c) which require exporting +** are marked with this flag. They may also be added to any explicit +** build-time export list but need not be. All of these APIs are +** intended for use only within the project's own JS/WASM code, and +** not by client code, so an argument can be made for reducing their +** visibility by not including them in any build-time export lists. +** +** 2022-09-11: it's not yet _proven_ that this approach works in +** non-Emscripten builds. If not, such builds will need to export +** those using the --export=... wasm-ld flag (or equivalent). As of +** this writing we are tied to Emscripten for various reasons +** and cannot test the library with other build environments. +*/ +#define SQLITE_WASM_KEEP __attribute__((used,visibility("default"))) +// See also: +//__attribute__((export_name("theExportedName"), used, visibility("default"))) + + +#if 0 +/* +** An EXPERIMENT in implementing a stack-based allocator analog to +** Emscripten's stackSave(), stackAlloc(), stackRestore(). +** Unfortunately, this cannot work together with Emscripten because +** Emscripten defines its own native one and we'd stomp on each +** other's memory. Other than that complication, basic tests show it +** to work just fine. +** +** Another option is to malloc() a chunk of our own and call that our +** "stack". +*/ +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_end(void){ + extern void __heap_base + /* see https://stackoverflow.com/questions/10038964 */; + return &__heap_base; +} +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_begin(void){ + extern void __data_end; + return &__data_end; +} +static void * pWasmStackPtr = 0; +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_ptr(void){ + if(!pWasmStackPtr) pWasmStackPtr = sqlite3_wasm_stack_end(); + return pWasmStackPtr; +} +SQLITE_WASM_KEEP void sqlite3_wasm_stack_restore(void * p){ + pWasmStackPtr = p; +} +SQLITE_WASM_KEEP void * sqlite3_wasm_stack_alloc(int n){ + if(n<=0) return 0; + n = (n + 7) & ~7 /* align to 8-byte boundary */; + unsigned char * const p = (unsigned char *)sqlite3_wasm_stack_ptr(); + unsigned const char * const b = (unsigned const char *)sqlite3_wasm_stack_begin(); + if(b + n >= p || b + n < b/*overflow*/) return 0; + return pWasmStackPtr = p - n; +} +#endif /* stack allocator experiment */ + +/* +** State for the "pseudo-stack" allocator implemented in +** sqlite3_wasm_pstack_xyz(). In order to avoid colliding with +** Emscripten-controled stack space, it carves out a bit of stack +** memory to use for that purpose. This memory ends up in the +** WASM-managed memory, such that routines which manipulate the wasm +** heap can also be used to manipulate this memory. +** +** This particular allocator is intended for small allocations such as +** storage for output pointers. We cannot reasonably size it large +** enough for general-purpose string conversions because some of our +** tests use input files (strings) of 16MB+. +*/ +static unsigned char PStack_mem[512 * 8] = {0}; +static struct { + unsigned const char * const pBegin;/* Start (inclusive) of memory */ + unsigned const char * const pEnd; /* One-after-the-end of memory */ + unsigned char * pPos; /* Current stack pointer */ +} PStack = { + &PStack_mem[0], + &PStack_mem[0] + sizeof(PStack_mem), + &PStack_mem[0] + sizeof(PStack_mem) +}; +/* +** Returns the current pstack position. +*/ +SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_ptr(void){ + return PStack.pPos; +} +/* +** Sets the pstack position poitner to p. Results are undefined if the +** given value did not come from sqlite3_wasm_pstack_ptr(). +*/ +SQLITE_WASM_KEEP void sqlite3_wasm_pstack_restore(unsigned char * p){ + assert(p>=PStack.pBegin && p<=PStack.pEnd && p>=PStack.pPos); + assert(0==(p & 0x7)); + if(p>=PStack.pBegin && p<=PStack.pEnd /*&& p>=PStack.pPos*/){ + PStack.pPos = p; + } +} +/* +** Allocate and zero out n bytes from the pstack. Returns a pointer to +** the memory on success, 0 on error (including a negative n value). n +** is always adjusted to be a multiple of 8 and returned memory is +** always zeroed out before returning (because this keeps the client +** JS code from having to do so, and most uses of the pstack will +** call for doing so). +*/ +SQLITE_WASM_KEEP void * sqlite3_wasm_pstack_alloc(int n){ + if( n<=0 ) return 0; + //if( n & 0x7 ) n += 8 - (n & 0x7) /* align to 8-byte boundary */; + n = (n + 7) & ~7 /* align to 8-byte boundary */; + if( PStack.pBegin + n > PStack.pPos /*not enough space left*/ + || PStack.pBegin + n <= PStack.pBegin /*overflow*/ ) return 0; + memset((PStack.pPos = PStack.pPos - n), 0, (unsigned int)n); + return PStack.pPos; +} +/* +** Return the number of bytes left which can be +** sqlite3_wasm_pstack_alloc()'d. +*/ +SQLITE_WASM_KEEP int sqlite3_wasm_pstack_remaining(void){ + assert(PStack.pPos >= PStack.pBegin); + assert(PStack.pPos <= PStack.pEnd); + return (int)(PStack.pPos - PStack.pBegin); +} + +/* +** Return the total number of bytes available in the pstack, including +** any space which is currently allocated. This value is a +** compile-time constant. +*/ +SQLITE_WASM_KEEP int sqlite3_wasm_pstack_quota(void){ + return (int)(PStack.pEnd - PStack.pBegin); +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** For purposes of certain hand-crafted C/Wasm function bindings, we +** need a way of reporting errors which is consistent with the rest of +** the C API, as opposed to throwing JS exceptions. To that end, this +** internal-use-only function is a thin proxy around +** sqlite3ErrorWithMessage(). The intent is that it only be used from +** Wasm bindings such as sqlite3_prepare_v2/v3(), and definitely not +** from client code. +** +** Returns err_code. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){ + if( 0!=zMsg ){ + const int nMsg = sqlite3Strlen30(zMsg); + sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg); + }else{ + sqlite3ErrorWithMsg(db, err_code, NULL); + } + return err_code; +} + +#if SQLITE_WASM_TESTS +struct WasmTestStruct { + int v4; + void * ppV; + const char * cstr; + int64_t v8; + void (*xFunc)(void*); +}; +typedef struct WasmTestStruct WasmTestStruct; +SQLITE_WASM_KEEP +void sqlite3_wasm_test_struct(WasmTestStruct * s){ + if(s){ + s->v4 *= 2; + s->v8 = s->v4 * 2; + s->ppV = s; + s->cstr = __FILE__; + if(s->xFunc) s->xFunc(s); + } + return; +} +#endif /* SQLITE_WASM_TESTS */ + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. Unlike the +** rest of the sqlite3 API, this part requires C99 for snprintf() and +** variadic macros. +** +** Returns a string containing a JSON-format "enum" of C-level +** constants and struct-related metadata intended to be imported into +** the JS environment. The JSON is initialized the first time this +** function is called and that result is reused for all future calls. +** +** If this function returns NULL then it means that the internal +** buffer is not large enough for the generated JSON and needs to be +** increased. In debug builds that will trigger an assert(). +*/ +SQLITE_WASM_KEEP +const char * sqlite3_wasm_enum_json(void){ + static char aBuffer[1024 * 12] = {0} /* where the JSON goes */; + int n = 0, nChildren = 0, nStruct = 0 + /* output counters for figuring out where commas go */; + char * zPos = &aBuffer[1] /* skip first byte for now to help protect + ** against a small race condition */; + char const * const zEnd = &aBuffer[0] + sizeof(aBuffer) /* one-past-the-end */; + if(aBuffer[0]) return aBuffer; + /* Leave aBuffer[0] at 0 until the end to help guard against a tiny + ** race condition. If this is called twice concurrently, they might + ** end up both writing to aBuffer, but they'll both write the same + ** thing, so that's okay. If we set byte 0 up front then the 2nd + ** instance might return and use the string before the 1st instance + ** is done filling it. */ + +/* Core output macros... */ +#define lenCheck assert(zPos < zEnd - 128 \ + && "sqlite3_wasm_enum_json() buffer is too small."); \ + if( zPos >= zEnd - 128 ) return 0 +#define outf(format,...) \ + zPos += snprintf(zPos, ((size_t)(zEnd - zPos)), format, __VA_ARGS__); \ + lenCheck +#define out(TXT) outf("%s",TXT) +#define CloseBrace(LEVEL) \ + assert(LEVEL<5); memset(zPos, '}', LEVEL); zPos+=LEVEL; lenCheck + +/* Macros for emitting maps of integer- and string-type macros to +** their values. */ +#define DefGroup(KEY) n = 0; \ + outf("%s\"" #KEY "\": {",(nChildren++ ? "," : "")); +#define DefInt(KEY) \ + outf("%s\"%s\": %d", (n++ ? ", " : ""), #KEY, (int)KEY) +#define DefStr(KEY) \ + outf("%s\"%s\": \"%s\"", (n++ ? ", " : ""), #KEY, KEY) +#define _DefGroup CloseBrace(1) + + /* The following groups are sorted alphabetic by group name. */ + DefGroup(access){ + DefInt(SQLITE_ACCESS_EXISTS); + DefInt(SQLITE_ACCESS_READWRITE); + DefInt(SQLITE_ACCESS_READ)/*docs say this is unused*/; + } _DefGroup; + + DefGroup(blobFinalizers) { + /* SQLITE_STATIC/TRANSIENT need to be handled explicitly as + ** integers to avoid casting-related warnings. */ + out("\"SQLITE_STATIC\":0, \"SQLITE_TRANSIENT\":-1"); + } _DefGroup; + + DefGroup(dataTypes) { + DefInt(SQLITE_INTEGER); + DefInt(SQLITE_FLOAT); + DefInt(SQLITE_TEXT); + DefInt(SQLITE_BLOB); + DefInt(SQLITE_NULL); + } _DefGroup; + + DefGroup(encodings) { + /* Noting that the wasm binding only aims to support UTF-8. */ + DefInt(SQLITE_UTF8); + DefInt(SQLITE_UTF16LE); + DefInt(SQLITE_UTF16BE); + DefInt(SQLITE_UTF16); + /*deprecated DefInt(SQLITE_ANY); */ + DefInt(SQLITE_UTF16_ALIGNED); + } _DefGroup; + + DefGroup(fcntl) { + DefInt(SQLITE_FCNTL_LOCKSTATE); + DefInt(SQLITE_FCNTL_GET_LOCKPROXYFILE); + DefInt(SQLITE_FCNTL_SET_LOCKPROXYFILE); + DefInt(SQLITE_FCNTL_LAST_ERRNO); + DefInt(SQLITE_FCNTL_SIZE_HINT); + DefInt(SQLITE_FCNTL_CHUNK_SIZE); + DefInt(SQLITE_FCNTL_FILE_POINTER); + DefInt(SQLITE_FCNTL_SYNC_OMITTED); + DefInt(SQLITE_FCNTL_WIN32_AV_RETRY); + DefInt(SQLITE_FCNTL_PERSIST_WAL); + DefInt(SQLITE_FCNTL_OVERWRITE); + DefInt(SQLITE_FCNTL_VFSNAME); + DefInt(SQLITE_FCNTL_POWERSAFE_OVERWRITE); + DefInt(SQLITE_FCNTL_PRAGMA); + DefInt(SQLITE_FCNTL_BUSYHANDLER); + DefInt(SQLITE_FCNTL_TEMPFILENAME); + DefInt(SQLITE_FCNTL_MMAP_SIZE); + DefInt(SQLITE_FCNTL_TRACE); + DefInt(SQLITE_FCNTL_HAS_MOVED); + DefInt(SQLITE_FCNTL_SYNC); + DefInt(SQLITE_FCNTL_COMMIT_PHASETWO); + DefInt(SQLITE_FCNTL_WIN32_SET_HANDLE); + DefInt(SQLITE_FCNTL_WAL_BLOCK); + DefInt(SQLITE_FCNTL_ZIPVFS); + DefInt(SQLITE_FCNTL_RBU); + DefInt(SQLITE_FCNTL_VFS_POINTER); + DefInt(SQLITE_FCNTL_JOURNAL_POINTER); + DefInt(SQLITE_FCNTL_WIN32_GET_HANDLE); + DefInt(SQLITE_FCNTL_PDB); + DefInt(SQLITE_FCNTL_BEGIN_ATOMIC_WRITE); + DefInt(SQLITE_FCNTL_COMMIT_ATOMIC_WRITE); + DefInt(SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE); + DefInt(SQLITE_FCNTL_LOCK_TIMEOUT); + DefInt(SQLITE_FCNTL_DATA_VERSION); + DefInt(SQLITE_FCNTL_SIZE_LIMIT); + DefInt(SQLITE_FCNTL_CKPT_DONE); + DefInt(SQLITE_FCNTL_RESERVE_BYTES); + DefInt(SQLITE_FCNTL_CKPT_START); + DefInt(SQLITE_FCNTL_EXTERNAL_READER); + DefInt(SQLITE_FCNTL_CKSM_FILE); + } _DefGroup; + + DefGroup(flock) { + DefInt(SQLITE_LOCK_NONE); + DefInt(SQLITE_LOCK_SHARED); + DefInt(SQLITE_LOCK_RESERVED); + DefInt(SQLITE_LOCK_PENDING); + DefInt(SQLITE_LOCK_EXCLUSIVE); + } _DefGroup; + + DefGroup(ioCap) { + DefInt(SQLITE_IOCAP_ATOMIC); + DefInt(SQLITE_IOCAP_ATOMIC512); + DefInt(SQLITE_IOCAP_ATOMIC1K); + DefInt(SQLITE_IOCAP_ATOMIC2K); + DefInt(SQLITE_IOCAP_ATOMIC4K); + DefInt(SQLITE_IOCAP_ATOMIC8K); + DefInt(SQLITE_IOCAP_ATOMIC16K); + DefInt(SQLITE_IOCAP_ATOMIC32K); + DefInt(SQLITE_IOCAP_ATOMIC64K); + DefInt(SQLITE_IOCAP_SAFE_APPEND); + DefInt(SQLITE_IOCAP_SEQUENTIAL); + DefInt(SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN); + DefInt(SQLITE_IOCAP_POWERSAFE_OVERWRITE); + DefInt(SQLITE_IOCAP_IMMUTABLE); + DefInt(SQLITE_IOCAP_BATCH_ATOMIC); + } _DefGroup; + + DefGroup(openFlags) { + /* Noting that not all of these will have any effect in + ** WASM-space. */ + DefInt(SQLITE_OPEN_READONLY); + DefInt(SQLITE_OPEN_READWRITE); + DefInt(SQLITE_OPEN_CREATE); + DefInt(SQLITE_OPEN_URI); + DefInt(SQLITE_OPEN_MEMORY); + DefInt(SQLITE_OPEN_NOMUTEX); + DefInt(SQLITE_OPEN_FULLMUTEX); + DefInt(SQLITE_OPEN_SHAREDCACHE); + DefInt(SQLITE_OPEN_PRIVATECACHE); + DefInt(SQLITE_OPEN_EXRESCODE); + DefInt(SQLITE_OPEN_NOFOLLOW); + /* OPEN flags for use with VFSes... */ + DefInt(SQLITE_OPEN_MAIN_DB); + DefInt(SQLITE_OPEN_MAIN_JOURNAL); + DefInt(SQLITE_OPEN_TEMP_DB); + DefInt(SQLITE_OPEN_TEMP_JOURNAL); + DefInt(SQLITE_OPEN_TRANSIENT_DB); + DefInt(SQLITE_OPEN_SUBJOURNAL); + DefInt(SQLITE_OPEN_SUPER_JOURNAL); + DefInt(SQLITE_OPEN_WAL); + DefInt(SQLITE_OPEN_DELETEONCLOSE); + DefInt(SQLITE_OPEN_EXCLUSIVE); + } _DefGroup; + + DefGroup(prepareFlags) { + DefInt(SQLITE_PREPARE_PERSISTENT); + DefInt(SQLITE_PREPARE_NORMALIZE); + DefInt(SQLITE_PREPARE_NO_VTAB); + } _DefGroup; + + DefGroup(resultCodes) { + DefInt(SQLITE_OK); + DefInt(SQLITE_ERROR); + DefInt(SQLITE_INTERNAL); + DefInt(SQLITE_PERM); + DefInt(SQLITE_ABORT); + DefInt(SQLITE_BUSY); + DefInt(SQLITE_LOCKED); + DefInt(SQLITE_NOMEM); + DefInt(SQLITE_READONLY); + DefInt(SQLITE_INTERRUPT); + DefInt(SQLITE_IOERR); + DefInt(SQLITE_CORRUPT); + DefInt(SQLITE_NOTFOUND); + DefInt(SQLITE_FULL); + DefInt(SQLITE_CANTOPEN); + DefInt(SQLITE_PROTOCOL); + DefInt(SQLITE_EMPTY); + DefInt(SQLITE_SCHEMA); + DefInt(SQLITE_TOOBIG); + DefInt(SQLITE_CONSTRAINT); + DefInt(SQLITE_MISMATCH); + DefInt(SQLITE_MISUSE); + DefInt(SQLITE_NOLFS); + DefInt(SQLITE_AUTH); + DefInt(SQLITE_FORMAT); + DefInt(SQLITE_RANGE); + DefInt(SQLITE_NOTADB); + DefInt(SQLITE_NOTICE); + DefInt(SQLITE_WARNING); + DefInt(SQLITE_ROW); + DefInt(SQLITE_DONE); + // Extended Result Codes + DefInt(SQLITE_ERROR_MISSING_COLLSEQ); + DefInt(SQLITE_ERROR_RETRY); + DefInt(SQLITE_ERROR_SNAPSHOT); + DefInt(SQLITE_IOERR_READ); + DefInt(SQLITE_IOERR_SHORT_READ); + DefInt(SQLITE_IOERR_WRITE); + DefInt(SQLITE_IOERR_FSYNC); + DefInt(SQLITE_IOERR_DIR_FSYNC); + DefInt(SQLITE_IOERR_TRUNCATE); + DefInt(SQLITE_IOERR_FSTAT); + DefInt(SQLITE_IOERR_UNLOCK); + DefInt(SQLITE_IOERR_RDLOCK); + DefInt(SQLITE_IOERR_DELETE); + DefInt(SQLITE_IOERR_BLOCKED); + DefInt(SQLITE_IOERR_NOMEM); + DefInt(SQLITE_IOERR_ACCESS); + DefInt(SQLITE_IOERR_CHECKRESERVEDLOCK); + DefInt(SQLITE_IOERR_LOCK); + DefInt(SQLITE_IOERR_CLOSE); + DefInt(SQLITE_IOERR_DIR_CLOSE); + DefInt(SQLITE_IOERR_SHMOPEN); + DefInt(SQLITE_IOERR_SHMSIZE); + DefInt(SQLITE_IOERR_SHMLOCK); + DefInt(SQLITE_IOERR_SHMMAP); + DefInt(SQLITE_IOERR_SEEK); + DefInt(SQLITE_IOERR_DELETE_NOENT); + DefInt(SQLITE_IOERR_MMAP); + DefInt(SQLITE_IOERR_GETTEMPPATH); + DefInt(SQLITE_IOERR_CONVPATH); + DefInt(SQLITE_IOERR_VNODE); + DefInt(SQLITE_IOERR_AUTH); + DefInt(SQLITE_IOERR_BEGIN_ATOMIC); + DefInt(SQLITE_IOERR_COMMIT_ATOMIC); + DefInt(SQLITE_IOERR_ROLLBACK_ATOMIC); + DefInt(SQLITE_IOERR_DATA); + DefInt(SQLITE_IOERR_CORRUPTFS); + DefInt(SQLITE_LOCKED_SHAREDCACHE); + DefInt(SQLITE_LOCKED_VTAB); + DefInt(SQLITE_BUSY_RECOVERY); + DefInt(SQLITE_BUSY_SNAPSHOT); + DefInt(SQLITE_BUSY_TIMEOUT); + DefInt(SQLITE_CANTOPEN_NOTEMPDIR); + DefInt(SQLITE_CANTOPEN_ISDIR); + DefInt(SQLITE_CANTOPEN_FULLPATH); + DefInt(SQLITE_CANTOPEN_CONVPATH); + //DefInt(SQLITE_CANTOPEN_DIRTYWAL)/*docs say not used*/; + DefInt(SQLITE_CANTOPEN_SYMLINK); + DefInt(SQLITE_CORRUPT_VTAB); + DefInt(SQLITE_CORRUPT_SEQUENCE); + DefInt(SQLITE_CORRUPT_INDEX); + DefInt(SQLITE_READONLY_RECOVERY); + DefInt(SQLITE_READONLY_CANTLOCK); + DefInt(SQLITE_READONLY_ROLLBACK); + DefInt(SQLITE_READONLY_DBMOVED); + DefInt(SQLITE_READONLY_CANTINIT); + DefInt(SQLITE_READONLY_DIRECTORY); + DefInt(SQLITE_ABORT_ROLLBACK); + DefInt(SQLITE_CONSTRAINT_CHECK); + DefInt(SQLITE_CONSTRAINT_COMMITHOOK); + DefInt(SQLITE_CONSTRAINT_FOREIGNKEY); + DefInt(SQLITE_CONSTRAINT_FUNCTION); + DefInt(SQLITE_CONSTRAINT_NOTNULL); + DefInt(SQLITE_CONSTRAINT_PRIMARYKEY); + DefInt(SQLITE_CONSTRAINT_TRIGGER); + DefInt(SQLITE_CONSTRAINT_UNIQUE); + DefInt(SQLITE_CONSTRAINT_VTAB); + DefInt(SQLITE_CONSTRAINT_ROWID); + DefInt(SQLITE_CONSTRAINT_PINNED); + DefInt(SQLITE_CONSTRAINT_DATATYPE); + DefInt(SQLITE_NOTICE_RECOVER_WAL); + DefInt(SQLITE_NOTICE_RECOVER_ROLLBACK); + DefInt(SQLITE_WARNING_AUTOINDEX); + DefInt(SQLITE_AUTH_USER); + DefInt(SQLITE_OK_LOAD_PERMANENTLY); + //DefInt(SQLITE_OK_SYMLINK) /* internal use only */; + } _DefGroup; + + DefGroup(serialize){ + DefInt(SQLITE_SERIALIZE_NOCOPY); + DefInt(SQLITE_DESERIALIZE_FREEONCLOSE); + DefInt(SQLITE_DESERIALIZE_READONLY); + DefInt(SQLITE_DESERIALIZE_RESIZEABLE); + } _DefGroup; + + DefGroup(syncFlags) { + DefInt(SQLITE_SYNC_NORMAL); + DefInt(SQLITE_SYNC_FULL); + DefInt(SQLITE_SYNC_DATAONLY); + } _DefGroup; + + DefGroup(trace) { + DefInt(SQLITE_TRACE_STMT); + DefInt(SQLITE_TRACE_PROFILE); + DefInt(SQLITE_TRACE_ROW); + DefInt(SQLITE_TRACE_CLOSE); + } _DefGroup; + + DefGroup(udfFlags) { + DefInt(SQLITE_DETERMINISTIC); + DefInt(SQLITE_DIRECTONLY); + DefInt(SQLITE_INNOCUOUS); + } _DefGroup; + + DefGroup(version) { + DefInt(SQLITE_VERSION_NUMBER); + DefStr(SQLITE_VERSION); + DefStr(SQLITE_SOURCE_ID); + } _DefGroup; + +#undef DefGroup +#undef DefStr +#undef DefInt +#undef _DefGroup + + /* + ** Emit an array of "StructBinder" struct descripions, which look + ** like: + ** + ** { + ** "name": "MyStruct", + ** "sizeof": 16, + ** "members": { + ** "member1": {"offset": 0,"sizeof": 4,"signature": "i"}, + ** "member2": {"offset": 4,"sizeof": 4,"signature": "p"}, + ** "member3": {"offset": 8,"sizeof": 8,"signature": "j"} + ** } + ** } + ** + ** Detailed documentation for those bits are in the docs for the + ** Jaccwabyt JS-side component. + */ + + /** Macros for emitting StructBinder description. */ +#define StructBinder__(TYPE) \ + n = 0; \ + outf("%s{", (nStruct++ ? ", " : "")); \ + out("\"name\": \"" # TYPE "\","); \ + outf("\"sizeof\": %d", (int)sizeof(TYPE)); \ + out(",\"members\": {"); +#define StructBinder_(T) StructBinder__(T) + /** ^^^ indirection needed to expand CurrentStruct */ +#define StructBinder StructBinder_(CurrentStruct) +#define _StructBinder CloseBrace(2) +#define M(MEMBER,SIG) \ + outf("%s\"%s\": " \ + "{\"offset\":%d,\"sizeof\": %d,\"signature\":\"%s\"}", \ + (n++ ? ", " : ""), #MEMBER, \ + (int)offsetof(CurrentStruct,MEMBER), \ + (int)sizeof(((CurrentStruct*)0)->MEMBER), \ + SIG) + + nStruct = 0; + out(", \"structs\": ["); { + +#define CurrentStruct sqlite3_vfs + StructBinder { + M(iVersion,"i"); + M(szOsFile,"i"); + M(mxPathname,"i"); + M(pNext,"p"); + M(zName,"s"); + M(pAppData,"p"); + M(xOpen,"i(pppip)"); + M(xDelete,"i(ppi)"); + M(xAccess,"i(ppip)"); + M(xFullPathname,"i(ppip)"); + M(xDlOpen,"p(pp)"); + M(xDlError,"p(pip)"); + M(xDlSym,"p()"); + M(xDlClose,"v(pp)"); + M(xRandomness,"i(pip)"); + M(xSleep,"i(pi)"); + M(xCurrentTime,"i(pp)"); + M(xGetLastError,"i(pip)"); + M(xCurrentTimeInt64,"i(pp)"); + M(xSetSystemCall,"i(ppp)"); + M(xGetSystemCall,"p(pp)"); + M(xNextSystemCall,"p(pp)"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_io_methods + StructBinder { + M(iVersion,"i"); + M(xClose,"i(p)"); + M(xRead,"i(ppij)"); + M(xWrite,"i(ppij)"); + M(xTruncate,"i(pj)"); + M(xSync,"i(pi)"); + M(xFileSize,"i(pp)"); + M(xLock,"i(pi)"); + M(xUnlock,"i(pi)"); + M(xCheckReservedLock,"i(pp)"); + M(xFileControl,"i(pip)"); + M(xSectorSize,"i(p)"); + M(xDeviceCharacteristics,"i(p)"); + M(xShmMap,"i(piiip)"); + M(xShmLock,"i(piii)"); + M(xShmBarrier,"v(p)"); + M(xShmUnmap,"i(pi)"); + M(xFetch,"i(pjip)"); + M(xUnfetch,"i(pjp)"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_file + StructBinder { + M(pMethods,"p"); + } _StructBinder; +#undef CurrentStruct + +#define CurrentStruct sqlite3_kvvfs_methods + StructBinder { + M(xRead,"i(sspi)"); + M(xWrite,"i(sss)"); + M(xDelete,"i(ss)"); + M(nKeySize,"i"); + } _StructBinder; +#undef CurrentStruct + +#if SQLITE_WASM_TESTS +#define CurrentStruct WasmTestStruct + StructBinder { + M(v4,"i"); + M(cstr,"s"); + M(ppV,"p"); + M(v8,"j"); + M(xFunc,"v(p)"); + } _StructBinder; +#undef CurrentStruct +#endif + + } out( "]"/*structs*/); + + out("}"/*top-level object*/); + *zPos = 0; + aBuffer[0] = '{'/*end of the race-condition workaround*/; + return aBuffer; +#undef StructBinder +#undef StructBinder_ +#undef StructBinder__ +#undef M +#undef _StructBinder +#undef CloseBrace +#undef out +#undef outf +#undef lenCheck +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function invokes the xDelete method of the given VFS (or the +** default VFS if pVfs is NULL), passing on the given filename. If +** zName is NULL, no default VFS is found, or it has no xDelete +** method, SQLITE_MISUSE is returned, else the result of the xDelete() +** call is returned. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_vfs_unlink(sqlite3_vfs *pVfs, const char * zName){ + int rc = SQLITE_MISUSE /* ??? */; + if( 0==pVfs && 0!=zName ) pVfs = sqlite3_vfs_find(0); + if( zName && pVfs && pVfs->xDelete ){ + rc = pVfs->xDelete(pVfs, zName, 1); + } + return rc; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Returns a pointer to the given DB's VFS for the given DB name, +** defaulting to "main" if zDbName is 0. Returns 0 if no db with the +** given name is open. +*/ +SQLITE_WASM_KEEP +sqlite3_vfs * sqlite3_wasm_db_vfs(sqlite3 *pDb, const char *zDbName){ + sqlite3_vfs * pVfs = 0; + sqlite3_file_control(pDb, zDbName ? zDbName : "main", + SQLITE_FCNTL_VFS_POINTER, &pVfs); + return pVfs; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** This function resets the given db pointer's database as described at +** +** https://www.sqlite.org/c3ref/c_dbconfig_defensive.html#sqlitedbconfigresetdatabase +** +** Returns 0 on success, an SQLITE_xxx code on error. Returns +** SQLITE_MISUSE if pDb is NULL. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_reset(sqlite3*pDb){ + int rc = SQLITE_MISUSE; + if( pDb ){ + rc = sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); + if( 0==rc ) rc = sqlite3_exec(pDb, "VACUUM", 0, 0, 0); + sqlite3_db_config(pDb, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); + } + return rc; +} + +/* +** Uses the given database's VFS xRead to stream the db file's +** contents out to the given callback. The callback gets a single +** chunk of size n (its 2nd argument) on each call and must return 0 +** on success, non-0 on error. This function returns 0 on success, +** SQLITE_NOTFOUND if no db is open, or propagates any other non-0 +** code from the callback. Note that this is not thread-friendly: it +** expects that it will be the only thread reading the db file and +** takes no measures to ensure that is the case. +** +** This implementation appears to work fine, but +** sqlite3_wasm_db_serialize() is arguably the better way to achieve +** this. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_export_chunked( sqlite3* pDb, + int (*xCallback)(unsigned const char *zOut, int n) ){ + sqlite3_int64 nSize = 0; + sqlite3_int64 nPos = 0; + sqlite3_file * pFile = 0; + unsigned char buf[1024 * 8]; + int nBuf = (int)sizeof(buf); + int rc = pDb + ? sqlite3_file_control(pDb, "main", + SQLITE_FCNTL_FILE_POINTER, &pFile) + : SQLITE_NOTFOUND; + if( rc ) return rc; + rc = pFile->pMethods->xFileSize(pFile, &nSize); + if( rc ) return rc; + if(nSize % nBuf){ + /* DB size is not an even multiple of the buffer size. Reduce + ** buffer size so that we do not unduly inflate the db size + ** with zero-padding when exporting. */ + if(0 == nSize % 4096) nBuf = 4096; + else if(0 == nSize % 2048) nBuf = 2048; + else if(0 == nSize % 1024) nBuf = 1024; + else nBuf = 512; + } + for( ; 0==rc && nPos<nSize; nPos += nBuf ){ + rc = pFile->pMethods->xRead(pFile, buf, nBuf, nPos); + if(SQLITE_IOERR_SHORT_READ == rc){ + rc = (nPos + nBuf) < nSize ? rc : 0/*assume EOF*/; + } + if( 0==rc ) rc = xCallback(buf, nBuf); + } + return rc; +} + +/* +** A proxy for sqlite3_serialize() which serializes the "main" schema +** of pDb, placing the serialized output in pOut and nOut. nOut may be +** NULL. If pDb or pOut are NULL then SQLITE_MISUSE is returned. If +** allocation of the serialized copy fails, SQLITE_NOMEM is returned. +** On success, 0 is returned and `*pOut` will contain a pointer to the +** memory unless mFlags includes SQLITE_SERIALIZE_NOCOPY and the +** database has no contiguous memory representation, in which case +** `*pOut` will be NULL but 0 will be returned. +** +** If `*pOut` is not NULL, the caller is responsible for passing it to +** sqlite3_free() to free it. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_db_serialize( sqlite3 *pDb, unsigned char **pOut, + sqlite3_int64 *nOut, unsigned int mFlags ){ + unsigned char * z; + if( !pDb || !pOut ) return SQLITE_MISUSE; + if(nOut) *nOut = 0; + z = sqlite3_serialize(pDb, "main", nOut, mFlags); + if( z || (SQLITE_SERIALIZE_NOCOPY & mFlags) ){ + *pOut = z; + return 0; + }else{ + return SQLITE_NOMEM; + } +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Creates a new file using the I/O API of the given VFS, containing +** the given number of bytes of the given data. If the file exists, +** it is truncated to the given length and populated with the given +** data. +** +** This function exists so that we can implement the equivalent of +** Emscripten's FS.createDataFile() in a VFS-agnostic way. This +** functionality is intended for use in uploading database files. +** +** If pVfs is NULL, sqlite3_vfs_find(0) is used. +** +** If zFile is NULL, pVfs is NULL (and sqlite3_vfs_find(0) returns +** NULL), or nData is negative, SQLITE_MISUSE are returned. +** +** On success, it creates a new file with the given name, populated +** with the fist nData bytes of pData. If pData is NULL, the file is +** created and/or truncated to nData bytes. +** +** Whether or not directory components of zFilename are created +** automatically or not is unspecified: that detail is left to the +** VFS. The "opfs" VFS, for example, create them. +** +** Not all VFSes support this functionality, e.g. the "kvvfs" does +** not. +** +** If an error happens while populating or truncating the file, the +** target file will be deleted (if needed) if this function created +** it. If this function did not create it, it is not deleted but may +** be left in an undefined state. +** +** Returns 0 on success. On error, it returns a code described above +** or propagates a code from one of the I/O methods. +** +** Design note: nData is an integer, instead of int64, for WASM +** portability, so that the API can still work in builds where BigInt +** support is disabled or unavailable. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_vfs_create_file( sqlite3_vfs *pVfs, + const char *zFilename, + const unsigned char * pData, + int nData ){ + int rc; + sqlite3_file *pFile = 0; + sqlite3_io_methods const *pIo; + const int openFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + int flagsOut = 0; + int fileExisted = 0; + int doUnlock = 0; + const unsigned char *pPos = pData; + const int blockSize = 512 + /* Because we are using pFile->pMethods->xWrite() for writing, and + ** it may have a buffer limit related to sqlite3's pager size, we + ** conservatively write in 512-byte blocks (smallest page + ** size). */; + + if( !pVfs ) pVfs = sqlite3_vfs_find(0); + if( !pVfs || !zFilename || nData<0 ) return SQLITE_MISUSE; + pVfs->xAccess(pVfs, zFilename, SQLITE_ACCESS_EXISTS, &fileExisted); + rc = sqlite3OsOpenMalloc(pVfs, zFilename, &pFile, openFlags, &flagsOut); + if(rc) return rc; + pIo = pFile->pMethods; + if( pIo->xLock ) { + /* We need xLock() in order to accommodate the OPFS VFS, as it + ** obtains a writeable handle via the lock operation and releases + ** it in xUnlock(). If we don't do those here, we have to add code + ** to the VFS to account check whether it was locked before + ** xFileSize(), xTruncate(), and the like, and release the lock + ** only if it was unlocked when the op was started. */ + rc = pIo->xLock(pFile, SQLITE_LOCK_EXCLUSIVE); + doUnlock = 0==rc; + } + if( 0==rc) rc = pIo->xTruncate(pFile, nData); + if( 0==rc && 0!=pData && nData>0 ){ + while( 0==rc && nData>0 ){ + const int n = nData>=blockSize ? blockSize : nData; + rc = pIo->xWrite(pFile, pPos, n, (sqlite3_int64)(pPos - pData)); + nData -= n; + pPos += n; + } + if( 0==rc && nData>0 ){ + assert( nData<blockSize ); + rc = pIo->xWrite(pFile, pPos, nData, (sqlite3_int64)(pPos - pData)); + } + } + if( pIo->xUnlock && doUnlock!=0 ) pIo->xUnlock(pFile, SQLITE_LOCK_NONE); + pIo->xClose(pFile); + if( rc!=0 && 0==fileExisted ){ + pVfs->xDelete(pVfs, zFilename, 1); + } + return rc; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Allocates sqlite3KvvfsMethods.nKeySize bytes from +** sqlite3_wasm_pstack_alloc() and returns 0 if that allocation fails, +** else it passes that string to kvstorageMakeKey() and returns a +** NUL-terminated pointer to that string. It is up to the caller to +** use sqlite3_wasm_pstack_restore() to free the returned pointer. +*/ +SQLITE_WASM_KEEP +char * sqlite3_wasm_kvvfsMakeKeyOnPstack(const char *zClass, + const char *zKeyIn){ + assert(sqlite3KvvfsMethods.nKeySize>24); + char *zKeyOut = + (char *)sqlite3_wasm_pstack_alloc(sqlite3KvvfsMethods.nKeySize); + if(zKeyOut){ + kvstorageMakeKey(zClass, zKeyIn, zKeyOut); + } + return zKeyOut; +} + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings. +** +** Returns the pointer to the singleton object which holds the kvvfs +** I/O methods and associated state. +*/ +SQLITE_WASM_KEEP +sqlite3_kvvfs_methods * sqlite3_wasm_kvvfs_methods(void){ + return &sqlite3KvvfsMethods; +} + +#if defined(__EMSCRIPTEN__) && defined(SQLITE_ENABLE_WASMFS) +#include <emscripten/wasmfs.h> + +/* +** This function is NOT part of the sqlite3 public API. It is strictly +** for use by the sqlite project's own JS/WASM bindings, specifically +** only when building with Emscripten's WASMFS support. +** +** This function should only be called if the JS side detects the +** existence of the Origin-Private FileSystem (OPFS) APIs in the +** client. The first time it is called, this function instantiates a +** WASMFS backend impl for OPFS. On success, subsequent calls are +** no-ops. +** +** This function may be passed a "mount point" name, which must have a +** leading "/" and is currently restricted to a single path component, +** e.g. "/foo" is legal but "/foo/" and "/foo/bar" are not. If it is +** NULL or empty, it defaults to "/opfs". +** +** Returns 0 on success, SQLITE_NOMEM if instantiation of the backend +** object fails, SQLITE_IOERR if mkdir() of the zMountPoint dir in +** the virtual FS fails. In builds compiled without SQLITE_ENABLE_WASMFS +** defined, SQLITE_NOTFOUND is returned without side effects. +*/ +SQLITE_WASM_KEEP +int sqlite3_wasm_init_wasmfs(const char *zMountPoint){ + static backend_t pOpfs = 0; + if( !zMountPoint || !*zMountPoint ) zMountPoint = "/opfs"; + if( !pOpfs ){ + pOpfs = wasmfs_create_opfs_backend(); + } + /** It's not enough to instantiate the backend. We have to create a + mountpoint in the VFS and attach the backend to it. */ + if( pOpfs && 0!=access(zMountPoint, F_OK) ){ + /* Note that this check and is not robust but it will + hypothetically suffice for the transient wasm-based virtual + filesystem we're currently running in. */ + const int rc = wasmfs_create_directory(zMountPoint, 0777, pOpfs); + /*emscripten_console_logf("OPFS mkdir(%s) rc=%d", zMountPoint, rc);*/ + if(rc) return SQLITE_IOERR; + } + return pOpfs ? 0 : SQLITE_NOMEM; +} +#else +SQLITE_WASM_KEEP +int sqlite3_wasm_init_wasmfs(const char *zUnused){ + //emscripten_console_warn("WASMFS OPFS is not compiled in."); + if(zUnused){/*unused*/} + return SQLITE_NOTFOUND; +} +#endif /* __EMSCRIPTEN__ && SQLITE_ENABLE_WASMFS */ + +#if SQLITE_WASM_TESTS + +SQLITE_WASM_KEEP +int sqlite3_wasm_test_intptr(int * p){ + return *p = *p * 2; +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64_max(void){ + return (int64_t)0x7fffffffffffffff; +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64_min(void){ + return ~sqlite3_wasm_test_int64_max(); +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64_times2(int64_t x){ + return x * 2; +} + +SQLITE_WASM_KEEP +void sqlite3_wasm_test_int64_minmax(int64_t * min, int64_t *max){ + *max = sqlite3_wasm_test_int64_max(); + *min = sqlite3_wasm_test_int64_min(); + /*printf("minmax: min=%lld, max=%lld\n", *min, *max);*/ +} + +SQLITE_WASM_KEEP +int64_t sqlite3_wasm_test_int64ptr(int64_t * p){ + /*printf("sqlite3_wasm_test_int64ptr( @%lld = 0x%llx )\n", (int64_t)p, *p);*/ + return *p = *p * 2; +} + +SQLITE_WASM_KEEP +void sqlite3_wasm_test_stack_overflow(int recurse){ + if(recurse) sqlite3_wasm_test_stack_overflow(recurse); +} + +/* For testing the 'string-free' whwasmutil.xWrap() conversion. */ +SQLITE_WASM_KEEP +char * sqlite3_wasm_test_str_hello(int fail){ + char * s = fail ? 0 : (char *)malloc(6); + if(s){ + memcpy(s, "hello", 5); + s[5] = 0; + } + return s; +} +#endif /* SQLITE_WASM_TESTS */ + +#undef SQLITE_WASM_KEEP diff --git a/ext/wasm/api/sqlite3-worker1-promiser.js b/ext/wasm/api/sqlite3-worker1-promiser.js new file mode 100644 index 0000000..7360512 --- /dev/null +++ b/ext/wasm/api/sqlite3-worker1-promiser.js @@ -0,0 +1,259 @@ +/* + 2022-08-24 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This file implements a Promise-based proxy for the sqlite3 Worker + API #1. It is intended to be included either from the main thread or + a Worker, but only if (A) the environment supports nested Workers + and (B) it's _not_ a Worker which loads the sqlite3 WASM/JS + module. This file's features will load that module and provide a + slightly simpler client-side interface than the slightly-lower-level + Worker API does. + + This script necessarily exposes one global symbol, but clients may + freely `delete` that symbol after calling it. +*/ +'use strict'; +/** + Configures an sqlite3 Worker API #1 Worker such that it can be + manipulated via a Promise-based interface and returns a factory + function which returns Promises for communicating with the worker. + This proxy has an _almost_ identical interface to the normal + worker API, with any exceptions documented below. + + It requires a configuration object with the following properties: + + - `worker` (required): a Worker instance which loads + `sqlite3-worker1.js` or a functional equivalent. Note that the + promiser factory replaces the worker.onmessage property. This + config option may alternately be a function, in which case this + function re-assigns this property with the result of calling that + function, enabling delayed instantiation of a Worker. + + - `onready` (optional, but...): this callback is called with no + arguments when the worker fires its initial + 'sqlite3-api'/'worker1-ready' message, which it does when + sqlite3.initWorker1API() completes its initialization. This is + the simplest way to tell the worker to kick off work at the + earliest opportunity. + + - `onunhandled` (optional): a callback which gets passed the + message event object for any worker.onmessage() events which + are not handled by this proxy. Ideally that "should" never + happen, as this proxy aims to handle all known message types. + + - `generateMessageId` (optional): a function which, when passed an + about-to-be-posted message object, generates a _unique_ message ID + for the message, which this API then assigns as the messageId + property of the message. It _must_ generate unique IDs on each call + so that dispatching can work. If not defined, a default generator + is used (which should be sufficient for most or all cases). + + - `debug` (optional): a console.debug()-style function for logging + information about messages. + + This function returns a stateful factory function with the + following interfaces: + + - Promise function(messageType, messageArgs) + - Promise function({message object}) + + The first form expects the "type" and "args" values for a Worker + message. The second expects an object in the form {type:..., + args:...} plus any other properties the client cares to set. This + function will always set the `messageId` property on the object, + even if it's already set, and will set the `dbId` property to the + current database ID if it is _not_ set in the message object. + + The function throws on error. + + The function installs a temporary message listener, posts a + message to the configured Worker, and handles the message's + response via the temporary message listener. The then() callback + of the returned Promise is passed the `message.data` property from + the resulting message, i.e. the payload from the worker, stripped + of the lower-level event state which the onmessage() handler + receives. + + Example usage: + + ``` + const config = {...}; + const sq3Promiser = sqlite3Worker1Promiser(config); + sq3Promiser('open', {filename:"/foo.db"}).then(function(msg){ + console.log("open response",msg); // => {type:'open', result: {filename:'/foo.db'}, ...} + }); + sq3Promiser({type:'close'}).then((msg)=>{ + console.log("close response",msg); // => {type:'close', result: {filename:'/foo.db'}, ...} + }); + ``` + + Differences from Worker API #1: + + - exec's {callback: STRING} option does not work via this + interface (it triggers an exception), but {callback: function} + does and works exactly like the STRING form does in the Worker: + the callback is called one time for each row of the result set, + passed the same worker message format as the worker API emits: + + {type:typeString, + row:VALUE, + rowNumber:1-based-#, + columnNames: array} + + Where `typeString` is an internally-synthesized message type string + used temporarily for worker message dispatching. It can be ignored + by all client code except that which tests this API. The `row` + property contains the row result in the form implied by the + `rowMode` option (defaulting to `'array'`). The `rowNumber` is a + 1-based integer value incremented by 1 on each call into th + callback. + + At the end of the result set, the same event is fired with + (row=undefined, rowNumber=null) to indicate that + the end of the result set has been reached. Note that the rows + arrive via worker-posted messages, with all the implications + of that. +*/ +self.sqlite3Worker1Promiser = function callee(config = callee.defaultConfig){ + // Inspired by: https://stackoverflow.com/a/52439530 + if(1===arguments.length && 'function'===typeof arguments[0]){ + const f = config; + config = Object.assign(Object.create(null), callee.defaultConfig); + config.onready = f; + }else{ + config = Object.assign(Object.create(null), callee.defaultConfig, config); + } + const handlerMap = Object.create(null); + const noop = function(){}; + const err = config.onerror + || noop /* config.onerror is intentionally undocumented + pending finding a less ambiguous name */; + const debug = config.debug || noop; + const idTypeMap = config.generateMessageId ? undefined : Object.create(null); + const genMsgId = config.generateMessageId || function(msg){ + return msg.type+'#'+(idTypeMap[msg.type] = (idTypeMap[msg.type]||0) + 1); + }; + const toss = (...args)=>{throw new Error(args.join(' '))}; + if(!config.worker) config.worker = callee.defaultConfig.worker; + if('function'===typeof config.worker) config.worker = config.worker(); + let dbId; + config.worker.onmessage = function(ev){ + ev = ev.data; + debug('worker1.onmessage',ev); + let msgHandler = handlerMap[ev.messageId]; + if(!msgHandler){ + if(ev && 'sqlite3-api'===ev.type && 'worker1-ready'===ev.result) { + /*fired one time when the Worker1 API initializes*/ + if(config.onready) config.onready(); + return; + } + msgHandler = handlerMap[ev.type] /* check for exec per-row callback */; + if(msgHandler && msgHandler.onrow){ + msgHandler.onrow(ev); + return; + } + if(config.onunhandled) config.onunhandled(arguments[0]); + else err("sqlite3Worker1Promiser() unhandled worker message:",ev); + return; + } + delete handlerMap[ev.messageId]; + switch(ev.type){ + case 'error': + msgHandler.reject(ev); + return; + case 'open': + if(!dbId) dbId = ev.dbId; + break; + case 'close': + if(ev.dbId===dbId) dbId = undefined; + break; + default: + break; + } + try {msgHandler.resolve(ev)} + catch(e){msgHandler.reject(e)} + }/*worker.onmessage()*/; + return function(/*(msgType, msgArgs) || (msgEnvelope)*/){ + let msg; + if(1===arguments.length){ + msg = arguments[0]; + }else if(2===arguments.length){ + msg = { + type: arguments[0], + args: arguments[1] + }; + }else{ + toss("Invalid arugments for sqlite3Worker1Promiser()-created factory."); + } + if(!msg.dbId) msg.dbId = dbId; + msg.messageId = genMsgId(msg); + msg.departureTime = performance.now(); + const proxy = Object.create(null); + proxy.message = msg; + let rowCallbackId /* message handler ID for exec on-row callback proxy */; + if('exec'===msg.type && msg.args){ + if('function'===typeof msg.args.callback){ + rowCallbackId = msg.messageId+':row'; + proxy.onrow = msg.args.callback; + msg.args.callback = rowCallbackId; + handlerMap[rowCallbackId] = proxy; + }else if('string' === typeof msg.args.callback){ + toss("exec callback may not be a string when using the Promise interface."); + /** + Design note: the reason for this limitation is that this + API takes over worker.onmessage() and the client has no way + of adding their own message-type handlers to it. Per-row + callbacks are implemented as short-lived message.type + mappings for worker.onmessage(). + + We "could" work around this by providing a new + config.fallbackMessageHandler (or some such) which contains + a map of event type names to callbacks. Seems like overkill + for now, seeing as the client can pass callback functions + to this interface (whereas the string-form "callback" is + needed for the over-the-Worker interface). + */ + } + } + //debug("requestWork", msg); + let p = new Promise(function(resolve, reject){ + proxy.resolve = resolve; + proxy.reject = reject; + handlerMap[msg.messageId] = proxy; + debug("Posting",msg.type,"message to Worker dbId="+(dbId||'default')+':',msg); + config.worker.postMessage(msg); + }); + if(rowCallbackId) p = p.finally(()=>delete handlerMap[rowCallbackId]); + return p; + }; +}/*sqlite3Worker1Promiser()*/; +self.sqlite3Worker1Promiser.defaultConfig = { + worker: function(){ + let theJs = "sqlite3-worker1.js"; + if(this.currentScript){ + const src = this.currentScript.src.split('/'); + src.pop(); + theJs = src.join('/')+'/' + theJs; + //console.warn("promiser currentScript, theJs =",this.currentScript,theJs); + }else{ + //console.warn("promiser self.location =",self.location); + const urlParams = new URL(self.location.href).searchParams; + if(urlParams.has('sqlite3.dir')){ + theJs = urlParams.get('sqlite3.dir') + '/' + theJs; + } + } + return new Worker(theJs + self.location.search); + }.bind({ + currentScript: self?.document?.currentScript + }), + onerror: (...args)=>console.error('worker1 promiser error',...args) +}; diff --git a/ext/wasm/api/sqlite3-worker1.js b/ext/wasm/api/sqlite3-worker1.js new file mode 100644 index 0000000..9424379 --- /dev/null +++ b/ext/wasm/api/sqlite3-worker1.js @@ -0,0 +1,49 @@ +/* + 2022-05-23 + + The author disclaims copyright to this source code. In place of a + legal notice, here is a blessing: + + * May you do good and not evil. + * May you find forgiveness for yourself and forgive others. + * May you share freely, never taking more than you give. + + *********************************************************************** + + This is a JS Worker file for the main sqlite3 api. It loads + sqlite3.js, initializes the module, and postMessage()'s a message + after the module is initialized: + + {type: 'sqlite3-api', result: 'worker1-ready'} + + This seemingly superfluous level of indirection is necessary when + loading sqlite3.js via a Worker. Instantiating a worker with new + Worker("sqlite.js") will not (cannot) call sqlite3InitModule() to + initialize the module due to a timing/order-of-operations conflict + (and that symbol is not exported in a way that a Worker loading it + that way can see it). Thus JS code wanting to load the sqlite3 + Worker-specific API needs to pass _this_ file (or equivalent) to the + Worker constructor and then listen for an event in the form shown + above in order to know when the module has completed initialization. + + This file accepts a URL arguments to adjust how it loads sqlite3.js: + + - `sqlite3.dir`, if set, treats the given directory name as the + directory from which `sqlite3.js` will be loaded. +*/ +"use strict"; +(()=>{ + const urlParams = new URL(self.location.href).searchParams; + let theJs = 'sqlite3.js'; + if(urlParams.has('sqlite3.dir')){ + theJs = urlParams.get('sqlite3.dir') + '/' + theJs; + } + //console.warn("worker1 theJs =",theJs); + importScripts(theJs); + sqlite3InitModule().then((sqlite3)=>{ + if(sqlite3.capi.sqlite3_wasmfs_opfs_dir){ + sqlite3.capi.sqlite3_wasmfs_opfs_dir(); + } + sqlite3.initWorker1API(); + }); +})(); |