Adding upstream version 1.43.0.upstream/1.43.0

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

40 files changed, 11696 insertions, 4239 deletions

diff --git a/database/contexts/api_v1.c b/database/contexts/api_v1.c
index bc7fee496..f144e6f7b 100644
--- a/database/contexts/api_v1.c
+++ b/database/contexts/api_v1.c
@@ -213,7 +213,7 @@ static inline int rrdinstance_to_json_callback(const DICTIONARY_ITEM *item, void
         buffer_json_array_close(wb);
     }
 
-    if(options & RRDCONTEXT_OPTION_SHOW_LABELS && ri->rrdlabels && dictionary_entries(ri->rrdlabels)) {
+    if(options & RRDCONTEXT_OPTION_SHOW_LABELS && ri->rrdlabels && rrdlabels_entries(ri->rrdlabels)) {
         buffer_json_member_add_object(wb, "labels");
         rrdlabels_to_buffer_json_members(ri->rrdlabels, wb);
         buffer_json_object_close(wb);
@@ -366,7 +366,7 @@ int rrdcontext_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before, R
     RRDCONTEXT *rc = rrdcontext_acquired_value(rca);
 
     if(after != 0 && before != 0)
-        rrdr_relative_window_to_absolute(&after, &before, NULL, false);
+        rrdr_relative_window_to_absolute_query(&after, &before, NULL, false);
 
     buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     struct rrdcontext_to_json t_contexts = {
@@ -403,7 +403,7 @@ int rrdcontexts_to_json(RRDHOST *host, BUFFER *wb, time_t after, time_t before,
         uuid_unparse(*host->node_id, node_uuid);
 
     if(after != 0 && before != 0)
-        rrdr_relative_window_to_absolute(&after, &before, NULL, false);
+        rrdr_relative_window_to_absolute_query(&after, &before, NULL, false);
 
     buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
     buffer_json_member_add_string(wb, "hostname", rrdhost_hostname(host));
diff --git a/database/contexts/api_v2.c b/database/contexts/api_v2.c
index 08739160d..d0b27a2aa 100644
--- a/database/contexts/api_v2.c
+++ b/database/contexts/api_v2.c
@@ -184,6 +184,7 @@ struct alert_v2_entry {
     RRDCALC *tmp;
 
     STRING *name;
+    STRING *summary;
 
     size_t ati;
 
@@ -315,6 +316,7 @@ static void alerts_v2_insert_callback(const DICTIONARY_ITEM *item __maybe_unused
     struct alert_v2_entry *t = value;
     RRDCALC *rc = t->tmp;
     t->name = rc->name;
+    t->summary = rc->summary;
     t->ati = ctl->alerts.ati++;
 
     t->nodes = dictionary_create(DICT_OPTION_SINGLE_THREADED|DICT_OPTION_VALUE_LINK_DONT_CLONE|DICT_OPTION_NAME_LINK_DONT_CLONE);
@@ -355,6 +357,7 @@ static void alert_instances_v2_insert_callback(const DICTIONARY_ITEM *item __may
     t->status = rc->status;
     t->flags = rc->run_flags;
     t->info = rc->info;
+    t->summary = rc->summary;
     t->value = rc->value;
     t->last_updated = rc->last_updated;
     t->last_status_change = rc->last_status_change;
@@ -418,7 +421,7 @@ static FTS_MATCH rrdcontext_to_json_v2_full_text_search(struct rrdcontext_to_jso
         dfe_done(rm);
 
         size_t label_searches = 0;
-        if(unlikely(ri->rrdlabels && dictionary_entries(ri->rrdlabels) &&
+        if(unlikely(ri->rrdlabels && rrdlabels_entries(ri->rrdlabels) &&
                     rrdlabels_match_simple_pattern_parsed(ri->rrdlabels, q, ':', &label_searches))) {
             ctl->q.fts.searches += label_searches;
             ctl->q.fts.char_searches += label_searches;
@@ -1009,8 +1012,8 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
             STORAGE_ENGINE *eng = localhost->db[tier].eng;
             if (!eng) continue;
 
-            size_t max = storage_engine_disk_space_max(eng->backend, localhost->db[tier].instance);
-            size_t used = storage_engine_disk_space_used(eng->backend, localhost->db[tier].instance);
+            uint64_t max = storage_engine_disk_space_max(eng->backend, localhost->db[tier].instance);
+            uint64_t used = storage_engine_disk_space_used(eng->backend, localhost->db[tier].instance);
             time_t first_time_s = storage_engine_global_first_time_s(eng->backend, localhost->db[tier].instance);
             size_t currently_collected_metrics = storage_engine_collected_metrics(eng->backend, localhost->db[tier].instance);
 
@@ -1280,6 +1283,7 @@ static void contexts_v2_alert_config_to_json_from_sql_alert_config_data(struct s
         buffer_json_member_add_string(wb, "component", t->component);
         buffer_json_member_add_string(wb, "type", t->type);
         buffer_json_member_add_string(wb, "info", t->info);
+        buffer_json_member_add_string(wb, "summary", t->summary);
         // buffer_json_member_add_string(wb, "source", t->source); // moved to alert instance
     }
     
@@ -1343,6 +1347,7 @@ static int contexts_v2_alert_instance_to_json_callback(const DICTIONARY_ITEM *it
             buffer_json_member_add_string(wb, "units", string2str(t->units));
             buffer_json_member_add_string(wb, "fami", string2str(t->family));
             buffer_json_member_add_string(wb, "info", string2str(t->info));
+            buffer_json_member_add_string(wb, "sum", string2str(t->summary));
             buffer_json_member_add_string(wb, "ctx", string2str(t->context));
             buffer_json_member_add_string(wb, "st", rrdcalc_status2string(t->status));
             buffer_json_member_add_uuid(wb, "tr_i", &t->last_transition_id);
@@ -1397,6 +1402,7 @@ static void contexts_v2_alerts_to_json(BUFFER *wb, struct rrdcontext_to_json_v2_
                         {
                             buffer_json_member_add_uint64(wb, "ati", t->ati);
                             buffer_json_member_add_string(wb, "nm", string2str(t->name));
+                            buffer_json_member_add_string(wb, "sum", string2str(t->summary));
 
                             buffer_json_member_add_uint64(wb, "cr", t->critical);
                             buffer_json_member_add_uint64(wb, "wr", t->warning);
@@ -1438,6 +1444,7 @@ struct sql_alert_transition_fixed_size {
     char units[SQL_TRANSITION_DATA_SMALL_STRING];
     char exec[SQL_TRANSITION_DATA_BIG_STRING];
     char info[SQL_TRANSITION_DATA_BIG_STRING];
+    char summary[SQL_TRANSITION_DATA_BIG_STRING];
     char classification[SQL_TRANSITION_DATA_SMALL_STRING];
     char type[SQL_TRANSITION_DATA_SMALL_STRING];
     char component[SQL_TRANSITION_DATA_SMALL_STRING];
@@ -1477,6 +1484,7 @@ static struct sql_alert_transition_fixed_size *contexts_v2_alert_transition_dup(
     strncpyz(n->units, t->units ? t->units : "", sizeof(n->units) - 1);
     strncpyz(n->exec, t->exec ? t->exec : "", sizeof(n->exec) - 1);
     strncpyz(n->info, t->info ? t->info : "", sizeof(n->info) - 1);
+    strncpyz(n->summary, t->summary ? t->summary : "", sizeof(n->summary) - 1);
     strncpyz(n->classification, t->classification ? t->classification : "", sizeof(n->classification) - 1);
     strncpyz(n->type, t->type ? t->type : "", sizeof(n->type) - 1);
     strncpyz(n->component, t->component ? t->component : "", sizeof(n->component) - 1);
@@ -1734,6 +1742,7 @@ static void contexts_v2_alert_transitions_to_json(BUFFER *wb, struct rrdcontext_
 
             buffer_json_member_add_time_t(wb, "when", t->when_key);
             buffer_json_member_add_string(wb, "info", *t->info ? t->info : "");
+            buffer_json_member_add_string(wb, "summary", *t->summary ? t->summary : "");
             buffer_json_member_add_string(wb, "units", *t->units ? t->units : NULL);
             buffer_json_member_add_object(wb, "new");
             {
@@ -1934,7 +1943,9 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
     }
 
     if(req->after || req->before) {
-        ctl.window.relative = rrdr_relative_window_to_absolute(&ctl.window.after, &ctl.window.before, &ctl.now, false);
+        ctl.window.relative = rrdr_relative_window_to_absolute_query(&ctl.window.after, &ctl.window.before, &ctl.now
+                                                                     , false
+                                                                    );
         ctl.window.enabled = !(mode & CONTEXTS_V2_ALERT_TRANSITIONS);
     }
     else
@@ -2023,7 +2034,7 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
         }
         else {
             buffer_strcat(wb, "query interrupted");
-            resp = HTTP_RESP_BACKEND_FETCH_FAILED;
+            resp = HTTP_RESP_CLIENT_CLOSED_REQUEST;
         }
         goto cleanup;
     }
diff --git a/database/contexts/instance.c b/database/contexts/instance.c
index 7e572fb80..8a60ce662 100644
--- a/database/contexts/instance.c
+++ b/database/contexts/instance.c
@@ -35,7 +35,7 @@ inline STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria) {
     return string_dup(ri->units);
 }
 
-inline DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) {
+inline RRDLABELS *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria) {
     RRDINSTANCE *ri = rrdinstance_acquired_value(ria);
     return ri->rrdlabels;
 }
@@ -68,7 +68,7 @@ inline time_t rrdinstance_acquired_update_every(RRDINSTANCE_ACQUIRED *ria) {
 static void rrdinstance_free(RRDINSTANCE *ri) {
 
     if(rrd_flag_check(ri, RRD_FLAG_OWN_LABELS))
-        dictionary_destroy(ri->rrdlabels);
+        rrdlabels_destroy(ri->rrdlabels);
 
     rrdmetrics_destroy_from_rrdinstance(ri);
     string_freez(ri->id);
@@ -211,7 +211,7 @@ static bool rrdinstance_conflict_callback(const DICTIONARY_ITEM *item __maybe_un
         ri->rrdset = ri_new->rrdset;
 
         if(ri->rrdset && rrd_flag_check(ri, RRD_FLAG_OWN_LABELS)) {
-            DICTIONARY *old = ri->rrdlabels;
+            RRDLABELS *old = ri->rrdlabels;
             ri->rrdlabels = ri->rrdset->rrdlabels;
             rrd_flag_clear(ri, RRD_FLAG_OWN_LABELS);
             rrdlabels_destroy(old);
@@ -494,7 +494,7 @@ inline void rrdinstance_updated_rrdset_flags(RRDSET *st) {
     RRDINSTANCE *ri = rrdset_get_rrdinstance(st);
     if(unlikely(!ri)) return;
 
-    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED|RRDSET_FLAG_OBSOLETE)))
+    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))
         rrd_flag_set_archived(ri);
 
     rrdinstance_updated_rrdset_flags_no_action(ri, st);
diff --git a/database/contexts/internal.h b/database/contexts/internal.h
index 04ad0883a..293659fdd 100644
--- a/database/contexts/internal.h
+++ b/database/contexts/internal.h
@@ -230,7 +230,7 @@ typedef struct rrdinstance {
     time_t update_every_s;              // data collection frequency
     RRDSET *rrdset;                     // pointer to RRDSET when collected, or NULL
 
-    DICTIONARY *rrdlabels;              // linked to RRDSET->chart_labels or own version
+    RRDLABELS *rrdlabels;              // linked to RRDSET->chart_labels or own version
 
     struct rrdcontext *rc;
     DICTIONARY *rrdmetrics;
diff --git a/database/contexts/query_target.c b/database/contexts/query_target.c
index 829640b90..d969691dd 100644
--- a/database/contexts/query_target.c
+++ b/database/contexts/query_target.c
@@ -1052,8 +1052,9 @@ QUERY_TARGET *query_target_create(QUERY_TARGET_REQUEST *qtr) {
     if(query_target_has_percentage_of_group(qt))
         qt->window.options &= ~RRDR_OPTION_PERCENTAGE;
 
-    qt->internal.relative = rrdr_relative_window_to_absolute(&qt->window.after, &qt->window.before, &qt->window.now,
-                                                             unittest_running);
+    qt->internal.relative = rrdr_relative_window_to_absolute_query(&qt->window.after, &qt->window.before
+                                                                   , &qt->window.now, unittest_running
+                                                                  );
 
     // prepare our local variables - we need these across all these functions
     QUERY_TARGET_LOCALS qtl = {
diff --git a/database/contexts/rrdcontext.h b/database/contexts/rrdcontext.h
index 0bcdb68de..9c497a5a5 100644
--- a/database/contexts/rrdcontext.h
+++ b/database/contexts/rrdcontext.h
@@ -40,7 +40,7 @@ const char *rrdinstance_acquired_name(RRDINSTANCE_ACQUIRED *ria);
 bool rrdinstance_acquired_has_name(RRDINSTANCE_ACQUIRED *ria);
 const char *rrdinstance_acquired_units(RRDINSTANCE_ACQUIRED *ria);
 STRING *rrdinstance_acquired_units_dup(RRDINSTANCE_ACQUIRED *ria);
-DICTIONARY *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria);
+RRDLABELS *rrdinstance_acquired_labels(RRDINSTANCE_ACQUIRED *ria);
 DICTIONARY *rrdinstance_acquired_functions(RRDINSTANCE_ACQUIRED *ria);
 RRDHOST *rrdinstance_acquired_rrdhost(RRDINSTANCE_ACQUIRED *ria);
 RRDSET *rrdinstance_acquired_rrdset(RRDINSTANCE_ACQUIRED *ria);
@@ -432,6 +432,7 @@ struct sql_alert_transition_data {
     const char *units;
     const char *exec;
     const char *info;
+    const char *summary;
     const char *classification;
     const char *type;
     const char *component;
@@ -472,6 +473,7 @@ struct sql_alert_config_data {
     const char *classification;
     const char *component;
     const char *type;
+    const char *summary;
 
     struct {
         struct {
@@ -531,6 +533,7 @@ struct sql_alert_instance_v2_entry {
     RRDCALC_STATUS status;
     RRDCALC_FLAGS flags;
     STRING *info;
+    STRING *summary;
     NETDATA_DOUBLE value;
     time_t last_updated;
     time_t last_status_change;
diff --git a/database/engine/cache.h b/database/engine/cache.h
index 1486fdc16..c10e09928 100644
--- a/database/engine/cache.h
+++ b/database/engine/cache.h
@@ -71,8 +71,8 @@ struct pgc_statistics {
 
     PGC_CACHE_LINE_PADDING(3);
 
-    size_t entries;                 // all the entries (includes clean, dirty, host)
-    size_t size;                    // all the entries (includes clean, dirty, host)
+    size_t entries;                 // all the entries (includes clean, dirty, hot)
+    size_t size;                    // all the entries (includes clean, dirty, hot)
 
     size_t evicting_entries;
     size_t evicting_size;
diff --git a/database/engine/datafile.c b/database/engine/datafile.c
index d5c1285be..fcda84bd6 100644
--- a/database/engine/datafile.c
+++ b/database/engine/datafile.c
@@ -112,7 +112,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
                                          "but it has %u lockers (oc:%u, pd:%u), "
                                          "%zu clean and %zu hot open cache pages "
                                          "- will be deleted shortly "
-                                         "(scanned open cache in %llu usecs)",
+                                         "(scanned open cache in %"PRIu64" usecs)",
                                    df->fileno, df->ctx->config.tier,
                                    df->users.lockers,
                                    df->users.lockers_by_reason[DATAFILE_ACQUIRE_OPEN_CACHE],
@@ -129,7 +129,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
                                          "but it has %u lockers (oc:%u, pd:%u), "
                                          "%zu clean and %zu hot open cache pages "
                                          "- will be deleted now "
-                                         "(scanned open cache in %llu usecs)",
+                                         "(scanned open cache in %"PRIu64" usecs)",
                                    df->fileno, df->ctx->config.tier,
                                    df->users.lockers,
                                    df->users.lockers_by_reason[DATAFILE_ACQUIRE_OPEN_CACHE],
@@ -143,7 +143,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df) {
                 internal_error(true, "DBENGINE: datafile %u of tier %d "
                                      "has %u lockers (oc:%u, pd:%u), "
                                      "%zu clean and %zu hot open cache pages "
-                                     "(scanned open cache in %llu usecs)",
+                                     "(scanned open cache in %"PRIu64" usecs)",
                                df->fileno, df->ctx->config.tier,
                                df->users.lockers,
                                df->users.lockers_by_reason[DATAFILE_ACQUIRE_OPEN_CACHE],
diff --git a/database/engine/metric.c b/database/engine/metric.c
index 0b248c09b..69b8f3116 100644
--- a/database/engine/metric.c
+++ b/database/engine/metric.c
@@ -923,7 +923,7 @@ int mrg_unittest(void) {
     netdata_log_info("DBENGINE METRIC: did %zu additions, %zu duplicate additions, "
          "%zu deletions, %zu wrong deletions, "
          "%zu successful searches, %zu wrong searches, "
-         "in %llu usecs",
+         "in %"PRIu64" usecs",
         stats.additions, stats.additions_duplicate,
         stats.deletions, stats.delete_misses,
         stats.search_hits, stats.search_misses,
diff --git a/database/engine/rrdengineapi.c b/database/engine/rrdengineapi.c
index c6b1fa2dd..318a933f1 100755
--- a/database/engine/rrdengineapi.c
+++ b/database/engine/rrdengineapi.c
@@ -534,7 +534,8 @@ static void rrdeng_store_metric_append_point(STORAGE_COLLECT_HANDLE *collection_
     timing_step(TIMING_STEP_DBENGINE_MRG_UPDATE);
 }
 
-static void store_metric_next_error_log(struct rrdeng_collect_handle *handle, usec_t point_in_time_ut, const char *msg) {
+static void store_metric_next_error_log(struct rrdeng_collect_handle *handle __maybe_unused, usec_t point_in_time_ut __maybe_unused, const char *msg __maybe_unused) {
+#ifdef NETDATA_INTERNAL_CHECKS
     time_t point_in_time_s = (time_t)(point_in_time_ut / USEC_PER_SEC);
     char uuid[UUID_STR_LEN + 1];
     uuid_unparse(*mrg_metric_uuid(main_mrg, handle->metric), uuid);
@@ -562,6 +563,9 @@ static void store_metric_next_error_log(struct rrdeng_collect_handle *handle, us
     );
 
     buffer_free(wb);
+#else
+    ;
+#endif
 }
 
 void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle,
@@ -808,12 +812,14 @@ static bool rrdeng_load_page_next(struct storage_engine_query_handle *rrddim_han
     if (unlikely(handle->now_s > rrddim_handle->end_time_s))
         return false;
 
-    size_t entries;
+    size_t entries = 0;
     handle->page = pg_cache_lookup_next(ctx, handle->pdc, handle->now_s, handle->dt_s, &entries);
-    if (unlikely(!handle->page))
-        return false;
 
-    internal_fatal(pgc_page_data(handle->page) == DBENGINE_EMPTY_PAGE, "Empty page returned");
+    internal_fatal(handle->page && (pgc_page_data(handle->page) == DBENGINE_EMPTY_PAGE || !entries),
+                   "A page was returned, but it is empty - pg_cache_lookup_next() should be handling this case");
+
+    if (unlikely(!handle->page || pgc_page_data(handle->page) == DBENGINE_EMPTY_PAGE || !entries))
+        return false;
 
     time_t page_start_time_s = pgc_page_start_time_s(handle->page);
     time_t page_end_time_s = pgc_page_end_time_s(handle->page);
@@ -1002,12 +1008,12 @@ bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *dim_
     return true;
 }
 
-size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance) {
+uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance) {
     struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
     return ctx->config.max_disk_space;
 }
 
-size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance) {
+uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance) {
     struct rrdengine_instance *ctx = (struct rrdengine_instance *)db_instance;
     return __atomic_load_n(&ctx->atomic.current_disk_space, __ATOMIC_RELAXED);
 }
diff --git a/database/engine/rrdengineapi.h b/database/engine/rrdengineapi.h
index 12f1becd1..61449426f 100644
--- a/database/engine/rrdengineapi.h
+++ b/database/engine/rrdengineapi.h
@@ -222,7 +222,7 @@ RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
 size_t rrdeng_collectors_running(struct rrdengine_instance *ctx);
 bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance);
 
-size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
-size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
+uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
+uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
 
 #endif /* NETDATA_RRDENGINEAPI_H */
diff --git a/database/rrd.h b/database/rrd.h
index 11f3aa98c..197ec45cb 100644
--- a/database/rrd.h
+++ b/database/rrd.h
@@ -23,6 +23,8 @@ typedef struct rrdcalc RRDCALC;
 typedef struct rrdcalctemplate RRDCALCTEMPLATE;
 typedef struct alarm_entry ALARM_ENTRY;
 
+typedef struct rrdlabels RRDLABELS;
+
 typedef struct rrdfamily_acquired RRDFAMILY_ACQUIRED;
 typedef struct rrdvar_acquired RRDVAR_ACQUIRED;
 typedef struct rrdsetvar_acquired RRDSETVAR_ACQUIRED;
@@ -113,6 +115,7 @@ struct ml_metrics_statistics {
 #include "rrddimvar.h"
 #include "rrdcalc.h"
 #include "rrdcalctemplate.h"
+#include "rrdlabels.h"
 #include "streaming/rrdpush.h"
 #include "aclk/aclk_rrdhost_state.h"
 #include "sqlite/sqlite_health.h"
@@ -265,60 +268,6 @@ typedef enum __attribute__ ((__packed__)) rrddim_flags {
 #define rrddim_flag_set(rd, flag)   __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
 #define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
 
-typedef enum __attribute__ ((__packed__)) rrdlabel_source {
-    RRDLABEL_SRC_AUTO       = (1 << 0), // set when Netdata found the label by some automation
-    RRDLABEL_SRC_CONFIG     = (1 << 1), // set when the user configured the label
-    RRDLABEL_SRC_K8S        = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
-    RRDLABEL_SRC_ACLK       = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
-
-    // more sources can be added here
-
-    RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
-    RRDLABEL_FLAG_OLD       = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
-    RRDLABEL_FLAG_NEW       = (1 << 31)  // marks for rrdlabels internal use - they are not exposed outside rrdlabels
-} RRDLABEL_SRC;
-
-#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
-
-size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
-
-DICTIONARY *rrdlabels_create(void);
-void rrdlabels_destroy(DICTIONARY *labels_dict);
-void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
-void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
-void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
-void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key);
-void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key);
-void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key);
-STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key);
-STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset);
-void rrdlabels_flush(DICTIONARY *labels_dict);
-
-void rrdlabels_unmark_all(DICTIONARY *labels);
-void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
-
-int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
-int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
-
-void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
-bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
-
-bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
-int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
-void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb);
-
-void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
-void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
-
-void reload_host_labels(void);
-void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
-void rrdset_save_rrdlabels_to_sql(RRDSET *st);
-void rrdhost_set_is_parent_label(void);
-int rrdlabels_unittest(void);
-
-// unfortunately this break when defined in exporting_engine.h
-bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
-
 // ----------------------------------------------------------------------------
 // engine-specific iterator state for dimension data collection
 typedef struct storage_collect_handle {
@@ -505,8 +454,8 @@ static inline void storage_engine_store_metric(
                                        count, anomaly_count, flags);
 }
 
-size_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
-static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+uint64_t rrdeng_disk_space_max(STORAGE_INSTANCE *db_instance);
+static inline uint64_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance __maybe_unused) {
 #ifdef ENABLE_DBENGINE
     if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
         return rrdeng_disk_space_max(db_instance);
@@ -515,8 +464,8 @@ static inline size_t storage_engine_disk_space_max(STORAGE_ENGINE_BACKEND backen
     return 0;
 }
 
-size_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
-static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+uint64_t rrdeng_disk_space_used(STORAGE_INSTANCE *db_instance);
+static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance __maybe_unused) {
 #ifdef ENABLE_DBENGINE
     if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
         return rrdeng_disk_space_used(db_instance);
@@ -527,7 +476,7 @@ static inline size_t storage_engine_disk_space_used(STORAGE_ENGINE_BACKEND backe
 }
 
 time_t rrdeng_global_first_time_s(STORAGE_INSTANCE *db_instance);
-static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance __maybe_unused) {
 #ifdef ENABLE_DBENGINE
     if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
         return rrdeng_global_first_time_s(db_instance);
@@ -537,7 +486,7 @@ static inline time_t storage_engine_global_first_time_s(STORAGE_ENGINE_BACKEND b
 }
 
 size_t rrdeng_currently_collected_metrics(STORAGE_INSTANCE *db_instance);
-static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance) {
+static inline size_t storage_engine_collected_metrics(STORAGE_ENGINE_BACKEND backend __maybe_unused, STORAGE_INSTANCE *db_instance __maybe_unused) {
 #ifdef ENABLE_DBENGINE
     if(likely(backend == STORAGE_ENGINE_BACKEND_DBENGINE))
         return rrdeng_currently_collected_metrics(db_instance);
@@ -716,13 +665,6 @@ STORAGE_ENGINE* storage_engine_find(const char* name);
 
 #define rrddim_foreach_read(rd, st) \
     dfe_start_read((st)->rrddim_root_index, rd)
-
-#define rrddim_foreach_write(rd, st) \
-    dfe_start_write((st)->rrddim_root_index, rd)
-
-#define rrddim_foreach_reentrant(rd, st) \
-    dfe_start_reentrant((st)->rrddim_root_index, rd)
-
 #define rrddim_foreach_done(rd) \
     dfe_done(rd)
 
@@ -751,9 +693,7 @@ typedef enum __attribute__ ((__packed__)) rrdset_flags {
     RRDSET_FLAG_HIDDEN                           = (1 << 12), // if set, do not show this chart on the dashboard, but use it for exporting
     RRDSET_FLAG_SYNC_CLOCK                       = (1 << 13), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
     RRDSET_FLAG_OBSOLETE_DIMENSIONS              = (1 << 14), // this is marked by the collector/module when a chart has obsolete dimensions
-                                                              // No new values have been collected for this chart since agent start, or it was marked RRDSET_FLAG_OBSOLETE at
-                                                              // least rrdset_free_obsolete_time seconds ago.
-    RRDSET_FLAG_ARCHIVED                         = (1 << 15),
+
     RRDSET_FLAG_METADATA_UPDATE                  = (1 << 16), // Mark that metadata needs to be stored
     RRDSET_FLAG_ANOMALY_DETECTION                = (1 << 18), // flag to identify anomaly detection charts.
     RRDSET_FLAG_INDEXED_ID                       = (1 << 19), // the rrdset is indexed by its id
@@ -804,7 +744,7 @@ struct rrdset {
     int32_t priority;                               // the sorting priority of this chart
     int32_t update_every;                           // data collection frequency
 
-    DICTIONARY *rrdlabels;                          // chart labels
+    RRDLABELS *rrdlabels;                           // chart labels
     DICTIONARY *rrdsetvar_root_index;               // chart variables
     DICTIONARY *rrddimvar_root_index;               // dimension variables
                                                     // we use this dictionary to manage their allocation
@@ -993,6 +933,8 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {
 
     RRDHOST_FLAG_METADATA_CLAIMID               = (1 << 28), // metadata needs to be stored in the database
     RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED  = (1 << 29), // set when the receiver part is disconnected
+
+    RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED       = (1 << 30), // set when the host has updated global functions
 } RRDHOST_FLAGS;
 
 #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
@@ -1050,7 +992,6 @@ struct alarm_entry {
     STRING *chart;
     STRING *chart_context;
     STRING *chart_name;
-    STRING *family;
 
     STRING *classification;
     STRING *component;
@@ -1064,6 +1005,7 @@ struct alarm_entry {
 
     STRING *source;
     STRING *units;
+    STRING *summary;
     STRING *info;
 
     NETDATA_DOUBLE old_value;
@@ -1094,14 +1036,12 @@ struct alarm_entry {
 #define ae_chart_id(ae) string2str((ae)->chart)
 #define ae_chart_name(ae) string2str((ae)->chart_name)
 #define ae_chart_context(ae) string2str((ae)->chart_context)
-#define ae_family(ae) string2str((ae)->family)
 #define ae_classification(ae) string2str((ae)->classification)
-#define ae_component(ae) string2str((ae)->component)
-#define ae_type(ae) string2str((ae)->type)
 #define ae_exec(ae) string2str((ae)->exec)
 #define ae_recipient(ae) string2str((ae)->recipient)
 #define ae_source(ae) string2str((ae)->source)
 #define ae_units(ae) string2str((ae)->units)
+#define ae_summary(ae) string2str((ae)->summary)
 #define ae_info(ae) string2str((ae)->info)
 #define ae_old_value_string(ae) string2str((ae)->old_value_string)
 #define ae_new_value_string(ae) string2str((ae)->new_value_string)
@@ -1117,13 +1057,14 @@ typedef struct alarm_log {
 } ALARM_LOG;
 
 typedef struct health {
-    unsigned int health_enabled;                   // 1 when this host has health enabled
     time_t health_delay_up_to;                     // a timestamp to delay alarms processing up to
     STRING *health_default_exec;                   // the full path of the alarms notifications program
     STRING *health_default_recipient;              // the default recipient for all alarms
-    size_t health_log_entries_written;             // the number of alarm events written to the alarms event log
+    int health_log_entries_written;                // the number of alarm events written to the alarms event log
     uint32_t health_default_warn_repeat_every;     // the default value for the interval between repeating warning notifications
     uint32_t health_default_crit_repeat_every;     // the default value for the interval between repeating critical notifications
+    unsigned int health_enabled;                   // 1 when this host has health enabled
+    bool use_summary_for_notifications;            // whether or not to use the summary field as a subject for notifications
 } HEALTH;
 
 // ----------------------------------------------------------------------------
@@ -1167,7 +1108,7 @@ struct rrdhost_system_info {
     int mc_version;
 };
 
-struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels);
+struct rrdhost_system_info *rrdhost_labels_to_system_info(RRDLABELS *labels);
 
 struct rrdhost {
     char machine_guid[GUID_LEN + 1];                // the unique ID of this host
@@ -1235,6 +1176,7 @@ struct rrdhost {
     // ------------------------------------------------------------------------
     // streaming of data from remote hosts - rrdpush receiver
 
+    time_t last_connected;                          // last time child connected (stored in db)
     time_t child_connect_time;                      // the time the last sender was connected
     time_t child_last_chart_command;                // the time of the last CHART streaming command
     time_t child_disconnected_time;                 // the time the last sender was disconnected
@@ -1274,7 +1216,7 @@ struct rrdhost {
 
     // ------------------------------------------------------------------------
     // Support for host-level labels
-    DICTIONARY *rrdlabels;
+    RRDLABELS *rrdlabels;
 
     // ------------------------------------------------------------------------
     // Support for functions
@@ -1310,6 +1252,8 @@ struct rrdhost {
     netdata_mutex_t aclk_state_lock;
     aclk_rrdhost_state aclk_state;
 
+    DICTIONARY *configurable_plugins;               // configurable plugins for this host
+
     struct rrdhost *next;
     struct rrdhost *prev;
 };
@@ -1465,8 +1409,6 @@ void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);
 static inline RRDSET *rrdset_find_active_localhost(const char *id)
 {
     RRDSET *st = rrdset_find_localhost(id);
-    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
-        return NULL;
     return st;
 }
 
@@ -1476,8 +1418,6 @@ RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
 static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
 {
     RRDSET *st = rrdset_find_bytype_localhost(type, id);
-    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
-        return NULL;
     return st;
 }
 
@@ -1487,8 +1427,6 @@ RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
 static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)
 {
     RRDSET *st = rrdset_find_byname_localhost(name);
-    if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
-        return NULL;
     return st;
 }
 
@@ -1504,9 +1442,8 @@ void rrdset_is_obsolete(RRDSET *st);
 void rrdset_isnot_obsolete(RRDSET *st);
 
 // checks if the RRDSET should be offered to viewers
-#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
-#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
-#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && rrdset_number_of_dimensions(st))
+#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
+#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st))
 
 time_t rrddim_first_entry_s(RRDDIM *rd);
 time_t rrddim_first_entry_s_of_tier(RRDDIM *rd, size_t tier);
@@ -1520,6 +1457,8 @@ time_t rrdset_last_entry_s_of_tier(RRDSET *st, size_t tier);
 
 void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_time_s, time_t *last_time_s, time_t now_s, size_t tier);
 
+void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels);
+
 // ----------------------------------------------------------------------------
 // RRD DIMENSION functions
 
@@ -1574,6 +1513,8 @@ void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n,
 // Miscellaneous functions
 
 char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
+void reload_host_labels(void);
+void rrdhost_set_is_parent_label(void);
 
 // ----------------------------------------------------------------------------
 // RRD internal functions
diff --git a/database/rrdcalc.c b/database/rrdcalc.c
index 398ddb32b..620883ec2 100644
--- a/database/rrdcalc.c
+++ b/database/rrdcalc.c
@@ -98,7 +98,7 @@ uint32_t rrdcalc_get_unique_id(RRDHOST *host, STRING *chart, STRING *name, uint3
 }
 
 // ----------------------------------------------------------------------------
-// RRDCALC replacing info text variables with RRDSET labels
+// RRDCALC replacing info/summary text variables with RRDSET labels
 
 static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RRDCALC *rc) {
     if (!line || !*line)
@@ -135,6 +135,7 @@ static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RR
             label_val[i - RRDCALC_VAR_LABEL_LEN - 1] = '\0';
 
             if(likely(rc->rrdset && rc->rrdset->rrdlabels)) {
+                lbl_value = NULL;
                 rrdlabels_get_value_strdup_or_null(rc->rrdset->rrdlabels, &lbl_value, label_val);
                 if (lbl_value) {
                     char *buf = find_and_replace(temp, var, lbl_value, m);
@@ -155,12 +156,20 @@ static STRING *rrdcalc_replace_variables_with_rrdset_labels(const char *line, RR
 void rrdcalc_update_info_using_rrdset_labels(RRDCALC *rc) {
     if(!rc->rrdset || !rc->original_info || !rc->rrdset->rrdlabels) return;
 
-    size_t labels_version = dictionary_version(rc->rrdset->rrdlabels);
+    size_t labels_version = rrdlabels_version(rc->rrdset->rrdlabels);
     if(rc->labels_version != labels_version) {
 
-        STRING *old = rc->info;
-        rc->info = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_info(rc), rc);
-        string_freez(old);
+        if (rc->original_info) {
+            STRING *old = rc->info;
+            rc->info = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_info(rc), rc);
+            string_freez(old);
+        }
+
+         if (rc->original_summary) {
+            STRING *old = rc->summary;
+            rc->summary = rrdcalc_replace_variables_with_rrdset_labels(rrdcalc_original_summary(rc), rc);
+            string_freez(old);
+        }
 
         rc->labels_version = labels_version;
     }
@@ -285,6 +294,11 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) {
 
     rrdcalc_update_info_using_rrdset_labels(rc);
 
+    if(!rc->summary) {
+        rc->summary = string_dup(rc->name);
+        rc->original_summary = string_dup(rc->name);
+    }
+
     time_t now = now_realtime_sec();
 
     ALARM_ENTRY *ae = health_create_alarm_entry(
@@ -297,7 +311,6 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) {
         rc->rrdset->id,
         rc->rrdset->context,
         rc->rrdset->name,
-        rc->rrdset->family,
         rc->classification,
         rc->component,
         rc->type,
@@ -310,6 +323,7 @@ static void rrdcalc_link_to_rrdset(RRDSET *st, RRDCALC *rc) {
         rc->status,
         rc->source,
         rc->units,
+        rc->summary,
         rc->info,
         0,
         rrdcalc_isrepeating(rc)?HEALTH_ENTRY_FLAG_IS_REPEATING:0);
@@ -343,7 +357,6 @@ static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) {
             rc->rrdset->id,
             rc->rrdset->context,
             rc->rrdset->name,
-            rc->rrdset->family,
             rc->classification,
             rc->component,
             rc->type,
@@ -356,6 +369,7 @@ static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) {
             RRDCALC_STATUS_REMOVED,
             rc->source,
             rc->units,
+            rc->summary,
             rc->info,
             0,
             0);
@@ -512,6 +526,11 @@ static void rrdcalc_rrdhost_insert_callback(const DICTIONARY_ITEM *item __maybe_
         rc->info = string_dup(rt->info);
         rc->original_info = string_dup(rt->info);
 
+        if (!rt->summary)
+            rt->summary = string_dup(rc->name);
+        rc->summary = string_dup(rt->summary);
+        rc->original_summary = string_dup(rt->summary);
+
         rc->classification = string_dup(rt->classification);
         rc->component = string_dup(rt->component);
         rc->type = string_dup(rt->type);
diff --git a/database/rrdcalc.h b/database/rrdcalc.h
index 2081452c7..71f43633c 100644
--- a/database/rrdcalc.h
+++ b/database/rrdcalc.h
@@ -64,8 +64,10 @@ struct rrdcalc {
 
     STRING *source;                 // the source of this alarm
     STRING *units;                  // the units of the alarm
+    STRING *summary;                // a short alert summary
+    STRING *original_summary;       // the original summary field before any variable replacement
     STRING *original_info;          // the original info field before any variable replacement
-    STRING *info;                   // a short description of the alarm
+    STRING *info;                   // a description of the alarm
 
     int update_every;               // update frequency for the alarm
 
@@ -170,6 +172,8 @@ struct rrdcalc {
 #define rrdcalc_module_match(rc) string2str((rc)->module_match)
 #define rrdcalc_source(rc) string2str((rc)->source)
 #define rrdcalc_units(rc) string2str((rc)->units)
+#define rrdcalc_original_summary(rc) string2str((rc)->original_summary)
+#define rrdcalc_summary(rc) string2str((rc)->summary)
 #define rrdcalc_original_info(rc) string2str((rc)->original_info)
 #define rrdcalc_info(rc) string2str((rc)->info)
 #define rrdcalc_dimensions(rc) string2str((rc)->dimensions)
@@ -192,7 +196,6 @@ struct alert_config {
     STRING *os;
     STRING *host;
     STRING *on;
-    STRING *families;
     STRING *plugin;
     STRING *module;
     STRING *charts;
@@ -206,6 +209,7 @@ struct alert_config {
     STRING *exec;
     STRING *to;
     STRING *units;
+    STRING *summary;
     STRING *info;
     STRING *classification;
     STRING *component;
diff --git a/database/rrdcalctemplate.c b/database/rrdcalctemplate.c
index a87403963..f0e5da80b 100644
--- a/database/rrdcalctemplate.c
+++ b/database/rrdcalctemplate.c
@@ -37,9 +37,6 @@ bool rrdcalctemplate_check_rrdset_conditions(RRDCALCTEMPLATE *rt, RRDSET *st, RR
     if (rt->charts_pattern && !simple_pattern_matches_string(rt->charts_pattern, st->name) && !simple_pattern_matches_string(rt->charts_pattern, st->id))
         return false;
 
-    if (rt->family_pattern && !simple_pattern_matches_string(rt->family_pattern, st->family))
-        return false;
-
     if (rt->module_pattern && !simple_pattern_matches_string(rt->module_pattern, st->module_name))
         return false;
 
@@ -100,9 +97,6 @@ static void rrdcalctemplate_free_internals(RRDCALCTEMPLATE *rt) {
     expression_free(rt->warning);
     expression_free(rt->critical);
 
-    string_freez(rt->family_match);
-    simple_pattern_free(rt->family_pattern);
-
     string_freez(rt->plugin_match);
     simple_pattern_free(rt->plugin_pattern);
 
@@ -217,10 +211,6 @@ inline void rrdcalctemplate_delete_all(RRDHOST *host) {
 }
 
 #define RRDCALCTEMPLATE_MAX_KEY_SIZE 1024
-static size_t rrdcalctemplate_key(char *dst, size_t dst_len, const char *name, const char *family_match) {
-    return snprintfz(dst, dst_len, "%s/%s", name, (family_match && *family_match)?family_match:"*");
-}
-
 void rrdcalctemplate_add_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
     if(unlikely(!rt->context)) {
         netdata_log_error("Health configuration for template '%s' does not have a context", rrdcalctemplate_name(rt));
@@ -238,7 +228,7 @@ void rrdcalctemplate_add_from_config(RRDHOST *host, RRDCALCTEMPLATE *rt) {
     }
 
     char key[RRDCALCTEMPLATE_MAX_KEY_SIZE + 1];
-    size_t key_len = rrdcalctemplate_key(key, RRDCALCTEMPLATE_MAX_KEY_SIZE, rrdcalctemplate_name(rt), rrdcalctemplate_family_match(rt));
+    size_t key_len = snprintfz(key, RRDCALCTEMPLATE_MAX_KEY_SIZE, "%s", rrdcalctemplate_name(rt));
 
     bool added = false;
     dictionary_set_advanced(host->rrdcalctemplate_root_index, key, (ssize_t)(key_len + 1), rt, sizeof(*rt), &added);
diff --git a/database/rrdcalctemplate.h b/database/rrdcalctemplate.h
index 965a818a1..ca2c43656 100644
--- a/database/rrdcalctemplate.h
+++ b/database/rrdcalctemplate.h
@@ -22,9 +22,6 @@ struct rrdcalctemplate {
 
     STRING *context;
 
-    STRING *family_match;
-    SIMPLE_PATTERN *family_pattern;
-
     STRING *plugin_match;
     SIMPLE_PATTERN *plugin_pattern;
 
@@ -36,7 +33,8 @@ struct rrdcalctemplate {
 
     STRING *source;                 // the source of this alarm
     STRING *units;                  // the units of the alarm
-    STRING *info;                   // a short description of the alarm
+    STRING *summary;                // a short summary of the alarm
+    STRING *info;                   // a description of the alarm
 
     int update_every;               // update frequency for the alarm
 
@@ -100,11 +98,11 @@ struct rrdcalctemplate {
 #define rrdcalctemplate_classification(rt) string2str((rt)->classification)
 #define rrdcalctemplate_component(rt) string2str((rt)->component)
 #define rrdcalctemplate_type(rt) string2str((rt)->type)
-#define rrdcalctemplate_family_match(rt) string2str((rt)->family_match)
 #define rrdcalctemplate_plugin_match(rt) string2str((rt)->plugin_match)
 #define rrdcalctemplate_module_match(rt) string2str((rt)->module_match)
 #define rrdcalctemplate_charts_match(rt) string2str((rt)->charts_match)
 #define rrdcalctemplate_units(rt) string2str((rt)->units)
+#define rrdcalctemplate_summary(rt) string2str((rt)->summary)
 #define rrdcalctemplate_info(rt) string2str((rt)->info)
 #define rrdcalctemplate_source(rt) string2str((rt)->source)
 #define rrdcalctemplate_dimensions(rt) string2str((rt)->dimensions)
diff --git a/database/rrdfunctions.c b/database/rrdfunctions.c
index d32a4b8c9..6c5baf346 100644
--- a/database/rrdfunctions.c
+++ b/database/rrdfunctions.c
@@ -1,3 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
 
@@ -37,17 +38,17 @@ static unsigned char functions_allowed_chars[256] = {
     [30] = '_', //
     [31] = '_', //
     [32] = ' ', // SPACE keep
-    [33] = '_', // !
-    [34] = '_', // "
-    [35] = '_', // #
-    [36] = '_', // $
-    [37] = '_', // %
-    [38] = '_', // &
-    [39] = '_', // '
-    [40] = '_', // (
-    [41] = '_', // )
-    [42] = '_', // *
-    [43] = '_', // +
+    [33] = '!', // ! keep
+    [34] = '"', // " keep
+    [35] = '#', // # keep
+    [36] = '$', // $ keep
+    [37] = '%', // % keep
+    [38] = '&', // & keep
+    [39] = '\'', // ' keep
+    [40] = '(', // ( keep
+    [41] = ')', // ) keep
+    [42] = '*', // * keep
+    [43] = '+', // + keep
     [44] = ',', // , keep
     [45] = '-', // - keep
     [46] = '.', // . keep
@@ -63,12 +64,12 @@ static unsigned char functions_allowed_chars[256] = {
     [56] = '8', // 8 keep
     [57] = '9', // 9 keep
     [58] = ':', // : keep
-    [59] = ':', // ; convert ; to :
-    [60] = '_', // <
-    [61] = ':', // = convert = to :
-    [62] = '_', // >
-    [63] = '_', // ?
-    [64] = '_', // @
+    [59] = ';', // ; keep
+    [60] = '<', // < keep
+    [61] = '=', // = keep
+    [62] = '>', // > keep
+    [63] = '?', // ? keep
+    [64] = '@', // @ keep
     [65] = 'A', // A keep
     [66] = 'B', // B keep
     [67] = 'C', // C keep
@@ -95,12 +96,12 @@ static unsigned char functions_allowed_chars[256] = {
     [88] = 'X', // X keep
     [89] = 'Y', // Y keep
     [90] = 'Z', // Z keep
-    [91] = '_', // [
-    [92] = '/', // backslash convert \ to /
-    [93] = '_', // ]
-    [94] = '_', // ^
+    [91] = '[', // [ keep
+    [92] = '\\', // backslash keep
+    [93] = ']', // ] keep
+    [94] = '^', // ^ keep
     [95] = '_', // _ keep
-    [96] = '_', // `
+    [96] = '`', // ` keep
     [97] = 'a', // a keep
     [98] = 'b', // b keep
     [99] = 'c', // c keep
@@ -127,10 +128,10 @@ static unsigned char functions_allowed_chars[256] = {
     [120] = 'x', // x keep
     [121] = 'y', // y keep
     [122] = 'z', // z keep
-    [123] = '_', // {
-    [124] = '_', // |
-    [125] = '_', // }
-    [126] = '_', // ~
+    [123] = '{', // { keep
+    [124] = '|', // | keep
+    [125] = '}', // } keep
+    [126] = '~', // ~ keep
     [127] = '_', //
     [128] = '_', //
     [129] = '_', //
@@ -277,16 +278,15 @@ typedef enum __attribute__((packed)) {
     // this is 8-bit
 } RRD_FUNCTION_OPTIONS;
 
-struct rrd_collector_function {
+struct rrd_host_function {
     bool sync;                      // when true, the function is called synchronously
     RRD_FUNCTION_OPTIONS options;   // RRD_FUNCTION_OPTIONS
     STRING *help;
     int timeout;                    // the default timeout of the function
 
-    int (*function)(BUFFER *wb, int timeout, const char *function, void *collector_data,
-                    function_data_ready_callback callback, void *callback_data);
+    rrd_function_execute_cb_t execute_cb;
 
-    void *collector_data;
+    void *execute_cb_data;
     struct rrd_collector *collector;
 };
 
@@ -299,6 +299,7 @@ struct rrd_collector_function {
 
 struct rrd_collector {
     int32_t refcount;
+    int32_t refcount_canceller;
     pid_t tid;
     bool running;
 };
@@ -310,8 +311,11 @@ struct rrd_collector {
 static __thread struct rrd_collector *thread_rrd_collector = NULL;
 
 static void rrd_collector_free(struct rrd_collector *rdc) {
+    if(rdc->running)
+        return;
+
     int32_t expected = 0;
-    if(likely(!__atomic_compare_exchange_n(&rdc->refcount, &expected, -1, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST))) {
+    if(!__atomic_compare_exchange_n(&rdc->refcount, &expected, -1, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
         // the collector is still referenced by charts.
         // leave it hanging there, the last chart will actually free it.
         return;
@@ -323,11 +327,11 @@ static void rrd_collector_free(struct rrd_collector *rdc) {
 
 // called once per collector
 void rrd_collector_started(void) {
-    if(likely(thread_rrd_collector)) return;
+    if(!thread_rrd_collector)
+        thread_rrd_collector = callocz(1, sizeof(struct rrd_collector));
 
-    thread_rrd_collector = callocz(1, sizeof(struct rrd_collector));
     thread_rrd_collector->tid = gettid();
-    thread_rrd_collector->running = true;
+    __atomic_store_n(&thread_rrd_collector->running, true, __ATOMIC_RELAXED);
 }
 
 // called once per collector
@@ -335,65 +339,110 @@ void rrd_collector_finished(void) {
     if(!thread_rrd_collector)
         return;
 
-    thread_rrd_collector->running = false;
+    __atomic_store_n(&thread_rrd_collector->running, false, __ATOMIC_RELAXED);
+
+    int32_t expected = 0;
+    while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount_canceller, &expected, -1, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) {
+        expected = 0;
+        sleep_usec(1 * USEC_PER_MS);
+    }
+
     rrd_collector_free(thread_rrd_collector);
     thread_rrd_collector = NULL;
 }
 
+#define rrd_collector_running(c) __atomic_load_n(&(c)->running, __ATOMIC_RELAXED)
+
 static struct rrd_collector *rrd_collector_acquire(void) {
-    __atomic_add_fetch(&thread_rrd_collector->refcount, 1, __ATOMIC_SEQ_CST);
+    rrd_collector_started();
+
+    int32_t expected = __atomic_load_n(&thread_rrd_collector->refcount, __ATOMIC_RELAXED), wanted = 0;
+    do {
+        if(expected < 0 || !rrd_collector_running(thread_rrd_collector)) {
+            internal_fatal(true, "FUNCTIONS: Trying to acquire a collector that is exiting.");
+            return thread_rrd_collector;
+        }
+
+        wanted = expected + 1;
+
+    } while(!__atomic_compare_exchange_n(&thread_rrd_collector->refcount, &expected, wanted, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED));
+
     return thread_rrd_collector;
 }
 
 static void rrd_collector_release(struct rrd_collector *rdc) {
     if(unlikely(!rdc)) return;
 
-    int32_t refcount = __atomic_sub_fetch(&rdc->refcount, 1, __ATOMIC_SEQ_CST);
-    if(refcount == 0 && !rdc->running)
+    int32_t expected = __atomic_load_n(&rdc->refcount, __ATOMIC_RELAXED), wanted = 0;
+    do {
+        if(expected < 0) {
+            internal_fatal(true, "FUNCTIONS: Trying to release a collector that is exiting.");
+            return;
+        }
+
+        if(expected == 0) {
+            internal_fatal(true, "FUNCTIONS: Trying to release a collector that is not acquired.");
+            return;
+        }
+
+        wanted = expected - 1;
+
+    } while(!__atomic_compare_exchange_n(&rdc->refcount, &expected, wanted, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED));
+
+    if(wanted == 0)
         rrd_collector_free(rdc);
 }
 
-static void rrd_functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused,
-                                          void *rrdhost __maybe_unused) {
-    struct rrd_collector_function *rdcf = func;
-
-    if(!thread_rrd_collector)
-        fatal("RRDSET_COLLECTOR: called %s() for function '%s' without calling rrd_collector_started() first.",
-              __FUNCTION__, dictionary_acquired_item_name(item));
+static void rrd_functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *rrdhost) {
+    RRDHOST *host = rrdhost; (void)host;
+    struct rrd_host_function *rdcf = func;
 
+    rrd_collector_started();
     rdcf->collector = rrd_collector_acquire();
+
+//    internal_error(true, "FUNCTIONS: adding function '%s' on host '%s', collection tid %d, %s",
+//                   dictionary_acquired_item_name(item), rrdhost_hostname(host),
+//                   rdcf->collector->tid, rdcf->collector->running ? "running" : "NOT running");
 }
 
-static void rrd_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused,
+static void rrd_functions_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func,
                                           void *rrdhost __maybe_unused) {
-    struct rrd_collector_function *rdcf = func;
+    struct rrd_host_function *rdcf = func;
     rrd_collector_release(rdcf->collector);
 }
 
-static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func __maybe_unused,
-                                            void *new_func __maybe_unused, void *rrdhost __maybe_unused) {
-    struct rrd_collector_function *rdcf = func;
-    struct rrd_collector_function *new_rdcf = new_func;
+static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func,
+                                            void *new_func, void *rrdhost) {
+    RRDHOST *host = rrdhost; (void)host;
+    struct rrd_host_function *rdcf = func;
+    struct rrd_host_function *new_rdcf = new_func;
 
-    if(!thread_rrd_collector)
-        fatal("RRDSET_COLLECTOR: called %s() for function '%s' without calling rrd_collector_started() first.",
-              __FUNCTION__, dictionary_acquired_item_name(item));
+    rrd_collector_started();
 
     bool changed = false;
 
     if(rdcf->collector != thread_rrd_collector) {
+        netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed collector from %d to %d",
+                         dictionary_acquired_item_name(item), rrdhost_hostname(host), rdcf->collector->tid, thread_rrd_collector->tid);
+
         struct rrd_collector *old_rdc = rdcf->collector;
         rdcf->collector = rrd_collector_acquire();
         rrd_collector_release(old_rdc);
         changed = true;
     }
 
-    if(rdcf->function != new_rdcf->function) {
-        rdcf->function = new_rdcf->function;
+    if(rdcf->execute_cb != new_rdcf->execute_cb) {
+        netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed execute callback",
+                         dictionary_acquired_item_name(item), rrdhost_hostname(host));
+
+        rdcf->execute_cb = new_rdcf->execute_cb;
         changed = true;
     }
 
     if(rdcf->help != new_rdcf->help) {
+        netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed help text",
+                         dictionary_acquired_item_name(item), rrdhost_hostname(host));
+
         STRING *old = rdcf->help;
         rdcf->help = new_rdcf->help;
         string_freez(old);
@@ -403,41 +452,53 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_
         string_freez(new_rdcf->help);
 
     if(rdcf->timeout != new_rdcf->timeout) {
+        netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed timeout",
+                         dictionary_acquired_item_name(item), rrdhost_hostname(host));
+
         rdcf->timeout = new_rdcf->timeout;
         changed = true;
     }
 
     if(rdcf->sync != new_rdcf->sync) {
+        netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed sync/async mode",
+                         dictionary_acquired_item_name(item), rrdhost_hostname(host));
+
         rdcf->sync = new_rdcf->sync;
         changed = true;
     }
 
-    if(rdcf->collector_data != new_rdcf->collector_data) {
-        rdcf->collector_data = new_rdcf->collector_data;
+    if(rdcf->execute_cb_data != new_rdcf->execute_cb_data) {
+        netdata_log_info("FUNCTIONS: function '%s' of host '%s' changed execute callback data",
+                         dictionary_acquired_item_name(item), rrdhost_hostname(host));
+
+        rdcf->execute_cb_data = new_rdcf->execute_cb_data;
         changed = true;
     }
 
+//    internal_error(true, "FUNCTIONS: adding function '%s' on host '%s', collection tid %d, %s",
+//                   dictionary_acquired_item_name(item), rrdhost_hostname(host),
+//                   rdcf->collector->tid, rdcf->collector->running ? "running" : "NOT running");
+
     return changed;
 }
 
-
-void rrdfunctions_init(RRDHOST *host) {
+void rrdfunctions_host_init(RRDHOST *host) {
     if(host->functions) return;
 
     host->functions = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
-                                                 &dictionary_stats_category_functions, sizeof(struct rrd_collector_function));
+                                                 &dictionary_stats_category_functions, sizeof(struct rrd_host_function));
 
     dictionary_register_insert_callback(host->functions, rrd_functions_insert_callback, host);
     dictionary_register_delete_callback(host->functions, rrd_functions_delete_callback, host);
     dictionary_register_conflict_callback(host->functions, rrd_functions_conflict_callback, host);
 }
 
-void rrdfunctions_destroy(RRDHOST *host) {
+void rrdfunctions_host_destroy(RRDHOST *host) {
     dictionary_destroy(host->functions);
 }
 
-void rrd_collector_add_function(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help,
-                                bool sync, function_execute_at_collector function, void *collector_data) {
+void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help,
+                      bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data) {
 
     // RRDSET *st may be NULL in this function
     // to create a GLOBAL function
@@ -448,18 +509,20 @@ void rrd_collector_add_function(RRDHOST *host, RRDSET *st, const char *name, int
     char key[PLUGINSD_LINE_MAX + 1];
     sanitize_function_text(key, name, PLUGINSD_LINE_MAX);
 
-    struct rrd_collector_function tmp = {
+    struct rrd_host_function tmp = {
         .sync = sync,
         .timeout = timeout,
         .options = (st)?RRD_FUNCTION_LOCAL:RRD_FUNCTION_GLOBAL,
-        .function = function,
-        .collector_data = collector_data,
+        .execute_cb = execute_cb,
+        .execute_cb_data = execute_cb_data,
         .help = string_strdupz(help),
     };
     const DICTIONARY_ITEM *item = dictionary_set_and_acquire_item(host->functions, key, &tmp, sizeof(tmp));
 
     if(st)
         dictionary_view_set(st->functions_view, key, item);
+    else
+        rrdhost_flag_set(host, RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED);
 
     dictionary_acquired_item_release(host->functions, item);
 }
@@ -468,7 +531,7 @@ void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb) {
     if(!st->functions_view)
         return;
 
-    struct rrd_collector_function *tmp;
+    struct rrd_host_function *tmp;
     dfe_start_read(st->functions_view, tmp) {
         buffer_sprintf(wb
                        , PLUGINSD_KEYWORD_FUNCTION " \"%s\" %d \"%s\"\n"
@@ -481,7 +544,9 @@ void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb) {
 }
 
 void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb) {
-    struct rrd_collector_function *tmp;
+    rrdhost_flag_clear(host, RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED);
+
+    struct rrd_host_function *tmp;
     dfe_start_read(host->functions, tmp) {
         if(!(tmp->options & RRD_FUNCTION_GLOBAL))
             continue;
@@ -496,20 +561,6 @@ void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb) {
     dfe_done(tmp);
 }
 
-struct rrd_function_call_wait {
-    bool free_with_signal;
-    bool data_are_ready;
-    netdata_mutex_t mutex;
-    pthread_cond_t cond;
-    int code;
-};
-
-static void rrd_function_call_wait_free(struct rrd_function_call_wait *tmp) {
-    pthread_cond_destroy(&tmp->cond);
-    netdata_mutex_destroy(&tmp->mutex);
-    freez(tmp);
-}
-
 struct {
     const char *format;
     HTTP_CONTENT_TYPE content_type;
@@ -558,41 +609,171 @@ int rrd_call_function_error(BUFFER *wb, const char *msg, int code) {
     return code;
 }
 
-static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, size_t key_length, struct rrd_collector_function **rdcf) {
+static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, size_t key_length, const DICTIONARY_ITEM **item) {
     char buffer[MAX_FUNCTION_LENGTH + 1];
 
     strncpyz(buffer, name, MAX_FUNCTION_LENGTH);
     char *s = NULL;
 
-    *rdcf = NULL;
-    while(!(*rdcf) && buffer[0]) {
-        *rdcf = dictionary_get(host->functions, buffer);
-        if(*rdcf) break;
+    bool found = false;
+    *item = NULL;
+    if(host->functions) {
+        while (buffer[0]) {
+            if((*item = dictionary_get_and_acquire_item(host->functions, buffer))) {
+                found = true;
+
+                struct rrd_host_function *rdcf = dictionary_acquired_item_value(*item);
+                if(rrd_collector_running(rdcf->collector)) {
+                    break;
+                }
+                else {
+                    dictionary_acquired_item_release(host->functions, *item);
+                    *item = NULL;
+                }
+            }
 
-        // if s == NULL, set it to the end of the buffer
-        // this should happen only the first time
-        if(unlikely(!s))
-            s = &buffer[key_length - 1];
+            // if s == NULL, set it to the end of the buffer
+            // this should happen only the first time
+            if (unlikely(!s))
+                s = &buffer[key_length - 1];
 
-        // skip a word from the end
-        while(s >= buffer && !isspace(*s)) *s-- = '\0';
+            // skip a word from the end
+            while (s >= buffer && !isspace(*s)) *s-- = '\0';
 
-        // skip all spaces
-        while(s >= buffer && isspace(*s)) *s-- = '\0';
+            // skip all spaces
+            while (s >= buffer && isspace(*s)) *s-- = '\0';
+        }
     }
 
     buffer_flush(wb);
 
-    if(!(*rdcf))
-        return rrd_call_function_error(wb, "No collector is supplying this function on this host at this time.", HTTP_RESP_NOT_FOUND);
-
-    if(!(*rdcf)->collector->running)
-        return rrd_call_function_error(wb, "The collector that registered this function, is not currently running.", HTTP_RESP_BACKEND_FETCH_FAILED);
+    if(!(*item)) {
+        if(found)
+            return rrd_call_function_error(wb,
+                                           "The collector that registered this function, is not currently running.",
+                                           HTTP_RESP_SERVICE_UNAVAILABLE);
+        else
+            return rrd_call_function_error(wb,
+                                           "No collector is supplying this function on this host at this time.",
+                                           HTTP_RESP_NOT_FOUND);
+    }
 
     return HTTP_RESP_OK;
 }
 
-static void rrd_call_function_signal_when_ready(BUFFER *temp_wb __maybe_unused, int code, void *callback_data) {
+// ----------------------------------------------------------------------------
+
+struct rrd_function_inflight {
+    bool used;
+
+    RRDHOST *host;
+    const char *transaction;
+    const char *cmd;
+    const char *sanitized_cmd;
+    size_t sanitized_cmd_length;
+    int timeout;
+    bool cancelled;
+
+    const DICTIONARY_ITEM *host_function_acquired;
+
+    // the collector
+    // we acquire this structure at the beginning,
+    // and we release it at the end
+    struct rrd_host_function *rdcf;
+
+    struct {
+        BUFFER *wb;
+
+        // in async mode,
+        // the function to call to send the result back
+        rrd_function_result_callback_t cb;
+        void *data;
+    } result;
+
+    struct {
+        // to be called in sync mode
+        // while the function is running
+        // to check if the function has been cancelled
+        rrd_function_is_cancelled_cb_t cb;
+        void *data;
+    } is_cancelled;
+
+    struct {
+        // to be registered by the function itself
+        // used to signal the function to cancel
+        rrd_function_canceller_cb_t cb;
+        void *data;
+    } canceller;
+};
+
+static DICTIONARY *rrd_functions_inflight_requests = NULL;
+
+static void rrd_functions_inflight_delete_cb(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
+    struct rrd_function_inflight *r = value;
+
+    // internal_error(true, "FUNCTIONS: transaction '%s' finished", r->transaction);
+
+    freez((void *)r->transaction);
+    freez((void *)r->cmd);
+    freez((void *)r->sanitized_cmd);
+    dictionary_acquired_item_release(r->host->functions, r->host_function_acquired);
+}
+
+void rrd_functions_inflight_init(void) {
+    if(rrd_functions_inflight_requests)
+        return;
+
+    rrd_functions_inflight_requests = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct rrd_function_inflight));
+
+    dictionary_register_delete_callback(rrd_functions_inflight_requests, rrd_functions_inflight_delete_cb, NULL);
+}
+
+void rrd_functions_inflight_destroy(void) {
+    if(!rrd_functions_inflight_requests)
+        return;
+
+    dictionary_destroy(rrd_functions_inflight_requests);
+    rrd_functions_inflight_requests = NULL;
+}
+
+static void rrd_inflight_async_function_register_canceller_cb(void *register_canceller_cb_data, rrd_function_canceller_cb_t canceller_cb, void *canceller_cb_data) {
+    struct rrd_function_inflight *r = register_canceller_cb_data;
+    r->canceller.cb = canceller_cb;
+    r->canceller.data = canceller_cb_data;
+}
+
+// ----------------------------------------------------------------------------
+// waiting for async function completion
+
+struct rrd_function_call_wait {
+    RRDHOST *host;
+    const DICTIONARY_ITEM *host_function_acquired;
+    char *transaction;
+
+    bool free_with_signal;
+    bool data_are_ready;
+    netdata_mutex_t mutex;
+    pthread_cond_t cond;
+    int code;
+};
+
+static void rrd_inflight_function_cleanup(RRDHOST *host __maybe_unused,
+                                          const DICTIONARY_ITEM *host_function_acquired __maybe_unused,
+                                          const char *transaction) {
+    dictionary_del(rrd_functions_inflight_requests, transaction);
+    dictionary_garbage_collect(rrd_functions_inflight_requests);
+}
+
+static void rrd_function_call_wait_free(struct rrd_function_call_wait *tmp) {
+    rrd_inflight_function_cleanup(tmp->host, tmp->host_function_acquired, tmp->transaction);
+    freez(tmp->transaction);
+
+    pthread_cond_destroy(&tmp->cond);
+    netdata_mutex_destroy(&tmp->mutex);
+    freez(tmp);
+}
+
+static void rrd_async_function_signal_when_ready(BUFFER *temp_wb __maybe_unused, int code, void *callback_data) {
     struct rrd_function_call_wait *tmp = callback_data;
     bool we_should_free = false;
 
@@ -618,115 +799,308 @@ static void rrd_call_function_signal_when_ready(BUFFER *temp_wb __maybe_unused,
     }
 }
 
-int rrd_call_function_and_wait(RRDHOST *host, BUFFER *wb, int timeout, const char *name) {
-    int code;
+static void rrd_inflight_async_function_nowait_finished(BUFFER *wb, int code, void *data) {
+    struct rrd_function_inflight *r = data;
 
-    struct rrd_collector_function *rdcf = NULL;
+    if(r->result.cb)
+        r->result.cb(wb, code, r->result.data);
 
-    char key[PLUGINSD_LINE_MAX + 1];
-    size_t key_length = sanitize_function_text(key, name, PLUGINSD_LINE_MAX);
-    code = rrd_call_function_find(host, wb, key, key_length, &rdcf);
-    if(code != HTTP_RESP_OK)
-        return code;
+    rrd_inflight_function_cleanup(r->host, r->host_function_acquired, r->transaction);
+}
 
-    if(timeout <= 0)
-        timeout = rdcf->timeout;
+static bool rrd_inflight_async_function_is_cancelled(void *data) {
+    struct rrd_function_inflight *r = data;
+    return __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
+}
 
-    struct timespec tp;
-    clock_gettime(CLOCK_REALTIME, &tp);
-    tp.tv_sec += (time_t)timeout;
+static inline int rrd_call_function_async_and_dont_wait(struct rrd_function_inflight *r) {
+    int code = r->rdcf->execute_cb(r->result.wb, r->timeout, r->sanitized_cmd, r->rdcf->execute_cb_data,
+                                   rrd_inflight_async_function_nowait_finished, r,
+                                   rrd_inflight_async_function_is_cancelled, r,
+                                   rrd_inflight_async_function_register_canceller_cb, r);
 
-    if(rdcf->sync) {
-        code = rdcf->function(wb, timeout, key, rdcf->collector_data, NULL, NULL);
+    if(code != HTTP_RESP_OK) {
+        if (!buffer_strlen(r->result.wb))
+            rrd_call_function_error(r->result.wb, "Failed to send request to the collector.", code);
+
+        rrd_inflight_function_cleanup(r->host, r->host_function_acquired, r->transaction);
     }
-    else {
-        struct rrd_function_call_wait *tmp = mallocz(sizeof(struct rrd_function_call_wait));
-        tmp->free_with_signal = false;
-        tmp->data_are_ready = false;
-        netdata_mutex_init(&tmp->mutex);
-        pthread_cond_init(&tmp->cond, NULL);
-
-        bool we_should_free = true;
-        BUFFER *temp_wb  = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions); // we need it because we may give up on it
-        temp_wb->content_type = wb->content_type;
-        code = rdcf->function(temp_wb, timeout, key, rdcf->collector_data, rrd_call_function_signal_when_ready, tmp);
-        if (code == HTTP_RESP_OK) {
-            netdata_mutex_lock(&tmp->mutex);
-
-            int rc = 0;
-            while (rc == 0 && !tmp->data_are_ready) {
-                // the mutex is unlocked within pthread_cond_timedwait()
-                rc = pthread_cond_timedwait(&tmp->cond, &tmp->mutex, &tp);
-                // the mutex is again ours
-            }
 
-            if (tmp->data_are_ready) {
-                // we have a response
-                buffer_fast_strcat(wb, buffer_tostring(temp_wb), buffer_strlen(temp_wb));
-                wb->content_type = temp_wb->content_type;
-                wb->expires = temp_wb->expires;
+    return code;
+}
 
-                if(wb->expires)
-                    buffer_cacheable(wb);
-                else
-                    buffer_no_cacheable(wb);
+static int rrd_call_function_async_and_wait(struct rrd_function_inflight *r) {
+    struct timespec tp;
+    clock_gettime(CLOCK_REALTIME, &tp);
+    usec_t now_ut = tp.tv_sec * USEC_PER_SEC + tp.tv_nsec / NSEC_PER_USEC;
+    usec_t end_ut = now_ut + r->timeout * USEC_PER_SEC + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT;
+
+    struct rrd_function_call_wait *tmp = mallocz(sizeof(struct rrd_function_call_wait));
+    tmp->free_with_signal = false;
+    tmp->data_are_ready = false;
+    tmp->host = r->host;
+    tmp->host_function_acquired = r->host_function_acquired;
+    tmp->transaction = strdupz(r->transaction);
+    netdata_mutex_init(&tmp->mutex);
+    pthread_cond_init(&tmp->cond, NULL);
+
+    // we need a temporary BUFFER, because we may time out and the caller supplied one may vanish
+    // so, we create a new one we guarantee will survive until the collector finishes...
+
+    bool we_should_free = true;
+    BUFFER *temp_wb  = buffer_create(PLUGINSD_LINE_MAX + 1, &netdata_buffers_statistics.buffers_functions); // we need it because we may give up on it
+    temp_wb->content_type = r->result.wb->content_type;
+
+    int code = r->rdcf->execute_cb(temp_wb, r->timeout, r->sanitized_cmd, r->rdcf->execute_cb_data,
+            // we overwrite the result callbacks,
+            // so that we can clean up the allocations made
+                                   rrd_async_function_signal_when_ready, tmp,
+                                   rrd_inflight_async_function_is_cancelled, r,
+                                   rrd_inflight_async_function_register_canceller_cb, r);
+
+    if (code == HTTP_RESP_OK) {
+        netdata_mutex_lock(&tmp->mutex);
+
+        bool cancelled = false;
+        int rc = 0;
+        while (rc == 0 && !cancelled && !tmp->data_are_ready) {
+            clock_gettime(CLOCK_REALTIME, &tp);
+            now_ut = tp.tv_sec * USEC_PER_SEC + tp.tv_nsec / NSEC_PER_USEC;
+
+            if(now_ut >= end_ut) {
+                rc = ETIMEDOUT;
+                break;
+            }
 
-                code = tmp->code;
+            tp.tv_nsec += 10 * NSEC_PER_MSEC;
+            if(tp.tv_nsec > (long)(1 * NSEC_PER_SEC)) {
+                tp.tv_sec++;
+                tp.tv_nsec -= 1 * NSEC_PER_SEC;
             }
-            else if (rc == ETIMEDOUT) {
-                // timeout
-                // we will go away and let the callback free the structure
-                tmp->free_with_signal = true;
-                we_should_free = false;
-                code = rrd_call_function_error(wb, "Timeout while waiting for a response from the collector.", HTTP_RESP_GATEWAY_TIMEOUT);
+
+            // the mutex is unlocked within pthread_cond_timedwait()
+            rc = pthread_cond_timedwait(&tmp->cond, &tmp->mutex, &tp);
+            // the mutex is again ours
+
+            if(rc == ETIMEDOUT) {
+                rc = 0;
+                if (!tmp->data_are_ready && r->is_cancelled.cb &&
+                    r->is_cancelled.cb(r->is_cancelled.data)) {
+//                    internal_error(true, "FUNCTIONS: transaction '%s' is cancelled while waiting for response",
+//                                   r->transaction);
+                    rc = 0;
+                    cancelled = true;
+                    rrd_function_cancel(r->transaction);
+                    break;
+                }
             }
+        }
+
+        if (tmp->data_are_ready) {
+            // we have a response
+            buffer_fast_strcat(r->result.wb, buffer_tostring(temp_wb), buffer_strlen(temp_wb));
+            r->result.wb->content_type = temp_wb->content_type;
+            r->result.wb->expires = temp_wb->expires;
+
+            if(r->result.wb->expires)
+                buffer_cacheable(r->result.wb);
             else
-                code = rrd_call_function_error(wb, "Failed to get the response from the collector.", HTTP_RESP_INTERNAL_SERVER_ERROR);
+                buffer_no_cacheable(r->result.wb);
 
-            netdata_mutex_unlock(&tmp->mutex);
+            code = tmp->code;
         }
-        else {
-            if(!buffer_strlen(wb))
-                rrd_call_function_error(wb, "Failed to send request to the collector.", code);
+        else if (rc == ETIMEDOUT || cancelled) {
+            // timeout
+            // we will go away and let the callback free the structure
+            tmp->free_with_signal = true;
+            we_should_free = false;
+
+            if(cancelled)
+                code = rrd_call_function_error(r->result.wb,
+                                               "Request cancelled",
+                                               HTTP_RESP_CLIENT_CLOSED_REQUEST);
+            else
+                code = rrd_call_function_error(r->result.wb,
+                                               "Timeout while waiting for a response from the collector.",
+                                               HTTP_RESP_GATEWAY_TIMEOUT);
         }
+        else
+            code = rrd_call_function_error(r->result.wb,
+                                           "Internal error while communicating with the collector",
+                                           HTTP_RESP_INTERNAL_SERVER_ERROR);
 
-        if (we_should_free) {
-            rrd_function_call_wait_free(tmp);
-            buffer_free(temp_wb);
-        }
+        netdata_mutex_unlock(&tmp->mutex);
+    }
+    else {
+        if(!buffer_strlen(r->result.wb))
+            rrd_call_function_error(r->result.wb, "The collector returned an error.", code);
+    }
+
+    if (we_should_free) {
+        rrd_function_call_wait_free(tmp);
+        buffer_free(temp_wb);
     }
 
     return code;
 }
 
-int rrd_call_function_async(RRDHOST *host, BUFFER *wb, int timeout, const char *name,
-    rrd_call_function_async_callback callback, void *callback_data) {
+static inline int rrd_call_function_async(struct rrd_function_inflight *r, bool wait) {
+    if(wait)
+        return rrd_call_function_async_and_wait(r);
+    else
+        return rrd_call_function_async_and_dont_wait(r);
+}
+
+
+void call_virtual_function_async(BUFFER *wb, RRDHOST *host, const char *name, const char *payload, rrd_function_result_callback_t callback, void *callback_data);
+// ----------------------------------------------------------------------------
+
+int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd,
+                     bool wait, const char *transaction,
+                     rrd_function_result_callback_t result_cb, void *result_cb_data,
+                     rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload) {
+
     int code;
+    char sanitized_cmd[PLUGINSD_LINE_MAX + 1];
+    const DICTIONARY_ITEM *host_function_acquired = NULL;
 
-    struct rrd_collector_function *rdcf = NULL;
-    char key[PLUGINSD_LINE_MAX + 1];
-    size_t key_length = sanitize_function_text(key, name, PLUGINSD_LINE_MAX);
-    code = rrd_call_function_find(host, wb, key, key_length, &rdcf);
+    // ------------------------------------------------------------------------
+    // find the function
+
+    size_t sanitized_cmd_length = sanitize_function_text(sanitized_cmd, cmd, PLUGINSD_LINE_MAX);
+
+    if (is_dyncfg_function(sanitized_cmd, DYNCFG_FUNCTION_TYPE_ALL)) {
+        call_virtual_function_async(result_wb, host, sanitized_cmd, payload, result_cb, result_cb_data);
+        return HTTP_RESP_OK;
+    }
+
+    code = rrd_call_function_find(host, result_wb, sanitized_cmd, sanitized_cmd_length, &host_function_acquired);
     if(code != HTTP_RESP_OK)
         return code;
 
+    struct rrd_host_function *rdcf = dictionary_acquired_item_value(host_function_acquired);
+
     if(timeout <= 0)
         timeout = rdcf->timeout;
 
-    code = rdcf->function(wb, timeout, key, rdcf->collector_data, callback, callback_data);
 
-    if(code != HTTP_RESP_OK) {
-        if (!buffer_strlen(wb))
-            rrd_call_function_error(wb, "Failed to send request to the collector.", code);
+    // ------------------------------------------------------------------------
+    // the function can only be executed in sync mode
+
+    if(rdcf->sync) {
+        // the caller has to wait
+
+        code = rdcf->execute_cb(result_wb, timeout, sanitized_cmd, rdcf->execute_cb_data,
+                                NULL, NULL,                             // no callback needed, it is synchronous
+                                is_cancelled_cb, is_cancelled_cb_data,  // it is ok to pass these, we block the caller
+                                NULL, NULL);                            // no need to pass, we will wait
+
+        if (code != HTTP_RESP_OK && !buffer_strlen(result_wb))
+            rrd_call_function_error(result_wb, "Collector reported error.", code);
+
+        dictionary_acquired_item_release(host->functions, host_function_acquired);
+        return code;
     }
 
-    return code;
+
+    // ------------------------------------------------------------------------
+    // the function can only be executed in async mode
+    // put the function into the inflight requests
+
+    char uuid_str[UUID_STR_LEN];
+    if(!transaction) {
+        uuid_t uuid;
+        uuid_generate_random(uuid);
+        uuid_unparse_lower(uuid, uuid_str);
+        transaction = uuid_str;
+    }
+
+    // put the request into the inflight requests
+    struct rrd_function_inflight t = {
+            .used = false,
+            .host = host,
+            .cmd = strdupz(cmd),
+            .sanitized_cmd = strdupz(sanitized_cmd),
+            .sanitized_cmd_length = sanitized_cmd_length,
+            .transaction = strdupz(transaction),
+            .timeout = timeout,
+            .cancelled = false,
+            .host_function_acquired = host_function_acquired,
+            .rdcf = rdcf,
+            .result = {
+                    .wb = result_wb,
+                    .cb = result_cb,
+                    .data = result_cb_data,
+            },
+            .is_cancelled = {
+                    .cb = is_cancelled_cb,
+                    .data = is_cancelled_cb_data,
+            }
+    };
+    struct rrd_function_inflight *r = dictionary_set(rrd_functions_inflight_requests, transaction, &t, sizeof(t));
+    if(r->used) {
+        netdata_log_info("FUNCTIONS: duplicate transaction '%s', function: '%s'", t.transaction, t.cmd);
+        code = rrd_call_function_error(result_wb, "duplicate transaction", HTTP_RESP_BAD_REQUEST);
+        freez((void *)t.transaction);
+        freez((void *)t.cmd);
+        freez((void *)t.sanitized_cmd);
+        dictionary_acquired_item_release(r->host->functions, t.host_function_acquired);
+        return code;
+    }
+    r->used = true;
+    // internal_error(true, "FUNCTIONS: transaction '%s' started", r->transaction);
+
+    return rrd_call_function_async(r, wait);
 }
 
+void rrd_function_cancel(const char *transaction) {
+    // internal_error(true, "FUNCTIONS: request to cancel transaction '%s'", transaction);
+
+    const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(rrd_functions_inflight_requests, transaction);
+    if(!item) {
+        netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but the transaction is not running.",
+                         transaction);
+        return;
+    }
+
+    struct rrd_function_inflight *r = dictionary_acquired_item_value(item);
+
+    bool cancelled = __atomic_load_n(&r->cancelled, __ATOMIC_RELAXED);
+    if(cancelled) {
+        netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but it is already cancelled.",
+                         transaction);
+        goto cleanup;
+    }
+
+    __atomic_store_n(&r->cancelled, true, __ATOMIC_RELAXED);
+
+    int32_t expected = __atomic_load_n(&r->rdcf->collector->refcount_canceller, __ATOMIC_RELAXED);
+    int32_t wanted;
+    do {
+        if(expected < 0) {
+            netdata_log_info("FUNCTIONS: received a cancel request for transaction '%s', but the collector is not running.",
+                             transaction);
+            goto cleanup;
+        }
+
+        wanted = expected + 1;
+    } while(!__atomic_compare_exchange_n(&r->rdcf->collector->refcount_canceller, &expected, wanted, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
+
+    if(r->canceller.cb)
+        r->canceller.cb(r->canceller.data);
+
+    __atomic_sub_fetch(&r->rdcf->collector->refcount_canceller, 1, __ATOMIC_RELAXED);
+
+cleanup:
+    dictionary_acquired_item_release(rrd_functions_inflight_requests, item);
+}
+
+// ----------------------------------------------------------------------------
+
 static void functions2json(DICTIONARY *functions, BUFFER *wb, const char *ident, const char *kq, const char *sq) {
-    struct rrd_collector_function *t;
+    struct rrd_host_function *t;
     dfe_start_read(functions, t) {
-        if(!t->collector->running) continue;
+        if(!rrd_collector_running(t->collector)) continue;
 
         if(t_dfe.counter)
             buffer_strcat(wb, ",\n");
@@ -759,9 +1133,9 @@ void host_functions2json(RRDHOST *host, BUFFER *wb) {
 
     buffer_json_member_add_object(wb, "functions");
 
-    struct rrd_collector_function *t;
+    struct rrd_host_function *t;
     dfe_start_read(host->functions, t) {
-        if(!t->collector->running) continue;
+        if(!rrd_collector_running(t->collector)) continue;
 
         buffer_json_member_add_object(wb, t_dfe.name);
         buffer_json_member_add_string(wb, "help", string2str(t->help));
@@ -782,9 +1156,9 @@ void host_functions2json(RRDHOST *host, BUFFER *wb) {
 void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst, void *value, size_t value_size) {
     if(!rrdset_functions_view || !dst) return;
 
-    struct rrd_collector_function *t;
+    struct rrd_host_function *t;
     dfe_start_read(rrdset_functions_view, t) {
-        if(!t->collector->running) continue;
+        if(!rrd_collector_running(t->collector)) continue;
 
         dictionary_set(dst, t_dfe.name, value, value_size);
     }
@@ -794,9 +1168,9 @@ void chart_functions_to_dict(DICTIONARY *rrdset_functions_view, DICTIONARY *dst,
 void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t value_size, STRING **help) {
     if(!host || !host->functions || !dictionary_entries(host->functions) || !dst) return;
 
-    struct rrd_collector_function *t;
+    struct rrd_host_function *t;
     dfe_start_read(host->functions, t) {
-        if(!t->collector->running) continue;
+        if(!rrd_collector_running(t->collector)) continue;
 
         if(help)
             *help = t->help;
@@ -806,10 +1180,15 @@ void host_functions_to_dict(RRDHOST *host, DICTIONARY *dst, void *value, size_t
     dfe_done(t);
 }
 
+// ----------------------------------------------------------------------------
 
 int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const char *function __maybe_unused,
                                void *collector_data __maybe_unused,
-                               function_data_ready_callback callback __maybe_unused, void *callback_data __maybe_unused) {
+                               rrd_function_result_callback_t result_cb, void *result_cb_data,
+                               rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
+                               rrd_function_register_canceller_cb_t register_canceller_cb __maybe_unused,
+                               void *register_canceller_cb_data __maybe_unused) {
+
     time_t now = now_realtime_sec();
 
     buffer_flush(wb);
@@ -1428,8 +1807,14 @@ int rrdhost_function_streaming(BUFFER *wb, int timeout __maybe_unused, const cha
     buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + 1);
     buffer_json_finalize(wb);
 
-    if(callback)
-        callback(wb, HTTP_RESP_OK, callback_data);
+    int response = HTTP_RESP_OK;
+    if(is_cancelled_cb && is_cancelled_cb(is_cancelled_cb_data)) {
+        buffer_flush(wb);
+        response = HTTP_RESP_CLIENT_CLOSED_REQUEST;
+    }
 
-    return HTTP_RESP_OK;
+    if(result_cb)
+        result_cb(wb, response, result_cb_data);
+
+    return response;
 }
diff --git a/database/rrdfunctions.h b/database/rrdfunctions.h
index 71ad96507..96aa3965e 100644
--- a/database/rrdfunctions.h
+++ b/database/rrdfunctions.h
@@ -1,26 +1,41 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
 #ifndef NETDATA_RRDFUNCTIONS_H
 #define NETDATA_RRDFUNCTIONS_H 1
 
+// ----------------------------------------------------------------------------
+
 #include "rrd.h"
 
-void rrdfunctions_init(RRDHOST *host);
-void rrdfunctions_destroy(RRDHOST *host);
+#define RRDFUNCTIONS_TIMEOUT_EXTENSION_UT (1 * USEC_PER_SEC)
 
-void rrd_collector_started(void);
-void rrd_collector_finished(void);
+typedef void (*rrd_function_result_callback_t)(BUFFER *wb, int code, void *result_cb_data);
+typedef bool (*rrd_function_is_cancelled_cb_t)(void *is_cancelled_cb_data);
+typedef void (*rrd_function_canceller_cb_t)(void *data);
+typedef void (*rrd_function_register_canceller_cb_t)(void *register_cancel_cb_data, rrd_function_canceller_cb_t cancel_cb, void *cancel_cb_data);
+typedef int (*rrd_function_execute_cb_t)(BUFFER *wb, int timeout, const char *function, void *collector_data,
+                                         rrd_function_result_callback_t result_cb, void *result_cb_data,
+                                         rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
+                                         rrd_function_register_canceller_cb_t register_cancel_cb, void *register_cancel_db_data);
 
-typedef void (*function_data_ready_callback)(BUFFER *wb, int code, void *callback_data);
+void rrd_functions_inflight_init(void);
+void rrdfunctions_host_init(RRDHOST *host);
+void rrdfunctions_host_destroy(RRDHOST *host);
 
-typedef int (*function_execute_at_collector)(BUFFER *wb, int timeout, const char *function, void *collector_data,
-                                             function_data_ready_callback callback, void *callback_data);
+void rrd_collector_started(void);
+void rrd_collector_finished(void);
 
-void rrd_collector_add_function(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help,
-                                       bool sync, function_execute_at_collector function, void *collector_data);
+// add a function, to be run from the collector
+void rrd_function_add(RRDHOST *host, RRDSET *st, const char *name, int timeout, const char *help,
+                      bool sync, rrd_function_execute_cb_t execute_cb, void *execute_cb_data);
 
-int rrd_call_function_and_wait(RRDHOST *host, BUFFER *wb, int timeout, const char *name);
+// call a function, to be run from anywhere
+int rrd_function_run(RRDHOST *host, BUFFER *result_wb, int timeout, const char *cmd,
+                     bool wait, const char *transaction,
+                     rrd_function_result_callback_t result_cb, void *result_cb_data,
+                     rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data, const char *payload);
 
-typedef void (*rrd_call_function_async_callback)(BUFFER *wb, int code, void *callback_data);
-int rrd_call_function_async(RRDHOST *host, BUFFER *wb, int timeout, const char *name, rrd_call_function_async_callback, void *callback_data);
+// cancel a running function, to be run from anywhere
+void rrd_function_cancel(const char *transaction);
 
 void rrd_functions_expose_rrdpush(RRDSET *st, BUFFER *wb);
 void rrd_functions_expose_global_rrdpush(RRDHOST *host, BUFFER *wb);
@@ -35,7 +50,9 @@ const char *functions_content_type_to_format(HTTP_CONTENT_TYPE content_type);
 int rrd_call_function_error(BUFFER *wb, const char *msg, int code);
 
 int rrdhost_function_streaming(BUFFER *wb, int timeout, const char *function, void *collector_data,
-                               function_data_ready_callback callback, void *callback_data);
+                               rrd_function_result_callback_t result_cb, void *result_cb_data,
+                               rrd_function_is_cancelled_cb_t is_cancelled_cb, void *is_cancelled_cb_data,
+                               rrd_function_register_canceller_cb_t register_canceller_cb, void *register_canceller_cb_data);
 
 #define RRDFUNCTIONS_STREAMING_HELP "Streaming status for parents and children."
 
diff --git a/database/rrdhost.c b/database/rrdhost.c
index bec821ccc..6abd3b816 100644
--- a/database/rrdhost.c
+++ b/database/rrdhost.c
@@ -331,8 +331,12 @@ int is_legacy = 1;
     host->rrd_history_entries        = align_entries_to_pagesize(memory_mode, entries);
     host->health.health_enabled      = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health_enabled;
 
+    netdata_mutex_init(&host->aclk_state_lock);
+    netdata_mutex_init(&host->receiver_lock);
+
     if (likely(!archived)) {
-        rrdfunctions_init(host);
+        rrdfunctions_host_init(host);
+        host->last_connected = now_realtime_sec();
         host->rrdlabels = rrdlabels_create();
         rrdhost_initialize_rrdpush_sender(
             host, rrdpush_enabled, rrdpush_destination, rrdpush_api_key, rrdpush_send_charts_matching);
@@ -361,9 +365,6 @@ int is_legacy = 1;
             break;
     }
 
-    netdata_mutex_init(&host->aclk_state_lock);
-    netdata_mutex_init(&host->receiver_lock);
-
     host->system_info = system_info;
 
     rrdset_index_init(host);
@@ -561,6 +562,9 @@ int is_legacy = 1;
          , string2str(host->health.health_default_recipient)
     );
 
+    host->configurable_plugins = dyncfg_dictionary_create();
+    dictionary_register_delete_callback(host->configurable_plugins, plugin_del_cb, NULL);
+
     if(!archived) {
         metaqueue_host_update_info(host);
         rrdhost_load_rrdcontext_data(host);
@@ -662,10 +666,12 @@ static void rrdhost_update(RRDHOST *host
     if(!host->rrdvars)
         host->rrdvars = rrdvariables_create();
 
+    host->last_connected = now_realtime_sec();
+
     if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
         rrdhost_flag_clear(host, RRDHOST_FLAG_ARCHIVED);
 
-        rrdfunctions_init(host);
+        rrdfunctions_host_init(host);
 
         if(!host->rrdlabels)
             host->rrdlabels = rrdlabels_create();
@@ -1070,9 +1076,9 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info, bool unitt
     // we register this only on localhost
     // for the other nodes, the origin server should register it
     rrd_collector_started(); // this creates a collector that runs for as long as netdata runs
-    rrd_collector_add_function(localhost, NULL, "streaming", 10,
-                               RRDFUNCTIONS_STREAMING_HELP, true,
-                               rrdhost_function_streaming, NULL);
+    rrd_function_add(localhost, NULL, "streaming", 10,
+                     RRDFUNCTIONS_STREAMING_HELP, true,
+                     rrdhost_function_streaming, NULL);
 #endif
 
     if (likely(system_info)) {
@@ -1160,9 +1166,11 @@ static void rrdhost_streaming_sender_structures_free(RRDHOST *host)
 
     rrdpush_sender_thread_stop(host, STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, true); // stop a possibly running thread
     cbuffer_free(host->sender->buffer);
+
 #ifdef ENABLE_RRDPUSH_COMPRESSION
     rrdpush_compressor_destroy(&host->sender->compressor);
 #endif
+
     replication_cleanup_sender(host->sender);
 
     __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(*host->sender), __ATOMIC_RELAXED);
@@ -1266,7 +1274,7 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force) {
     freez(host->node_id);
 
     rrdfamily_index_destroy(host);
-    rrdfunctions_destroy(host);
+    rrdfunctions_host_destroy(host);
     rrdvariables_destroy(host->rrdvars);
     if (host == localhost)
         rrdvariables_destroy(health_rrdvars);
@@ -1317,7 +1325,7 @@ void rrdhost_save_charts(RRDHOST *host) {
     rrdset_foreach_done(st);
 }
 
-struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
+struct rrdhost_system_info *rrdhost_labels_to_system_info(RRDLABELS *labels) {
     struct rrdhost_system_info *info = callocz(1, sizeof(struct rrdhost_system_info));
     info->hops = 1;
 
@@ -1345,7 +1353,7 @@ struct rrdhost_system_info *rrdhost_labels_to_system_info(DICTIONARY *labels) {
 }
 
 static void rrdhost_load_auto_labels(void) {
-    DICTIONARY *labels = localhost->rrdlabels;
+    RRDLABELS *labels = localhost->rrdlabels;
 
     if (localhost->system_info->cloud_provider_type)
         rrdlabels_add(labels, "_cloud_provider_type", localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
@@ -1418,7 +1426,7 @@ void rrdhost_set_is_parent_label(void) {
     int count = __atomic_load_n(&localhost->connected_children_count, __ATOMIC_RELAXED);
 
     if (count == 0 || count == 1) {
-        DICTIONARY *labels = localhost->rrdlabels;
+        RRDLABELS *labels = localhost->rrdlabels;
         rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);
 
         //queue a node info
@@ -1855,7 +1863,9 @@ void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
 
         s->stream.since = host->sender->last_state_since_t;
         s->stream.peers = socket_peers(host->sender->rrdpush_sender_socket);
+#ifdef ENABLE_HTTPS
         s->stream.ssl = SSL_connection(&host->sender->ssl);
+#endif
 
         memcpy(s->stream.sent_bytes_on_this_connection_per_type,
                host->sender->sent_bytes_on_this_connection_per_type,
diff --git a/database/rrdlabels.c b/database/rrdlabels.c
index 77d9a91f0..243b16c69 100644
--- a/database/rrdlabels.c
+++ b/database/rrdlabels.c
@@ -3,6 +3,91 @@
 #define NETDATA_RRD_INTERNALS
 #include "rrd.h"
 
+// Key OF HS ARRRAY
+
+struct {
+    Pvoid_t JudyHS;
+    SPINLOCK spinlock;
+} global_labels = {
+    .JudyHS = (Pvoid_t) NULL,
+    .spinlock = NETDATA_SPINLOCK_INITIALIZER
+};
+
+typedef struct label_registry_idx {
+    STRING *key;
+    STRING *value;
+} LABEL_REGISTRY_IDX;
+
+typedef struct labels_registry_entry {
+    LABEL_REGISTRY_IDX index;
+} RRDLABEL;
+
+// Value of HS array
+typedef struct labels_registry_idx_entry {
+    RRDLABEL label;
+    size_t refcount;
+} RRDLABEL_IDX;
+
+typedef struct rrdlabels {
+    SPINLOCK spinlock;
+    size_t version;
+    Pvoid_t JudyL;
+} RRDLABELS;
+
+#define lfe_start_nolock(label_list, label, ls)                                                                        \
+    do {                                                                                                               \
+        bool _first_then_next = true;                                                                                  \
+        Pvoid_t *_PValue;                                                                                              \
+        Word_t _Index = 0;                                                                                             \
+        while ((_PValue = JudyLFirstThenNext((label_list)->JudyL, &_Index, &_first_then_next))) {                      \
+            (ls) = *(RRDLABEL_SRC *)_PValue;                                                                           \
+            (void)(ls);                                                                                                \
+            (label) = (void *)_Index;
+
+#define lfe_done_nolock()                                                                                              \
+        }                                                                                                              \
+    }                                                                                                                  \
+    while (0)
+
+#define lfe_start_read(label_list, label, ls)                                                                          \
+    do {                                                                                                               \
+        spinlock_lock(&(label_list)->spinlock);                                                                        \
+        bool _first_then_next = true;                                                                                  \
+        Pvoid_t *_PValue;                                                                                              \
+        Word_t _Index = 0;                                                                                             \
+        while ((_PValue = JudyLFirstThenNext((label_list)->JudyL, &_Index, &_first_then_next))) {                      \
+            (ls) = *(RRDLABEL_SRC *)_PValue;                                                                           \
+            (void)(ls);                                                                                                \
+            (label) = (void *)_Index;
+
+#define lfe_done(label_list)                                                                                           \
+        }                                                                                                              \
+        spinlock_unlock(&(label_list)->spinlock);                                                                      \
+    }                                                                                                                  \
+    while (0)
+
+static inline void STATS_PLUS_MEMORY(struct dictionary_stats *stats, size_t key_size, size_t item_size, size_t value_size) {
+    if(key_size)
+        __atomic_fetch_add(&stats->memory.index, (long)JUDYHS_INDEX_SIZE_ESTIMATE(key_size), __ATOMIC_RELAXED);
+
+    if(item_size)
+        __atomic_fetch_add(&stats->memory.dict, (long)item_size, __ATOMIC_RELAXED);
+
+    if(value_size)
+        __atomic_fetch_add(&stats->memory.values, (long)value_size, __ATOMIC_RELAXED);
+}
+
+static inline void STATS_MINUS_MEMORY(struct dictionary_stats *stats, size_t key_size, size_t item_size, size_t value_size) {
+    if(key_size)
+        __atomic_fetch_sub(&stats->memory.index, (long)JUDYHS_INDEX_SIZE_ESTIMATE(key_size), __ATOMIC_RELAXED);
+
+    if(item_size)
+        __atomic_fetch_sub(&stats->memory.dict, (long)item_size, __ATOMIC_RELAXED);
+
+    if(value_size)
+        __atomic_fetch_sub(&stats->memory.values, (long)value_size, __ATOMIC_RELAXED);
+}
+
 // ----------------------------------------------------------------------------
 // labels sanitization
 
@@ -369,6 +454,12 @@ __attribute__((constructor)) void initialize_labels_keys_char_map(void) {
 
 }
 
+__attribute__((constructor)) void initialize_label_stats(void) {
+    dictionary_stats_category_rrdlabels.memory.dict = 0;
+    dictionary_stats_category_rrdlabels.memory.index = 0;
+    dictionary_stats_category_rrdlabels.memory.values = 0;
+}
+
 size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length) {
     if(unlikely(!dst_size)) return 0;
 
@@ -484,93 +575,164 @@ static inline size_t rrdlabels_sanitize_value(char *dst, const char *src, size_t
 // ----------------------------------------------------------------------------
 // rrdlabels_create()
 
-typedef struct rrdlabel {
-    STRING *label_value;
-    RRDLABEL_SRC label_source;
-} RRDLABEL;
-
-static void rrdlabel_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *dict_ptr __maybe_unused) {
-    RRDLABEL *lb = (RRDLABEL *)value;
-
-    // label_value is already allocated by the STRING
-    lb->label_source |=  RRDLABEL_FLAG_NEW;
-    lb->label_source &= ~RRDLABEL_FLAG_OLD;
+RRDLABELS *rrdlabels_create(void)
+{
+    RRDLABELS *labels = callocz(1, sizeof(*labels));
+    STATS_PLUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, sizeof(RRDLABELS), 0);
+    return labels;
 }
 
-static void rrdlabel_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *dict_ptr __maybe_unused) {
-    RRDLABEL *lb = (RRDLABEL *)value;
-
-    string_freez(lb->label_value);
-    lb->label_value = NULL;
-}
-
-static bool rrdlabel_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused, void *oldvalue, void *newvalue, void *dict_ptr __maybe_unused) {
-    RRDLABEL *lbold = (RRDLABEL *)oldvalue;
-    RRDLABEL *lbnew = (RRDLABEL *)newvalue;
-
-    if(lbold->label_value == lbnew->label_value) {
-        // they are the same
-
-        lbold->label_source |=  lbnew->label_source;
-        lbold->label_source |=  RRDLABEL_FLAG_OLD;
-        lbold->label_source &= ~RRDLABEL_FLAG_NEW;
+static void dup_label(RRDLABEL *label_index)
+{
+    if (!label_index)
+        return;
 
-        // free the new one
-        string_freez(lbnew->label_value);
+    spinlock_lock(&global_labels.spinlock);
 
-        return false;
+    Pvoid_t *PValue = JudyHSGet(global_labels.JudyHS, (void *)label_index, sizeof(*label_index));
+    if (PValue && *PValue) {
+        RRDLABEL_IDX *rrdlabel = *PValue;
+        __atomic_add_fetch(&rrdlabel->refcount, 1, __ATOMIC_RELAXED);
     }
 
-    // they are different
-
-    string_freez(lbold->label_value);
-    lbold->label_value  =   lbnew->label_value;
-    lbold->label_source =   lbnew->label_source;
-    lbold->label_source |=  RRDLABEL_FLAG_NEW;
-    lbold->label_source &= ~RRDLABEL_FLAG_OLD;
-
-    return true;
+    spinlock_unlock(&global_labels.spinlock);
 }
 
-DICTIONARY *rrdlabels_create(void) {
-    DICTIONARY *dict = dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE,
-                                                  &dictionary_stats_category_rrdlabels, sizeof(RRDLABEL));
+static RRDLABEL *add_label_name_value(const char *name, const char *value)
+{
+    RRDLABEL_IDX *rrdlabel = NULL;
+    LABEL_REGISTRY_IDX label_index;
+    label_index.key = string_strdupz(name);
+    label_index.value = string_strdupz(value);
+
+    spinlock_lock(&global_labels.spinlock);
+
+    Pvoid_t *PValue = JudyHSIns(&global_labels.JudyHS, (void *)&label_index, sizeof(label_index), PJE0);
+    if(unlikely(!PValue || PValue == PJERR))
+        fatal("RRDLABELS: corrupted judyHS array");
+
+    if (*PValue) {
+        rrdlabel = *PValue;
+        string_freez(label_index.key);
+        string_freez(label_index.value);
+    } else {
+        rrdlabel = callocz(1, sizeof(*rrdlabel));
+        rrdlabel->label.index = label_index;
+        *PValue = rrdlabel;
+        STATS_PLUS_MEMORY(&dictionary_stats_category_rrdlabels, sizeof(LABEL_REGISTRY_IDX), sizeof(RRDLABEL_IDX), 0);
+    }
+    __atomic_add_fetch(&rrdlabel->refcount, 1, __ATOMIC_RELAXED);
 
-    dictionary_register_insert_callback(dict, rrdlabel_insert_callback, dict);
-    dictionary_register_delete_callback(dict, rrdlabel_delete_callback, dict);
-    dictionary_register_conflict_callback(dict, rrdlabel_conflict_callback, dict);
-    return dict;
+    spinlock_unlock(&global_labels.spinlock);
+    return &rrdlabel->label;
 }
 
+static void delete_label(RRDLABEL *label)
+{
+    spinlock_lock(&global_labels.spinlock);
+
+    Pvoid_t *PValue = JudyHSGet(global_labels.JudyHS, &label->index, sizeof(label->index));
+    if (PValue && *PValue) {
+        RRDLABEL_IDX *rrdlabel = *PValue;
+        size_t refcount = __atomic_sub_fetch(&rrdlabel->refcount, 1, __ATOMIC_RELAXED);
+        if (refcount == 0) {
+            int ret = JudyHSDel(&global_labels.JudyHS, (void *)label, sizeof(*label), PJE0);
+            if (unlikely(ret == JERR))
+                STATS_MINUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, sizeof(*rrdlabel), 0);
+            else
+                STATS_MINUS_MEMORY(&dictionary_stats_category_rrdlabels, sizeof(LABEL_REGISTRY_IDX), sizeof(*rrdlabel), 0);
+            string_freez(label->index.key);
+            string_freez(label->index.value);
+            freez(rrdlabel);
+        }
+    }
+    spinlock_unlock(&global_labels.spinlock);
+}
 
 // ----------------------------------------------------------------------------
 // rrdlabels_destroy()
 
-void rrdlabels_destroy(DICTIONARY *labels_dict) {
-    dictionary_destroy(labels_dict);
+void rrdlabels_destroy(RRDLABELS *labels)
+{
+    if (unlikely(!labels))
+        return;
+
+    spinlock_lock(&labels->spinlock);
+
+    Pvoid_t *PValue;
+    Word_t Index = 0;
+    bool first_then_next = true;
+    while ((PValue = JudyLFirstThenNext(labels->JudyL, &Index, &first_then_next))) {
+        delete_label((RRDLABEL *)Index);
+    }
+    size_t memory_freed = JudyLFreeArray(&labels->JudyL, PJE0);
+    STATS_MINUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, memory_freed + sizeof(RRDLABELS), 0);
+    spinlock_unlock(&labels->spinlock);
+    freez(labels);
 }
 
-void rrdlabels_flush(DICTIONARY *labels_dict) {
-    dictionary_flush(labels_dict);
+static RRDLABEL *rrdlabels_find_label_with_key_unsafe(RRDLABELS *labels, RRDLABEL *label)
+{
+    if (unlikely(!labels))
+        return NULL;
+
+    Pvoid_t *PValue;
+    Word_t Index = 0;
+    bool first_then_next = true;
+    RRDLABEL *found = NULL;
+    while ((PValue = JudyLFirstThenNext(labels->JudyL, &Index, &first_then_next))) {
+        RRDLABEL *lb = (RRDLABEL *)Index;
+        if (lb->index.key == label->index.key && lb->index.value != label->index.value) {
+            found = (RRDLABEL *)Index;
+            break;
+        }
+    }
+    return found;
 }
 
 // ----------------------------------------------------------------------------
 // rrdlabels_add()
 
-static void labels_add_already_sanitized(DICTIONARY *dict, const char *key, const char *value, RRDLABEL_SRC ls) {
-    if(ls & RRDLABEL_FLAG_NEW) ls &= ~RRDLABEL_FLAG_NEW;
-    if(ls & RRDLABEL_FLAG_OLD) ls &= ~RRDLABEL_FLAG_OLD;
+static void labels_add_already_sanitized(RRDLABELS *labels, const char *key, const char *value, RRDLABEL_SRC ls)
+{
+    RRDLABEL *label = add_label_name_value(key, value);
 
-    RRDLABEL tmp = {
-        .label_source = ls,
-        .label_value = string_strdupz(value)
-    };
-    dictionary_set(dict, key, &tmp, sizeof(RRDLABEL));
-}
+    spinlock_lock(&labels->spinlock);
+
+    RRDLABEL *old_key = rrdlabels_find_label_with_key_unsafe(labels, label);
+
+    size_t mem_before_judyl = JudyLMemUsed(labels->JudyL);
+
+    Pvoid_t *PValue = JudyLIns(&labels->JudyL, (Word_t) label, PJE0);
+    if(unlikely(!PValue || PValue == PJERR))
+        fatal("RRDLABELS: corrupted labels JudyL array");
+
+    if (!*PValue) {
+        RRDLABEL_SRC new_ls;
+        if (old_key)
+            new_ls = ((ls & ~(RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_OLD)) | RRDLABEL_FLAG_OLD);
+        else
+            new_ls = ((ls & ~(RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_OLD)) | RRDLABEL_FLAG_NEW);
+        *((RRDLABEL_SRC *)PValue) = new_ls;
 
+        labels->version++;
 
-void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls) {
-    if(!dict) {
+        if (old_key) {
+            (void)JudyLDel(&labels->JudyL, (Word_t) old_key, PJE0);
+            delete_label((RRDLABEL *)old_key);
+        }
+        size_t mem_after_judyl = JudyLMemUsed(labels->JudyL);
+        STATS_PLUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, mem_after_judyl - mem_before_judyl, 0);
+    }
+    else
+        delete_label(label);
+
+    spinlock_unlock(&labels->spinlock);
+}
+
+void rrdlabels_add(RRDLABELS *labels, const char *name, const char *value, RRDLABEL_SRC ls)
+{
+    if(!labels) {
         netdata_log_error("%s(): called with NULL dictionary.", __FUNCTION__ );
         return;
     }
@@ -584,7 +746,30 @@ void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLAB
         return;
     }
 
-    labels_add_already_sanitized(dict, n, v, ls);
+    labels_add_already_sanitized(labels, n, v, ls);
+}
+
+bool rrdlabels_exist(RRDLABELS *labels, const char *key)
+{
+    if (!labels)
+        return false;
+
+    STRING *this_key = string_strdupz(key);
+
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
+
+    bool found = false;
+    lfe_start_read(labels, lb, ls)
+    {
+        if (lb->index.key == this_key) {
+            found = true;
+            break;
+        }
+    }
+    lfe_done(labels);
+    string_freez(this_key);
+    return found;
 }
 
 static const char *get_quoted_string_up_to(char *dst, size_t dst_size, const char *string, char upto1, char upto2) {
@@ -619,8 +804,9 @@ static const char *get_quoted_string_up_to(char *dst, size_t dst_size, const cha
     return string;
 }
 
-void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) {
-    if(!dict) {
+void rrdlabels_add_pair(RRDLABELS *labels, const char *string, RRDLABEL_SRC ls)
+{
+    if(!labels) {
         netdata_log_error("%s(): called with NULL dictionary.", __FUNCTION__ );
         return;
     }
@@ -631,199 +817,243 @@ void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) {
     char value[RRDLABELS_MAX_VALUE_LENGTH + 1];
     get_quoted_string_up_to(value, RRDLABELS_MAX_VALUE_LENGTH, string, '\0', '\0');
 
-    rrdlabels_add(dict, name, value, ls);
+    rrdlabels_add(labels, name, value, ls);
 }
 
 // ----------------------------------------------------------------------------
-// rrdlabels_get_value_to_buffer_or_null()
 
-void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null) {
+void rrdlabels_value_to_buffer_array_item_or_null(RRDLABELS *labels, BUFFER *wb, const char *key) {
     if(!labels) return;
 
-    const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
-    RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
-
-    if(lb && lb->label_value)
-        buffer_sprintf(wb, "%s%s%s", quote, string2str(lb->label_value), quote);
-    else
-        buffer_strcat(wb, null);
+    STRING *this_key = string_strdupz(key);
 
-    dictionary_acquired_item_release(labels, acquired_item);
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
+    lfe_start_read(labels, lb, ls)
+    {
+        if (lb->index.key == this_key) {
+            if (lb->index.value)
+                buffer_json_add_array_item_string(wb, string2str(lb->index.value));
+            else
+                buffer_json_add_array_item_string(wb, NULL);
+            break;
+        }
+    }
+    lfe_done(labels);
+    string_freez(this_key);
 }
 
-void rrdlabels_value_to_buffer_array_item_or_null(DICTIONARY *labels, BUFFER *wb, const char *key) {
+// ----------------------------------------------------------------------------
+
+void rrdlabels_get_value_strcpyz(RRDLABELS *labels, char *dst, size_t dst_len, const char *key) {
     if(!labels) return;
 
-    const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
-    RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
+    STRING *this_key = string_strdupz(key);
 
-    if(lb && lb->label_value)
-        buffer_json_add_array_item_string(wb, string2str(lb->label_value));
-    else
-        buffer_json_add_array_item_string(wb, NULL);
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
 
-    dictionary_acquired_item_release(labels, acquired_item);
+    lfe_start_read(labels, lb, ls)
+    {
+        if (lb->index.key == this_key) {
+            if (lb->index.value)
+                strncpyz(dst, string2str(lb->index.value), dst_len);
+            else
+                dst[0] = '\0';
+            break;
+        }
+    }
+    lfe_done(labels);
+    string_freez(this_key);
 }
 
-// ----------------------------------------------------------------------------
-// rrdlabels_get_value_to_char_or_null()
-
-void rrdlabels_get_value_strdup_or_null(DICTIONARY *labels, char **value, const char *key) {
-    const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
-    RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
+void rrdlabels_get_value_strdup_or_null(RRDLABELS *labels, char **value, const char *key)
+{
+    if(!labels) return;
 
-    *value = (lb && lb->label_value) ? strdupz(string2str(lb->label_value)) : NULL;
+    STRING *this_key = string_strdupz(key);
 
-    dictionary_acquired_item_release(labels, acquired_item);
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
+    lfe_start_read(labels, lb, ls)
+    {
+        if (lb->index.key == this_key) {
+            *value = (lb->index.value) ? strdupz(string2str(lb->index.value)) : NULL;
+            break;
+        }
+    }
+    lfe_done(labels);
+    string_freez(this_key);
 }
 
-void rrdlabels_get_value_strcpyz(DICTIONARY *labels, char *dst, size_t dst_len, const char *key) {
-    const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
-    RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
+void rrdlabels_get_value_to_buffer_or_unset(RRDLABELS *labels, BUFFER *wb, const char *key, const char *unset)
+{
+    if(!labels || !key || !wb) return;
 
-    if(lb && lb->label_value)
-        strncpyz(dst, string2str(lb->label_value), dst_len);
-    else
-        dst[0] = '\0';
+    STRING *this_key = string_strdupz(key);
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
 
-    dictionary_acquired_item_release(labels, acquired_item);
+    lfe_start_read(labels, lb, ls)
+    {
+        if (lb->index.key == this_key) {
+            if (lb->index.value)
+                buffer_strcat(wb, string2str(lb->index.value));
+            else
+                buffer_strcat(wb, unset);
+            break;
+        }
+    }
+    lfe_done(labels);
+    string_freez(this_key);
 }
 
-STRING *rrdlabels_get_value_string_dup(DICTIONARY *labels, const char *key) {
-    const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
-    RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
-
-    STRING *ret = NULL;
-    if(lb && lb->label_value)
-        ret = string_dup(lb->label_value);
-
-    dictionary_acquired_item_release(labels, acquired_item);
-
-    return ret;
+static void rrdlabels_unmark_all_unsafe(RRDLABELS *labels)
+{
+    Pvoid_t *PValue;
+    Word_t Index = 0;
+    bool first_then_next = true;
+    while ((PValue = JudyLFirstThenNext(labels->JudyL, &Index, &first_then_next)))
+        *((RRDLABEL_SRC *)PValue) &= ~(RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW);
 }
 
-STRING *rrdlabels_get_value_to_buffer_or_unset(DICTIONARY *labels, BUFFER *wb, const char *key, const char *unset) {
-    const DICTIONARY_ITEM *acquired_item = dictionary_get_and_acquire_item(labels, key);
-    RRDLABEL *lb = dictionary_acquired_item_value(acquired_item);
-
-    STRING *ret = NULL;
-    if(lb && lb->label_value)
-        buffer_strcat(wb, string2str(lb->label_value));
-    else
-        buffer_strcat(wb, unset);
+void rrdlabels_unmark_all(RRDLABELS *labels)
+{
+    spinlock_lock(&labels->spinlock);
 
-    dictionary_acquired_item_release(labels, acquired_item);
+    rrdlabels_unmark_all_unsafe(labels);
 
-    return ret;
+    spinlock_unlock(&labels->spinlock);
 }
 
-// ----------------------------------------------------------------------------
-// rrdlabels_unmark_all()
-// remove labels RRDLABEL_FLAG_OLD and RRDLABEL_FLAG_NEW from all dictionary items
+static void rrdlabels_remove_all_unmarked_unsafe(RRDLABELS *labels)
+{
+    Pvoid_t *PValue;
+    Word_t Index = 0;
+    bool first_then_next = true;
 
-static int remove_flags_old_new(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
-    RRDLABEL *lb = (RRDLABEL *)value;
+    while ((PValue = JudyLFirstThenNext(labels->JudyL, &Index, &first_then_next))) {
+        if (!((*((RRDLABEL_SRC *)PValue)) & (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT))) {
 
-    if(lb->label_source & RRDLABEL_FLAG_OLD) lb->label_source &= ~RRDLABEL_FLAG_OLD;
-    if(lb->label_source & RRDLABEL_FLAG_NEW) lb->label_source &= ~RRDLABEL_FLAG_NEW;
+            size_t mem_before_judyl = JudyLMemUsed(labels->JudyL);
+            (void)JudyLDel(&labels->JudyL, Index, PJE0);
+            size_t mem_after_judyl = JudyLMemUsed(labels->JudyL);
 
-    return 1;
-}
+            STATS_MINUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, mem_before_judyl - mem_after_judyl, 0);
 
-void rrdlabels_unmark_all(DICTIONARY *labels) {
-    dictionary_walkthrough_read(labels, remove_flags_old_new, NULL);
+            delete_label((RRDLABEL *)Index);
+            if (labels->JudyL != (Pvoid_t) NULL) {
+                Index = 0;
+                first_then_next = true;
+            }
+        }
+    }
 }
 
+void rrdlabels_remove_all_unmarked(RRDLABELS *labels)
+{
+    spinlock_lock(&labels->spinlock);
+    rrdlabels_remove_all_unmarked_unsafe(labels);
+    spinlock_unlock(&labels->spinlock);
+}
 
 // ----------------------------------------------------------------------------
-// rrdlabels_remove_all_unmarked()
-// remove dictionary items that are neither old, nor new
+// rrdlabels_walkthrough_read()
 
-static int remove_not_old_not_new_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
-    DICTIONARY *dict = (DICTIONARY *)data;
-    RRDLABEL *lb = (RRDLABEL *)value;
+int rrdlabels_walkthrough_read(RRDLABELS *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data)
+{
+    int ret = 0;
 
-    if(!(lb->label_source & (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT))) {
-        dictionary_del(dict, name);
-        return 1;
-    }
+    if(unlikely(!labels || !callback)) return 0;
 
-    return 0;
-}
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
+    lfe_start_read(labels, lb, ls)
+    {
+        ret = callback(string2str(lb->index.key), string2str(lb->index.value), ls, data);
+        if (ret < 0)
+            break;
+    }
+    lfe_done(labels);
 
-void rrdlabels_remove_all_unmarked(DICTIONARY *labels) {
-    dictionary_walkthrough_write(labels, remove_not_old_not_new_callback, labels);
+    return ret;
 }
 
-
 // ----------------------------------------------------------------------------
-// rrdlabels_walkthrough_read()
-
-struct labels_walkthrough {
-    int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
-    void *data;
-};
+// rrdlabels_migrate_to_these()
+// migrate an existing label list to a new list
 
-static int labels_walkthrough_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
-    struct labels_walkthrough *d = (struct labels_walkthrough *)data;
-    RRDLABEL *lb = (RRDLABEL *)value;
+void rrdlabels_migrate_to_these(RRDLABELS *dst, RRDLABELS *src) {
+    if (!dst || !src)
+        return;
 
-    RRDLABEL_SRC ls = lb->label_source;
-    if(ls & RRDLABEL_FLAG_NEW) ls &= ~RRDLABEL_FLAG_NEW;
-    if(ls & RRDLABEL_FLAG_OLD) ls &= ~RRDLABEL_FLAG_OLD;
+    spinlock_lock(&dst->spinlock);
+    spinlock_lock(&src->spinlock);
 
-    return d->callback(name, string2str(lb->label_value), ls, d->data);
-}
+    rrdlabels_unmark_all_unsafe(dst);
 
-int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data) {
-    struct labels_walkthrough d = {
-        .callback = callback,
-        .data = data
-    };
-    return dictionary_walkthrough_read(labels, labels_walkthrough_callback, &d);
-}
+    RRDLABEL *label;
+    Pvoid_t *PValue;
 
-int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data) {
-    struct labels_walkthrough d = {
-        .callback = callback,
-        .data = data
-    };
-    return dictionary_sorted_walkthrough_read(labels, labels_walkthrough_callback, &d);
-}
+    RRDLABEL_SRC ls;
+    lfe_start_nolock(src, label, ls)
+    {
+        size_t mem_before_judyl = JudyLMemUsed(dst->JudyL);
+        PValue = JudyLIns(&dst->JudyL, (Word_t)label, PJE0);
+        if(unlikely(!PValue || PValue == PJERR))
+            fatal("RRDLABELS migrate: corrupted labels array");
+
+        RRDLABEL_SRC flag = RRDLABEL_FLAG_NEW;
+        if (!*PValue) {
+            dup_label(label);
+            size_t mem_after_judyl = JudyLMemUsed(dst->JudyL);
+            STATS_PLUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, mem_after_judyl - mem_before_judyl, 0);
+        }
+        else
+            flag = RRDLABEL_FLAG_OLD;
 
+        *((RRDLABEL_SRC *)PValue) |= flag;
+    }
+    lfe_done_nolock();
 
-// ----------------------------------------------------------------------------
-// rrdlabels_migrate_to_these()
-// migrate an existing label list to a new list, INPLACE
+    rrdlabels_remove_all_unmarked_unsafe(dst);
+    dst->version = src->version;
 
-static int copy_label_to_dictionary_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
-    DICTIONARY *dst = (DICTIONARY *)data;
-    RRDLABEL *lb = (RRDLABEL *)value;
-    labels_add_already_sanitized(dst, name, string2str(lb->label_value), lb->label_source);
-    return 1;
+    spinlock_unlock(&src->spinlock);
+    spinlock_unlock(&dst->spinlock);
 }
 
-void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src) {
-    if(!dst || !src) return;
-
-    // remove the RRDLABEL_FLAG_OLD and RRDLABEL_FLAG_NEW from all items
-    rrdlabels_unmark_all(dst);
+void rrdlabels_copy(RRDLABELS *dst, RRDLABELS *src)
+{
+    if (!dst || !src)
+        return;
 
-    // Mark the existing ones as RRDLABEL_FLAG_OLD,
-    // or the newly added ones as RRDLABEL_FLAG_NEW
-    dictionary_walkthrough_read(src, copy_label_to_dictionary_callback, dst);
+    RRDLABEL *label;
+    RRDLABEL_SRC ls;
 
-    // remove the unmarked dst
-    rrdlabels_remove_all_unmarked(dst);
-}
+    spinlock_lock(&dst->spinlock);
+    spinlock_lock(&src->spinlock);
+
+    lfe_start_nolock(src, label, ls)
+    {
+        size_t mem_before_judyl = JudyLMemUsed(dst->JudyL);
+        Pvoid_t *PValue = JudyLIns(&dst->JudyL, (Word_t)label, PJE0);
+        if(unlikely(!PValue || PValue == PJERR))
+            fatal("RRDLABELS: corrupted labels array");
+
+        if (!*PValue) {
+            dup_label(label);
+            size_t mem_after_judyl = JudyLMemUsed(dst->JudyL);
+            STATS_PLUS_MEMORY(&dictionary_stats_category_rrdlabels, 0, mem_after_judyl - mem_before_judyl, 0);
+            *((RRDLABEL_SRC *)PValue) = ls;
+        }
+    }
+    lfe_done_nolock();
 
-void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src) {
-    if(!dst || !src) return;
+    dst->version = src->version;
 
-    dictionary_walkthrough_read(src, copy_label_to_dictionary_callback, dst);
+    spinlock_unlock(&src->spinlock);
+    spinlock_unlock(&dst->spinlock);
 }
 
 
@@ -837,8 +1067,7 @@ struct simple_pattern_match_name_value {
     char equal;
 };
 
-static int simple_pattern_match_name_only_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
+static int simple_pattern_match_name_only_callback(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
     struct simple_pattern_match_name_value *t = (struct simple_pattern_match_name_value *)data;
     (void)value;
 
@@ -849,10 +1078,8 @@ static int simple_pattern_match_name_only_callback(const DICTIONARY_ITEM *item,
     return 0;
 }
 
-static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
+static int simple_pattern_match_name_and_value_callback(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
     struct simple_pattern_match_name_value *t = (struct simple_pattern_match_name_value *)data;
-    RRDLABEL *lb = (RRDLABEL *)value;
 
     // we return -1 to stop the walkthrough on first match
     t->searches++;
@@ -860,7 +1087,7 @@ static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *i
 
     size_t len = RRDLABELS_MAX_NAME_LENGTH + RRDLABELS_MAX_VALUE_LENGTH + 2; // +1 for =, +1 for \0
     char tmp[len], *dst = &tmp[0];
-    const char *v = string2str(lb->label_value);
+    const char *v = value;
 
     // copy the name
     while(*name) *dst++ = *name++;
@@ -881,7 +1108,7 @@ static int simple_pattern_match_name_and_value_callback(const DICTIONARY_ITEM *i
     return 0;
 }
 
-bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches) {
+bool rrdlabels_match_simple_pattern_parsed(RRDLABELS *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches) {
     if (!labels) return false;
 
     struct simple_pattern_match_name_value t = {
@@ -890,7 +1117,7 @@ bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *p
         .equal = equal
     };
 
-    int ret = dictionary_walkthrough_read(labels, equal?simple_pattern_match_name_and_value_callback:simple_pattern_match_name_only_callback, &t);
+    int ret = rrdlabels_walkthrough_read(labels, equal?simple_pattern_match_name_and_value_callback:simple_pattern_match_name_only_callback, &t);
 
     if(searches)
         *searches = t.searches;
@@ -898,7 +1125,7 @@ bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *p
     return (ret == -1)?true:false;
 }
 
-bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt) {
+bool rrdlabels_match_simple_pattern(RRDLABELS *labels, const char *simple_pattern_txt) {
     if (!labels) return false;
 
     SIMPLE_PATTERN *pattern = simple_pattern_create(simple_pattern_txt, " ,|\t\r\n\f\v", SIMPLE_PATTERN_EXACT, true);
@@ -923,39 +1150,23 @@ bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_patte
 // ----------------------------------------------------------------------------
 // Log all labels
 
-static int rrdlabels_log_label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
-
+static int rrdlabels_log_label_to_buffer_callback(const char *name, const char *value, void *data) {
     BUFFER *wb = (BUFFER *)data;
-    RRDLABEL *lb = (RRDLABEL *)value;
-
-    buffer_sprintf(wb, "Label: %s: \"%s\" (", name, string2str(lb->label_value));
-
-    size_t sources = 0;
-    if(lb->label_source & RRDLABEL_SRC_AUTO) {
-        buffer_sprintf(wb, "auto");
-        sources++;
-    }
-
-    if(lb->label_source & RRDLABEL_SRC_CONFIG)
-        buffer_sprintf(wb, "%snetdata.conf", sources++?",":"");
-
-    if(lb->label_source & RRDLABEL_SRC_K8S)
-        buffer_sprintf(wb, "%sk8s", sources++?",":"");
-
-    if(lb->label_source & RRDLABEL_SRC_ACLK)
-        buffer_sprintf(wb, "%saclk", sources++?",":"");
-
-    if(!sources)
-        buffer_strcat(wb, "unknown");
 
+    buffer_sprintf(wb, "Label: %s: \"%s\" (", name, value);
+    buffer_strcat(wb, "unknown");
     buffer_strcat(wb, ")\n");
 
     return 1;
 }
 
-void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb) {
-    dictionary_sorted_walkthrough_read(labels, rrdlabels_log_label_to_buffer_callback, wb);
+void rrdlabels_log_to_buffer(RRDLABELS *labels, BUFFER *wb)
+{
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
+    lfe_start_read(labels, lb, ls)
+        rrdlabels_log_label_to_buffer_callback((void *) string2str(lb->index.key), (void *) string2str(lb->index.value), wb);
+    lfe_done(labels);
 }
 
 
@@ -975,10 +1186,10 @@ struct labels_to_buffer {
     size_t count;
 };
 
-static int label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, void *data) {
-    const char *name = dictionary_acquired_item_name(item);
+static int label_to_buffer_callback(const RRDLABEL *lb, void *value __maybe_unused, RRDLABEL_SRC ls, void *data)
+{
+
     struct labels_to_buffer *t = (struct labels_to_buffer *)data;
-    RRDLABEL *lb = (RRDLABEL *)value;
 
     size_t n_size = (t->name_sanitizer ) ? ( RRDLABELS_MAX_NAME_LENGTH  * 2 ) : 1;
     size_t v_size = (t->value_sanitizer) ? ( RRDLABELS_MAX_VALUE_LENGTH * 2 ) : 1;
@@ -986,7 +1197,9 @@ static int label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, vo
     char n[n_size];
     char v[v_size];
 
-    const char *nn = name, *vv = string2str(lb->label_value);
+    const char *name = string2str(lb->index.key);
+
+    const char *nn = name, *vv = string2str(lb->index.value);
 
     if(t->name_sanitizer) {
         t->name_sanitizer(n, name, n_size);
@@ -994,11 +1207,11 @@ static int label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, vo
     }
 
     if(t->value_sanitizer) {
-        t->value_sanitizer(v, string2str(lb->label_value), v_size);
+        t->value_sanitizer(v, string2str(lb->index.value), v_size);
         vv = v;
     }
 
-    if(!t->filter_callback || t->filter_callback(name, string2str(lb->label_value), lb->label_source, t->filter_data)) {
+    if(!t->filter_callback || t->filter_callback(name, string2str(lb->index.value), ls, t->filter_data)) {
         buffer_sprintf(t->wb, "%s%s%s%s%s%s%s%s%s", t->count++?t->between_them:"", t->before_each, t->quote, nn, t->quote, t->equal, t->quote, vv, t->quote);
         return 1;
     }
@@ -1006,7 +1219,26 @@ static int label_to_buffer_callback(const DICTIONARY_ITEM *item, void *value, vo
     return 0;
 }
 
-int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size)) {
+
+int label_walkthrough_read(RRDLABELS *labels, int (*callback)(const RRDLABEL *item, void *entry, RRDLABEL_SRC ls, void *data), void *data)
+{
+    int ret = 0;
+
+    if(unlikely(!labels || !callback)) return 0;
+
+    RRDLABEL *lb;
+    RRDLABEL_SRC ls;
+    lfe_start_read(labels, lb, ls)
+    {
+        ret = callback((const RRDLABEL *)lb, (void *)string2str(lb->index.value), ls, data);
+        if (ret < 0)
+            break;
+    }
+    lfe_done(labels);
+    return ret;
+}
+
+int rrdlabels_to_buffer(RRDLABELS *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size)) {
     struct labels_to_buffer tmp = {
         .wb = wb,
         .filter_callback = filter_callback,
@@ -1019,18 +1251,33 @@ int rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each,
         .between_them = between_them,
         .count = 0
     };
-    return dictionary_walkthrough_read(labels, label_to_buffer_callback, (void *)&tmp);
+    return label_walkthrough_read(labels, label_to_buffer_callback, (void *)&tmp);
 }
 
-void rrdlabels_to_buffer_json_members(DICTIONARY *labels, BUFFER *wb) {
+void rrdlabels_to_buffer_json_members(RRDLABELS *labels, BUFFER *wb)
+{
     RRDLABEL *lb;
-    dfe_start_read(labels, lb) {
-        buffer_json_member_add_string(wb, lb_dfe.name, string2str(lb->label_value));
-    }
-    dfe_done(lb);
+    RRDLABEL_SRC ls;
+    lfe_start_read(labels, lb, ls)
+        buffer_json_member_add_string(wb, string2str(lb->index.key), string2str(lb->index.value));
+    lfe_done(labels);
+}
+
+size_t rrdlabels_entries(RRDLABELS *labels __maybe_unused)
+{
+    size_t count;
+    spinlock_lock(&labels->spinlock);
+    count = JudyLCount(labels->JudyL, 0, -1, PJE0);
+    spinlock_unlock(&labels->spinlock);
+    return count;
 }
 
-void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
+size_t rrdlabels_version(RRDLABELS *labels __maybe_unused)
+{
+    return (size_t) labels->version;
+}
+
+void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels) {
     if(!st->rrdlabels)
         st->rrdlabels = rrdlabels_create();
 
@@ -1051,16 +1298,14 @@ struct rrdlabels_unittest_add_a_pair {
     const char *expected_value;
     const char *name;
     const char *value;
-    RRDLABEL_SRC ls;
     int errors;
 };
 
-int rrdlabels_unittest_add_a_pair_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+int rrdlabels_unittest_add_a_pair_callback(const char *name, const char *value, RRDLABEL_SRC ls __maybe_unused, void *data) {
     struct rrdlabels_unittest_add_a_pair *t = (struct rrdlabels_unittest_add_a_pair *)data;
 
     t->name = name;
     t->value = value;
-    t->ls = ls;
 
     if(strcmp(name, t->expected_name) != 0) {
         fprintf(stderr, "name is wrong, found \"%s\", expected \"%s\"", name, t->expected_name);
@@ -1083,7 +1328,7 @@ int rrdlabels_unittest_add_a_pair_callback(const char *name, const char *value,
 }
 
 int rrdlabels_unittest_add_a_pair(const char *pair, const char *name, const char *value) {
-    DICTIONARY *labels = rrdlabels_create();
+    RRDLABELS *labels = rrdlabels_create();
     int errors;
 
     fprintf(stderr, "rrdlabels_add_pair(labels, %s) ... ", pair);
@@ -1160,7 +1405,7 @@ int rrdlabels_unittest_add_pairs() {
     return errors;
 }
 
-int rrdlabels_unittest_check_simple_pattern(DICTIONARY *labels, const char *pattern, bool expected) {
+int rrdlabels_unittest_check_simple_pattern(RRDLABELS *labels, const char *pattern, bool expected) {
     fprintf(stderr, "rrdlabels_match_simple_pattern(labels, \"%s\") ... ", pattern);
 
     bool ret = rrdlabels_match_simple_pattern(labels, pattern);
@@ -1174,7 +1419,7 @@ int rrdlabels_unittest_simple_pattern() {
 
     int errors = 0;
 
-    DICTIONARY *labels = rrdlabels_create();
+    RRDLABELS *labels = rrdlabels_create();
     rrdlabels_add(labels, "tag1", "value1", RRDLABEL_SRC_CONFIG);
     rrdlabels_add(labels, "tag2", "value2", RRDLABEL_SRC_CONFIG);
     rrdlabels_add(labels, "tag3", "value3", RRDLABEL_SRC_CONFIG);
diff --git a/database/rrdlabels.h b/database/rrdlabels.h
new file mode 100644
index 000000000..c65fbb2c3
--- /dev/null
+++ b/database/rrdlabels.h
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_RRDLABELS_H
+#define NETDATA_RRDLABELS_H
+
+#include "rrd.h"
+
+typedef enum __attribute__ ((__packed__)) rrdlabel_source {
+    RRDLABEL_SRC_AUTO       = (1 << 0), // set when Netdata found the label by some automation
+    RRDLABEL_SRC_CONFIG     = (1 << 1), // set when the user configured the label
+    RRDLABEL_SRC_K8S        = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
+    RRDLABEL_SRC_ACLK       = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
+
+    // more sources can be added here
+
+    RRDLABEL_FLAG_PERMANENT = (1 << 29), // set when this label should never be removed (can be overwritten though)
+    RRDLABEL_FLAG_OLD       = (1 << 30), // marks for rrdlabels internal use - they are not exposed outside rrdlabels
+    RRDLABEL_FLAG_NEW       = (1 << 31)  // marks for rrdlabels internal use - they are not exposed outside rrdlabels
+} RRDLABEL_SRC;
+
+#define RRDLABEL_FLAG_INTERNAL (RRDLABEL_FLAG_OLD | RRDLABEL_FLAG_NEW | RRDLABEL_FLAG_PERMANENT)
+
+size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf, const char *empty, size_t *multibyte_length);
+
+RRDLABELS *rrdlabels_create(void);
+void rrdlabels_destroy(RRDLABELS *labels_dict);
+void rrdlabels_add(RRDLABELS *labels, const char *name, const char *value, RRDLABEL_SRC ls);
+void rrdlabels_add_pair(RRDLABELS *labels, const char *string, RRDLABEL_SRC ls);
+void rrdlabels_value_to_buffer_array_item_or_null(RRDLABELS *labels, BUFFER *wb, const char *key);
+void rrdlabels_get_value_strdup_or_null(RRDLABELS *labels, char **value, const char *key);
+void rrdlabels_get_value_to_buffer_or_unset(RRDLABELS *labels, BUFFER *wb, const char *key, const char *unset);
+bool rrdlabels_exist(RRDLABELS *labels, const char *key);
+size_t rrdlabels_entries(RRDLABELS *labels __maybe_unused);
+size_t rrdlabels_version(RRDLABELS *labels __maybe_unused);
+void rrdlabels_get_value_strcpyz(RRDLABELS *labels, char *dst, size_t dst_len, const char *key);
+
+void rrdlabels_unmark_all(RRDLABELS *labels);
+void rrdlabels_remove_all_unmarked(RRDLABELS *labels);
+
+int rrdlabels_walkthrough_read(RRDLABELS *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
+void rrdlabels_log_to_buffer(RRDLABELS *labels, BUFFER *wb);
+bool rrdlabels_match_simple_pattern(RRDLABELS *labels, const char *simple_pattern_txt);
+
+bool rrdlabels_match_simple_pattern_parsed(RRDLABELS *labels, SIMPLE_PATTERN *pattern, char equal, size_t *searches);
+int rrdlabels_to_buffer(RRDLABELS *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them,
+                        bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data,
+                        void (*name_sanitizer)(char *dst, const char *src, size_t dst_size),
+                        void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
+void rrdlabels_to_buffer_json_members(RRDLABELS *labels, BUFFER *wb);
+
+void rrdlabels_migrate_to_these(RRDLABELS *dst, RRDLABELS *src);
+void rrdlabels_copy(RRDLABELS *dst, RRDLABELS *src);
+
+int rrdlabels_unittest(void);
+
+// unfortunately this break when defined in exporting_engine.h
+bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
+
+#endif /* NETDATA_RRDLABELS_H */
diff --git a/database/rrdset.c b/database/rrdset.c
index 1e00d5c8a..cf8b9ef85 100644
--- a/database/rrdset.c
+++ b/database/rrdset.c
@@ -292,11 +292,6 @@ static bool rrdset_conflict_callback(const DICTIONARY_ITEM *item __maybe_unused,
 
     ctr->react_action = RRDSET_REACT_NONE;
 
-    if (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
-        rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
-        ctr->react_action |= RRDSET_REACT_CHART_ACTIVATED;
-    }
-
     if (rrdset_reset_name(st, (ctr->name && *ctr->name) ? ctr->name : ctr->id) == 2)
         ctr->react_action |= RRDSET_REACT_UPDATED;
 
@@ -657,11 +652,6 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_
 }
 
 inline void rrdset_is_obsolete(RRDSET *st) {
-    if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED))) {
-        netdata_log_info("Cannot obsolete already archived chart %s", rrdset_name(st));
-        return;
-    }
-
     if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
         rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
         rrdhost_flag_set(st->rrdhost, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS);
@@ -1046,7 +1036,7 @@ void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_las
                 last_time_s = now.tv_sec;
 
                 if(min_delta > permanent_min_delta) {
-                    netdata_log_info("MINIMUM MICROSECONDS DELTA of thread %d increased from %lld to %lld (+%lld)", gettid(), permanent_min_delta, min_delta, min_delta - permanent_min_delta);
+                    netdata_log_info("MINIMUM MICROSECONDS DELTA of thread %d increased from %"PRIi64" to %"PRIi64" (+%"PRIi64")", gettid(), permanent_min_delta, min_delta, min_delta - permanent_min_delta);
                     permanent_min_delta = min_delta;
                 }
 
@@ -1056,12 +1046,12 @@ void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t duration_since_las
 #endif
     }
 
-    netdata_log_debug(D_RRD_CALLS, "rrdset_timed_next() for chart %s with duration since last update %llu usec", rrdset_name(st), duration_since_last_update);
-    rrdset_debug(st, "NEXT: %llu microseconds", duration_since_last_update);
+    netdata_log_debug(D_RRD_CALLS, "rrdset_timed_next() for chart %s with duration since last update %"PRIu64" usec", rrdset_name(st), duration_since_last_update);
+    rrdset_debug(st, "NEXT: %"PRIu64" microseconds", duration_since_last_update);
 
     internal_error(discarded && discarded != duration_since_last_update,
-                   "host '%s', chart '%s': discarded data collection time of %llu usec, "
-                   "replaced with %llu usec, reason: '%s'"
+                   "host '%s', chart '%s': discarded data collection time of %"PRIu64" usec, "
+                   "replaced with %"PRIu64" usec, reason: '%s'"
                    , rrdhost_hostname(st->rrdhost)
                    , rrdset_id(st)
                    , discarded
@@ -1333,7 +1323,7 @@ static inline size_t rrdset_done_interpolate(
 
         internal_error(iterations < 0,
                        "RRDSET: '%s': iterations calculation wrapped! "
-                       "first_ut = %llu, last_stored_ut = %llu, next_store_ut = %llu, now_collect_ut = %llu"
+                       "first_ut = %"PRIu64", last_stored_ut = %"PRIu64", next_store_ut = %"PRIu64", now_collect_ut = %"PRIu64""
                        , rrdset_id(st)
                        , first_ut
                        , last_stored_ut
@@ -1366,8 +1356,8 @@ static inline size_t rrdset_done_interpolate(
 
                     rrdset_debug(st, "%s: CALC2 INC " NETDATA_DOUBLE_FORMAT " = "
                                  NETDATA_DOUBLE_FORMAT
-                                " * (%llu - %llu)"
-                                " / (%llu - %llu)"
+                                " * (%"PRIu64" - %"PRIu64")"
+                                " / (%"PRIu64" - %"PRIu64""
                               , rrddim_name(rd)
                               , new_value
                               , rd->collector.calculated_value
@@ -1416,8 +1406,8 @@ static inline size_t rrdset_done_interpolate(
 
                         rrdset_debug(st, "%s: CALC2 DEF " NETDATA_DOUBLE_FORMAT " = ((("
                                             "(" NETDATA_DOUBLE_FORMAT " - " NETDATA_DOUBLE_FORMAT ")"
-                                            " * %llu"
-                                            " / %llu) + " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
+                                            " * %"PRIu64""
+                                            " / %"PRIu64") + " NETDATA_DOUBLE_FORMAT, rrddim_name(rd)
                                   , new_value
                                   , rd->collector.calculated_value, rd->collector.last_calculated_value
                                   , (next_store_ut - first_ut)
@@ -1551,7 +1541,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)
         first_entry = 1;
     }
 
-    rrdset_debug(st, "microseconds since last update: %llu", st->usec_since_last_update);
+    rrdset_debug(st, "microseconds since last update: %"PRIu64"", st->usec_since_last_update);
 
     // set last_collected_time
     if(unlikely(!st->last_collected_time.tv_sec)) {
diff --git a/database/sqlite/dbdata.c b/database/sqlite/dbdata.c
new file mode 100644
index 000000000..1ad742e04
--- /dev/null
+++ b/database/sqlite/dbdata.c
@@ -0,0 +1,959 @@
+/*
+** 2019-04-17
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+******************************************************************************
+**
+** This file contains an implementation of two eponymous virtual tables,
+** "sqlite_dbdata" and "sqlite_dbptr". Both modules require that the
+** "sqlite_dbpage" eponymous virtual table be available.
+**
+** SQLITE_DBDATA:
+**   sqlite_dbdata is used to extract data directly from a database b-tree
+**   page and its associated overflow pages, bypassing the b-tree layer.
+**   The table schema is equivalent to:
+**
+**     CREATE TABLE sqlite_dbdata(
+**       pgno INTEGER,
+**       cell INTEGER,
+**       field INTEGER,
+**       value ANY,
+**       schema TEXT HIDDEN
+**     );
+**
+**   IMPORTANT: THE VIRTUAL TABLE SCHEMA ABOVE IS SUBJECT TO CHANGE. IN THE
+**   FUTURE NEW NON-HIDDEN COLUMNS MAY BE ADDED BETWEEN "value" AND
+**   "schema".
+**
+**   Each page of the database is inspected. If it cannot be interpreted as
+**   a b-tree page, or if it is a b-tree page containing 0 entries, the
+**   sqlite_dbdata table contains no rows for that page.  Otherwise, the
+**   table contains one row for each field in the record associated with
+**   each cell on the page. For intkey b-trees, the key value is stored in
+**   field -1.
+**
+**   For example, for the database:
+**
+**     CREATE TABLE t1(a, b);     -- root page is page 2
+**     INSERT INTO t1(rowid, a, b) VALUES(5, 'v', 'five');
+**     INSERT INTO t1(rowid, a, b) VALUES(10, 'x', 'ten');
+**
+**   the sqlite_dbdata table contains, as well as from entries related to 
+**   page 1, content equivalent to:
+**
+**     INSERT INTO sqlite_dbdata(pgno, cell, field, value) VALUES
+**         (2, 0, -1, 5     ),
+**         (2, 0,  0, 'v'   ),
+**         (2, 0,  1, 'five'),
+**         (2, 1, -1, 10    ),
+**         (2, 1,  0, 'x'   ),
+**         (2, 1,  1, 'ten' );
+**
+**   If database corruption is encountered, this module does not report an
+**   error. Instead, it attempts to extract as much data as possible and
+**   ignores the corruption.
+**
+** SQLITE_DBPTR:
+**   The sqlite_dbptr table has the following schema:
+**
+**     CREATE TABLE sqlite_dbptr(
+**       pgno INTEGER,
+**       child INTEGER,
+**       schema TEXT HIDDEN
+**     );
+**
+**   It contains one entry for each b-tree pointer between a parent and
+**   child page in the database.
+*/
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#if !defined(SQLITEINT_H)
+#include "sqlite3.h"
+
+typedef unsigned char u8;
+typedef unsigned int u32;
+
+#endif
+#include <string.h>
+#include <assert.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+#define DBDATA_PADDING_BYTES 100 
+
+typedef struct DbdataTable DbdataTable;
+typedef struct DbdataCursor DbdataCursor;
+
+/* Cursor object */
+struct DbdataCursor {
+  sqlite3_vtab_cursor base;       /* Base class.  Must be first */
+  sqlite3_stmt *pStmt;            /* For fetching database pages */
+
+  int iPgno;                      /* Current page number */
+  u8 *aPage;                      /* Buffer containing page */
+  int nPage;                      /* Size of aPage[] in bytes */
+  int nCell;                      /* Number of cells on aPage[] */
+  int iCell;                      /* Current cell number */
+  int bOnePage;                   /* True to stop after one page */
+  int szDb;
+  sqlite3_int64 iRowid;
+
+  /* Only for the sqlite_dbdata table */
+  u8 *pRec;                       /* Buffer containing current record */
+  sqlite3_int64 nRec;             /* Size of pRec[] in bytes */
+  sqlite3_int64 nHdr;             /* Size of header in bytes */
+  int iField;                     /* Current field number */
+  u8 *pHdrPtr;
+  u8 *pPtr;
+  u32 enc;                        /* Text encoding */
+  
+  sqlite3_int64 iIntkey;          /* Integer key value */
+};
+
+/* Table object */
+struct DbdataTable {
+  sqlite3_vtab base;              /* Base class.  Must be first */
+  sqlite3 *db;                    /* The database connection */
+  sqlite3_stmt *pStmt;            /* For fetching database pages */
+  int bPtr;                       /* True for sqlite3_dbptr table */
+};
+
+/* Column and schema definitions for sqlite_dbdata */
+#define DBDATA_COLUMN_PGNO        0
+#define DBDATA_COLUMN_CELL        1
+#define DBDATA_COLUMN_FIELD       2
+#define DBDATA_COLUMN_VALUE       3
+#define DBDATA_COLUMN_SCHEMA      4
+#define DBDATA_SCHEMA             \
+      "CREATE TABLE x("           \
+      "  pgno INTEGER,"           \
+      "  cell INTEGER,"           \
+      "  field INTEGER,"          \
+      "  value ANY,"              \
+      "  schema TEXT HIDDEN"      \
+      ")"
+
+/* Column and schema definitions for sqlite_dbptr */
+#define DBPTR_COLUMN_PGNO         0
+#define DBPTR_COLUMN_CHILD        1
+#define DBPTR_COLUMN_SCHEMA       2
+#define DBPTR_SCHEMA              \
+      "CREATE TABLE x("           \
+      "  pgno INTEGER,"           \
+      "  child INTEGER,"          \
+      "  schema TEXT HIDDEN"      \
+      ")"
+
+/*
+** Connect to an sqlite_dbdata (pAux==0) or sqlite_dbptr (pAux!=0) virtual 
+** table.
+*/
+static int dbdataConnect(
+  sqlite3 *db,
+  void *pAux,
+  int argc, const char *const*argv,
+  sqlite3_vtab **ppVtab,
+  char **pzErr
+){
+  DbdataTable *pTab = 0;
+  int rc = sqlite3_declare_vtab(db, pAux ? DBPTR_SCHEMA : DBDATA_SCHEMA);
+
+  (void)argc;
+  (void)argv;
+  (void)pzErr;
+  sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS);
+  if( rc==SQLITE_OK ){
+    pTab = (DbdataTable*)sqlite3_malloc64(sizeof(DbdataTable));
+    if( pTab==0 ){
+      rc = SQLITE_NOMEM;
+    }else{
+      memset(pTab, 0, sizeof(DbdataTable));
+      pTab->db = db;
+      pTab->bPtr = (pAux!=0);
+    }
+  }
+
+  *ppVtab = (sqlite3_vtab*)pTab;
+  return rc;
+}
+
+/*
+** Disconnect from or destroy a sqlite_dbdata or sqlite_dbptr virtual table.
+*/
+static int dbdataDisconnect(sqlite3_vtab *pVtab){
+  DbdataTable *pTab = (DbdataTable*)pVtab;
+  if( pTab ){
+    sqlite3_finalize(pTab->pStmt);
+    sqlite3_free(pVtab);
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This function interprets two types of constraints:
+**
+**       schema=?
+**       pgno=?
+**
+** If neither are present, idxNum is set to 0. If schema=? is present,
+** the 0x01 bit in idxNum is set. If pgno=? is present, the 0x02 bit
+** in idxNum is set.
+**
+** If both parameters are present, schema is in position 0 and pgno in
+** position 1.
+*/
+static int dbdataBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdx){
+  DbdataTable *pTab = (DbdataTable*)tab;
+  int i;
+  int iSchema = -1;
+  int iPgno = -1;
+  int colSchema = (pTab->bPtr ? DBPTR_COLUMN_SCHEMA : DBDATA_COLUMN_SCHEMA);
+
+  for(i=0; i<pIdx->nConstraint; i++){
+    struct sqlite3_index_constraint *p = &pIdx->aConstraint[i];
+    if( p->op==SQLITE_INDEX_CONSTRAINT_EQ ){
+      if( p->iColumn==colSchema ){
+        if( p->usable==0 ) return SQLITE_CONSTRAINT;
+        iSchema = i;
+      }
+      if( p->iColumn==DBDATA_COLUMN_PGNO && p->usable ){
+        iPgno = i;
+      }
+    }
+  }
+
+  if( iSchema>=0 ){
+    pIdx->aConstraintUsage[iSchema].argvIndex = 1;
+    pIdx->aConstraintUsage[iSchema].omit = 1;
+  }
+  if( iPgno>=0 ){
+    pIdx->aConstraintUsage[iPgno].argvIndex = 1 + (iSchema>=0);
+    pIdx->aConstraintUsage[iPgno].omit = 1;
+    pIdx->estimatedCost = 100;
+    pIdx->estimatedRows =  50;
+
+    if( pTab->bPtr==0 && pIdx->nOrderBy && pIdx->aOrderBy[0].desc==0 ){
+      int iCol = pIdx->aOrderBy[0].iColumn;
+      if( pIdx->nOrderBy==1 ){
+        pIdx->orderByConsumed = (iCol==0 || iCol==1);
+      }else if( pIdx->nOrderBy==2 && pIdx->aOrderBy[1].desc==0 && iCol==0 ){
+        pIdx->orderByConsumed = (pIdx->aOrderBy[1].iColumn==1);
+      }
+    }
+
+  }else{
+    pIdx->estimatedCost = 100000000;
+    pIdx->estimatedRows = 1000000000;
+  }
+  pIdx->idxNum = (iSchema>=0 ? 0x01 : 0x00) | (iPgno>=0 ? 0x02 : 0x00);
+  return SQLITE_OK;
+}
+
+/*
+** Open a new sqlite_dbdata or sqlite_dbptr cursor.
+*/
+static int dbdataOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){
+  DbdataCursor *pCsr;
+
+  pCsr = (DbdataCursor*)sqlite3_malloc64(sizeof(DbdataCursor));
+  if( pCsr==0 ){
+    return SQLITE_NOMEM;
+  }else{
+    memset(pCsr, 0, sizeof(DbdataCursor));
+    pCsr->base.pVtab = pVTab;
+  }
+
+  *ppCursor = (sqlite3_vtab_cursor *)pCsr;
+  return SQLITE_OK;
+}
+
+/*
+** Restore a cursor object to the state it was in when first allocated 
+** by dbdataOpen().
+*/
+static void dbdataResetCursor(DbdataCursor *pCsr){
+  DbdataTable *pTab = (DbdataTable*)(pCsr->base.pVtab);
+  if( pTab->pStmt==0 ){
+    pTab->pStmt = pCsr->pStmt;
+  }else{
+    sqlite3_finalize(pCsr->pStmt);
+  }
+  pCsr->pStmt = 0;
+  pCsr->iPgno = 1;
+  pCsr->iCell = 0;
+  pCsr->iField = 0;
+  pCsr->bOnePage = 0;
+  sqlite3_free(pCsr->aPage);
+  sqlite3_free(pCsr->pRec);
+  pCsr->pRec = 0;
+  pCsr->aPage = 0;
+}
+
+/*
+** Close an sqlite_dbdata or sqlite_dbptr cursor.
+*/
+static int dbdataClose(sqlite3_vtab_cursor *pCursor){
+  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
+  dbdataResetCursor(pCsr);
+  sqlite3_free(pCsr);
+  return SQLITE_OK;
+}
+
+/* 
+** Utility methods to decode 16 and 32-bit big-endian unsigned integers. 
+*/
+static u32 get_uint16(unsigned char *a){
+  return (a[0]<<8)|a[1];
+}
+static u32 get_uint32(unsigned char *a){
+  return ((u32)a[0]<<24)
+       | ((u32)a[1]<<16)
+       | ((u32)a[2]<<8)
+       | ((u32)a[3]);
+}
+
+/*
+** Load page pgno from the database via the sqlite_dbpage virtual table.
+** If successful, set (*ppPage) to point to a buffer containing the page
+** data, (*pnPage) to the size of that buffer in bytes and return
+** SQLITE_OK. In this case it is the responsibility of the caller to
+** eventually free the buffer using sqlite3_free().
+**
+** Or, if an error occurs, set both (*ppPage) and (*pnPage) to 0 and
+** return an SQLite error code.
+*/
+static int dbdataLoadPage(
+  DbdataCursor *pCsr,             /* Cursor object */
+  u32 pgno,                       /* Page number of page to load */
+  u8 **ppPage,                    /* OUT: pointer to page buffer */
+  int *pnPage                     /* OUT: Size of (*ppPage) in bytes */
+){
+  int rc2;
+  int rc = SQLITE_OK;
+  sqlite3_stmt *pStmt = pCsr->pStmt;
+
+  *ppPage = 0;
+  *pnPage = 0;
+  if( pgno>0 ){
+    sqlite3_bind_int64(pStmt, 2, pgno);
+    if( SQLITE_ROW==sqlite3_step(pStmt) ){
+      int nCopy = sqlite3_column_bytes(pStmt, 0);
+      if( nCopy>0 ){
+        u8 *pPage;
+        pPage = (u8*)sqlite3_malloc64(nCopy + DBDATA_PADDING_BYTES);
+        if( pPage==0 ){
+          rc = SQLITE_NOMEM;
+        }else{
+          const u8 *pCopy = sqlite3_column_blob(pStmt, 0);
+          memcpy(pPage, pCopy, nCopy);
+          memset(&pPage[nCopy], 0, DBDATA_PADDING_BYTES);
+        }
+        *ppPage = pPage;
+        *pnPage = nCopy;
+      }
+    }
+    rc2 = sqlite3_reset(pStmt);
+    if( rc==SQLITE_OK ) rc = rc2;
+  }
+
+  return rc;
+}
+
+/*
+** Read a varint.  Put the value in *pVal and return the number of bytes.
+*/
+static int dbdataGetVarint(const u8 *z, sqlite3_int64 *pVal){
+  sqlite3_uint64 u = 0;
+  int i;
+  for(i=0; i<8; i++){
+    u = (u<<7) + (z[i]&0x7f);
+    if( (z[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
+  }
+  u = (u<<8) + (z[i]&0xff);
+  *pVal = (sqlite3_int64)u;
+  return 9;
+}
+
+/*
+** Like dbdataGetVarint(), but set the output to 0 if it is less than 0
+** or greater than 0xFFFFFFFF. This can be used for all varints in an
+** SQLite database except for key values in intkey tables.
+*/
+static int dbdataGetVarintU32(const u8 *z, sqlite3_int64 *pVal){
+  sqlite3_int64 val;
+  int nRet = dbdataGetVarint(z, &val);
+  if( val<0 || val>0xFFFFFFFF ) val = 0;
+  *pVal = val;
+  return nRet;
+}
+
+/*
+** Return the number of bytes of space used by an SQLite value of type
+** eType.
+*/
+static int dbdataValueBytes(int eType){
+  switch( eType ){
+    case 0: case 8: case 9:
+    case 10: case 11:
+      return 0;
+    case 1:
+      return 1;
+    case 2:
+      return 2;
+    case 3:
+      return 3;
+    case 4:
+      return 4;
+    case 5:
+      return 6;
+    case 6:
+    case 7:
+      return 8;
+    default:
+      if( eType>0 ){
+        return ((eType-12) / 2);
+      }
+      return 0;
+  }
+}
+
+/*
+** Load a value of type eType from buffer pData and use it to set the
+** result of context object pCtx.
+*/
+static void dbdataValue(
+  sqlite3_context *pCtx, 
+  u32 enc,
+  int eType, 
+  u8 *pData,
+  sqlite3_int64 nData
+){
+  if( eType>=0 && dbdataValueBytes(eType)<=nData ){
+    switch( eType ){
+      case 0: 
+      case 10: 
+      case 11: 
+        sqlite3_result_null(pCtx);
+        break;
+      
+      case 8: 
+        sqlite3_result_int(pCtx, 0);
+        break;
+      case 9:
+        sqlite3_result_int(pCtx, 1);
+        break;
+  
+      case 1: case 2: case 3: case 4: case 5: case 6: case 7: {
+        sqlite3_uint64 v = (signed char)pData[0];
+        pData++;
+        switch( eType ){
+          case 7:
+          case 6:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
+          case 5:  v = (v<<16) + (pData[0]<<8) + pData[1];  pData += 2;
+          case 4:  v = (v<<8) + pData[0];  pData++;
+          case 3:  v = (v<<8) + pData[0];  pData++;
+          case 2:  v = (v<<8) + pData[0];  pData++;
+        }
+  
+        if( eType==7 ){
+          double r;
+          memcpy(&r, &v, sizeof(r));
+          sqlite3_result_double(pCtx, r);
+        }else{
+          sqlite3_result_int64(pCtx, (sqlite3_int64)v);
+        }
+        break;
+      }
+  
+      default: {
+        int n = ((eType-12) / 2);
+        if( eType % 2 ){
+          switch( enc ){
+#ifndef SQLITE_OMIT_UTF16
+            case SQLITE_UTF16BE:
+              sqlite3_result_text16be(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
+              break;
+            case SQLITE_UTF16LE:
+              sqlite3_result_text16le(pCtx, (void*)pData, n, SQLITE_TRANSIENT);
+              break;
+#endif
+            default:
+              sqlite3_result_text(pCtx, (char*)pData, n, SQLITE_TRANSIENT);
+              break;
+          }
+        }else{
+          sqlite3_result_blob(pCtx, pData, n, SQLITE_TRANSIENT);
+        }
+      }
+    }
+  }
+}
+
+/*
+** Move an sqlite_dbdata or sqlite_dbptr cursor to the next entry.
+*/
+static int dbdataNext(sqlite3_vtab_cursor *pCursor){
+  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
+  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
+
+  pCsr->iRowid++;
+  while( 1 ){
+    int rc;
+    int iOff = (pCsr->iPgno==1 ? 100 : 0);
+    int bNextPage = 0;
+
+    if( pCsr->aPage==0 ){
+      while( 1 ){
+        if( pCsr->bOnePage==0 && pCsr->iPgno>pCsr->szDb ) return SQLITE_OK;
+        rc = dbdataLoadPage(pCsr, pCsr->iPgno, &pCsr->aPage, &pCsr->nPage);
+        if( rc!=SQLITE_OK ) return rc;
+        if( pCsr->aPage && pCsr->nPage>=256 ) break;
+        sqlite3_free(pCsr->aPage);
+        pCsr->aPage = 0;
+        if( pCsr->bOnePage ) return SQLITE_OK;
+        pCsr->iPgno++;
+      }
+
+      assert( iOff+3+2<=pCsr->nPage );
+      pCsr->iCell = pTab->bPtr ? -2 : 0;
+      pCsr->nCell = get_uint16(&pCsr->aPage[iOff+3]);
+    }
+
+    if( pTab->bPtr ){
+      if( pCsr->aPage[iOff]!=0x02 && pCsr->aPage[iOff]!=0x05 ){
+        pCsr->iCell = pCsr->nCell;
+      }
+      pCsr->iCell++;
+      if( pCsr->iCell>=pCsr->nCell ){
+        sqlite3_free(pCsr->aPage);
+        pCsr->aPage = 0;
+        if( pCsr->bOnePage ) return SQLITE_OK;
+        pCsr->iPgno++;
+      }else{
+        return SQLITE_OK;
+      }
+    }else{
+      /* If there is no record loaded, load it now. */
+      if( pCsr->pRec==0 ){
+        int bHasRowid = 0;
+        int nPointer = 0;
+        sqlite3_int64 nPayload = 0;
+        sqlite3_int64 nHdr = 0;
+        int iHdr;
+        int U, X;
+        int nLocal;
+  
+        switch( pCsr->aPage[iOff] ){
+          case 0x02:
+            nPointer = 4;
+            break;
+          case 0x0a:
+            break;
+          case 0x0d:
+            bHasRowid = 1;
+            break;
+          default:
+            /* This is not a b-tree page with records on it. Continue. */
+            pCsr->iCell = pCsr->nCell;
+            break;
+        }
+
+        if( pCsr->iCell>=pCsr->nCell ){
+          bNextPage = 1;
+        }else{
+  
+          iOff += 8 + nPointer + pCsr->iCell*2;
+          if( iOff>pCsr->nPage ){
+            bNextPage = 1;
+          }else{
+            iOff = get_uint16(&pCsr->aPage[iOff]);
+          }
+    
+          /* For an interior node cell, skip past the child-page number */
+          iOff += nPointer;
+    
+          /* Load the "byte of payload including overflow" field */
+          if( bNextPage || iOff>pCsr->nPage ){
+            bNextPage = 1;
+          }else{
+            iOff += dbdataGetVarintU32(&pCsr->aPage[iOff], &nPayload);
+          }
+    
+          /* If this is a leaf intkey cell, load the rowid */
+          if( bHasRowid && !bNextPage && iOff<pCsr->nPage ){
+            iOff += dbdataGetVarint(&pCsr->aPage[iOff], &pCsr->iIntkey);
+          }
+    
+          /* Figure out how much data to read from the local page */
+          U = pCsr->nPage;
+          if( bHasRowid ){
+            X = U-35;
+          }else{
+            X = ((U-12)*64/255)-23;
+          }
+          if( nPayload<=X ){
+            nLocal = nPayload;
+          }else{
+            int M, K;
+            M = ((U-12)*32/255)-23;
+            K = M+((nPayload-M)%(U-4));
+            if( K<=X ){
+              nLocal = K;
+            }else{
+              nLocal = M;
+            }
+          }
+
+          if( bNextPage || nLocal+iOff>pCsr->nPage ){
+            bNextPage = 1;
+          }else{
+
+            /* Allocate space for payload. And a bit more to catch small buffer
+            ** overruns caused by attempting to read a varint or similar from 
+            ** near the end of a corrupt record.  */
+            pCsr->pRec = (u8*)sqlite3_malloc64(nPayload+DBDATA_PADDING_BYTES);
+            if( pCsr->pRec==0 ) return SQLITE_NOMEM;
+            memset(pCsr->pRec, 0, nPayload+DBDATA_PADDING_BYTES);
+            pCsr->nRec = nPayload;
+
+            /* Load the nLocal bytes of payload */
+            memcpy(pCsr->pRec, &pCsr->aPage[iOff], nLocal);
+            iOff += nLocal;
+
+            /* Load content from overflow pages */
+            if( nPayload>nLocal ){
+              sqlite3_int64 nRem = nPayload - nLocal;
+              u32 pgnoOvfl = get_uint32(&pCsr->aPage[iOff]);
+              while( nRem>0 ){
+                u8 *aOvfl = 0;
+                int nOvfl = 0;
+                int nCopy;
+                rc = dbdataLoadPage(pCsr, pgnoOvfl, &aOvfl, &nOvfl);
+                assert( rc!=SQLITE_OK || aOvfl==0 || nOvfl==pCsr->nPage );
+                if( rc!=SQLITE_OK ) return rc;
+                if( aOvfl==0 ) break;
+
+                nCopy = U-4;
+                if( nCopy>nRem ) nCopy = nRem;
+                memcpy(&pCsr->pRec[nPayload-nRem], &aOvfl[4], nCopy);
+                nRem -= nCopy;
+
+                pgnoOvfl = get_uint32(aOvfl);
+                sqlite3_free(aOvfl);
+              }
+            }
+    
+            iHdr = dbdataGetVarintU32(pCsr->pRec, &nHdr);
+            if( nHdr>nPayload ) nHdr = 0;
+            pCsr->nHdr = nHdr;
+            pCsr->pHdrPtr = &pCsr->pRec[iHdr];
+            pCsr->pPtr = &pCsr->pRec[pCsr->nHdr];
+            pCsr->iField = (bHasRowid ? -1 : 0);
+          }
+        }
+      }else{
+        pCsr->iField++;
+        if( pCsr->iField>0 ){
+          sqlite3_int64 iType;
+          if( pCsr->pHdrPtr>&pCsr->pRec[pCsr->nRec] ){
+            bNextPage = 1;
+          }else{
+            int szField = 0;
+            pCsr->pHdrPtr += dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
+            szField = dbdataValueBytes(iType);
+            if( (pCsr->nRec - (pCsr->pPtr - pCsr->pRec))<szField ){
+              pCsr->pPtr = &pCsr->pRec[pCsr->nRec];
+            }else{
+              pCsr->pPtr += szField;
+            }
+          }
+        }
+      }
+
+      if( bNextPage ){
+        sqlite3_free(pCsr->aPage);
+        sqlite3_free(pCsr->pRec);
+        pCsr->aPage = 0;
+        pCsr->pRec = 0;
+        if( pCsr->bOnePage ) return SQLITE_OK;
+        pCsr->iPgno++;
+      }else{
+        if( pCsr->iField<0 || pCsr->pHdrPtr<&pCsr->pRec[pCsr->nHdr] ){
+          return SQLITE_OK;
+        }
+
+        /* Advance to the next cell. The next iteration of the loop will load
+        ** the record and so on. */
+        sqlite3_free(pCsr->pRec);
+        pCsr->pRec = 0;
+        pCsr->iCell++;
+      }
+    }
+  }
+
+  assert( !"can't get here" );
+  return SQLITE_OK;
+}
+
+/* 
+** Return true if the cursor is at EOF.
+*/
+static int dbdataEof(sqlite3_vtab_cursor *pCursor){
+  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
+  return pCsr->aPage==0;
+}
+
+/*
+** Return true if nul-terminated string zSchema ends in "()". Or false
+** otherwise.
+*/
+static int dbdataIsFunction(const char *zSchema){
+  size_t n = strlen(zSchema);
+  if( n>2 && zSchema[n-2]=='(' && zSchema[n-1]==')' ){
+    return (int)n-2;
+  }
+  return 0;
+}
+
+/* 
+** Determine the size in pages of database zSchema (where zSchema is
+** "main", "temp" or the name of an attached database) and set 
+** pCsr->szDb accordingly. If successful, return SQLITE_OK. Otherwise,
+** an SQLite error code.
+*/
+static int dbdataDbsize(DbdataCursor *pCsr, const char *zSchema){
+  DbdataTable *pTab = (DbdataTable*)pCsr->base.pVtab;
+  char *zSql = 0;
+  int rc, rc2;
+  int nFunc = 0;
+  sqlite3_stmt *pStmt = 0;
+
+  if( (nFunc = dbdataIsFunction(zSchema))>0 ){
+    zSql = sqlite3_mprintf("SELECT %.*s(0)", nFunc, zSchema);
+  }else{
+    zSql = sqlite3_mprintf("PRAGMA %Q.page_count", zSchema);
+  }
+  if( zSql==0 ) return SQLITE_NOMEM;
+
+  rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pStmt, 0);
+  sqlite3_free(zSql);
+  if( rc==SQLITE_OK && sqlite3_step(pStmt)==SQLITE_ROW ){
+    pCsr->szDb = sqlite3_column_int(pStmt, 0);
+  }
+  rc2 = sqlite3_finalize(pStmt);
+  if( rc==SQLITE_OK ) rc = rc2;
+  return rc;
+}
+
+/*
+** Attempt to figure out the encoding of the database by retrieving page 1
+** and inspecting the header field. If successful, set the pCsr->enc variable
+** and return SQLITE_OK. Otherwise, return an SQLite error code.
+*/
+static int dbdataGetEncoding(DbdataCursor *pCsr){
+  int rc = SQLITE_OK;
+  int nPg1 = 0;
+  u8 *aPg1 = 0;
+  rc = dbdataLoadPage(pCsr, 1, &aPg1, &nPg1);
+  if( rc==SQLITE_OK && nPg1>=(56+4) ){
+    pCsr->enc = get_uint32(&aPg1[56]);
+  }
+  sqlite3_free(aPg1);
+  return rc;
+}
+
+
+/* 
+** xFilter method for sqlite_dbdata and sqlite_dbptr.
+*/
+static int dbdataFilter(
+  sqlite3_vtab_cursor *pCursor, 
+  int idxNum, const char *idxStr,
+  int argc, sqlite3_value **argv
+){
+  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
+  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
+  int rc = SQLITE_OK;
+  const char *zSchema = "main";
+  (void)idxStr;
+  (void)argc;
+
+  dbdataResetCursor(pCsr);
+  assert( pCsr->iPgno==1 );
+  if( idxNum & 0x01 ){
+    zSchema = (const char*)sqlite3_value_text(argv[0]);
+    if( zSchema==0 ) zSchema = "";
+  }
+  if( idxNum & 0x02 ){
+    pCsr->iPgno = sqlite3_value_int(argv[(idxNum & 0x01)]);
+    pCsr->bOnePage = 1;
+  }else{
+    rc = dbdataDbsize(pCsr, zSchema);
+  }
+
+  if( rc==SQLITE_OK ){
+    int nFunc = 0;
+    if( pTab->pStmt ){
+      pCsr->pStmt = pTab->pStmt;
+      pTab->pStmt = 0;
+    }else if( (nFunc = dbdataIsFunction(zSchema))>0 ){
+      char *zSql = sqlite3_mprintf("SELECT %.*s(?2)", nFunc, zSchema);
+      if( zSql==0 ){
+        rc = SQLITE_NOMEM;
+      }else{
+        rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pStmt, 0);
+        sqlite3_free(zSql);
+      }
+    }else{
+      rc = sqlite3_prepare_v2(pTab->db, 
+          "SELECT data FROM sqlite_dbpage(?) WHERE pgno=?", -1,
+          &pCsr->pStmt, 0
+      );
+    }
+  }
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_bind_text(pCsr->pStmt, 1, zSchema, -1, SQLITE_TRANSIENT);
+  }
+
+  /* Try to determine the encoding of the db by inspecting the header
+  ** field on page 1. */
+  if( rc==SQLITE_OK ){
+    rc = dbdataGetEncoding(pCsr);
+  }
+
+  if( rc!=SQLITE_OK ){
+    pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db));
+  }
+
+  if( rc==SQLITE_OK ){
+    rc = dbdataNext(pCursor);
+  }
+  return rc;
+}
+
+/*
+** Return a column for the sqlite_dbdata or sqlite_dbptr table.
+*/
+static int dbdataColumn(
+  sqlite3_vtab_cursor *pCursor, 
+  sqlite3_context *ctx, 
+  int i
+){
+  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
+  DbdataTable *pTab = (DbdataTable*)pCursor->pVtab;
+  if( pTab->bPtr ){
+    switch( i ){
+      case DBPTR_COLUMN_PGNO:
+        sqlite3_result_int64(ctx, pCsr->iPgno);
+        break;
+      case DBPTR_COLUMN_CHILD: {
+        int iOff = pCsr->iPgno==1 ? 100 : 0;
+        if( pCsr->iCell<0 ){
+          iOff += 8;
+        }else{
+          iOff += 12 + pCsr->iCell*2;
+          if( iOff>pCsr->nPage ) return SQLITE_OK;
+          iOff = get_uint16(&pCsr->aPage[iOff]);
+        }
+        if( iOff<=pCsr->nPage ){
+          sqlite3_result_int64(ctx, get_uint32(&pCsr->aPage[iOff]));
+        }
+        break;
+      }
+    }
+  }else{
+    switch( i ){
+      case DBDATA_COLUMN_PGNO:
+        sqlite3_result_int64(ctx, pCsr->iPgno);
+        break;
+      case DBDATA_COLUMN_CELL:
+        sqlite3_result_int(ctx, pCsr->iCell);
+        break;
+      case DBDATA_COLUMN_FIELD:
+        sqlite3_result_int(ctx, pCsr->iField);
+        break;
+      case DBDATA_COLUMN_VALUE: {
+        if( pCsr->iField<0 ){
+          sqlite3_result_int64(ctx, pCsr->iIntkey);
+        }else if( &pCsr->pRec[pCsr->nRec] >= pCsr->pPtr ){
+          sqlite3_int64 iType;
+          dbdataGetVarintU32(pCsr->pHdrPtr, &iType);
+          dbdataValue(
+              ctx, pCsr->enc, iType, pCsr->pPtr, 
+              &pCsr->pRec[pCsr->nRec] - pCsr->pPtr
+          );
+        }
+        break;
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/* 
+** Return the rowid for an sqlite_dbdata or sqlite_dptr table.
+*/
+static int dbdataRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){
+  DbdataCursor *pCsr = (DbdataCursor*)pCursor;
+  *pRowid = pCsr->iRowid;
+  return SQLITE_OK;
+}
+
+
+/*
+** Invoke this routine to register the "sqlite_dbdata" virtual table module
+*/
+static int sqlite3DbdataRegister(sqlite3 *db){
+  static sqlite3_module dbdata_module = {
+    0,                            /* iVersion */
+    0,                            /* xCreate */
+    dbdataConnect,                /* xConnect */
+    dbdataBestIndex,              /* xBestIndex */
+    dbdataDisconnect,             /* xDisconnect */
+    0,                            /* xDestroy */
+    dbdataOpen,                   /* xOpen - open a cursor */
+    dbdataClose,                  /* xClose - close a cursor */
+    dbdataFilter,                 /* xFilter - configure scan constraints */
+    dbdataNext,                   /* xNext - advance a cursor */
+    dbdataEof,                    /* xEof - check for end of scan */
+    dbdataColumn,                 /* xColumn - read data */
+    dbdataRowid,                  /* xRowid - read data */
+    0,                            /* xUpdate */
+    0,                            /* xBegin */
+    0,                            /* xSync */
+    0,                            /* xCommit */
+    0,                            /* xRollback */
+    0,                            /* xFindMethod */
+    0,                            /* xRename */
+    0,                            /* xSavepoint */
+    0,                            /* xRelease */
+    0,                            /* xRollbackTo */
+    0                             /* xShadowName */
+  };
+
+  int rc = sqlite3_create_module(db, "sqlite_dbdata", &dbdata_module, 0);
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_create_module(db, "sqlite_dbptr", &dbdata_module, (void*)1);
+  }
+  return rc;
+}
+
+int sqlite3_dbdata_init(
+  sqlite3 *db, 
+  char **pzErrMsg, 
+  const sqlite3_api_routines *pApi
+){
+  (void)pzErrMsg;
+  return sqlite3DbdataRegister(db);
+}
+
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+#pragma GCC diagnostic pop
diff --git a/database/sqlite/sqlite3.c b/database/sqlite/sqlite3.c
index 005aab85a..da8c38d09 100644
--- a/database/sqlite/sqlite3.c
+++ b/database/sqlite/sqlite3.c
@@ -1,6 +1,6 @@
 /******************************************************************************
 ** This file is an amalgamation of many separate C source files from SQLite
-** version 3.41.2.  By combining all the individual C code files into this
+** version 3.42.0.  By combining all the individual C code files into this
 ** single large file, the entire code can be compiled as a single translation
 ** unit.  This allows many compilers to do optimizations that would not be
 ** possible if the files were compiled separately.  Performance improvements
@@ -17,6 +17,9 @@
 ** language. The code for the "sqlite3" command-line shell is also in a
 ** separate file. This file contains only code for the core SQLite library.
 */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wimplicit-fallthrough"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
 #define SQLITE_CORE 1
 #define SQLITE_AMALGAMATION 1
 #ifndef SQLITE_PRIVATE
@@ -26,6 +29,7 @@
 #define SQLITE_ENABLE_UPDATE_DELETE_LIMIT 1
 #define SQLITE_OMIT_LOAD_EXTENSION 1
 #define SQLITE_ENABLE_DBSTAT_VTAB 1
+#define SQLITE_ENABLE_DBPAGE_VTAB 1
 /************** Begin file sqliteInt.h ***************************************/
 /*
 ** 2001 September 15
@@ -127,6 +131,10 @@
 #define SQLITE_4_BYTE_ALIGNED_MALLOC
 #endif /* defined(_MSC_VER) && !defined(_WIN64) */
 
+#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800
+#define HAVE_LOG2 0
+#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */
+
 #endif /* SQLITE_MSVC_H */
 
 /************** End of msvc.h ************************************************/
@@ -456,9 +464,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.41.2"
-#define SQLITE_VERSION_NUMBER 3041002
-#define SQLITE_SOURCE_ID      "2023-03-22 11:56:21 0d1fc92f94cb6b76bffe3ec34d69cffde2924203304e8ffc4155597af0c191da"
+#define SQLITE_VERSION        "3.42.0"
+#define SQLITE_VERSION_NUMBER 3042000
+#define SQLITE_SOURCE_ID      "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -1965,20 +1973,23 @@ SQLITE_API int sqlite3_os_end(void);
 ** must ensure that no other SQLite interfaces are invoked by other
 ** threads while sqlite3_config() is running.</b>
 **
-** The sqlite3_config() interface
-** may only be invoked prior to library initialization using
-** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
-** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
-** [sqlite3_shutdown()] then it will return SQLITE_MISUSE.
-** Note, however, that ^sqlite3_config() can be called as part of the
-** implementation of an application-defined [sqlite3_os_init()].
-**
 ** The first argument to sqlite3_config() is an integer
 ** [configuration option] that determines
 ** what property of SQLite is to be configured.  Subsequent arguments
 ** vary depending on the [configuration option]
 ** in the first argument.
 **
+** For most configuration options, the sqlite3_config() interface
+** may only be invoked prior to library initialization using
+** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
+** The exceptional configuration options that may be invoked at any time
+** are called "anytime configuration options".
+** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
+** [sqlite3_shutdown()] with a first argument that is not an anytime
+** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE.
+** Note, however, that ^sqlite3_config() can be called as part of the
+** implementation of an application-defined [sqlite3_os_init()].
+**
 ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK].
 ** ^If the option is unknown or SQLite is unable to set the option
 ** then this routine returns a non-zero [error code].
@@ -2086,6 +2097,23 @@ struct sqlite3_mem_methods {
 ** These constants are the available integer configuration options that
 ** can be passed as the first argument to the [sqlite3_config()] interface.
 **
+** Most of the configuration options for sqlite3_config()
+** will only work if invoked prior to [sqlite3_initialize()] or after
+** [sqlite3_shutdown()].  The few exceptions to this rule are called
+** "anytime configuration options".
+** ^Calling [sqlite3_config()] with a first argument that is not an
+** anytime configuration option in between calls to [sqlite3_initialize()] and
+** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE.
+**
+** The set of anytime configuration options can change (by insertions
+** and/or deletions) from one release of SQLite to the next.
+** As of SQLite version 3.42.0, the complete set of anytime configuration
+** options is:
+** <ul>
+** <li> SQLITE_CONFIG_LOG
+** <li> SQLITE_CONFIG_PCACHE_HDRSZ
+** </ul>
+**
 ** New configuration options may be added in future releases of SQLite.
 ** Existing configuration options might be discontinued.  Applications
 ** should check the return code from [sqlite3_config()] to make sure that
@@ -2432,28 +2460,28 @@ struct sqlite3_mem_methods {
 ** compile-time option is not set, then the default maximum is 1073741824.
 ** </dl>
 */
-#define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
-#define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
-#define SQLITE_CONFIG_SERIALIZED    3  /* nil */
-#define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
-#define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
-#define SQLITE_CONFIG_SCRATCH       6  /* No longer used */
-#define SQLITE_CONFIG_PAGECACHE     7  /* void*, int sz, int N */
-#define SQLITE_CONFIG_HEAP          8  /* void*, int nByte, int min */
-#define SQLITE_CONFIG_MEMSTATUS     9  /* boolean */
-#define SQLITE_CONFIG_MUTEX        10  /* sqlite3_mutex_methods* */
-#define SQLITE_CONFIG_GETMUTEX     11  /* sqlite3_mutex_methods* */
-/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */
-#define SQLITE_CONFIG_LOOKASIDE    13  /* int int */
-#define SQLITE_CONFIG_PCACHE       14  /* no-op */
-#define SQLITE_CONFIG_GETPCACHE    15  /* no-op */
-#define SQLITE_CONFIG_LOG          16  /* xFunc, void* */
-#define SQLITE_CONFIG_URI          17  /* int */
-#define SQLITE_CONFIG_PCACHE2      18  /* sqlite3_pcache_methods2* */
-#define SQLITE_CONFIG_GETPCACHE2   19  /* sqlite3_pcache_methods2* */
+#define SQLITE_CONFIG_SINGLETHREAD         1  /* nil */
+#define SQLITE_CONFIG_MULTITHREAD          2  /* nil */
+#define SQLITE_CONFIG_SERIALIZED           3  /* nil */
+#define SQLITE_CONFIG_MALLOC               4  /* sqlite3_mem_methods* */
+#define SQLITE_CONFIG_GETMALLOC            5  /* sqlite3_mem_methods* */
+#define SQLITE_CONFIG_SCRATCH              6  /* No longer used */
+#define SQLITE_CONFIG_PAGECACHE            7  /* void*, int sz, int N */
+#define SQLITE_CONFIG_HEAP                 8  /* void*, int nByte, int min */
+#define SQLITE_CONFIG_MEMSTATUS            9  /* boolean */
+#define SQLITE_CONFIG_MUTEX               10  /* sqlite3_mutex_methods* */
+#define SQLITE_CONFIG_GETMUTEX            11  /* sqlite3_mutex_methods* */
+/* previously SQLITE_CONFIG_CHUNKALLOC    12 which is now unused. */
+#define SQLITE_CONFIG_LOOKASIDE           13  /* int int */
+#define SQLITE_CONFIG_PCACHE              14  /* no-op */
+#define SQLITE_CONFIG_GETPCACHE           15  /* no-op */
+#define SQLITE_CONFIG_LOG                 16  /* xFunc, void* */
+#define SQLITE_CONFIG_URI                 17  /* int */
+#define SQLITE_CONFIG_PCACHE2             18  /* sqlite3_pcache_methods2* */
+#define SQLITE_CONFIG_GETPCACHE2          19  /* sqlite3_pcache_methods2* */
 #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20  /* int */
-#define SQLITE_CONFIG_SQLLOG       21  /* xSqllog, void* */
-#define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
+#define SQLITE_CONFIG_SQLLOG              21  /* xSqllog, void* */
+#define SQLITE_CONFIG_MMAP_SIZE           22  /* sqlite3_int64, sqlite3_int64 */
 #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
 #define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
 #define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
@@ -2688,7 +2716,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_DQS_DML]]
-** <dt>SQLITE_DBCONFIG_DQS_DML</td>
+** <dt>SQLITE_DBCONFIG_DQS_DML</dt>
 ** <dd>The SQLITE_DBCONFIG_DQS_DML option activates or deactivates
 ** the legacy [double-quoted string literal] misfeature for DML statements
 ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The
@@ -2697,7 +2725,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_DQS_DDL]]
-** <dt>SQLITE_DBCONFIG_DQS_DDL</td>
+** <dt>SQLITE_DBCONFIG_DQS_DDL</dt>
 ** <dd>The SQLITE_DBCONFIG_DQS option activates or deactivates
 ** the legacy [double-quoted string literal] misfeature for DDL statements,
 ** such as CREATE TABLE and CREATE INDEX. The
@@ -2706,7 +2734,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]]
-** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</td>
+** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</dt>
 ** <dd>The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to
 ** assume that database schemas are untainted by malicious content.
 ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite
@@ -2726,7 +2754,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]]
-** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</td>
+** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates
 ** the legacy file format flag.  When activated, this flag causes all newly
 ** created database file to have a schema format version number (the 4-byte
@@ -2735,7 +2763,7 @@ struct sqlite3_mem_methods {
 ** any SQLite version back to 3.0.0 ([dateof:3.0.0]).  Without this setting,
 ** newly created databases are generally not understandable by SQLite versions
 ** prior to 3.3.0 ([dateof:3.3.0]).  As these words are written, there
-** is now scarcely any need to generated database files that are compatible
+** is now scarcely any need to generate database files that are compatible
 ** all the way back to version 3.0.0, and so this setting is of little
 ** practical use, but is provided so that SQLite can continue to claim the
 ** ability to generate new database files that are compatible with  version
@@ -2746,6 +2774,38 @@ struct sqlite3_mem_methods {
 ** not considered a bug since SQLite versions 3.3.0 and earlier do not support
 ** either generated columns or decending indexes.
 ** </dd>
+**
+** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]]
+** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt>
+** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in
+** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears
+** a flag that enables collection of the sqlite3_stmt_scanstatus_v2()
+** statistics. For statistics to be collected, the flag must be set on
+** the database handle both when the SQL statement is prepared and when it
+** is stepped. The flag is set (collection of statistics is enabled)
+** by default.  This option takes two arguments: an integer and a pointer to
+** an integer..  The first argument is 1, 0, or -1 to enable, disable, or
+** leave unchanged the statement scanstatus option.  If the second argument
+** is not NULL, then the value of the statement scanstatus setting after
+** processing the first argument is written into the integer that the second
+** argument points to.
+** </dd>
+**
+** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]]
+** <dt>SQLITE_DBCONFIG_REVERSE_SCANORDER</dt>
+** <dd>The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order
+** in which tables and indexes are scanned so that the scans start at the end
+** and work toward the beginning rather than starting at the beginning and
+** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the
+** same as setting [PRAGMA reverse_unordered_selects].  This option takes
+** two arguments which are an integer and a pointer to an integer.  The first
+** argument is 1, 0, or -1 to enable, disable, or leave unchanged the
+** reverse scan order flag, respectively.  If the second argument is not NULL,
+** then 0 or 1 is written into the integer that the second argument points to
+** depending on if the reverse scan order flag is set after processing the
+** first argument.
+** </dd>
+**
 ** </dl>
 */
 #define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
@@ -2766,7 +2826,9 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_VIEW           1015 /* int int* */
 #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT    1016 /* int int* */
 #define SQLITE_DBCONFIG_TRUSTED_SCHEMA        1017 /* int int* */
-#define SQLITE_DBCONFIG_MAX                   1017 /* Largest DBCONFIG */
+#define SQLITE_DBCONFIG_STMT_SCANSTATUS       1018 /* int int* */
+#define SQLITE_DBCONFIG_REVERSE_SCANORDER     1019 /* int int* */
+#define SQLITE_DBCONFIG_MAX                   1019 /* Largest DBCONFIG */
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
@@ -6511,6 +6573,13 @@ SQLITE_API void sqlite3_activate_cerod(
 ** of the default VFS is not implemented correctly, or not implemented at
 ** all, then the behavior of sqlite3_sleep() may deviate from the description
 ** in the previous paragraphs.
+**
+** If a negative argument is passed to sqlite3_sleep() the results vary by
+** VFS and operating system.  Some system treat a negative argument as an
+** instruction to sleep forever.  Others understand it to mean do not sleep
+** at all. ^In SQLite version 3.42.0 and later, a negative
+** argument passed into sqlite3_sleep() is changed to zero before it is relayed
+** down into the xSleep method of the VFS.
 */
 SQLITE_API int sqlite3_sleep(int);
 
@@ -8138,9 +8207,9 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*);
 ** is undefined if the mutex is not currently entered by the
 ** calling thread or is not currently allocated.
 **
-** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or
-** sqlite3_mutex_leave() is a NULL pointer, then all three routines
-** behave as no-ops.
+** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(),
+** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer,
+** then any of the four routines behaves as a no-op.
 **
 ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()].
 */
@@ -9874,18 +9943,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...);
 ** [[SQLITE_VTAB_INNOCUOUS]]<dt>SQLITE_VTAB_INNOCUOUS</dt>
 ** <dd>Calls of the form
 ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the
-** the [xConnect] or [xCreate] methods of a [virtual table] implmentation
+** the [xConnect] or [xCreate] methods of a [virtual table] implementation
 ** identify that virtual table as being safe to use from within triggers
 ** and views.  Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the
 ** virtual table can do no serious harm even if it is controlled by a
 ** malicious hacker.  Developers should avoid setting the SQLITE_VTAB_INNOCUOUS
 ** flag unless absolutely necessary.
 ** </dd>
+**
+** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]<dt>SQLITE_VTAB_USES_ALL_SCHEMAS</dt>
+** <dd>Calls of the form
+** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the
+** the [xConnect] or [xCreate] methods of a [virtual table] implementation
+** instruct the query planner to begin at least a read transaction on
+** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the
+** virtual table is used.
+** </dd>
 ** </dl>
 */
 #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1
 #define SQLITE_VTAB_INNOCUOUS          2
 #define SQLITE_VTAB_DIRECTONLY         3
+#define SQLITE_VTAB_USES_ALL_SCHEMAS   4
 
 /*
 ** CAPI3REF: Determine The Virtual Table Conflict Policy
@@ -11060,16 +11139,20 @@ SQLITE_API int sqlite3session_create(
 SQLITE_API void sqlite3session_delete(sqlite3_session *pSession);
 
 /*
-** CAPIREF: Conigure a Session Object
+** CAPI3REF: Configure a Session Object
 ** METHOD: sqlite3_session
 **
 ** This method is used to configure a session object after it has been
-** created. At present the only valid value for the second parameter is
-** [SQLITE_SESSION_OBJCONFIG_SIZE].
+** created. At present the only valid values for the second parameter are
+** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID].
 **
-** Arguments for sqlite3session_object_config()
+*/
+SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg);
+
+/*
+** CAPI3REF: Options for sqlite3session_object_config
 **
-** The following values may passed as the the 4th parameter to
+** The following values may passed as the the 2nd parameter to
 ** sqlite3session_object_config().
 **
 ** <dt>SQLITE_SESSION_OBJCONFIG_SIZE <dd>
@@ -11085,12 +11168,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession);
 **
 **   It is an error (SQLITE_MISUSE) to attempt to modify this setting after
 **   the first table has been attached to the session object.
+**
+** <dt>SQLITE_SESSION_OBJCONFIG_ROWID <dd>
+**   This option is used to set, clear or query the flag that enables
+**   collection of data for tables with no explicit PRIMARY KEY.
+**
+**   Normally, tables with no explicit PRIMARY KEY are simply ignored
+**   by the sessions module. However, if this flag is set, it behaves
+**   as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted
+**   as their leftmost columns.
+**
+**   It is an error (SQLITE_MISUSE) to attempt to modify this setting after
+**   the first table has been attached to the session object.
 */
-SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg);
-
-/*
-*/
-#define SQLITE_SESSION_OBJCONFIG_SIZE 1
+#define SQLITE_SESSION_OBJCONFIG_SIZE  1
+#define SQLITE_SESSION_OBJCONFIG_ROWID 2
 
 /*
 ** CAPI3REF: Enable Or Disable A Session Object
@@ -12223,9 +12315,23 @@ SQLITE_API int sqlite3changeset_apply_v2(
 **   Invert the changeset before applying it. This is equivalent to inverting
 **   a changeset using sqlite3changeset_invert() before applying it. It is
 **   an error to specify this flag with a patchset.
+**
+** <dt>SQLITE_CHANGESETAPPLY_IGNORENOOP <dd>
+**   Do not invoke the conflict handler callback for any changes that
+**   would not actually modify the database even if they were applied.
+**   Specifically, this means that the conflict handler is not invoked
+**   for:
+**    <ul>
+**    <li>a delete change if the row being deleted cannot be found,
+**    <li>an update change if the modified fields are already set to
+**        their new values in the conflicting row, or
+**    <li>an insert change if all fields of the conflicting row match
+**        the row being inserted.
+**    </ul>
 */
 #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT   0x0001
 #define SQLITE_CHANGESETAPPLY_INVERT        0x0002
+#define SQLITE_CHANGESETAPPLY_IGNORENOOP    0x0004
 
 /*
 ** CAPI3REF: Constants Passed To The Conflict Handler
@@ -13522,8 +13628,8 @@ struct fts5_api {
 #endif
 
 /*
-** WAL mode depends on atomic aligned 32-bit loads and stores in a few
-** places.  The following macros try to make this explicit.
+** A few places in the code require atomic load/store of aligned
+** integer values.
 */
 #ifndef __has_extension
 # define __has_extension(x) 0     /* compatibility with non-clang compilers */
@@ -13579,15 +13685,22 @@ struct fts5_api {
 #endif
 
 /*
-** A macro to hint to the compiler that a function should not be
+** Macros to hint to the compiler that a function should or should not be
 ** inlined.
 */
 #if defined(__GNUC__)
 #  define SQLITE_NOINLINE  __attribute__((noinline))
+#  define SQLITE_INLINE    __attribute__((always_inline)) inline
 #elif defined(_MSC_VER) && _MSC_VER>=1310
 #  define SQLITE_NOINLINE  __declspec(noinline)
+#  define SQLITE_INLINE    __forceinline
 #else
 #  define SQLITE_NOINLINE
+#  define SQLITE_INLINE
+#endif
+#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__)
+# undef SQLITE_INLINE
+# define SQLITE_INLINE
 #endif
 
 /*
@@ -16548,6 +16661,10 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int);
 SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*);
 #endif
 
+#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG)
+SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr);
+#endif
+
 #endif /* SQLITE_VDBE_H */
 
 /************** End of vdbe.h ************************************************/
@@ -17257,7 +17374,7 @@ struct sqlite3 {
 #define SQLITE_NullCallback   0x00000100  /* Invoke the callback once if the */
                                           /*   result set is empty */
 #define SQLITE_IgnoreChecks   0x00000200  /* Do not enforce check constraints */
-#define SQLITE_ReadUncommit   0x00000400  /* READ UNCOMMITTED in shared-cache */
+#define SQLITE_StmtScanStatus 0x00000400  /* Enable stmt_scanstats() counters */
 #define SQLITE_NoCkptOnClose  0x00000800  /* No checkpoint on close()/DETACH */
 #define SQLITE_ReverseOrder   0x00001000  /* Reverse unordered SELECTs */
 #define SQLITE_RecTriggers    0x00002000  /* Enable recursive triggers */
@@ -17283,6 +17400,7 @@ struct sqlite3 {
                                           /*   DELETE, or UPDATE and return */
                                           /*   the count using a callback. */
 #define SQLITE_CorruptRdOnly  HI(0x00002) /* Prohibit writes due to error */
+#define SQLITE_ReadUncommit   HI(0x00004) /* READ UNCOMMITTED in shared-cache */
 
 /* Flags used only if debugging */
 #ifdef SQLITE_DEBUG
@@ -17339,6 +17457,7 @@ struct sqlite3 {
    /* TH3 expects this value  ^^^^^^^^^^ See flatten04.test */
 #define SQLITE_IndexedExpr    0x01000000 /* Pull exprs from index when able */
 #define SQLITE_Coroutines     0x02000000 /* Co-routines for subqueries */
+#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */
 #define SQLITE_AllOpts        0xffffffff /* All optimizations */
 
 /*
@@ -17810,6 +17929,7 @@ struct VTable {
   sqlite3_vtab *pVtab;      /* Pointer to vtab instance */
   int nRef;                 /* Number of pointers to this structure */
   u8 bConstraint;           /* True if constraints are supported */
+  u8 bAllSchemas;           /* True if might use any attached schema */
   u8 eVtabRisk;             /* Riskiness of allowing hacker access */
   int iSavepoint;           /* Depth of the SAVEPOINT stack */
   VTable *pNext;            /* Next in linked list (see above) */
@@ -18190,6 +18310,7 @@ struct Index {
                            ** expression, or a reference to a VIRTUAL column */
 #ifdef SQLITE_ENABLE_STAT4
   int nSample;             /* Number of elements in aSample[] */
+  int mxSample;            /* Number of slots allocated to aSample[] */
   int nSampleCol;          /* Size of IndexSample.anEq[] and so on */
   tRowcnt *aAvgEq;         /* Average nEq values for keys not in aSample */
   IndexSample *aSample;    /* Samples of the left-most key */
@@ -19676,6 +19797,7 @@ struct Walker {
     struct CoveringIndexCheck *pCovIdxCk;     /* Check for covering index */
     SrcItem *pSrcItem;                        /* A single FROM clause item */
     DbFixer *pFix;                            /* See sqlite3FixSelect() */
+    Mem *aMem;                                /* See sqlite3BtreeCursorHint() */
   } u;
 };
 
@@ -19945,6 +20067,8 @@ SQLITE_PRIVATE   int sqlite3CorruptPgnoError(int,Pgno);
 # define sqlite3Isxdigit(x)  (sqlite3CtypeMap[(unsigned char)(x)]&0x08)
 # define sqlite3Tolower(x)   (sqlite3UpperToLower[(unsigned char)(x)])
 # define sqlite3Isquote(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x80)
+# define sqlite3JsonId1(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x42)
+# define sqlite3JsonId2(x)   (sqlite3CtypeMap[(unsigned char)(x)]&0x46)
 #else
 # define sqlite3Toupper(x)   toupper((unsigned char)(x))
 # define sqlite3Isspace(x)   isspace((unsigned char)(x))
@@ -19954,6 +20078,8 @@ SQLITE_PRIVATE   int sqlite3CorruptPgnoError(int,Pgno);
 # define sqlite3Isxdigit(x)  isxdigit((unsigned char)(x))
 # define sqlite3Tolower(x)   tolower((unsigned char)(x))
 # define sqlite3Isquote(x)   ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`')
+# define sqlite3JsonId1(x)   (sqlite3IsIdChar(x)&&(x)<'0')
+# define sqlite3JsonId2(x)   sqlite3IsIdChar(x)
 #endif
 SQLITE_PRIVATE int sqlite3IsIdChar(u8);
 
@@ -20147,6 +20273,10 @@ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int);
 SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int);
 SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int);
 SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*);
+SQLITE_PRIVATE void sqlite3TouchRegister(Parse*,int);
+#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG)
+SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse*,int);
+#endif
 #ifdef SQLITE_DEBUG
 SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int);
 #endif
@@ -20297,7 +20427,7 @@ SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList
                          Expr*,ExprList*,u32,Expr*);
 SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*);
 SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*);
-SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int);
+SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*);
 SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int);
 #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY)
 SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*);
@@ -20386,7 +20516,7 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*);
 SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8);
 SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*);
 SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int);
-SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr*,const SrcItem*);
+SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int);
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
 SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*);
 #endif
@@ -20834,10 +20964,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *);
 SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *);
 
 SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*);
-#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \
-    && !defined(SQLITE_OMIT_VIRTUALTABLE)
-SQLITE_PRIVATE   void sqlite3VtabUsesAllSchemas(sqlite3_index_info*);
-#endif
+SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse*);
 SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*);
 SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int);
 SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *);
@@ -21084,6 +21211,12 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void);
 SQLITE_PRIVATE sqlite3_uint64 sqlite3Hwtime(void);
 #endif
 
+#ifdef SQLITE_ENABLE_STMT_SCANSTATUS
+# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus)
+#else
+# define IS_STMT_SCANSTATUS(db) 0
+#endif
+
 #endif /* SQLITEINT_H */
 
 /************** End of sqliteInt.h *******************************************/
@@ -22079,7 +22212,7 @@ SQLITE_PRIVATE const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP
 **   isalnum()                        0x06
 **   isxdigit()                       0x08
 **   toupper()                        0x20
-**   SQLite identifier character      0x40
+**   SQLite identifier character      0x40   $, _, or non-ascii
 **   Quote character                  0x80
 **
 ** Bit 0x20 is set if the mapped character requires translation to upper
@@ -22273,7 +22406,7 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = {
    SQLITE_DEFAULT_SORTERREF_SIZE,   /* szSorterRef */
    0,                         /* iPrngSeed */
 #ifdef SQLITE_DEBUG
-   {0,0,0,0,0,0}              /* aTune */
+   {0,0,0,0,0,0},             /* aTune */
 #endif
 };
 
@@ -23572,6 +23705,7 @@ struct DateTime {
   char validTZ;       /* True (1) if tz is valid */
   char tzSet;         /* Timezone was set explicitly */
   char isError;       /* An overflow has occurred */
+  char useSubsec;     /* Display subsecond precision */
 };
 
 
@@ -23886,6 +24020,11 @@ static int parseDateOrTime(
   }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){
     setRawDateNumber(p, r);
     return 0;
+  }else if( (sqlite3StrICmp(zDate,"subsec")==0
+             || sqlite3StrICmp(zDate,"subsecond")==0)
+           && sqlite3NotPureFunc(context) ){
+    p->useSubsec = 1;
+    return setDateTimeToCurrent(context, p);
   }
   return 1;
 }
@@ -24300,8 +24439,22 @@ static int parseModifier(
       **
       ** Move the date backwards to the beginning of the current day,
       ** or month or year.
+      **
+      **    subsecond
+      **    subsec
+      **
+      ** Show subsecond precision in the output of datetime() and
+      ** unixepoch() and strftime('%s').
       */
-      if( sqlite3_strnicmp(z, "start of ", 9)!=0 ) break;
+      if( sqlite3_strnicmp(z, "start of ", 9)!=0 ){
+        if( sqlite3_stricmp(z, "subsec")==0
+         || sqlite3_stricmp(z, "subsecond")==0
+        ){
+          p->useSubsec = 1;
+          rc = 0;
+        }
+        break;
+      }
       if( !p->validJD && !p->validYMD && !p->validHMS ) break;
       z += 9;
       computeYMD(p);
@@ -24499,7 +24652,11 @@ static void unixepochFunc(
   DateTime x;
   if( isDate(context, argc, argv, &x)==0 ){
     computeJD(&x);
-    sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000);
+    if( x.useSubsec ){
+      sqlite3_result_double(context, (x.iJD - 21086676*(i64)10000000)/1000.0);
+    }else{
+      sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000);
+    }
   }
 }
 
@@ -24515,8 +24672,8 @@ static void datetimeFunc(
 ){
   DateTime x;
   if( isDate(context, argc, argv, &x)==0 ){
-    int Y, s;
-    char zBuf[24];
+    int Y, s, n;
+    char zBuf[32];
     computeYMD_HMS(&x);
     Y = x.Y;
     if( Y<0 ) Y = -Y;
@@ -24537,15 +24694,28 @@ static void datetimeFunc(
     zBuf[15] = '0' + (x.m/10)%10;
     zBuf[16] = '0' + (x.m)%10;
     zBuf[17] = ':';
-    s = (int)x.s;
-    zBuf[18] = '0' + (s/10)%10;
-    zBuf[19] = '0' + (s)%10;
-    zBuf[20] = 0;
+    if( x.useSubsec ){
+      s = (int)1000.0*x.s;
+      zBuf[18] = '0' + (s/10000)%10;
+      zBuf[19] = '0' + (s/1000)%10;
+      zBuf[20] = '.';
+      zBuf[21] = '0' + (s/100)%10;
+      zBuf[22] = '0' + (s/10)%10;
+      zBuf[23] = '0' + (s)%10;
+      zBuf[24] = 0;
+      n = 24;
+    }else{
+      s = (int)x.s;
+      zBuf[18] = '0' + (s/10)%10;
+      zBuf[19] = '0' + (s)%10;
+      zBuf[20] = 0;
+      n = 20;
+    }
     if( x.Y<0 ){
       zBuf[0] = '-';
-      sqlite3_result_text(context, zBuf, 20, SQLITE_TRANSIENT);
+      sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT);
     }else{
-      sqlite3_result_text(context, &zBuf[1], 19, SQLITE_TRANSIENT);
+      sqlite3_result_text(context, &zBuf[1], n-1, SQLITE_TRANSIENT);
     }
   }
 }
@@ -24562,7 +24732,7 @@ static void timeFunc(
 ){
   DateTime x;
   if( isDate(context, argc, argv, &x)==0 ){
-    int s;
+    int s, n;
     char zBuf[16];
     computeHMS(&x);
     zBuf[0] = '0' + (x.h/10)%10;
@@ -24571,11 +24741,24 @@ static void timeFunc(
     zBuf[3] = '0' + (x.m/10)%10;
     zBuf[4] = '0' + (x.m)%10;
     zBuf[5] = ':';
-    s = (int)x.s;
-    zBuf[6] = '0' + (s/10)%10;
-    zBuf[7] = '0' + (s)%10;
-    zBuf[8] = 0;
-    sqlite3_result_text(context, zBuf, 8, SQLITE_TRANSIENT);
+    if( x.useSubsec ){
+      s = (int)1000.0*x.s;
+      zBuf[6] = '0' + (s/10000)%10;
+      zBuf[7] = '0' + (s/1000)%10;
+      zBuf[8] = '.';
+      zBuf[9] = '0' + (s/100)%10;
+      zBuf[10] = '0' + (s/10)%10;
+      zBuf[11] = '0' + (s)%10;
+      zBuf[12] = 0;
+      n = 12;
+    }else{
+      s = (int)x.s;
+      zBuf[6] = '0' + (s/10)%10;
+      zBuf[7] = '0' + (s)%10;
+      zBuf[8] = 0;
+      n = 8;
+    }
+    sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT);
   }
 }
 
@@ -24706,8 +24889,13 @@ static void strftimeFunc(
         break;
       }
       case 's': {
-        i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000);
-        sqlite3_str_appendf(&sRes,"%lld",iS);
+        if( x.useSubsec ){
+          sqlite3_str_appendf(&sRes,"%.3f",
+                (x.iJD - 21086676*(i64)10000000)/1000.0);
+        }else{
+          i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000);
+          sqlite3_str_appendf(&sRes,"%lld",iS);
+        }
         break;
       }
       case 'S': {
@@ -30078,6 +30266,20 @@ static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){
 }
 #endif /* SQLITE_OMIT_FLOATING_POINT */
 
+#ifndef SQLITE_OMIT_FLOATING_POINT
+/*
+** "*val" is a u64.  *msd is a divisor used to extract the
+** most significant digit of *val.  Extract that most significant
+** digit and return it.
+*/
+static char et_getdigit_int(u64 *val, u64 *msd){
+  u64 x = (*val)/(*msd);
+  *val -= x*(*msd);
+  if( *msd>=10 ) *msd /= 10;
+  return '0' + (char)(x & 15);
+}
+#endif /* SQLITE_OMIT_FLOATING_POINT */
+
 /*
 ** Set the StrAccum object to an error mode.
 */
@@ -30170,6 +30372,8 @@ SQLITE_API void sqlite3_str_vappendf(
   char prefix;               /* Prefix character.  "+" or "-" or " " or '\0'. */
   sqlite_uint64 longvalue;   /* Value for integer types */
   LONGDOUBLE_TYPE realvalue; /* Value for real types */
+  sqlite_uint64 msd;         /* Divisor to get most-significant-digit
+                             ** of longvalue */
   const et_info *infop;      /* Pointer to the appropriate info structure */
   char *zOut;                /* Rendering buffer */
   int nOut;                  /* Size of the rendering buffer */
@@ -30476,52 +30680,78 @@ SQLITE_API void sqlite3_str_vappendf(
         }else{
           prefix = flag_prefix;
         }
+        exp = 0;
         if( xtype==etGENERIC && precision>0 ) precision--;
         testcase( precision>0xfff );
-        idx = precision & 0xfff;
-        rounder = arRound[idx%10];
-        while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; }
-        if( xtype==etFLOAT ){
-          double rx = (double)realvalue;
-          sqlite3_uint64 u;
-          int ex;
-          memcpy(&u, &rx, sizeof(u));
-          ex = -1023 + (int)((u>>52)&0x7ff);
-          if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16;
-          realvalue += rounder;
-        }
-        /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */
-        exp = 0;
-        if( sqlite3IsNaN((double)realvalue) ){
-          bufpt = "NaN";
-          length = 3;
-          break;
-        }
-        if( realvalue>0.0 ){
-          LONGDOUBLE_TYPE scale = 1.0;
-          while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;}
-          while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; }
-          while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; }
-          realvalue /= scale;
-          while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; }
-          while( realvalue<1.0 ){ realvalue *= 10.0; exp--; }
-          if( exp>350 ){
-            bufpt = buf;
-            buf[0] = prefix;
-            memcpy(buf+(prefix!=0),"Inf",4);
-            length = 3+(prefix!=0);
+        if( realvalue<1.0e+16
+         && realvalue==(LONGDOUBLE_TYPE)(longvalue = (u64)realvalue)
+        ){
+          /* Number is a pure integer that can be represented as u64 */
+          for(msd=1; msd*10<=longvalue; msd *= 10, exp++){}
+          if( exp>precision && xtype!=etFLOAT ){
+            u64 rnd = msd/2;
+            int kk = precision;
+            while( kk-- > 0 ){  rnd /= 10; }
+            longvalue += rnd;
+          }
+        }else{
+          msd = 0;
+          longvalue = 0;  /* To prevent a compiler warning */
+          idx = precision & 0xfff;
+          rounder = arRound[idx%10];
+          while( idx>=10 ){ rounder *= 1.0e-10; idx -= 10; }
+          if( xtype==etFLOAT ){
+            double rx = (double)realvalue;
+            sqlite3_uint64 u;
+            int ex;
+            memcpy(&u, &rx, sizeof(u));
+            ex = -1023 + (int)((u>>52)&0x7ff);
+            if( precision+(ex/3) < 15 ) rounder += realvalue*3e-16;
+            realvalue += rounder;
+          }
+          if( sqlite3IsNaN((double)realvalue) ){
+            if( flag_zeropad ){
+              bufpt = "null";
+              length = 4;
+            }else{
+              bufpt = "NaN";
+              length = 3;
+            }
             break;
           }
+
+          /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */
+          if( ALWAYS(realvalue>0.0) ){
+            LONGDOUBLE_TYPE scale = 1.0;
+            while( realvalue>=1e100*scale && exp<=350){ scale*=1e100;exp+=100;}
+            while( realvalue>=1e10*scale && exp<=350 ){ scale*=1e10; exp+=10; }
+            while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; }
+            realvalue /= scale;
+            while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; }
+            while( realvalue<1.0 ){ realvalue *= 10.0; exp--; }
+            if( exp>350 ){
+              if( flag_zeropad ){
+                realvalue = 9.0;
+                exp = 999;
+              }else{
+                bufpt = buf;
+                buf[0] = prefix;
+                memcpy(buf+(prefix!=0),"Inf",4);
+                length = 3+(prefix!=0);
+                break;
+              }
+            }
+            if( xtype!=etFLOAT ){
+              realvalue += rounder;
+              if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; }
+            }
+          }
         }
-        bufpt = buf;
+
         /*
         ** If the field type is etGENERIC, then convert to either etEXP
         ** or etFLOAT, as appropriate.
         */
-        if( xtype!=etFLOAT ){
-          realvalue += rounder;
-          if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; }
-        }
         if( xtype==etGENERIC ){
           flag_rtz = !flag_alternateform;
           if( exp<-4 || exp>precision ){
@@ -30538,16 +30768,18 @@ SQLITE_API void sqlite3_str_vappendf(
         }else{
           e2 = exp;
         }
+        nsd = 16 + flag_altform2*10;
+        bufpt = buf;
         {
           i64 szBufNeeded;           /* Size of a temporary buffer needed */
           szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15;
+          if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3;
           if( szBufNeeded > etBUFSIZE ){
             bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded);
             if( bufpt==0 ) return;
           }
         }
         zOut = bufpt;
-        nsd = 16 + flag_altform2*10;
         flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2;
         /* The sign in front of the number */
         if( prefix ){
@@ -30556,9 +30788,15 @@ SQLITE_API void sqlite3_str_vappendf(
         /* Digits prior to the decimal point */
         if( e2<0 ){
           *(bufpt++) = '0';
+        }else if( msd>0 ){
+          for(; e2>=0; e2--){
+            *(bufpt++) = et_getdigit_int(&longvalue,&msd);
+            if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ',';
+          }
         }else{
           for(; e2>=0; e2--){
             *(bufpt++) = et_getdigit(&realvalue,&nsd);
+            if( cThousand && (e2%3)==0 && e2>1 ) *(bufpt++) = ',';
           }
         }
         /* The decimal point */
@@ -30572,8 +30810,14 @@ SQLITE_API void sqlite3_str_vappendf(
           *(bufpt++) = '0';
         }
         /* Significant digits after the decimal point */
-        while( (precision--)>0 ){
-          *(bufpt++) = et_getdigit(&realvalue,&nsd);
+        if( msd>0 ){
+          while( (precision--)>0 ){
+            *(bufpt++) = et_getdigit_int(&longvalue,&msd);
+          }
+        }else{
+          while( (precision--)>0 ){
+            *(bufpt++) = et_getdigit(&realvalue,&nsd);
+          }
         }
         /* Remove trailing zeros and the "." if no digits follow the "." */
         if( flag_rtz && flag_dp ){
@@ -31254,12 +31498,22 @@ SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_li
   return zBuf;
 }
 SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){
-  char *z;
+  StrAccum acc;
   va_list ap;
+  if( n<=0 ) return zBuf;
+#ifdef SQLITE_ENABLE_API_ARMOR
+  if( zBuf==0 || zFormat==0 ) {
+    (void)SQLITE_MISUSE_BKPT;
+    if( zBuf ) zBuf[0] = 0;
+    return zBuf;
+  }
+#endif
+  sqlite3StrAccumInit(&acc, 0, zBuf, n, 0);
   va_start(ap,zFormat);
-  z = sqlite3_vsnprintf(n, zBuf, zFormat, ap);
+  sqlite3_str_vappendf(&acc, zFormat, ap);
   va_end(ap);
-  return z;
+  zBuf[acc.nChar] = 0;
+  return zBuf;
 }
 
 /*
@@ -34289,13 +34543,15 @@ SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){
   }
   i = sizeof(zTemp)-2;
   zTemp[sizeof(zTemp)-1] = 0;
-  do{
-    zTemp[i--] = (x%10) + '0';
+  while( 1 /*exit-by-break*/ ){
+    zTemp[i] = (x%10) + '0';
     x = x/10;
-  }while( x );
-  if( v<0 ) zTemp[i--] = '-';
-  memcpy(zOut, &zTemp[i+1], sizeof(zTemp)-1-i);
-  return sizeof(zTemp)-2-i;
+    if( x==0 ) break;
+    i--;
+  };
+  if( v<0 ) zTemp[--i] = '-';
+  memcpy(zOut, &zTemp[i], sizeof(zTemp)-i);
+  return sizeof(zTemp)-1-i;
 }
 
 /*
@@ -34460,7 +34716,9 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){
       u = u*16 + sqlite3HexToInt(z[k]);
     }
     memcpy(pOut, &u, 8);
-    return (z[k]==0 && k-i<=16) ? 0 : 2;
+    if( k-i>16 ) return 2;
+    if( z[k]!=0 ) return 1;
+    return 0;
   }else
 #endif /* SQLITE_OMIT_HEX_INTEGER */
   {
@@ -34496,7 +34754,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){
     u32 u = 0;
     zNum += 2;
     while( zNum[0]=='0' ) zNum++;
-    for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){
+    for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){
       u = u*16 + sqlite3HexToInt(zNum[i]);
     }
     if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){
@@ -36992,7 +37250,7 @@ SQLITE_PRIVATE int sqlite3KvvfsInit(void){
 #endif
 
 /* Use pread() and pwrite() if they are available */
-#if defined(__APPLE__)
+#if defined(__APPLE__) || defined(__linux__)
 # define HAVE_PREAD 1
 # define HAVE_PWRITE 1
 #endif
@@ -40242,12 +40500,6 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){
 ** Seek to the offset passed as the second argument, then read cnt
 ** bytes into pBuf. Return the number of bytes actually read.
 **
-** NB:  If you define USE_PREAD or USE_PREAD64, then it might also
-** be necessary to define _XOPEN_SOURCE to be 500.  This varies from
-** one system to another.  Since SQLite does not define USE_PREAD
-** in any form by default, we will not attempt to define _XOPEN_SOURCE.
-** See tickets #2741 and #2681.
-**
 ** To avoid stomping the errno value on a failed read the lastErrno value
 ** is set before returning.
 */
@@ -50274,7 +50526,7 @@ static int winOpen(
       if( isReadWrite ){
         int rc2, isRO = 0;
         sqlite3BeginBenignMalloc();
-        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO);
         sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
@@ -50291,7 +50543,7 @@ static int winOpen(
       if( isReadWrite ){
         int rc2, isRO = 0;
         sqlite3BeginBenignMalloc();
-        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO);
         sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
@@ -50311,7 +50563,7 @@ static int winOpen(
       if( isReadWrite ){
         int rc2, isRO = 0;
         sqlite3BeginBenignMalloc();
-        rc2 = winAccess(pVfs, zName, SQLITE_ACCESS_READ, &isRO);
+        rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO);
         sqlite3EndBenignMalloc();
         if( rc2==SQLITE_OK && isRO ) break;
       }
@@ -50534,6 +50786,13 @@ static int winAccess(
   OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n",
            zFilename, flags, pResOut));
 
+  if( zFilename==0 ){
+    *pResOut = 0;
+    OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n",
+             zFilename, pResOut, *pResOut));
+    return SQLITE_OK;
+  }
+
   zConverted = winConvertFromUtf8Filename(zFilename);
   if( zConverted==0 ){
     OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename));
@@ -52690,11 +52949,15 @@ struct PCache {
     PgHdr *pPg;
     unsigned char *a;
     int j;
-    pPg = (PgHdr*)pLower->pExtra;
-    printf("%3lld: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags);
-    a = (unsigned char *)pLower->pBuf;
-    for(j=0; j<12; j++) printf("%02x", a[j]);
-    printf(" ptr %p\n", pPg);
+    if( pLower==0 ){
+      printf("%3d: NULL\n", i);
+    }else{
+      pPg = (PgHdr*)pLower->pExtra;
+      printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags);
+      a = (unsigned char *)pLower->pBuf;
+      for(j=0; j<12; j++) printf("%02x", a[j]);
+      printf(" ptr %p\n", pPg);
+    }
   }
   static void pcacheDump(PCache *pCache){
     int N;
@@ -52707,9 +52970,8 @@ struct PCache {
     if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump;
     for(i=1; i<=N; i++){
        pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0);
-       if( pLower==0 ) continue;
        pcachePageTrace(i, pLower);
-       if( ((PgHdr*)pLower)->pPage==0 ){
+       if( pLower && ((PgHdr*)pLower)->pPage==0 ){
          sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0);
        }
     }
@@ -58097,6 +58359,8 @@ static int pager_truncate(Pager *pPager, Pgno nPage){
   int rc = SQLITE_OK;
   assert( pPager->eState!=PAGER_ERROR );
   assert( pPager->eState!=PAGER_READER );
+  PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage));
+
 
   if( isOpen(pPager->fd)
    && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN)
@@ -61014,6 +61278,10 @@ static int getPageNormal(
     if( !isOpen(pPager->fd) || pPager->dbSize<pgno || noContent ){
       if( pgno>pPager->mxPgno ){
         rc = SQLITE_FULL;
+        if( pgno<=pPager->dbSize ){
+          sqlite3PcacheRelease(pPg);
+          pPg = 0;
+        }
         goto pager_acquire_err;
       }
       if( noContent ){
@@ -61178,10 +61446,12 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){
 /*
 ** Release a page reference.
 **
-** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be
-** used if we know that the page being released is not the last page.
+** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used
+** if we know that the page being released is not the last reference to page1.
 ** The btree layer always holds page1 open until the end, so these first
-** to routines can be used to release any page other than BtShared.pPage1.
+** two routines can be used to release any page other than BtShared.pPage1.
+** The assert() at tag-20230419-2 proves that this constraint is always
+** honored.
 **
 ** Use sqlite3PagerUnrefPageOne() to release page1.  This latter routine
 ** checks the total number of outstanding pages and if the number of
@@ -61197,7 +61467,7 @@ SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){
     sqlite3PcacheRelease(pPg);
   }
   /* Do not use this routine to release the last reference to page1 */
-  assert( sqlite3PcacheRefCount(pPager->pPCache)>0 );
+  assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */
 }
 SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){
   if( pPg ) sqlite3PagerUnrefNotNull(pPg);
@@ -62957,13 +63227,15 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){
 */
 static int pagerExclusiveLock(Pager *pPager){
   int rc;                         /* Return code */
+  u8 eOrigLock;                   /* Original lock */
 
-  assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK );
+  assert( pPager->eLock>=SHARED_LOCK );
+  eOrigLock = pPager->eLock;
   rc = pagerLockDb(pPager, EXCLUSIVE_LOCK);
   if( rc!=SQLITE_OK ){
     /* If the attempt to grab the exclusive lock failed, release the
     ** pending lock that may have been obtained instead.  */
-    pagerUnlockDb(pPager, SHARED_LOCK);
+    pagerUnlockDb(pPager, eOrigLock);
   }
 
   return rc;
@@ -63968,19 +64240,40 @@ static void walChecksumBytes(
   assert( nByte>=8 );
   assert( (nByte&0x00000007)==0 );
   assert( nByte<=65536 );
+  assert( nByte%4==0 );
 
-  if( nativeCksum ){
+  if( !nativeCksum ){
+    do {
+      s1 += BYTESWAP32(aData[0]) + s2;
+      s2 += BYTESWAP32(aData[1]) + s1;
+      aData += 2;
+    }while( aData<aEnd );
+  }else if( nByte%64==0 ){
     do {
       s1 += *aData++ + s2;
       s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
     }while( aData<aEnd );
   }else{
     do {
-      s1 += BYTESWAP32(aData[0]) + s2;
-      s2 += BYTESWAP32(aData[1]) + s1;
-      aData += 2;
+      s1 += *aData++ + s2;
+      s2 += *aData++ + s1;
     }while( aData<aEnd );
   }
+  assert( aData==aEnd );
 
   aOut[0] = s1;
   aOut[1] = s2;
@@ -66911,7 +67204,9 @@ SQLITE_PRIVATE int sqlite3WalFrames(
       if( rc ) return rc;
     }
   }
-  assert( (int)pWal->szPage==szPage );
+  if( (int)pWal->szPage!=szPage ){
+    return SQLITE_CORRUPT_BKPT;  /* TH3 test case: cov1/corrupt155.test */
+  }
 
   /* Setup information needed to write frames into the WAL */
   w.pWal = pWal;
@@ -67571,7 +67866,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){
 ** byte are used.  The integer consists of all bytes that have bit 8 set and
 ** the first byte with bit 8 clear.  The most significant byte of the integer
 ** appears first.  A variable-length integer may not be more than 9 bytes long.
-** As a special case, all 8 bytes of the 9th byte are used as data.  This
+** As a special case, all 8 bits of the 9th byte are used as data.  This
 ** allows a 64-bit integer to be encoded in 9 bytes.
 **
 **    0x00                      becomes  0x00000000
@@ -67955,7 +68250,7 @@ struct BtCursor {
 #define BTCF_WriteFlag    0x01   /* True if a write cursor */
 #define BTCF_ValidNKey    0x02   /* True if info.nKey is valid */
 #define BTCF_ValidOvfl    0x04   /* True if aOverflow is valid */
-#define BTCF_AtLast       0x08   /* Cursor is pointing ot the last entry */
+#define BTCF_AtLast       0x08   /* Cursor is pointing to the last entry */
 #define BTCF_Incrblob     0x10   /* True if an incremental I/O handle */
 #define BTCF_Multiple     0x20   /* Maybe another cursor on the same btree */
 #define BTCF_Pinned       0x40   /* Cursor is busy and cannot be moved */
@@ -68100,8 +68395,9 @@ struct IntegrityCk {
   int rc;           /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */
   u32 nStep;        /* Number of steps into the integrity_check process */
   const char *zPfx; /* Error message prefix */
-  Pgno v1;          /* Value for first %u substitution in zPfx */
-  int v2;           /* Value for second %d substitution in zPfx */
+  Pgno v0;          /* Value for first %u substitution in zPfx (root page) */
+  Pgno v1;          /* Value for second %u substitution in zPfx (current pg) */
+  int v2;           /* Value for third %d substitution in zPfx */
   StrAccum errMsg;  /* Accumulate the error message text here */
   u32 *heap;        /* Min-heap used for analyzing cell coverage */
   sqlite3 *db;      /* Database connection running the check */
@@ -68564,8 +68860,8 @@ SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){
 int corruptPageError(int lineno, MemPage *p){
   char *zMsg;
   sqlite3BeginBenignMalloc();
-  zMsg = sqlite3_mprintf("database corruption page %d of %s",
-      (int)p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0)
+  zMsg = sqlite3_mprintf("database corruption page %u of %s",
+             p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0)
   );
   sqlite3EndBenignMalloc();
   if( zMsg ){
@@ -69374,8 +69670,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow)
 */
 SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){
   /* Used only by system that substitute their own storage engine */
+#ifdef SQLITE_DEBUG
+  if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){
+    va_list ap;
+    Expr *pExpr;
+    Walker w;
+    memset(&w, 0, sizeof(w));
+    w.xExprCallback = sqlite3CursorRangeHintExprCheck;
+    va_start(ap, eHintType);
+    pExpr = va_arg(ap, Expr*);
+    w.u.aMem = va_arg(ap, Mem*);
+    va_end(ap);
+    assert( pExpr!=0 );
+    assert( w.u.aMem!=0 );
+    sqlite3WalkExpr(&w, pExpr);
+  }
+#endif /* SQLITE_DEBUG */
 }
-#endif
+#endif /* SQLITE_ENABLE_CURSOR_HINTS */
+
 
 /*
 ** Provide flag hints to the cursor.
@@ -69460,7 +69773,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){
   pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage);
 
   if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){
-    TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent));
+    TRACE(("PTRMAP_UPDATE: %u->(%u,%u)\n", key, eType, parent));
     *pRC= rc = sqlite3PagerWrite(pDbPage);
     if( rc==SQLITE_OK ){
       pPtrmap[offset] = eType;
@@ -69659,27 +69972,31 @@ static void btreeParseCellPtr(
   iKey = *pIter;
   if( iKey>=0x80 ){
     u8 x;
-    iKey = ((iKey&0x7f)<<7) | ((x = *++pIter) & 0x7f);
+    iKey = (iKey<<7) ^ (x = *++pIter);
     if( x>=0x80 ){
-      iKey = (iKey<<7) | ((x =*++pIter) & 0x7f);
+      iKey = (iKey<<7) ^ (x = *++pIter);
       if( x>=0x80 ){
-        iKey = (iKey<<7) | ((x = *++pIter) & 0x7f);
+        iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++pIter);
         if( x>=0x80 ){
-          iKey = (iKey<<7) | ((x = *++pIter) & 0x7f);
+          iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter);
           if( x>=0x80 ){
-            iKey = (iKey<<7) | ((x = *++pIter) & 0x7f);
+            iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter);
             if( x>=0x80 ){
-              iKey = (iKey<<7) | ((x = *++pIter) & 0x7f);
+              iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter);
               if( x>=0x80 ){
-                iKey = (iKey<<7) | ((x = *++pIter) & 0x7f);
+                iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter);
                 if( x>=0x80 ){
-                  iKey = (iKey<<8) | (*++pIter);
+                  iKey = (iKey<<8) ^ 0x8000 ^ (*++pIter);
                 }
               }
             }
           }
         }
+      }else{
+        iKey ^= 0x204000;
       }
+    }else{
+      iKey ^= 0x4000;
     }
   }
   pIter++;
@@ -69756,10 +70073,11 @@ static void btreeParseCell(
 **
 ** cellSizePtrNoPayload()    =>   table internal nodes
 ** cellSizePtrTableLeaf()    =>   table leaf nodes
-** cellSizePtr()             =>   all index nodes & table leaf nodes
+** cellSizePtr()             =>   index internal nodes
+** cellSizeIdxLeaf()         =>   index leaf nodes
 */
 static u16 cellSizePtr(MemPage *pPage, u8 *pCell){
-  u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */
+  u8 *pIter = pCell + 4;                   /* For looping over bytes of pCell */
   u8 *pEnd;                                /* End mark for a varint */
   u32 nSize;                               /* Size value to return */
 
@@ -69772,6 +70090,49 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){
   pPage->xParseCell(pPage, pCell, &debuginfo);
 #endif
 
+  assert( pPage->childPtrSize==4 );
+  nSize = *pIter;
+  if( nSize>=0x80 ){
+    pEnd = &pIter[8];
+    nSize &= 0x7f;
+    do{
+      nSize = (nSize<<7) | (*++pIter & 0x7f);
+    }while( *(pIter)>=0x80 && pIter<pEnd );
+  }
+  pIter++;
+  testcase( nSize==pPage->maxLocal );
+  testcase( nSize==(u32)pPage->maxLocal+1 );
+  if( nSize<=pPage->maxLocal ){
+    nSize += (u32)(pIter - pCell);
+    assert( nSize>4 );
+  }else{
+    int minLocal = pPage->minLocal;
+    nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4);
+    testcase( nSize==pPage->maxLocal );
+    testcase( nSize==(u32)pPage->maxLocal+1 );
+    if( nSize>pPage->maxLocal ){
+      nSize = minLocal;
+    }
+    nSize += 4 + (u16)(pIter - pCell);
+  }
+  assert( nSize==debuginfo.nSize || CORRUPT_DB );
+  return (u16)nSize;
+}
+static u16 cellSizePtrIdxLeaf(MemPage *pPage, u8 *pCell){
+  u8 *pIter = pCell;                       /* For looping over bytes of pCell */
+  u8 *pEnd;                                /* End mark for a varint */
+  u32 nSize;                               /* Size value to return */
+
+#ifdef SQLITE_DEBUG
+  /* The value returned by this function should always be the same as
+  ** the (CellInfo.nSize) value found by doing a full parse of the
+  ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of
+  ** this function verifies that this invariant is not violated. */
+  CellInfo debuginfo;
+  pPage->xParseCell(pPage, pCell, &debuginfo);
+#endif
+
+  assert( pPage->childPtrSize==0 );
   nSize = *pIter;
   if( nSize>=0x80 ){
     pEnd = &pIter[8];
@@ -70008,10 +70369,10 @@ static int defragmentPage(MemPage *pPage, int nMaxFrag){
       /* These conditions have already been verified in btreeInitPage()
       ** if PRAGMA cell_size_check=ON.
       */
-      if( pc<iCellStart || pc>iCellLast ){
+      if( pc>iCellLast ){
         return SQLITE_CORRUPT_PAGE(pPage);
       }
-      assert( pc>=iCellStart && pc<=iCellLast );
+      assert( pc>=0 && pc<=iCellLast );
       size = pPage->xCellSize(pPage, &src[pc]);
       cbrk -= size;
       if( cbrk<iCellStart || pc+size>usableSize ){
@@ -70126,7 +70487,7 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){
 ** allocation is being made in order to insert a new cell, so we will
 ** also end up needing a new cell pointer.
 */
-static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
+static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
   const int hdr = pPage->hdrOffset;    /* Local cache of pPage->hdrOffset */
   u8 * const data = pPage->aData;      /* Local cache of pPage->aData */
   int top;                             /* First byte of cell content area */
@@ -70152,13 +70513,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){
   ** integer, so a value of 0 is used in its place. */
   pTmp = &data[hdr+5];
   top = get2byte(pTmp);
-  assert( top<=(int)pPage->pBt->usableSize ); /* by btreeComputeFreeSpace() */
   if( gap>top ){
     if( top==0 && pPage->pBt->usableSize==65536 ){
       top = 65536;
     }else{
       return SQLITE_CORRUPT_PAGE(pPage);
     }
+  }else if( top>(int)pPage->pBt->usableSize ){
+    return SQLITE_CORRUPT_PAGE(pPage);
   }
 
   /* If there is enough space between gap and top for one more cell pointer,
@@ -70241,7 +70603,7 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize );
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   assert( iSize>=4 );   /* Minimum cell size is 4 */
-  assert( iStart<=pPage->pBt->usableSize-4 );
+  assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 );
 
   /* The list of freeblocks must be in ascending order.  Find the
   ** spot on the list where iStart should be inserted.
@@ -70298,6 +70660,11 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   }
   pTmp = &data[hdr+5];
   x = get2byte(pTmp);
+  if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){
+    /* Overwrite deleted information with zeros when the secure_delete
+    ** option is enabled */
+    memset(&data[iStart], 0, iSize);
+  }
   if( iStart<=x ){
     /* The new freeblock is at the beginning of the cell content area,
     ** so just extend the cell content area rather than create another
@@ -70309,14 +70676,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){
   }else{
     /* Insert the new freeblock into the freelist */
     put2byte(&data[iPtr], iStart);
+    put2byte(&data[iStart], iFreeBlk);
+    put2byte(&data[iStart+2], iSize);
   }
-  if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){
-    /* Overwrite deleted information with zeros when the secure_delete
-    ** option is enabled */
-    memset(&data[iStart], 0, iSize);
-  }
-  put2byte(&data[iStart], iFreeBlk);
-  put2byte(&data[iStart+2], iSize);
   pPage->nFree += iOrigSize;
   return SQLITE_OK;
 }
@@ -70353,14 +70715,14 @@ static int decodeFlags(MemPage *pPage, int flagByte){
     }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){
       pPage->intKey = 0;
       pPage->intKeyLeaf = 0;
-      pPage->xCellSize = cellSizePtr;
+      pPage->xCellSize = cellSizePtrIdxLeaf;
       pPage->xParseCell = btreeParseCellPtrIndex;
       pPage->maxLocal = pBt->maxLocal;
       pPage->minLocal = pBt->minLocal;
     }else{
       pPage->intKey = 0;
       pPage->intKeyLeaf = 0;
-      pPage->xCellSize = cellSizePtr;
+      pPage->xCellSize = cellSizePtrIdxLeaf;
       pPage->xParseCell = btreeParseCellPtrIndex;
       return SQLITE_CORRUPT_PAGE(pPage);
     }
@@ -72226,7 +72588,7 @@ static int relocatePage(
   if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT;
 
   /* Move page iDbPage from its current location to page number iFreePage */
-  TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n",
+  TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n",
       iDbPage, iFreePage, iPtrPage, eType));
   rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit);
   if( rc!=SQLITE_OK ){
@@ -74512,7 +74874,8 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){
 
   pPage = pCur->pPage;
   idx = ++pCur->ix;
-  if( !pPage->isInit || sqlite3FaultSim(412) ){
+  if( sqlite3FaultSim(412) ) pPage->isInit = 0;
+  if( !pPage->isInit ){
     return SQLITE_CORRUPT_BKPT;
   }
 
@@ -74775,7 +75138,7 @@ static int allocateBtreePage(
         memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4);
         *ppPage = pTrunk;
         pTrunk = 0;
-        TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1));
+        TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1));
       }else if( k>(u32)(pBt->usableSize/4 - 2) ){
         /* Value of k is out of range.  Database corruption */
         rc = SQLITE_CORRUPT_PGNO(iTrunk);
@@ -74841,7 +75204,7 @@ static int allocateBtreePage(
           }
         }
         pTrunk = 0;
-        TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1));
+        TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1));
 #endif
       }else if( k>0 ){
         /* Extract a leaf from the trunk */
@@ -74886,8 +75249,8 @@ static int allocateBtreePage(
         ){
           int noContent;
           *pPgno = iPage;
-          TRACE(("ALLOCATE: %d was leaf %d of %d on trunk %d"
-                 ": %d more free pages\n",
+          TRACE(("ALLOCATE: %u was leaf %u of %u on trunk %u"
+                 ": %u more free pages\n",
                  *pPgno, closest+1, k, pTrunk->pgno, n-1));
           rc = sqlite3PagerWrite(pTrunk->pDbPage);
           if( rc ) goto end_allocate_page;
@@ -74943,7 +75306,7 @@ static int allocateBtreePage(
       ** becomes a new pointer-map page, the second is used by the caller.
       */
       MemPage *pPg = 0;
-      TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage));
+      TRACE(("ALLOCATE: %u from end of file (pointer-map page)\n", pBt->nPage));
       assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) );
       rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent);
       if( rc==SQLITE_OK ){
@@ -74966,7 +75329,7 @@ static int allocateBtreePage(
       releasePage(*ppPage);
       *ppPage = 0;
     }
-    TRACE(("ALLOCATE: %d from end of file\n", *pPgno));
+    TRACE(("ALLOCATE: %u from end of file\n", *pPgno));
   }
 
   assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) );
@@ -75094,7 +75457,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
         }
         rc = btreeSetHasContent(pBt, iPage);
       }
-      TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno));
+      TRACE(("FREE-PAGE: %u leaf on trunk page %u\n",pPage->pgno,pTrunk->pgno));
       goto freepage_out;
     }
   }
@@ -75115,7 +75478,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){
   put4byte(pPage->aData, iTrunk);
   put4byte(&pPage->aData[4], 0);
   put4byte(&pPage1->aData[32], iPage);
-  TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk));
+  TRACE(("FREE-PAGE: %u new trunk page replacing %u\n", pPage->pgno, iTrunk));
 
 freepage_out:
   if( pPage ){
@@ -75474,6 +75837,14 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){
 ** in pTemp or the original pCell) and also record its index.
 ** Allocating a new entry in pPage->aCell[] implies that
 ** pPage->nOverflow is incremented.
+**
+** The insertCellFast() routine below works exactly the same as
+** insertCell() except that it lacks the pTemp and iChild parameters
+** which are assumed zero.  Other than that, the two routines are the
+** same.
+**
+** Fixes or enhancements to this routine should be reflected in
+** insertCellFast()!
 */
 static int insertCell(
   MemPage *pPage,   /* Page into which we are copying */
@@ -75496,14 +75867,103 @@ static int insertCell(
   assert( sqlite3_mutex_held(pPage->pBt->mutex) );
   assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB );
   assert( pPage->nFree>=0 );
+  assert( iChild>0 );
   if( pPage->nOverflow || sz+2>pPage->nFree ){
     if( pTemp ){
       memcpy(pTemp, pCell, sz);
       pCell = pTemp;
     }
-    if( iChild ){
-      put4byte(pCell, iChild);
+    put4byte(pCell, iChild);
+    j = pPage->nOverflow++;
+    /* Comparison against ArraySize-1 since we hold back one extra slot
+    ** as a contingency.  In other words, never need more than 3 overflow
+    ** slots but 4 are allocated, just to be safe. */
+    assert( j < ArraySize(pPage->apOvfl)-1 );
+    pPage->apOvfl[j] = pCell;
+    pPage->aiOvfl[j] = (u16)i;
+
+    /* When multiple overflows occur, they are always sequential and in
+    ** sorted order.  This invariants arise because multiple overflows can
+    ** only occur when inserting divider cells into the parent page during
+    ** balancing, and the dividers are adjacent and sorted.
+    */
+    assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */
+    assert( j==0 || i==pPage->aiOvfl[j-1]+1 );   /* Overflows are sequential */
+  }else{
+    int rc = sqlite3PagerWrite(pPage->pDbPage);
+    if( NEVER(rc!=SQLITE_OK) ){
+      return rc;
+    }
+    assert( sqlite3PagerIswriteable(pPage->pDbPage) );
+    data = pPage->aData;
+    assert( &data[pPage->cellOffset]==pPage->aCellIdx );
+    rc = allocateSpace(pPage, sz, &idx);
+    if( rc ){ return rc; }
+    /* The allocateSpace() routine guarantees the following properties
+    ** if it returns successfully */
+    assert( idx >= 0 );
+    assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
+    assert( idx+sz <= (int)pPage->pBt->usableSize );
+    pPage->nFree -= (u16)(2 + sz);
+    /* In a corrupt database where an entry in the cell index section of
+    ** a btree page has a value of 3 or less, the pCell value might point
+    ** as many as 4 bytes in front of the start of the aData buffer for
+    ** the source page.  Make sure this does not cause problems by not
+    ** reading the first 4 bytes */
+    memcpy(&data[idx+4], pCell+4, sz-4);
+    put4byte(&data[idx], iChild);
+    pIns = pPage->aCellIdx + i*2;
+    memmove(pIns+2, pIns, 2*(pPage->nCell - i));
+    put2byte(pIns, idx);
+    pPage->nCell++;
+    /* increment the cell count */
+    if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++;
+    assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB );
+#ifndef SQLITE_OMIT_AUTOVACUUM
+    if( pPage->pBt->autoVacuum ){
+      int rc2 = SQLITE_OK;
+      /* The cell may contain a pointer to an overflow page. If so, write
+      ** the entry for the overflow page into the pointer map.
+      */
+      ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2);
+      if( rc2 ) return rc2;
     }
+#endif
+  }
+  return SQLITE_OK;
+}
+
+/*
+** This variant of insertCell() assumes that the pTemp and iChild
+** parameters are both zero.  Use this variant in sqlite3BtreeInsert()
+** for performance improvement, and also so that this variant is only
+** called from that one place, and is thus inlined, and thus runs must
+** faster.
+**
+** Fixes or enhancements to this routine should be reflected into
+** the insertCell() routine.
+*/
+static int insertCellFast(
+  MemPage *pPage,   /* Page into which we are copying */
+  int i,            /* New cell becomes the i-th cell of the page */
+  u8 *pCell,        /* Content of the new cell */
+  int sz            /* Bytes of content in pCell */
+){
+  int idx = 0;      /* Where to write new cell content in data[] */
+  int j;            /* Loop counter */
+  u8 *data;         /* The content of the whole page */
+  u8 *pIns;         /* The point in pPage->aCellIdx[] where no cell inserted */
+
+  assert( i>=0 && i<=pPage->nCell+pPage->nOverflow );
+  assert( MX_CELL(pPage->pBt)<=10921 );
+  assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB );
+  assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) );
+  assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) );
+  assert( sqlite3_mutex_held(pPage->pBt->mutex) );
+  assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB );
+  assert( pPage->nFree>=0 );
+  assert( pPage->nOverflow==0 );
+  if( sz+2>pPage->nFree ){
     j = pPage->nOverflow++;
     /* Comparison against ArraySize-1 since we hold back one extra slot
     ** as a contingency.  In other words, never need more than 3 overflow
@@ -75535,17 +75995,7 @@ static int insertCell(
     assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB );
     assert( idx+sz <= (int)pPage->pBt->usableSize );
     pPage->nFree -= (u16)(2 + sz);
-    if( iChild ){
-      /* In a corrupt database where an entry in the cell index section of
-      ** a btree page has a value of 3 or less, the pCell value might point
-      ** as many as 4 bytes in front of the start of the aData buffer for
-      ** the source page.  Make sure this does not cause problems by not
-      ** reading the first 4 bytes */
-      memcpy(&data[idx+4], pCell+4, sz-4);
-      put4byte(&data[idx], iChild);
-    }else{
-      memcpy(&data[idx], pCell, sz);
-    }
+    memcpy(&data[idx], pCell, sz);
     pIns = pPage->aCellIdx + i*2;
     memmove(pIns+2, pIns, 2*(pPage->nCell - i));
     put2byte(pIns, idx);
@@ -75730,7 +76180,7 @@ static int rebuildPage(
 
   assert( i<iEnd );
   j = get2byte(&aData[hdr+5]);
-  if( j>(u32)usableSize ){ j = 0; }
+  if( NEVER(j>(u32)usableSize) ){ j = 0; }
   memcpy(&pTmp[j], &aData[j], usableSize - j);
 
   for(k=0; pCArray->ixNx[k]<=i && ALWAYS(k<NB*2); k++){}
@@ -75874,42 +76324,50 @@ static int pageFreeArray(
   u8 * const pEnd = &aData[pPg->pBt->usableSize];
   u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize];
   int nRet = 0;
-  int i;
+  int i, j;
   int iEnd = iFirst + nCell;
-  u8 *pFree = 0;                  /* \__ Parameters for pending call to */
-  int szFree = 0;                 /* /   freeSpace()                    */
+  int nFree = 0;
+  int aOfst[10];
+  int aAfter[10];
 
   for(i=iFirst; i<iEnd; i++){
     u8 *pCell = pCArray->apCell[i];
     if( SQLITE_WITHIN(pCell, pStart, pEnd) ){
       int sz;
+      int iAfter;
+      int iOfst;
       /* No need to use cachedCellSize() here.  The sizes of all cells that
       ** are to be freed have already been computing while deciding which
       ** cells need freeing */
       sz = pCArray->szCell[i];  assert( sz>0 );
-      if( pFree!=(pCell + sz) ){
-        if( pFree ){
-          assert( pFree>aData && (pFree - aData)<65536 );
-          freeSpace(pPg, (u16)(pFree - aData), szFree);
-        }
-        pFree = pCell;
-        szFree = sz;
-        if( pFree+sz>pEnd ){
-          return 0;
+      iOfst = (u16)(pCell - aData);
+      iAfter = iOfst+sz;
+      for(j=0; j<nFree; j++){
+        if( aOfst[j]==iAfter ){
+          aOfst[j] = iOfst;
+          break;
+        }else if( aAfter[j]==iOfst ){
+          aAfter[j] = iAfter;
+          break;
         }
-      }else{
-        /* The current cell is adjacent to and before the pFree cell.
-        ** Combine the two regions into one to reduce the number of calls
-        ** to freeSpace(). */
-        pFree = pCell;
-        szFree += sz;
+      }
+      if( j>=nFree ){
+        if( nFree>=(int)(sizeof(aOfst)/sizeof(aOfst[0])) ){
+          for(j=0; j<nFree; j++){
+            freeSpace(pPg, aOfst[j], aAfter[j]-aOfst[j]);
+          }
+          nFree = 0;
+        }
+        aOfst[nFree] = iOfst;
+        aAfter[nFree] = iAfter;
+        if( &aData[iAfter]>pEnd ) return 0;
+        nFree++;
       }
       nRet++;
     }
   }
-  if( pFree ){
-    assert( pFree>aData && (pFree - aData)<65536 );
-    freeSpace(pPg, (u16)(pFree - aData), szFree);
+  for(j=0; j<nFree; j++){
+    freeSpace(pPg, aOfst[j], aAfter[j]-aOfst[j]);
   }
   return nRet;
 }
@@ -75962,9 +76420,9 @@ static int editPage(
     nCell -= nTail;
   }
 
-  pData = &aData[get2byteNotZero(&aData[hdr+5])];
+  pData = &aData[get2byte(&aData[hdr+5])];
   if( pData<pBegin ) goto editpage_fail;
-  if( pData>pPg->aDataEnd ) goto editpage_fail;
+  if( NEVER(pData>pPg->aDataEnd) ) goto editpage_fail;
 
   /* Add cells to the start of the page */
   if( iNew<iOld ){
@@ -76701,7 +77159,7 @@ static int balance_nonroot(
   **        that page.
   */
   assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB);
-  TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n",
+  TRACE(("BALANCE: old: %u(nc=%u) %u(nc=%u) %u(nc=%u)\n",
     apOld[0]->pgno, apOld[0]->nCell,
     nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0,
     nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0
@@ -76785,8 +77243,8 @@ static int balance_nonroot(
     }
   }
 
-  TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) "
-         "%d(%d nc=%d) %d(%d nc=%d)\n",
+  TRACE(("BALANCE: new: %u(%u nc=%u) %u(%u nc=%u) %u(%u nc=%u) "
+         "%u(%u nc=%u) %u(%u nc=%u)\n",
     apNew[0]->pgno, szNew[0], cntNew[0],
     nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0,
     nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0,
@@ -77031,7 +77489,7 @@ static int balance_nonroot(
   }
 
   assert( pParent->isInit );
-  TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n",
+  TRACE(("BALANCE: finished: old=%u new=%u cells=%u\n",
           nOld, nNew, b.nCell));
 
   /* Free any old pages that were not reused as new pages.
@@ -77116,7 +77574,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){
   assert( sqlite3PagerIswriteable(pRoot->pDbPage) );
   assert( pChild->nCell==pRoot->nCell || CORRUPT_DB );
 
-  TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno));
+  TRACE(("BALANCE: copy root %u into %u\n", pRoot->pgno, pChild->pgno));
 
   /* Copy the overflow cells from pRoot to pChild */
   memcpy(pChild->aiOvfl, pRoot->aiOvfl,
@@ -77599,7 +78057,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
     }
   }
   assert( pCur->eState==CURSOR_VALID
-       || (pCur->eState==CURSOR_INVALID && loc) );
+       || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB );
 
   pPage = pCur->pPage;
   assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) );
@@ -77614,7 +78072,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
     if( rc ) return rc;
   }
 
-  TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n",
+  TRACE(("INSERT: table=%u nkey=%lld ndata=%u page=%u %s\n",
           pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno,
           loc==0 ? "overwrite" : "new entry"));
   assert( pPage->isInit || CORRUPT_DB );
@@ -77690,7 +78148,7 @@ SQLITE_PRIVATE int sqlite3BtreeInsert(
   }else{
     assert( pPage->leaf );
   }
-  rc = insertCell(pPage, idx, newCell, szNew, 0, 0);
+  rc = insertCellFast(pPage, idx, newCell, szNew);
   assert( pPage->nOverflow==0 || rc==SQLITE_OK );
   assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 );
 
@@ -77914,6 +78372,9 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){
   if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){
     return SQLITE_CORRUPT_BKPT;
   }
+  if( pCell<&pPage->aCellIdx[pPage->nCell] ){
+    return SQLITE_CORRUPT_BKPT;
+  }
 
   /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must
   ** be preserved following this delete operation. If the current delete
@@ -78662,7 +79123,8 @@ static void checkAppendMsg(
     sqlite3_str_append(&pCheck->errMsg, "\n", 1);
   }
   if( pCheck->zPfx ){
-    sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2);
+    sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx,
+                        pCheck->v0, pCheck->v1, pCheck->v2);
   }
   sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap);
   va_end(ap);
@@ -78702,11 +79164,11 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){
 */
 static int checkRef(IntegrityCk *pCheck, Pgno iPage){
   if( iPage>pCheck->nPage || iPage==0 ){
-    checkAppendMsg(pCheck, "invalid page number %d", iPage);
+    checkAppendMsg(pCheck, "invalid page number %u", iPage);
     return 1;
   }
   if( getPageReferenced(pCheck, iPage) ){
-    checkAppendMsg(pCheck, "2nd reference to page %d", iPage);
+    checkAppendMsg(pCheck, "2nd reference to page %u", iPage);
     return 1;
   }
   setPageReferenced(pCheck, iPage);
@@ -78732,13 +79194,13 @@ static void checkPtrmap(
   rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent);
   if( rc!=SQLITE_OK ){
     if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck);
-    checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild);
+    checkAppendMsg(pCheck, "Failed to read ptrmap key=%u", iChild);
     return;
   }
 
   if( ePtrmapType!=eType || iPtrmapParent!=iParent ){
     checkAppendMsg(pCheck,
-      "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)",
+      "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)",
       iChild, eType, iParent, ePtrmapType, iPtrmapParent);
   }
 }
@@ -78763,7 +79225,7 @@ static void checkList(
     if( checkRef(pCheck, iPage) ) break;
     N--;
     if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){
-      checkAppendMsg(pCheck, "failed to get page %d", iPage);
+      checkAppendMsg(pCheck, "failed to get page %u", iPage);
       break;
     }
     pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage);
@@ -78776,7 +79238,7 @@ static void checkList(
 #endif
       if( n>pCheck->pBt->usableSize/4-2 ){
         checkAppendMsg(pCheck,
-           "freelist leaf count too big on page %d", iPage);
+           "freelist leaf count too big on page %u", iPage);
         N--;
       }else{
         for(i=0; i<(int)n; i++){
@@ -78808,7 +79270,7 @@ static void checkList(
   }
   if( N && nErrAtStart==pCheck->nErr ){
     checkAppendMsg(pCheck,
-      "%s is %d but should be %d",
+      "%s is %u but should be %u",
       isFreeList ? "size" : "overflow list length",
       expected-N, expected);
   }
@@ -78923,8 +79385,8 @@ static int checkTreePage(
   usableSize = pBt->usableSize;
   if( iPage==0 ) return 0;
   if( checkRef(pCheck, iPage) ) return 0;
-  pCheck->zPfx = "Page %u: ";
-  pCheck->v1 = iPage;
+  pCheck->zPfx = "Tree %u page %u: ";
+  pCheck->v0 = pCheck->v1 = iPage;
   if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){
     checkAppendMsg(pCheck,
        "unable to get the page. error code=%d", rc);
@@ -78950,7 +79412,7 @@ static int checkTreePage(
   hdr = pPage->hdrOffset;
 
   /* Set up for cell analysis */
-  pCheck->zPfx = "On tree page %u cell %d: ";
+  pCheck->zPfx = "Tree %u page %u cell %u: ";
   contentOffset = get2byteNotZero(&data[hdr+5]);
   assert( contentOffset<=usableSize );  /* Enforced by btreeInitPage() */
 
@@ -78970,7 +79432,7 @@ static int checkTreePage(
     pgno = get4byte(&data[hdr+8]);
 #ifndef SQLITE_OMIT_AUTOVACUUM
     if( pBt->autoVacuum ){
-      pCheck->zPfx = "On page %u at right child: ";
+      pCheck->zPfx = "Tree %u page %u right child: ";
       checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage);
     }
 #endif
@@ -78994,7 +79456,7 @@ static int checkTreePage(
     pc = get2byteAligned(pCellIdx);
     pCellIdx -= 2;
     if( pc<contentOffset || pc>usableSize-4 ){
-      checkAppendMsg(pCheck, "Offset %d out of range %d..%d",
+      checkAppendMsg(pCheck, "Offset %u out of range %u..%u",
                              pc, contentOffset, usableSize-4);
       doCoverageCheck = 0;
       continue;
@@ -79126,7 +79588,7 @@ static int checkTreePage(
     */
     if( heap[0]==0 && nFrag!=data[hdr+7] ){
       checkAppendMsg(pCheck,
-          "Fragmentation of %d bytes reported as %d on page %u",
+          "Fragmentation of %u bytes reported as %u on page %u",
           nFrag, data[hdr+7], iPage);
     }
   }
@@ -79223,7 +79685,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck(
   /* Check the integrity of the freelist
   */
   if( bCkFreelist ){
-    sCheck.zPfx = "Main freelist: ";
+    sCheck.zPfx = "Freelist: ";
     checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]),
               get4byte(&pBt->pPage1->aData[36]));
     sCheck.zPfx = 0;
@@ -79240,7 +79702,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck(
       mxInHdr = get4byte(&pBt->pPage1->aData[52]);
       if( mx!=mxInHdr ){
         checkAppendMsg(&sCheck,
-          "max rootpage (%d) disagrees with header (%d)",
+          "max rootpage (%u) disagrees with header (%u)",
           mx, mxInHdr
         );
       }
@@ -79271,7 +79733,7 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck(
     for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){
 #ifdef SQLITE_OMIT_AUTOVACUUM
       if( getPageReferenced(&sCheck, i)==0 ){
-        checkAppendMsg(&sCheck, "Page %d is never used", i);
+        checkAppendMsg(&sCheck, "Page %u: never used", i);
       }
 #else
       /* If the database supports auto-vacuum, make sure no tables contain
@@ -79279,11 +79741,11 @@ SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck(
       */
       if( getPageReferenced(&sCheck, i)==0 &&
          (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){
-        checkAppendMsg(&sCheck, "Page %d is never used", i);
+        checkAppendMsg(&sCheck, "Page %u: never used", i);
       }
       if( getPageReferenced(&sCheck, i)!=0 &&
          (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){
-        checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i);
+        checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i);
       }
 #endif
     }
@@ -79845,13 +80307,7 @@ static int backupOnePage(
   assert( !isFatalError(p->rc) );
   assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) );
   assert( zSrcData );
-
-  /* Catch the case where the destination is an in-memory database and the
-  ** page sizes of the source and destination differ.
-  */
-  if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){
-    rc = SQLITE_READONLY;
-  }
+  assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 );
 
   /* This loop runs once for each destination page spanned by the source
   ** page. For each iteration, variable iOff is set to the byte offset
@@ -79984,7 +80440,10 @@ SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){
     pgszSrc = sqlite3BtreeGetPageSize(p->pSrc);
     pgszDest = sqlite3BtreeGetPageSize(p->pDest);
     destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest));
-    if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){
+    if( SQLITE_OK==rc
+     && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager))
+     && pgszSrc!=pgszDest
+    ){
       rc = SQLITE_READONLY;
     }
 
@@ -80533,6 +80992,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){
   char *z;
   int i, j, incr;
   if( (p->flags & MEM_Str)==0 ) return 1;
+  if( p->db && p->db->mallocFailed ) return 1;
   if( p->flags & MEM_Term ){
     /* Insure that the string is properly zero-terminated.  Pay particular
     ** attention to the case where p->n is odd */
@@ -80815,7 +81275,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){
 
   vdbeMemRenderNum(nByte, pMem->z, pMem);
   assert( pMem->z!=0 );
-  assert( pMem->n==sqlite3Strlen30NN(pMem->z) );
+  assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) );
   pMem->enc = SQLITE_UTF8;
   pMem->flags |= MEM_Str|MEM_Term;
   if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal);
@@ -81859,6 +82319,9 @@ static int valueFromFunction(
   if( pList ) nVal = pList->nExpr;
   assert( !ExprHasProperty(p, EP_IntValue) );
   pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0);
+#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
+  if( pFunc==0 ) return SQLITE_OK;
+#endif
   assert( pFunc );
   if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0
    || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)
@@ -81895,16 +82358,11 @@ static int valueFromFunction(
   }else{
     sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8);
     assert( rc==SQLITE_OK );
-    assert( enc==pVal->enc
-         || (pVal->flags & MEM_Str)==0
-         || db->mallocFailed  );
-#if 0  /* Not reachable except after a prior failure */
     rc = sqlite3VdbeChangeEncoding(pVal, enc);
-    if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){
+    if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){
       rc = SQLITE_TOOBIG;
       pCtx->pParse->nErr++;
     }
-#endif
   }
 
  value_from_function_out:
@@ -81968,6 +82426,13 @@ static int valueFromExpr(
     rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx);
     testcase( rc!=SQLITE_OK );
     if( *ppVal ){
+#ifdef SQLITE_ENABLE_STAT4
+      rc = ExpandBlob(*ppVal);
+#else
+      /* zero-blobs only come from functions, not literal values.  And
+      ** functions are only processed under STAT4 */
+      assert( (ppVal[0][0].flags & MEM_Zero)==0 );
+#endif
       sqlite3VdbeMemCast(*ppVal, aff, enc);
       sqlite3ValueApplyAffinity(*ppVal, affinity, enc);
     }
@@ -82814,10 +83279,10 @@ SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){
 */
 SQLITE_PRIVATE int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){
   int addr = 0;
-#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+#if !defined(SQLITE_DEBUG)
   /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined.
   ** But omit them (for performance) during production builds */
-  if( pParse->explain==2 )
+  if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) )
 #endif
   {
     char *zMsg;
@@ -83191,6 +83656,8 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
   Op *pOp;
   Parse *pParse = p->pParse;
   int *aLabel = pParse->aLabel;
+
+  assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */
   p->readOnly = 1;
   p->bIsReader = 0;
   pOp = &p->aOp[p->nOp-1];
@@ -83250,6 +83717,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){
             ** have non-negative values for P2. */
             assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 );
             assert( ADDR(pOp->p2)<-pParse->nLabel );
+            assert( aLabel!=0 );  /* True because of tag-20230419-1 */
             pOp->p2 = aLabel[ADDR(pOp->p2)];
           }
           break;
@@ -83493,18 +83961,20 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(
   LogEst nEst,                    /* Estimated number of output rows */
   const char *zName               /* Name of table or index being scanned */
 ){
-  sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus);
-  ScanStatus *aNew;
-  aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte);
-  if( aNew ){
-    ScanStatus *pNew = &aNew[p->nScan++];
-    memset(pNew, 0, sizeof(ScanStatus));
-    pNew->addrExplain = addrExplain;
-    pNew->addrLoop = addrLoop;
-    pNew->addrVisit = addrVisit;
-    pNew->nEst = nEst;
-    pNew->zName = sqlite3DbStrDup(p->db, zName);
-    p->aScan = aNew;
+  if( IS_STMT_SCANSTATUS(p->db) ){
+    sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus);
+    ScanStatus *aNew;
+    aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte);
+    if( aNew ){
+      ScanStatus *pNew = &aNew[p->nScan++];
+      memset(pNew, 0, sizeof(ScanStatus));
+      pNew->addrExplain = addrExplain;
+      pNew->addrLoop = addrLoop;
+      pNew->addrVisit = addrVisit;
+      pNew->nEst = nEst;
+      pNew->zName = sqlite3DbStrDup(p->db, zName);
+      p->aScan = aNew;
+    }
   }
 }
 
@@ -83521,20 +83991,22 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusRange(
   int addrStart,
   int addrEnd
 ){
-  ScanStatus *pScan = 0;
-  int ii;
-  for(ii=p->nScan-1; ii>=0; ii--){
-    pScan = &p->aScan[ii];
-    if( pScan->addrExplain==addrExplain ) break;
-    pScan = 0;
-  }
-  if( pScan ){
-    if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1;
-    for(ii=0; ii<ArraySize(pScan->aAddrRange); ii+=2){
-      if( pScan->aAddrRange[ii]==0 ){
-        pScan->aAddrRange[ii] = addrStart;
-        pScan->aAddrRange[ii+1] = addrEnd;
-        break;
+  if( IS_STMT_SCANSTATUS(p->db) ){
+    ScanStatus *pScan = 0;
+    int ii;
+    for(ii=p->nScan-1; ii>=0; ii--){
+      pScan = &p->aScan[ii];
+      if( pScan->addrExplain==addrExplain ) break;
+      pScan = 0;
+    }
+    if( pScan ){
+      if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1;
+      for(ii=0; ii<ArraySize(pScan->aAddrRange); ii+=2){
+        if( pScan->aAddrRange[ii]==0 ){
+          pScan->aAddrRange[ii] = addrStart;
+          pScan->aAddrRange[ii+1] = addrEnd;
+          break;
+        }
       }
     }
   }
@@ -83551,19 +84023,21 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters(
   int addrLoop,
   int addrVisit
 ){
-  ScanStatus *pScan = 0;
-  int ii;
-  for(ii=p->nScan-1; ii>=0; ii--){
-    pScan = &p->aScan[ii];
-    if( pScan->addrExplain==addrExplain ) break;
-    pScan = 0;
-  }
-  if( pScan ){
-    pScan->addrLoop = addrLoop;
-    pScan->addrVisit = addrVisit;
+  if( IS_STMT_SCANSTATUS(p->db) ){
+    ScanStatus *pScan = 0;
+    int ii;
+    for(ii=p->nScan-1; ii>=0; ii--){
+      pScan = &p->aScan[ii];
+      if( pScan->addrExplain==addrExplain ) break;
+      pScan = 0;
+    }
+    if( pScan ){
+      pScan->addrLoop = addrLoop;
+      pScan->addrVisit = addrVisit;
+    }
   }
 }
-#endif
+#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */
 
 
 /*
@@ -83987,7 +84461,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){
 
 /* Return the most recently added opcode
 */
-VdbeOp * sqlite3VdbeGetLastOp(Vdbe *p){
+SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){
   return sqlite3VdbeGetOp(p, p->nOp - 1);
 }
 
@@ -85691,6 +86165,8 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){
           db->flags &= ~(u64)SQLITE_DeferFKs;
           sqlite3CommitInternalChanges(db);
         }
+      }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){
+        p->nChange = 0;
       }else{
         sqlite3RollbackAll(db, SQLITE_OK);
         p->nChange = 0;
@@ -86009,9 +86485,9 @@ static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){
 #ifdef SQLITE_ENABLE_NORMALIZE
   sqlite3DbFree(db, p->zNormSql);
   {
-    DblquoteStr *pThis, *pNext;
-    for(pThis=p->pDblStr; pThis; pThis=pNext){
-      pNext = pThis->pNextStr;
+    DblquoteStr *pThis, *pNxt;
+    for(pThis=p->pDblStr; pThis; pThis=pNxt){
+      pNxt = pThis->pNextStr;
       sqlite3DbFree(db, pThis);
     }
   }
@@ -87638,6 +88114,20 @@ SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){
   return 1;
 }
 
+#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG)
+/*
+** This Walker callback is used to help verify that calls to
+** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have
+** byte-code register values correctly initialized.
+*/
+SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){
+  if( pExpr->op==TK_REGISTER ){
+    assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 );
+  }
+  return WRC_Continue;
+}
+#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */
+
 #ifndef SQLITE_OMIT_VIRTUALTABLE
 /*
 ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored
@@ -87700,6 +88190,16 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(
   PreUpdate preupdate;
   const char *zTbl = pTab->zName;
   static const u8 fakeSortOrder = 0;
+#ifdef SQLITE_DEBUG
+  int nRealCol;
+  if( pTab->tabFlags & TF_WithoutRowid ){
+    nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn;
+  }else if( pTab->tabFlags & TF_HasVirtual ){
+    nRealCol = pTab->nNVCol;
+  }else{
+    nRealCol = pTab->nCol;
+  }
+#endif
 
   assert( db->pPreUpdate==0 );
   memset(&preupdate, 0, sizeof(PreUpdate));
@@ -87716,8 +88216,8 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(
 
   assert( pCsr!=0 );
   assert( pCsr->eCurType==CURTYPE_BTREE );
-  assert( pCsr->nField==pTab->nCol
-       || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1)
+  assert( pCsr->nField==nRealCol
+       || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1)
   );
 
   preupdate.v = v;
@@ -88024,7 +88524,7 @@ SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){
      SQLITE_NULL,     /* 0x1f (not possible) */
      SQLITE_FLOAT,    /* 0x20 INTREAL */
      SQLITE_NULL,     /* 0x21 (not possible) */
-     SQLITE_TEXT,     /* 0x22 INTREAL + TEXT */
+     SQLITE_FLOAT,    /* 0x22 INTREAL + TEXT */
      SQLITE_NULL,     /* 0x23 (not possible) */
      SQLITE_FLOAT,    /* 0x24 (not possible) */
      SQLITE_NULL,     /* 0x25 (not possible) */
@@ -89090,9 +89590,9 @@ static const void *columnName(
   assert( db!=0 );
   n = sqlite3_column_count(pStmt);
   if( N<n && N>=0 ){
+    u8 prior_mallocFailed = db->mallocFailed;
     N += useType*n;
     sqlite3_mutex_enter(db->mutex);
-    assert( db->mallocFailed==0 );
 #ifndef SQLITE_OMIT_UTF16
     if( useUtf16 ){
       ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]);
@@ -89104,7 +89604,8 @@ static const void *columnName(
     /* A malloc may have failed inside of the _text() call. If this
     ** is the case, clear the mallocFailed flag and return NULL.
     */
-    if( db->mallocFailed ){
+    assert( db->mallocFailed==0 || db->mallocFailed==1 );
+    if( db->mallocFailed > prior_mallocFailed ){
       sqlite3OomClear(db);
       ret = 0;
     }
@@ -89891,15 +90392,24 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
   void *pOut                      /* OUT: Write the answer here */
 ){
   Vdbe *p = (Vdbe*)pStmt;
-  ScanStatus *pScan;
+  VdbeOp *aOp = p->aOp;
+  int nOp = p->nOp;
+  ScanStatus *pScan = 0;
   int idx;
 
+  if( p->pFrame ){
+    VdbeFrame *pFrame;
+    for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent);
+    aOp = pFrame->aOp;
+    nOp = pFrame->nOp;
+  }
+
   if( iScan<0 ){
     int ii;
     if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){
       i64 res = 0;
-      for(ii=0; ii<p->nOp; ii++){
-        res += p->aOp[ii].nCycle;
+      for(ii=0; ii<nOp; ii++){
+        res += aOp[ii].nCycle;
       }
       *(i64*)pOut = res;
       return 0;
@@ -89925,7 +90435,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
   switch( iScanStatusOp ){
     case SQLITE_SCANSTAT_NLOOP: {
       if( pScan->addrLoop>0 ){
-        *(sqlite3_int64*)pOut = p->aOp[pScan->addrLoop].nExec;
+        *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec;
       }else{
         *(sqlite3_int64*)pOut = -1;
       }
@@ -89933,7 +90443,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
     }
     case SQLITE_SCANSTAT_NVISIT: {
       if( pScan->addrVisit>0 ){
-        *(sqlite3_int64*)pOut = p->aOp[pScan->addrVisit].nExec;
+        *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec;
       }else{
         *(sqlite3_int64*)pOut = -1;
       }
@@ -89955,7 +90465,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
     }
     case SQLITE_SCANSTAT_EXPLAIN: {
       if( pScan->addrExplain ){
-        *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z;
+        *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z;
       }else{
         *(const char**)pOut = 0;
       }
@@ -89963,7 +90473,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
     }
     case SQLITE_SCANSTAT_SELECTID: {
       if( pScan->addrExplain ){
-        *(int*)pOut = p->aOp[ pScan->addrExplain ].p1;
+        *(int*)pOut = aOp[ pScan->addrExplain ].p1;
       }else{
         *(int*)pOut = -1;
       }
@@ -89971,7 +90481,7 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
     }
     case SQLITE_SCANSTAT_PARENTID: {
       if( pScan->addrExplain ){
-        *(int*)pOut = p->aOp[ pScan->addrExplain ].p2;
+        *(int*)pOut = aOp[ pScan->addrExplain ].p2;
       }else{
         *(int*)pOut = -1;
       }
@@ -89989,18 +90499,18 @@ SQLITE_API int sqlite3_stmt_scanstatus_v2(
           if( iIns==0 ) break;
           if( iIns>0 ){
             while( iIns<=iEnd ){
-              res += p->aOp[iIns].nCycle;
+              res += aOp[iIns].nCycle;
               iIns++;
             }
           }else{
             int iOp;
-            for(iOp=0; iOp<p->nOp; iOp++){
-              Op *pOp = &p->aOp[iOp];
+            for(iOp=0; iOp<nOp; iOp++){
+              Op *pOp = &aOp[iOp];
               if( pOp->p1!=iEnd ) continue;
               if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){
                 continue;
               }
-              res += p->aOp[iOp].nCycle;
+              res += aOp[iOp].nCycle;
             }
           }
         }
@@ -90923,7 +91433,10 @@ static u64 filterHash(const Mem *aMem, const Op *pOp){
     }else if( p->flags & MEM_Real ){
       h += sqlite3VdbeIntValue(p);
     }else if( p->flags & (MEM_Str|MEM_Blob) ){
-      /* no-op */
+      /* All strings have the same hash and all blobs have the same hash,
+      ** though, at least, those hashes are different from each other and
+      ** from NULL. */
+      h += 4093 + (p->flags & (MEM_Str|MEM_Blob));
     }
   }
   return h;
@@ -90973,6 +91486,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec(
   Mem *pOut = 0;             /* Output operand */
 #if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
   u64 *pnCycle = 0;
+  int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0;
 #endif
   /*** INSERT STACK UNION HERE ***/
 
@@ -91037,13 +91551,17 @@ SQLITE_PRIVATE int sqlite3VdbeExec(
 
     assert( pOp>=aOp && pOp<&aOp[p->nOp]);
     nVmStep++;
-#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE)
+
+#if defined(VDBE_PROFILE)
     pOp->nExec++;
     pnCycle = &pOp->nCycle;
-# ifdef VDBE_PROFILE
-    if( sqlite3NProfileCnt==0 )
-# endif
+    if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime();
+#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+    if( bStmtScanStatus ){
+      pOp->nExec++;
+      pnCycle = &pOp->nCycle;
       *pnCycle -= sqlite3Hwtime();
+    }
 #endif
 
     /* Only allow tracing if SQLITE_DEBUG is defined.
@@ -92631,7 +93149,7 @@ case OP_Compare: {
 /* Opcode: Jump P1 P2 P3 * *
 **
 ** Jump to the instruction at address P1, P2, or P3 depending on whether
-** in the most recent OP_Compare instruction the P1 vector was less than
+** in the most recent OP_Compare instruction the P1 vector was less than,
 ** equal to, or greater than the P2 vector, respectively.
 **
 ** This opcode must immediately follow an OP_Compare opcode.
@@ -92858,6 +93376,12 @@ case OP_IsNull: {            /* same as TK_ISNULL, jump, in1 */
 ** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04.
 ** SQLITE_BLOB is 0x08.  SQLITE_NULL is 0x10.
 **
+** WARNING: This opcode does not reliably distinguish between NULL and REAL
+** when P1>=0.  If the database contains a NaN value, this opcode will think
+** that the datatype is REAL when it should be NULL.  When P1<0 and the value
+** is already stored in register P3, then this opcode does reliably
+** distinguish between NULL and REAL.  The problem only arises then P1>=0.
+**
 ** Take the jump to address P2 if and only if the datatype of the
 ** value determined by P1 and P3 corresponds to one of the bits in the
 ** P5 bitmask.
@@ -92971,7 +93495,7 @@ case OP_IfNullRow: {         /* jump */
   VdbeCursor *pC;
   assert( pOp->p1>=0 && pOp->p1<p->nCursor );
   pC = p->apCsr[pOp->p1];
-  if( ALWAYS(pC) && pC->nullRow ){
+  if( pC && pC->nullRow ){
     sqlite3VdbeMemSetNull(aMem + pOp->p3);
     goto jump_to_p2;
   }
@@ -93466,7 +93990,7 @@ case OP_Affinity: {
       }else{
         pIn1->u.r = (double)pIn1->u.i;
         pIn1->flags |= MEM_Real;
-        pIn1->flags &= ~MEM_Int;
+        pIn1->flags &= ~(MEM_Int|MEM_Str);
       }
     }
     REGISTER_TRACE((int)(pIn1-aMem), pIn1);
@@ -95205,6 +95729,7 @@ case OP_SeekScan: {          /* ncycle */
       break;
     }
     nStep--;
+    pC->cacheStatus = CACHE_STALE;
     rc = sqlite3BtreeNext(pC->uc.pCursor, 0);
     if( rc ){
       if( rc==SQLITE_DONE ){
@@ -97857,6 +98382,7 @@ case OP_AggFinal: {
   }
   sqlite3VdbeChangeEncoding(pMem, encoding);
   UPDATE_MAX_BLOBSIZE(pMem);
+  REGISTER_TRACE((int)(pMem-aMem), pMem);
   break;
 }
 
@@ -98995,8 +99521,10 @@ default: {          /* This is really OP_Noop, OP_Explain */
     *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime();
     pnCycle = 0;
 #elif defined(SQLITE_ENABLE_STMT_SCANSTATUS)
-    *pnCycle += sqlite3Hwtime();
-    pnCycle = 0;
+    if( pnCycle ){
+      *pnCycle += sqlite3Hwtime();
+      pnCycle = 0;
+    }
 #endif
 
     /* The following code adds nothing to the actual functionality
@@ -99475,7 +100003,7 @@ blob_open_out:
     if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt);
     sqlite3DbFree(db, pBlob);
   }
-  sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr);
+  sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr);
   sqlite3DbFree(db, zErr);
   sqlite3ParseObjectReset(&sParse);
   rc = sqlite3ApiExit(db, rc);
@@ -99634,7 +100162,7 @@ SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){
     ((Vdbe*)p->pStmt)->rc = SQLITE_OK;
     rc = blobSeekToRow(p, iRow, &zErr);
     if( rc!=SQLITE_OK ){
-      sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr);
+      sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr);
       sqlite3DbFree(db, zErr);
     }
     assert( rc!=SQLITE_SCHEMA );
@@ -104022,7 +104550,8 @@ static int lookupName(
         assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT );
         if( pParse->bReturning ){
           if( (pNC->ncFlags & NC_UBaseReg)!=0
-           && (zTab==0 || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0)
+           && ALWAYS(zTab==0
+                     || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0)
           ){
             pExpr->iTable = op!=TK_DELETE;
             pTab = pParse->pTriggerTab;
@@ -105996,11 +106525,10 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){
       }else{
         Expr *pNext  = p->pRight;
         /* The Expr.x union is never used at the same time as Expr.pRight */
-        assert( ExprUseXList(p) );
-        assert( p->x.pList==0 || p->pRight==0 );
-        if( p->x.pList!=0 && !db->mallocFailed ){
+        assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 );
+        if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){
           int i;
-          for(i=0; ALWAYS(i<p->x.pList->nExpr); i++){
+          for(i=0; i<p->x.pList->nExpr; i++){
             if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){
               pNext = p->x.pList->a[i].pExpr;
               break;
@@ -106832,9 +107360,9 @@ SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprLis
 ** Join two expressions using an AND operator.  If either expression is
 ** NULL, then just return the other expression.
 **
-** If one side or the other of the AND is known to be false, then instead
-** of returning an AND expression, just return a constant expression with
-** a value of false.
+** If one side or the other of the AND is known to be false, and neither side
+** is part of an ON clause, then instead of returning an AND expression,
+** just return a constant expression with a value of false.
 */
 SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){
   sqlite3 *db = pParse->db;
@@ -106842,14 +107370,17 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){
     return pRight;
   }else if( pRight==0 ){
     return pLeft;
-  }else if( (ExprAlwaysFalse(pLeft) || ExprAlwaysFalse(pRight))
-         && !IN_RENAME_OBJECT
-  ){
-    sqlite3ExprDeferredDelete(pParse, pLeft);
-    sqlite3ExprDeferredDelete(pParse, pRight);
-    return sqlite3Expr(db, TK_INTEGER, "0");
   }else{
-    return sqlite3PExpr(pParse, TK_AND, pLeft, pRight);
+    u32 f = pLeft->flags | pRight->flags;
+    if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse
+     && !IN_RENAME_OBJECT
+    ){
+      sqlite3ExprDeferredDelete(pParse, pLeft);
+      sqlite3ExprDeferredDelete(pParse, pRight);
+      return sqlite3Expr(db, TK_INTEGER, "0");
+    }else{
+      return sqlite3PExpr(pParse, TK_AND, pLeft, pRight);
+    }
   }
 }
 
@@ -108094,12 +108625,17 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){
 }
 
 /*
-** Check pExpr to see if it is an invariant constraint on data source pSrc.
+** Check pExpr to see if it is an constraint on the single data source
+** pSrc = &pSrcList->a[iSrc].  In other words, check to see if pExpr
+** constrains pSrc but does not depend on any other tables or data
+** sources anywhere else in the query.  Return true (non-zero) if pExpr
+** is a constraint on pSrc only.
+**
 ** This is an optimization.  False negatives will perhaps cause slower
 ** queries, but false positives will yield incorrect answers.  So when in
 ** doubt, return 0.
 **
-** To be an invariant constraint, the following must be true:
+** To be an single-source constraint, the following must be true:
 **
 **   (1)  pExpr cannot refer to any table other than pSrc->iCursor.
 **
@@ -108110,13 +108646,31 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){
 **
 **   (4)  If pSrc is the right operand of a LEFT JOIN, then...
 **         (4a)  pExpr must come from an ON clause..
-           (4b)  and specifically the ON clause associated with the LEFT JOIN.
+**         (4b)  and specifically the ON clause associated with the LEFT JOIN.
 **
 **   (5)  If pSrc is not the right operand of a LEFT JOIN or the left
 **        operand of a RIGHT JOIN, then pExpr must be from the WHERE
 **        clause, not an ON clause.
+**
+**   (6) Either:
+**
+**       (6a) pExpr does not originate in an ON or USING clause, or
+**
+**       (6b) The ON or USING clause from which pExpr is derived is
+**            not to the left of a RIGHT JOIN (or FULL JOIN).
+**
+**       Without this restriction, accepting pExpr as a single-table
+**       constraint might move the the ON/USING filter expression
+**       from the left side of a RIGHT JOIN over to the right side,
+**       which leads to incorrect answers.  See also restriction (9)
+**       on push-down.
 */
-SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc){
+SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(
+  Expr *pExpr,                 /* The constraint */
+  const SrcList *pSrcList,     /* Complete FROM clause */
+  int iSrc                     /* Which element of pSrcList to use */
+){
+  const SrcItem *pSrc = &pSrcList->a[iSrc];
   if( pSrc->fg.jointype & JT_LTORJ ){
     return 0;  /* rule (3) */
   }
@@ -108126,6 +108680,19 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstraint(Expr *pExpr, const SrcItem *pSrc
   }else{
     if( ExprHasProperty(pExpr, EP_OuterON) ) return 0;    /* rule (5) */
   }
+  if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON)  /* (6a) */
+   && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0     /* Fast pre-test of (6b) */
+  ){
+    int jj;
+    for(jj=0; jj<iSrc; jj++){
+      if( pExpr->w.iJoin==pSrcList->a[jj].iCursor ){
+        if( (pSrcList->a[jj].fg.jointype & JT_LTORJ)!=0 ){
+          return 0;  /* restriction (6) */
+        }
+        break;
+      }
+    }
+  }
   return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor); /* rules (1), (2) */
 }
 
@@ -108368,7 +108935,7 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){
 ** pX is the RHS of an IN operator.  If pX is a SELECT statement
 ** that can be simplified to a direct table access, then return
 ** a pointer to the SELECT statement.  If pX is not a SELECT statement,
-** or if the SELECT statement needs to be manifested into a transient
+** or if the SELECT statement needs to be materialized into a transient
 ** table, then return NULL.
 */
 #ifndef SQLITE_OMIT_SUBQUERY
@@ -108654,7 +109221,6 @@ SQLITE_PRIVATE int sqlite3FindInIndex(
             CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs);
             int j;
 
-            assert( pReq!=0 || pRhs->iColumn==XN_ROWID || pParse->nErr );
             for(j=0; j<nExpr; j++){
               if( pIdx->aiColumn[j]!=pRhs->iColumn ) continue;
               assert( pIdx->azColl[j] );
@@ -109940,7 +110506,19 @@ expr_code_doover:
       AggInfo *pAggInfo = pExpr->pAggInfo;
       struct AggInfo_col *pCol;
       assert( pAggInfo!=0 );
-      assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn );
+      assert( pExpr->iAgg>=0 );
+      if( pExpr->iAgg>=pAggInfo->nColumn ){
+        /* Happens when the left table of a RIGHT JOIN is null and
+        ** is using an expression index */
+        sqlite3VdbeAddOp2(v, OP_Null, 0, target);
+#ifdef SQLITE_VDBE_COVERAGE
+        /* Verify that the OP_Null above is exercised by tests
+        ** tag-20230325-2 */
+        sqlite3VdbeAddOp2(v, OP_NotNull, target, 1);
+        VdbeCoverageNeverTaken(v);
+#endif
+        break;
+      }
       pCol = &pAggInfo->aCol[pExpr->iAgg];
       if( !pAggInfo->directMode ){
         return AggInfoColumnReg(pAggInfo, pExpr->iAgg);
@@ -110115,11 +110693,8 @@ expr_code_doover:
 #ifndef SQLITE_OMIT_CAST
     case TK_CAST: {
       /* Expressions of the form:   CAST(pLeft AS token) */
-      inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
-      if( inReg!=target ){
-        sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target);
-        inReg = target;
-      }
+      sqlite3ExprCode(pParse, pExpr->pLeft, target);
+      assert( inReg==target );
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
       sqlite3VdbeAddOp2(v, OP_Cast, target,
                         sqlite3AffinityType(pExpr->u.zToken, 0));
@@ -110458,13 +111033,9 @@ expr_code_doover:
         ** Clear subtypes as subtypes may not cross a subquery boundary.
         */
         assert( pExpr->pLeft );
-        inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
-        if( inReg!=target ){
-          sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target);
-          inReg = target;
-        }
-        sqlite3VdbeAddOp1(v, OP_ClrSubtype, inReg);
-        return inReg;
+        sqlite3ExprCode(pParse, pExpr->pLeft, target);
+        sqlite3VdbeAddOp1(v, OP_ClrSubtype, target);
+        return target;
       }else{
         pExpr = pExpr->pLeft;
         goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */
@@ -110574,12 +111145,9 @@ expr_code_doover:
       **        "target" and not someplace else.
       */
       pParse->okConstFactor = 0;   /* note (1) above */
-      inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target);
+      sqlite3ExprCode(pParse, pExpr->pLeft, target);
+      assert( target==inReg );
       pParse->okConstFactor = okConstFactor;
-      if( inReg!=target ){         /* note (2) above */
-        sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target);
-        inReg = target;
-      }
       sqlite3VdbeJumpHere(v, addrINR);
       break;
     }
@@ -110817,7 +111385,9 @@ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){
   inReg = sqlite3ExprCodeTarget(pParse, pExpr, target);
   if( inReg!=target ){
     u8 op;
-    if( ALWAYS(pExpr) && ExprHasProperty(pExpr,EP_Subquery) ){
+    if( ALWAYS(pExpr)
+     && (ExprHasProperty(pExpr,EP_Subquery) || pExpr->op==TK_REGISTER)
+    ){
       op = OP_Copy;
     }else{
       op = OP_SCopy;
@@ -112002,9 +112572,11 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){
     int iAgg = pExpr->iAgg;
     Parse *pParse = pWalker->pParse;
     sqlite3 *db = pParse->db;
+    assert( iAgg>=0 );
     if( pExpr->op!=TK_AGG_FUNCTION ){
-      assert( iAgg>=0 && iAgg<pAggInfo->nColumn );
-      if( pAggInfo->aCol[iAgg].pCExpr==pExpr ){
+      if( iAgg<pAggInfo->nColumn
+       && pAggInfo->aCol[iAgg].pCExpr==pExpr
+      ){
         pExpr = sqlite3ExprDup(db, pExpr, 0);
         if( pExpr ){
           pAggInfo->aCol[iAgg].pCExpr = pExpr;
@@ -112013,8 +112585,9 @@ static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){
       }
     }else{
       assert( pExpr->op==TK_AGG_FUNCTION );
-      assert( iAgg>=0 && iAgg<pAggInfo->nFunc );
-      if( pAggInfo->aFunc[iAgg].pFExpr==pExpr ){
+      if( ALWAYS(iAgg<pAggInfo->nFunc)
+       && pAggInfo->aFunc[iAgg].pFExpr==pExpr
+      ){
         pExpr = sqlite3ExprDup(db, pExpr, 0);
         if( pExpr ){
           pAggInfo->aFunc[iAgg].pFExpr = pExpr;
@@ -112164,7 +112737,12 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
       }
       if( pIEpr==0 ) break;
       if( NEVER(!ExprUseYTab(pExpr)) ) break;
-      if( pExpr->pAggInfo!=0 ) break; /* Already resolved by outer context */
+      for(i=0; i<pSrcList->nSrc; i++){
+         if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break;
+      }
+      if( i>=pSrcList->nSrc ) break;
+      if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */
+      if( pParse->nErr ){ return WRC_Abort; }
 
       /* If we reach this point, it means that expression pExpr can be
       ** translated into a reference to an index column as described by
@@ -112175,6 +112753,9 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
       tmp.iTable = pIEpr->iIdxCur;
       tmp.iColumn = pIEpr->iIdxCol;
       findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp);
+      if( pParse->nErr ){ return WRC_Abort; }
+      assert( pAggInfo->aCol!=0 );
+      assert( tmp.iAgg<pAggInfo->nColumn );
       pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr;
       pExpr->pAggInfo = pAggInfo;
       pExpr->iAgg = tmp.iAgg;
@@ -112198,7 +112779,7 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){
           } /* endif pExpr->iTable==pItem->iCursor */
         } /* end loop over pSrcList */
       }
-      return WRC_Prune;
+      return WRC_Continue;
     }
     case TK_AGG_FUNCTION: {
       if( (pNC->ncFlags & NC_InAggFunc)==0
@@ -112352,6 +112933,37 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){
 }
 
 /*
+** Make sure sufficient registers have been allocated so that
+** iReg is a valid register number.
+*/
+SQLITE_PRIVATE void sqlite3TouchRegister(Parse *pParse, int iReg){
+  if( pParse->nMem<iReg ) pParse->nMem = iReg;
+}
+
+#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG)
+/*
+** Return the latest reusable register in the set of all registers.
+** The value returned is no less than iMin.  If any register iMin or
+** greater is in permanent use, then return one more than that last
+** permanent register.
+*/
+SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){
+  const ExprList *pList = pParse->pConstExpr;
+  if( pList ){
+    int i;
+    for(i=0; i<pList->nExpr; i++){
+      if( pList->a[i].u.iConstExprReg>=iMin ){
+        iMin = pList->a[i].u.iConstExprReg + 1;
+      }
+    }
+  }
+  pParse->nTempReg = 0;
+  pParse->nRangeReg = 0;
+  return iMin;
+}
+#endif /* SQLITE_ENABLE_STAT4 || SQLITE_DEBUG */
+
+/*
 ** Validate that no temporary register falls within the range of
 ** iFirst..iLast, inclusive.  This routine is only call from within assert()
 ** statements.
@@ -112370,6 +112982,14 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){
       return 0;
     }
   }
+  if( pParse->pConstExpr ){
+    ExprList *pList = pParse->pConstExpr;
+    for(i=0; i<pList->nExpr; i++){
+      int iReg = pList->a[i].u.iConstExprReg;
+      if( iReg==0 ) continue;
+      if( iReg>=iFirst && iReg<=iLast ) return 0;
+    }
+  }
   return 1;
 }
 #endif /* SQLITE_DEBUG */
@@ -113658,6 +114278,19 @@ static int renameEditSql(
 }
 
 /*
+** Set all pEList->a[].fg.eEName fields in the expression-list to val.
+*/
+static void renameSetENames(ExprList *pEList, int val){
+  if( pEList ){
+    int i;
+    for(i=0; i<pEList->nExpr; i++){
+      assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME );
+      pEList->a[i].fg.eEName = val;
+    }
+  }
+}
+
+/*
 ** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming
 ** it was read from the schema of database zDb. Return SQLITE_OK if
 ** successful. Otherwise, return an SQLite error code and leave an error
@@ -113704,7 +114337,17 @@ static int renameResolveTrigger(Parse *pParse){
           pSrc = 0;
           rc = SQLITE_NOMEM;
         }else{
+          /* pStep->pExprList contains an expression-list used for an UPDATE
+          ** statement. So the a[].zEName values are the RHS of the
+          ** "<col> = <expr>" clauses of the UPDATE statement. So, before
+          ** running SelectPrep(), change all the eEName values in
+          ** pStep->pExprList to ENAME_SPAN (from their current value of
+          ** ENAME_NAME). This is to prevent any ids in ON() clauses that are
+          ** part of pSrc from being incorrectly resolved against the
+          ** a[].zEName values as if they were column aliases.  */
+          renameSetENames(pStep->pExprList, ENAME_SPAN);
           sqlite3SelectPrep(pParse, pSel, 0);
+          renameSetENames(pStep->pExprList, ENAME_NAME);
           rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK;
           assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList );
           assert( pSrc==pSel->pSrc );
@@ -115653,11 +116296,15 @@ static void analyzeOneTable(
   int regIdxname = iMem++;     /* Register containing index name */
   int regStat1 = iMem++;       /* Value for the stat column of sqlite_stat1 */
   int regPrev = iMem;          /* MUST BE LAST (see below) */
+#ifdef SQLITE_ENABLE_STAT4
+  int doOnce = 1;              /* Flag for a one-time computation */
+#endif
 #ifdef SQLITE_ENABLE_PREUPDATE_HOOK
   Table *pStat1 = 0;
 #endif
 
-  pParse->nMem = MAX(pParse->nMem, iMem);
+  sqlite3TouchRegister(pParse, iMem);
+  assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) );
   v = sqlite3GetVdbe(pParse);
   if( v==0 || NEVER(pTab==0) ){
     return;
@@ -115763,7 +116410,7 @@ static void analyzeOneTable(
     ** the regPrev array and a trailing rowid (the rowid slot is required
     ** when building a record to insert into the sample column of
     ** the sqlite_stat4 table.  */
-    pParse->nMem = MAX(pParse->nMem, regPrev+nColTest);
+    sqlite3TouchRegister(pParse, regPrev+nColTest);
 
     /* Open a read-only cursor on the index being analyzed. */
     assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) );
@@ -115935,7 +116582,35 @@ static void analyzeOneTable(
       int addrIsNull;
       u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound;
 
-      pParse->nMem = MAX(pParse->nMem, regCol+nCol);
+      if( doOnce ){
+        int mxCol = nCol;
+        Index *pX;
+
+        /* Compute the maximum number of columns in any index */
+        for(pX=pTab->pIndex; pX; pX=pX->pNext){
+          int nColX;                     /* Number of columns in pX */
+          if( !HasRowid(pTab) && IsPrimaryKeyIndex(pX) ){
+            nColX = pX->nKeyCol;
+          }else{
+            nColX = pX->nColumn;
+          }
+          if( nColX>mxCol ) mxCol = nColX;
+        }
+
+        /* Allocate space to compute results for the largest index */
+        sqlite3TouchRegister(pParse, regCol+mxCol);
+        doOnce = 0;
+#ifdef SQLITE_DEBUG
+        /* Verify that the call to sqlite3ClearTempRegCache() below
+        ** really is needed.
+        ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25)
+        */
+        testcase( !sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) );
+#endif
+        sqlite3ClearTempRegCache(pParse);  /* tag-20230325-1 */
+        assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) );
+      }
+      assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) );
 
       addrNext = sqlite3VdbeCurrentAddr(v);
       callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid);
@@ -116016,6 +116691,11 @@ static void analyzeDatabase(Parse *pParse, int iDb){
   for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){
     Table *pTab = (Table*)sqliteHashData(k);
     analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab);
+#ifdef SQLITE_ENABLE_STAT4
+    iMem = sqlite3FirstAvailableRegister(pParse, iMem);
+#else
+    assert( iMem==sqlite3FirstAvailableRegister(pParse,iMem) );
+#endif
   }
   loadAnalysis(pParse, iDb);
 }
@@ -116403,6 +117083,10 @@ static int loadStatTbl(
     pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
     assert( pIdx==0 || pIdx->nSample==0 );
     if( pIdx==0 ) continue;
+    if( pIdx->aSample!=0 ){
+      /* The same index appears in sqlite_stat4 under multiple names */
+      continue;
+    }
     assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 );
     if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){
       nIdxCol = pIdx->nKeyCol;
@@ -116410,6 +117094,7 @@ static int loadStatTbl(
       nIdxCol = pIdx->nColumn;
     }
     pIdx->nSampleCol = nIdxCol;
+    pIdx->mxSample = nSample;
     nByte = sizeof(IndexSample) * nSample;
     nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample;
     nByte += nIdxCol * sizeof(tRowcnt);     /* Space for Index.aAvgEq[] */
@@ -116449,6 +117134,11 @@ static int loadStatTbl(
     if( zIndex==0 ) continue;
     pIdx = findIndexOrPrimaryKey(db, zIndex, zDb);
     if( pIdx==0 ) continue;
+    if( pIdx->nSample>=pIdx->mxSample ){
+      /* Too many slots used because the same index appears in
+      ** sqlite_stat4 using multiple names */
+      continue;
+    }
     /* This next condition is true if data has already been loaded from
     ** the sqlite_stat4 table. */
     nCol = pIdx->nSampleCol;
@@ -116492,11 +117182,12 @@ static int loadStat4(sqlite3 *db, const char *zDb){
   const Table *pStat4;
 
   assert( db->lookaside.bDisable );
-  if( (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0
+  if( OptimizationEnabled(db, SQLITE_Stat4)
+   && (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0
    && IsOrdinaryTable(pStat4)
   ){
     rc = loadStatTbl(db,
-      "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx",
+      "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase",
       "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4",
       zDb
     );
@@ -118340,7 +119031,7 @@ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){
   if( IsOrdinaryTable(pTable) ){
     sqlite3FkDelete(db, pTable);
   }
-#ifndef SQLITE_OMIT_VIRTUAL_TABLE
+#ifndef SQLITE_OMIT_VIRTUALTABLE
   else if( IsVirtual(pTable) ){
     sqlite3VtabClear(db, pTable);
   }
@@ -123372,6 +124063,7 @@ SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){
   ** strings is BINARY.
   */
   db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0);
+  sqlite3ExpirePreparedStatements(db, 1);
 }
 
 /*
@@ -123843,13 +124535,15 @@ static int tabIsReadOnly(Parse *pParse, Table *pTab){
 ** If pTab is writable but other errors have occurred -> return 1.
 ** If pTab is writable and no prior errors -> return 0;
 */
-SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){
+SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){
   if( tabIsReadOnly(pParse, pTab) ){
     sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName);
     return 1;
   }
 #ifndef SQLITE_OMIT_VIEW
-  if( !viewOk && IsView(pTab) ){
+  if( IsView(pTab)
+   && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0))
+  ){
     sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName);
     return 1;
   }
@@ -124103,7 +124797,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom(
     goto delete_from_cleanup;
   }
 
-  if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){
+  if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){
     goto delete_from_cleanup;
   }
   iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
@@ -126266,7 +126960,7 @@ static void trimFunc(
 /*
 ** The "unknown" function is automatically substituted in place of
 ** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN
-** when the SQLITE_ENABLE_UNKNOWN_FUNCTION compile-time option is used.
+** when the SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION compile-time option is used.
 ** When the "sqlite3" command-line shell is built using this functionality,
 ** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries
 ** involving application-defined functions to be examined in a generic
@@ -128569,22 +129263,22 @@ static Trigger *fkActionTrigger(
 
     if( action==OE_Restrict ){
       int iDb = sqlite3SchemaToIndex(db, pTab->pSchema);
-      Token tFrom;
-      Token tDb;
+      SrcList *pSrc;
       Expr *pRaise;
 
-      tFrom.z = zFrom;
-      tFrom.n = nFrom;
-      tDb.z = db->aDb[iDb].zDbSName;
-      tDb.n = sqlite3Strlen30(tDb.z);
-
       pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed");
       if( pRaise ){
         pRaise->affExpr = OE_Abort;
       }
+      pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0);
+      if( pSrc ){
+        assert( pSrc->nSrc==1 );
+        pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom);
+        pSrc->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName);
+      }
       pSelect = sqlite3SelectNew(pParse,
           sqlite3ExprListAppend(pParse, 0, pRaise),
-          sqlite3SrcListAppend(pParse, 0, &tDb, &tFrom),
+          pSrc,
           pWhere,
           0, 0, 0, 0, 0
       );
@@ -128800,45 +129494,47 @@ SQLITE_PRIVATE void sqlite3OpenTable(
 ** is managed along with the rest of the Index structure. It will be
 ** released when sqlite3DeleteIndex() is called.
 */
-SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){
+static SQLITE_NOINLINE const char *computeIndexAffStr(sqlite3 *db, Index *pIdx){
+  /* The first time a column affinity string for a particular index is
+  ** required, it is allocated and populated here. It is then stored as
+  ** a member of the Index structure for subsequent use.
+  **
+  ** The column affinity string will eventually be deleted by
+  ** sqliteDeleteIndex() when the Index structure itself is cleaned
+  ** up.
+  */
+  int n;
+  Table *pTab = pIdx->pTable;
+  pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1);
   if( !pIdx->zColAff ){
-    /* The first time a column affinity string for a particular index is
-    ** required, it is allocated and populated here. It is then stored as
-    ** a member of the Index structure for subsequent use.
-    **
-    ** The column affinity string will eventually be deleted by
-    ** sqliteDeleteIndex() when the Index structure itself is cleaned
-    ** up.
-    */
-    int n;
-    Table *pTab = pIdx->pTable;
-    pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1);
-    if( !pIdx->zColAff ){
-      sqlite3OomFault(db);
-      return 0;
-    }
-    for(n=0; n<pIdx->nColumn; n++){
-      i16 x = pIdx->aiColumn[n];
-      char aff;
-      if( x>=0 ){
-        aff = pTab->aCol[x].affinity;
-      }else if( x==XN_ROWID ){
-        aff = SQLITE_AFF_INTEGER;
-      }else{
-        assert( x==XN_EXPR );
-        assert( pIdx->bHasExpr );
-        assert( pIdx->aColExpr!=0 );
-        aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr);
-      }
-      if( aff<SQLITE_AFF_BLOB ) aff = SQLITE_AFF_BLOB;
-      if( aff>SQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC;
-      pIdx->zColAff[n] = aff;
+    sqlite3OomFault(db);
+    return 0;
+  }
+  for(n=0; n<pIdx->nColumn; n++){
+    i16 x = pIdx->aiColumn[n];
+    char aff;
+    if( x>=0 ){
+      aff = pTab->aCol[x].affinity;
+    }else if( x==XN_ROWID ){
+      aff = SQLITE_AFF_INTEGER;
+    }else{
+      assert( x==XN_EXPR );
+      assert( pIdx->bHasExpr );
+      assert( pIdx->aColExpr!=0 );
+      aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr);
     }
-    pIdx->zColAff[n] = 0;
+    if( aff<SQLITE_AFF_BLOB ) aff = SQLITE_AFF_BLOB;
+    if( aff>SQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC;
+    pIdx->zColAff[n] = aff;
   }
-
+  pIdx->zColAff[n] = 0;
   return pIdx->zColAff;
 }
+SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){
+  if( !pIdx->zColAff ) return computeIndexAffStr(db, pIdx);
+  return pIdx->zColAff;
+}
+
 
 /*
 ** Compute an affinity string for a table.   Space is obtained
@@ -129524,7 +130220,7 @@ SQLITE_PRIVATE void sqlite3Insert(
 
   /* Cannot insert into a read-only table.
   */
-  if( sqlite3IsReadOnly(pParse, pTab, tmask) ){
+  if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){
     goto insert_cleanup;
   }
 
@@ -129971,7 +130667,7 @@ SQLITE_PRIVATE void sqlite3Insert(
     }
 
     /* Copy the new data already generated. */
-    assert( pTab->nNVCol>0 );
+    assert( pTab->nNVCol>0 || pParse->nErr>0 );
     sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1);
 
 #ifndef SQLITE_OMIT_GENERATED_COLUMNS
@@ -133334,7 +134030,11 @@ static int sqlite3LoadExtension(
   /* tag-20210611-1.  Some dlopen() implementations will segfault if given
   ** an oversize filename.  Most filesystems have a pathname limit of 4K,
   ** so limit the extension filename length to about twice that.
-  ** https://sqlite.org/forum/forumpost/08a0d6d9bf */
+  ** https://sqlite.org/forum/forumpost/08a0d6d9bf
+  **
+  ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix.
+  ** See https://sqlite.org/forum/forumpost/24083b579d.
+  */
   if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found;
 
   handle = sqlite3OsDlOpen(pVfs, zFile);
@@ -133342,7 +134042,9 @@ static int sqlite3LoadExtension(
   for(ii=0; ii<ArraySize(azEndings) && handle==0; ii++){
     char *zAltFile = sqlite3_mprintf("%s.%s", zFile, azEndings[ii]);
     if( zAltFile==0 ) return SQLITE_NOMEM_BKPT;
-    handle = sqlite3OsDlOpen(pVfs, zAltFile);
+    if( nMsg+strlen(azEndings[ii])+1<=SQLITE_MAX_PATHLEN ){
+      handle = sqlite3OsDlOpen(pVfs, zAltFile);
+    }
     sqlite3_free(zAltFile);
   }
 #endif
@@ -135837,7 +136539,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
       zDb = db->aDb[iDb].zDbSName;
       sqlite3CodeVerifySchema(pParse, iDb);
       sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName);
-      if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow;
+      sqlite3TouchRegister(pParse, pTab->nCol+regRow);
       sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead);
       sqlite3VdbeLoadString(v, regResult, pTab->zName);
       assert( IsOrdinaryTable(pTab) );
@@ -135878,7 +136580,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
         ** regRow..regRow+n. If any of the child key values are NULL, this
         ** row cannot cause an FK violation. Jump directly to addrOk in
         ** this case. */
-        if( regRow+pFK->nCol>pParse->nMem ) pParse->nMem = regRow+pFK->nCol;
+        sqlite3TouchRegister(pParse, regRow + pFK->nCol);
         for(j=0; j<pFK->nCol; j++){
           int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom;
           sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j);
@@ -136007,6 +136709,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
       if( iDb>=0 && i!=iDb ) continue;
 
       sqlite3CodeVerifySchema(pParse, i);
+      pParse->okConstFactor = 0;  /* tag-20230327-1 */
 
       /* Do an integrity check of the B-Tree
       **
@@ -136042,7 +136745,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
       aRoot[0] = cnt;
 
       /* Make sure sufficient number of registers have been allocated */
-      pParse->nMem = MAX( pParse->nMem, 8+mxIdx );
+      sqlite3TouchRegister(pParse, 8+mxIdx);
       sqlite3ClearTempRegCache(pParse);
 
       /* Do the b-tree integrity checks */
@@ -136192,15 +136895,29 @@ SQLITE_PRIVATE void sqlite3Pragma(
           labelOk = sqlite3VdbeMakeLabel(pParse);
           if( pCol->notNull ){
             /* (1) NOT NULL columns may not contain a NULL */
+            int jmp3;
             int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4);
-            sqlite3VdbeChangeP5(v, 0x0f);
             VdbeCoverage(v);
+            if( p1<0 ){
+              sqlite3VdbeChangeP5(v, 0x0f); /* INT, REAL, TEXT, or BLOB */
+              jmp3 = jmp2;
+            }else{
+              sqlite3VdbeChangeP5(v, 0x0d); /* INT, TEXT, or BLOB */
+              /* OP_IsType does not detect NaN values in the database file
+              ** which should be treated as a NULL.  So if the header type
+              ** is REAL, we have to load the actual data using OP_Column
+              ** to reliably determine if the value is a NULL. */
+              sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3);
+              jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk);
+              VdbeCoverage(v);
+            }
             zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName,
                                 pCol->zCnName);
             sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC);
             if( doTypeCheck ){
               sqlite3VdbeGoto(v, labelError);
               sqlite3VdbeJumpHere(v, jmp2);
+              sqlite3VdbeJumpHere(v, jmp3);
             }else{
               /* VDBE byte code will fall thru */
             }
@@ -136308,7 +137025,7 @@ SQLITE_PRIVATE void sqlite3Pragma(
               int jmp7;
               sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3);
               jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1);
-              VdbeCoverage(v);
+              VdbeCoverageNeverNull(v);
               sqlite3VdbeLoadString(v, 3,
                  "rowid not at end-of-record for row ");
               sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3);
@@ -137514,7 +138231,9 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
 #else
       encoding = SQLITE_UTF8;
 #endif
-      if( db->nVdbeActive>0 && encoding!=ENC(db) ){
+      if( db->nVdbeActive>0 && encoding!=ENC(db)
+       && (db->mDbFlags & DBFLAG_Vacuum)==0
+      ){
         rc = SQLITE_LOCKED;
         goto initone_error_out;
       }else{
@@ -137908,7 +138627,11 @@ static int sqlite3Prepare(
   sParse.db = db;
   sParse.pReprepare = pReprepare;
   assert( ppStmt && *ppStmt==0 );
-  if( db->mallocFailed ) sqlite3ErrorMsg(&sParse, "out of memory");
+  if( db->mallocFailed ){
+    sqlite3ErrorMsg(&sParse, "out of memory");
+    db->errCode = rc = SQLITE_NOMEM;
+    goto end_prepare;
+  }
   assert( sqlite3_mutex_held(db->mutex) );
 
   /* For a long-term use prepared statement avoid the use of
@@ -138997,7 +139720,7 @@ static void pushOntoSorter(
   **   (2) All output columns are included in the sort record.  In that
   **       case regData==regOrigData.
   **   (3) Some output columns are omitted from the sort record due to
-  **       the SQLITE_ENABLE_SORTER_REFERENCE optimization, or due to the
+  **       the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the
   **       SQLITE_ECEL_OMITREF optimization, or due to the
   **       SortCtx.pDeferredRowLoad optimiation.  In any of these cases
   **       regOrigData is 0 to prevent this routine from trying to copy
@@ -140598,7 +141321,7 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes(
   assert( (pSelect->selFlags & SF_Resolved)!=0 );
   assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 );
   assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB );
-  if( db->mallocFailed ) return;
+  if( db->mallocFailed || IN_RENAME_OBJECT ) return;
   while( pSelect->pPrior ) pSelect = pSelect->pPrior;
   a = pSelect->pEList->a;
   memset(&sNC, 0, sizeof(sNC));
@@ -140643,18 +141366,16 @@ SQLITE_PRIVATE void sqlite3SubqueryColumnTypes(
             break;
           }
         }
-       }
-     }
-     if( zType ){
-       i64 m = sqlite3Strlen30(zType);
-       n = sqlite3Strlen30(pCol->zCnName);
-       pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2);
-       if( pCol->zCnName ){
-         memcpy(&pCol->zCnName[n+1], zType, m+1);
-         pCol->colFlags |= COLFLAG_HASTYPE;
-       }else{
-         testcase( pCol->colFlags & COLFLAG_HASTYPE );
-        pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL);
+      }
+    }
+    if( zType ){
+      i64 m = sqlite3Strlen30(zType);
+      n = sqlite3Strlen30(pCol->zCnName);
+      pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+m+2);
+      pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL);
+      if( pCol->zCnName ){
+        memcpy(&pCol->zCnName[n+1], zType, m+1);
+        pCol->colFlags |= COLFLAG_HASTYPE;
       }
     }
     pColl = sqlite3ExprCollSeq(pParse, p);
@@ -142521,8 +143242,7 @@ static int compoundHasDifferentAffinities(Select *p){
 **              query or there are no RIGHT or FULL JOINs in any arm
 **              of the subquery.  (This is a duplicate of condition (27b).)
 **        (17h) The corresponding result set expressions in all arms of the
-**              compound must have the same affinity. (See restriction (9)
-**              on the push-down optimization.)
+**              compound must have the same affinity.
 **
 **        The parent and sub-query may contain WHERE clauses. Subject to
 **        rules (11), (13) and (14), they may also contain ORDER BY,
@@ -143390,10 +144110,24 @@ static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){
 **       or EXCEPT, then all of the result set columns for all arms of
 **       the compound must use the BINARY collating sequence.
 **
-**   (9) If the subquery is a compound, then all arms of the compound must
-**       have the same affinity.  (This is the same as restriction (17h)
-**       for query flattening.)
+**   (9) All three of the following are true:
+**
+**       (9a) The WHERE clause expression originates in the ON or USING clause
+**            of a join (either an INNER or an OUTER join), and
+**
+**       (9b) The subquery is to the right of the ON/USING clause
 **
+**       (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING
+**            clause and the subquery.
+**
+**       Without this restriction, the push-down optimization might move
+**       the ON/USING filter expression from the left side of a RIGHT JOIN
+**       over to the right side, which leads to incorrect answers.  See
+**       also restriction (6) in sqlite3ExprIsSingleTableConstraint().
+**
+**  (10) The inner query is not the right-hand table of a RIGHT JOIN.
+**
+**  (11) The subquery is not a VALUES clause
 **
 ** Return 0 if no changes are made and non-zero if one or more WHERE clause
 ** terms are duplicated into the subquery.
@@ -143402,13 +144136,20 @@ static int pushDownWhereTerms(
   Parse *pParse,        /* Parse context (for malloc() and error reporting) */
   Select *pSubq,        /* The subquery whose WHERE clause is to be augmented */
   Expr *pWhere,         /* The WHERE clause of the outer query */
-  SrcItem *pSrc         /* The subquery term of the outer FROM clause */
+  SrcList *pSrcList,    /* The complete from clause of the outer query */
+  int iSrc              /* Which FROM clause term to try to push into  */
 ){
   Expr *pNew;
+  SrcItem *pSrc;        /* The subquery FROM term into which WHERE is pushed */
   int nChng = 0;
+  pSrc = &pSrcList->a[iSrc];
   if( pWhere==0 ) return 0;
-  if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ) return 0;
-  if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ) return 0;
+  if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ){
+    return 0;           /* restrictions (2) and (11) */
+  }
+  if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ){
+    return 0;           /* restrictions (10) */
+  }
 
   if( pSubq->pPrior ){
     Select *pSel;
@@ -143424,9 +144165,6 @@ static int pushDownWhereTerms(
       if( pSel->pWin ) return 0;    /* restriction (6b) */
 #endif
     }
-    if( compoundHasDifferentAffinities(pSubq) ){
-      return 0;  /* restriction (9) */
-    }
     if( notUnionAll ){
       /* If any of the compound arms are connected using UNION, INTERSECT,
       ** or EXCEPT, then we must ensure that none of the columns use a
@@ -143466,11 +144204,28 @@ static int pushDownWhereTerms(
     return 0; /* restriction (3) */
   }
   while( pWhere->op==TK_AND ){
-    nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrc);
+    nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrcList, iSrc);
     pWhere = pWhere->pLeft;
   }
 
-#if 0  /* Legacy code. Checks now done by sqlite3ExprIsTableConstraint() */
+#if 0 /* These checks now done by sqlite3ExprIsSingleTableConstraint() */
+  if( ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) /* (9a) */
+   && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0     /* Fast pre-test of (9c) */
+  ){
+    int jj;
+    for(jj=0; jj<iSrc; jj++){
+      if( pWhere->w.iJoin==pSrcList->a[jj].iCursor ){
+        /* If we reach this point, both (9a) and (9b) are satisfied.
+        ** The following loop checks (9c):
+        */
+        for(jj++; jj<iSrc; jj++){
+          if( (pSrcList->a[jj].fg.jointype & JT_RIGHT)!=0 ){
+            return 0;  /* restriction (9) */
+          }
+        }
+      }
+    }
+  }
   if( isLeftJoin
    && (ExprHasProperty(pWhere,EP_OuterON)==0
          || pWhere->w.iJoin!=iCursor)
@@ -143484,7 +144239,7 @@ static int pushDownWhereTerms(
   }
 #endif
 
-  if( sqlite3ExprIsTableConstraint(pWhere, pSrc) ){
+  if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc) ){
     nChng++;
     pSubq->selFlags |= SF_PushDown;
     while( pSubq ){
@@ -143519,6 +144274,78 @@ static int pushDownWhereTerms(
 #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */
 
 /*
+** Check to see if a subquery contains result-set columns that are
+** never used.  If it does, change the value of those result-set columns
+** to NULL so that they do not cause unnecessary work to compute.
+**
+** Return the number of column that were changed to NULL.
+*/
+static int disableUnusedSubqueryResultColumns(SrcItem *pItem){
+  int nCol;
+  Select *pSub;      /* The subquery to be simplified */
+  Select *pX;        /* For looping over compound elements of pSub */
+  Table *pTab;       /* The table that describes the subquery */
+  int j;             /* Column number */
+  int nChng = 0;     /* Number of columns converted to NULL */
+  Bitmask colUsed;   /* Columns that may not be NULLed out */
+
+  assert( pItem!=0 );
+  if( pItem->fg.isCorrelated || pItem->fg.isCte ){
+    return 0;
+  }
+  assert( pItem->pTab!=0 );
+  pTab = pItem->pTab;
+  assert( pItem->pSelect!=0 );
+  pSub = pItem->pSelect;
+  assert( pSub->pEList->nExpr==pTab->nCol );
+  if( (pSub->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){
+    testcase( pSub->selFlags & SF_Distinct );
+    testcase( pSub->selFlags & SF_Aggregate );
+    return 0;
+  }
+  for(pX=pSub; pX; pX=pX->pPrior){
+    if( pX->pPrior && pX->op!=TK_ALL ){
+      /* This optimization does not work for compound subqueries that
+      ** use UNION, INTERSECT, or EXCEPT.  Only UNION ALL is allowed. */
+      return 0;
+    }
+#ifndef SQLITE_OMIT_WINDOWFUNC
+    if( pX->pWin ){
+      /* This optimization does not work for subqueries that use window
+      ** functions. */
+      return 0;
+    }
+#endif
+  }
+  colUsed = pItem->colUsed;
+  if( pSub->pOrderBy ){
+    ExprList *pList = pSub->pOrderBy;
+    for(j=0; j<pList->nExpr; j++){
+      u16 iCol = pList->a[j].u.x.iOrderByCol;
+      if( iCol>0 ){
+        iCol--;
+        colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol);
+      }
+    }
+  }
+  nCol = pTab->nCol;
+  for(j=0; j<nCol; j++){
+    Bitmask m = j<BMS-1 ? MASKBIT(j) : TOPBIT;
+    if( (m & colUsed)!=0 ) continue;
+    for(pX=pSub; pX; pX=pX->pPrior) {
+      Expr *pY = pX->pEList->a[j].pExpr;
+      if( pY->op==TK_NULL ) continue;
+      pY->op = TK_NULL;
+      ExprClearProperty(pY, EP_Skip|EP_Unlikely);
+      pX->selFlags |= SF_PushDown;
+      nChng++;
+    }
+  }
+  return nChng;
+}
+
+
+/*
 ** The pFunc is the only aggregate function in the query.  Check to see
 ** if the query is a candidate for the min/max optimization.
 **
@@ -144664,12 +145491,13 @@ static void optimizeAggregateUseOfIndexedExpr(
   assert( pSelect->pGroupBy!=0 );
   pAggInfo->nColumn = pAggInfo->nAccumulator;
   if( ALWAYS(pAggInfo->nSortingColumn>0) ){
-    if( pAggInfo->nColumn==0 ){
-      pAggInfo->nSortingColumn = pSelect->pGroupBy->nExpr;
-    }else{
-      pAggInfo->nSortingColumn =
-        pAggInfo->aCol[pAggInfo->nColumn-1].iSorterColumn+1;
+    int mx = pSelect->pGroupBy->nExpr - 1;
+    int j, k;
+    for(j=0; j<pAggInfo->nColumn; j++){
+      k = pAggInfo->aCol[j].iSorterColumn;
+      if( k>mx ) mx = k;
     }
+    pAggInfo->nSortingColumn = mx+1;
   }
   analyzeAggFuncArgs(pAggInfo, pNC);
 #if TREETRACE_ENABLED
@@ -144703,11 +145531,13 @@ static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){
   if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue;
   if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue;
   pAggInfo = pExpr->pAggInfo;
-  assert( pExpr->iAgg>=0 && pExpr->iAgg<pAggInfo->nColumn );
+  if( NEVER(pExpr->iAgg>=pAggInfo->nColumn) ) return WRC_Continue;
+  assert( pExpr->iAgg>=0 );
   pCol = &pAggInfo->aCol[pExpr->iAgg];
   pExpr->op = TK_AGG_COLUMN;
   pExpr->iTable = pCol->iTable;
   pExpr->iColumn = pCol->iColumn;
+  ExprClearProperty(pExpr, EP_Skip|EP_Collate);
   return WRC_Prune;
 }
 
@@ -145061,7 +145891,6 @@ static void agginfoFree(sqlite3 *db, AggInfo *p){
   sqlite3DbFreeNN(db, p);
 }
 
-#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION
 /*
 ** Attempt to transform a query of the form
 **
@@ -145089,6 +145918,7 @@ static int countOfViewOptimization(Parse *pParse, Select *p){
   if( (p->selFlags & SF_Aggregate)==0 ) return 0;   /* This is an aggregate */
   if( p->pEList->nExpr!=1 ) return 0;               /* Single result column */
   if( p->pWhere ) return 0;
+  if( p->pHaving ) return 0;
   if( p->pGroupBy ) return 0;
   if( p->pOrderBy ) return 0;
   pExpr = p->pEList->a[0].pExpr;
@@ -145108,7 +145938,8 @@ static int countOfViewOptimization(Parse *pParse, Select *p){
     if( pSub->pWhere ) return 0;                      /* No WHERE clause */
     if( pSub->pLimit ) return 0;                      /* No LIMIT clause */
     if( pSub->selFlags & SF_Aggregate ) return 0;     /* Not an aggregate */
-    pSub = pSub->pPrior;                              /* Repeat over compound */
+    assert( pSub->pHaving==0 );  /* Due to the previous */
+   pSub = pSub->pPrior;                              /* Repeat over compound */
   }while( pSub );
 
   /* If we reach this point then it is OK to perform the transformation */
@@ -145151,7 +145982,6 @@ static int countOfViewOptimization(Parse *pParse, Select *p){
 #endif
   return 1;
 }
-#endif /* SQLITE_COUNTOFVIEW_OPTIMIZATION */
 
 /*
 ** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same
@@ -145407,7 +146237,7 @@ SQLITE_PRIVATE int sqlite3Select(
                     pTabList->a[0].fg.jointype & JT_LTORJ);
     }
 
-    /* No futher action if this term of the FROM clause is no a subquery */
+    /* No futher action if this term of the FROM clause is not a subquery */
     if( pSub==0 ) continue;
 
     /* Catch mismatch in the declared columns of a view and the number of
@@ -145540,14 +146370,12 @@ SQLITE_PRIVATE int sqlite3Select(
     TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n"));
   }
 
-#ifdef SQLITE_COUNTOFVIEW_OPTIMIZATION
   if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView)
    && countOfViewOptimization(pParse, p)
   ){
     if( db->mallocFailed ) goto select_end;
     pTabList = p->pSrc;
   }
-#endif
 
   /* For each term in the FROM clause, do two things:
   ** (1) Authorized unreferenced tables
@@ -145606,7 +146434,7 @@ SQLITE_PRIVATE int sqlite3Select(
     if( OptimizationEnabled(db, SQLITE_PushDown)
      && (pItem->fg.isCte==0
          || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2))
-     && pushDownWhereTerms(pParse, pSub, p->pWhere, pItem)
+     && pushDownWhereTerms(pParse, pSub, p->pWhere, pTabList, i)
     ){
 #if TREETRACE_ENABLED
       if( sqlite3TreeTrace & 0x4000 ){
@@ -145620,6 +146448,22 @@ SQLITE_PRIVATE int sqlite3Select(
       TREETRACE(0x4000,pParse,p,("Push-down not possible\n"));
     }
 
+    /* Convert unused result columns of the subquery into simple NULL
+    ** expressions, to avoid unneeded searching and computation.
+    */
+    if( OptimizationEnabled(db, SQLITE_NullUnusedCols)
+     && disableUnusedSubqueryResultColumns(pItem)
+    ){
+#if TREETRACE_ENABLED
+      if( sqlite3TreeTrace & 0x4000 ){
+        TREETRACE(0x4000,pParse,p,
+            ("Change unused result columns to NULL for subquery %d:\n",
+             pSub->selId));
+        sqlite3TreeViewSelect(0, p, 0);
+      }
+#endif
+    }
+
     zSavedAuthContext = pParse->zAuthContext;
     pParse->zAuthContext = pItem->zName;
 
@@ -148157,6 +149001,9 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(
   Trigger *p;
 
   assert( isNew==1 || isNew==0 );
+  if( IsView(pTab) ){
+    return 0xffffffff;
+  }
   for(p=pTrigger; p; p=p->pNext){
     if( p->op==op
      && (tr_tm&p->tr_tm)
@@ -148591,7 +149438,7 @@ SQLITE_PRIVATE void sqlite3Update(
   if( sqlite3ViewGetColumnNames(pParse, pTab) ){
     goto update_cleanup;
   }
-  if( sqlite3IsReadOnly(pParse, pTab, tmask) ){
+  if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){
     goto update_cleanup;
   }
 
@@ -151382,7 +152229,10 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){
             break;
         }
         if( xMethod && pVTab->iSavepoint>iSavepoint ){
+          u64 savedFlags = (db->flags & SQLITE_Defensive);
+          db->flags &= ~(u64)SQLITE_Defensive;
           rc = xMethod(pVTab->pVtab, iSavepoint);
+          db->flags |= savedFlags;
         }
         sqlite3VtabUnlock(pVTab);
       }
@@ -151611,6 +152461,10 @@ SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){
         p->pVTable->eVtabRisk = SQLITE_VTABRISK_High;
         break;
       }
+      case SQLITE_VTAB_USES_ALL_SCHEMAS: {
+        p->pVTable->bAllSchemas = 1;
+        break;
+      }
       default: {
         rc = SQLITE_MISUSE_BKPT;
         break;
@@ -152384,9 +153238,9 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){
 
 /*
 ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN
-** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was
-** defined at compile-time. If it is not a no-op, a single OP_Explain opcode
-** is added to the output to describe the table scan strategy in pLevel.
+** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG
+** was defined at compile-time. If it is not a no-op, a single OP_Explain
+** opcode is added to the output to describe the table scan strategy in pLevel.
 **
 ** If an OP_Explain opcode is added to the VM, its address is returned.
 ** Otherwise, if no OP_Explain is coded, zero is returned.
@@ -152398,8 +153252,8 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan(
   u16 wctrlFlags                  /* Flags passed to sqlite3WhereBegin() */
 ){
   int ret = 0;
-#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS)
-  if( sqlite3ParseToplevel(pParse)->explain==2 )
+#if !defined(SQLITE_DEBUG)
+  if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) )
 #endif
   {
     SrcItem *pItem = &pTabList->a[pLevel->iFrom];
@@ -152565,27 +153419,29 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus(
   WhereLevel *pLvl,               /* Level to add scanstatus() entry for */
   int addrExplain                 /* Address of OP_Explain (or 0) */
 ){
-  const char *zObj = 0;
-  WhereLoop *pLoop = pLvl->pWLoop;
-  int wsFlags = pLoop->wsFlags;
-  int viaCoroutine = 0;
+  if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){
+    const char *zObj = 0;
+    WhereLoop *pLoop = pLvl->pWLoop;
+    int wsFlags = pLoop->wsFlags;
+    int viaCoroutine = 0;
 
-  if( (wsFlags & WHERE_VIRTUALTABLE)==0  &&  pLoop->u.btree.pIndex!=0 ){
-    zObj = pLoop->u.btree.pIndex->zName;
-  }else{
-    zObj = pSrclist->a[pLvl->iFrom].zName;
-    viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine;
-  }
-  sqlite3VdbeScanStatus(
-      v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj
-  );
-
-  if( viaCoroutine==0 ){
-    if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){
-      sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur);
+    if( (wsFlags & WHERE_VIRTUALTABLE)==0  &&  pLoop->u.btree.pIndex!=0 ){
+      zObj = pLoop->u.btree.pIndex->zName;
+    }else{
+      zObj = pSrclist->a[pLvl->iFrom].zName;
+      viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine;
     }
-    if( wsFlags & WHERE_INDEXED ){
-      sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur);
+    sqlite3VdbeScanStatus(
+        v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj
+    );
+
+    if( viaCoroutine==0 ){
+      if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){
+        sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur);
+      }
+      if( wsFlags & WHERE_INDEXED ){
+        sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur);
+      }
     }
   }
 }
@@ -153282,11 +154138,12 @@ static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){
 */
 static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
   int rc = WRC_Continue;
+  int reg;
   struct CCurHint *pHint = pWalker->u.pCCurHint;
   if( pExpr->op==TK_COLUMN ){
     if( pExpr->iTable!=pHint->iTabCur ){
-      int reg = ++pWalker->pParse->nMem;   /* Register for column value */
-      sqlite3ExprCode(pWalker->pParse, pExpr, reg);
+      reg = ++pWalker->pParse->nMem;   /* Register for column value */
+      reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg);
       pExpr->op = TK_REGISTER;
       pExpr->iTable = reg;
     }else if( pHint->pIdx!=0 ){
@@ -153294,15 +154151,15 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){
       pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn);
       assert( pExpr->iColumn>=0 );
     }
-  }else if( pExpr->op==TK_AGG_FUNCTION ){
-    /* An aggregate function in the WHERE clause of a query means this must
-    ** be a correlated sub-query, and expression pExpr is an aggregate from
-    ** the parent context. Do not walk the function arguments in this case.
-    **
-    ** todo: It should be possible to replace this node with a TK_REGISTER
-    ** expression, as the result of the expression must be stored in a
-    ** register at this point. The same holds for TK_AGG_COLUMN nodes. */
+  }else if( pExpr->pAggInfo ){
     rc = WRC_Prune;
+    reg = ++pWalker->pParse->nMem;   /* Register for column value */
+    reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg);
+    pExpr->op = TK_REGISTER;
+    pExpr->iTable = reg;
+  }else if( pExpr->op==TK_TRUEFALSE ){
+    /* Do not walk disabled expressions.  tag-20230504-1 */
+    return WRC_Prune;
   }
   return rc;
 }
@@ -153404,7 +154261,7 @@ static void codeCursorHint(
   }
   if( pExpr!=0 ){
     sWalker.xExprCallback = codeCursorHintFixExpr;
-    sqlite3WalkExpr(&sWalker, pExpr);
+    if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr);
     sqlite3VdbeAddOp4(v, OP_CursorHint,
                       (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0,
                       (const char*)pExpr, P4_EXPR);
@@ -154198,7 +155055,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
         ** guess. */
         addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan,
                                          (pIdx->aiRowLogEst[0]+9)/10);
-        if( pRangeStart ){
+        if( pRangeStart || pRangeEnd ){
           sqlite3VdbeChangeP5(v, 1);
           sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1);
           addrSeekScan = 0;
@@ -154239,16 +155096,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
     assert( pLevel->p2==0 );
     if( pRangeEnd ){
       Expr *pRight = pRangeEnd->pExpr->pRight;
-      if( addrSeekScan ){
-        /* For a seek-scan that has a range on the lowest term of the index,
-        ** we have to make the top of the loop be code that sets the end
-        ** condition of the range.  Otherwise, the OP_SeekScan might jump
-        ** over that initialization, leaving the range-end value set to the
-        ** range-start value, resulting in a wrong answer.
-        ** See ticket 5981a8c041a3c2f3 (2021-11-02).
-        */
-        pLevel->p2 = sqlite3VdbeCurrentAddr(v);
-      }
+      assert( addrSeekScan==0 );
       codeExprOrVector(pParse, pRight, regBase+nEq, nTop);
       whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd);
       if( (pRangeEnd->wtFlags & TERM_VNULL)==0
@@ -154282,7 +155130,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart(
     if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff);
 
     /* Top of the loop body */
-    if( pLevel->p2==0 ) pLevel->p2 = sqlite3VdbeCurrentAddr(v);
+    pLevel->p2 = sqlite3VdbeCurrentAddr(v);
 
     /* Check if the index cursor is past the end of the range. */
     if( nConstraint ){
@@ -156279,7 +157127,7 @@ static void exprAnalyze(
      && 0==sqlite3ExprCanBeNull(pLeft)
     ){
       assert( !ExprHasProperty(pExpr, EP_IntValue) );
-      pExpr->op = TK_TRUEFALSE;
+      pExpr->op = TK_TRUEFALSE;  /* See tag-20230504-1 */
       pExpr->u.zToken = "false";
       ExprSetProperty(pExpr, EP_IsFalse);
       pTerm->prereqAll = 0;
@@ -156924,9 +157772,12 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(
     pRhs = sqlite3PExpr(pParse, TK_UPLUS,
         sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0);
     pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs);
-    if( pItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){
+    if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){
+      testcase( pItem->fg.jointype & JT_LEFT );  /* testtag-20230227a */
+      testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */
       joinType = EP_OuterON;
     }else{
+      testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */
       joinType = EP_InnerON;
     }
     sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType);
@@ -157769,7 +158620,7 @@ static void explainAutomaticIndex(
   int bPartial,                   /* True if pIdx is a partial index */
   int *pAddrExplain               /* OUT: Address of OP_Explain */
 ){
-  if( pParse->explain!=2 ){
+  if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){
     Table *pTab = pIdx->pTable;
     const char *zSep = "";
     char *zText = 0;
@@ -157808,8 +158659,7 @@ static void explainAutomaticIndex(
 */
 static SQLITE_NOINLINE void constructAutomaticIndex(
   Parse *pParse,              /* The parsing context */
-  const WhereClause *pWC,     /* The WHERE clause */
-  const SrcItem *pSrc,        /* The FROM clause term to get the next index */
+  WhereClause *pWC,           /* The WHERE clause */
   const Bitmask notReady,     /* Mask of cursors that are not available */
   WhereLevel *pLevel          /* Write new index here */
 ){
@@ -157830,10 +158680,12 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   char *zNotUsed;             /* Extra space on the end of pIdx */
   Bitmask idxCols;            /* Bitmap of columns used for indexing */
   Bitmask extraCols;          /* Bitmap of additional columns */
-  u8 sentWarning = 0;         /* True if a warnning has been issued */
+  u8 sentWarning = 0;         /* True if a warning has been issued */
+  u8 useBloomFilter = 0;      /* True to also add a Bloom filter */
   Expr *pPartial = 0;         /* Partial Index Expression */
   int iContinue = 0;          /* Jump here to skip excluded rows */
-  SrcItem *pTabItem;          /* FROM clause term being indexed */
+  SrcList *pTabList;          /* The complete FROM clause */
+  SrcItem *pSrc;              /* The FROM clause term to get the next index */
   int addrCounter = 0;        /* Address where integer counter is initialized */
   int regBase;                /* Array of registers where record is assembled */
 #ifdef SQLITE_ENABLE_STMT_SCANSTATUS
@@ -157849,6 +158701,8 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   /* Count the number of columns that will be added to the index
   ** and used to match WHERE clause constraints */
   nKeyCol = 0;
+  pTabList = pWC->pWInfo->pTabList;
+  pSrc = &pTabList->a[pLevel->iFrom];
   pTable = pSrc->pTab;
   pWCEnd = &pWC->a[pWC->nTerm];
   pLoop = pLevel->pWLoop;
@@ -157859,7 +158713,7 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
     ** WHERE clause (or the ON clause of a LEFT join) that constrain which
     ** rows of the target table (pSrc) that can be used. */
     if( (pTerm->wtFlags & TERM_VIRTUAL)==0
-     && sqlite3ExprIsTableConstraint(pExpr, pSrc)
+     && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom)
     ){
       pPartial = sqlite3ExprAnd(pParse, pPartial,
                                 sqlite3ExprDup(pParse->db, pExpr, 0));
@@ -157900,7 +158754,11 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   ** original table changes and the index and table cannot both be used
   ** if they go out of sync.
   */
-  extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
+  if( IsView(pTable) ){
+    extraCols = ALLBITS;
+  }else{
+    extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1));
+  }
   mxBitCol = MIN(BMS-1,pTable->nCol);
   testcase( pTable->nCol==BMS-1 );
   testcase( pTable->nCol==BMS-2 );
@@ -157936,6 +158794,16 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
         assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */
         pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY;
         n++;
+        if( ALWAYS(pX->pLeft!=0)
+         && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT
+        ){
+          /* TUNING: only use a Bloom filter on an automatic index
+          ** if one or more key columns has the ability to hold numeric
+          ** values, since strings all have the same hash in the Bloom
+          ** filter implementation and hence a Bloom filter on a text column
+          ** is not usually helpful. */
+          useBloomFilter = 1;
+        }
       }
     }
   }
@@ -157968,20 +158836,21 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1);
   sqlite3VdbeSetP4KeyInfo(pParse, pIdx);
   VdbeComment((v, "for %s", pTable->zName));
-  if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){
+  if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){
+    sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel);
     pLevel->regFilter = ++pParse->nMem;
     sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter);
   }
 
   /* Fill the automatic index with content */
-  pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom];
-  if( pTabItem->fg.viaCoroutine ){
-    int regYield = pTabItem->regReturn;
+  assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] );
+  if( pSrc->fg.viaCoroutine ){
+    int regYield = pSrc->regReturn;
     addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0);
-    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub);
+    sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSrc->addrFillSub);
     addrTop =  sqlite3VdbeAddOp1(v, OP_Yield, regYield);
     VdbeCoverage(v);
-    VdbeComment((v, "next row of %s", pTabItem->pTab->zName));
+    VdbeComment((v, "next row of %s", pSrc->pTab->zName));
   }else{
     addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v);
   }
@@ -158002,14 +158871,14 @@ static SQLITE_NOINLINE void constructAutomaticIndex(
   sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord);
   sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT);
   if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue);
-  if( pTabItem->fg.viaCoroutine ){
+  if( pSrc->fg.viaCoroutine ){
     sqlite3VdbeChangeP2(v, addrCounter, regBase+n);
     testcase( pParse->db->mallocFailed );
     assert( pLevel->iIdxCur>0 );
     translateColumnToCopy(pParse, addrTop, pLevel->iTabCur,
-                          pTabItem->regResult, pLevel->iIdxCur);
+                          pSrc->regResult, pLevel->iIdxCur);
     sqlite3VdbeGoto(v, addrTop);
-    pTabItem->fg.viaCoroutine = 0;
+    pSrc->fg.viaCoroutine = 0;
   }else{
     sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v);
     sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX);
@@ -158072,9 +158941,11 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
 
   addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v);
   do{
+    const SrcList *pTabList;
     const SrcItem *pItem;
     const Table *pTab;
     u64 sz;
+    int iSrc;
     sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel);
     addrCont = sqlite3VdbeMakeLabel(pParse);
     iCur = pLevel->iTabCur;
@@ -158088,7 +158959,9 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
     ** testing complicated.  By basing the blob size on the value in the
     ** sqlite_stat1 table, testing is much easier.
     */
-    pItem = &pWInfo->pTabList->a[pLevel->iFrom];
+    pTabList = pWInfo->pTabList;
+    iSrc = pLevel->iFrom;
+    pItem = &pTabList->a[iSrc];
     assert( pItem!=0 );
     pTab = pItem->pTab;
     assert( pTab!=0 );
@@ -158105,7 +158978,7 @@ static SQLITE_NOINLINE void sqlite3ConstructBloomFilter(
     for(pTerm=pWInfo->sWC.a; pTerm<pWCEnd; pTerm++){
       Expr *pExpr = pTerm->pExpr;
       if( (pTerm->wtFlags & TERM_VIRTUAL)==0
-       && sqlite3ExprIsTableConstraint(pExpr, pItem)
+       && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc)
       ){
         sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL);
       }
@@ -158409,6 +159282,9 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){
       sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg);
     }
   }
+  if( pTab->u.vtab.p->bAllSchemas ){
+    sqlite3VtabUsesAllSchemas(pParse);
+  }
   sqlite3_free(pVtab->zErrMsg);
   pVtab->zErrMsg = 0;
   return rc;
@@ -158939,7 +159815,7 @@ static int whereRangeScanEst(
   UNUSED_PARAMETER(pBuilder);
   assert( pLower || pUpper );
 #endif
-  assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 );
+  assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 );
   nNew = whereRangeAdjust(pLower, nOut);
   nNew = whereRangeAdjust(pUpper, nNew);
 
@@ -161040,8 +161916,6 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
   return pHidden->eDistinct;
 }
 
-#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \
-    && !defined(SQLITE_OMIT_VIRTUALTABLE)
 /*
 ** Cause the prepared statement that is associated with a call to
 ** xBestIndex to potentially use all schemas.  If the statement being
@@ -161051,9 +161925,7 @@ SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){
 **
 ** This is used by the (built-in) sqlite_dbpage virtual table.
 */
-SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){
-  HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1];
-  Parse *pParse = pHidden->pParse;
+SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse *pParse){
   int nDb = pParse->db->nDb;
   int i;
   for(i=0; i<nDb; i++){
@@ -161065,7 +161937,6 @@ SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(sqlite3_index_info *pIdxInfo){
     }
   }
 }
-#endif
 
 /*
 ** Add all WhereLoop objects for a table of the join identified by
@@ -162446,6 +163317,13 @@ static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){
 **      at most a single row.
 **   4) The table must not be referenced by any part of the query apart
 **      from its own USING or ON clause.
+**   5) The table must not have an inner-join ON or USING clause if there is
+**      a RIGHT JOIN anywhere in the query.  Otherwise the ON/USING clause
+**      might move from the right side to the left side of the RIGHT JOIN.
+**      Note: Due to (2), this condition can only arise if the table is
+**      the right-most table of a subquery that was flattened into the
+**      main query and that subquery was the right-hand operand of an
+**      inner join that held an ON or USING clause.
 **
 ** For example, given:
 **
@@ -162471,6 +163349,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
 ){
   int i;
   Bitmask tabUsed;
+  int hasRightJoin;
 
   /* Preconditions checked by the caller */
   assert( pWInfo->nLevel>=2 );
@@ -162485,6 +163364,7 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
   if( pWInfo->pOrderBy ){
     tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy);
   }
+  hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0;
   for(i=pWInfo->nLevel-1; i>=1; i--){
     WhereTerm *pTerm, *pEnd;
     SrcItem *pItem;
@@ -162507,6 +163387,12 @@ static SQLITE_NOINLINE Bitmask whereOmitNoopJoin(
           break;
         }
       }
+      if( hasRightJoin
+       && ExprHasProperty(pTerm->pExpr, EP_InnerON)
+       && pTerm->pExpr->w.iJoin==pItem->iCursor
+      ){
+        break;  /* restriction (5) */
+      }
     }
     if( pTerm<pEnd ) continue;
     WHERETRACE(0xffffffff, ("-> drop loop %c not used\n", pLoop->cId));
@@ -162906,22 +163792,45 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
   }
   if( pParse->nErr ) goto whereBeginError;
 
-  /* Special case: WHERE terms that do not refer to any tables in the join
-  ** (constant expressions). Evaluate each such term, and jump over all the
-  ** generated code if the result is not true.
+  /* The False-WHERE-Term-Bypass optimization:
   **
-  ** Do not do this if the expression contains non-deterministic functions
-  ** that are not within a sub-select. This is not strictly required, but
-  ** preserves SQLite's legacy behaviour in the following two cases:
+  ** If there are WHERE terms that are false, then no rows will be output,
+  ** so skip over all of the code generated here.
   **
-  **   FROM ... WHERE random()>0;           -- eval random() once per row
-  **   FROM ... WHERE (SELECT random())>0;  -- eval random() once overall
+  ** Conditions:
+  **
+  **   (1)  The WHERE term must not refer to any tables in the join.
+  **   (2)  The term must not come from an ON clause on the
+  **        right-hand side of a LEFT or FULL JOIN.
+  **   (3)  The term must not come from an ON clause, or there must be
+  **        no RIGHT or FULL OUTER joins in pTabList.
+  **   (4)  If the expression contains non-deterministic functions
+  **        that are not within a sub-select. This is not required
+  **        for correctness but rather to preserves SQLite's legacy
+  **        behaviour in the following two cases:
+  **
+  **          WHERE random()>0;           -- eval random() once per row
+  **          WHERE (SELECT random())>0;  -- eval random() just once overall
+  **
+  ** Note that the Where term need not be a constant in order for this
+  ** optimization to apply, though it does need to be constant relative to
+  ** the current subquery (condition 1).  The term might include variables
+  ** from outer queries so that the value of the term changes from one
+  ** invocation of the current subquery to the next.
   */
   for(ii=0; ii<sWLB.pWC->nBase; ii++){
-    WhereTerm *pT = &sWLB.pWC->a[ii];
+    WhereTerm *pT = &sWLB.pWC->a[ii];  /* A term of the WHERE clause */
+    Expr *pX;                          /* The expression of pT */
     if( pT->wtFlags & TERM_VIRTUAL ) continue;
-    if( pT->prereqAll==0 && (nTabList==0 || exprIsDeterministic(pT->pExpr)) ){
-      sqlite3ExprIfFalse(pParse, pT->pExpr, pWInfo->iBreak, SQLITE_JUMPIFNULL);
+    pX = pT->pExpr;
+    assert( pX!=0 );
+    assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) );
+    if( pT->prereqAll==0                           /* Conditions (1) and (2) */
+     && (nTabList==0 || exprIsDeterministic(pX))   /* Condition (4) */
+     && !(ExprHasProperty(pX, EP_InnerON)          /* Condition (3) */
+          && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 )
+    ){
+      sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL);
       pT->wtFlags |= TERM_CODED;
     }
   }
@@ -163164,7 +164073,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
         assert( n<=pTab->nCol );
       }
 #ifdef SQLITE_ENABLE_CURSOR_HINTS
-      if( pLoop->u.btree.pIndex!=0 ){
+      if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){
         sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete);
       }else
 #endif
@@ -163301,11 +164210,11 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(
         sqlite3VdbeJumpHere(v, iOnce);
       }
     }
+    assert( pTabList == pWInfo->pTabList );
     if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){
       if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){
 #ifndef SQLITE_OMIT_AUTOMATIC_INDEX
-        constructAutomaticIndex(pParse, &pWInfo->sWC,
-                  &pTabList->a[pLevel->iFrom], notReady, pLevel);
+        constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel);
 #endif
       }else{
         sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady);
@@ -163622,7 +164531,8 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){
       k = pLevel->addrBody + 1;
 #ifdef SQLITE_DEBUG
       if( db->flags & SQLITE_VdbeAddopTrace ){
-        printf("TRANSLATE opcodes in range %d..%d\n", k, last-1);
+        printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n",
+                pLevel->iTabCur, pLevel->iIdxCur, k, last-1);
       }
       /* Proof that the "+1" on the k value above is safe */
       pOp = sqlite3VdbeGetOp(v, k - 1);
@@ -164497,6 +165407,7 @@ static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){
       }
       /* no break */ deliberate_fall_through
 
+    case TK_IF_NULL_ROW:
     case TK_AGG_FUNCTION:
     case TK_COLUMN: {
       int iCol = -1;
@@ -167325,18 +168236,18 @@ typedef union {
 #define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse;
 #define sqlite3ParserCTX_STORE yypParser->pParse=pParse;
 #define YYFALLBACK 1
-#define YYNSTATE             580
-#define YYNRULE              405
-#define YYNRULE_WITH_ACTION  342
+#define YYNSTATE             579
+#define YYNRULE              403
+#define YYNRULE_WITH_ACTION  340
 #define YYNTOKEN             185
-#define YY_MAX_SHIFT         579
-#define YY_MIN_SHIFTREDUCE   839
-#define YY_MAX_SHIFTREDUCE   1243
-#define YY_ERROR_ACTION      1244
-#define YY_ACCEPT_ACTION     1245
-#define YY_NO_ACTION         1246
-#define YY_MIN_REDUCE        1247
-#define YY_MAX_REDUCE        1651
+#define YY_MAX_SHIFT         578
+#define YY_MIN_SHIFTREDUCE   837
+#define YY_MAX_SHIFTREDUCE   1239
+#define YY_ERROR_ACTION      1240
+#define YY_ACCEPT_ACTION     1241
+#define YY_NO_ACTION         1242
+#define YY_MIN_REDUCE        1243
+#define YY_MAX_REDUCE        1645
 /************* End control #defines *******************************************/
 #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0])))
 
@@ -167403,219 +168314,218 @@ typedef union {
 **  yy_default[]       Default action for each state.
 **
 *********** Begin parsing tables **********************************************/
-#define YY_ACTTAB_COUNT (2101)
+#define YY_ACTTAB_COUNT (2096)
 static const YYACTIONTYPE yy_action[] = {
  /*     0 */   572,  208,  572,  118,  115,  229,  572,  118,  115,  229,
- /*    10 */   572, 1318,  381, 1297,  412,  566,  566,  566,  572,  413,
- /*    20 */   382, 1318, 1280,   41,   41,   41,   41,  208, 1530,   71,
- /*    30 */    71,  975,  423,   41,   41,  495,  303,  279,  303,  976,
- /*    40 */   401,   71,   71,  125,  126,   80, 1221, 1221, 1054, 1057,
- /*    50 */  1044, 1044,  123,  123,  124,  124,  124,  124,  480,  413,
- /*    60 */  1245,    1,    1,  579,    2, 1249,  554,  118,  115,  229,
- /*    70 */   317,  484,  146,  484,  528,  118,  115,  229,  533, 1331,
- /*    80 */   421,  527,  142,  125,  126,   80, 1221, 1221, 1054, 1057,
- /*    90 */  1044, 1044,  123,  123,  124,  124,  124,  124,  118,  115,
+ /*    10 */   572, 1314,  381, 1293,  412,  566,  566,  566,  572,  413,
+ /*    20 */   382, 1314, 1276,   41,   41,   41,   41,  208, 1524,   71,
+ /*    30 */    71,  973,  423,   41,   41,  495,  303,  279,  303,  974,
+ /*    40 */   401,   71,   71,  125,  126,   80, 1216, 1216, 1051, 1054,
+ /*    50 */  1041, 1041,  123,  123,  124,  124,  124,  124,  480,  413,
+ /*    60 */  1241,    1,    1,  578,    2, 1245,  554,  118,  115,  229,
+ /*    70 */   317,  484,  146,  484,  528,  118,  115,  229,  533, 1327,
+ /*    80 */   421,  527,  142,  125,  126,   80, 1216, 1216, 1051, 1054,
+ /*    90 */  1041, 1041,  123,  123,  124,  124,  124,  124,  118,  115,
  /*   100 */   229,  327,  122,  122,  122,  122,  121,  121,  120,  120,
  /*   110 */   120,  119,  116,  448,  284,  284,  284,  284,  446,  446,
- /*   120 */   446, 1571,  380, 1573, 1196,  379, 1167,  569, 1167,  569,
- /*   130 */   413, 1571,  541,  259,  226,  448,  101,  145,  453,  316,
+ /*   120 */   446, 1565,  380, 1567, 1192,  379, 1163,  569, 1163,  569,
+ /*   130 */   413, 1565,  541,  259,  226,  448,  101,  145,  453,  316,
  /*   140 */   563,  240,  122,  122,  122,  122,  121,  121,  120,  120,
- /*   150 */   120,  119,  116,  448,  125,  126,   80, 1221, 1221, 1054,
- /*   160 */  1057, 1044, 1044,  123,  123,  124,  124,  124,  124,  142,
- /*   170 */   294, 1196,  343,  452,  120,  120,  120,  119,  116,  448,
- /*   180 */   127, 1196, 1197, 1198,  148,  445,  444,  572,  119,  116,
+ /*   150 */   120,  119,  116,  448,  125,  126,   80, 1216, 1216, 1051,
+ /*   160 */  1054, 1041, 1041,  123,  123,  124,  124,  124,  124,  142,
+ /*   170 */   294, 1192,  343,  452,  120,  120,  120,  119,  116,  448,
+ /*   180 */   127, 1192, 1193, 1192,  148,  445,  444,  572,  119,  116,
  /*   190 */   448,  124,  124,  124,  124,  117,  122,  122,  122,  122,
  /*   200 */   121,  121,  120,  120,  120,  119,  116,  448,  458,  113,
  /*   210 */    13,   13,  550,  122,  122,  122,  122,  121,  121,  120,
- /*   220 */   120,  120,  119,  116,  448,  426,  316,  563, 1196, 1197,
- /*   230 */  1198,  149, 1228,  413, 1228,  124,  124,  124,  124,  122,
+ /*   220 */   120,  120,  119,  116,  448,  426,  316,  563, 1192, 1193,
+ /*   230 */  1192,  149, 1224,  413, 1224,  124,  124,  124,  124,  122,
  /*   240 */   122,  122,  122,  121,  121,  120,  120,  120,  119,  116,
- /*   250 */   448,  469,  346, 1041, 1041, 1055, 1058,  125,  126,   80,
- /*   260 */  1221, 1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,
- /*   270 */   124,  124, 1283,  526,  222, 1196,  572,  413,  224,  518,
+ /*   250 */   448,  469,  346, 1038, 1038, 1052, 1055,  125,  126,   80,
+ /*   260 */  1216, 1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,
+ /*   270 */   124,  124, 1279,  526,  222, 1192,  572,  413,  224,  518,
  /*   280 */   175,   82,   83,  122,  122,  122,  122,  121,  121,  120,
- /*   290 */   120,  120,  119,  116,  448, 1011,   16,   16, 1196,  133,
- /*   300 */   133,  125,  126,   80, 1221, 1221, 1054, 1057, 1044, 1044,
+ /*   290 */   120,  120,  119,  116,  448, 1009,   16,   16, 1192,  133,
+ /*   300 */   133,  125,  126,   80, 1216, 1216, 1051, 1054, 1041, 1041,
  /*   310 */   123,  123,  124,  124,  124,  124,  122,  122,  122,  122,
- /*   320 */   121,  121,  120,  120,  120,  119,  116,  448, 1045,  550,
- /*   330 */  1196,  377, 1196, 1197, 1198,  252, 1438,  403,  508,  505,
- /*   340 */   504,  111,  564,  570,    4,  930,  930,  437,  503,  344,
- /*   350 */   464,  330,  364,  398, 1241, 1196, 1197, 1198,  567,  572,
+ /*   320 */   121,  121,  120,  120,  120,  119,  116,  448, 1042,  550,
+ /*   330 */  1192,  377, 1192, 1193, 1192,  252, 1433,  403,  508,  505,
+ /*   340 */   504,  111,  564,  570,    4,  928,  928,  437,  503,  344,
+ /*   350 */   464,  330,  364,  398, 1237, 1192, 1193, 1192,  567,  572,
  /*   360 */   122,  122,  122,  122,  121,  121,  120,  120,  120,  119,
- /*   370 */   116,  448,  284,  284,  373, 1584, 1611,  445,  444,  154,
- /*   380 */   413,  449,   71,   71, 1290,  569, 1225, 1196, 1197, 1198,
- /*   390 */    85, 1227,  271,  561,  547,  519, 1565,  572,   98, 1226,
- /*   400 */     6, 1282,  476,  142,  125,  126,   80, 1221, 1221, 1054,
- /*   410 */  1057, 1044, 1044,  123,  123,  124,  124,  124,  124,  554,
- /*   420 */    13,   13, 1031,  511, 1228, 1196, 1228,  553,  109,  109,
- /*   430 */   222,  572, 1242,  175,  572,  431,  110,  197,  449,  574,
- /*   440 */   573,  434, 1556, 1021,  325,  555, 1196,  270,  287,  372,
+ /*   370 */   116,  448,  284,  284,  373, 1578, 1604,  445,  444,  154,
+ /*   380 */   413,  449,   71,   71, 1286,  569, 1221, 1192, 1193, 1192,
+ /*   390 */    85, 1223,  271,  561,  547,  519, 1559,  572,   98, 1222,
+ /*   400 */     6, 1278,  476,  142,  125,  126,   80, 1216, 1216, 1051,
+ /*   410 */  1054, 1041, 1041,  123,  123,  124,  124,  124,  124,  554,
+ /*   420 */    13,   13, 1028,  511, 1224, 1192, 1224,  553,  109,  109,
+ /*   430 */   222,  572, 1238,  175,  572,  431,  110,  197,  449,  573,
+ /*   440 */   449,  434, 1550, 1018,  325,  555, 1192,  270,  287,  372,
  /*   450 */   514,  367,  513,  257,   71,   71,  547,   71,   71,  363,
- /*   460 */   316,  563, 1617,  122,  122,  122,  122,  121,  121,  120,
- /*   470 */   120,  120,  119,  116,  448, 1021, 1021, 1023, 1024,   27,
- /*   480 */   284,  284, 1196, 1197, 1198, 1162,  572, 1616,  413,  905,
- /*   490 */   190,  554,  360,  569,  554,  941,  537,  521, 1162,  520,
- /*   500 */   417, 1162,  556, 1196, 1197, 1198,  572,  548, 1558,   51,
- /*   510 */    51,  214,  125,  126,   80, 1221, 1221, 1054, 1057, 1044,
- /*   520 */  1044,  123,  123,  124,  124,  124,  124, 1196,  478,  135,
- /*   530 */   135,  413,  284,  284, 1494,  509,  121,  121,  120,  120,
- /*   540 */   120,  119,  116,  448, 1011,  569,  522,  217,  545, 1565,
- /*   550 */   316,  563,  142,    6,  536,  125,  126,   80, 1221, 1221,
- /*   560 */  1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,  124,
- /*   570 */  1559,  122,  122,  122,  122,  121,  121,  120,  120,  120,
- /*   580 */   119,  116,  448,  489, 1196, 1197, 1198,  486,  281, 1271,
- /*   590 */   961,  252, 1196,  377,  508,  505,  504, 1196,  344,  575,
- /*   600 */  1196,  575,  413,  292,  503,  961,  880,  191,  484,  316,
+ /*   460 */   316,  563, 1610,  122,  122,  122,  122,  121,  121,  120,
+ /*   470 */   120,  120,  119,  116,  448, 1018, 1018, 1020, 1021,   27,
+ /*   480 */   284,  284, 1192, 1193, 1192, 1158,  572, 1609,  413,  903,
+ /*   490 */   190,  554,  360,  569,  554,  939,  537,  521, 1158,  520,
+ /*   500 */   417, 1158,  556, 1192, 1193, 1192,  572,  548, 1552,   51,
+ /*   510 */    51,  214,  125,  126,   80, 1216, 1216, 1051, 1054, 1041,
+ /*   520 */  1041,  123,  123,  124,  124,  124,  124, 1192,  478,  135,
+ /*   530 */   135,  413,  284,  284, 1488,  509,  121,  121,  120,  120,
+ /*   540 */   120,  119,  116,  448, 1009,  569,  522,  217,  545, 1559,
+ /*   550 */   316,  563,  142,    6,  536,  125,  126,   80, 1216, 1216,
+ /*   560 */  1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,  124,
+ /*   570 */  1553,  122,  122,  122,  122,  121,  121,  120,  120,  120,
+ /*   580 */   119,  116,  448,  489, 1192, 1193, 1192,  486,  281, 1267,
+ /*   590 */   959,  252, 1192,  377,  508,  505,  504, 1192,  344,  574,
+ /*   600 */  1192,  574,  413,  292,  503,  959,  878,  191,  484,  316,
  /*   610 */   563,  388,  290,  384,  122,  122,  122,  122,  121,  121,
- /*   620 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1221,
- /*   630 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*   640 */   124,  413,  398, 1140, 1196,  873,  100,  284,  284, 1196,
- /*   650 */  1197, 1198,  377, 1097, 1196, 1197, 1198, 1196, 1197, 1198,
- /*   660 */   569,  459,   32,  377,  233,  125,  126,   80, 1221, 1221,
- /*   670 */  1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,  124,
- /*   680 */  1437,  963,  572,  228,  962,  122,  122,  122,  122,  121,
- /*   690 */   121,  120,  120,  120,  119,  116,  448, 1162,  228, 1196,
- /*   700 */   157, 1196, 1197, 1198, 1557,   13,   13,  301,  961, 1236,
- /*   710 */  1162,  153,  413, 1162,  377, 1587, 1180,    5,  373, 1584,
- /*   720 */   433, 1242,    3,  961,  122,  122,  122,  122,  121,  121,
- /*   730 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1221,
- /*   740 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*   750 */   124,  413,  208,  571, 1196, 1032, 1196, 1197, 1198, 1196,
- /*   760 */   392,  856,  155, 1556,  286,  406, 1102, 1102,  492,  572,
- /*   770 */   469,  346, 1323, 1323, 1556,  125,  126,   80, 1221, 1221,
- /*   780 */  1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,  124,
+ /*   620 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1216,
+ /*   630 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*   640 */   124,  413,  398, 1136, 1192,  871,  100,  284,  284, 1192,
+ /*   650 */  1193, 1192,  377, 1093, 1192, 1193, 1192, 1192, 1193, 1192,
+ /*   660 */   569,  459,   32,  377,  233,  125,  126,   80, 1216, 1216,
+ /*   670 */  1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,  124,
+ /*   680 */  1432,  961,  572,  228,  960,  122,  122,  122,  122,  121,
+ /*   690 */   121,  120,  120,  120,  119,  116,  448, 1158,  228, 1192,
+ /*   700 */   157, 1192, 1193, 1192, 1551,   13,   13,  301,  959, 1232,
+ /*   710 */  1158,  153,  413, 1158,  377, 1581, 1176,    5,  373, 1578,
+ /*   720 */   433, 1238,    3,  959,  122,  122,  122,  122,  121,  121,
+ /*   730 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1216,
+ /*   740 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*   750 */   124,  413,  208,  571, 1192, 1029, 1192, 1193, 1192, 1192,
+ /*   760 */   392,  854,  155, 1550,  286,  406, 1098, 1098,  492,  572,
+ /*   770 */   469,  346, 1319, 1319, 1550,  125,  126,   80, 1216, 1216,
+ /*   780 */  1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,  124,
  /*   790 */   129,  572,   13,   13,  378,  122,  122,  122,  122,  121,
  /*   800 */   121,  120,  120,  120,  119,  116,  448,  302,  572,  457,
- /*   810 */   532, 1196, 1197, 1198,   13,   13, 1196, 1197, 1198, 1301,
- /*   820 */   467, 1271,  413, 1321, 1321, 1556, 1016,  457,  456,  200,
- /*   830 */   299,   71,   71, 1269,  122,  122,  122,  122,  121,  121,
- /*   840 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1221,
- /*   850 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*   860 */   124,  413,  227, 1077, 1162,  284,  284,  423,  312,  278,
- /*   870 */   278,  285,  285, 1423,  410,  409,  386, 1162,  569,  572,
- /*   880 */  1162, 1200,  569, 1604,  569,  125,  126,   80, 1221, 1221,
- /*   890 */  1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,  124,
- /*   900 */   457, 1486,   13,   13, 1540,  122,  122,  122,  122,  121,
+ /*   810 */   532, 1192, 1193, 1192,   13,   13, 1192, 1193, 1192, 1297,
+ /*   820 */   467, 1267,  413, 1317, 1317, 1550, 1014,  457,  456,  200,
+ /*   830 */   299,   71,   71, 1265,  122,  122,  122,  122,  121,  121,
+ /*   840 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1216,
+ /*   850 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*   860 */   124,  413,  227, 1073, 1158,  284,  284,  423,  312,  278,
+ /*   870 */   278,  285,  285, 1419,  410,  409,  386, 1158,  569,  572,
+ /*   880 */  1158, 1195,  569, 1598,  569,  125,  126,   80, 1216, 1216,
+ /*   890 */  1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,  124,
+ /*   900 */   457, 1480,   13,   13, 1534,  122,  122,  122,  122,  121,
  /*   910 */   121,  120,  120,  120,  119,  116,  448,  201,  572,  358,
- /*   920 */  1590,  579,    2, 1249,  844,  845,  846, 1566,  317, 1216,
- /*   930 */   146,    6,  413,  255,  254,  253,  206, 1331,    9, 1200,
+ /*   920 */  1584,  578,    2, 1245,  842,  843,  844, 1560,  317, 1211,
+ /*   930 */   146,    6,  413,  255,  254,  253,  206, 1327,    9, 1195,
  /*   940 */   262,   71,   71,  428,  122,  122,  122,  122,  121,  121,
- /*   950 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1221,
- /*   960 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*   970 */   124,  572,  284,  284,  572, 1217,  413,  578,  313, 1249,
- /*   980 */   353, 1300,  356,  423,  317,  569,  146,  495,  529, 1647,
- /*   990 */   399,  375,  495, 1331,   70,   70, 1299,   71,   71,  240,
- /*  1000 */  1329,  104,   80, 1221, 1221, 1054, 1057, 1044, 1044,  123,
+ /*   950 */   120,  120,  120,  119,  116,  448,  125,  126,   80, 1216,
+ /*   960 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*   970 */   124,  572,  284,  284,  572, 1212,  413,  577,  313, 1245,
+ /*   980 */   353, 1296,  356,  423,  317,  569,  146,  495,  529, 1641,
+ /*   990 */   399,  375,  495, 1327,   70,   70, 1295,   71,   71,  240,
+ /*  1000 */  1325,  104,   80, 1216, 1216, 1051, 1054, 1041, 1041,  123,
  /*  1010 */   123,  124,  124,  124,  124,  122,  122,  122,  122,  121,
- /*  1020 */   121,  120,  120,  120,  119,  116,  448, 1118,  284,  284,
- /*  1030 */   432,  452, 1529, 1217,  443,  284,  284, 1493, 1356,  311,
- /*  1040 */   478,  569, 1119,  975,  495,  495,  217, 1267,  569, 1542,
- /*  1050 */   572,  976,  207,  572, 1031,  240,  387, 1120,  523,  122,
+ /*  1020 */   121,  120,  120,  120,  119,  116,  448, 1114,  284,  284,
+ /*  1030 */   432,  452, 1523, 1212,  443,  284,  284, 1487, 1352,  311,
+ /*  1040 */   478,  569, 1115,  973,  495,  495,  217, 1263,  569, 1536,
+ /*  1050 */   572,  974,  207,  572, 1028,  240,  387, 1116,  523,  122,
  /*  1060 */   122,  122,  122,  121,  121,  120,  120,  120,  119,  116,
- /*  1070 */   448, 1022,  107,   71,   71, 1021,   13,   13,  916,  572,
- /*  1080 */  1499,  572,  284,  284,   97,  530,  495,  452,  917, 1330,
- /*  1090 */  1326,  549,  413,  284,  284,  569,  151,  209, 1499, 1501,
- /*  1100 */   262,  454,   55,   55,   56,   56,  569, 1021, 1021, 1023,
- /*  1110 */   447,  336,  413,  531,   12,  295,  125,  126,   80, 1221,
- /*  1120 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*  1130 */   124,  351,  413,  868, 1538, 1217,  125,  126,   80, 1221,
- /*  1140 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*  1150 */   124, 1141, 1645,  478, 1645,  375,  125,  114,   80, 1221,
- /*  1160 */  1221, 1054, 1057, 1044, 1044,  123,  123,  124,  124,  124,
- /*  1170 */   124, 1499,  333,  478,  335,  122,  122,  122,  122,  121,
- /*  1180 */   121,  120,  120,  120,  119,  116,  448,  203, 1423,  572,
- /*  1190 */  1298,  868,  468, 1217,  440,  122,  122,  122,  122,  121,
- /*  1200 */   121,  120,  120,  120,  119,  116,  448,  557, 1141, 1646,
- /*  1210 */   543, 1646,   15,   15,  896,  122,  122,  122,  122,  121,
+ /*  1070 */   448, 1019,  107,   71,   71, 1018,   13,   13,  914,  572,
+ /*  1080 */  1493,  572,  284,  284,   97,  530,  495,  452,  915, 1326,
+ /*  1090 */  1322,  549,  413,  284,  284,  569,  151,  209, 1493, 1495,
+ /*  1100 */   262,  454,   55,   55,   56,   56,  569, 1018, 1018, 1020,
+ /*  1110 */   447,  336,  413,  531,   12,  295,  125,  126,   80, 1216,
+ /*  1120 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*  1130 */   124,  351,  413,  866, 1532, 1212,  125,  126,   80, 1216,
+ /*  1140 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*  1150 */   124, 1137, 1639,  478, 1639,  375,  125,  114,   80, 1216,
+ /*  1160 */  1216, 1051, 1054, 1041, 1041,  123,  123,  124,  124,  124,
+ /*  1170 */   124, 1493,  333,  478,  335,  122,  122,  122,  122,  121,
+ /*  1180 */   121,  120,  120,  120,  119,  116,  448,  203, 1419,  572,
+ /*  1190 */  1294,  866,  468, 1212,  440,  122,  122,  122,  122,  121,
+ /*  1200 */   121,  120,  120,  120,  119,  116,  448,  557, 1137, 1640,
+ /*  1210 */   543, 1640,   15,   15,  894,  122,  122,  122,  122,  121,
  /*  1220 */   121,  120,  120,  120,  119,  116,  448,  572,  298,  542,
- /*  1230 */  1139, 1423, 1563, 1564, 1335,  413,    6,    6, 1173, 1272,
- /*  1240 */   419,  320,  284,  284, 1423,  512,  569,  529,  300,  461,
- /*  1250 */    43,   43,  572,  897,   12,  569,  334,  482,  429,  411,
- /*  1260 */   126,   80, 1221, 1221, 1054, 1057, 1044, 1044,  123,  123,
- /*  1270 */   124,  124,  124,  124,  572,   57,   57,  288, 1196, 1423,
- /*  1280 */   500,  462,  396,  396,  395,  273,  393, 1139, 1562,  853,
- /*  1290 */  1173,  411,    6,  572,  321, 1162,  474,   44,   44, 1561,
- /*  1300 */  1118,  430,  234,    6,  323,  256,  544,  256, 1162,  435,
- /*  1310 */   572, 1162,  322,   17,  491, 1119,   58,   58,  122,  122,
+ /*  1230 */  1135, 1419, 1557, 1558, 1331,  413,    6,    6, 1169, 1268,
+ /*  1240 */   419,  320,  284,  284, 1419,  512,  569,  529,  300,  461,
+ /*  1250 */    43,   43,  572,  895,   12,  569,  334,  482,  429,  411,
+ /*  1260 */   126,   80, 1216, 1216, 1051, 1054, 1041, 1041,  123,  123,
+ /*  1270 */   124,  124,  124,  124,  572,   57,   57,  288, 1192, 1419,
+ /*  1280 */   500,  462,  396,  396,  395,  273,  393, 1135, 1556,  851,
+ /*  1290 */  1169,  411,    6,  572,  321, 1158,  474,   44,   44, 1555,
+ /*  1300 */  1114,  430,  234,    6,  323,  256,  544,  256, 1158,  435,
+ /*  1310 */   572, 1158,  322,   17,  491, 1115,   58,   58,  122,  122,
  /*  1320 */   122,  122,  121,  121,  120,  120,  120,  119,  116,  448,
- /*  1330 */  1120,  216,  485,   59,   59, 1196, 1197, 1198,  111,  564,
+ /*  1330 */  1116,  216,  485,   59,   59, 1192, 1193, 1192,  111,  564,
  /*  1340 */   324,    4,  236,  460,  530,  572,  237,  460,  572,  441,
- /*  1350 */   168,  560,  424,  141,  483,  567,  572,  293,  572, 1099,
- /*  1360 */   572,  293,  572, 1099,  535,  572,  876,    8,   60,   60,
+ /*  1350 */   168,  560,  424,  141,  483,  567,  572,  293,  572, 1095,
+ /*  1360 */   572,  293,  572, 1095,  535,  572,  874,    8,   60,   60,
  /*  1370 */   235,   61,   61,  572,  418,  572,  418,  572,  449,   62,
  /*  1380 */    62,   45,   45,   46,   46,   47,   47,  199,   49,   49,
  /*  1390 */   561,  572,  363,  572,  100,  490,   50,   50,   63,   63,
- /*  1400 */    64,   64,  565,  419,  539,  414,  572, 1031,  572,  538,
- /*  1410 */   316,  563,  316,  563,   65,   65,   14,   14,  572, 1031,
- /*  1420 */   572,  516,  936,  876, 1022,  109,  109,  935, 1021,   66,
- /*  1430 */    66,  131,  131,  110,  455,  449,  574,  573,  420,  177,
- /*  1440 */  1021,  132,  132,   67,   67,  572,  471,  572,  936,  475,
- /*  1450 */  1368,  283,  226,  935,  315, 1367,  411,  572,  463,  411,
- /*  1460 */  1021, 1021, 1023,  239,  411,   86,  213, 1354,   52,   52,
- /*  1470 */    68,   68, 1021, 1021, 1023, 1024,   27, 1589, 1184,  451,
- /*  1480 */    69,   69,  288,   97,  108, 1545,  106,  396,  396,  395,
- /*  1490 */   273,  393,  572,  883,  853,  887,  572,  111,  564,  470,
- /*  1500 */     4,  572,  152,   30,   38,  572, 1136,  234,  400,  323,
+ /*  1400 */    64,   64,  565,  419,  539,  414,  572, 1028,  572,  538,
+ /*  1410 */   316,  563,  316,  563,   65,   65,   14,   14,  572, 1028,
+ /*  1420 */   572,  516,  934,  874, 1019,  109,  109,  933, 1018,   66,
+ /*  1430 */    66,  131,  131,  110,  455,  449,  573,  449,  420,  177,
+ /*  1440 */  1018,  132,  132,   67,   67,  572,  471,  572,  934,  475,
+ /*  1450 */  1364,  283,  226,  933,  315, 1363,  411,  572,  463,  411,
+ /*  1460 */  1018, 1018, 1020,  239,  411,   86,  213, 1350,   52,   52,
+ /*  1470 */    68,   68, 1018, 1018, 1020, 1021,   27, 1583, 1180,  451,
+ /*  1480 */    69,   69,  288,   97,  108, 1539,  106,  396,  396,  395,
+ /*  1490 */   273,  393,  572,  881,  851,  885,  572,  111,  564,  470,
+ /*  1500 */     4,  572,  152,   30,   38,  572, 1132,  234,  400,  323,
  /*  1510 */   111,  564,  531,    4,  567,   53,   53,  322,  572,  163,
  /*  1520 */   163,  572,  341,  472,  164,  164,  337,  567,   76,   76,
- /*  1530 */   572,  289, 1518,  572,   31, 1517,  572,  449,  342,  487,
- /*  1540 */   100,   54,   54,  348,   72,   72,  296,  236, 1084,  561,
- /*  1550 */   449,  883, 1364,  134,  134,  168,   73,   73,  141,  161,
- /*  1560 */   161, 1578,  561,  539,  572,  319,  572,  352,  540, 1013,
- /*  1570 */   477,  261,  261,  895,  894,  235,  539,  572, 1031,  572,
+ /*  1530 */   572,  289, 1512,  572,   31, 1511,  572,  449,  342,  487,
+ /*  1540 */   100,   54,   54,  348,   72,   72,  296,  236, 1080,  561,
+ /*  1550 */   449,  881, 1360,  134,  134,  168,   73,   73,  141,  161,
+ /*  1560 */   161, 1572,  561,  539,  572,  319,  572,  352,  540, 1011,
+ /*  1570 */   477,  261,  261,  893,  892,  235,  539,  572, 1028,  572,
  /*  1580 */   479,  538,  261,  371,  109,  109,  525,  136,  136,  130,
- /*  1590 */   130, 1031,  110,  370,  449,  574,  573,  109,  109, 1021,
- /*  1600 */   162,  162,  156,  156,  572,  110, 1084,  449,  574,  573,
- /*  1610 */   414,  355, 1021,  572,  357,  316,  563,  572,  347,  572,
- /*  1620 */   100,  501,  361,  258,  100,  902,  903,  140,  140,  359,
- /*  1630 */  1314, 1021, 1021, 1023, 1024,   27,  139,  139,  366,  455,
- /*  1640 */   137,  137,  138,  138, 1021, 1021, 1023, 1024,   27, 1184,
- /*  1650 */   451,  572,  376,  288,  111,  564, 1025,    4,  396,  396,
- /*  1660 */   395,  273,  393,  572, 1145,  853,  572, 1080,  572,  258,
- /*  1670 */   496,  567,  572,  211,   75,   75,  559,  966,  234,  261,
- /*  1680 */   323,  111,  564,  933,    4,  113,   77,   77,  322,   74,
- /*  1690 */    74,   42,   42, 1377,  449,   48,   48, 1422,  567,  978,
- /*  1700 */   979, 1096, 1095, 1096, 1095,  866,  561,  150,  934, 1350,
- /*  1710 */   113, 1362,  558, 1428, 1025, 1279, 1270, 1258,  236, 1257,
- /*  1720 */  1259,  449, 1597, 1347,  308,  276,  168,  309,   11,  141,
- /*  1730 */   397,  310,  232,  561, 1409, 1031,  339,  291,  329,  219,
- /*  1740 */   340,  109,  109,  940,  297, 1414,  235,  345,  481,  110,
- /*  1750 */   506,  449,  574,  573,  332, 1413, 1021,  404, 1297,  369,
- /*  1760 */   223, 1490, 1031, 1489, 1359, 1360, 1358, 1357,  109,  109,
- /*  1770 */   204, 1600, 1236,  562,  265,  218,  110,  205,  449,  574,
- /*  1780 */   573,  414,  391, 1021, 1537,  179,  316,  563, 1021, 1021,
- /*  1790 */  1023, 1024,   27,  230, 1535, 1233,   79,  564,   85,    4,
- /*  1800 */   422,  215,  552,   81,   84,  188, 1410,  128, 1404,  550,
- /*  1810 */   455,   35,  328,  567,  173, 1021, 1021, 1023, 1024,   27,
- /*  1820 */   181, 1495, 1397,  331,  465,  183,  184,  185,  186,  466,
- /*  1830 */   499,  242,   98,  402, 1416, 1418,  449, 1415,  473,   36,
- /*  1840 */   192,  488,  405, 1506,  246,   91,  494,  196,  561, 1484,
- /*  1850 */   350,  497,  277,  354,  248,  249,  111,  564, 1260,    4,
- /*  1860 */   250,  407,  515,  436, 1317, 1308,   93, 1316, 1315,  887,
- /*  1870 */  1307,  224, 1583,  567,  438,  524,  439, 1031,  263,  264,
- /*  1880 */   442, 1615,   10,  109,  109, 1287,  408, 1614, 1286,  368,
- /*  1890 */  1285,  110, 1613,  449,  574,  573,  449,  306, 1021,  307,
- /*  1900 */   374, 1382, 1569, 1470, 1381,  385,  105,  314,  561,   99,
- /*  1910 */  1568,  534,   34,  576, 1190,  272, 1340,  551,  383,  274,
- /*  1920 */  1339,  210,  389,  390,  275,  577, 1255, 1250,  415,  165,
- /*  1930 */  1021, 1021, 1023, 1024,   27,  147, 1522, 1031,  166, 1523,
- /*  1940 */   416, 1521,  178,  109,  109, 1520,  304,  167,  840,  450,
- /*  1950 */   220,  110,  221,  449,  574,  573,  212,   78, 1021,  318,
- /*  1960 */   231, 1094, 1092,  144,  180,  326,  169, 1216,  241,  182,
- /*  1970 */   919,  338,  238, 1108,  187,  170,  171,  425,  427,  189,
- /*  1980 */    87,   88,   89,   90,  172, 1111,  243, 1107,  244,  158,
- /*  1990 */  1021, 1021, 1023, 1024,   27,   18,  245, 1230,  493,  349,
- /*  2000 */  1100,  261,  247,  193,  194,   37,  370,  855,  498,  251,
- /*  2010 */   195,  510,   92,   19,  174,  362,  502,   20,  507,  885,
- /*  2020 */   365,  898,   94,  305,  159,   95,  517,   96, 1178,  160,
- /*  2030 */  1060, 1147,   39, 1146,  225,  280,  282,  970,  198,  964,
- /*  2040 */   113, 1164, 1168,  260, 1166,   21, 1172,    7,   22, 1152,
- /*  2050 */    33,   23,   24,   25, 1171,  546,   26,  202,  100,  102,
- /*  2060 */  1075,  103, 1061, 1059, 1063, 1117, 1064, 1116,  266,  267,
- /*  2070 */    28,   40,  929, 1026,  867,  112,   29,  568,  394,  143,
- /*  2080 */  1186,  268,  176, 1185,  269, 1246, 1246, 1246, 1246, 1246,
- /*  2090 */  1246, 1246, 1246, 1246, 1246, 1606, 1246, 1246, 1246, 1246,
- /*  2100 */  1605,
+ /*  1590 */   130, 1028,  110,  370,  449,  573,  449,  109,  109, 1018,
+ /*  1600 */   162,  162,  156,  156,  572,  110, 1080,  449,  573,  449,
+ /*  1610 */   414,  355, 1018,  572,  357,  316,  563,  572,  347,  572,
+ /*  1620 */   100,  501,  361,  258,  100,  900,  901,  140,  140,  359,
+ /*  1630 */  1310, 1018, 1018, 1020, 1021,   27,  139,  139,  366,  455,
+ /*  1640 */   137,  137,  138,  138, 1018, 1018, 1020, 1021,   27, 1180,
+ /*  1650 */   451,  572,  376,  288,  111,  564, 1022,    4,  396,  396,
+ /*  1660 */   395,  273,  393,  572, 1141,  851,  572, 1076,  572,  258,
+ /*  1670 */   496,  567,  572,  211,   75,   75,  559,  964,  234,  261,
+ /*  1680 */   323,  111,  564,  931,    4,  113,   77,   77,  322,   74,
+ /*  1690 */    74,   42,   42, 1373,  449,   48,   48, 1418,  567,  976,
+ /*  1700 */   977, 1092, 1091, 1092, 1091,  864,  561,  150,  932, 1346,
+ /*  1710 */   113, 1358,  558, 1423, 1022, 1275, 1266, 1254,  236, 1253,
+ /*  1720 */  1255,  449, 1591, 1343,  308,  276,  168,  309,   11,  141,
+ /*  1730 */   397,  310,  232,  561, 1405, 1028,  339,  291,  329,  219,
+ /*  1740 */   340,  109,  109,  938,  297, 1410,  235,  345,  481,  110,
+ /*  1750 */   506,  449,  573,  449,  332, 1409, 1018,  404, 1293,  369,
+ /*  1760 */   223, 1484, 1028, 1483, 1355, 1356, 1354, 1353,  109,  109,
+ /*  1770 */   204, 1594, 1232,  562,  265,  218,  110,  205,  449,  573,
+ /*  1780 */   449,  414,  391, 1018, 1531,  179,  316,  563, 1018, 1018,
+ /*  1790 */  1020, 1021,   27,  230, 1529, 1229,   79,  564,   85,    4,
+ /*  1800 */   422,  215,  552,   81,   84,  188, 1406,  128, 1400,  550,
+ /*  1810 */   455,   35,  328,  567,  173, 1018, 1018, 1020, 1021,   27,
+ /*  1820 */   181, 1489, 1393,  331,  465,  183,  184,  185,  186,  466,
+ /*  1830 */   499,  242,   98,  402, 1412, 1414,  449, 1411,  473,   36,
+ /*  1840 */   192,  488,  405, 1500,  246,   91,  494,  196,  561, 1478,
+ /*  1850 */   350,  497,  277,  354,  248,  249,  111,  564, 1256,    4,
+ /*  1860 */   250,  407,  515,  436, 1313, 1304,   93, 1312, 1311,  885,
+ /*  1870 */  1303,  224, 1577,  567,  438,  524,  439, 1028,  263,  264,
+ /*  1880 */   442, 1608,   10,  109,  109, 1283,  408, 1607, 1282,  368,
+ /*  1890 */  1281,  110, 1606,  449,  573,  449,  449,  306, 1018,  307,
+ /*  1900 */   374, 1378, 1563, 1465, 1377,  385,  105,  314,  561,   99,
+ /*  1910 */  1562,  534,   34,  575, 1186,  272, 1336,  551,  383,  274,
+ /*  1920 */  1335,  210,  389,  390,  275,  576, 1251, 1246,  415,  165,
+ /*  1930 */  1018, 1018, 1020, 1021,   27,  147, 1516, 1028,  166, 1517,
+ /*  1940 */   416, 1515,  178,  109,  109, 1514,  304,  167,  838,  450,
+ /*  1950 */   220,  110,  221,  449,  573,  449,  212,   78, 1018,  318,
+ /*  1960 */   231, 1090, 1088,  144,  180,  326,  169, 1211,  241,  182,
+ /*  1970 */   917,  338,  238, 1104,  187,  170,  171,  425,  427,  189,
+ /*  1980 */    87,   88,   89,   90,  172, 1107,  243, 1103,  244,  158,
+ /*  1990 */  1018, 1018, 1020, 1021,   27,   18,  245, 1226,  493,  349,
+ /*  2000 */  1096,  261,  247,  193,  194,   37,  370,  853,  498,  251,
+ /*  2010 */   195,  510,   92,   19,  174,  362,  502,   20,  507,  883,
+ /*  2020 */   365,  896,   94,  305,  159,   95,  517,   96, 1174,  160,
+ /*  2030 */  1057, 1143,   39, 1142,  225,  280,  282,  968,  198,  962,
+ /*  2040 */   113, 1160, 1164,  260, 1162,   21, 1168,    7,   22, 1148,
+ /*  2050 */    33,   23,   24,   25, 1167,  546,   26,  202,  100,  102,
+ /*  2060 */  1071,  103, 1058, 1056, 1060, 1113, 1061, 1112,  266,  267,
+ /*  2070 */    28,   40,  927, 1023,  865,  112,   29,  568,  394,  143,
+ /*  2080 */  1182,  268,  176, 1181,  269, 1242, 1242, 1242, 1242, 1242,
+ /*  2090 */  1242, 1242, 1242, 1242, 1242, 1599,
 };
 static const YYCODETYPE yy_lookahead[] = {
  /*     0 */   193,  193,  193,  274,  275,  276,  193,  274,  275,  276,
@@ -167828,7 +168738,7 @@ static const YYCODETYPE yy_lookahead[] = {
  /*  2070 */    22,   22,  135,   23,   23,   22,   22,   25,   15,   23,
  /*  2080 */     1,  141,   25,    1,  141,  319,  319,  319,  319,  319,
  /*  2090 */   319,  319,  319,  319,  319,  141,  319,  319,  319,  319,
- /*  2100 */   141,  319,  319,  319,  319,  319,  319,  319,  319,  319,
+ /*  2100 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
  /*  2110 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
  /*  2120 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
  /*  2130 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
@@ -167846,9 +168756,9 @@ static const YYCODETYPE yy_lookahead[] = {
  /*  2250 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
  /*  2260 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
  /*  2270 */   319,  319,  319,  319,  319,  319,  319,  319,  319,  319,
- /*  2280 */   319,  319,  319,  319,  319,  319,
+ /*  2280 */   319,
 };
-#define YY_SHIFT_COUNT    (579)
+#define YY_SHIFT_COUNT    (578)
 #define YY_SHIFT_MIN      (0)
 #define YY_SHIFT_MAX      (2082)
 static const unsigned short int yy_shift_ofst[] = {
@@ -167868,12 +168778,12 @@ static const unsigned short int yy_shift_ofst[] = {
  /*   130 */   137,  181,  181,  181,  181,  181,  181,  181,   94,  430,
  /*   140 */    66,   65,  112,  366,  533,  533,  740, 1261,  533,  533,
  /*   150 */    79,   79,  533,  412,  412,  412,   77,  412,  123,  113,
- /*   160 */   113,   22,   22, 2101, 2101,  328,  328,  328,  239,  468,
+ /*   160 */   113,   22,   22, 2096, 2096,  328,  328,  328,  239,  468,
  /*   170 */   468,  468,  468, 1015, 1015,  409,  366, 1129, 1186,  533,
  /*   180 */   533,  533,  533,  533,  533,  533,  533,  533,  533,  533,
  /*   190 */   533,  533,  533,  533,  533,  533,  533,  533,  533,  969,
  /*   200 */   621,  621,  533,  642,  788,  788, 1228, 1228,  822,  822,
- /*   210 */    67, 1274, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 1307,
+ /*   210 */    67, 1274, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 1307,
  /*   220 */   954,  954,  585,  472,  640,  387,  695,  538,  541,  700,
  /*   230 */   533,  533,  533,  533,  533,  533,  533,  533,  533,  533,
  /*   240 */   222,  533,  533,  533,  533,  533,  533,  533,  533,  533,
@@ -167891,9 +168801,9 @@ static const unsigned short int yy_shift_ofst[] = {
  /*   360 */  1747, 1747, 1747, 1799, 1844, 1844, 1825, 1747, 1743, 1747,
  /*   370 */  1799, 1747, 1747, 1706, 1850, 1763, 1763, 1825, 1633, 1788,
  /*   380 */  1788, 1798, 1798, 1659, 1664, 1860, 1633, 1748, 1659, 1762,
- /*   390 */  1765, 1683, 1887, 1901, 1901, 1918, 1918, 1918, 2101, 2101,
- /*   400 */  2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101, 2101,
- /*   410 */  2101, 2101, 2101,  207, 1095,  331,  620,  903,  806, 1074,
+ /*   390 */  1765, 1683, 1887, 1901, 1901, 1918, 1918, 1918, 2096, 2096,
+ /*   400 */  2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096, 2096,
+ /*   410 */  2096, 2096, 2096,  207, 1095,  331,  620,  903,  806, 1074,
  /*   420 */  1483, 1432, 1481, 1322, 1370, 1394, 1515, 1291, 1546, 1547,
  /*   430 */  1557, 1595, 1598, 1599, 1434, 1453, 1618, 1462, 1567, 1489,
  /*   440 */  1644, 1654, 1616, 1660, 1548, 1549, 1682, 1685, 1597,  742,
@@ -167909,7 +168819,7 @@ static const unsigned short int yy_shift_ofst[] = {
  /*   540 */  1958, 2003, 1971, 1961, 2019, 2026, 2028, 2031, 2032, 2033,
  /*   550 */  2022, 1917, 1919, 2037, 2015, 2039, 2040, 2041, 2042, 2043,
  /*   560 */  2044, 2047, 2055, 2048, 2049, 2050, 2051, 2053, 2054, 2052,
- /*   570 */  1937, 1940, 1943, 1954, 1959, 2057, 2056, 2063, 2079, 2082,
+ /*   570 */  1937, 1940, 1943, 1954, 2057, 2056, 2063, 2079, 2082,
 };
 #define YY_REDUCE_COUNT (412)
 #define YY_REDUCE_MIN   (-271)
@@ -167959,64 +168869,64 @@ static const short yy_reduce_ofst[] = {
  /*   410 */  1738, 1744, 1740,
 };
 static const YYACTIONTYPE yy_default[] = {
- /*     0 */  1651, 1651, 1651, 1479, 1244, 1355, 1244, 1244, 1244, 1479,
- /*    10 */  1479, 1479, 1244, 1385, 1385, 1532, 1277, 1244, 1244, 1244,
- /*    20 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1478, 1244, 1244,
- /*    30 */  1244, 1244, 1567, 1567, 1244, 1244, 1244, 1244, 1244, 1244,
- /*    40 */  1244, 1244, 1394, 1244, 1401, 1244, 1244, 1244, 1244, 1244,
- /*    50 */  1480, 1481, 1244, 1244, 1244, 1531, 1533, 1496, 1408, 1407,
- /*    60 */  1406, 1405, 1514, 1373, 1399, 1392, 1396, 1474, 1475, 1473,
- /*    70 */  1477, 1481, 1480, 1244, 1395, 1442, 1458, 1441, 1244, 1244,
- /*    80 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*    90 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   100 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   110 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   120 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   130 */  1450, 1457, 1456, 1455, 1464, 1454, 1451, 1444, 1443, 1445,
- /*   140 */  1446, 1244, 1244, 1268, 1244, 1244, 1265, 1319, 1244, 1244,
- /*   150 */  1244, 1244, 1244, 1551, 1550, 1244, 1447, 1244, 1277, 1436,
- /*   160 */  1435, 1461, 1448, 1460, 1459, 1539, 1603, 1602, 1497, 1244,
- /*   170 */  1244, 1244, 1244, 1244, 1244, 1567, 1244, 1244, 1244, 1244,
- /*   180 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   190 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1375,
- /*   200 */  1567, 1567, 1244, 1277, 1567, 1567, 1376, 1376, 1273, 1273,
- /*   210 */  1379, 1244, 1546, 1346, 1346, 1346, 1346, 1355, 1346, 1244,
- /*   220 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   230 */  1244, 1244, 1244, 1244, 1536, 1534, 1244, 1244, 1244, 1244,
- /*   240 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   250 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   260 */  1244, 1244, 1244, 1351, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   270 */  1244, 1244, 1244, 1244, 1244, 1596, 1244, 1509, 1333, 1351,
- /*   280 */  1351, 1351, 1351, 1353, 1334, 1332, 1345, 1278, 1251, 1643,
- /*   290 */  1411, 1400, 1352, 1400, 1640, 1398, 1411, 1411, 1398, 1411,
- /*   300 */  1352, 1640, 1294, 1619, 1289, 1385, 1385, 1385, 1375, 1375,
- /*   310 */  1375, 1375, 1379, 1379, 1476, 1352, 1345, 1244, 1643, 1643,
- /*   320 */  1361, 1361, 1642, 1642, 1361, 1497, 1627, 1420, 1393, 1379,
- /*   330 */  1322, 1393, 1379, 1328, 1328, 1328, 1328, 1361, 1262, 1398,
- /*   340 */  1627, 1627, 1398, 1420, 1322, 1398, 1322, 1398, 1361, 1262,
- /*   350 */  1513, 1637, 1361, 1262, 1487, 1361, 1262, 1361, 1262, 1487,
- /*   360 */  1320, 1320, 1320, 1309, 1244, 1244, 1487, 1320, 1294, 1320,
- /*   370 */  1309, 1320, 1320, 1585, 1244, 1491, 1491, 1487, 1361, 1577,
- /*   380 */  1577, 1388, 1388, 1393, 1379, 1482, 1361, 1244, 1393, 1391,
- /*   390 */  1389, 1398, 1312, 1599, 1599, 1595, 1595, 1595, 1648, 1648,
- /*   400 */  1546, 1612, 1277, 1277, 1277, 1277, 1612, 1296, 1296, 1278,
- /*   410 */  1278, 1277, 1612, 1244, 1244, 1244, 1244, 1244, 1244, 1607,
- /*   420 */  1244, 1541, 1498, 1365, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   430 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1552, 1244,
- /*   440 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1425,
- /*   450 */  1244, 1247, 1543, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   460 */  1244, 1402, 1403, 1366, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   470 */  1244, 1417, 1244, 1244, 1244, 1412, 1244, 1244, 1244, 1244,
- /*   480 */  1244, 1244, 1244, 1244, 1639, 1244, 1244, 1244, 1244, 1244,
- /*   490 */  1244, 1512, 1511, 1244, 1244, 1363, 1244, 1244, 1244, 1244,
- /*   500 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1292,
- /*   510 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   520 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   530 */  1244, 1244, 1244, 1390, 1244, 1244, 1244, 1244, 1244, 1244,
- /*   540 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1582, 1380,
- /*   550 */  1244, 1244, 1244, 1244, 1630, 1244, 1244, 1244, 1244, 1244,
- /*   560 */  1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1244, 1623,
- /*   570 */  1336, 1427, 1244, 1426, 1430, 1266, 1244, 1256, 1244, 1244,
+ /*     0 */  1645, 1645, 1645, 1473, 1240, 1351, 1240, 1240, 1240, 1473,
+ /*    10 */  1473, 1473, 1240, 1381, 1381, 1526, 1273, 1240, 1240, 1240,
+ /*    20 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1472, 1240, 1240,
+ /*    30 */  1240, 1240, 1561, 1561, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*    40 */  1240, 1240, 1390, 1240, 1397, 1240, 1240, 1240, 1240, 1240,
+ /*    50 */  1474, 1475, 1240, 1240, 1240, 1525, 1527, 1490, 1404, 1403,
+ /*    60 */  1402, 1401, 1508, 1369, 1395, 1388, 1392, 1469, 1470, 1468,
+ /*    70 */  1623, 1475, 1474, 1240, 1391, 1437, 1453, 1436, 1240, 1240,
+ /*    80 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*    90 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   100 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   110 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   120 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   130 */  1445, 1452, 1451, 1450, 1459, 1449, 1446, 1439, 1438, 1440,
+ /*   140 */  1441, 1240, 1240, 1264, 1240, 1240, 1261, 1315, 1240, 1240,
+ /*   150 */  1240, 1240, 1240, 1545, 1544, 1240, 1442, 1240, 1273, 1431,
+ /*   160 */  1430, 1456, 1443, 1455, 1454, 1533, 1597, 1596, 1491, 1240,
+ /*   170 */  1240, 1240, 1240, 1240, 1240, 1561, 1240, 1240, 1240, 1240,
+ /*   180 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   190 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1371,
+ /*   200 */  1561, 1561, 1240, 1273, 1561, 1561, 1372, 1372, 1269, 1269,
+ /*   210 */  1375, 1240, 1540, 1342, 1342, 1342, 1342, 1351, 1342, 1240,
+ /*   220 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   230 */  1240, 1240, 1240, 1240, 1530, 1528, 1240, 1240, 1240, 1240,
+ /*   240 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   250 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   260 */  1240, 1240, 1240, 1347, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   270 */  1240, 1240, 1240, 1240, 1240, 1590, 1240, 1503, 1329, 1347,
+ /*   280 */  1347, 1347, 1347, 1349, 1330, 1328, 1341, 1274, 1247, 1637,
+ /*   290 */  1407, 1396, 1348, 1396, 1634, 1394, 1407, 1407, 1394, 1407,
+ /*   300 */  1348, 1634, 1290, 1612, 1285, 1381, 1381, 1381, 1371, 1371,
+ /*   310 */  1371, 1371, 1375, 1375, 1471, 1348, 1341, 1240, 1637, 1637,
+ /*   320 */  1357, 1357, 1636, 1636, 1357, 1491, 1620, 1416, 1389, 1375,
+ /*   330 */  1318, 1389, 1375, 1324, 1324, 1324, 1324, 1357, 1258, 1394,
+ /*   340 */  1620, 1620, 1394, 1416, 1318, 1394, 1318, 1394, 1357, 1258,
+ /*   350 */  1507, 1631, 1357, 1258, 1481, 1357, 1258, 1357, 1258, 1481,
+ /*   360 */  1316, 1316, 1316, 1305, 1240, 1240, 1481, 1316, 1290, 1316,
+ /*   370 */  1305, 1316, 1316, 1579, 1240, 1485, 1485, 1481, 1357, 1571,
+ /*   380 */  1571, 1384, 1384, 1389, 1375, 1476, 1357, 1240, 1389, 1387,
+ /*   390 */  1385, 1394, 1308, 1593, 1593, 1589, 1589, 1589, 1642, 1642,
+ /*   400 */  1540, 1605, 1273, 1273, 1273, 1273, 1605, 1292, 1292, 1274,
+ /*   410 */  1274, 1273, 1605, 1240, 1240, 1240, 1240, 1240, 1240, 1600,
+ /*   420 */  1240, 1535, 1492, 1361, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   430 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1546, 1240,
+ /*   440 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1421,
+ /*   450 */  1240, 1243, 1537, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   460 */  1240, 1398, 1399, 1362, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   470 */  1240, 1413, 1240, 1240, 1240, 1408, 1240, 1240, 1240, 1240,
+ /*   480 */  1240, 1240, 1240, 1240, 1633, 1240, 1240, 1240, 1240, 1240,
+ /*   490 */  1240, 1506, 1505, 1240, 1240, 1359, 1240, 1240, 1240, 1240,
+ /*   500 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1288,
+ /*   510 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   520 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   530 */  1240, 1240, 1240, 1386, 1240, 1240, 1240, 1240, 1240, 1240,
+ /*   540 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1576, 1376,
+ /*   550 */  1240, 1240, 1240, 1240, 1624, 1240, 1240, 1240, 1240, 1240,
+ /*   560 */  1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1240, 1616,
+ /*   570 */  1332, 1422, 1240, 1425, 1262, 1240, 1252, 1240, 1240,
 };
 /********** End of lemon-generated parsing tables *****************************/
 
@@ -168813,233 +169723,231 @@ static const char *const yyRuleName[] = {
  /* 175 */ "idlist ::= idlist COMMA nm",
  /* 176 */ "idlist ::= nm",
  /* 177 */ "expr ::= LP expr RP",
- /* 178 */ "expr ::= ID|INDEXED",
- /* 179 */ "expr ::= JOIN_KW",
- /* 180 */ "expr ::= nm DOT nm",
- /* 181 */ "expr ::= nm DOT nm DOT nm",
- /* 182 */ "term ::= NULL|FLOAT|BLOB",
- /* 183 */ "term ::= STRING",
- /* 184 */ "term ::= INTEGER",
- /* 185 */ "expr ::= VARIABLE",
- /* 186 */ "expr ::= expr COLLATE ID|STRING",
- /* 187 */ "expr ::= CAST LP expr AS typetoken RP",
- /* 188 */ "expr ::= ID|INDEXED LP distinct exprlist RP",
- /* 189 */ "expr ::= ID|INDEXED LP STAR RP",
- /* 190 */ "expr ::= ID|INDEXED LP distinct exprlist RP filter_over",
- /* 191 */ "expr ::= ID|INDEXED LP STAR RP filter_over",
- /* 192 */ "term ::= CTIME_KW",
- /* 193 */ "expr ::= LP nexprlist COMMA expr RP",
- /* 194 */ "expr ::= expr AND expr",
- /* 195 */ "expr ::= expr OR expr",
- /* 196 */ "expr ::= expr LT|GT|GE|LE expr",
- /* 197 */ "expr ::= expr EQ|NE expr",
- /* 198 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr",
- /* 199 */ "expr ::= expr PLUS|MINUS expr",
- /* 200 */ "expr ::= expr STAR|SLASH|REM expr",
- /* 201 */ "expr ::= expr CONCAT expr",
- /* 202 */ "likeop ::= NOT LIKE_KW|MATCH",
- /* 203 */ "expr ::= expr likeop expr",
- /* 204 */ "expr ::= expr likeop expr ESCAPE expr",
- /* 205 */ "expr ::= expr ISNULL|NOTNULL",
- /* 206 */ "expr ::= expr NOT NULL",
- /* 207 */ "expr ::= expr IS expr",
- /* 208 */ "expr ::= expr IS NOT expr",
- /* 209 */ "expr ::= expr IS NOT DISTINCT FROM expr",
- /* 210 */ "expr ::= expr IS DISTINCT FROM expr",
- /* 211 */ "expr ::= NOT expr",
- /* 212 */ "expr ::= BITNOT expr",
- /* 213 */ "expr ::= PLUS|MINUS expr",
- /* 214 */ "expr ::= expr PTR expr",
- /* 215 */ "between_op ::= BETWEEN",
- /* 216 */ "between_op ::= NOT BETWEEN",
- /* 217 */ "expr ::= expr between_op expr AND expr",
- /* 218 */ "in_op ::= IN",
- /* 219 */ "in_op ::= NOT IN",
- /* 220 */ "expr ::= expr in_op LP exprlist RP",
- /* 221 */ "expr ::= LP select RP",
- /* 222 */ "expr ::= expr in_op LP select RP",
- /* 223 */ "expr ::= expr in_op nm dbnm paren_exprlist",
- /* 224 */ "expr ::= EXISTS LP select RP",
- /* 225 */ "expr ::= CASE case_operand case_exprlist case_else END",
- /* 226 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr",
- /* 227 */ "case_exprlist ::= WHEN expr THEN expr",
- /* 228 */ "case_else ::= ELSE expr",
- /* 229 */ "case_else ::=",
- /* 230 */ "case_operand ::= expr",
- /* 231 */ "case_operand ::=",
- /* 232 */ "exprlist ::=",
- /* 233 */ "nexprlist ::= nexprlist COMMA expr",
- /* 234 */ "nexprlist ::= expr",
- /* 235 */ "paren_exprlist ::=",
- /* 236 */ "paren_exprlist ::= LP exprlist RP",
- /* 237 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt",
- /* 238 */ "uniqueflag ::= UNIQUE",
- /* 239 */ "uniqueflag ::=",
- /* 240 */ "eidlist_opt ::=",
- /* 241 */ "eidlist_opt ::= LP eidlist RP",
- /* 242 */ "eidlist ::= eidlist COMMA nm collate sortorder",
- /* 243 */ "eidlist ::= nm collate sortorder",
- /* 244 */ "collate ::=",
- /* 245 */ "collate ::= COLLATE ID|STRING",
- /* 246 */ "cmd ::= DROP INDEX ifexists fullname",
- /* 247 */ "cmd ::= VACUUM vinto",
- /* 248 */ "cmd ::= VACUUM nm vinto",
- /* 249 */ "vinto ::= INTO expr",
- /* 250 */ "vinto ::=",
- /* 251 */ "cmd ::= PRAGMA nm dbnm",
- /* 252 */ "cmd ::= PRAGMA nm dbnm EQ nmnum",
- /* 253 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP",
- /* 254 */ "cmd ::= PRAGMA nm dbnm EQ minus_num",
- /* 255 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP",
- /* 256 */ "plus_num ::= PLUS INTEGER|FLOAT",
- /* 257 */ "minus_num ::= MINUS INTEGER|FLOAT",
- /* 258 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END",
- /* 259 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause",
- /* 260 */ "trigger_time ::= BEFORE|AFTER",
- /* 261 */ "trigger_time ::= INSTEAD OF",
- /* 262 */ "trigger_time ::=",
- /* 263 */ "trigger_event ::= DELETE|INSERT",
- /* 264 */ "trigger_event ::= UPDATE",
- /* 265 */ "trigger_event ::= UPDATE OF idlist",
- /* 266 */ "when_clause ::=",
- /* 267 */ "when_clause ::= WHEN expr",
- /* 268 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI",
- /* 269 */ "trigger_cmd_list ::= trigger_cmd SEMI",
- /* 270 */ "trnm ::= nm DOT nm",
- /* 271 */ "tridxby ::= INDEXED BY nm",
- /* 272 */ "tridxby ::= NOT INDEXED",
- /* 273 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt",
- /* 274 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt",
- /* 275 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt",
- /* 276 */ "trigger_cmd ::= scanpt select scanpt",
- /* 277 */ "expr ::= RAISE LP IGNORE RP",
- /* 278 */ "expr ::= RAISE LP raisetype COMMA nm RP",
- /* 279 */ "raisetype ::= ROLLBACK",
- /* 280 */ "raisetype ::= ABORT",
- /* 281 */ "raisetype ::= FAIL",
- /* 282 */ "cmd ::= DROP TRIGGER ifexists fullname",
- /* 283 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt",
- /* 284 */ "cmd ::= DETACH database_kw_opt expr",
- /* 285 */ "key_opt ::=",
- /* 286 */ "key_opt ::= KEY expr",
- /* 287 */ "cmd ::= REINDEX",
- /* 288 */ "cmd ::= REINDEX nm dbnm",
- /* 289 */ "cmd ::= ANALYZE",
- /* 290 */ "cmd ::= ANALYZE nm dbnm",
- /* 291 */ "cmd ::= ALTER TABLE fullname RENAME TO nm",
- /* 292 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist",
- /* 293 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm",
- /* 294 */ "add_column_fullname ::= fullname",
- /* 295 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm",
- /* 296 */ "cmd ::= create_vtab",
- /* 297 */ "cmd ::= create_vtab LP vtabarglist RP",
- /* 298 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm",
- /* 299 */ "vtabarg ::=",
- /* 300 */ "vtabargtoken ::= ANY",
- /* 301 */ "vtabargtoken ::= lp anylist RP",
- /* 302 */ "lp ::= LP",
- /* 303 */ "with ::= WITH wqlist",
- /* 304 */ "with ::= WITH RECURSIVE wqlist",
- /* 305 */ "wqas ::= AS",
- /* 306 */ "wqas ::= AS MATERIALIZED",
- /* 307 */ "wqas ::= AS NOT MATERIALIZED",
- /* 308 */ "wqitem ::= nm eidlist_opt wqas LP select RP",
- /* 309 */ "wqlist ::= wqitem",
- /* 310 */ "wqlist ::= wqlist COMMA wqitem",
- /* 311 */ "windowdefn_list ::= windowdefn",
- /* 312 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn",
- /* 313 */ "windowdefn ::= nm AS LP window RP",
- /* 314 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt",
- /* 315 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt",
- /* 316 */ "window ::= ORDER BY sortlist frame_opt",
- /* 317 */ "window ::= nm ORDER BY sortlist frame_opt",
- /* 318 */ "window ::= frame_opt",
- /* 319 */ "window ::= nm frame_opt",
- /* 320 */ "frame_opt ::=",
- /* 321 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt",
- /* 322 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt",
- /* 323 */ "range_or_rows ::= RANGE|ROWS|GROUPS",
- /* 324 */ "frame_bound_s ::= frame_bound",
- /* 325 */ "frame_bound_s ::= UNBOUNDED PRECEDING",
- /* 326 */ "frame_bound_e ::= frame_bound",
- /* 327 */ "frame_bound_e ::= UNBOUNDED FOLLOWING",
- /* 328 */ "frame_bound ::= expr PRECEDING|FOLLOWING",
- /* 329 */ "frame_bound ::= CURRENT ROW",
- /* 330 */ "frame_exclude_opt ::=",
- /* 331 */ "frame_exclude_opt ::= EXCLUDE frame_exclude",
- /* 332 */ "frame_exclude ::= NO OTHERS",
- /* 333 */ "frame_exclude ::= CURRENT ROW",
- /* 334 */ "frame_exclude ::= GROUP|TIES",
- /* 335 */ "window_clause ::= WINDOW windowdefn_list",
- /* 336 */ "filter_over ::= filter_clause over_clause",
- /* 337 */ "filter_over ::= over_clause",
- /* 338 */ "filter_over ::= filter_clause",
- /* 339 */ "over_clause ::= OVER LP window RP",
- /* 340 */ "over_clause ::= OVER nm",
- /* 341 */ "filter_clause ::= FILTER LP WHERE expr RP",
- /* 342 */ "input ::= cmdlist",
- /* 343 */ "cmdlist ::= cmdlist ecmd",
- /* 344 */ "cmdlist ::= ecmd",
- /* 345 */ "ecmd ::= SEMI",
- /* 346 */ "ecmd ::= cmdx SEMI",
- /* 347 */ "ecmd ::= explain cmdx SEMI",
- /* 348 */ "trans_opt ::=",
- /* 349 */ "trans_opt ::= TRANSACTION",
- /* 350 */ "trans_opt ::= TRANSACTION nm",
- /* 351 */ "savepoint_opt ::= SAVEPOINT",
- /* 352 */ "savepoint_opt ::=",
- /* 353 */ "cmd ::= create_table create_table_args",
- /* 354 */ "table_option_set ::= table_option",
- /* 355 */ "columnlist ::= columnlist COMMA columnname carglist",
- /* 356 */ "columnlist ::= columnname carglist",
- /* 357 */ "nm ::= ID|INDEXED",
- /* 358 */ "nm ::= STRING",
- /* 359 */ "nm ::= JOIN_KW",
- /* 360 */ "typetoken ::= typename",
- /* 361 */ "typename ::= ID|STRING",
- /* 362 */ "signed ::= plus_num",
- /* 363 */ "signed ::= minus_num",
- /* 364 */ "carglist ::= carglist ccons",
- /* 365 */ "carglist ::=",
- /* 366 */ "ccons ::= NULL onconf",
- /* 367 */ "ccons ::= GENERATED ALWAYS AS generated",
- /* 368 */ "ccons ::= AS generated",
- /* 369 */ "conslist_opt ::= COMMA conslist",
- /* 370 */ "conslist ::= conslist tconscomma tcons",
- /* 371 */ "conslist ::= tcons",
- /* 372 */ "tconscomma ::=",
- /* 373 */ "defer_subclause_opt ::= defer_subclause",
- /* 374 */ "resolvetype ::= raisetype",
- /* 375 */ "selectnowith ::= oneselect",
- /* 376 */ "oneselect ::= values",
- /* 377 */ "sclp ::= selcollist COMMA",
- /* 378 */ "as ::= ID|STRING",
- /* 379 */ "indexed_opt ::= indexed_by",
- /* 380 */ "returning ::=",
- /* 381 */ "expr ::= term",
- /* 382 */ "likeop ::= LIKE_KW|MATCH",
- /* 383 */ "exprlist ::= nexprlist",
- /* 384 */ "nmnum ::= plus_num",
- /* 385 */ "nmnum ::= nm",
- /* 386 */ "nmnum ::= ON",
- /* 387 */ "nmnum ::= DELETE",
- /* 388 */ "nmnum ::= DEFAULT",
- /* 389 */ "plus_num ::= INTEGER|FLOAT",
- /* 390 */ "foreach_clause ::=",
- /* 391 */ "foreach_clause ::= FOR EACH ROW",
- /* 392 */ "trnm ::= nm",
- /* 393 */ "tridxby ::=",
- /* 394 */ "database_kw_opt ::= DATABASE",
- /* 395 */ "database_kw_opt ::=",
- /* 396 */ "kwcolumn_opt ::=",
- /* 397 */ "kwcolumn_opt ::= COLUMNKW",
- /* 398 */ "vtabarglist ::= vtabarg",
- /* 399 */ "vtabarglist ::= vtabarglist COMMA vtabarg",
- /* 400 */ "vtabarg ::= vtabarg vtabargtoken",
- /* 401 */ "anylist ::=",
- /* 402 */ "anylist ::= anylist LP anylist RP",
- /* 403 */ "anylist ::= anylist ANY",
- /* 404 */ "with ::=",
+ /* 178 */ "expr ::= ID|INDEXED|JOIN_KW",
+ /* 179 */ "expr ::= nm DOT nm",
+ /* 180 */ "expr ::= nm DOT nm DOT nm",
+ /* 181 */ "term ::= NULL|FLOAT|BLOB",
+ /* 182 */ "term ::= STRING",
+ /* 183 */ "term ::= INTEGER",
+ /* 184 */ "expr ::= VARIABLE",
+ /* 185 */ "expr ::= expr COLLATE ID|STRING",
+ /* 186 */ "expr ::= CAST LP expr AS typetoken RP",
+ /* 187 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP",
+ /* 188 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP",
+ /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over",
+ /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over",
+ /* 191 */ "term ::= CTIME_KW",
+ /* 192 */ "expr ::= LP nexprlist COMMA expr RP",
+ /* 193 */ "expr ::= expr AND expr",
+ /* 194 */ "expr ::= expr OR expr",
+ /* 195 */ "expr ::= expr LT|GT|GE|LE expr",
+ /* 196 */ "expr ::= expr EQ|NE expr",
+ /* 197 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr",
+ /* 198 */ "expr ::= expr PLUS|MINUS expr",
+ /* 199 */ "expr ::= expr STAR|SLASH|REM expr",
+ /* 200 */ "expr ::= expr CONCAT expr",
+ /* 201 */ "likeop ::= NOT LIKE_KW|MATCH",
+ /* 202 */ "expr ::= expr likeop expr",
+ /* 203 */ "expr ::= expr likeop expr ESCAPE expr",
+ /* 204 */ "expr ::= expr ISNULL|NOTNULL",
+ /* 205 */ "expr ::= expr NOT NULL",
+ /* 206 */ "expr ::= expr IS expr",
+ /* 207 */ "expr ::= expr IS NOT expr",
+ /* 208 */ "expr ::= expr IS NOT DISTINCT FROM expr",
+ /* 209 */ "expr ::= expr IS DISTINCT FROM expr",
+ /* 210 */ "expr ::= NOT expr",
+ /* 211 */ "expr ::= BITNOT expr",
+ /* 212 */ "expr ::= PLUS|MINUS expr",
+ /* 213 */ "expr ::= expr PTR expr",
+ /* 214 */ "between_op ::= BETWEEN",
+ /* 215 */ "between_op ::= NOT BETWEEN",
+ /* 216 */ "expr ::= expr between_op expr AND expr",
+ /* 217 */ "in_op ::= IN",
+ /* 218 */ "in_op ::= NOT IN",
+ /* 219 */ "expr ::= expr in_op LP exprlist RP",
+ /* 220 */ "expr ::= LP select RP",
+ /* 221 */ "expr ::= expr in_op LP select RP",
+ /* 222 */ "expr ::= expr in_op nm dbnm paren_exprlist",
+ /* 223 */ "expr ::= EXISTS LP select RP",
+ /* 224 */ "expr ::= CASE case_operand case_exprlist case_else END",
+ /* 225 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr",
+ /* 226 */ "case_exprlist ::= WHEN expr THEN expr",
+ /* 227 */ "case_else ::= ELSE expr",
+ /* 228 */ "case_else ::=",
+ /* 229 */ "case_operand ::=",
+ /* 230 */ "exprlist ::=",
+ /* 231 */ "nexprlist ::= nexprlist COMMA expr",
+ /* 232 */ "nexprlist ::= expr",
+ /* 233 */ "paren_exprlist ::=",
+ /* 234 */ "paren_exprlist ::= LP exprlist RP",
+ /* 235 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt",
+ /* 236 */ "uniqueflag ::= UNIQUE",
+ /* 237 */ "uniqueflag ::=",
+ /* 238 */ "eidlist_opt ::=",
+ /* 239 */ "eidlist_opt ::= LP eidlist RP",
+ /* 240 */ "eidlist ::= eidlist COMMA nm collate sortorder",
+ /* 241 */ "eidlist ::= nm collate sortorder",
+ /* 242 */ "collate ::=",
+ /* 243 */ "collate ::= COLLATE ID|STRING",
+ /* 244 */ "cmd ::= DROP INDEX ifexists fullname",
+ /* 245 */ "cmd ::= VACUUM vinto",
+ /* 246 */ "cmd ::= VACUUM nm vinto",
+ /* 247 */ "vinto ::= INTO expr",
+ /* 248 */ "vinto ::=",
+ /* 249 */ "cmd ::= PRAGMA nm dbnm",
+ /* 250 */ "cmd ::= PRAGMA nm dbnm EQ nmnum",
+ /* 251 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP",
+ /* 252 */ "cmd ::= PRAGMA nm dbnm EQ minus_num",
+ /* 253 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP",
+ /* 254 */ "plus_num ::= PLUS INTEGER|FLOAT",
+ /* 255 */ "minus_num ::= MINUS INTEGER|FLOAT",
+ /* 256 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END",
+ /* 257 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause",
+ /* 258 */ "trigger_time ::= BEFORE|AFTER",
+ /* 259 */ "trigger_time ::= INSTEAD OF",
+ /* 260 */ "trigger_time ::=",
+ /* 261 */ "trigger_event ::= DELETE|INSERT",
+ /* 262 */ "trigger_event ::= UPDATE",
+ /* 263 */ "trigger_event ::= UPDATE OF idlist",
+ /* 264 */ "when_clause ::=",
+ /* 265 */ "when_clause ::= WHEN expr",
+ /* 266 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI",
+ /* 267 */ "trigger_cmd_list ::= trigger_cmd SEMI",
+ /* 268 */ "trnm ::= nm DOT nm",
+ /* 269 */ "tridxby ::= INDEXED BY nm",
+ /* 270 */ "tridxby ::= NOT INDEXED",
+ /* 271 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt",
+ /* 272 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt",
+ /* 273 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt",
+ /* 274 */ "trigger_cmd ::= scanpt select scanpt",
+ /* 275 */ "expr ::= RAISE LP IGNORE RP",
+ /* 276 */ "expr ::= RAISE LP raisetype COMMA nm RP",
+ /* 277 */ "raisetype ::= ROLLBACK",
+ /* 278 */ "raisetype ::= ABORT",
+ /* 279 */ "raisetype ::= FAIL",
+ /* 280 */ "cmd ::= DROP TRIGGER ifexists fullname",
+ /* 281 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt",
+ /* 282 */ "cmd ::= DETACH database_kw_opt expr",
+ /* 283 */ "key_opt ::=",
+ /* 284 */ "key_opt ::= KEY expr",
+ /* 285 */ "cmd ::= REINDEX",
+ /* 286 */ "cmd ::= REINDEX nm dbnm",
+ /* 287 */ "cmd ::= ANALYZE",
+ /* 288 */ "cmd ::= ANALYZE nm dbnm",
+ /* 289 */ "cmd ::= ALTER TABLE fullname RENAME TO nm",
+ /* 290 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist",
+ /* 291 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm",
+ /* 292 */ "add_column_fullname ::= fullname",
+ /* 293 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm",
+ /* 294 */ "cmd ::= create_vtab",
+ /* 295 */ "cmd ::= create_vtab LP vtabarglist RP",
+ /* 296 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm",
+ /* 297 */ "vtabarg ::=",
+ /* 298 */ "vtabargtoken ::= ANY",
+ /* 299 */ "vtabargtoken ::= lp anylist RP",
+ /* 300 */ "lp ::= LP",
+ /* 301 */ "with ::= WITH wqlist",
+ /* 302 */ "with ::= WITH RECURSIVE wqlist",
+ /* 303 */ "wqas ::= AS",
+ /* 304 */ "wqas ::= AS MATERIALIZED",
+ /* 305 */ "wqas ::= AS NOT MATERIALIZED",
+ /* 306 */ "wqitem ::= nm eidlist_opt wqas LP select RP",
+ /* 307 */ "wqlist ::= wqitem",
+ /* 308 */ "wqlist ::= wqlist COMMA wqitem",
+ /* 309 */ "windowdefn_list ::= windowdefn",
+ /* 310 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn",
+ /* 311 */ "windowdefn ::= nm AS LP window RP",
+ /* 312 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt",
+ /* 313 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt",
+ /* 314 */ "window ::= ORDER BY sortlist frame_opt",
+ /* 315 */ "window ::= nm ORDER BY sortlist frame_opt",
+ /* 316 */ "window ::= frame_opt",
+ /* 317 */ "window ::= nm frame_opt",
+ /* 318 */ "frame_opt ::=",
+ /* 319 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt",
+ /* 320 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt",
+ /* 321 */ "range_or_rows ::= RANGE|ROWS|GROUPS",
+ /* 322 */ "frame_bound_s ::= frame_bound",
+ /* 323 */ "frame_bound_s ::= UNBOUNDED PRECEDING",
+ /* 324 */ "frame_bound_e ::= frame_bound",
+ /* 325 */ "frame_bound_e ::= UNBOUNDED FOLLOWING",
+ /* 326 */ "frame_bound ::= expr PRECEDING|FOLLOWING",
+ /* 327 */ "frame_bound ::= CURRENT ROW",
+ /* 328 */ "frame_exclude_opt ::=",
+ /* 329 */ "frame_exclude_opt ::= EXCLUDE frame_exclude",
+ /* 330 */ "frame_exclude ::= NO OTHERS",
+ /* 331 */ "frame_exclude ::= CURRENT ROW",
+ /* 332 */ "frame_exclude ::= GROUP|TIES",
+ /* 333 */ "window_clause ::= WINDOW windowdefn_list",
+ /* 334 */ "filter_over ::= filter_clause over_clause",
+ /* 335 */ "filter_over ::= over_clause",
+ /* 336 */ "filter_over ::= filter_clause",
+ /* 337 */ "over_clause ::= OVER LP window RP",
+ /* 338 */ "over_clause ::= OVER nm",
+ /* 339 */ "filter_clause ::= FILTER LP WHERE expr RP",
+ /* 340 */ "input ::= cmdlist",
+ /* 341 */ "cmdlist ::= cmdlist ecmd",
+ /* 342 */ "cmdlist ::= ecmd",
+ /* 343 */ "ecmd ::= SEMI",
+ /* 344 */ "ecmd ::= cmdx SEMI",
+ /* 345 */ "ecmd ::= explain cmdx SEMI",
+ /* 346 */ "trans_opt ::=",
+ /* 347 */ "trans_opt ::= TRANSACTION",
+ /* 348 */ "trans_opt ::= TRANSACTION nm",
+ /* 349 */ "savepoint_opt ::= SAVEPOINT",
+ /* 350 */ "savepoint_opt ::=",
+ /* 351 */ "cmd ::= create_table create_table_args",
+ /* 352 */ "table_option_set ::= table_option",
+ /* 353 */ "columnlist ::= columnlist COMMA columnname carglist",
+ /* 354 */ "columnlist ::= columnname carglist",
+ /* 355 */ "nm ::= ID|INDEXED|JOIN_KW",
+ /* 356 */ "nm ::= STRING",
+ /* 357 */ "typetoken ::= typename",
+ /* 358 */ "typename ::= ID|STRING",
+ /* 359 */ "signed ::= plus_num",
+ /* 360 */ "signed ::= minus_num",
+ /* 361 */ "carglist ::= carglist ccons",
+ /* 362 */ "carglist ::=",
+ /* 363 */ "ccons ::= NULL onconf",
+ /* 364 */ "ccons ::= GENERATED ALWAYS AS generated",
+ /* 365 */ "ccons ::= AS generated",
+ /* 366 */ "conslist_opt ::= COMMA conslist",
+ /* 367 */ "conslist ::= conslist tconscomma tcons",
+ /* 368 */ "conslist ::= tcons",
+ /* 369 */ "tconscomma ::=",
+ /* 370 */ "defer_subclause_opt ::= defer_subclause",
+ /* 371 */ "resolvetype ::= raisetype",
+ /* 372 */ "selectnowith ::= oneselect",
+ /* 373 */ "oneselect ::= values",
+ /* 374 */ "sclp ::= selcollist COMMA",
+ /* 375 */ "as ::= ID|STRING",
+ /* 376 */ "indexed_opt ::= indexed_by",
+ /* 377 */ "returning ::=",
+ /* 378 */ "expr ::= term",
+ /* 379 */ "likeop ::= LIKE_KW|MATCH",
+ /* 380 */ "case_operand ::= expr",
+ /* 381 */ "exprlist ::= nexprlist",
+ /* 382 */ "nmnum ::= plus_num",
+ /* 383 */ "nmnum ::= nm",
+ /* 384 */ "nmnum ::= ON",
+ /* 385 */ "nmnum ::= DELETE",
+ /* 386 */ "nmnum ::= DEFAULT",
+ /* 387 */ "plus_num ::= INTEGER|FLOAT",
+ /* 388 */ "foreach_clause ::=",
+ /* 389 */ "foreach_clause ::= FOR EACH ROW",
+ /* 390 */ "trnm ::= nm",
+ /* 391 */ "tridxby ::=",
+ /* 392 */ "database_kw_opt ::= DATABASE",
+ /* 393 */ "database_kw_opt ::=",
+ /* 394 */ "kwcolumn_opt ::=",
+ /* 395 */ "kwcolumn_opt ::= COLUMNKW",
+ /* 396 */ "vtabarglist ::= vtabarg",
+ /* 397 */ "vtabarglist ::= vtabarglist COMMA vtabarg",
+ /* 398 */ "vtabarg ::= vtabarg vtabargtoken",
+ /* 399 */ "anylist ::=",
+ /* 400 */ "anylist ::= anylist LP anylist RP",
+ /* 401 */ "anylist ::= anylist ANY",
+ /* 402 */ "with ::=",
 };
 #endif /* NDEBUG */
 
@@ -169724,233 +170632,231 @@ static const YYCODETYPE yyRuleInfoLhs[] = {
    263,  /* (175) idlist ::= idlist COMMA nm */
    263,  /* (176) idlist ::= nm */
    217,  /* (177) expr ::= LP expr RP */
-   217,  /* (178) expr ::= ID|INDEXED */
-   217,  /* (179) expr ::= JOIN_KW */
-   217,  /* (180) expr ::= nm DOT nm */
-   217,  /* (181) expr ::= nm DOT nm DOT nm */
-   216,  /* (182) term ::= NULL|FLOAT|BLOB */
-   216,  /* (183) term ::= STRING */
-   216,  /* (184) term ::= INTEGER */
-   217,  /* (185) expr ::= VARIABLE */
-   217,  /* (186) expr ::= expr COLLATE ID|STRING */
-   217,  /* (187) expr ::= CAST LP expr AS typetoken RP */
-   217,  /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */
-   217,  /* (189) expr ::= ID|INDEXED LP STAR RP */
-   217,  /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */
-   217,  /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */
-   216,  /* (192) term ::= CTIME_KW */
-   217,  /* (193) expr ::= LP nexprlist COMMA expr RP */
-   217,  /* (194) expr ::= expr AND expr */
-   217,  /* (195) expr ::= expr OR expr */
-   217,  /* (196) expr ::= expr LT|GT|GE|LE expr */
-   217,  /* (197) expr ::= expr EQ|NE expr */
-   217,  /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */
-   217,  /* (199) expr ::= expr PLUS|MINUS expr */
-   217,  /* (200) expr ::= expr STAR|SLASH|REM expr */
-   217,  /* (201) expr ::= expr CONCAT expr */
-   274,  /* (202) likeop ::= NOT LIKE_KW|MATCH */
-   217,  /* (203) expr ::= expr likeop expr */
-   217,  /* (204) expr ::= expr likeop expr ESCAPE expr */
-   217,  /* (205) expr ::= expr ISNULL|NOTNULL */
-   217,  /* (206) expr ::= expr NOT NULL */
-   217,  /* (207) expr ::= expr IS expr */
-   217,  /* (208) expr ::= expr IS NOT expr */
-   217,  /* (209) expr ::= expr IS NOT DISTINCT FROM expr */
-   217,  /* (210) expr ::= expr IS DISTINCT FROM expr */
-   217,  /* (211) expr ::= NOT expr */
-   217,  /* (212) expr ::= BITNOT expr */
-   217,  /* (213) expr ::= PLUS|MINUS expr */
-   217,  /* (214) expr ::= expr PTR expr */
-   275,  /* (215) between_op ::= BETWEEN */
-   275,  /* (216) between_op ::= NOT BETWEEN */
-   217,  /* (217) expr ::= expr between_op expr AND expr */
-   276,  /* (218) in_op ::= IN */
-   276,  /* (219) in_op ::= NOT IN */
-   217,  /* (220) expr ::= expr in_op LP exprlist RP */
-   217,  /* (221) expr ::= LP select RP */
-   217,  /* (222) expr ::= expr in_op LP select RP */
-   217,  /* (223) expr ::= expr in_op nm dbnm paren_exprlist */
-   217,  /* (224) expr ::= EXISTS LP select RP */
-   217,  /* (225) expr ::= CASE case_operand case_exprlist case_else END */
-   279,  /* (226) case_exprlist ::= case_exprlist WHEN expr THEN expr */
-   279,  /* (227) case_exprlist ::= WHEN expr THEN expr */
-   280,  /* (228) case_else ::= ELSE expr */
-   280,  /* (229) case_else ::= */
-   278,  /* (230) case_operand ::= expr */
-   278,  /* (231) case_operand ::= */
-   261,  /* (232) exprlist ::= */
-   253,  /* (233) nexprlist ::= nexprlist COMMA expr */
-   253,  /* (234) nexprlist ::= expr */
-   277,  /* (235) paren_exprlist ::= */
-   277,  /* (236) paren_exprlist ::= LP exprlist RP */
-   190,  /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
-   281,  /* (238) uniqueflag ::= UNIQUE */
-   281,  /* (239) uniqueflag ::= */
-   221,  /* (240) eidlist_opt ::= */
-   221,  /* (241) eidlist_opt ::= LP eidlist RP */
-   232,  /* (242) eidlist ::= eidlist COMMA nm collate sortorder */
-   232,  /* (243) eidlist ::= nm collate sortorder */
-   282,  /* (244) collate ::= */
-   282,  /* (245) collate ::= COLLATE ID|STRING */
-   190,  /* (246) cmd ::= DROP INDEX ifexists fullname */
-   190,  /* (247) cmd ::= VACUUM vinto */
-   190,  /* (248) cmd ::= VACUUM nm vinto */
-   283,  /* (249) vinto ::= INTO expr */
-   283,  /* (250) vinto ::= */
-   190,  /* (251) cmd ::= PRAGMA nm dbnm */
-   190,  /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */
-   190,  /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */
-   190,  /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */
-   190,  /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */
-   211,  /* (256) plus_num ::= PLUS INTEGER|FLOAT */
-   212,  /* (257) minus_num ::= MINUS INTEGER|FLOAT */
-   190,  /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
-   285,  /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
-   287,  /* (260) trigger_time ::= BEFORE|AFTER */
-   287,  /* (261) trigger_time ::= INSTEAD OF */
-   287,  /* (262) trigger_time ::= */
-   288,  /* (263) trigger_event ::= DELETE|INSERT */
-   288,  /* (264) trigger_event ::= UPDATE */
-   288,  /* (265) trigger_event ::= UPDATE OF idlist */
-   290,  /* (266) when_clause ::= */
-   290,  /* (267) when_clause ::= WHEN expr */
-   286,  /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
-   286,  /* (269) trigger_cmd_list ::= trigger_cmd SEMI */
-   292,  /* (270) trnm ::= nm DOT nm */
-   293,  /* (271) tridxby ::= INDEXED BY nm */
-   293,  /* (272) tridxby ::= NOT INDEXED */
-   291,  /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
-   291,  /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
-   291,  /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
-   291,  /* (276) trigger_cmd ::= scanpt select scanpt */
-   217,  /* (277) expr ::= RAISE LP IGNORE RP */
-   217,  /* (278) expr ::= RAISE LP raisetype COMMA nm RP */
-   236,  /* (279) raisetype ::= ROLLBACK */
-   236,  /* (280) raisetype ::= ABORT */
-   236,  /* (281) raisetype ::= FAIL */
-   190,  /* (282) cmd ::= DROP TRIGGER ifexists fullname */
-   190,  /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
-   190,  /* (284) cmd ::= DETACH database_kw_opt expr */
-   295,  /* (285) key_opt ::= */
-   295,  /* (286) key_opt ::= KEY expr */
-   190,  /* (287) cmd ::= REINDEX */
-   190,  /* (288) cmd ::= REINDEX nm dbnm */
-   190,  /* (289) cmd ::= ANALYZE */
-   190,  /* (290) cmd ::= ANALYZE nm dbnm */
-   190,  /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */
-   190,  /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
-   190,  /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
-   296,  /* (294) add_column_fullname ::= fullname */
-   190,  /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
-   190,  /* (296) cmd ::= create_vtab */
-   190,  /* (297) cmd ::= create_vtab LP vtabarglist RP */
-   298,  /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
-   300,  /* (299) vtabarg ::= */
-   301,  /* (300) vtabargtoken ::= ANY */
-   301,  /* (301) vtabargtoken ::= lp anylist RP */
-   302,  /* (302) lp ::= LP */
-   266,  /* (303) with ::= WITH wqlist */
-   266,  /* (304) with ::= WITH RECURSIVE wqlist */
-   305,  /* (305) wqas ::= AS */
-   305,  /* (306) wqas ::= AS MATERIALIZED */
-   305,  /* (307) wqas ::= AS NOT MATERIALIZED */
-   304,  /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */
-   241,  /* (309) wqlist ::= wqitem */
-   241,  /* (310) wqlist ::= wqlist COMMA wqitem */
-   306,  /* (311) windowdefn_list ::= windowdefn */
-   306,  /* (312) windowdefn_list ::= windowdefn_list COMMA windowdefn */
-   307,  /* (313) windowdefn ::= nm AS LP window RP */
-   308,  /* (314) window ::= PARTITION BY nexprlist orderby_opt frame_opt */
-   308,  /* (315) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
-   308,  /* (316) window ::= ORDER BY sortlist frame_opt */
-   308,  /* (317) window ::= nm ORDER BY sortlist frame_opt */
-   308,  /* (318) window ::= frame_opt */
-   308,  /* (319) window ::= nm frame_opt */
-   309,  /* (320) frame_opt ::= */
-   309,  /* (321) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
-   309,  /* (322) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
-   313,  /* (323) range_or_rows ::= RANGE|ROWS|GROUPS */
-   315,  /* (324) frame_bound_s ::= frame_bound */
-   315,  /* (325) frame_bound_s ::= UNBOUNDED PRECEDING */
-   316,  /* (326) frame_bound_e ::= frame_bound */
-   316,  /* (327) frame_bound_e ::= UNBOUNDED FOLLOWING */
-   314,  /* (328) frame_bound ::= expr PRECEDING|FOLLOWING */
-   314,  /* (329) frame_bound ::= CURRENT ROW */
-   317,  /* (330) frame_exclude_opt ::= */
-   317,  /* (331) frame_exclude_opt ::= EXCLUDE frame_exclude */
-   318,  /* (332) frame_exclude ::= NO OTHERS */
-   318,  /* (333) frame_exclude ::= CURRENT ROW */
-   318,  /* (334) frame_exclude ::= GROUP|TIES */
-   251,  /* (335) window_clause ::= WINDOW windowdefn_list */
-   273,  /* (336) filter_over ::= filter_clause over_clause */
-   273,  /* (337) filter_over ::= over_clause */
-   273,  /* (338) filter_over ::= filter_clause */
-   312,  /* (339) over_clause ::= OVER LP window RP */
-   312,  /* (340) over_clause ::= OVER nm */
-   311,  /* (341) filter_clause ::= FILTER LP WHERE expr RP */
-   185,  /* (342) input ::= cmdlist */
-   186,  /* (343) cmdlist ::= cmdlist ecmd */
-   186,  /* (344) cmdlist ::= ecmd */
-   187,  /* (345) ecmd ::= SEMI */
-   187,  /* (346) ecmd ::= cmdx SEMI */
-   187,  /* (347) ecmd ::= explain cmdx SEMI */
-   192,  /* (348) trans_opt ::= */
-   192,  /* (349) trans_opt ::= TRANSACTION */
-   192,  /* (350) trans_opt ::= TRANSACTION nm */
-   194,  /* (351) savepoint_opt ::= SAVEPOINT */
-   194,  /* (352) savepoint_opt ::= */
-   190,  /* (353) cmd ::= create_table create_table_args */
-   203,  /* (354) table_option_set ::= table_option */
-   201,  /* (355) columnlist ::= columnlist COMMA columnname carglist */
-   201,  /* (356) columnlist ::= columnname carglist */
-   193,  /* (357) nm ::= ID|INDEXED */
-   193,  /* (358) nm ::= STRING */
-   193,  /* (359) nm ::= JOIN_KW */
-   208,  /* (360) typetoken ::= typename */
-   209,  /* (361) typename ::= ID|STRING */
-   210,  /* (362) signed ::= plus_num */
-   210,  /* (363) signed ::= minus_num */
-   207,  /* (364) carglist ::= carglist ccons */
-   207,  /* (365) carglist ::= */
-   215,  /* (366) ccons ::= NULL onconf */
-   215,  /* (367) ccons ::= GENERATED ALWAYS AS generated */
-   215,  /* (368) ccons ::= AS generated */
-   202,  /* (369) conslist_opt ::= COMMA conslist */
-   228,  /* (370) conslist ::= conslist tconscomma tcons */
-   228,  /* (371) conslist ::= tcons */
-   229,  /* (372) tconscomma ::= */
-   233,  /* (373) defer_subclause_opt ::= defer_subclause */
-   235,  /* (374) resolvetype ::= raisetype */
-   239,  /* (375) selectnowith ::= oneselect */
-   240,  /* (376) oneselect ::= values */
-   254,  /* (377) sclp ::= selcollist COMMA */
-   255,  /* (378) as ::= ID|STRING */
-   264,  /* (379) indexed_opt ::= indexed_by */
-   272,  /* (380) returning ::= */
-   217,  /* (381) expr ::= term */
-   274,  /* (382) likeop ::= LIKE_KW|MATCH */
-   261,  /* (383) exprlist ::= nexprlist */
-   284,  /* (384) nmnum ::= plus_num */
-   284,  /* (385) nmnum ::= nm */
-   284,  /* (386) nmnum ::= ON */
-   284,  /* (387) nmnum ::= DELETE */
-   284,  /* (388) nmnum ::= DEFAULT */
-   211,  /* (389) plus_num ::= INTEGER|FLOAT */
-   289,  /* (390) foreach_clause ::= */
-   289,  /* (391) foreach_clause ::= FOR EACH ROW */
-   292,  /* (392) trnm ::= nm */
-   293,  /* (393) tridxby ::= */
-   294,  /* (394) database_kw_opt ::= DATABASE */
-   294,  /* (395) database_kw_opt ::= */
-   297,  /* (396) kwcolumn_opt ::= */
-   297,  /* (397) kwcolumn_opt ::= COLUMNKW */
-   299,  /* (398) vtabarglist ::= vtabarg */
-   299,  /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */
-   300,  /* (400) vtabarg ::= vtabarg vtabargtoken */
-   303,  /* (401) anylist ::= */
-   303,  /* (402) anylist ::= anylist LP anylist RP */
-   303,  /* (403) anylist ::= anylist ANY */
-   266,  /* (404) with ::= */
+   217,  /* (178) expr ::= ID|INDEXED|JOIN_KW */
+   217,  /* (179) expr ::= nm DOT nm */
+   217,  /* (180) expr ::= nm DOT nm DOT nm */
+   216,  /* (181) term ::= NULL|FLOAT|BLOB */
+   216,  /* (182) term ::= STRING */
+   216,  /* (183) term ::= INTEGER */
+   217,  /* (184) expr ::= VARIABLE */
+   217,  /* (185) expr ::= expr COLLATE ID|STRING */
+   217,  /* (186) expr ::= CAST LP expr AS typetoken RP */
+   217,  /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */
+   217,  /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */
+   217,  /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */
+   217,  /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */
+   216,  /* (191) term ::= CTIME_KW */
+   217,  /* (192) expr ::= LP nexprlist COMMA expr RP */
+   217,  /* (193) expr ::= expr AND expr */
+   217,  /* (194) expr ::= expr OR expr */
+   217,  /* (195) expr ::= expr LT|GT|GE|LE expr */
+   217,  /* (196) expr ::= expr EQ|NE expr */
+   217,  /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */
+   217,  /* (198) expr ::= expr PLUS|MINUS expr */
+   217,  /* (199) expr ::= expr STAR|SLASH|REM expr */
+   217,  /* (200) expr ::= expr CONCAT expr */
+   274,  /* (201) likeop ::= NOT LIKE_KW|MATCH */
+   217,  /* (202) expr ::= expr likeop expr */
+   217,  /* (203) expr ::= expr likeop expr ESCAPE expr */
+   217,  /* (204) expr ::= expr ISNULL|NOTNULL */
+   217,  /* (205) expr ::= expr NOT NULL */
+   217,  /* (206) expr ::= expr IS expr */
+   217,  /* (207) expr ::= expr IS NOT expr */
+   217,  /* (208) expr ::= expr IS NOT DISTINCT FROM expr */
+   217,  /* (209) expr ::= expr IS DISTINCT FROM expr */
+   217,  /* (210) expr ::= NOT expr */
+   217,  /* (211) expr ::= BITNOT expr */
+   217,  /* (212) expr ::= PLUS|MINUS expr */
+   217,  /* (213) expr ::= expr PTR expr */
+   275,  /* (214) between_op ::= BETWEEN */
+   275,  /* (215) between_op ::= NOT BETWEEN */
+   217,  /* (216) expr ::= expr between_op expr AND expr */
+   276,  /* (217) in_op ::= IN */
+   276,  /* (218) in_op ::= NOT IN */
+   217,  /* (219) expr ::= expr in_op LP exprlist RP */
+   217,  /* (220) expr ::= LP select RP */
+   217,  /* (221) expr ::= expr in_op LP select RP */
+   217,  /* (222) expr ::= expr in_op nm dbnm paren_exprlist */
+   217,  /* (223) expr ::= EXISTS LP select RP */
+   217,  /* (224) expr ::= CASE case_operand case_exprlist case_else END */
+   279,  /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */
+   279,  /* (226) case_exprlist ::= WHEN expr THEN expr */
+   280,  /* (227) case_else ::= ELSE expr */
+   280,  /* (228) case_else ::= */
+   278,  /* (229) case_operand ::= */
+   261,  /* (230) exprlist ::= */
+   253,  /* (231) nexprlist ::= nexprlist COMMA expr */
+   253,  /* (232) nexprlist ::= expr */
+   277,  /* (233) paren_exprlist ::= */
+   277,  /* (234) paren_exprlist ::= LP exprlist RP */
+   190,  /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
+   281,  /* (236) uniqueflag ::= UNIQUE */
+   281,  /* (237) uniqueflag ::= */
+   221,  /* (238) eidlist_opt ::= */
+   221,  /* (239) eidlist_opt ::= LP eidlist RP */
+   232,  /* (240) eidlist ::= eidlist COMMA nm collate sortorder */
+   232,  /* (241) eidlist ::= nm collate sortorder */
+   282,  /* (242) collate ::= */
+   282,  /* (243) collate ::= COLLATE ID|STRING */
+   190,  /* (244) cmd ::= DROP INDEX ifexists fullname */
+   190,  /* (245) cmd ::= VACUUM vinto */
+   190,  /* (246) cmd ::= VACUUM nm vinto */
+   283,  /* (247) vinto ::= INTO expr */
+   283,  /* (248) vinto ::= */
+   190,  /* (249) cmd ::= PRAGMA nm dbnm */
+   190,  /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */
+   190,  /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */
+   190,  /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */
+   190,  /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */
+   211,  /* (254) plus_num ::= PLUS INTEGER|FLOAT */
+   212,  /* (255) minus_num ::= MINUS INTEGER|FLOAT */
+   190,  /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
+   285,  /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
+   287,  /* (258) trigger_time ::= BEFORE|AFTER */
+   287,  /* (259) trigger_time ::= INSTEAD OF */
+   287,  /* (260) trigger_time ::= */
+   288,  /* (261) trigger_event ::= DELETE|INSERT */
+   288,  /* (262) trigger_event ::= UPDATE */
+   288,  /* (263) trigger_event ::= UPDATE OF idlist */
+   290,  /* (264) when_clause ::= */
+   290,  /* (265) when_clause ::= WHEN expr */
+   286,  /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
+   286,  /* (267) trigger_cmd_list ::= trigger_cmd SEMI */
+   292,  /* (268) trnm ::= nm DOT nm */
+   293,  /* (269) tridxby ::= INDEXED BY nm */
+   293,  /* (270) tridxby ::= NOT INDEXED */
+   291,  /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
+   291,  /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
+   291,  /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
+   291,  /* (274) trigger_cmd ::= scanpt select scanpt */
+   217,  /* (275) expr ::= RAISE LP IGNORE RP */
+   217,  /* (276) expr ::= RAISE LP raisetype COMMA nm RP */
+   236,  /* (277) raisetype ::= ROLLBACK */
+   236,  /* (278) raisetype ::= ABORT */
+   236,  /* (279) raisetype ::= FAIL */
+   190,  /* (280) cmd ::= DROP TRIGGER ifexists fullname */
+   190,  /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
+   190,  /* (282) cmd ::= DETACH database_kw_opt expr */
+   295,  /* (283) key_opt ::= */
+   295,  /* (284) key_opt ::= KEY expr */
+   190,  /* (285) cmd ::= REINDEX */
+   190,  /* (286) cmd ::= REINDEX nm dbnm */
+   190,  /* (287) cmd ::= ANALYZE */
+   190,  /* (288) cmd ::= ANALYZE nm dbnm */
+   190,  /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */
+   190,  /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
+   190,  /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
+   296,  /* (292) add_column_fullname ::= fullname */
+   190,  /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
+   190,  /* (294) cmd ::= create_vtab */
+   190,  /* (295) cmd ::= create_vtab LP vtabarglist RP */
+   298,  /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
+   300,  /* (297) vtabarg ::= */
+   301,  /* (298) vtabargtoken ::= ANY */
+   301,  /* (299) vtabargtoken ::= lp anylist RP */
+   302,  /* (300) lp ::= LP */
+   266,  /* (301) with ::= WITH wqlist */
+   266,  /* (302) with ::= WITH RECURSIVE wqlist */
+   305,  /* (303) wqas ::= AS */
+   305,  /* (304) wqas ::= AS MATERIALIZED */
+   305,  /* (305) wqas ::= AS NOT MATERIALIZED */
+   304,  /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */
+   241,  /* (307) wqlist ::= wqitem */
+   241,  /* (308) wqlist ::= wqlist COMMA wqitem */
+   306,  /* (309) windowdefn_list ::= windowdefn */
+   306,  /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */
+   307,  /* (311) windowdefn ::= nm AS LP window RP */
+   308,  /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */
+   308,  /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
+   308,  /* (314) window ::= ORDER BY sortlist frame_opt */
+   308,  /* (315) window ::= nm ORDER BY sortlist frame_opt */
+   308,  /* (316) window ::= frame_opt */
+   308,  /* (317) window ::= nm frame_opt */
+   309,  /* (318) frame_opt ::= */
+   309,  /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
+   309,  /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
+   313,  /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */
+   315,  /* (322) frame_bound_s ::= frame_bound */
+   315,  /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */
+   316,  /* (324) frame_bound_e ::= frame_bound */
+   316,  /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */
+   314,  /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */
+   314,  /* (327) frame_bound ::= CURRENT ROW */
+   317,  /* (328) frame_exclude_opt ::= */
+   317,  /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */
+   318,  /* (330) frame_exclude ::= NO OTHERS */
+   318,  /* (331) frame_exclude ::= CURRENT ROW */
+   318,  /* (332) frame_exclude ::= GROUP|TIES */
+   251,  /* (333) window_clause ::= WINDOW windowdefn_list */
+   273,  /* (334) filter_over ::= filter_clause over_clause */
+   273,  /* (335) filter_over ::= over_clause */
+   273,  /* (336) filter_over ::= filter_clause */
+   312,  /* (337) over_clause ::= OVER LP window RP */
+   312,  /* (338) over_clause ::= OVER nm */
+   311,  /* (339) filter_clause ::= FILTER LP WHERE expr RP */
+   185,  /* (340) input ::= cmdlist */
+   186,  /* (341) cmdlist ::= cmdlist ecmd */
+   186,  /* (342) cmdlist ::= ecmd */
+   187,  /* (343) ecmd ::= SEMI */
+   187,  /* (344) ecmd ::= cmdx SEMI */
+   187,  /* (345) ecmd ::= explain cmdx SEMI */
+   192,  /* (346) trans_opt ::= */
+   192,  /* (347) trans_opt ::= TRANSACTION */
+   192,  /* (348) trans_opt ::= TRANSACTION nm */
+   194,  /* (349) savepoint_opt ::= SAVEPOINT */
+   194,  /* (350) savepoint_opt ::= */
+   190,  /* (351) cmd ::= create_table create_table_args */
+   203,  /* (352) table_option_set ::= table_option */
+   201,  /* (353) columnlist ::= columnlist COMMA columnname carglist */
+   201,  /* (354) columnlist ::= columnname carglist */
+   193,  /* (355) nm ::= ID|INDEXED|JOIN_KW */
+   193,  /* (356) nm ::= STRING */
+   208,  /* (357) typetoken ::= typename */
+   209,  /* (358) typename ::= ID|STRING */
+   210,  /* (359) signed ::= plus_num */
+   210,  /* (360) signed ::= minus_num */
+   207,  /* (361) carglist ::= carglist ccons */
+   207,  /* (362) carglist ::= */
+   215,  /* (363) ccons ::= NULL onconf */
+   215,  /* (364) ccons ::= GENERATED ALWAYS AS generated */
+   215,  /* (365) ccons ::= AS generated */
+   202,  /* (366) conslist_opt ::= COMMA conslist */
+   228,  /* (367) conslist ::= conslist tconscomma tcons */
+   228,  /* (368) conslist ::= tcons */
+   229,  /* (369) tconscomma ::= */
+   233,  /* (370) defer_subclause_opt ::= defer_subclause */
+   235,  /* (371) resolvetype ::= raisetype */
+   239,  /* (372) selectnowith ::= oneselect */
+   240,  /* (373) oneselect ::= values */
+   254,  /* (374) sclp ::= selcollist COMMA */
+   255,  /* (375) as ::= ID|STRING */
+   264,  /* (376) indexed_opt ::= indexed_by */
+   272,  /* (377) returning ::= */
+   217,  /* (378) expr ::= term */
+   274,  /* (379) likeop ::= LIKE_KW|MATCH */
+   278,  /* (380) case_operand ::= expr */
+   261,  /* (381) exprlist ::= nexprlist */
+   284,  /* (382) nmnum ::= plus_num */
+   284,  /* (383) nmnum ::= nm */
+   284,  /* (384) nmnum ::= ON */
+   284,  /* (385) nmnum ::= DELETE */
+   284,  /* (386) nmnum ::= DEFAULT */
+   211,  /* (387) plus_num ::= INTEGER|FLOAT */
+   289,  /* (388) foreach_clause ::= */
+   289,  /* (389) foreach_clause ::= FOR EACH ROW */
+   292,  /* (390) trnm ::= nm */
+   293,  /* (391) tridxby ::= */
+   294,  /* (392) database_kw_opt ::= DATABASE */
+   294,  /* (393) database_kw_opt ::= */
+   297,  /* (394) kwcolumn_opt ::= */
+   297,  /* (395) kwcolumn_opt ::= COLUMNKW */
+   299,  /* (396) vtabarglist ::= vtabarg */
+   299,  /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */
+   300,  /* (398) vtabarg ::= vtabarg vtabargtoken */
+   303,  /* (399) anylist ::= */
+   303,  /* (400) anylist ::= anylist LP anylist RP */
+   303,  /* (401) anylist ::= anylist ANY */
+   266,  /* (402) with ::= */
 };
 
 /* For rule J, yyRuleInfoNRhs[J] contains the negative of the number
@@ -170134,233 +171040,231 @@ static const signed char yyRuleInfoNRhs[] = {
    -3,  /* (175) idlist ::= idlist COMMA nm */
    -1,  /* (176) idlist ::= nm */
    -3,  /* (177) expr ::= LP expr RP */
-   -1,  /* (178) expr ::= ID|INDEXED */
-   -1,  /* (179) expr ::= JOIN_KW */
-   -3,  /* (180) expr ::= nm DOT nm */
-   -5,  /* (181) expr ::= nm DOT nm DOT nm */
-   -1,  /* (182) term ::= NULL|FLOAT|BLOB */
-   -1,  /* (183) term ::= STRING */
-   -1,  /* (184) term ::= INTEGER */
-   -1,  /* (185) expr ::= VARIABLE */
-   -3,  /* (186) expr ::= expr COLLATE ID|STRING */
-   -6,  /* (187) expr ::= CAST LP expr AS typetoken RP */
-   -5,  /* (188) expr ::= ID|INDEXED LP distinct exprlist RP */
-   -4,  /* (189) expr ::= ID|INDEXED LP STAR RP */
-   -6,  /* (190) expr ::= ID|INDEXED LP distinct exprlist RP filter_over */
-   -5,  /* (191) expr ::= ID|INDEXED LP STAR RP filter_over */
-   -1,  /* (192) term ::= CTIME_KW */
-   -5,  /* (193) expr ::= LP nexprlist COMMA expr RP */
-   -3,  /* (194) expr ::= expr AND expr */
-   -3,  /* (195) expr ::= expr OR expr */
-   -3,  /* (196) expr ::= expr LT|GT|GE|LE expr */
-   -3,  /* (197) expr ::= expr EQ|NE expr */
-   -3,  /* (198) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */
-   -3,  /* (199) expr ::= expr PLUS|MINUS expr */
-   -3,  /* (200) expr ::= expr STAR|SLASH|REM expr */
-   -3,  /* (201) expr ::= expr CONCAT expr */
-   -2,  /* (202) likeop ::= NOT LIKE_KW|MATCH */
-   -3,  /* (203) expr ::= expr likeop expr */
-   -5,  /* (204) expr ::= expr likeop expr ESCAPE expr */
-   -2,  /* (205) expr ::= expr ISNULL|NOTNULL */
-   -3,  /* (206) expr ::= expr NOT NULL */
-   -3,  /* (207) expr ::= expr IS expr */
-   -4,  /* (208) expr ::= expr IS NOT expr */
-   -6,  /* (209) expr ::= expr IS NOT DISTINCT FROM expr */
-   -5,  /* (210) expr ::= expr IS DISTINCT FROM expr */
-   -2,  /* (211) expr ::= NOT expr */
-   -2,  /* (212) expr ::= BITNOT expr */
-   -2,  /* (213) expr ::= PLUS|MINUS expr */
-   -3,  /* (214) expr ::= expr PTR expr */
-   -1,  /* (215) between_op ::= BETWEEN */
-   -2,  /* (216) between_op ::= NOT BETWEEN */
-   -5,  /* (217) expr ::= expr between_op expr AND expr */
-   -1,  /* (218) in_op ::= IN */
-   -2,  /* (219) in_op ::= NOT IN */
-   -5,  /* (220) expr ::= expr in_op LP exprlist RP */
-   -3,  /* (221) expr ::= LP select RP */
-   -5,  /* (222) expr ::= expr in_op LP select RP */
-   -5,  /* (223) expr ::= expr in_op nm dbnm paren_exprlist */
-   -4,  /* (224) expr ::= EXISTS LP select RP */
-   -5,  /* (225) expr ::= CASE case_operand case_exprlist case_else END */
-   -5,  /* (226) case_exprlist ::= case_exprlist WHEN expr THEN expr */
-   -4,  /* (227) case_exprlist ::= WHEN expr THEN expr */
-   -2,  /* (228) case_else ::= ELSE expr */
-    0,  /* (229) case_else ::= */
-   -1,  /* (230) case_operand ::= expr */
-    0,  /* (231) case_operand ::= */
-    0,  /* (232) exprlist ::= */
-   -3,  /* (233) nexprlist ::= nexprlist COMMA expr */
-   -1,  /* (234) nexprlist ::= expr */
-    0,  /* (235) paren_exprlist ::= */
-   -3,  /* (236) paren_exprlist ::= LP exprlist RP */
-  -12,  /* (237) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
-   -1,  /* (238) uniqueflag ::= UNIQUE */
-    0,  /* (239) uniqueflag ::= */
-    0,  /* (240) eidlist_opt ::= */
-   -3,  /* (241) eidlist_opt ::= LP eidlist RP */
-   -5,  /* (242) eidlist ::= eidlist COMMA nm collate sortorder */
-   -3,  /* (243) eidlist ::= nm collate sortorder */
-    0,  /* (244) collate ::= */
-   -2,  /* (245) collate ::= COLLATE ID|STRING */
-   -4,  /* (246) cmd ::= DROP INDEX ifexists fullname */
-   -2,  /* (247) cmd ::= VACUUM vinto */
-   -3,  /* (248) cmd ::= VACUUM nm vinto */
-   -2,  /* (249) vinto ::= INTO expr */
-    0,  /* (250) vinto ::= */
-   -3,  /* (251) cmd ::= PRAGMA nm dbnm */
-   -5,  /* (252) cmd ::= PRAGMA nm dbnm EQ nmnum */
-   -6,  /* (253) cmd ::= PRAGMA nm dbnm LP nmnum RP */
-   -5,  /* (254) cmd ::= PRAGMA nm dbnm EQ minus_num */
-   -6,  /* (255) cmd ::= PRAGMA nm dbnm LP minus_num RP */
-   -2,  /* (256) plus_num ::= PLUS INTEGER|FLOAT */
-   -2,  /* (257) minus_num ::= MINUS INTEGER|FLOAT */
-   -5,  /* (258) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
-  -11,  /* (259) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
-   -1,  /* (260) trigger_time ::= BEFORE|AFTER */
-   -2,  /* (261) trigger_time ::= INSTEAD OF */
-    0,  /* (262) trigger_time ::= */
-   -1,  /* (263) trigger_event ::= DELETE|INSERT */
-   -1,  /* (264) trigger_event ::= UPDATE */
-   -3,  /* (265) trigger_event ::= UPDATE OF idlist */
-    0,  /* (266) when_clause ::= */
-   -2,  /* (267) when_clause ::= WHEN expr */
-   -3,  /* (268) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
-   -2,  /* (269) trigger_cmd_list ::= trigger_cmd SEMI */
-   -3,  /* (270) trnm ::= nm DOT nm */
-   -3,  /* (271) tridxby ::= INDEXED BY nm */
-   -2,  /* (272) tridxby ::= NOT INDEXED */
-   -9,  /* (273) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
-   -8,  /* (274) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
-   -6,  /* (275) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
-   -3,  /* (276) trigger_cmd ::= scanpt select scanpt */
-   -4,  /* (277) expr ::= RAISE LP IGNORE RP */
-   -6,  /* (278) expr ::= RAISE LP raisetype COMMA nm RP */
-   -1,  /* (279) raisetype ::= ROLLBACK */
-   -1,  /* (280) raisetype ::= ABORT */
-   -1,  /* (281) raisetype ::= FAIL */
-   -4,  /* (282) cmd ::= DROP TRIGGER ifexists fullname */
-   -6,  /* (283) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
-   -3,  /* (284) cmd ::= DETACH database_kw_opt expr */
-    0,  /* (285) key_opt ::= */
-   -2,  /* (286) key_opt ::= KEY expr */
-   -1,  /* (287) cmd ::= REINDEX */
-   -3,  /* (288) cmd ::= REINDEX nm dbnm */
-   -1,  /* (289) cmd ::= ANALYZE */
-   -3,  /* (290) cmd ::= ANALYZE nm dbnm */
-   -6,  /* (291) cmd ::= ALTER TABLE fullname RENAME TO nm */
-   -7,  /* (292) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
-   -6,  /* (293) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
-   -1,  /* (294) add_column_fullname ::= fullname */
-   -8,  /* (295) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
-   -1,  /* (296) cmd ::= create_vtab */
-   -4,  /* (297) cmd ::= create_vtab LP vtabarglist RP */
-   -8,  /* (298) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
-    0,  /* (299) vtabarg ::= */
-   -1,  /* (300) vtabargtoken ::= ANY */
-   -3,  /* (301) vtabargtoken ::= lp anylist RP */
-   -1,  /* (302) lp ::= LP */
-   -2,  /* (303) with ::= WITH wqlist */
-   -3,  /* (304) with ::= WITH RECURSIVE wqlist */
-   -1,  /* (305) wqas ::= AS */
-   -2,  /* (306) wqas ::= AS MATERIALIZED */
-   -3,  /* (307) wqas ::= AS NOT MATERIALIZED */
-   -6,  /* (308) wqitem ::= nm eidlist_opt wqas LP select RP */
-   -1,  /* (309) wqlist ::= wqitem */
-   -3,  /* (310) wqlist ::= wqlist COMMA wqitem */
-   -1,  /* (311) windowdefn_list ::= windowdefn */
-   -3,  /* (312) windowdefn_list ::= windowdefn_list COMMA windowdefn */
-   -5,  /* (313) windowdefn ::= nm AS LP window RP */
-   -5,  /* (314) window ::= PARTITION BY nexprlist orderby_opt frame_opt */
-   -6,  /* (315) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
-   -4,  /* (316) window ::= ORDER BY sortlist frame_opt */
-   -5,  /* (317) window ::= nm ORDER BY sortlist frame_opt */
-   -1,  /* (318) window ::= frame_opt */
-   -2,  /* (319) window ::= nm frame_opt */
-    0,  /* (320) frame_opt ::= */
-   -3,  /* (321) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
-   -6,  /* (322) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
-   -1,  /* (323) range_or_rows ::= RANGE|ROWS|GROUPS */
-   -1,  /* (324) frame_bound_s ::= frame_bound */
-   -2,  /* (325) frame_bound_s ::= UNBOUNDED PRECEDING */
-   -1,  /* (326) frame_bound_e ::= frame_bound */
-   -2,  /* (327) frame_bound_e ::= UNBOUNDED FOLLOWING */
-   -2,  /* (328) frame_bound ::= expr PRECEDING|FOLLOWING */
-   -2,  /* (329) frame_bound ::= CURRENT ROW */
-    0,  /* (330) frame_exclude_opt ::= */
-   -2,  /* (331) frame_exclude_opt ::= EXCLUDE frame_exclude */
-   -2,  /* (332) frame_exclude ::= NO OTHERS */
-   -2,  /* (333) frame_exclude ::= CURRENT ROW */
-   -1,  /* (334) frame_exclude ::= GROUP|TIES */
-   -2,  /* (335) window_clause ::= WINDOW windowdefn_list */
-   -2,  /* (336) filter_over ::= filter_clause over_clause */
-   -1,  /* (337) filter_over ::= over_clause */
-   -1,  /* (338) filter_over ::= filter_clause */
-   -4,  /* (339) over_clause ::= OVER LP window RP */
-   -2,  /* (340) over_clause ::= OVER nm */
-   -5,  /* (341) filter_clause ::= FILTER LP WHERE expr RP */
-   -1,  /* (342) input ::= cmdlist */
-   -2,  /* (343) cmdlist ::= cmdlist ecmd */
-   -1,  /* (344) cmdlist ::= ecmd */
-   -1,  /* (345) ecmd ::= SEMI */
-   -2,  /* (346) ecmd ::= cmdx SEMI */
-   -3,  /* (347) ecmd ::= explain cmdx SEMI */
-    0,  /* (348) trans_opt ::= */
-   -1,  /* (349) trans_opt ::= TRANSACTION */
-   -2,  /* (350) trans_opt ::= TRANSACTION nm */
-   -1,  /* (351) savepoint_opt ::= SAVEPOINT */
-    0,  /* (352) savepoint_opt ::= */
-   -2,  /* (353) cmd ::= create_table create_table_args */
-   -1,  /* (354) table_option_set ::= table_option */
-   -4,  /* (355) columnlist ::= columnlist COMMA columnname carglist */
-   -2,  /* (356) columnlist ::= columnname carglist */
-   -1,  /* (357) nm ::= ID|INDEXED */
-   -1,  /* (358) nm ::= STRING */
-   -1,  /* (359) nm ::= JOIN_KW */
-   -1,  /* (360) typetoken ::= typename */
-   -1,  /* (361) typename ::= ID|STRING */
-   -1,  /* (362) signed ::= plus_num */
-   -1,  /* (363) signed ::= minus_num */
-   -2,  /* (364) carglist ::= carglist ccons */
-    0,  /* (365) carglist ::= */
-   -2,  /* (366) ccons ::= NULL onconf */
-   -4,  /* (367) ccons ::= GENERATED ALWAYS AS generated */
-   -2,  /* (368) ccons ::= AS generated */
-   -2,  /* (369) conslist_opt ::= COMMA conslist */
-   -3,  /* (370) conslist ::= conslist tconscomma tcons */
-   -1,  /* (371) conslist ::= tcons */
-    0,  /* (372) tconscomma ::= */
-   -1,  /* (373) defer_subclause_opt ::= defer_subclause */
-   -1,  /* (374) resolvetype ::= raisetype */
-   -1,  /* (375) selectnowith ::= oneselect */
-   -1,  /* (376) oneselect ::= values */
-   -2,  /* (377) sclp ::= selcollist COMMA */
-   -1,  /* (378) as ::= ID|STRING */
-   -1,  /* (379) indexed_opt ::= indexed_by */
-    0,  /* (380) returning ::= */
-   -1,  /* (381) expr ::= term */
-   -1,  /* (382) likeop ::= LIKE_KW|MATCH */
-   -1,  /* (383) exprlist ::= nexprlist */
-   -1,  /* (384) nmnum ::= plus_num */
-   -1,  /* (385) nmnum ::= nm */
-   -1,  /* (386) nmnum ::= ON */
-   -1,  /* (387) nmnum ::= DELETE */
-   -1,  /* (388) nmnum ::= DEFAULT */
-   -1,  /* (389) plus_num ::= INTEGER|FLOAT */
-    0,  /* (390) foreach_clause ::= */
-   -3,  /* (391) foreach_clause ::= FOR EACH ROW */
-   -1,  /* (392) trnm ::= nm */
-    0,  /* (393) tridxby ::= */
-   -1,  /* (394) database_kw_opt ::= DATABASE */
-    0,  /* (395) database_kw_opt ::= */
-    0,  /* (396) kwcolumn_opt ::= */
-   -1,  /* (397) kwcolumn_opt ::= COLUMNKW */
-   -1,  /* (398) vtabarglist ::= vtabarg */
-   -3,  /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */
-   -2,  /* (400) vtabarg ::= vtabarg vtabargtoken */
-    0,  /* (401) anylist ::= */
-   -4,  /* (402) anylist ::= anylist LP anylist RP */
-   -2,  /* (403) anylist ::= anylist ANY */
-    0,  /* (404) with ::= */
+   -1,  /* (178) expr ::= ID|INDEXED|JOIN_KW */
+   -3,  /* (179) expr ::= nm DOT nm */
+   -5,  /* (180) expr ::= nm DOT nm DOT nm */
+   -1,  /* (181) term ::= NULL|FLOAT|BLOB */
+   -1,  /* (182) term ::= STRING */
+   -1,  /* (183) term ::= INTEGER */
+   -1,  /* (184) expr ::= VARIABLE */
+   -3,  /* (185) expr ::= expr COLLATE ID|STRING */
+   -6,  /* (186) expr ::= CAST LP expr AS typetoken RP */
+   -5,  /* (187) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */
+   -4,  /* (188) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */
+   -6,  /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */
+   -5,  /* (190) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */
+   -1,  /* (191) term ::= CTIME_KW */
+   -5,  /* (192) expr ::= LP nexprlist COMMA expr RP */
+   -3,  /* (193) expr ::= expr AND expr */
+   -3,  /* (194) expr ::= expr OR expr */
+   -3,  /* (195) expr ::= expr LT|GT|GE|LE expr */
+   -3,  /* (196) expr ::= expr EQ|NE expr */
+   -3,  /* (197) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */
+   -3,  /* (198) expr ::= expr PLUS|MINUS expr */
+   -3,  /* (199) expr ::= expr STAR|SLASH|REM expr */
+   -3,  /* (200) expr ::= expr CONCAT expr */
+   -2,  /* (201) likeop ::= NOT LIKE_KW|MATCH */
+   -3,  /* (202) expr ::= expr likeop expr */
+   -5,  /* (203) expr ::= expr likeop expr ESCAPE expr */
+   -2,  /* (204) expr ::= expr ISNULL|NOTNULL */
+   -3,  /* (205) expr ::= expr NOT NULL */
+   -3,  /* (206) expr ::= expr IS expr */
+   -4,  /* (207) expr ::= expr IS NOT expr */
+   -6,  /* (208) expr ::= expr IS NOT DISTINCT FROM expr */
+   -5,  /* (209) expr ::= expr IS DISTINCT FROM expr */
+   -2,  /* (210) expr ::= NOT expr */
+   -2,  /* (211) expr ::= BITNOT expr */
+   -2,  /* (212) expr ::= PLUS|MINUS expr */
+   -3,  /* (213) expr ::= expr PTR expr */
+   -1,  /* (214) between_op ::= BETWEEN */
+   -2,  /* (215) between_op ::= NOT BETWEEN */
+   -5,  /* (216) expr ::= expr between_op expr AND expr */
+   -1,  /* (217) in_op ::= IN */
+   -2,  /* (218) in_op ::= NOT IN */
+   -5,  /* (219) expr ::= expr in_op LP exprlist RP */
+   -3,  /* (220) expr ::= LP select RP */
+   -5,  /* (221) expr ::= expr in_op LP select RP */
+   -5,  /* (222) expr ::= expr in_op nm dbnm paren_exprlist */
+   -4,  /* (223) expr ::= EXISTS LP select RP */
+   -5,  /* (224) expr ::= CASE case_operand case_exprlist case_else END */
+   -5,  /* (225) case_exprlist ::= case_exprlist WHEN expr THEN expr */
+   -4,  /* (226) case_exprlist ::= WHEN expr THEN expr */
+   -2,  /* (227) case_else ::= ELSE expr */
+    0,  /* (228) case_else ::= */
+    0,  /* (229) case_operand ::= */
+    0,  /* (230) exprlist ::= */
+   -3,  /* (231) nexprlist ::= nexprlist COMMA expr */
+   -1,  /* (232) nexprlist ::= expr */
+    0,  /* (233) paren_exprlist ::= */
+   -3,  /* (234) paren_exprlist ::= LP exprlist RP */
+  -12,  /* (235) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
+   -1,  /* (236) uniqueflag ::= UNIQUE */
+    0,  /* (237) uniqueflag ::= */
+    0,  /* (238) eidlist_opt ::= */
+   -3,  /* (239) eidlist_opt ::= LP eidlist RP */
+   -5,  /* (240) eidlist ::= eidlist COMMA nm collate sortorder */
+   -3,  /* (241) eidlist ::= nm collate sortorder */
+    0,  /* (242) collate ::= */
+   -2,  /* (243) collate ::= COLLATE ID|STRING */
+   -4,  /* (244) cmd ::= DROP INDEX ifexists fullname */
+   -2,  /* (245) cmd ::= VACUUM vinto */
+   -3,  /* (246) cmd ::= VACUUM nm vinto */
+   -2,  /* (247) vinto ::= INTO expr */
+    0,  /* (248) vinto ::= */
+   -3,  /* (249) cmd ::= PRAGMA nm dbnm */
+   -5,  /* (250) cmd ::= PRAGMA nm dbnm EQ nmnum */
+   -6,  /* (251) cmd ::= PRAGMA nm dbnm LP nmnum RP */
+   -5,  /* (252) cmd ::= PRAGMA nm dbnm EQ minus_num */
+   -6,  /* (253) cmd ::= PRAGMA nm dbnm LP minus_num RP */
+   -2,  /* (254) plus_num ::= PLUS INTEGER|FLOAT */
+   -2,  /* (255) minus_num ::= MINUS INTEGER|FLOAT */
+   -5,  /* (256) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
+  -11,  /* (257) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
+   -1,  /* (258) trigger_time ::= BEFORE|AFTER */
+   -2,  /* (259) trigger_time ::= INSTEAD OF */
+    0,  /* (260) trigger_time ::= */
+   -1,  /* (261) trigger_event ::= DELETE|INSERT */
+   -1,  /* (262) trigger_event ::= UPDATE */
+   -3,  /* (263) trigger_event ::= UPDATE OF idlist */
+    0,  /* (264) when_clause ::= */
+   -2,  /* (265) when_clause ::= WHEN expr */
+   -3,  /* (266) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
+   -2,  /* (267) trigger_cmd_list ::= trigger_cmd SEMI */
+   -3,  /* (268) trnm ::= nm DOT nm */
+   -3,  /* (269) tridxby ::= INDEXED BY nm */
+   -2,  /* (270) tridxby ::= NOT INDEXED */
+   -9,  /* (271) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
+   -8,  /* (272) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
+   -6,  /* (273) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
+   -3,  /* (274) trigger_cmd ::= scanpt select scanpt */
+   -4,  /* (275) expr ::= RAISE LP IGNORE RP */
+   -6,  /* (276) expr ::= RAISE LP raisetype COMMA nm RP */
+   -1,  /* (277) raisetype ::= ROLLBACK */
+   -1,  /* (278) raisetype ::= ABORT */
+   -1,  /* (279) raisetype ::= FAIL */
+   -4,  /* (280) cmd ::= DROP TRIGGER ifexists fullname */
+   -6,  /* (281) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
+   -3,  /* (282) cmd ::= DETACH database_kw_opt expr */
+    0,  /* (283) key_opt ::= */
+   -2,  /* (284) key_opt ::= KEY expr */
+   -1,  /* (285) cmd ::= REINDEX */
+   -3,  /* (286) cmd ::= REINDEX nm dbnm */
+   -1,  /* (287) cmd ::= ANALYZE */
+   -3,  /* (288) cmd ::= ANALYZE nm dbnm */
+   -6,  /* (289) cmd ::= ALTER TABLE fullname RENAME TO nm */
+   -7,  /* (290) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
+   -6,  /* (291) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
+   -1,  /* (292) add_column_fullname ::= fullname */
+   -8,  /* (293) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
+   -1,  /* (294) cmd ::= create_vtab */
+   -4,  /* (295) cmd ::= create_vtab LP vtabarglist RP */
+   -8,  /* (296) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
+    0,  /* (297) vtabarg ::= */
+   -1,  /* (298) vtabargtoken ::= ANY */
+   -3,  /* (299) vtabargtoken ::= lp anylist RP */
+   -1,  /* (300) lp ::= LP */
+   -2,  /* (301) with ::= WITH wqlist */
+   -3,  /* (302) with ::= WITH RECURSIVE wqlist */
+   -1,  /* (303) wqas ::= AS */
+   -2,  /* (304) wqas ::= AS MATERIALIZED */
+   -3,  /* (305) wqas ::= AS NOT MATERIALIZED */
+   -6,  /* (306) wqitem ::= nm eidlist_opt wqas LP select RP */
+   -1,  /* (307) wqlist ::= wqitem */
+   -3,  /* (308) wqlist ::= wqlist COMMA wqitem */
+   -1,  /* (309) windowdefn_list ::= windowdefn */
+   -3,  /* (310) windowdefn_list ::= windowdefn_list COMMA windowdefn */
+   -5,  /* (311) windowdefn ::= nm AS LP window RP */
+   -5,  /* (312) window ::= PARTITION BY nexprlist orderby_opt frame_opt */
+   -6,  /* (313) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
+   -4,  /* (314) window ::= ORDER BY sortlist frame_opt */
+   -5,  /* (315) window ::= nm ORDER BY sortlist frame_opt */
+   -1,  /* (316) window ::= frame_opt */
+   -2,  /* (317) window ::= nm frame_opt */
+    0,  /* (318) frame_opt ::= */
+   -3,  /* (319) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
+   -6,  /* (320) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
+   -1,  /* (321) range_or_rows ::= RANGE|ROWS|GROUPS */
+   -1,  /* (322) frame_bound_s ::= frame_bound */
+   -2,  /* (323) frame_bound_s ::= UNBOUNDED PRECEDING */
+   -1,  /* (324) frame_bound_e ::= frame_bound */
+   -2,  /* (325) frame_bound_e ::= UNBOUNDED FOLLOWING */
+   -2,  /* (326) frame_bound ::= expr PRECEDING|FOLLOWING */
+   -2,  /* (327) frame_bound ::= CURRENT ROW */
+    0,  /* (328) frame_exclude_opt ::= */
+   -2,  /* (329) frame_exclude_opt ::= EXCLUDE frame_exclude */
+   -2,  /* (330) frame_exclude ::= NO OTHERS */
+   -2,  /* (331) frame_exclude ::= CURRENT ROW */
+   -1,  /* (332) frame_exclude ::= GROUP|TIES */
+   -2,  /* (333) window_clause ::= WINDOW windowdefn_list */
+   -2,  /* (334) filter_over ::= filter_clause over_clause */
+   -1,  /* (335) filter_over ::= over_clause */
+   -1,  /* (336) filter_over ::= filter_clause */
+   -4,  /* (337) over_clause ::= OVER LP window RP */
+   -2,  /* (338) over_clause ::= OVER nm */
+   -5,  /* (339) filter_clause ::= FILTER LP WHERE expr RP */
+   -1,  /* (340) input ::= cmdlist */
+   -2,  /* (341) cmdlist ::= cmdlist ecmd */
+   -1,  /* (342) cmdlist ::= ecmd */
+   -1,  /* (343) ecmd ::= SEMI */
+   -2,  /* (344) ecmd ::= cmdx SEMI */
+   -3,  /* (345) ecmd ::= explain cmdx SEMI */
+    0,  /* (346) trans_opt ::= */
+   -1,  /* (347) trans_opt ::= TRANSACTION */
+   -2,  /* (348) trans_opt ::= TRANSACTION nm */
+   -1,  /* (349) savepoint_opt ::= SAVEPOINT */
+    0,  /* (350) savepoint_opt ::= */
+   -2,  /* (351) cmd ::= create_table create_table_args */
+   -1,  /* (352) table_option_set ::= table_option */
+   -4,  /* (353) columnlist ::= columnlist COMMA columnname carglist */
+   -2,  /* (354) columnlist ::= columnname carglist */
+   -1,  /* (355) nm ::= ID|INDEXED|JOIN_KW */
+   -1,  /* (356) nm ::= STRING */
+   -1,  /* (357) typetoken ::= typename */
+   -1,  /* (358) typename ::= ID|STRING */
+   -1,  /* (359) signed ::= plus_num */
+   -1,  /* (360) signed ::= minus_num */
+   -2,  /* (361) carglist ::= carglist ccons */
+    0,  /* (362) carglist ::= */
+   -2,  /* (363) ccons ::= NULL onconf */
+   -4,  /* (364) ccons ::= GENERATED ALWAYS AS generated */
+   -2,  /* (365) ccons ::= AS generated */
+   -2,  /* (366) conslist_opt ::= COMMA conslist */
+   -3,  /* (367) conslist ::= conslist tconscomma tcons */
+   -1,  /* (368) conslist ::= tcons */
+    0,  /* (369) tconscomma ::= */
+   -1,  /* (370) defer_subclause_opt ::= defer_subclause */
+   -1,  /* (371) resolvetype ::= raisetype */
+   -1,  /* (372) selectnowith ::= oneselect */
+   -1,  /* (373) oneselect ::= values */
+   -2,  /* (374) sclp ::= selcollist COMMA */
+   -1,  /* (375) as ::= ID|STRING */
+   -1,  /* (376) indexed_opt ::= indexed_by */
+    0,  /* (377) returning ::= */
+   -1,  /* (378) expr ::= term */
+   -1,  /* (379) likeop ::= LIKE_KW|MATCH */
+   -1,  /* (380) case_operand ::= expr */
+   -1,  /* (381) exprlist ::= nexprlist */
+   -1,  /* (382) nmnum ::= plus_num */
+   -1,  /* (383) nmnum ::= nm */
+   -1,  /* (384) nmnum ::= ON */
+   -1,  /* (385) nmnum ::= DELETE */
+   -1,  /* (386) nmnum ::= DEFAULT */
+   -1,  /* (387) plus_num ::= INTEGER|FLOAT */
+    0,  /* (388) foreach_clause ::= */
+   -3,  /* (389) foreach_clause ::= FOR EACH ROW */
+   -1,  /* (390) trnm ::= nm */
+    0,  /* (391) tridxby ::= */
+   -1,  /* (392) database_kw_opt ::= DATABASE */
+    0,  /* (393) database_kw_opt ::= */
+    0,  /* (394) kwcolumn_opt ::= */
+   -1,  /* (395) kwcolumn_opt ::= COLUMNKW */
+   -1,  /* (396) vtabarglist ::= vtabarg */
+   -3,  /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */
+   -2,  /* (398) vtabarg ::= vtabarg vtabargtoken */
+    0,  /* (399) anylist ::= */
+   -4,  /* (400) anylist ::= anylist LP anylist RP */
+   -2,  /* (401) anylist ::= anylist ANY */
+    0,  /* (402) with ::= */
 };
 
 static void yy_accept(yyParser*);  /* Forward Declaration */
@@ -170420,7 +171324,7 @@ static YYACTIONTYPE yy_reduce(
       case 5: /* transtype ::= DEFERRED */
       case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6);
       case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7);
-      case 323: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==323);
+      case 321: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==321);
 {yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/}
         break;
       case 8: /* cmd ::= COMMIT|END trans_opt */
@@ -170457,7 +171361,7 @@ static YYACTIONTYPE yy_reduce(
       case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72);
       case 81: /* ifexists ::= */ yytestcase(yyruleno==81);
       case 98: /* distinct ::= */ yytestcase(yyruleno==98);
-      case 244: /* collate ::= */ yytestcase(yyruleno==244);
+      case 242: /* collate ::= */ yytestcase(yyruleno==242);
 {yymsp[1].minor.yy394 = 0;}
         break;
       case 16: /* ifnotexists ::= IF NOT EXISTS */
@@ -170641,9 +171545,9 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */
       case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80);
-      case 216: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==216);
-      case 219: /* in_op ::= NOT IN */ yytestcase(yyruleno==219);
-      case 245: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==245);
+      case 215: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==215);
+      case 218: /* in_op ::= NOT IN */ yytestcase(yyruleno==218);
+      case 243: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==243);
 {yymsp[-1].minor.yy394 = 1;}
         break;
       case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */
@@ -170793,9 +171697,9 @@ static YYACTIONTYPE yy_reduce(
       case 99: /* sclp ::= */
       case 132: /* orderby_opt ::= */ yytestcase(yyruleno==132);
       case 142: /* groupby_opt ::= */ yytestcase(yyruleno==142);
-      case 232: /* exprlist ::= */ yytestcase(yyruleno==232);
-      case 235: /* paren_exprlist ::= */ yytestcase(yyruleno==235);
-      case 240: /* eidlist_opt ::= */ yytestcase(yyruleno==240);
+      case 230: /* exprlist ::= */ yytestcase(yyruleno==230);
+      case 233: /* paren_exprlist ::= */ yytestcase(yyruleno==233);
+      case 238: /* eidlist_opt ::= */ yytestcase(yyruleno==238);
 {yymsp[1].minor.yy322 = 0;}
         break;
       case 100: /* selcollist ::= sclp scanpt expr scanpt as */
@@ -170821,8 +171725,8 @@ static YYACTIONTYPE yy_reduce(
         break;
       case 103: /* as ::= AS nm */
       case 115: /* dbnm ::= DOT nm */ yytestcase(yyruleno==115);
-      case 256: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==256);
-      case 257: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==257);
+      case 254: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==254);
+      case 255: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==255);
 {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;}
         break;
       case 105: /* from ::= */
@@ -170866,7 +171770,7 @@ static YYACTIONTYPE yy_reduce(
 {
     if( yymsp[-5].minor.yy131==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy561.pOn==0 && yymsp[0].minor.yy561.pUsing==0 ){
       yymsp[-5].minor.yy131 = yymsp[-3].minor.yy131;
-    }else if( yymsp[-3].minor.yy131->nSrc==1 ){
+    }else if( ALWAYS(yymsp[-3].minor.yy131!=0) && yymsp[-3].minor.yy131->nSrc==1 ){
       yymsp[-5].minor.yy131 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy131,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy561);
       if( yymsp[-5].minor.yy131 ){
         SrcItem *pNew = &yymsp[-5].minor.yy131->a[yymsp[-5].minor.yy131->nSrc-1];
@@ -170994,16 +171898,16 @@ static YYACTIONTYPE yy_reduce(
       case 146: /* limit_opt ::= */ yytestcase(yyruleno==146);
       case 151: /* where_opt ::= */ yytestcase(yyruleno==151);
       case 153: /* where_opt_ret ::= */ yytestcase(yyruleno==153);
-      case 229: /* case_else ::= */ yytestcase(yyruleno==229);
-      case 231: /* case_operand ::= */ yytestcase(yyruleno==231);
-      case 250: /* vinto ::= */ yytestcase(yyruleno==250);
+      case 228: /* case_else ::= */ yytestcase(yyruleno==228);
+      case 229: /* case_operand ::= */ yytestcase(yyruleno==229);
+      case 248: /* vinto ::= */ yytestcase(yyruleno==248);
 {yymsp[1].minor.yy528 = 0;}
         break;
       case 145: /* having_opt ::= HAVING expr */
       case 152: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==152);
       case 154: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==154);
-      case 228: /* case_else ::= ELSE expr */ yytestcase(yyruleno==228);
-      case 249: /* vinto ::= INTO expr */ yytestcase(yyruleno==249);
+      case 227: /* case_else ::= ELSE expr */ yytestcase(yyruleno==227);
+      case 247: /* vinto ::= INTO expr */ yytestcase(yyruleno==247);
 {yymsp[-1].minor.yy528 = yymsp[0].minor.yy528;}
         break;
       case 147: /* limit_opt ::= LIMIT expr */
@@ -171129,11 +172033,10 @@ static YYACTIONTYPE yy_reduce(
       case 177: /* expr ::= LP expr RP */
 {yymsp[-2].minor.yy528 = yymsp[-1].minor.yy528;}
         break;
-      case 178: /* expr ::= ID|INDEXED */
-      case 179: /* expr ::= JOIN_KW */ yytestcase(yyruleno==179);
+      case 178: /* expr ::= ID|INDEXED|JOIN_KW */
 {yymsp[0].minor.yy528=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/}
         break;
-      case 180: /* expr ::= nm DOT nm */
+      case 179: /* expr ::= nm DOT nm */
 {
   Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0);
   Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0);
@@ -171141,7 +172044,7 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[-2].minor.yy528 = yylhsminor.yy528;
         break;
-      case 181: /* expr ::= nm DOT nm DOT nm */
+      case 180: /* expr ::= nm DOT nm DOT nm */
 {
   Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0);
   Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0);
@@ -171154,18 +172057,18 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[-4].minor.yy528 = yylhsminor.yy528;
         break;
-      case 182: /* term ::= NULL|FLOAT|BLOB */
-      case 183: /* term ::= STRING */ yytestcase(yyruleno==183);
+      case 181: /* term ::= NULL|FLOAT|BLOB */
+      case 182: /* term ::= STRING */ yytestcase(yyruleno==182);
 {yymsp[0].minor.yy528=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/}
         break;
-      case 184: /* term ::= INTEGER */
+      case 183: /* term ::= INTEGER */
 {
   yylhsminor.yy528 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1);
   if( yylhsminor.yy528 ) yylhsminor.yy528->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail);
 }
   yymsp[0].minor.yy528 = yylhsminor.yy528;
         break;
-      case 185: /* expr ::= VARIABLE */
+      case 184: /* expr ::= VARIABLE */
 {
   if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){
     u32 n = yymsp[0].minor.yy0.n;
@@ -171187,50 +172090,50 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 186: /* expr ::= expr COLLATE ID|STRING */
+      case 185: /* expr ::= expr COLLATE ID|STRING */
 {
   yymsp[-2].minor.yy528 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy528, &yymsp[0].minor.yy0, 1);
 }
         break;
-      case 187: /* expr ::= CAST LP expr AS typetoken RP */
+      case 186: /* expr ::= CAST LP expr AS typetoken RP */
 {
   yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1);
   sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy528, yymsp[-3].minor.yy528, 0);
 }
         break;
-      case 188: /* expr ::= ID|INDEXED LP distinct exprlist RP */
+      case 187: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */
 {
   yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy394);
 }
   yymsp[-4].minor.yy528 = yylhsminor.yy528;
         break;
-      case 189: /* expr ::= ID|INDEXED LP STAR RP */
+      case 188: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */
 {
   yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0);
 }
   yymsp[-3].minor.yy528 = yylhsminor.yy528;
         break;
-      case 190: /* expr ::= ID|INDEXED LP distinct exprlist RP filter_over */
+      case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */
 {
   yylhsminor.yy528 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy322, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy394);
   sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41);
 }
   yymsp[-5].minor.yy528 = yylhsminor.yy528;
         break;
-      case 191: /* expr ::= ID|INDEXED LP STAR RP filter_over */
+      case 190: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */
 {
   yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0);
   sqlite3WindowAttach(pParse, yylhsminor.yy528, yymsp[0].minor.yy41);
 }
   yymsp[-4].minor.yy528 = yylhsminor.yy528;
         break;
-      case 192: /* term ::= CTIME_KW */
+      case 191: /* term ::= CTIME_KW */
 {
   yylhsminor.yy528 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0);
 }
   yymsp[0].minor.yy528 = yylhsminor.yy528;
         break;
-      case 193: /* expr ::= LP nexprlist COMMA expr RP */
+      case 192: /* expr ::= LP nexprlist COMMA expr RP */
 {
   ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528);
   yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0);
@@ -171244,22 +172147,22 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 194: /* expr ::= expr AND expr */
+      case 193: /* expr ::= expr AND expr */
 {yymsp[-2].minor.yy528=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);}
         break;
-      case 195: /* expr ::= expr OR expr */
-      case 196: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==196);
-      case 197: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==197);
-      case 198: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==198);
-      case 199: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==199);
-      case 200: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==200);
-      case 201: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==201);
+      case 194: /* expr ::= expr OR expr */
+      case 195: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==195);
+      case 196: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==196);
+      case 197: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==197);
+      case 198: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==198);
+      case 199: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==199);
+      case 200: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==200);
 {yymsp[-2].minor.yy528=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);}
         break;
-      case 202: /* likeop ::= NOT LIKE_KW|MATCH */
+      case 201: /* likeop ::= NOT LIKE_KW|MATCH */
 {yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/}
         break;
-      case 203: /* expr ::= expr likeop expr */
+      case 202: /* expr ::= expr likeop expr */
 {
   ExprList *pList;
   int bNot = yymsp[-1].minor.yy0.n & 0x80000000;
@@ -171271,7 +172174,7 @@ static YYACTIONTYPE yy_reduce(
   if( yymsp[-2].minor.yy528 ) yymsp[-2].minor.yy528->flags |= EP_InfixFunc;
 }
         break;
-      case 204: /* expr ::= expr likeop expr ESCAPE expr */
+      case 203: /* expr ::= expr likeop expr ESCAPE expr */
 {
   ExprList *pList;
   int bNot = yymsp[-3].minor.yy0.n & 0x80000000;
@@ -171284,47 +172187,47 @@ static YYACTIONTYPE yy_reduce(
   if( yymsp[-4].minor.yy528 ) yymsp[-4].minor.yy528->flags |= EP_InfixFunc;
 }
         break;
-      case 205: /* expr ::= expr ISNULL|NOTNULL */
+      case 204: /* expr ::= expr ISNULL|NOTNULL */
 {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy528,0);}
         break;
-      case 206: /* expr ::= expr NOT NULL */
+      case 205: /* expr ::= expr NOT NULL */
 {yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy528,0);}
         break;
-      case 207: /* expr ::= expr IS expr */
+      case 206: /* expr ::= expr IS expr */
 {
   yymsp[-2].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy528,yymsp[0].minor.yy528);
   binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-2].minor.yy528, TK_ISNULL);
 }
         break;
-      case 208: /* expr ::= expr IS NOT expr */
+      case 207: /* expr ::= expr IS NOT expr */
 {
   yymsp[-3].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy528,yymsp[0].minor.yy528);
   binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-3].minor.yy528, TK_NOTNULL);
 }
         break;
-      case 209: /* expr ::= expr IS NOT DISTINCT FROM expr */
+      case 208: /* expr ::= expr IS NOT DISTINCT FROM expr */
 {
   yymsp[-5].minor.yy528 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy528,yymsp[0].minor.yy528);
   binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-5].minor.yy528, TK_ISNULL);
 }
         break;
-      case 210: /* expr ::= expr IS DISTINCT FROM expr */
+      case 209: /* expr ::= expr IS DISTINCT FROM expr */
 {
   yymsp[-4].minor.yy528 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy528,yymsp[0].minor.yy528);
   binaryToUnaryIfNull(pParse, yymsp[0].minor.yy528, yymsp[-4].minor.yy528, TK_NOTNULL);
 }
         break;
-      case 211: /* expr ::= NOT expr */
-      case 212: /* expr ::= BITNOT expr */ yytestcase(yyruleno==212);
+      case 210: /* expr ::= NOT expr */
+      case 211: /* expr ::= BITNOT expr */ yytestcase(yyruleno==211);
 {yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy528, 0);/*A-overwrites-B*/}
         break;
-      case 213: /* expr ::= PLUS|MINUS expr */
+      case 212: /* expr ::= PLUS|MINUS expr */
 {
   yymsp[-1].minor.yy528 = sqlite3PExpr(pParse, yymsp[-1].major==TK_PLUS ? TK_UPLUS : TK_UMINUS, yymsp[0].minor.yy528, 0);
   /*A-overwrites-B*/
 }
         break;
-      case 214: /* expr ::= expr PTR expr */
+      case 213: /* expr ::= expr PTR expr */
 {
   ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy528);
   pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy528);
@@ -171332,11 +172235,11 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[-2].minor.yy528 = yylhsminor.yy528;
         break;
-      case 215: /* between_op ::= BETWEEN */
-      case 218: /* in_op ::= IN */ yytestcase(yyruleno==218);
+      case 214: /* between_op ::= BETWEEN */
+      case 217: /* in_op ::= IN */ yytestcase(yyruleno==217);
 {yymsp[0].minor.yy394 = 0;}
         break;
-      case 217: /* expr ::= expr between_op expr AND expr */
+      case 216: /* expr ::= expr between_op expr AND expr */
 {
   ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528);
   pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy528);
@@ -171349,7 +172252,7 @@ static YYACTIONTYPE yy_reduce(
   if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0);
 }
         break;
-      case 220: /* expr ::= expr in_op LP exprlist RP */
+      case 219: /* expr ::= expr in_op LP exprlist RP */
 {
     if( yymsp[-1].minor.yy322==0 ){
       /* Expressions of the form
@@ -171395,20 +172298,20 @@ static YYACTIONTYPE yy_reduce(
     }
   }
         break;
-      case 221: /* expr ::= LP select RP */
+      case 220: /* expr ::= LP select RP */
 {
     yymsp[-2].minor.yy528 = sqlite3PExpr(pParse, TK_SELECT, 0, 0);
     sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy528, yymsp[-1].minor.yy47);
   }
         break;
-      case 222: /* expr ::= expr in_op LP select RP */
+      case 221: /* expr ::= expr in_op LP select RP */
 {
     yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy528, 0);
     sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy528, yymsp[-1].minor.yy47);
     if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0);
   }
         break;
-      case 223: /* expr ::= expr in_op nm dbnm paren_exprlist */
+      case 222: /* expr ::= expr in_op nm dbnm paren_exprlist */
 {
     SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);
     Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0);
@@ -171418,14 +172321,14 @@ static YYACTIONTYPE yy_reduce(
     if( yymsp[-3].minor.yy394 ) yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy528, 0);
   }
         break;
-      case 224: /* expr ::= EXISTS LP select RP */
+      case 223: /* expr ::= EXISTS LP select RP */
 {
     Expr *p;
     p = yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0);
     sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy47);
   }
         break;
-      case 225: /* expr ::= CASE case_operand case_exprlist case_else END */
+      case 224: /* expr ::= CASE case_operand case_exprlist case_else END */
 {
   yymsp[-4].minor.yy528 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy528, 0);
   if( yymsp[-4].minor.yy528 ){
@@ -171437,32 +172340,29 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 226: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */
+      case 225: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */
 {
   yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[-2].minor.yy528);
   yymsp[-4].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy322, yymsp[0].minor.yy528);
 }
         break;
-      case 227: /* case_exprlist ::= WHEN expr THEN expr */
+      case 226: /* case_exprlist ::= WHEN expr THEN expr */
 {
   yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy528);
   yymsp[-3].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy322, yymsp[0].minor.yy528);
 }
         break;
-      case 230: /* case_operand ::= expr */
-{yymsp[0].minor.yy528 = yymsp[0].minor.yy528; /*A-overwrites-X*/}
-        break;
-      case 233: /* nexprlist ::= nexprlist COMMA expr */
+      case 231: /* nexprlist ::= nexprlist COMMA expr */
 {yymsp[-2].minor.yy322 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy322,yymsp[0].minor.yy528);}
         break;
-      case 234: /* nexprlist ::= expr */
+      case 232: /* nexprlist ::= expr */
 {yymsp[0].minor.yy322 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy528); /*A-overwrites-Y*/}
         break;
-      case 236: /* paren_exprlist ::= LP exprlist RP */
-      case 241: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==241);
+      case 234: /* paren_exprlist ::= LP exprlist RP */
+      case 239: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==239);
 {yymsp[-2].minor.yy322 = yymsp[-1].minor.yy322;}
         break;
-      case 237: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
+      case 235: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */
 {
   sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0,
                      sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy322, yymsp[-10].minor.yy394,
@@ -171472,48 +172372,48 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 238: /* uniqueflag ::= UNIQUE */
-      case 280: /* raisetype ::= ABORT */ yytestcase(yyruleno==280);
+      case 236: /* uniqueflag ::= UNIQUE */
+      case 278: /* raisetype ::= ABORT */ yytestcase(yyruleno==278);
 {yymsp[0].minor.yy394 = OE_Abort;}
         break;
-      case 239: /* uniqueflag ::= */
+      case 237: /* uniqueflag ::= */
 {yymsp[1].minor.yy394 = OE_None;}
         break;
-      case 242: /* eidlist ::= eidlist COMMA nm collate sortorder */
+      case 240: /* eidlist ::= eidlist COMMA nm collate sortorder */
 {
   yymsp[-4].minor.yy322 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy322, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394);
 }
         break;
-      case 243: /* eidlist ::= nm collate sortorder */
+      case 241: /* eidlist ::= nm collate sortorder */
 {
   yymsp[-2].minor.yy322 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy394, yymsp[0].minor.yy394); /*A-overwrites-Y*/
 }
         break;
-      case 246: /* cmd ::= DROP INDEX ifexists fullname */
+      case 244: /* cmd ::= DROP INDEX ifexists fullname */
 {sqlite3DropIndex(pParse, yymsp[0].minor.yy131, yymsp[-1].minor.yy394);}
         break;
-      case 247: /* cmd ::= VACUUM vinto */
+      case 245: /* cmd ::= VACUUM vinto */
 {sqlite3Vacuum(pParse,0,yymsp[0].minor.yy528);}
         break;
-      case 248: /* cmd ::= VACUUM nm vinto */
+      case 246: /* cmd ::= VACUUM nm vinto */
 {sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy528);}
         break;
-      case 251: /* cmd ::= PRAGMA nm dbnm */
+      case 249: /* cmd ::= PRAGMA nm dbnm */
 {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);}
         break;
-      case 252: /* cmd ::= PRAGMA nm dbnm EQ nmnum */
+      case 250: /* cmd ::= PRAGMA nm dbnm EQ nmnum */
 {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);}
         break;
-      case 253: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */
+      case 251: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */
 {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);}
         break;
-      case 254: /* cmd ::= PRAGMA nm dbnm EQ minus_num */
+      case 252: /* cmd ::= PRAGMA nm dbnm EQ minus_num */
 {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);}
         break;
-      case 255: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */
+      case 253: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */
 {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);}
         break;
-      case 258: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
+      case 256: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */
 {
   Token all;
   all.z = yymsp[-3].minor.yy0.z;
@@ -171521,50 +172421,50 @@ static YYACTIONTYPE yy_reduce(
   sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy33, &all);
 }
         break;
-      case 259: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
+      case 257: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */
 {
   sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy394, yymsp[-4].minor.yy180.a, yymsp[-4].minor.yy180.b, yymsp[-2].minor.yy131, yymsp[0].minor.yy528, yymsp[-10].minor.yy394, yymsp[-8].minor.yy394);
   yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/
 }
         break;
-      case 260: /* trigger_time ::= BEFORE|AFTER */
+      case 258: /* trigger_time ::= BEFORE|AFTER */
 { yymsp[0].minor.yy394 = yymsp[0].major; /*A-overwrites-X*/ }
         break;
-      case 261: /* trigger_time ::= INSTEAD OF */
+      case 259: /* trigger_time ::= INSTEAD OF */
 { yymsp[-1].minor.yy394 = TK_INSTEAD;}
         break;
-      case 262: /* trigger_time ::= */
+      case 260: /* trigger_time ::= */
 { yymsp[1].minor.yy394 = TK_BEFORE; }
         break;
-      case 263: /* trigger_event ::= DELETE|INSERT */
-      case 264: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==264);
+      case 261: /* trigger_event ::= DELETE|INSERT */
+      case 262: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==262);
 {yymsp[0].minor.yy180.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy180.b = 0;}
         break;
-      case 265: /* trigger_event ::= UPDATE OF idlist */
+      case 263: /* trigger_event ::= UPDATE OF idlist */
 {yymsp[-2].minor.yy180.a = TK_UPDATE; yymsp[-2].minor.yy180.b = yymsp[0].minor.yy254;}
         break;
-      case 266: /* when_clause ::= */
-      case 285: /* key_opt ::= */ yytestcase(yyruleno==285);
+      case 264: /* when_clause ::= */
+      case 283: /* key_opt ::= */ yytestcase(yyruleno==283);
 { yymsp[1].minor.yy528 = 0; }
         break;
-      case 267: /* when_clause ::= WHEN expr */
-      case 286: /* key_opt ::= KEY expr */ yytestcase(yyruleno==286);
+      case 265: /* when_clause ::= WHEN expr */
+      case 284: /* key_opt ::= KEY expr */ yytestcase(yyruleno==284);
 { yymsp[-1].minor.yy528 = yymsp[0].minor.yy528; }
         break;
-      case 268: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
+      case 266: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */
 {
   assert( yymsp[-2].minor.yy33!=0 );
   yymsp[-2].minor.yy33->pLast->pNext = yymsp[-1].minor.yy33;
   yymsp[-2].minor.yy33->pLast = yymsp[-1].minor.yy33;
 }
         break;
-      case 269: /* trigger_cmd_list ::= trigger_cmd SEMI */
+      case 267: /* trigger_cmd_list ::= trigger_cmd SEMI */
 {
   assert( yymsp[-1].minor.yy33!=0 );
   yymsp[-1].minor.yy33->pLast = yymsp[-1].minor.yy33;
 }
         break;
-      case 270: /* trnm ::= nm DOT nm */
+      case 268: /* trnm ::= nm DOT nm */
 {
   yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;
   sqlite3ErrorMsg(pParse,
@@ -171572,39 +172472,39 @@ static YYACTIONTYPE yy_reduce(
         "statements within triggers");
 }
         break;
-      case 271: /* tridxby ::= INDEXED BY nm */
+      case 269: /* tridxby ::= INDEXED BY nm */
 {
   sqlite3ErrorMsg(pParse,
         "the INDEXED BY clause is not allowed on UPDATE or DELETE statements "
         "within triggers");
 }
         break;
-      case 272: /* tridxby ::= NOT INDEXED */
+      case 270: /* tridxby ::= NOT INDEXED */
 {
   sqlite3ErrorMsg(pParse,
         "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements "
         "within triggers");
 }
         break;
-      case 273: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
+      case 271: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */
 {yylhsminor.yy33 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy131, yymsp[-3].minor.yy322, yymsp[-1].minor.yy528, yymsp[-7].minor.yy394, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy522);}
   yymsp[-8].minor.yy33 = yylhsminor.yy33;
         break;
-      case 274: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
+      case 272: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */
 {
    yylhsminor.yy33 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy254,yymsp[-2].minor.yy47,yymsp[-6].minor.yy394,yymsp[-1].minor.yy444,yymsp[-7].minor.yy522,yymsp[0].minor.yy522);/*yylhsminor.yy33-overwrites-yymsp[-6].minor.yy394*/
 }
   yymsp[-7].minor.yy33 = yylhsminor.yy33;
         break;
-      case 275: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
+      case 273: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */
 {yylhsminor.yy33 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy528, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy522);}
   yymsp[-5].minor.yy33 = yylhsminor.yy33;
         break;
-      case 276: /* trigger_cmd ::= scanpt select scanpt */
+      case 274: /* trigger_cmd ::= scanpt select scanpt */
 {yylhsminor.yy33 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy47, yymsp[-2].minor.yy522, yymsp[0].minor.yy522); /*yylhsminor.yy33-overwrites-yymsp[-1].minor.yy47*/}
   yymsp[-2].minor.yy33 = yylhsminor.yy33;
         break;
-      case 277: /* expr ::= RAISE LP IGNORE RP */
+      case 275: /* expr ::= RAISE LP IGNORE RP */
 {
   yymsp[-3].minor.yy528 = sqlite3PExpr(pParse, TK_RAISE, 0, 0);
   if( yymsp[-3].minor.yy528 ){
@@ -171612,7 +172512,7 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 278: /* expr ::= RAISE LP raisetype COMMA nm RP */
+      case 276: /* expr ::= RAISE LP raisetype COMMA nm RP */
 {
   yymsp[-5].minor.yy528 = sqlite3ExprAlloc(pParse->db, TK_RAISE, &yymsp[-1].minor.yy0, 1);
   if( yymsp[-5].minor.yy528 ) {
@@ -171620,118 +172520,118 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 279: /* raisetype ::= ROLLBACK */
+      case 277: /* raisetype ::= ROLLBACK */
 {yymsp[0].minor.yy394 = OE_Rollback;}
         break;
-      case 281: /* raisetype ::= FAIL */
+      case 279: /* raisetype ::= FAIL */
 {yymsp[0].minor.yy394 = OE_Fail;}
         break;
-      case 282: /* cmd ::= DROP TRIGGER ifexists fullname */
+      case 280: /* cmd ::= DROP TRIGGER ifexists fullname */
 {
   sqlite3DropTrigger(pParse,yymsp[0].minor.yy131,yymsp[-1].minor.yy394);
 }
         break;
-      case 283: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
+      case 281: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */
 {
   sqlite3Attach(pParse, yymsp[-3].minor.yy528, yymsp[-1].minor.yy528, yymsp[0].minor.yy528);
 }
         break;
-      case 284: /* cmd ::= DETACH database_kw_opt expr */
+      case 282: /* cmd ::= DETACH database_kw_opt expr */
 {
   sqlite3Detach(pParse, yymsp[0].minor.yy528);
 }
         break;
-      case 287: /* cmd ::= REINDEX */
+      case 285: /* cmd ::= REINDEX */
 {sqlite3Reindex(pParse, 0, 0);}
         break;
-      case 288: /* cmd ::= REINDEX nm dbnm */
+      case 286: /* cmd ::= REINDEX nm dbnm */
 {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);}
         break;
-      case 289: /* cmd ::= ANALYZE */
+      case 287: /* cmd ::= ANALYZE */
 {sqlite3Analyze(pParse, 0, 0);}
         break;
-      case 290: /* cmd ::= ANALYZE nm dbnm */
+      case 288: /* cmd ::= ANALYZE nm dbnm */
 {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);}
         break;
-      case 291: /* cmd ::= ALTER TABLE fullname RENAME TO nm */
+      case 289: /* cmd ::= ALTER TABLE fullname RENAME TO nm */
 {
   sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy131,&yymsp[0].minor.yy0);
 }
         break;
-      case 292: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
+      case 290: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */
 {
   yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n;
   sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0);
 }
         break;
-      case 293: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
+      case 291: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */
 {
   sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy131, &yymsp[0].minor.yy0);
 }
         break;
-      case 294: /* add_column_fullname ::= fullname */
+      case 292: /* add_column_fullname ::= fullname */
 {
   disableLookaside(pParse);
   sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy131);
 }
         break;
-      case 295: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
+      case 293: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */
 {
   sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy131, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0);
 }
         break;
-      case 296: /* cmd ::= create_vtab */
+      case 294: /* cmd ::= create_vtab */
 {sqlite3VtabFinishParse(pParse,0);}
         break;
-      case 297: /* cmd ::= create_vtab LP vtabarglist RP */
+      case 295: /* cmd ::= create_vtab LP vtabarglist RP */
 {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);}
         break;
-      case 298: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
+      case 296: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */
 {
     sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy394);
 }
         break;
-      case 299: /* vtabarg ::= */
+      case 297: /* vtabarg ::= */
 {sqlite3VtabArgInit(pParse);}
         break;
-      case 300: /* vtabargtoken ::= ANY */
-      case 301: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==301);
-      case 302: /* lp ::= LP */ yytestcase(yyruleno==302);
+      case 298: /* vtabargtoken ::= ANY */
+      case 299: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==299);
+      case 300: /* lp ::= LP */ yytestcase(yyruleno==300);
 {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);}
         break;
-      case 303: /* with ::= WITH wqlist */
-      case 304: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==304);
+      case 301: /* with ::= WITH wqlist */
+      case 302: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==302);
 { sqlite3WithPush(pParse, yymsp[0].minor.yy521, 1); }
         break;
-      case 305: /* wqas ::= AS */
+      case 303: /* wqas ::= AS */
 {yymsp[0].minor.yy516 = M10d_Any;}
         break;
-      case 306: /* wqas ::= AS MATERIALIZED */
+      case 304: /* wqas ::= AS MATERIALIZED */
 {yymsp[-1].minor.yy516 = M10d_Yes;}
         break;
-      case 307: /* wqas ::= AS NOT MATERIALIZED */
+      case 305: /* wqas ::= AS NOT MATERIALIZED */
 {yymsp[-2].minor.yy516 = M10d_No;}
         break;
-      case 308: /* wqitem ::= nm eidlist_opt wqas LP select RP */
+      case 306: /* wqitem ::= nm eidlist_opt wqas LP select RP */
 {
   yymsp[-5].minor.yy385 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy322, yymsp[-1].minor.yy47, yymsp[-3].minor.yy516); /*A-overwrites-X*/
 }
         break;
-      case 309: /* wqlist ::= wqitem */
+      case 307: /* wqlist ::= wqitem */
 {
   yymsp[0].minor.yy521 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy385); /*A-overwrites-X*/
 }
         break;
-      case 310: /* wqlist ::= wqlist COMMA wqitem */
+      case 308: /* wqlist ::= wqlist COMMA wqitem */
 {
   yymsp[-2].minor.yy521 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy521, yymsp[0].minor.yy385);
 }
         break;
-      case 311: /* windowdefn_list ::= windowdefn */
+      case 309: /* windowdefn_list ::= windowdefn */
 { yylhsminor.yy41 = yymsp[0].minor.yy41; }
   yymsp[0].minor.yy41 = yylhsminor.yy41;
         break;
-      case 312: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */
+      case 310: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */
 {
   assert( yymsp[0].minor.yy41!=0 );
   sqlite3WindowChain(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy41);
@@ -171740,7 +172640,7 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[-2].minor.yy41 = yylhsminor.yy41;
         break;
-      case 313: /* windowdefn ::= nm AS LP window RP */
+      case 311: /* windowdefn ::= nm AS LP window RP */
 {
   if( ALWAYS(yymsp[-1].minor.yy41) ){
     yymsp[-1].minor.yy41->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n);
@@ -171749,90 +172649,90 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[-4].minor.yy41 = yylhsminor.yy41;
         break;
-      case 314: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */
+      case 312: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */
 {
   yymsp[-4].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, 0);
 }
         break;
-      case 315: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
+      case 313: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */
 {
   yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, yymsp[-2].minor.yy322, yymsp[-1].minor.yy322, &yymsp[-5].minor.yy0);
 }
   yymsp[-5].minor.yy41 = yylhsminor.yy41;
         break;
-      case 316: /* window ::= ORDER BY sortlist frame_opt */
+      case 314: /* window ::= ORDER BY sortlist frame_opt */
 {
   yymsp[-3].minor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, 0);
 }
         break;
-      case 317: /* window ::= nm ORDER BY sortlist frame_opt */
+      case 315: /* window ::= nm ORDER BY sortlist frame_opt */
 {
   yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, yymsp[-1].minor.yy322, &yymsp[-4].minor.yy0);
 }
   yymsp[-4].minor.yy41 = yylhsminor.yy41;
         break;
-      case 318: /* window ::= frame_opt */
-      case 337: /* filter_over ::= over_clause */ yytestcase(yyruleno==337);
+      case 316: /* window ::= frame_opt */
+      case 335: /* filter_over ::= over_clause */ yytestcase(yyruleno==335);
 {
   yylhsminor.yy41 = yymsp[0].minor.yy41;
 }
   yymsp[0].minor.yy41 = yylhsminor.yy41;
         break;
-      case 319: /* window ::= nm frame_opt */
+      case 317: /* window ::= nm frame_opt */
 {
   yylhsminor.yy41 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy41, 0, 0, &yymsp[-1].minor.yy0);
 }
   yymsp[-1].minor.yy41 = yylhsminor.yy41;
         break;
-      case 320: /* frame_opt ::= */
+      case 318: /* frame_opt ::= */
 {
   yymsp[1].minor.yy41 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0);
 }
         break;
-      case 321: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
+      case 319: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */
 {
   yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy394, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy516);
 }
   yymsp[-2].minor.yy41 = yylhsminor.yy41;
         break;
-      case 322: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
+      case 320: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */
 {
   yylhsminor.yy41 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy394, yymsp[-3].minor.yy595.eType, yymsp[-3].minor.yy595.pExpr, yymsp[-1].minor.yy595.eType, yymsp[-1].minor.yy595.pExpr, yymsp[0].minor.yy516);
 }
   yymsp[-5].minor.yy41 = yylhsminor.yy41;
         break;
-      case 324: /* frame_bound_s ::= frame_bound */
-      case 326: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==326);
+      case 322: /* frame_bound_s ::= frame_bound */
+      case 324: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==324);
 {yylhsminor.yy595 = yymsp[0].minor.yy595;}
   yymsp[0].minor.yy595 = yylhsminor.yy595;
         break;
-      case 325: /* frame_bound_s ::= UNBOUNDED PRECEDING */
-      case 327: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==327);
-      case 329: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==329);
+      case 323: /* frame_bound_s ::= UNBOUNDED PRECEDING */
+      case 325: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==325);
+      case 327: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==327);
 {yylhsminor.yy595.eType = yymsp[-1].major; yylhsminor.yy595.pExpr = 0;}
   yymsp[-1].minor.yy595 = yylhsminor.yy595;
         break;
-      case 328: /* frame_bound ::= expr PRECEDING|FOLLOWING */
+      case 326: /* frame_bound ::= expr PRECEDING|FOLLOWING */
 {yylhsminor.yy595.eType = yymsp[0].major; yylhsminor.yy595.pExpr = yymsp[-1].minor.yy528;}
   yymsp[-1].minor.yy595 = yylhsminor.yy595;
         break;
-      case 330: /* frame_exclude_opt ::= */
+      case 328: /* frame_exclude_opt ::= */
 {yymsp[1].minor.yy516 = 0;}
         break;
-      case 331: /* frame_exclude_opt ::= EXCLUDE frame_exclude */
+      case 329: /* frame_exclude_opt ::= EXCLUDE frame_exclude */
 {yymsp[-1].minor.yy516 = yymsp[0].minor.yy516;}
         break;
-      case 332: /* frame_exclude ::= NO OTHERS */
-      case 333: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==333);
+      case 330: /* frame_exclude ::= NO OTHERS */
+      case 331: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==331);
 {yymsp[-1].minor.yy516 = yymsp[-1].major; /*A-overwrites-X*/}
         break;
-      case 334: /* frame_exclude ::= GROUP|TIES */
+      case 332: /* frame_exclude ::= GROUP|TIES */
 {yymsp[0].minor.yy516 = yymsp[0].major; /*A-overwrites-X*/}
         break;
-      case 335: /* window_clause ::= WINDOW windowdefn_list */
+      case 333: /* window_clause ::= WINDOW windowdefn_list */
 { yymsp[-1].minor.yy41 = yymsp[0].minor.yy41; }
         break;
-      case 336: /* filter_over ::= filter_clause over_clause */
+      case 334: /* filter_over ::= filter_clause over_clause */
 {
   if( yymsp[0].minor.yy41 ){
     yymsp[0].minor.yy41->pFilter = yymsp[-1].minor.yy528;
@@ -171843,7 +172743,7 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[-1].minor.yy41 = yylhsminor.yy41;
         break;
-      case 338: /* filter_over ::= filter_clause */
+      case 336: /* filter_over ::= filter_clause */
 {
   yylhsminor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
   if( yylhsminor.yy41 ){
@@ -171855,13 +172755,13 @@ static YYACTIONTYPE yy_reduce(
 }
   yymsp[0].minor.yy41 = yylhsminor.yy41;
         break;
-      case 339: /* over_clause ::= OVER LP window RP */
+      case 337: /* over_clause ::= OVER LP window RP */
 {
   yymsp[-3].minor.yy41 = yymsp[-1].minor.yy41;
   assert( yymsp[-3].minor.yy41!=0 );
 }
         break;
-      case 340: /* over_clause ::= OVER nm */
+      case 338: /* over_clause ::= OVER nm */
 {
   yymsp[-1].minor.yy41 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window));
   if( yymsp[-1].minor.yy41 ){
@@ -171869,73 +172769,73 @@ static YYACTIONTYPE yy_reduce(
   }
 }
         break;
-      case 341: /* filter_clause ::= FILTER LP WHERE expr RP */
+      case 339: /* filter_clause ::= FILTER LP WHERE expr RP */
 { yymsp[-4].minor.yy528 = yymsp[-1].minor.yy528; }
         break;
       default:
-      /* (342) input ::= cmdlist */ yytestcase(yyruleno==342);
-      /* (343) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==343);
-      /* (344) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=344);
-      /* (345) ecmd ::= SEMI */ yytestcase(yyruleno==345);
-      /* (346) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==346);
-      /* (347) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=347);
-      /* (348) trans_opt ::= */ yytestcase(yyruleno==348);
-      /* (349) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==349);
-      /* (350) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==350);
-      /* (351) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==351);
-      /* (352) savepoint_opt ::= */ yytestcase(yyruleno==352);
-      /* (353) cmd ::= create_table create_table_args */ yytestcase(yyruleno==353);
-      /* (354) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=354);
-      /* (355) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==355);
-      /* (356) columnlist ::= columnname carglist */ yytestcase(yyruleno==356);
-      /* (357) nm ::= ID|INDEXED */ yytestcase(yyruleno==357);
-      /* (358) nm ::= STRING */ yytestcase(yyruleno==358);
-      /* (359) nm ::= JOIN_KW */ yytestcase(yyruleno==359);
-      /* (360) typetoken ::= typename */ yytestcase(yyruleno==360);
-      /* (361) typename ::= ID|STRING */ yytestcase(yyruleno==361);
-      /* (362) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=362);
-      /* (363) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=363);
-      /* (364) carglist ::= carglist ccons */ yytestcase(yyruleno==364);
-      /* (365) carglist ::= */ yytestcase(yyruleno==365);
-      /* (366) ccons ::= NULL onconf */ yytestcase(yyruleno==366);
-      /* (367) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==367);
-      /* (368) ccons ::= AS generated */ yytestcase(yyruleno==368);
-      /* (369) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==369);
-      /* (370) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==370);
-      /* (371) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=371);
-      /* (372) tconscomma ::= */ yytestcase(yyruleno==372);
-      /* (373) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=373);
-      /* (374) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=374);
-      /* (375) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=375);
-      /* (376) oneselect ::= values */ yytestcase(yyruleno==376);
-      /* (377) sclp ::= selcollist COMMA */ yytestcase(yyruleno==377);
-      /* (378) as ::= ID|STRING */ yytestcase(yyruleno==378);
-      /* (379) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=379);
-      /* (380) returning ::= */ yytestcase(yyruleno==380);
-      /* (381) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=381);
-      /* (382) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==382);
-      /* (383) exprlist ::= nexprlist */ yytestcase(yyruleno==383);
-      /* (384) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=384);
-      /* (385) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=385);
-      /* (386) nmnum ::= ON */ yytestcase(yyruleno==386);
-      /* (387) nmnum ::= DELETE */ yytestcase(yyruleno==387);
-      /* (388) nmnum ::= DEFAULT */ yytestcase(yyruleno==388);
-      /* (389) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==389);
-      /* (390) foreach_clause ::= */ yytestcase(yyruleno==390);
-      /* (391) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==391);
-      /* (392) trnm ::= nm */ yytestcase(yyruleno==392);
-      /* (393) tridxby ::= */ yytestcase(yyruleno==393);
-      /* (394) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==394);
-      /* (395) database_kw_opt ::= */ yytestcase(yyruleno==395);
-      /* (396) kwcolumn_opt ::= */ yytestcase(yyruleno==396);
-      /* (397) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==397);
-      /* (398) vtabarglist ::= vtabarg */ yytestcase(yyruleno==398);
-      /* (399) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==399);
-      /* (400) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==400);
-      /* (401) anylist ::= */ yytestcase(yyruleno==401);
-      /* (402) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==402);
-      /* (403) anylist ::= anylist ANY */ yytestcase(yyruleno==403);
-      /* (404) with ::= */ yytestcase(yyruleno==404);
+      /* (340) input ::= cmdlist */ yytestcase(yyruleno==340);
+      /* (341) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==341);
+      /* (342) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=342);
+      /* (343) ecmd ::= SEMI */ yytestcase(yyruleno==343);
+      /* (344) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==344);
+      /* (345) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=345);
+      /* (346) trans_opt ::= */ yytestcase(yyruleno==346);
+      /* (347) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==347);
+      /* (348) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==348);
+      /* (349) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==349);
+      /* (350) savepoint_opt ::= */ yytestcase(yyruleno==350);
+      /* (351) cmd ::= create_table create_table_args */ yytestcase(yyruleno==351);
+      /* (352) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=352);
+      /* (353) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==353);
+      /* (354) columnlist ::= columnname carglist */ yytestcase(yyruleno==354);
+      /* (355) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==355);
+      /* (356) nm ::= STRING */ yytestcase(yyruleno==356);
+      /* (357) typetoken ::= typename */ yytestcase(yyruleno==357);
+      /* (358) typename ::= ID|STRING */ yytestcase(yyruleno==358);
+      /* (359) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=359);
+      /* (360) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=360);
+      /* (361) carglist ::= carglist ccons */ yytestcase(yyruleno==361);
+      /* (362) carglist ::= */ yytestcase(yyruleno==362);
+      /* (363) ccons ::= NULL onconf */ yytestcase(yyruleno==363);
+      /* (364) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==364);
+      /* (365) ccons ::= AS generated */ yytestcase(yyruleno==365);
+      /* (366) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==366);
+      /* (367) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==367);
+      /* (368) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=368);
+      /* (369) tconscomma ::= */ yytestcase(yyruleno==369);
+      /* (370) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=370);
+      /* (371) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=371);
+      /* (372) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=372);
+      /* (373) oneselect ::= values */ yytestcase(yyruleno==373);
+      /* (374) sclp ::= selcollist COMMA */ yytestcase(yyruleno==374);
+      /* (375) as ::= ID|STRING */ yytestcase(yyruleno==375);
+      /* (376) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=376);
+      /* (377) returning ::= */ yytestcase(yyruleno==377);
+      /* (378) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=378);
+      /* (379) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==379);
+      /* (380) case_operand ::= expr */ yytestcase(yyruleno==380);
+      /* (381) exprlist ::= nexprlist */ yytestcase(yyruleno==381);
+      /* (382) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=382);
+      /* (383) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=383);
+      /* (384) nmnum ::= ON */ yytestcase(yyruleno==384);
+      /* (385) nmnum ::= DELETE */ yytestcase(yyruleno==385);
+      /* (386) nmnum ::= DEFAULT */ yytestcase(yyruleno==386);
+      /* (387) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==387);
+      /* (388) foreach_clause ::= */ yytestcase(yyruleno==388);
+      /* (389) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==389);
+      /* (390) trnm ::= nm */ yytestcase(yyruleno==390);
+      /* (391) tridxby ::= */ yytestcase(yyruleno==391);
+      /* (392) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==392);
+      /* (393) database_kw_opt ::= */ yytestcase(yyruleno==393);
+      /* (394) kwcolumn_opt ::= */ yytestcase(yyruleno==394);
+      /* (395) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==395);
+      /* (396) vtabarglist ::= vtabarg */ yytestcase(yyruleno==396);
+      /* (397) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==397);
+      /* (398) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==398);
+      /* (399) anylist ::= */ yytestcase(yyruleno==399);
+      /* (400) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==400);
+      /* (401) anylist ::= anylist ANY */ yytestcase(yyruleno==401);
+      /* (402) with ::= */ yytestcase(yyruleno==402);
         break;
 /********** End reduce actions ************************************************/
   };
@@ -172511,7 +173411,7 @@ static const unsigned char aKWHash[127] = {
 /* aKWNext[] forms the hash collision chain.  If aKWHash[i]==0
 ** then the i-th keyword has no more hash collisions.  Otherwise,
 ** the next keyword with the same hash is aKWHash[i]-1. */
-static const unsigned char aKWNext[147] = {
+static const unsigned char aKWNext[148] = {0,
      0,   0,   0,   0,   4,   0,  43,   0,   0, 106, 114,   0,   0,
      0,   2,   0,   0, 143,   0,   0,   0,  13,   0,   0,   0,   0,
    141,   0,   0, 119,  52,   0,   0, 137,  12,   0,   0,  62,   0,
@@ -172526,7 +173426,7 @@ static const unsigned char aKWNext[147] = {
    102,   0,   0,  87,
 };
 /* aKWLen[i] is the length (in bytes) of the i-th keyword */
-static const unsigned char aKWLen[147] = {
+static const unsigned char aKWLen[148] = {0,
      7,   7,   5,   4,   6,   4,   5,   3,   6,   7,   3,   6,   6,
      7,   7,   3,   8,   2,   6,   5,   4,   4,   3,  10,   4,   7,
      6,   9,   4,   2,   6,   5,   9,   9,   4,   7,   3,   2,   4,
@@ -172542,7 +173442,7 @@ static const unsigned char aKWLen[147] = {
 };
 /* aKWOffset[i] is the index into zKWText[] of the start of
 ** the text for the i-th keyword. */
-static const unsigned short int aKWOffset[147] = {
+static const unsigned short int aKWOffset[148] = {0,
      0,   2,   2,   8,   9,  14,  16,  20,  23,  25,  25,  29,  33,
     36,  41,  46,  48,  53,  54,  59,  62,  65,  67,  69,  78,  81,
     86,  90,  90,  94,  99, 101, 105, 111, 119, 123, 123, 123, 126,
@@ -172557,7 +173457,7 @@ static const unsigned short int aKWOffset[147] = {
    648, 650, 655, 659,
 };
 /* aKWCode[i] is the parser symbol code for the i-th keyword */
-static const unsigned char aKWCode[147] = {
+static const unsigned char aKWCode[148] = {0,
   TK_REINDEX,    TK_INDEXED,    TK_INDEX,      TK_DESC,       TK_ESCAPE,
   TK_EACH,       TK_CHECK,      TK_KEY,        TK_BEFORE,     TK_FOREIGN,
   TK_FOR,        TK_IGNORE,     TK_LIKE_KW,    TK_EXPLAIN,    TK_INSTEAD,
@@ -172726,7 +173626,7 @@ static int keywordCode(const char *z, int n, int *pType){
   const char *zKW;
   if( n>=2 ){
     i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127;
-    for(i=((int)aKWHash[i])-1; i>=0; i=((int)aKWNext[i])-1){
+    for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){
       if( aKWLen[i]!=n ) continue;
       zKW = &zKWText[aKWOffset[i]];
 #ifdef SQLITE_ASCII
@@ -172742,153 +173642,153 @@ static int keywordCode(const char *z, int n, int *pType){
       while( j<n && toupper(z[j])==zKW[j] ){ j++; }
 #endif
       if( j<n ) continue;
-      testcase( i==0 ); /* REINDEX */
-      testcase( i==1 ); /* INDEXED */
-      testcase( i==2 ); /* INDEX */
-      testcase( i==3 ); /* DESC */
-      testcase( i==4 ); /* ESCAPE */
-      testcase( i==5 ); /* EACH */
-      testcase( i==6 ); /* CHECK */
-      testcase( i==7 ); /* KEY */
-      testcase( i==8 ); /* BEFORE */
-      testcase( i==9 ); /* FOREIGN */
-      testcase( i==10 ); /* FOR */
-      testcase( i==11 ); /* IGNORE */
-      testcase( i==12 ); /* REGEXP */
-      testcase( i==13 ); /* EXPLAIN */
-      testcase( i==14 ); /* INSTEAD */
-      testcase( i==15 ); /* ADD */
-      testcase( i==16 ); /* DATABASE */
-      testcase( i==17 ); /* AS */
-      testcase( i==18 ); /* SELECT */
-      testcase( i==19 ); /* TABLE */
-      testcase( i==20 ); /* LEFT */
-      testcase( i==21 ); /* THEN */
-      testcase( i==22 ); /* END */
-      testcase( i==23 ); /* DEFERRABLE */
-      testcase( i==24 ); /* ELSE */
-      testcase( i==25 ); /* EXCLUDE */
-      testcase( i==26 ); /* DELETE */
-      testcase( i==27 ); /* TEMPORARY */
-      testcase( i==28 ); /* TEMP */
-      testcase( i==29 ); /* OR */
-      testcase( i==30 ); /* ISNULL */
-      testcase( i==31 ); /* NULLS */
-      testcase( i==32 ); /* SAVEPOINT */
-      testcase( i==33 ); /* INTERSECT */
-      testcase( i==34 ); /* TIES */
-      testcase( i==35 ); /* NOTNULL */
-      testcase( i==36 ); /* NOT */
-      testcase( i==37 ); /* NO */
-      testcase( i==38 ); /* NULL */
-      testcase( i==39 ); /* LIKE */
-      testcase( i==40 ); /* EXCEPT */
-      testcase( i==41 ); /* TRANSACTION */
-      testcase( i==42 ); /* ACTION */
-      testcase( i==43 ); /* ON */
-      testcase( i==44 ); /* NATURAL */
-      testcase( i==45 ); /* ALTER */
-      testcase( i==46 ); /* RAISE */
-      testcase( i==47 ); /* EXCLUSIVE */
-      testcase( i==48 ); /* EXISTS */
-      testcase( i==49 ); /* CONSTRAINT */
-      testcase( i==50 ); /* INTO */
-      testcase( i==51 ); /* OFFSET */
-      testcase( i==52 ); /* OF */
-      testcase( i==53 ); /* SET */
-      testcase( i==54 ); /* TRIGGER */
-      testcase( i==55 ); /* RANGE */
-      testcase( i==56 ); /* GENERATED */
-      testcase( i==57 ); /* DETACH */
-      testcase( i==58 ); /* HAVING */
-      testcase( i==59 ); /* GLOB */
-      testcase( i==60 ); /* BEGIN */
-      testcase( i==61 ); /* INNER */
-      testcase( i==62 ); /* REFERENCES */
-      testcase( i==63 ); /* UNIQUE */
-      testcase( i==64 ); /* QUERY */
-      testcase( i==65 ); /* WITHOUT */
-      testcase( i==66 ); /* WITH */
-      testcase( i==67 ); /* OUTER */
-      testcase( i==68 ); /* RELEASE */
-      testcase( i==69 ); /* ATTACH */
-      testcase( i==70 ); /* BETWEEN */
-      testcase( i==71 ); /* NOTHING */
-      testcase( i==72 ); /* GROUPS */
-      testcase( i==73 ); /* GROUP */
-      testcase( i==74 ); /* CASCADE */
-      testcase( i==75 ); /* ASC */
-      testcase( i==76 ); /* DEFAULT */
-      testcase( i==77 ); /* CASE */
-      testcase( i==78 ); /* COLLATE */
-      testcase( i==79 ); /* CREATE */
-      testcase( i==80 ); /* CURRENT_DATE */
-      testcase( i==81 ); /* IMMEDIATE */
-      testcase( i==82 ); /* JOIN */
-      testcase( i==83 ); /* INSERT */
-      testcase( i==84 ); /* MATCH */
-      testcase( i==85 ); /* PLAN */
-      testcase( i==86 ); /* ANALYZE */
-      testcase( i==87 ); /* PRAGMA */
-      testcase( i==88 ); /* MATERIALIZED */
-      testcase( i==89 ); /* DEFERRED */
-      testcase( i==90 ); /* DISTINCT */
-      testcase( i==91 ); /* IS */
-      testcase( i==92 ); /* UPDATE */
-      testcase( i==93 ); /* VALUES */
-      testcase( i==94 ); /* VIRTUAL */
-      testcase( i==95 ); /* ALWAYS */
-      testcase( i==96 ); /* WHEN */
-      testcase( i==97 ); /* WHERE */
-      testcase( i==98 ); /* RECURSIVE */
-      testcase( i==99 ); /* ABORT */
-      testcase( i==100 ); /* AFTER */
-      testcase( i==101 ); /* RENAME */
-      testcase( i==102 ); /* AND */
-      testcase( i==103 ); /* DROP */
-      testcase( i==104 ); /* PARTITION */
-      testcase( i==105 ); /* AUTOINCREMENT */
-      testcase( i==106 ); /* TO */
-      testcase( i==107 ); /* IN */
-      testcase( i==108 ); /* CAST */
-      testcase( i==109 ); /* COLUMN */
-      testcase( i==110 ); /* COMMIT */
-      testcase( i==111 ); /* CONFLICT */
-      testcase( i==112 ); /* CROSS */
-      testcase( i==113 ); /* CURRENT_TIMESTAMP */
-      testcase( i==114 ); /* CURRENT_TIME */
-      testcase( i==115 ); /* CURRENT */
-      testcase( i==116 ); /* PRECEDING */
-      testcase( i==117 ); /* FAIL */
-      testcase( i==118 ); /* LAST */
-      testcase( i==119 ); /* FILTER */
-      testcase( i==120 ); /* REPLACE */
-      testcase( i==121 ); /* FIRST */
-      testcase( i==122 ); /* FOLLOWING */
-      testcase( i==123 ); /* FROM */
-      testcase( i==124 ); /* FULL */
-      testcase( i==125 ); /* LIMIT */
-      testcase( i==126 ); /* IF */
-      testcase( i==127 ); /* ORDER */
-      testcase( i==128 ); /* RESTRICT */
-      testcase( i==129 ); /* OTHERS */
-      testcase( i==130 ); /* OVER */
-      testcase( i==131 ); /* RETURNING */
-      testcase( i==132 ); /* RIGHT */
-      testcase( i==133 ); /* ROLLBACK */
-      testcase( i==134 ); /* ROWS */
-      testcase( i==135 ); /* ROW */
-      testcase( i==136 ); /* UNBOUNDED */
-      testcase( i==137 ); /* UNION */
-      testcase( i==138 ); /* USING */
-      testcase( i==139 ); /* VACUUM */
-      testcase( i==140 ); /* VIEW */
-      testcase( i==141 ); /* WINDOW */
-      testcase( i==142 ); /* DO */
-      testcase( i==143 ); /* BY */
-      testcase( i==144 ); /* INITIALLY */
-      testcase( i==145 ); /* ALL */
-      testcase( i==146 ); /* PRIMARY */
+      testcase( i==1 ); /* REINDEX */
+      testcase( i==2 ); /* INDEXED */
+      testcase( i==3 ); /* INDEX */
+      testcase( i==4 ); /* DESC */
+      testcase( i==5 ); /* ESCAPE */
+      testcase( i==6 ); /* EACH */
+      testcase( i==7 ); /* CHECK */
+      testcase( i==8 ); /* KEY */
+      testcase( i==9 ); /* BEFORE */
+      testcase( i==10 ); /* FOREIGN */
+      testcase( i==11 ); /* FOR */
+      testcase( i==12 ); /* IGNORE */
+      testcase( i==13 ); /* REGEXP */
+      testcase( i==14 ); /* EXPLAIN */
+      testcase( i==15 ); /* INSTEAD */
+      testcase( i==16 ); /* ADD */
+      testcase( i==17 ); /* DATABASE */
+      testcase( i==18 ); /* AS */
+      testcase( i==19 ); /* SELECT */
+      testcase( i==20 ); /* TABLE */
+      testcase( i==21 ); /* LEFT */
+      testcase( i==22 ); /* THEN */
+      testcase( i==23 ); /* END */
+      testcase( i==24 ); /* DEFERRABLE */
+      testcase( i==25 ); /* ELSE */
+      testcase( i==26 ); /* EXCLUDE */
+      testcase( i==27 ); /* DELETE */
+      testcase( i==28 ); /* TEMPORARY */
+      testcase( i==29 ); /* TEMP */
+      testcase( i==30 ); /* OR */
+      testcase( i==31 ); /* ISNULL */
+      testcase( i==32 ); /* NULLS */
+      testcase( i==33 ); /* SAVEPOINT */
+      testcase( i==34 ); /* INTERSECT */
+      testcase( i==35 ); /* TIES */
+      testcase( i==36 ); /* NOTNULL */
+      testcase( i==37 ); /* NOT */
+      testcase( i==38 ); /* NO */
+      testcase( i==39 ); /* NULL */
+      testcase( i==40 ); /* LIKE */
+      testcase( i==41 ); /* EXCEPT */
+      testcase( i==42 ); /* TRANSACTION */
+      testcase( i==43 ); /* ACTION */
+      testcase( i==44 ); /* ON */
+      testcase( i==45 ); /* NATURAL */
+      testcase( i==46 ); /* ALTER */
+      testcase( i==47 ); /* RAISE */
+      testcase( i==48 ); /* EXCLUSIVE */
+      testcase( i==49 ); /* EXISTS */
+      testcase( i==50 ); /* CONSTRAINT */
+      testcase( i==51 ); /* INTO */
+      testcase( i==52 ); /* OFFSET */
+      testcase( i==53 ); /* OF */
+      testcase( i==54 ); /* SET */
+      testcase( i==55 ); /* TRIGGER */
+      testcase( i==56 ); /* RANGE */
+      testcase( i==57 ); /* GENERATED */
+      testcase( i==58 ); /* DETACH */
+      testcase( i==59 ); /* HAVING */
+      testcase( i==60 ); /* GLOB */
+      testcase( i==61 ); /* BEGIN */
+      testcase( i==62 ); /* INNER */
+      testcase( i==63 ); /* REFERENCES */
+      testcase( i==64 ); /* UNIQUE */
+      testcase( i==65 ); /* QUERY */
+      testcase( i==66 ); /* WITHOUT */
+      testcase( i==67 ); /* WITH */
+      testcase( i==68 ); /* OUTER */
+      testcase( i==69 ); /* RELEASE */
+      testcase( i==70 ); /* ATTACH */
+      testcase( i==71 ); /* BETWEEN */
+      testcase( i==72 ); /* NOTHING */
+      testcase( i==73 ); /* GROUPS */
+      testcase( i==74 ); /* GROUP */
+      testcase( i==75 ); /* CASCADE */
+      testcase( i==76 ); /* ASC */
+      testcase( i==77 ); /* DEFAULT */
+      testcase( i==78 ); /* CASE */
+      testcase( i==79 ); /* COLLATE */
+      testcase( i==80 ); /* CREATE */
+      testcase( i==81 ); /* CURRENT_DATE */
+      testcase( i==82 ); /* IMMEDIATE */
+      testcase( i==83 ); /* JOIN */
+      testcase( i==84 ); /* INSERT */
+      testcase( i==85 ); /* MATCH */
+      testcase( i==86 ); /* PLAN */
+      testcase( i==87 ); /* ANALYZE */
+      testcase( i==88 ); /* PRAGMA */
+      testcase( i==89 ); /* MATERIALIZED */
+      testcase( i==90 ); /* DEFERRED */
+      testcase( i==91 ); /* DISTINCT */
+      testcase( i==92 ); /* IS */
+      testcase( i==93 ); /* UPDATE */
+      testcase( i==94 ); /* VALUES */
+      testcase( i==95 ); /* VIRTUAL */
+      testcase( i==96 ); /* ALWAYS */
+      testcase( i==97 ); /* WHEN */
+      testcase( i==98 ); /* WHERE */
+      testcase( i==99 ); /* RECURSIVE */
+      testcase( i==100 ); /* ABORT */
+      testcase( i==101 ); /* AFTER */
+      testcase( i==102 ); /* RENAME */
+      testcase( i==103 ); /* AND */
+      testcase( i==104 ); /* DROP */
+      testcase( i==105 ); /* PARTITION */
+      testcase( i==106 ); /* AUTOINCREMENT */
+      testcase( i==107 ); /* TO */
+      testcase( i==108 ); /* IN */
+      testcase( i==109 ); /* CAST */
+      testcase( i==110 ); /* COLUMN */
+      testcase( i==111 ); /* COMMIT */
+      testcase( i==112 ); /* CONFLICT */
+      testcase( i==113 ); /* CROSS */
+      testcase( i==114 ); /* CURRENT_TIMESTAMP */
+      testcase( i==115 ); /* CURRENT_TIME */
+      testcase( i==116 ); /* CURRENT */
+      testcase( i==117 ); /* PRECEDING */
+      testcase( i==118 ); /* FAIL */
+      testcase( i==119 ); /* LAST */
+      testcase( i==120 ); /* FILTER */
+      testcase( i==121 ); /* REPLACE */
+      testcase( i==122 ); /* FIRST */
+      testcase( i==123 ); /* FOLLOWING */
+      testcase( i==124 ); /* FROM */
+      testcase( i==125 ); /* FULL */
+      testcase( i==126 ); /* LIMIT */
+      testcase( i==127 ); /* IF */
+      testcase( i==128 ); /* ORDER */
+      testcase( i==129 ); /* RESTRICT */
+      testcase( i==130 ); /* OTHERS */
+      testcase( i==131 ); /* OVER */
+      testcase( i==132 ); /* RETURNING */
+      testcase( i==133 ); /* RIGHT */
+      testcase( i==134 ); /* ROLLBACK */
+      testcase( i==135 ); /* ROWS */
+      testcase( i==136 ); /* ROW */
+      testcase( i==137 ); /* UNBOUNDED */
+      testcase( i==138 ); /* UNION */
+      testcase( i==139 ); /* USING */
+      testcase( i==140 ); /* VACUUM */
+      testcase( i==141 ); /* VIEW */
+      testcase( i==142 ); /* WINDOW */
+      testcase( i==143 ); /* DO */
+      testcase( i==144 ); /* BY */
+      testcase( i==145 ); /* INITIALLY */
+      testcase( i==146 ); /* ALL */
+      testcase( i==147 ); /* PRIMARY */
       *pType = aKWCode[i];
       break;
     }
@@ -172903,6 +173803,7 @@ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){
 #define SQLITE_N_KEYWORD 147
 SQLITE_API int sqlite3_keyword_name(int i,const char **pzName,int *pnName){
   if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR;
+  i++;
   *pzName = zKWText + aKWOffset[i];
   *pnName = aKWLen[i];
   return SQLITE_OK;
@@ -174441,9 +175342,21 @@ SQLITE_API int sqlite3_config(int op, ...){
   va_list ap;
   int rc = SQLITE_OK;
 
-  /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while
-  ** the SQLite library is in use. */
-  if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT;
+  /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while
+  ** the SQLite library is in use.  Except, a few selected opcodes
+  ** are allowed.
+  */
+  if( sqlite3GlobalConfig.isInit ){
+    static const u64 mAnytimeConfigOption = 0
+       | MASKBIT64( SQLITE_CONFIG_LOG )
+       | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ )
+    ;
+    if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){
+      return SQLITE_MISUSE_BKPT;
+    }
+    testcase( op==SQLITE_CONFIG_LOG );
+    testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ );
+  }
 
   va_start(ap, op);
   switch( op ){
@@ -174512,6 +175425,7 @@ SQLITE_API int sqlite3_config(int op, ...){
       break;
     }
     case SQLITE_CONFIG_MEMSTATUS: {
+      assert( !sqlite3GlobalConfig.isInit );  /* Cannot change at runtime */
       /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes
       ** single argument of type int, interpreted as a boolean, which enables
       ** or disables the collection of memory allocation statistics. */
@@ -174635,8 +175549,10 @@ SQLITE_API int sqlite3_config(int op, ...){
       ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*));
       */
       typedef void(*LOGFUNC_t)(void*,int,const char*);
-      sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t);
-      sqlite3GlobalConfig.pLogArg = va_arg(ap, void*);
+      LOGFUNC_t xLog = va_arg(ap, LOGFUNC_t);
+      void *pLogArg = va_arg(ap, void*);
+      AtomicStore(&sqlite3GlobalConfig.xLog, xLog);
+      AtomicStore(&sqlite3GlobalConfig.pLogArg, pLogArg);
       break;
     }
 
@@ -174650,7 +175566,8 @@ SQLITE_API int sqlite3_config(int op, ...){
       ** argument of type int. If non-zero, then URI handling is globally
       ** enabled. If the parameter is zero, then URI handling is globally
       ** disabled. */
-      sqlite3GlobalConfig.bOpenUri = va_arg(ap, int);
+      int bOpenUri = va_arg(ap, int);
+      AtomicStore(&sqlite3GlobalConfig.bOpenUri, bOpenUri);
       break;
     }
 
@@ -174965,6 +175882,8 @@ SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){
         { SQLITE_DBCONFIG_DQS_DML,               SQLITE_DqsDML         },
         { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT,    SQLITE_LegacyFileFmt  },
         { SQLITE_DBCONFIG_TRUSTED_SCHEMA,        SQLITE_TrustedSchema  },
+        { SQLITE_DBCONFIG_STMT_SCANSTATUS,       SQLITE_StmtScanStatus },
+        { SQLITE_DBCONFIG_REVERSE_SCANORDER,     SQLITE_ReverseOrder   },
       };
       unsigned int i;
       rc = SQLITE_ERROR; /* IMP: R-42790-23372 */
@@ -176950,9 +177869,9 @@ SQLITE_PRIVATE int sqlite3ParseUri(
 
   assert( *pzErrMsg==0 );
 
-  if( ((flags & SQLITE_OPEN_URI)             /* IMP: R-48725-32206 */
-            || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */
-   && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */
+  if( ((flags & SQLITE_OPEN_URI)                     /* IMP: R-48725-32206 */
+       || AtomicLoad(&sqlite3GlobalConfig.bOpenUri)) /* IMP: R-51689-46548 */
+   && nUri>=5 && memcmp(zUri, "file:", 5)==0         /* IMP: R-57884-37496 */
   ){
     char *zOpt;
     int eState;                   /* Parser state when parsing URI */
@@ -177359,6 +178278,9 @@ static int openDatabase(
 #if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE)
                  | SQLITE_LegacyAlter
 #endif
+#if defined(SQLITE_ENABLE_STMT_SCANSTATUS)
+                 | SQLITE_StmtScanStatus
+#endif
       ;
   sqlite3HashInit(&db->aCollSeq);
 #ifndef SQLITE_OMIT_VIRTUALTABLE
@@ -177923,7 +178845,7 @@ SQLITE_API int sqlite3_sleep(int ms){
   /* This function works in milliseconds, but the underlying OsSleep()
   ** API uses microseconds. Hence the 1000's.
   */
-  rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000);
+  rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000);
   return rc;
 }
 
@@ -198927,6 +199849,7 @@ static const char * const jsonType[] = {
 #define JNODE_PATCH   0x10         /* Patch with JsonNode.u.pPatch */
 #define JNODE_APPEND  0x20         /* More ARRAY/OBJECT entries at u.iAppend */
 #define JNODE_LABEL   0x40         /* Is a label of an object */
+#define JNODE_JSON5   0x80         /* Node contains JSON5 enhancements */
 
 
 /* A single node of parsed JSON
@@ -198953,10 +199876,12 @@ struct JsonParse {
   JsonNode *aNode;   /* Array of nodes containing the parse */
   const char *zJson; /* Original JSON string */
   u32 *aUp;          /* Index of parent of each node */
-  u8 oom;            /* Set to true if out of memory */
-  u8 nErr;           /* Number of errors seen */
   u16 iDepth;        /* Nesting depth */
+  u8 nErr;           /* Number of errors seen */
+  u8 oom;            /* Set to true if out of memory */
+  u8 hasNonstd;      /* True if input uses non-standard features like JSON5 */
   int nJson;         /* Length of the zJson string in bytes */
+  u32 iErr;          /* Error location in zJson[] */
   u32 iHold;         /* Replace cache line with the lowest iHold value */
 };
 
@@ -198964,10 +199889,10 @@ struct JsonParse {
 ** Maximum nesting depth of JSON for this implementation.
 **
 ** This limit is needed to avoid a stack overflow in the recursive
-** descent parser.  A depth of 2000 is far deeper than any sane JSON
-** should go.
+** descent parser.  A depth of 1000 is far deeper than any sane JSON
+** should go.  Historical note: This limit was 2000 prior to version 3.42.0
 */
-#define JSON_MAX_DEPTH  2000
+#define JSON_MAX_DEPTH  1000
 
 /**************************************************************************
 ** Utility routines for dealing with JsonString objects
@@ -199118,6 +200043,129 @@ static void jsonAppendString(JsonString *p, const char *zIn, u32 N){
 }
 
 /*
+** The zIn[0..N] string is a JSON5 string literal.  Append to p a translation
+** of the string literal that standard JSON and that omits all JSON5
+** features.
+*/
+static void jsonAppendNormalizedString(JsonString *p, const char *zIn, u32 N){
+  u32 i;
+  jsonAppendChar(p, '"');
+  zIn++;
+  N -= 2;
+  while( N>0 ){
+    for(i=0; i<N && zIn[i]!='\\'; i++){}
+    if( i>0 ){
+      jsonAppendRaw(p, zIn, i);
+      zIn += i;
+      N -= i;
+      if( N==0 ) break;
+    }
+    assert( zIn[0]=='\\' );
+    switch( (u8)zIn[1] ){
+      case '\'':
+        jsonAppendChar(p, '\'');
+        break;
+      case 'v':
+        jsonAppendRaw(p, "\\u0009", 6);
+        break;
+      case 'x':
+        jsonAppendRaw(p, "\\u00", 4);
+        jsonAppendRaw(p, &zIn[2], 2);
+        zIn += 2;
+        N -= 2;
+        break;
+      case '0':
+        jsonAppendRaw(p, "\\u0000", 6);
+        break;
+      case '\r':
+        if( zIn[2]=='\n' ){
+          zIn++;
+          N--;
+        }
+        break;
+      case '\n':
+        break;
+      case 0xe2:
+        assert( N>=4 );
+        assert( 0x80==(u8)zIn[2] );
+        assert( 0xa8==(u8)zIn[3] || 0xa9==(u8)zIn[3] );
+        zIn += 2;
+        N -= 2;
+        break;
+      default:
+        jsonAppendRaw(p, zIn, 2);
+        break;
+    }
+    zIn += 2;
+    N -= 2;
+  }
+  jsonAppendChar(p, '"');
+}
+
+/*
+** The zIn[0..N] string is a JSON5 integer literal.  Append to p a translation
+** of the string literal that standard JSON and that omits all JSON5
+** features.
+*/
+static void jsonAppendNormalizedInt(JsonString *p, const char *zIn, u32 N){
+  if( zIn[0]=='+' ){
+    zIn++;
+    N--;
+  }else if( zIn[0]=='-' ){
+    jsonAppendChar(p, '-');
+    zIn++;
+    N--;
+  }
+  if( zIn[0]=='0' && (zIn[1]=='x' || zIn[1]=='X') ){
+    sqlite3_int64 i = 0;
+    int rc = sqlite3DecOrHexToI64(zIn, &i);
+    if( rc<=1 ){
+      jsonPrintf(100,p,"%lld",i);
+    }else{
+      assert( rc==2 );
+      jsonAppendRaw(p, "9.0e999", 7);
+    }
+    return;
+  }
+  jsonAppendRaw(p, zIn, N);
+}
+
+/*
+** The zIn[0..N] string is a JSON5 real literal.  Append to p a translation
+** of the string literal that standard JSON and that omits all JSON5
+** features.
+*/
+static void jsonAppendNormalizedReal(JsonString *p, const char *zIn, u32 N){
+  u32 i;
+  if( zIn[0]=='+' ){
+    zIn++;
+    N--;
+  }else if( zIn[0]=='-' ){
+    jsonAppendChar(p, '-');
+    zIn++;
+    N--;
+  }
+  if( zIn[0]=='.' ){
+    jsonAppendChar(p, '0');
+  }
+  for(i=0; i<N; i++){
+    if( zIn[i]=='.' && (i+1==N || !sqlite3Isdigit(zIn[i+1])) ){
+      i++;
+      jsonAppendRaw(p, zIn, i);
+      zIn += i;
+      N -= i;
+      jsonAppendChar(p, '0');
+      break;
+    }
+  }
+  if( N>0 ){
+    jsonAppendRaw(p, zIn, N);
+  }
+}
+
+
+
+/*
 ** Append a function parameter value to the JSON string under
 ** construction.
 */
@@ -199130,8 +200178,11 @@ static void jsonAppendValue(
       jsonAppendRaw(p, "null", 4);
       break;
     }
-    case SQLITE_INTEGER:
     case SQLITE_FLOAT: {
+      jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue));
+      break;
+    }
+    case SQLITE_INTEGER: {
       const char *z = (const char*)sqlite3_value_text(pValue);
       u32 n = (u32)sqlite3_value_bytes(pValue);
       jsonAppendRaw(p, z, n);
@@ -199244,17 +200295,38 @@ static void jsonRenderNode(
       break;
     }
     case JSON_STRING: {
+      assert( pNode->eU==1 );
       if( pNode->jnFlags & JNODE_RAW ){
-        assert( pNode->eU==1 );
-        jsonAppendString(pOut, pNode->u.zJContent, pNode->n);
-        break;
+        if( pNode->jnFlags & JNODE_LABEL ){
+          jsonAppendChar(pOut, '"');
+          jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+          jsonAppendChar(pOut, '"');
+        }else{
+          jsonAppendString(pOut, pNode->u.zJContent, pNode->n);
+        }
+      }else if( pNode->jnFlags & JNODE_JSON5 ){
+        jsonAppendNormalizedString(pOut, pNode->u.zJContent, pNode->n);
+      }else{
+        jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
       }
-      /* no break */ deliberate_fall_through
+      break;
+    }
+    case JSON_REAL: {
+      assert( pNode->eU==1 );
+      if( pNode->jnFlags & JNODE_JSON5 ){
+        jsonAppendNormalizedReal(pOut, pNode->u.zJContent, pNode->n);
+      }else{
+        jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+      }
+      break;
     }
-    case JSON_REAL:
     case JSON_INT: {
       assert( pNode->eU==1 );
-      jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+      if( pNode->jnFlags & JNODE_JSON5 ){
+        jsonAppendNormalizedInt(pOut, pNode->u.zJContent, pNode->n);
+      }else{
+        jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n);
+      }
       break;
     }
     case JSON_ARRAY: {
@@ -199370,59 +200442,41 @@ static void jsonReturn(
     }
     case JSON_INT: {
       sqlite3_int64 i = 0;
+      int rc;
+      int bNeg = 0;
       const char *z;
+
+
       assert( pNode->eU==1 );
       z = pNode->u.zJContent;
-      if( z[0]=='-' ){ z++; }
-      while( z[0]>='0' && z[0]<='9' ){
-        unsigned v = *(z++) - '0';
-        if( i>=LARGEST_INT64/10 ){
-          if( i>LARGEST_INT64/10 ) goto int_as_real;
-          if( z[0]>='0' && z[0]<='9' ) goto int_as_real;
-          if( v==9 ) goto int_as_real;
-          if( v==8 ){
-            if( pNode->u.zJContent[0]=='-' ){
-              sqlite3_result_int64(pCtx, SMALLEST_INT64);
-              goto int_done;
-            }else{
-              goto int_as_real;
-            }
-          }
-        }
-        i = i*10 + v;
+      if( z[0]=='-' ){ z++; bNeg = 1; }
+      else if( z[0]=='+' ){ z++; }
+      rc = sqlite3DecOrHexToI64(z, &i);
+      if( rc<=1 ){
+        sqlite3_result_int64(pCtx, bNeg ? -i : i);
+      }else if( rc==3 && bNeg ){
+        sqlite3_result_int64(pCtx, SMALLEST_INT64);
+      }else{
+        goto to_double;
       }
-      if( pNode->u.zJContent[0]=='-' ){ i = -i; }
-      sqlite3_result_int64(pCtx, i);
-      int_done:
       break;
-      int_as_real: ; /* no break */ deliberate_fall_through
     }
     case JSON_REAL: {
       double r;
-#ifdef SQLITE_AMALGAMATION
       const char *z;
       assert( pNode->eU==1 );
+    to_double:
       z = pNode->u.zJContent;
       sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8);
-#else
-      assert( pNode->eU==1 );
-      r = strtod(pNode->u.zJContent, 0);
-#endif
       sqlite3_result_double(pCtx, r);
       break;
     }
     case JSON_STRING: {
-#if 0 /* Never happens because JNODE_RAW is only set by json_set(),
-      ** json_insert() and json_replace() and those routines do not
-      ** call jsonReturn() */
       if( pNode->jnFlags & JNODE_RAW ){
         assert( pNode->eU==1 );
         sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n,
                             SQLITE_TRANSIENT);
-      }else
-#endif
-      assert( (pNode->jnFlags & JNODE_RAW)==0 );
-      if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){
+      }else if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){
         /* JSON formatted without any backslash-escapes */
         assert( pNode->eU==1 );
         sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2,
@@ -199434,18 +200488,17 @@ static void jsonReturn(
         const char *z;
         char *zOut;
         u32 j;
+        u32 nOut = n;
         assert( pNode->eU==1 );
         z = pNode->u.zJContent;
-        zOut = sqlite3_malloc( n+1 );
+        zOut = sqlite3_malloc( nOut+1 );
         if( zOut==0 ){
           sqlite3_result_error_nomem(pCtx);
           break;
         }
         for(i=1, j=0; i<n-1; i++){
           char c = z[i];
-          if( c!='\\' ){
-            zOut[j++] = c;
-          }else{
+          if( c=='\\' ){
             c = z[++i];
             if( c=='u' ){
               u32 v = jsonHexToInt4(z+i+1);
@@ -199477,22 +200530,40 @@ static void jsonReturn(
                   zOut[j++] = 0x80 | (v&0x3f);
                 }
               }
+              continue;
+            }else if( c=='b' ){
+              c = '\b';
+            }else if( c=='f' ){
+              c = '\f';
+            }else if( c=='n' ){
+              c = '\n';
+            }else if( c=='r' ){
+              c = '\r';
+            }else if( c=='t' ){
+              c = '\t';
+            }else if( c=='v' ){
+              c = '\v';
+            }else if( c=='\'' || c=='"' || c=='/' || c=='\\' ){
+              /* pass through unchanged */
+            }else if( c=='0' ){
+              c = 0;
+            }else if( c=='x' ){
+              c = (jsonHexToInt(z[i+1])<<4) | jsonHexToInt(z[i+2]);
+              i += 2;
+            }else if( c=='\r' && z[i+1]=='\n' ){
+              i++;
+              continue;
+            }else if( 0xe2==(u8)c ){
+              assert( 0x80==(u8)z[i+1] );
+              assert( 0xa8==(u8)z[i+2] || 0xa9==(u8)z[i+2] );
+              i += 2;
+              continue;
             }else{
-              if( c=='b' ){
-                c = '\b';
-              }else if( c=='f' ){
-                c = '\f';
-              }else if( c=='n' ){
-                c = '\n';
-              }else if( c=='r' ){
-                c = '\r';
-              }else if( c=='t' ){
-                c = '\t';
-              }
-              zOut[j++] = c;
+              continue;
             }
-          }
-        }
+          } /* end if( c=='\\' ) */
+          zOut[j++] = c;
+        } /* end for() */
         zOut[j] = 0;
         sqlite3_result_text(pCtx, zOut, j, sqlite3_free);
       }
@@ -199560,8 +200631,8 @@ static int jsonParseAddNode(
     return jsonParseAddNodeExpand(pParse, eType, n, zContent);
   }
   p = &pParse->aNode[pParse->nNode];
-  p->eType = (u8)eType;
-  p->jnFlags = 0;
+  p->eType = (u8)(eType & 0xff);
+  p->jnFlags = (u8)(eType >> 8);
   VVA( p->eU = zContent ? 1 : 0 );
   p->n = n;
   p->u.zJContent = zContent;
@@ -199569,21 +200640,177 @@ static int jsonParseAddNode(
 }
 
 /*
+** Return true if z[] begins with 2 (or more) hexadecimal digits
+*/
+static int jsonIs2Hex(const char *z){
+  return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]);
+}
+
+/*
 ** Return true if z[] begins with 4 (or more) hexadecimal digits
 */
 static int jsonIs4Hex(const char *z){
-  int i;
-  for(i=0; i<4; i++) if( !sqlite3Isxdigit(z[i]) ) return 0;
-  return 1;
+  return jsonIs2Hex(z) && jsonIs2Hex(&z[2]);
+}
+
+/*
+** Return the number of bytes of JSON5 whitespace at the beginning of
+** the input string z[].
+**
+** JSON5 whitespace consists of any of the following characters:
+**
+**    Unicode  UTF-8         Name
+**    U+0009   09            horizontal tab
+**    U+000a   0a            line feed
+**    U+000b   0b            vertical tab
+**    U+000c   0c            form feed
+**    U+000d   0d            carriage return
+**    U+0020   20            space
+**    U+00a0   c2 a0         non-breaking space
+**    U+1680   e1 9a 80      ogham space mark
+**    U+2000   e2 80 80      en quad
+**    U+2001   e2 80 81      em quad
+**    U+2002   e2 80 82      en space
+**    U+2003   e2 80 83      em space
+**    U+2004   e2 80 84      three-per-em space
+**    U+2005   e2 80 85      four-per-em space
+**    U+2006   e2 80 86      six-per-em space
+**    U+2007   e2 80 87      figure space
+**    U+2008   e2 80 88      punctuation space
+**    U+2009   e2 80 89      thin space
+**    U+200a   e2 80 8a      hair space
+**    U+2028   e2 80 a8      line separator
+**    U+2029   e2 80 a9      paragraph separator
+**    U+202f   e2 80 af      narrow no-break space (NNBSP)
+**    U+205f   e2 81 9f      medium mathematical space (MMSP)
+**    U+3000   e3 80 80      ideographical space
+**    U+FEFF   ef bb bf      byte order mark
+**
+** In addition, comments between '/', '*' and '*', '/' and
+** from '/', '/' to end-of-line are also considered to be whitespace.
+*/
+static int json5Whitespace(const char *zIn){
+  int n = 0;
+  const u8 *z = (u8*)zIn;
+  while( 1 /*exit by "goto whitespace_done"*/ ){
+    switch( z[n] ){
+      case 0x09:
+      case 0x0a:
+      case 0x0b:
+      case 0x0c:
+      case 0x0d:
+      case 0x20: {
+        n++;
+        break;
+      }
+      case '/': {
+        if( z[n+1]=='*' && z[n+2]!=0 ){
+          int j;
+          for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){
+            if( z[j]==0 ) goto whitespace_done;
+          }
+          n = j+1;
+          break;
+        }else if( z[n+1]=='/' ){
+          int j;
+          char c;
+          for(j=n+2; (c = z[j])!=0; j++){
+            if( c=='\n' || c=='\r' ) break;
+            if( 0xe2==(u8)c && 0x80==(u8)z[j+1]
+             && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])
+            ){
+              j += 2;
+              break;
+            }
+          }
+          n = j;
+          if( z[n] ) n++;
+          break;
+        }
+        goto whitespace_done;
+      }
+      case 0xc2: {
+        if( z[n+1]==0xa0 ){
+          n += 2;
+          break;
+        }
+        goto whitespace_done;
+      }
+      case 0xe1: {
+        if( z[n+1]==0x9a && z[n+2]==0x80 ){
+          n += 3;
+          break;
+        }
+        goto whitespace_done;
+      }
+      case 0xe2: {
+        if( z[n+1]==0x80 ){
+          u8 c = z[n+2];
+          if( c<0x80 ) goto whitespace_done;
+          if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){
+            n += 3;
+            break;
+          }
+        }else if( z[n+1]==0x81 && z[n+2]==0x9f ){
+          n += 3;
+          break;
+        }
+        goto whitespace_done;
+      }
+      case 0xe3: {
+        if( z[n+1]==0x80 && z[n+2]==0x80 ){
+          n += 3;
+          break;
+        }
+        goto whitespace_done;
+      }
+      case 0xef: {
+        if( z[n+1]==0xbb && z[n+2]==0xbf ){
+          n += 3;
+          break;
+        }
+        goto whitespace_done;
+      }
+      default: {
+        goto whitespace_done;
+      }
+    }
+  }
+  whitespace_done:
+  return n;
 }
 
 /*
+** Extra floating-point literals to allow in JSON.
+*/
+static const struct NanInfName {
+  char c1;
+  char c2;
+  char n;
+  char eType;
+  char nRepl;
+  char *zMatch;
+  char *zRepl;
+} aNanInfName[] = {
+  { 'i', 'I', 3, JSON_REAL, 7, "inf", "9.0e999" },
+  { 'i', 'I', 8, JSON_REAL, 7, "infinity", "9.0e999" },
+  { 'n', 'N', 3, JSON_NULL, 4, "NaN", "null" },
+  { 'q', 'Q', 4, JSON_NULL, 4, "QNaN", "null" },
+  { 's', 'S', 4, JSON_NULL, 4, "SNaN", "null" },
+};
+
+/*
 ** Parse a single JSON value which begins at pParse->zJson[i].  Return the
 ** index of the first character past the end of the value parsed.
 **
-** Return negative for a syntax error.  Special cases:  return -2 if the
-** first non-whitespace character is '}' and return -3 if the first
-** non-whitespace character is ']'.
+** Special return values:
+**
+**      0    End if input
+**     -1    Syntax error
+**     -2    '}' seen
+**     -3    ']' seen
+**     -4    ',' seen
+**     -5    ':' seen
 */
 static int jsonParseValue(JsonParse *pParse, u32 i){
   char c;
@@ -199592,151 +200819,430 @@ static int jsonParseValue(JsonParse *pParse, u32 i){
   int x;
   JsonNode *pNode;
   const char *z = pParse->zJson;
-  while( fast_isspace(z[i]) ){ i++; }
-  if( (c = z[i])=='{' ){
+json_parse_restart:
+  switch( (u8)z[i] ){
+  case '{': {
     /* Parse object */
     iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0);
     if( iThis<0 ) return -1;
+    if( ++pParse->iDepth > JSON_MAX_DEPTH ){
+      pParse->iErr = i;
+      return -1;
+    }
     for(j=i+1;;j++){
-      while( fast_isspace(z[j]) ){ j++; }
-      if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1;
+      u32 nNode = pParse->nNode;
       x = jsonParseValue(pParse, j);
-      if( x<0 ){
-        pParse->iDepth--;
-        if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1;
-        return -1;
+      if( x<=0 ){
+        if( x==(-2) ){
+          j = pParse->iErr;
+          if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1;
+          break;
+        }
+        j += json5Whitespace(&z[j]);
+        if( sqlite3JsonId1(z[j])
+         || (z[j]=='\\' && z[j+1]=='u' && jsonIs4Hex(&z[j+2]))
+        ){
+          int k = j+1;
+          while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0)
+            || (z[k]=='\\' && z[k+1]=='u' && jsonIs4Hex(&z[k+2]))
+          ){
+            k++;
+          }
+          jsonParseAddNode(pParse, JSON_STRING | (JNODE_RAW<<8), k-j, &z[j]);
+          pParse->hasNonstd = 1;
+          x = k;
+        }else{
+          if( x!=-1 ) pParse->iErr = j;
+          return -1;
+        }
       }
       if( pParse->oom ) return -1;
-      pNode = &pParse->aNode[pParse->nNode-1];
-      if( pNode->eType!=JSON_STRING ) return -1;
+      pNode = &pParse->aNode[nNode];
+      if( pNode->eType!=JSON_STRING ){
+        pParse->iErr = j;
+        return -1;
+      }
       pNode->jnFlags |= JNODE_LABEL;
       j = x;
-      while( fast_isspace(z[j]) ){ j++; }
-      if( z[j]!=':' ) return -1;
-      j++;
+      if( z[j]==':' ){
+        j++;
+      }else{
+        if( fast_isspace(z[j]) ){
+          do{ j++; }while( fast_isspace(z[j]) );
+          if( z[j]==':' ){
+            j++;
+            goto parse_object_value;
+          }
+        }
+        x = jsonParseValue(pParse, j);
+        if( x!=(-5) ){
+          if( x!=(-1) ) pParse->iErr = j;
+          return -1;
+        }
+        j = pParse->iErr+1;
+      }
+    parse_object_value:
       x = jsonParseValue(pParse, j);
-      pParse->iDepth--;
-      if( x<0 ) return -1;
+      if( x<=0 ){
+        if( x!=(-1) ) pParse->iErr = j;
+        return -1;
+      }
       j = x;
-      while( fast_isspace(z[j]) ){ j++; }
-      c = z[j];
-      if( c==',' ) continue;
-      if( c!='}' ) return -1;
-      break;
+      if( z[j]==',' ){
+        continue;
+      }else if( z[j]=='}' ){
+        break;
+      }else{
+        if( fast_isspace(z[j]) ){
+          do{ j++; }while( fast_isspace(z[j]) );
+          if( z[j]==',' ){
+            continue;
+          }else if( z[j]=='}' ){
+            break;
+          }
+        }
+        x = jsonParseValue(pParse, j);
+        if( x==(-4) ){
+          j = pParse->iErr;
+          continue;
+        }
+        if( x==(-2) ){
+          j = pParse->iErr;
+          break;
+        }
+      }
+      pParse->iErr = j;
+      return -1;
     }
     pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1;
+    pParse->iDepth--;
     return j+1;
-  }else if( c=='[' ){
+  }
+  case '[': {
     /* Parse array */
     iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0);
     if( iThis<0 ) return -1;
+    if( ++pParse->iDepth > JSON_MAX_DEPTH ){
+      pParse->iErr = i;
+      return -1;
+    }
     memset(&pParse->aNode[iThis].u, 0, sizeof(pParse->aNode[iThis].u));
     for(j=i+1;;j++){
-      while( fast_isspace(z[j]) ){ j++; }
-      if( ++pParse->iDepth > JSON_MAX_DEPTH ) return -1;
       x = jsonParseValue(pParse, j);
-      pParse->iDepth--;
-      if( x<0 ){
-        if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1;
+      if( x<=0 ){
+        if( x==(-3) ){
+          j = pParse->iErr;
+          if( pParse->nNode!=(u32)iThis+1 ) pParse->hasNonstd = 1;
+          break;
+        }
+        if( x!=(-1) ) pParse->iErr = j;
         return -1;
       }
       j = x;
-      while( fast_isspace(z[j]) ){ j++; }
-      c = z[j];
-      if( c==',' ) continue;
-      if( c!=']' ) return -1;
-      break;
+      if( z[j]==',' ){
+        continue;
+      }else if( z[j]==']' ){
+        break;
+      }else{
+        if( fast_isspace(z[j]) ){
+          do{ j++; }while( fast_isspace(z[j]) );
+          if( z[j]==',' ){
+            continue;
+          }else if( z[j]==']' ){
+            break;
+          }
+        }
+        x = jsonParseValue(pParse, j);
+        if( x==(-4) ){
+          j = pParse->iErr;
+          continue;
+        }
+        if( x==(-3) ){
+          j = pParse->iErr;
+          break;
+        }
+      }
+      pParse->iErr = j;
+      return -1;
     }
     pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1;
+    pParse->iDepth--;
     return j+1;
-  }else if( c=='"' ){
+  }
+  case '\'': {
+    u8 jnFlags;
+    char cDelim;
+    pParse->hasNonstd = 1;
+    jnFlags = JNODE_JSON5;
+    goto parse_string;
+  case '"':
     /* Parse string */
-    u8 jnFlags = 0;
+    jnFlags = 0;
+  parse_string:
+    cDelim = z[i];
     j = i+1;
     for(;;){
       c = z[j];
       if( (c & ~0x1f)==0 ){
         /* Control characters are not allowed in strings */
+        pParse->iErr = j;
         return -1;
       }
       if( c=='\\' ){
         c = z[++j];
         if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f'
            || c=='n' || c=='r' || c=='t'
-           || (c=='u' && jsonIs4Hex(z+j+1)) ){
-          jnFlags = JNODE_ESCAPE;
+           || (c=='u' && jsonIs4Hex(&z[j+1])) ){
+          jnFlags |= JNODE_ESCAPE;
+        }else if( c=='\'' || c=='0' || c=='v' || c=='\n'
+           || (0xe2==(u8)c && 0x80==(u8)z[j+1]
+                && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2]))
+           || (c=='x' && jsonIs2Hex(&z[j+1])) ){
+          jnFlags |= (JNODE_ESCAPE|JNODE_JSON5);
+          pParse->hasNonstd = 1;
+        }else if( c=='\r' ){
+          if( z[j+1]=='\n' ) j++;
+          jnFlags |= (JNODE_ESCAPE|JNODE_JSON5);
+          pParse->hasNonstd = 1;
         }else{
+          pParse->iErr = j;
           return -1;
         }
-      }else if( c=='"' ){
+      }else if( c==cDelim ){
         break;
       }
       j++;
     }
-    jsonParseAddNode(pParse, JSON_STRING, j+1-i, &z[i]);
-    if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags;
+    jsonParseAddNode(pParse, JSON_STRING | (jnFlags<<8), j+1-i, &z[i]);
     return j+1;
-  }else if( c=='n'
-         && strncmp(z+i,"null",4)==0
-         && !sqlite3Isalnum(z[i+4]) ){
-    jsonParseAddNode(pParse, JSON_NULL, 0, 0);
-    return i+4;
-  }else if( c=='t'
-         && strncmp(z+i,"true",4)==0
-         && !sqlite3Isalnum(z[i+4]) ){
-    jsonParseAddNode(pParse, JSON_TRUE, 0, 0);
-    return i+4;
-  }else if( c=='f'
-         && strncmp(z+i,"false",5)==0
-         && !sqlite3Isalnum(z[i+5]) ){
-    jsonParseAddNode(pParse, JSON_FALSE, 0, 0);
-    return i+5;
-  }else if( c=='-' || (c>='0' && c<='9') ){
+  }
+  case 't': {
+    if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){
+      jsonParseAddNode(pParse, JSON_TRUE, 0, 0);
+      return i+4;
+    }
+    pParse->iErr = i;
+    return -1;
+  }
+  case 'f': {
+    if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){
+      jsonParseAddNode(pParse, JSON_FALSE, 0, 0);
+      return i+5;
+    }
+    pParse->iErr = i;
+    return -1;
+  }
+  case '+': {
+    u8 seenDP, seenE, jnFlags;
+    pParse->hasNonstd = 1;
+    jnFlags = JNODE_JSON5;
+    goto parse_number;
+  case '.':
+    if( sqlite3Isdigit(z[i+1]) ){
+      pParse->hasNonstd = 1;
+      jnFlags = JNODE_JSON5;
+      seenE = 0;
+      seenDP = JSON_REAL;
+      goto parse_number_2;
+    }
+    pParse->iErr = i;
+    return -1;
+  case '-':
+  case '0':
+  case '1':
+  case '2':
+  case '3':
+  case '4':
+  case '5':
+  case '6':
+  case '7':
+  case '8':
+  case '9':
     /* Parse number */
-    u8 seenDP = 0;
-    u8 seenE = 0;
+    jnFlags = 0;
+  parse_number:
+    seenDP = JSON_INT;
+    seenE = 0;
     assert( '-' < '0' );
+    assert( '+' < '0' );
+    assert( '.' < '0' );
+    c = z[i];
+
     if( c<='0' ){
-      j = c=='-' ? i+1 : i;
-      if( z[j]=='0' && z[j+1]>='0' && z[j+1]<='9' ) return -1;
+      if( c=='0' ){
+        if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){
+          assert( seenDP==JSON_INT );
+          pParse->hasNonstd = 1;
+          jnFlags |= JNODE_JSON5;
+          for(j=i+3; sqlite3Isxdigit(z[j]); j++){}
+          goto parse_number_finish;
+        }else if( sqlite3Isdigit(z[i+1]) ){
+          pParse->iErr = i+1;
+          return -1;
+        }
+      }else{
+        if( !sqlite3Isdigit(z[i+1]) ){
+          /* JSON5 allows for "+Infinity" and "-Infinity" using exactly
+          ** that case.  SQLite also allows these in any case and it allows
+          ** "+inf" and "-inf". */
+          if( (z[i+1]=='I' || z[i+1]=='i')
+           && sqlite3StrNICmp(&z[i+1], "inf",3)==0
+          ){
+            pParse->hasNonstd = 1;
+            if( z[i]=='-' ){
+              jsonParseAddNode(pParse, JSON_REAL, 8, "-9.0e999");
+            }else{
+              jsonParseAddNode(pParse, JSON_REAL, 7, "9.0e999");
+            }
+            return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4);
+          }
+          if( z[i+1]=='.' ){
+            pParse->hasNonstd = 1;
+            jnFlags |= JNODE_JSON5;
+            goto parse_number_2;
+          }
+          pParse->iErr = i;
+          return -1;
+        }
+        if( z[i+1]=='0' ){
+          if( sqlite3Isdigit(z[i+2]) ){
+            pParse->iErr = i+1;
+            return -1;
+          }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){
+            pParse->hasNonstd = 1;
+            jnFlags |= JNODE_JSON5;
+            for(j=i+4; sqlite3Isxdigit(z[j]); j++){}
+            goto parse_number_finish;
+          }
+        }
+      }
     }
-    j = i+1;
-    for(;; j++){
+  parse_number_2:
+    for(j=i+1;; j++){
       c = z[j];
-      if( c>='0' && c<='9' ) continue;
+      if( sqlite3Isdigit(c) ) continue;
       if( c=='.' ){
-        if( z[j-1]=='-' ) return -1;
-        if( seenDP ) return -1;
-        seenDP = 1;
+        if( seenDP==JSON_REAL ){
+          pParse->iErr = j;
+          return -1;
+        }
+        seenDP = JSON_REAL;
         continue;
       }
       if( c=='e' || c=='E' ){
-        if( z[j-1]<'0' ) return -1;
-        if( seenE ) return -1;
-        seenDP = seenE = 1;
+        if( z[j-1]<'0' ){
+          if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){
+            pParse->hasNonstd = 1;
+            jnFlags |= JNODE_JSON5;
+          }else{
+            pParse->iErr = j;
+            return -1;
+          }
+        }
+        if( seenE ){
+          pParse->iErr = j;
+          return -1;
+        }
+        seenDP = JSON_REAL;
+        seenE = 1;
         c = z[j+1];
         if( c=='+' || c=='-' ){
           j++;
           c = z[j+1];
         }
-        if( c<'0' || c>'9' ) return -1;
+        if( c<'0' || c>'9' ){
+          pParse->iErr = j;
+          return -1;
+        }
         continue;
       }
       break;
     }
-    if( z[j-1]<'0' ) return -1;
-    jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT,
-                        j - i, &z[i]);
+    if( z[j-1]<'0' ){
+      if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){
+        pParse->hasNonstd = 1;
+        jnFlags |= JNODE_JSON5;
+      }else{
+        pParse->iErr = j;
+        return -1;
+      }
+    }
+  parse_number_finish:
+    jsonParseAddNode(pParse, seenDP | (jnFlags<<8), j - i, &z[i]);
     return j;
-  }else if( c=='}' ){
+  }
+  case '}': {
+    pParse->iErr = i;
     return -2;  /* End of {...} */
-  }else if( c==']' ){
+  }
+  case ']': {
+    pParse->iErr = i;
     return -3;  /* End of [...] */
-  }else if( c==0 ){
+  }
+  case ',': {
+    pParse->iErr = i;
+    return -4;  /* List separator */
+  }
+  case ':': {
+    pParse->iErr = i;
+    return -5;  /* Object label/value separator */
+  }
+  case 0: {
     return 0;   /* End of file */
-  }else{
+  }
+  case 0x09:
+  case 0x0a:
+  case 0x0d:
+  case 0x20: {
+    do{
+      i++;
+    }while( fast_isspace(z[i]) );
+    goto json_parse_restart;
+  }
+  case 0x0b:
+  case 0x0c:
+  case '/':
+  case 0xc2:
+  case 0xe1:
+  case 0xe2:
+  case 0xe3:
+  case 0xef: {
+    j = json5Whitespace(&z[i]);
+    if( j>0 ){
+      i += j;
+      pParse->hasNonstd = 1;
+      goto json_parse_restart;
+    }
+    pParse->iErr = i;
+    return -1;
+  }
+  case 'n': {
+    if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){
+      jsonParseAddNode(pParse, JSON_NULL, 0, 0);
+      return i+4;
+    }
+    /* fall-through into the default case that checks for NaN */
+  }
+  default: {
+    u32 k;
+    int nn;
+    c = z[i];
+    for(k=0; k<sizeof(aNanInfName)/sizeof(aNanInfName[0]); k++){
+      if( c!=aNanInfName[k].c1 && c!=aNanInfName[k].c2 ) continue;
+      nn = aNanInfName[k].n;
+      if( sqlite3StrNICmp(&z[i], aNanInfName[k].zMatch, nn)!=0 ){
+        continue;
+      }
+      if( sqlite3Isalnum(z[i+nn]) ) continue;
+      jsonParseAddNode(pParse, aNanInfName[k].eType,
+          aNanInfName[k].nRepl, aNanInfName[k].zRepl);
+      pParse->hasNonstd = 1;
+      return i + nn;
+    }
+    pParse->iErr = i;
     return -1;  /* Syntax error */
   }
+  } /* End switch(z[i]) */
 }
 
 /*
@@ -199760,7 +201266,14 @@ static int jsonParse(
   if( i>0 ){
     assert( pParse->iDepth==0 );
     while( fast_isspace(zJson[i]) ) i++;
-    if( zJson[i] ) i = -1;
+    if( zJson[i] ){
+      i += json5Whitespace(&zJson[i]);
+      if( zJson[i] ){
+        jsonParseReset(pParse);
+        return 1;
+      }
+      pParse->hasNonstd = 1;
+    }
   }
   if( i<=0 ){
     if( pCtx!=0 ){
@@ -199831,6 +201344,15 @@ static int jsonParseFindParents(JsonParse *pParse){
 ** is no longer valid, parse the JSON again and return the new parse,
 ** and also register the new parse so that it will be available for
 ** future sqlite3_get_auxdata() calls.
+**
+** If an error occurs and pErrCtx!=0 then report the error on pErrCtx
+** and return NULL.
+**
+** If an error occurs and pErrCtx==0 then return the Parse object with
+** JsonParse.nErr non-zero.  If the caller invokes this routine with
+** pErrCtx==0 and it gets back a JsonParse with nErr!=0, then the caller
+** is responsible for invoking jsonParseFree() on the returned value.
+** But the caller may invoke jsonParseFree() *only* if pParse->nErr!=0.
 */
 static JsonParse *jsonParseCached(
   sqlite3_context *pCtx,
@@ -199880,6 +201402,10 @@ static JsonParse *jsonParseCached(
   p->zJson = (char*)&p[1];
   memcpy((char*)p->zJson, zJson, nJson+1);
   if( jsonParse(p, pErrCtx, p->zJson) ){
+    if( pErrCtx==0 ){
+      p->nErr = 1;
+      return p;
+    }
     sqlite3_free(p);
     return 0;
   }
@@ -199894,7 +201420,7 @@ static JsonParse *jsonParseCached(
 ** Compare the OBJECT label at pNode against zKey,nKey.  Return true on
 ** a match.
 */
-static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){
+static int jsonLabelCompare(const JsonNode *pNode, const char *zKey, u32 nKey){
   assert( pNode->eU==1 );
   if( pNode->jnFlags & JNODE_RAW ){
     if( pNode->n!=nKey ) return 0;
@@ -199904,6 +201430,15 @@ static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){
     return strncmp(pNode->u.zJContent+1, zKey, nKey)==0;
   }
 }
+static int jsonSameLabel(const JsonNode *p1, const JsonNode *p2){
+  if( p1->jnFlags & JNODE_RAW ){
+    return jsonLabelCompare(p2, p1->u.zJContent, p1->n);
+  }else if( p2->jnFlags & JNODE_RAW ){
+    return jsonLabelCompare(p1, p2->u.zJContent, p2->n);
+  }else{
+    return p1->n==p2->n && strncmp(p1->u.zJContent,p2->u.zJContent,p1->n)==0;
+  }
+}
 
 /* forward declaration */
 static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**);
@@ -200374,7 +201909,7 @@ static void jsonExtractFunc(
     zPath = (const char*)sqlite3_value_text(argv[1]);
     if( zPath==0 ) return;
     if( flags & JSON_ABPATH ){
-      if( zPath[0]!='$' ){
+      if( zPath[0]!='$' || (zPath[1]!='.' && zPath[1]!='[' && zPath[1]!=0) ){
         /* The -> and ->> operators accept abbreviated PATH arguments.  This
         ** is mostly for compatibility with PostgreSQL, but also for
         ** convenience.
@@ -200465,12 +202000,10 @@ static JsonNode *jsonMergePatch(
     assert( pPatch[i].eU==1 );
     nKey = pPatch[i].n;
     zKey = pPatch[i].u.zJContent;
-    assert( (pPatch[i].jnFlags & JNODE_RAW)==0 );
     for(j=1; j<pTarget->n; j += jsonNodeSize(&pTarget[j+1])+1 ){
       assert( pTarget[j].eType==JSON_STRING );
       assert( pTarget[j].jnFlags & JNODE_LABEL );
-      assert( (pPatch[i].jnFlags & JNODE_RAW)==0 );
-      if( pTarget[j].n==nKey && strncmp(pTarget[j].u.zJContent,zKey,nKey)==0 ){
+      if( jsonSameLabel(&pPatch[i], &pTarget[j]) ){
         if( pTarget[j+1].jnFlags & (JNODE_REMOVE|JNODE_PATCH) ) break;
         if( pPatch[i+1].eType==JSON_NULL ){
           pTarget[j+1].jnFlags |= JNODE_REMOVE;
@@ -200757,8 +202290,8 @@ static void jsonTypeFunc(
 /*
 ** json_valid(JSON)
 **
-** Return 1 if JSON is a well-formed JSON string according to RFC-7159.
-** Return 0 otherwise.
+** Return 1 if JSON is a well-formed canonical JSON string according
+** to RFC-7159. Return 0 otherwise.
 */
 static void jsonValidFunc(
   sqlite3_context *ctx,
@@ -200767,8 +202300,69 @@ static void jsonValidFunc(
 ){
   JsonParse *p;          /* The parse */
   UNUSED_PARAMETER(argc);
+  if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
+  p = jsonParseCached(ctx, argv, 0);
+  if( p==0 || p->oom ){
+    sqlite3_result_error_nomem(ctx);
+    sqlite3_free(p);
+  }else{
+    sqlite3_result_int(ctx, p->nErr==0 && p->hasNonstd==0);
+    if( p->nErr ) jsonParseFree(p);
+  }
+}
+
+/*
+** json_error_position(JSON)
+**
+** If the argument is not an interpretable JSON string, then return the 1-based
+** character position at which the parser first recognized that the input
+** was in error.  The left-most character is 1.  If the string is valid
+** JSON, then return 0.
+**
+** Note that json_valid() is only true for strictly conforming canonical JSON.
+** But this routine returns zero if the input contains extension.  Thus:
+**
+** (1) If the input X is strictly conforming canonical JSON:
+**
+**         json_valid(X) returns true
+**         json_error_position(X) returns 0
+**
+** (2) If the input X is JSON but it includes extension (such as JSON5) that
+**     are not part of RFC-8259:
+**
+**         json_valid(X) returns false
+**         json_error_position(X) return 0
+**
+** (3) If the input X cannot be interpreted as JSON even taking extensions
+**     into account:
+**
+**         json_valid(X) return false
+**         json_error_position(X) returns 1 or more
+*/
+static void jsonErrorFunc(
+  sqlite3_context *ctx,
+  int argc,
+  sqlite3_value **argv
+){
+  JsonParse *p;          /* The parse */
+  UNUSED_PARAMETER(argc);
+  if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return;
   p = jsonParseCached(ctx, argv, 0);
-  sqlite3_result_int(ctx, p!=0);
+  if( p==0 || p->oom ){
+    sqlite3_result_error_nomem(ctx);
+    sqlite3_free(p);
+  }else if( p->nErr==0 ){
+    sqlite3_result_int(ctx, 0);
+  }else{
+    int n = 1;
+    u32 i;
+    const char *z = p->zJson;
+    for(i=0; i<p->iErr && ALWAYS(z[i]); i++){
+      if( (z[i]&0xc0)!=0x80 ) n++;
+    }
+    sqlite3_result_int(ctx, n);
+    jsonParseFree(p);
+  }
 }
 
 
@@ -201112,14 +202706,16 @@ static void jsonAppendObjectPathElement(
   assert( pNode->eU==1 );
   z = pNode->u.zJContent;
   nn = pNode->n;
-  assert( nn>=2 );
-  assert( z[0]=='"' );
-  assert( z[nn-1]=='"' );
-  if( nn>2 && sqlite3Isalpha(z[1]) ){
-    for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){}
-    if( jj==nn-1 ){
-      z++;
-      nn -= 2;
+  if( (pNode->jnFlags & JNODE_RAW)==0 ){
+    assert( nn>=2 );
+    assert( z[0]=='"' || z[0]=='\'' );
+    assert( z[nn-1]=='"' || z[0]=='\'' );
+    if( nn>2 && sqlite3Isalpha(z[1]) ){
+      for(jj=2; jj<nn-1 && sqlite3Isalnum(z[jj]); jj++){}
+      if( jj==nn-1 ){
+        z++;
+        nn -= 2;
+      }
     }
   }
   jsonPrintf(nn+2, pStr, ".%.*s", nn, z);
@@ -201479,6 +203075,7 @@ SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){
     JFUNCTION(json_array,        -1, 0,  jsonArrayFunc),
     JFUNCTION(json_array_length,  1, 0,  jsonArrayLengthFunc),
     JFUNCTION(json_array_length,  2, 0,  jsonArrayLengthFunc),
+    JFUNCTION(json_error_position,1, 0,  jsonErrorFunc),
     JFUNCTION(json_extract,      -1, 0,  jsonExtractFunc),
     JFUNCTION(->,                 2, JSON_JSON, jsonExtractFunc),
     JFUNCTION(->>,                2, JSON_SQL, jsonExtractFunc),
@@ -202003,16 +203600,17 @@ struct RtreeMatchArg {
 ** at run-time.
 */
 #ifndef SQLITE_BYTEORDER
-#if defined(i386)     || defined(__i386__)   || defined(_M_IX86) ||    \
-    defined(__x86_64) || defined(__x86_64__) || defined(_M_X64)  ||    \
-    defined(_M_AMD64) || defined(_M_ARM)     || defined(__x86)   ||    \
-    defined(__arm__)
-# define SQLITE_BYTEORDER    1234
-#elif defined(sparc)    || defined(__ppc__)
-# define SQLITE_BYTEORDER    4321
-#else
-# define SQLITE_BYTEORDER    0     /* 0 means "unknown at compile-time" */
-#endif
+# if defined(i386)      || defined(__i386__)      || defined(_M_IX86) ||    \
+     defined(__x86_64)  || defined(__x86_64__)    || defined(_M_X64)  ||    \
+     defined(_M_AMD64)  || defined(_M_ARM)        || defined(__x86)   ||    \
+     defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64)
+#   define SQLITE_BYTEORDER    1234
+# elif defined(sparc)     || defined(__ppc__) || \
+       defined(__ARMEB__) || defined(__AARCH64EB__)
+#   define SQLITE_BYTEORDER    4321
+# else
+#   define SQLITE_BYTEORDER 0
+# endif
 #endif
 
 
@@ -212557,6 +214155,11 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
   p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff);
 }
 
+/*
+** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER
+** in zipvfs.h.
+*/
+#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439
 
 /*
 ** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if
@@ -212565,9 +214168,20 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){
 static int rbuLockDatabase(sqlite3 *db){
   int rc = SQLITE_OK;
   sqlite3_file *fd = 0;
-  sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
 
-  if( fd->pMethods ){
+  sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd);
+  if( fd ){
+    sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
+    rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED);
+    if( rc==SQLITE_OK ){
+      rc = fd->pMethods->xUnlock(fd, SQLITE_LOCK_NONE);
+    }
+    sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd);
+  }else{
+    sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd);
+  }
+
+  if( rc==SQLITE_OK && fd->pMethods ){
     rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED);
     if( rc==SQLITE_OK ){
       rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE);
@@ -215804,6 +217418,7 @@ static int dbpageConnect(
   (void)pzErr;
 
   sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY);
+  sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS);
   rc = sqlite3_declare_vtab(db,
           "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)");
   if( rc==SQLITE_OK ){
@@ -215887,7 +217502,6 @@ static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){
   ){
     pIdxInfo->orderByConsumed = 1;
   }
-  sqlite3VtabUsesAllSchemas(pIdxInfo);
   return SQLITE_OK;
 }
 
@@ -216188,6 +217802,8 @@ typedef struct SessionInput SessionInput;
 # endif
 #endif
 
+#define SESSIONS_ROWID "_rowid_"
+
 static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE;
 
 typedef struct SessionHook SessionHook;
@@ -216209,6 +217825,7 @@ struct sqlite3_session {
   int bEnable;                    /* True if currently recording */
   int bIndirect;                  /* True if all changes are indirect */
   int bAutoAttach;                /* True to auto-attach tables */
+  int bImplicitPK;                /* True to handle tables with implicit PK */
   int rc;                         /* Non-zero if an error has occurred */
   void *pFilterCtx;               /* First argument to pass to xTableFilter */
   int (*xTableFilter)(void *pCtx, const char *zTab);
@@ -216285,6 +217902,7 @@ struct SessionTable {
   char *zName;                    /* Local name of table */
   int nCol;                       /* Number of columns in table zName */
   int bStat1;                     /* True if this is sqlite_stat1 */
+  int bRowid;                     /* True if this table uses rowid for PK */
   const char **azCol;             /* Column names */
   u8 *abPK;                       /* Array of primary key flags */
   int nEntry;                     /* Total number of entries in hash table */
@@ -216677,6 +218295,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){
 */
 static int sessionPreupdateHash(
   sqlite3_session *pSession,      /* Session object that owns pTab */
+  i64 iRowid,
   SessionTable *pTab,             /* Session table handle */
   int bNew,                       /* True to hash the new.* PK */
   int *piHash,                    /* OUT: Hash value */
@@ -216685,48 +218304,53 @@ static int sessionPreupdateHash(
   unsigned int h = 0;             /* Hash value to return */
   int i;                          /* Used to iterate through columns */
 
-  assert( *pbNullPK==0 );
-  assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) );
-  for(i=0; i<pTab->nCol; i++){
-    if( pTab->abPK[i] ){
-      int rc;
-      int eType;
-      sqlite3_value *pVal;
-
-      if( bNew ){
-        rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal);
-      }else{
-        rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal);
-      }
-      if( rc!=SQLITE_OK ) return rc;
+  if( pTab->bRowid ){
+    assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) );
+    h = sessionHashAppendI64(h, iRowid);
+  }else{
+    assert( *pbNullPK==0 );
+    assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) );
+    for(i=0; i<pTab->nCol; i++){
+      if( pTab->abPK[i] ){
+        int rc;
+        int eType;
+        sqlite3_value *pVal;
 
-      eType = sqlite3_value_type(pVal);
-      h = sessionHashAppendType(h, eType);
-      if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
-        i64 iVal;
-        if( eType==SQLITE_INTEGER ){
-          iVal = sqlite3_value_int64(pVal);
+        if( bNew ){
+          rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal);
         }else{
-          double rVal = sqlite3_value_double(pVal);
-          assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
-          memcpy(&iVal, &rVal, 8);
+          rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal);
         }
-        h = sessionHashAppendI64(h, iVal);
-      }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
-        const u8 *z;
-        int n;
-        if( eType==SQLITE_TEXT ){
-          z = (const u8 *)sqlite3_value_text(pVal);
+        if( rc!=SQLITE_OK ) return rc;
+
+        eType = sqlite3_value_type(pVal);
+        h = sessionHashAppendType(h, eType);
+        if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){
+          i64 iVal;
+          if( eType==SQLITE_INTEGER ){
+            iVal = sqlite3_value_int64(pVal);
+          }else{
+            double rVal = sqlite3_value_double(pVal);
+            assert( sizeof(iVal)==8 && sizeof(rVal)==8 );
+            memcpy(&iVal, &rVal, 8);
+          }
+          h = sessionHashAppendI64(h, iVal);
+        }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){
+          const u8 *z;
+          int n;
+          if( eType==SQLITE_TEXT ){
+            z = (const u8 *)sqlite3_value_text(pVal);
+          }else{
+            z = (const u8 *)sqlite3_value_blob(pVal);
+          }
+          n = sqlite3_value_bytes(pVal);
+          if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
+          h = sessionHashAppendBlob(h, n, z);
         }else{
-          z = (const u8 *)sqlite3_value_blob(pVal);
+          assert( eType==SQLITE_NULL );
+          assert( pTab->bStat1==0 || i!=1 );
+          *pbNullPK = 1;
         }
-        n = sqlite3_value_bytes(pVal);
-        if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM;
-        h = sessionHashAppendBlob(h, n, z);
-      }else{
-        assert( eType==SQLITE_NULL );
-        assert( pTab->bStat1==0 || i!=1 );
-        *pbNullPK = 1;
       }
     }
   }
@@ -217009,6 +218633,7 @@ static int sessionMergeUpdate(
 */
 static int sessionPreupdateEqual(
   sqlite3_session *pSession,      /* Session object that owns SessionTable */
+  i64 iRowid,                     /* Rowid value if pTab->bRowid */
   SessionTable *pTab,             /* Table associated with change */
   SessionChange *pChange,         /* Change to compare to */
   int op                          /* Current pre-update operation */
@@ -217016,6 +218641,11 @@ static int sessionPreupdateEqual(
   int iCol;                       /* Used to iterate through columns */
   u8 *a = pChange->aRecord;       /* Cursor used to scan change record */
 
+  if( pTab->bRowid ){
+    if( a[0]!=SQLITE_INTEGER ) return 0;
+    return sessionGetI64(&a[1])==iRowid;
+  }
+
   assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE );
   for(iCol=0; iCol<pTab->nCol; iCol++){
     if( !pTab->abPK[iCol] ){
@@ -217160,7 +218790,8 @@ static int sessionTableInfo(
   int *pnCol,                     /* OUT: number of columns */
   const char **pzTab,             /* OUT: Copy of zThis */
   const char ***pazCol,           /* OUT: Array of column names for table */
-  u8 **pabPK                      /* OUT: Array of booleans - true for PK col */
+  u8 **pabPK,                     /* OUT: Array of booleans - true for PK col */
+  int *pbRowid                    /* OUT: True if only PK is a rowid */
 ){
   char *zPragma;
   sqlite3_stmt *pStmt;
@@ -217172,6 +218803,7 @@ static int sessionTableInfo(
   u8 *pAlloc = 0;
   char **azCol = 0;
   u8 *abPK = 0;
+  int bRowid = 0;                 /* Set to true to use rowid as PK */
 
   assert( pazCol && pabPK );
 
@@ -217216,10 +218848,15 @@ static int sessionTableInfo(
   }
 
   nByte = nThis + 1;
+  bRowid = (pbRowid!=0);
   while( SQLITE_ROW==sqlite3_step(pStmt) ){
     nByte += sqlite3_column_bytes(pStmt, 1);
     nDbCol++;
+    if( sqlite3_column_int(pStmt, 5) ) bRowid = 0;
   }
+  if( nDbCol==0 ) bRowid = 0;
+  nDbCol += bRowid;
+  nByte += strlen(SESSIONS_ROWID);
   rc = sqlite3_reset(pStmt);
 
   if( rc==SQLITE_OK ){
@@ -217241,6 +218878,14 @@ static int sessionTableInfo(
     }
 
     i = 0;
+    if( bRowid ){
+      size_t nName = strlen(SESSIONS_ROWID);
+      memcpy(pAlloc, SESSIONS_ROWID, nName+1);
+      azCol[i] = (char*)pAlloc;
+      pAlloc += nName+1;
+      abPK[i] = 1;
+      i++;
+    }
     while( SQLITE_ROW==sqlite3_step(pStmt) ){
       int nName = sqlite3_column_bytes(pStmt, 1);
       const unsigned char *zName = sqlite3_column_text(pStmt, 1);
@@ -217252,7 +218897,6 @@ static int sessionTableInfo(
       i++;
     }
     rc = sqlite3_reset(pStmt);
-
   }
 
   /* If successful, populate the output variables. Otherwise, zero them and
@@ -217269,6 +218913,7 @@ static int sessionTableInfo(
     if( pzTab ) *pzTab = 0;
     sessionFree(pSession, azCol);
   }
+  if( pbRowid ) *pbRowid = bRowid;
   sqlite3_finalize(pStmt);
   return rc;
 }
@@ -217290,7 +218935,8 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){
     u8 *abPK;
     assert( pTab->azCol==0 || pTab->abPK==0 );
     pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb,
-        pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK
+        pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK,
+        (pSession->bImplicitPK ? &pTab->bRowid : 0)
     );
     if( pSession->rc==SQLITE_OK ){
       int i;
@@ -217362,6 +219008,7 @@ static int sessionUpdateMaxSize(
 ){
   i64 nNew = 2;
   if( pC->op==SQLITE_INSERT ){
+    if( pTab->bRowid ) nNew += 9;
     if( op!=SQLITE_DELETE ){
       int ii;
       for(ii=0; ii<pTab->nCol; ii++){
@@ -217378,12 +219025,16 @@ static int sessionUpdateMaxSize(
   }else{
     int ii;
     u8 *pCsr = pC->aRecord;
-    for(ii=0; ii<pTab->nCol; ii++){
+    if( pTab->bRowid ){
+      nNew += 9 + 1;
+      pCsr += 9;
+    }
+    for(ii=pTab->bRowid; ii<pTab->nCol; ii++){
       int bChanged = 1;
       int nOld = 0;
       int eType;
       sqlite3_value *p = 0;
-      pSession->hook.xNew(pSession->hook.pCtx, ii, &p);
+      pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p);
       if( p==0 ){
         return SQLITE_NOMEM;
       }
@@ -217462,6 +219113,7 @@ static int sessionUpdateMaxSize(
 */
 static void sessionPreupdateOneChange(
   int op,                         /* One of SQLITE_UPDATE, INSERT, DELETE */
+  i64 iRowid,
   sqlite3_session *pSession,      /* Session object pTab is attached to */
   SessionTable *pTab              /* Table that change applies to */
 ){
@@ -217477,7 +219129,7 @@ static void sessionPreupdateOneChange(
 
   /* Check the number of columns in this xPreUpdate call matches the
   ** number of columns in the table.  */
-  if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){
+  if( (pTab->nCol-pTab->bRowid)!=pSession->hook.xCount(pSession->hook.pCtx) ){
     pSession->rc = SQLITE_SCHEMA;
     return;
   }
@@ -217510,14 +219162,16 @@ static void sessionPreupdateOneChange(
   /* Calculate the hash-key for this change. If the primary key of the row
   ** includes a NULL value, exit early. Such changes are ignored by the
   ** session module. */
-  rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull);
+  rc = sessionPreupdateHash(
+      pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull
+  );
   if( rc!=SQLITE_OK ) goto error_out;
 
   if( bNull==0 ){
     /* Search the hash table for an existing record for this row. */
     SessionChange *pC;
     for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){
-      if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break;
+      if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break;
     }
 
     if( pC==0 ){
@@ -217532,7 +219186,7 @@ static void sessionPreupdateOneChange(
 
       /* Figure out how large an allocation is required */
       nByte = sizeof(SessionChange);
-      for(i=0; i<pTab->nCol; i++){
+      for(i=0; i<(pTab->nCol-pTab->bRowid); i++){
         sqlite3_value *p = 0;
         if( op!=SQLITE_INSERT ){
           TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p);
@@ -217547,6 +219201,9 @@ static void sessionPreupdateOneChange(
         rc = sessionSerializeValue(0, p, &nByte);
         if( rc!=SQLITE_OK ) goto error_out;
       }
+      if( pTab->bRowid ){
+        nByte += 9;               /* Size of rowid field - an integer */
+      }
 
       /* Allocate the change object */
       pC = (SessionChange *)sessionMalloc64(pSession, nByte);
@@ -217563,7 +219220,12 @@ static void sessionPreupdateOneChange(
       ** required values and encodings have already been cached in memory.
       ** It is not possible for an OOM to occur in this block. */
       nByte = 0;
-      for(i=0; i<pTab->nCol; i++){
+      if( pTab->bRowid ){
+        pC->aRecord[0] = SQLITE_INTEGER;
+        sessionPutI64(&pC->aRecord[1], iRowid);
+        nByte = 9;
+      }
+      for(i=0; i<(pTab->nCol-pTab->bRowid); i++){
         sqlite3_value *p = 0;
         if( op!=SQLITE_INSERT ){
           pSession->hook.xOld(pSession->hook.pCtx, i, &p);
@@ -217678,9 +219340,10 @@ static void xPreUpdate(
     pSession->rc = sessionFindTable(pSession, zName, &pTab);
     if( pTab ){
       assert( pSession->rc==SQLITE_OK );
-      sessionPreupdateOneChange(op, pSession, pTab);
+      assert( op==SQLITE_UPDATE || iKey1==iKey2 );
+      sessionPreupdateOneChange(op, iKey1, pSession, pTab);
       if( op==SQLITE_UPDATE ){
-        sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab);
+        sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab);
       }
     }
   }
@@ -217719,6 +219382,7 @@ static void sessionPreupdateHooks(
 typedef struct SessionDiffCtx SessionDiffCtx;
 struct SessionDiffCtx {
   sqlite3_stmt *pStmt;
+  int bRowid;
   int nOldOff;
 };
 
@@ -217727,17 +219391,17 @@ struct SessionDiffCtx {
 */
 static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){
   SessionDiffCtx *p = (SessionDiffCtx*)pCtx;
-  *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff);
+  *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid);
   return SQLITE_OK;
 }
 static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){
   SessionDiffCtx *p = (SessionDiffCtx*)pCtx;
-  *ppVal = sqlite3_column_value(p->pStmt, iVal);
+  *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid);
    return SQLITE_OK;
 }
 static int sessionDiffCount(void *pCtx){
   SessionDiffCtx *p = (SessionDiffCtx*)pCtx;
-  return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt);
+  return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid;
 }
 static int sessionDiffDepth(void *pCtx){
   (void)pCtx;
@@ -217816,14 +219480,16 @@ static char *sessionExprCompareOther(
 static char *sessionSelectFindNew(
   const char *zDb1,      /* Pick rows in this db only */
   const char *zDb2,      /* But not in this one */
+  int bRowid,
   const char *zTbl,      /* Table name */
   const char *zExpr
 ){
+  const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*");
   char *zRet = sqlite3_mprintf(
-      "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS ("
+      "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS ("
       "  SELECT 1 FROM \"%w\".\"%w\" WHERE %s"
       ")",
-      zDb1, zTbl, zDb2, zTbl, zExpr
+      zSel, zDb1, zTbl, zDb2, zTbl, zExpr
   );
   return zRet;
 }
@@ -217837,7 +219503,9 @@ static int sessionDiffFindNew(
   char *zExpr
 ){
   int rc = SQLITE_OK;
-  char *zStmt = sessionSelectFindNew(zDb1, zDb2, pTab->zName,zExpr);
+  char *zStmt = sessionSelectFindNew(
+      zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr
+  );
 
   if( zStmt==0 ){
     rc = SQLITE_NOMEM;
@@ -217848,8 +219516,10 @@ static int sessionDiffFindNew(
       SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx;
       pDiffCtx->pStmt = pStmt;
       pDiffCtx->nOldOff = 0;
+      pDiffCtx->bRowid = pTab->bRowid;
       while( SQLITE_ROW==sqlite3_step(pStmt) ){
-        sessionPreupdateOneChange(op, pSession, pTab);
+        i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0);
+        sessionPreupdateOneChange(op, iRowid, pSession, pTab);
       }
       rc = sqlite3_finalize(pStmt);
     }
@@ -217859,6 +219529,27 @@ static int sessionDiffFindNew(
   return rc;
 }
 
+/*
+** Return a comma-separated list of the fully-qualified (with both database
+** and table name) column names from table pTab. e.g.
+**
+**    "main"."t1"."a", "main"."t1"."b", "main"."t1"."c"
+*/
+static char *sessionAllCols(
+  const char *zDb,
+  SessionTable *pTab
+){
+  int ii;
+  char *zRet = 0;
+  for(ii=0; ii<pTab->nCol; ii++){
+    zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"",
+        zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii]
+    );
+    if( !zRet ) break;
+  }
+  return zRet;
+}
+
 static int sessionDiffFindModified(
   sqlite3_session *pSession,
   SessionTable *pTab,
@@ -217873,11 +219564,13 @@ static int sessionDiffFindModified(
   if( zExpr2==0 ){
     rc = SQLITE_NOMEM;
   }else{
+    char *z1 = sessionAllCols(pSession->zDb, pTab);
+    char *z2 = sessionAllCols(zFrom, pTab);
     char *zStmt = sqlite3_mprintf(
-        "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)",
-        pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2
+        "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)",
+        z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2
     );
-    if( zStmt==0 ){
+    if( zStmt==0 || z1==0 || z2==0 ){
       rc = SQLITE_NOMEM;
     }else{
       sqlite3_stmt *pStmt;
@@ -217888,12 +219581,15 @@ static int sessionDiffFindModified(
         pDiffCtx->pStmt = pStmt;
         pDiffCtx->nOldOff = pTab->nCol;
         while( SQLITE_ROW==sqlite3_step(pStmt) ){
-          sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab);
+          i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0);
+          sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab);
         }
         rc = sqlite3_finalize(pStmt);
       }
-      sqlite3_free(zStmt);
     }
+    sqlite3_free(zStmt);
+    sqlite3_free(z1);
+    sqlite3_free(z2);
   }
 
   return rc;
@@ -217932,9 +219628,12 @@ SQLITE_API int sqlite3session_diff(
       int bHasPk = 0;
       int bMismatch = 0;
       int nCol;                   /* Columns in zFrom.zTbl */
+      int bRowid = 0;
       u8 *abPK;
       const char **azCol = 0;
-      rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK);
+      rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, &abPK,
+          pSession->bImplicitPK ? &bRowid : 0
+      );
       if( rc==SQLITE_OK ){
         if( pTo->nCol!=nCol ){
           bMismatch = 1;
@@ -218276,9 +219975,10 @@ static void sessionAppendStr(
   int *pRc
 ){
   int nStr = sqlite3Strlen30(zStr);
-  if( 0==sessionBufferGrow(p, nStr, pRc) ){
+  if( 0==sessionBufferGrow(p, nStr+1, pRc) ){
     memcpy(&p->aBuf[p->nBuf], zStr, nStr);
     p->nBuf += nStr;
+    p->aBuf[p->nBuf] = 0x00;
   }
 }
 
@@ -218300,6 +220000,27 @@ static void sessionAppendInteger(
   sessionAppendStr(p, aBuf, pRc);
 }
 
+static void sessionAppendPrintf(
+  SessionBuffer *p,               /* Buffer to append to */
+  int *pRc,
+  const char *zFmt,
+  ...
+){
+  if( *pRc==SQLITE_OK ){
+    char *zApp = 0;
+    va_list ap;
+    va_start(ap, zFmt);
+    zApp = sqlite3_vmprintf(zFmt, ap);
+    if( zApp==0 ){
+      *pRc = SQLITE_NOMEM;
+    }else{
+      sessionAppendStr(p, zApp, pRc);
+    }
+    va_end(ap);
+    sqlite3_free(zApp);
+  }
+}
+
 /*
 ** This function is a no-op if *pRc is other than SQLITE_OK when it is
 ** called. Otherwise, append the string zStr enclosed in quotes (") and
@@ -218314,7 +220035,7 @@ static void sessionAppendIdent(
   const char *zStr,               /* String to quote, escape and append */
   int *pRc                        /* IN/OUT: Error code */
 ){
-  int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1;
+  int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2;
   if( 0==sessionBufferGrow(p, nStr, pRc) ){
     char *zOut = (char *)&p->aBuf[p->nBuf];
     const char *zIn = zStr;
@@ -218325,6 +220046,7 @@ static void sessionAppendIdent(
     }
     *zOut++ = '"';
     p->nBuf = (int)((u8 *)zOut - p->aBuf);
+    p->aBuf[p->nBuf] = 0x00;
   }
 }
 
@@ -218460,7 +220182,7 @@ static int sessionAppendUpdate(
     /* If at least one field has been modified, this is not a no-op. */
     if( bChanged ) bNoop = 0;
 
-    /* Add a field to the old.* record. This is omitted if this modules is
+    /* Add a field to the old.* record. This is omitted if this module is
     ** currently generating a patchset. */
     if( bPatchset==0 ){
       if( bChanged || abPK[i] ){
@@ -218549,12 +220271,20 @@ static int sessionAppendDelete(
 ** Formulate and prepare a SELECT statement to retrieve a row from table
 ** zTab in database zDb based on its primary key. i.e.
 **
-**   SELECT * FROM zDb.zTab WHERE pk1 = ? AND pk2 = ? AND ...
+**   SELECT *, <noop-test> FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...)
+**
+** where <noop-test> is:
+**
+**   1 AND (?A OR ?1 IS <column>) AND ...
+**
+** for each non-pk <column>.
 */
 static int sessionSelectStmt(
   sqlite3 *db,                    /* Database handle */
+  int bIgnoreNoop,
   const char *zDb,                /* Database name */
   const char *zTab,               /* Table name */
+  int bRowid,
   int nCol,                       /* Number of columns in table */
   const char **azCol,             /* Names of table columns */
   u8 *abPK,                       /* PRIMARY KEY  array */
@@ -218562,8 +220292,50 @@ static int sessionSelectStmt(
 ){
   int rc = SQLITE_OK;
   char *zSql = 0;
+  const char *zSep = "";
+  const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*";
   int nSql = -1;
+  int i;
+
+  SessionBuffer nooptest = {0, 0, 0};
+  SessionBuffer pkfield = {0, 0, 0};
+  SessionBuffer pkvar = {0, 0, 0};
+
+  sessionAppendStr(&nooptest, ", 1", &rc);
+
+  if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){
+    sessionAppendStr(&nooptest, " AND (?6 OR ?3 IS stat)", &rc);
+    sessionAppendStr(&pkfield, "tbl, idx", &rc);
+    sessionAppendStr(&pkvar,
+        "?1, (CASE WHEN ?2=X'' THEN NULL ELSE ?2 END)", &rc
+    );
+    zCols = "tbl, ?2, stat";
+  }else{
+    for(i=0; i<nCol; i++){
+      if( abPK[i] ){
+        sessionAppendStr(&pkfield, zSep, &rc);
+        sessionAppendStr(&pkvar, zSep, &rc);
+        zSep = ", ";
+        sessionAppendIdent(&pkfield, azCol[i], &rc);
+        sessionAppendPrintf(&pkvar, &rc, "?%d", i+1);
+      }else{
+        sessionAppendPrintf(&nooptest, &rc,
+            " AND (?%d OR ?%d IS %w.%w)", i+1+nCol, i+1, zTab, azCol[i]
+        );
+      }
+    }
+  }
+
+  if( rc==SQLITE_OK ){
+    zSql = sqlite3_mprintf(
+        "SELECT %s%s FROM %Q.%Q WHERE (%s) IS (%s)",
+        zCols, (bIgnoreNoop ? (char*)nooptest.aBuf : ""),
+        zDb, zTab, (char*)pkfield.aBuf, (char*)pkvar.aBuf
+    );
+    if( zSql==0 ) rc = SQLITE_NOMEM;
+  }
 
+#if 0
   if( 0==sqlite3_stricmp("sqlite_stat1", zTab) ){
     zSql = sqlite3_mprintf(
         "SELECT tbl, ?2, stat FROM %Q.sqlite_stat1 WHERE tbl IS ?1 AND "
@@ -218571,7 +220343,6 @@ static int sessionSelectStmt(
     );
     if( zSql==0 ) rc = SQLITE_NOMEM;
   }else{
-    int i;
     const char *zSep = "";
     SessionBuffer buf = {0, 0, 0};
 
@@ -218592,11 +220363,15 @@ static int sessionSelectStmt(
     zSql = (char*)buf.aBuf;
     nSql = buf.nBuf;
   }
+#endif
 
   if( rc==SQLITE_OK ){
     rc = sqlite3_prepare_v2(db, zSql, nSql, ppStmt, 0);
   }
   sqlite3_free(zSql);
+  sqlite3_free(nooptest.aBuf);
+  sqlite3_free(pkfield.aBuf);
+  sqlite3_free(pkvar.aBuf);
   return rc;
 }
 
@@ -218743,10 +220518,18 @@ static int sessionGenerateChangeset(
       sqlite3_stmt *pSel = 0;     /* SELECT statement to query table pTab */
       int nRewind = buf.nBuf;     /* Initial size of write buffer */
       int nNoop;                  /* Size of buffer after writing tbl header */
+      int bRowid = 0;
 
       /* Check the table schema is still Ok. */
-      rc = sessionTableInfo(0, db, pSession->zDb, zName, &nCol, 0,&azCol,&abPK);
-      if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){
+      rc = sessionTableInfo(
+          0, db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK,
+          (pSession->bImplicitPK ? &bRowid : 0)
+      );
+      if( rc==SQLITE_OK && (
+          pTab->nCol!=nCol
+       || pTab->bRowid!=bRowid
+       || memcmp(abPK, pTab->abPK, nCol)
+      )){
         rc = SQLITE_SCHEMA;
       }
 
@@ -218756,7 +220539,8 @@ static int sessionGenerateChangeset(
       /* Build and compile a statement to execute: */
       if( rc==SQLITE_OK ){
         rc = sessionSelectStmt(
-            db, pSession->zDb, zName, nCol, azCol, abPK, &pSel);
+            db, 0, pSession->zDb, zName, bRowid, nCol, azCol, abPK, &pSel
+        );
       }
 
       nNoop = buf.nBuf;
@@ -218839,7 +220623,7 @@ SQLITE_API int sqlite3session_changeset(
   int rc;
 
   if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE;
-  rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset,ppChangeset);
+  rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset);
   assert( rc || pnChangeset==0
        || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize
   );
@@ -218957,6 +220741,19 @@ SQLITE_API int sqlite3session_object_config(sqlite3_session *pSession, int op, v
       break;
     }
 
+    case SQLITE_SESSION_OBJCONFIG_ROWID: {
+      int iArg = *(int*)pArg;
+      if( iArg>=0 ){
+        if( pSession->pTable ){
+          rc = SQLITE_MISUSE;
+        }else{
+          pSession->bImplicitPK = (iArg!=0);
+        }
+      }
+      *(int*)pArg = pSession->bImplicitPK;
+      break;
+    }
+
     default:
       rc = SQLITE_MISUSE;
   }
@@ -219945,6 +221742,8 @@ struct SessionApplyCtx {
   SessionBuffer rebase;           /* Rebase information (if any) here */
   u8 bRebaseStarted;              /* If table header is already in rebase */
   u8 bRebase;                     /* True to collect rebase information */
+  u8 bIgnoreNoop;                 /* True to ignore no-op conflicts */
+  int bRowid;
 };
 
 /* Number of prepared UPDATE statements to cache. */
@@ -220195,8 +221994,10 @@ static int sessionSelectRow(
   const char *zTab,               /* Table name */
   SessionApplyCtx *p              /* Session changeset-apply context */
 ){
-  return sessionSelectStmt(
-      db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect);
+  /* TODO */
+  return sessionSelectStmt(db, p->bIgnoreNoop,
+      "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect
+  );
 }
 
 /*
@@ -220355,20 +222156,33 @@ static int sessionBindRow(
 */
 static int sessionSeekToRow(
   sqlite3_changeset_iter *pIter,  /* Changeset iterator */
-  u8 *abPK,                       /* Primary key flags array */
-  sqlite3_stmt *pSelect           /* SELECT statement from sessionSelectRow() */
+  SessionApplyCtx *p
 ){
+  sqlite3_stmt *pSelect = p->pSelect;
   int rc;                         /* Return code */
   int nCol;                       /* Number of columns in table */
   int op;                         /* Changset operation (SQLITE_UPDATE etc.) */
   const char *zDummy;             /* Unused */
 
+  sqlite3_clear_bindings(pSelect);
   sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0);
   rc = sessionBindRow(pIter,
       op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old,
-      nCol, abPK, pSelect
+      nCol, p->abPK, pSelect
   );
 
+  if( op!=SQLITE_DELETE && p->bIgnoreNoop ){
+    int ii;
+    for(ii=0; rc==SQLITE_OK && ii<nCol; ii++){
+      if( p->abPK[ii]==0 ){
+        sqlite3_value *pVal = 0;
+        sqlite3changeset_new(pIter, ii, &pVal);
+        sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0));
+        if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal);
+      }
+    }
+  }
+
   if( rc==SQLITE_OK ){
     rc = sqlite3_step(pSelect);
     if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect);
@@ -220483,16 +222297,22 @@ static int sessionConflictHandler(
 
   /* Bind the new.* PRIMARY KEY values to the SELECT statement. */
   if( pbReplace ){
-    rc = sessionSeekToRow(pIter, p->abPK, p->pSelect);
+    rc = sessionSeekToRow(pIter, p);
   }else{
     rc = SQLITE_OK;
   }
 
   if( rc==SQLITE_ROW ){
     /* There exists another row with the new.* primary key. */
-    pIter->pConflict = p->pSelect;
-    res = xConflict(pCtx, eType, pIter);
-    pIter->pConflict = 0;
+    if( p->bIgnoreNoop
+     && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1)
+    ){
+      res = SQLITE_CHANGESET_OMIT;
+    }else{
+      pIter->pConflict = p->pSelect;
+      res = xConflict(pCtx, eType, pIter);
+      pIter->pConflict = 0;
+    }
     rc = sqlite3_reset(p->pSelect);
   }else if( rc==SQLITE_OK ){
     if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){
@@ -220600,7 +222420,7 @@ static int sessionApplyOneOp(
 
     sqlite3_step(p->pDelete);
     rc = sqlite3_reset(p->pDelete);
-    if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){
+    if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){
       rc = sessionConflictHandler(
           SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry
       );
@@ -220657,7 +222477,7 @@ static int sessionApplyOneOp(
       /* Check if there is a conflicting row. For sqlite_stat1, this needs
       ** to be done using a SELECT, as there is no PRIMARY KEY in the
       ** database schema to throw an exception if a duplicate is inserted.  */
-      rc = sessionSeekToRow(pIter, p->abPK, p->pSelect);
+      rc = sessionSeekToRow(pIter, p);
       if( rc==SQLITE_ROW ){
         rc = SQLITE_CONSTRAINT;
         sqlite3_reset(p->pSelect);
@@ -220834,6 +222654,7 @@ static int sessionChangesetApply(
   memset(&sApply, 0, sizeof(sApply));
   sApply.bRebase = (ppRebase && pnRebase);
   sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT);
+  sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP);
   sqlite3_mutex_enter(sqlite3_db_mutex(db));
   if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){
     rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0);
@@ -220871,6 +222692,7 @@ static int sessionChangesetApply(
       sApply.bStat1 = 0;
       sApply.bDeferConstraints = 1;
       sApply.bRebaseStarted = 0;
+      sApply.bRowid = 0;
       memset(&sApply.constraints, 0, sizeof(SessionBuffer));
 
       /* If an xFilter() callback was specified, invoke it now. If the
@@ -220890,8 +222712,8 @@ static int sessionChangesetApply(
         int i;
 
         sqlite3changeset_pk(pIter, &abPK, 0);
-        rc = sessionTableInfo(0,
-            db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK
+        rc = sessionTableInfo(0, db, "main", zNew,
+            &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK, &sApply.bRowid
         );
         if( rc!=SQLITE_OK ) break;
         for(i=0; i<sApply.nCol; i++){
@@ -222773,6 +224595,7 @@ struct Fts5Config {
   int ePattern;                   /* FTS_PATTERN_XXX constant */
 
   /* Values loaded from the %_config table */
+  int iVersion;                   /* fts5 file format 'version' */
   int iCookie;                    /* Incremented when %_config is modified */
   int pgsz;                       /* Approximate page size used in %_data */
   int nAutomerge;                 /* 'automerge' setting */
@@ -222781,6 +224604,7 @@ struct Fts5Config {
   int nHashSize;                  /* Bytes of memory for in-memory hash */
   char *zRank;                    /* Name of rank function */
   char *zRankArgs;                /* Arguments to rank function */
+  int bSecureDelete;              /* 'secure-delete' */
 
   /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */
   char **pzErrmsg;
@@ -222790,8 +224614,11 @@ struct Fts5Config {
 #endif
 };
 
-/* Current expected value of %_config table 'version' field */
-#define FTS5_CURRENT_VERSION  4
+/* Current expected value of %_config table 'version' field. And
+** the expected version if the 'secure-delete' option has ever been
+** set on the table.  */
+#define FTS5_CURRENT_VERSION               4
+#define FTS5_CURRENT_VERSION_SECUREDELETE  5
 
 #define FTS5_CONTENT_NORMAL   0
 #define FTS5_CONTENT_NONE     1
@@ -222957,6 +224784,7 @@ struct Fts5IndexIter {
 ** above. */
 #define FTS5INDEX_QUERY_SKIPEMPTY  0x0010
 #define FTS5INDEX_QUERY_NOOUTPUT   0x0020
+#define FTS5INDEX_QUERY_SKIPHASH   0x0040
 
 /*
 ** Create/destroy an Fts5Index object.
@@ -223111,7 +224939,7 @@ static int sqlite3Fts5GetVarintLen(u32 iVal);
 static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*);
 static int sqlite3Fts5PutVarint(unsigned char *p, u64 v);
 
-#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b)
+#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b))
 #define fts5GetVarint    sqlite3Fts5GetVarint
 
 #define fts5FastGetVarint32(a, iOff, nVal) {      \
@@ -225090,7 +226918,7 @@ static int fts5HighlightCb(
   if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK;
   iPos = p->iPos++;
 
-  if( p->iRangeEnd>0 ){
+  if( p->iRangeEnd>=0 ){
     if( iPos<p->iRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK;
     if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff;
   }
@@ -225102,7 +226930,7 @@ static int fts5HighlightCb(
   }
 
   if( iPos==p->iter.iEnd ){
-    if( p->iRangeEnd && p->iter.iStart<p->iRangeStart ){
+    if( p->iRangeEnd>=0 && p->iter.iStart<p->iRangeStart ){
       fts5HighlightAppend(&rc, p, p->zOpen, -1);
     }
     fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
@@ -225113,7 +226941,7 @@ static int fts5HighlightCb(
     }
   }
 
-  if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){
+  if( p->iRangeEnd>=0 && iPos==p->iRangeEnd ){
     fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff);
     p->iOff = iEndOff;
     if( iPos>=p->iter.iStart && iPos<p->iter.iEnd ){
@@ -225148,6 +226976,7 @@ static void fts5HighlightFunction(
   memset(&ctx, 0, sizeof(HighlightContext));
   ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]);
   ctx.zClose = (const char*)sqlite3_value_text(apVal[2]);
+  ctx.iRangeEnd = -1;
   rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn);
 
   if( ctx.zIn ){
@@ -225333,6 +227162,7 @@ static void fts5SnippetFunction(
   iCol = sqlite3_value_int(apVal[0]);
   ctx.zOpen = fts5ValueToText(apVal[1]);
   ctx.zClose = fts5ValueToText(apVal[2]);
+  ctx.iRangeEnd = -1;
   zEllips = fts5ValueToText(apVal[3]);
   nToken = sqlite3_value_int(apVal[4]);
 
@@ -226601,6 +228431,7 @@ static int sqlite3Fts5ConfigParse(
     rc = SQLITE_ERROR;
   }
 
+  assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK );
   for(i=3; rc==SQLITE_OK && i<nArg; i++){
     const char *zOrig = azArg[i];
     const char *z;
@@ -226954,6 +228785,18 @@ static int sqlite3Fts5ConfigSetValue(
       rc = SQLITE_OK;
       *pbBadkey = 1;
     }
+  }
+
+  else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){
+    int bVal = -1;
+    if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){
+      bVal = sqlite3_value_int(pVal);
+    }
+    if( bVal<0 ){
+      *pbBadkey = 1;
+    }else{
+      pConfig->bSecureDelete = (bVal ? 1 : 0);
+    }
   }else{
     *pbBadkey = 1;
   }
@@ -226998,15 +228841,20 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
     rc = sqlite3_finalize(p);
   }
 
-  if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){
+  if( rc==SQLITE_OK
+   && iVersion!=FTS5_CURRENT_VERSION
+   && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE
+  ){
     rc = SQLITE_ERROR;
     if( pConfig->pzErrmsg ){
       assert( 0==*pConfig->pzErrmsg );
-      *pConfig->pzErrmsg = sqlite3_mprintf(
-          "invalid fts5 file format (found %d, expected %d) - run 'rebuild'",
-          iVersion, FTS5_CURRENT_VERSION
+      *pConfig->pzErrmsg = sqlite3_mprintf("invalid fts5 file format "
+          "(found %d, expected %d or %d) - run 'rebuild'",
+          iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE
       );
     }
+  }else{
+    pConfig->iVersion = iVersion;
   }
 
   if( rc==SQLITE_OK ){
@@ -227034,6 +228882,10 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){
 /* #include "fts5Int.h" */
 /* #include "fts5parse.h" */
 
+#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH
+# define SQLITE_FTS5_MAX_EXPR_DEPTH 256
+#endif
+
 /*
 ** All token types in the generated fts5parse.h file are greater than 0.
 */
@@ -227074,11 +228926,17 @@ struct Fts5Expr {
 **       FTS5_NOT                 (nChild, apChild valid)
 **       FTS5_STRING              (pNear valid)
 **       FTS5_TERM                (pNear valid)
+**
+** iHeight:
+**   Distance from this node to furthest leaf. This is always 0 for nodes
+**   of type FTS5_STRING and FTS5_TERM. For all other nodes it is one
+**   greater than the largest child value.
 */
 struct Fts5ExprNode {
   int eType;                      /* Node type */
   int bEof;                       /* True at EOF */
   int bNomatch;                   /* True if entry is not a match */
+  int iHeight;                    /* Distance to tree leaf nodes */
 
   /* Next method for this node. */
   int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64);
@@ -227148,6 +229006,31 @@ struct Fts5Parse {
   int bPhraseToAnd;               /* Convert "a+b" to "a AND b" */
 };
 
+/*
+** Check that the Fts5ExprNode.iHeight variables are set correctly in
+** the expression tree passed as the only argument.
+*/
+#ifndef NDEBUG
+static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){
+  if( rc==SQLITE_OK ){
+    if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){
+      assert( p->iHeight==0 );
+    }else{
+      int ii;
+      int iMaxChild = 0;
+      for(ii=0; ii<p->nChild; ii++){
+        Fts5ExprNode *pChild = p->apChild[ii];
+        iMaxChild = MAX(iMaxChild, pChild->iHeight);
+        assert_expr_depth_ok(SQLITE_OK, pChild);
+      }
+      assert( p->iHeight==iMaxChild+1 );
+    }
+  }
+}
+#else
+# define assert_expr_depth_ok(rc, p)
+#endif
+
 static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){
   va_list ap;
   va_start(ap, zFmt);
@@ -227262,6 +229145,8 @@ static int sqlite3Fts5ExprNew(
   }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF );
   sqlite3Fts5ParserFree(pEngine, fts5ParseFree);
 
+  assert_expr_depth_ok(sParse.rc, sParse.pExpr);
+
   /* If the LHS of the MATCH expression was a user column, apply the
   ** implicit column-filter.  */
   if( iCol<pConfig->nCol && sParse.pExpr && sParse.rc==SQLITE_OK ){
@@ -227424,7 +229309,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
   Fts5Parse sParse;
   memset(&sParse, 0, sizeof(sParse));
 
-  if( *pp1 ){
+  if( *pp1 && p2 ){
     Fts5Expr *p1 = *pp1;
     int nPhrase = p1->nPhrase + p2->nPhrase;
 
@@ -227449,7 +229334,7 @@ static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){
     }
     sqlite3_free(p2->apExprPhrase);
     sqlite3_free(p2);
-  }else{
+  }else if( p2 ){
     *pp1 = p2;
   }
 
@@ -229223,6 +231108,7 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){
 }
 
 static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
+  int ii = p->nChild;
   if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){
     int nByte = sizeof(Fts5ExprNode*) * pSub->nChild;
     memcpy(&p->apChild[p->nChild], pSub->apChild, nByte);
@@ -229231,6 +231117,9 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){
   }else{
     p->apChild[p->nChild++] = pSub;
   }
+  for( ; ii<p->nChild; ii++){
+    p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1);
+  }
 }
 
 /*
@@ -229261,6 +231150,7 @@ static Fts5ExprNode *fts5ParsePhraseToAnd(
   if( pRet ){
     pRet->eType = FTS5_AND;
     pRet->nChild = nTerm;
+    pRet->iHeight = 1;
     fts5ExprAssignXNext(pRet);
     pParse->nPhrase--;
     for(ii=0; ii<nTerm; ii++){
@@ -229366,6 +231256,14 @@ static Fts5ExprNode *sqlite3Fts5ParseNode(
         }else{
           fts5ExprAddChildren(pRet, pLeft);
           fts5ExprAddChildren(pRet, pRight);
+          if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){
+            sqlite3Fts5ParseError(pParse,
+                "fts5 expression tree is too large (maximum depth %d)",
+                SQLITE_FTS5_MAX_EXPR_DEPTH
+            );
+            sqlite3_free(pRet);
+            pRet = 0;
+          }
         }
       }
     }
@@ -230966,6 +232864,8 @@ struct Fts5Index {
   sqlite3_stmt *pIdxSelect;
   int nRead;                      /* Total number of blocks read */
 
+  sqlite3_stmt *pDeleteFromIdx;
+
   sqlite3_stmt *pDataVersion;
   i64 iStructVersion;             /* data_version when pStruct read */
   Fts5Structure *pStruct;         /* Current db structure (or NULL) */
@@ -231058,9 +232958,6 @@ struct Fts5CResult {
 ** iLeafOffset:
 **   Byte offset within the current leaf that is the first byte of the
 **   position list data (one byte passed the position-list size field).
-**   rowid field of the current entry. Usually this is the size field of the
-**   position list data. The exception is if the rowid for the current entry
-**   is the last thing on the leaf page.
 **
 ** pLeaf:
 **   Buffer containing current leaf page data. Set to NULL at EOF.
@@ -231619,6 +233516,7 @@ static int fts5StructureDecode(
             rc = FTS5_CORRUPT;
             break;
           }
+          assert( pSeg!=0 );
           i += fts5GetVarint32(&pData[i], pSeg->iSegid);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst);
           i += fts5GetVarint32(&pData[i], pSeg->pgnoLast);
@@ -231649,6 +233547,7 @@ static int fts5StructureDecode(
 */
 static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){
   fts5StructureMakeWritable(pRc, ppStruct);
+  assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK );
   if( *pRc==SQLITE_OK ){
     Fts5Structure *pStruct = *ppStruct;
     int nLevel = pStruct->nLevel;
@@ -232107,42 +234006,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){
     pLvl->bEof = 1;
   }else{
     u8 *a = pLvl->pData->p;
-    i64 iVal;
-    int iLimit;
-    int ii;
-    int nZero = 0;
-
-    /* Currently iOff points to the first byte of a varint. This block
-    ** decrements iOff until it points to the first byte of the previous
-    ** varint. Taking care not to read any memory locations that occur
-    ** before the buffer in memory.  */
-    iLimit = (iOff>9 ? iOff-9 : 0);
-    for(iOff--; iOff>iLimit; iOff--){
-      if( (a[iOff-1] & 0x80)==0 ) break;
-    }
-
-    fts5GetVarint(&a[iOff], (u64*)&iVal);
-    pLvl->iRowid -= iVal;
-    pLvl->iLeafPgno--;
-
-    /* Skip backwards past any 0x00 varints. */
-    for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){
-      nZero++;
-    }
-    if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){
-      /* The byte immediately before the last 0x00 byte has the 0x80 bit
-      ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80
-      ** bytes before a[ii]. */
-      int bZero = 0;              /* True if last 0x00 counts */
-      if( (ii-8)>=pLvl->iFirstOff ){
-        int j;
-        for(j=1; j<=8 && (a[ii-j] & 0x80); j++);
-        bZero = (j>8);
+
+    pLvl->iOff = 0;
+    fts5DlidxLvlNext(pLvl);
+    while( 1 ){
+      int nZero = 0;
+      int ii = pLvl->iOff;
+      u64 delta = 0;
+
+      while( a[ii]==0 ){
+        nZero++;
+        ii++;
       }
-      if( bZero==0 ) nZero--;
+      ii += sqlite3Fts5GetVarint(&a[ii], &delta);
+
+      if( ii>=iOff ) break;
+      pLvl->iLeafPgno += nZero+1;
+      pLvl->iRowid += delta;
+      pLvl->iOff = ii;
     }
-    pLvl->iLeafPgno -= nZero;
-    pLvl->iOff = iOff - nZero;
   }
 
   return pLvl->bEof;
@@ -232338,7 +234220,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){
   i64 iOff = pIter->iLeafOffset;
 
   ASSERT_SZLEAF_OK(pIter->pLeaf);
-  if( iOff>=pIter->pLeaf->szLeaf ){
+  while( iOff>=pIter->pLeaf->szLeaf ){
     fts5SegIterNextPage(p, pIter);
     if( pIter->pLeaf==0 ){
       if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT;
@@ -232437,10 +234319,12 @@ static void fts5SegIterInit(
     fts5SegIterSetNext(p, pIter);
     pIter->pSeg = pSeg;
     pIter->iLeafPgno = pSeg->pgnoFirst-1;
-    fts5SegIterNextPage(p, pIter);
+    do {
+      fts5SegIterNextPage(p, pIter);
+    }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 );
   }
 
-  if( p->rc==SQLITE_OK ){
+  if( p->rc==SQLITE_OK && pIter->pLeaf ){
     pIter->iLeafOffset = 4;
     assert( pIter->pLeaf!=0 );
     assert_nc( pIter->pLeaf->nn>4 );
@@ -232634,7 +234518,7 @@ static void fts5SegIterNext_None(
   iOff = pIter->iLeafOffset;
 
   /* Next entry is on the next page */
-  if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
+  while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){
     fts5SegIterNextPage(p, pIter);
     if( p->rc || pIter->pLeaf==0 ) return;
     pIter->iRowid = 0;
@@ -232827,7 +234711,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){
   Fts5Data *pLast = 0;
   int pgnoLast = 0;
 
-  if( pDlidx ){
+  if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){
     int iSegid = pIter->pSeg->iSegid;
     pgnoLast = fts5DlidxIterPgno(pDlidx);
     pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast));
@@ -233388,7 +235272,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){
 
 /*
 ** Move the seg-iter so that it points to the first rowid on page iLeafPgno.
-** It is an error if leaf iLeafPgno does not exist or contains no rowids.
+** It is an error if leaf iLeafPgno does not exist. Unless the db is
+** a 'secure-delete' db, if it contains no rowids then this is also an error.
 */
 static void fts5SegIterGotoPage(
   Fts5Index *p,                   /* FTS5 backend object */
@@ -233403,21 +235288,23 @@ static void fts5SegIterGotoPage(
     fts5DataRelease(pIter->pNextLeaf);
     pIter->pNextLeaf = 0;
     pIter->iLeafPgno = iLeafPgno-1;
-    fts5SegIterNextPage(p, pIter);
-    assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno );
 
-    if( p->rc==SQLITE_OK && ALWAYS(pIter->pLeaf!=0) ){
+    while( p->rc==SQLITE_OK ){
       int iOff;
-      u8 *a = pIter->pLeaf->p;
-      int n = pIter->pLeaf->szLeaf;
-
+      fts5SegIterNextPage(p, pIter);
+      if( pIter->pLeaf==0 ) break;
       iOff = fts5LeafFirstRowidOff(pIter->pLeaf);
-      if( iOff<4 || iOff>=n ){
-        p->rc = FTS5_CORRUPT;
-      }else{
-        iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
-        pIter->iLeafOffset = iOff;
-        fts5SegIterLoadNPos(p, pIter);
+      if( iOff>0 ){
+        u8 *a = pIter->pLeaf->p;
+        int n = pIter->pLeaf->szLeaf;
+        if( iOff<4 || iOff>=n ){
+          p->rc = FTS5_CORRUPT;
+        }else{
+          iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid);
+          pIter->iLeafOffset = iOff;
+          fts5SegIterLoadNPos(p, pIter);
+        }
+        break;
       }
     }
   }
@@ -234132,7 +236019,7 @@ static void fts5MultiIterNew(
     if( iLevel<0 ){
       assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) );
       nSeg = pStruct->nSegment;
-      nSeg += (p->pHash ? 1 : 0);
+      nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH));
     }else{
       nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment);
     }
@@ -234153,7 +236040,7 @@ static void fts5MultiIterNew(
   if( p->rc==SQLITE_OK ){
     if( iLevel<0 ){
       Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel];
-      if( p->pHash ){
+      if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){
         /* Add a segment iterator for the current contents of the hash table. */
         Fts5SegIter *pIter = &pNew->aSeg[iIter++];
         fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter);
@@ -234908,7 +236795,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){
           fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr);
           fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n);
           fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p);
-          fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff,&pData->p[iOff]);
+          fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]);
           if( p->rc==SQLITE_OK ){
             /* Set the szLeaf field */
             fts5PutU16(&buf.p[2], (u16)buf.n);
@@ -235186,16 +237073,16 @@ static void fts5IndexCrisismerge(
 ){
   const int nCrisis = p->pConfig->nCrisisMerge;
   Fts5Structure *pStruct = *ppStruct;
-  int iLvl = 0;
-
-  assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 );
-  while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
-    fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
-    assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) );
-    fts5StructurePromote(p, iLvl+1, pStruct);
-    iLvl++;
+  if( pStruct && pStruct->nLevel>0 ){
+    int iLvl = 0;
+    while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){
+      fts5IndexMergeLevel(p, &pStruct, iLvl, 0);
+      assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) );
+      fts5StructurePromote(p, iLvl+1, pStruct);
+      iLvl++;
+    }
+    *ppStruct = pStruct;
   }
-  *ppStruct = pStruct;
 }
 
 static int fts5IndexReturn(Fts5Index *p){
@@ -235230,6 +237117,413 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){
 }
 
 /*
+** Execute the SQL statement:
+**
+**    DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno);
+**
+** This is used when a secure-delete operation removes the last term
+** from a segment leaf page. In that case the %_idx entry is removed
+** too. This is done to ensure that if all instances of a token are
+** removed from an fts5 database in secure-delete mode, no trace of
+** the token itself remains in the database.
+*/
+static void fts5SecureDeleteIdxEntry(
+  Fts5Index *p,                   /* FTS5 backend object */
+  int iSegid,                     /* Id of segment to delete entry for */
+  int iPgno                       /* Page number within segment */
+){
+  if( iPgno!=1 ){
+    assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE );
+    if( p->pDeleteFromIdx==0 ){
+      fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf(
+          "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)",
+          p->pConfig->zDb, p->pConfig->zName
+      ));
+    }
+    if( p->rc==SQLITE_OK ){
+      sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid);
+      sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno);
+      sqlite3_step(p->pDeleteFromIdx);
+      p->rc = sqlite3_reset(p->pDeleteFromIdx);
+    }
+  }
+}
+
+/*
+** This is called when a secure-delete operation removes a position-list
+** that overflows onto segment page iPgno of segment pSeg. This function
+** rewrites node iPgno, and possibly one or more of its right-hand peers,
+** to remove this portion of the position list.
+**
+** Output variable (*pbLastInDoclist) is set to true if the position-list
+** removed is followed by a new term or the end-of-segment, or false if
+** it is followed by another rowid/position list.
+*/
+static void fts5SecureDeleteOverflow(
+  Fts5Index *p,
+  Fts5StructureSegment *pSeg,
+  int iPgno,
+  int *pbLastInDoclist
+){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
+  int pgno;
+  Fts5Data *pLeaf = 0;
+  assert( iPgno!=1 );
+
+  *pbLastInDoclist = 1;
+  for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){
+    i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno);
+    int iNext = 0;
+    u8 *aPg = 0;
+
+    pLeaf = fts5DataRead(p, iRowid);
+    if( pLeaf==0 ) break;
+    aPg = pLeaf->p;
+
+    iNext = fts5GetU16(&aPg[0]);
+    if( iNext!=0 ){
+      *pbLastInDoclist = 0;
+    }
+    if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){
+      fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext);
+    }
+
+    if( iNext==0 ){
+      /* The page contains no terms or rowids. Replace it with an empty
+      ** page and move on to the right-hand peer.  */
+      const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04};
+      assert_nc( bDetailNone==0 || pLeaf->nn==4 );
+      if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty));
+      fts5DataRelease(pLeaf);
+      pLeaf = 0;
+    }else if( bDetailNone ){
+      break;
+    }else if( iNext>=pLeaf->szLeaf || iNext<4 ){
+      p->rc = FTS5_CORRUPT;
+      break;
+    }else{
+      int nShift = iNext - 4;
+      int nPg;
+
+      int nIdx = 0;
+      u8 *aIdx = 0;
+
+      /* Unless the current page footer is 0 bytes in size (in which case
+      ** the new page footer will be as well), allocate and populate a
+      ** buffer containing the new page footer. Set stack variables aIdx
+      ** and nIdx accordingly.  */
+      if( pLeaf->nn>pLeaf->szLeaf ){
+        int iFirst = 0;
+        int i1 = pLeaf->szLeaf;
+        int i2 = 0;
+
+        aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2);
+        if( aIdx==0 ) break;
+        i1 += fts5GetVarint32(&aPg[i1], iFirst);
+        i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift);
+        if( i1<pLeaf->nn ){
+          memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1);
+          i2 += (pLeaf->nn-i1);
+        }
+        nIdx = i2;
+      }
+
+      /* Modify the contents of buffer aPg[]. Set nPg to the new size
+      ** in bytes. The new page is always smaller than the old.  */
+      nPg = pLeaf->szLeaf - nShift;
+      memmove(&aPg[4], &aPg[4+nShift], nPg-4);
+      fts5PutU16(&aPg[2], nPg);
+      if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4);
+      if( nIdx>0 ){
+        memcpy(&aPg[nPg], aIdx, nIdx);
+        nPg += nIdx;
+      }
+      sqlite3_free(aIdx);
+
+      /* Write the new page to disk and exit the loop */
+      assert( nPg>4 || fts5GetU16(aPg)==0 );
+      fts5DataWrite(p, iRowid, aPg, nPg);
+      break;
+    }
+  }
+  fts5DataRelease(pLeaf);
+}
+
+/*
+** Completely remove the entry that pSeg currently points to from
+** the database.
+*/
+static void fts5DoSecureDelete(
+  Fts5Index *p,
+  Fts5SegIter *pSeg
+){
+  const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE);
+  int iSegid = pSeg->pSeg->iSegid;
+  u8 *aPg = pSeg->pLeaf->p;
+  int nPg = pSeg->pLeaf->nn;
+  int iPgIdx = pSeg->pLeaf->szLeaf;
+
+  u64 iDelta = 0;
+  u64 iNextDelta = 0;
+  int iNextOff = 0;
+  int iOff = 0;
+  int nIdx = 0;
+  u8 *aIdx = 0;
+  int bLastInDoclist = 0;
+  int iIdx = 0;
+  int iStart = 0;
+  int iKeyOff = 0;
+  int iPrevKeyOff = 0;
+  int iDelKeyOff = 0;       /* Offset of deleted key, if any */
+
+  nIdx = nPg-iPgIdx;
+  aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16);
+  if( p->rc ) return;
+  memcpy(aIdx, &aPg[iPgIdx], nIdx);
+
+  /* At this point segment iterator pSeg points to the entry
+  ** this function should remove from the b-tree segment.
+  **
+  ** In detail=full or detail=column mode, pSeg->iLeafOffset is the
+  ** offset of the first byte in the position-list for the entry to
+  ** remove. Immediately before this comes two varints that will also
+  ** need to be removed:
+  **
+  **     + the rowid or delta rowid value for the entry, and
+  **     + the size of the position list in bytes.
+  **
+  ** Or, in detail=none mode, there is a single varint prior to
+  ** pSeg->iLeafOffset - the rowid or delta rowid value.
+  **
+  ** This block sets the following variables:
+  **
+  **   iStart:
+  **   iDelta:
+  */
+  {
+    int iSOP;
+    if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){
+      iStart = pSeg->iTermLeafOffset;
+    }else{
+      iStart = fts5GetU16(&aPg[0]);
+    }
+
+    iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+    assert_nc( iSOP<=pSeg->iLeafOffset );
+
+    if( bDetailNone ){
+      while( iSOP<pSeg->iLeafOffset ){
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        if( aPg[iSOP]==0x00 ) iSOP++;
+        iStart = iSOP;
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+      }
+
+      iNextOff = iSOP;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+      if( iNextOff<pSeg->iEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++;
+
+    }else{
+      int nPos = 0;
+      iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+      while( iSOP<pSeg->iLeafOffset ){
+        iStart = iSOP + (nPos/2);
+        iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta);
+        iSOP += fts5GetVarint32(&aPg[iSOP], nPos);
+      }
+      assert_nc( iSOP==pSeg->iLeafOffset );
+      iNextOff = pSeg->iLeafOffset + pSeg->nPos;
+    }
+  }
+
+  iOff = iStart;
+  if( iNextOff>=iPgIdx ){
+    int pgno = pSeg->iLeafPgno+1;
+    fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist);
+    iNextOff = iPgIdx;
+  }else{
+    /* Set bLastInDoclist to true if the entry being removed is the last
+    ** in its doclist.  */
+    for(iIdx=0, iKeyOff=0; iIdx<nIdx; /* no-op */){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      iKeyOff += iVal;
+      if( iKeyOff==iNextOff ){
+        bLastInDoclist = 1;
+      }
+    }
+  }
+
+  if( fts5GetU16(&aPg[0])==iStart && (bLastInDoclist||iNextOff==iPgIdx) ){
+    fts5PutU16(&aPg[0], 0);
+  }
+
+  if( bLastInDoclist==0 ){
+    if( iNextOff!=iPgIdx ){
+      iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta);
+      iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta);
+    }
+  }else if(
+      iStart==pSeg->iTermLeafOffset && pSeg->iLeafPgno==pSeg->iTermLeafPgno
+  ){
+    /* The entry being removed was the only position list in its
+    ** doclist. Therefore the term needs to be removed as well. */
+    int iKey = 0;
+    for(iIdx=0, iKeyOff=0; iIdx<nIdx; iKey++){
+      u32 iVal = 0;
+      iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+      if( (iKeyOff+iVal)>(u32)iStart ) break;
+      iKeyOff += iVal;
+    }
+
+    iDelKeyOff = iOff = iKeyOff;
+    if( iNextOff!=iPgIdx ){
+      int nPrefix = 0;
+      int nSuffix = 0;
+      int nPrefix2 = 0;
+      int nSuffix2 = 0;
+
+      iDelKeyOff = iNextOff;
+      iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2);
+      iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2);
+
+      if( iKey!=1 ){
+        iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix);
+      }
+      iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix);
+
+      nPrefix = MIN(nPrefix, nPrefix2);
+      nSuffix = (nPrefix2 + nSuffix2) - nPrefix;
+
+      if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){
+        p->rc = FTS5_CORRUPT;
+      }else{
+        if( iKey!=1 ){
+          iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix);
+        }
+        iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix);
+        if( nPrefix2>nPrefix ){
+          memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix);
+          iOff += (nPrefix2-nPrefix);
+        }
+        memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2);
+        iOff += nSuffix2;
+        iNextOff += nSuffix2;
+      }
+    }
+  }else if( iStart==4 ){
+      int iPgno;
+
+      assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno );
+      /* The entry being removed may be the only position list in
+      ** its doclist. */
+      for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){
+        Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno));
+        int bEmpty = (pPg && pPg->nn==4);
+        fts5DataRelease(pPg);
+        if( bEmpty==0 ) break;
+      }
+
+      if( iPgno==pSeg->iTermLeafPgno ){
+        i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno);
+        Fts5Data *pTerm = fts5DataRead(p, iId);
+        if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){
+          u8 *aTermIdx = &pTerm->p[pTerm->szLeaf];
+          int nTermIdx = pTerm->nn - pTerm->szLeaf;
+          int iTermIdx = 0;
+          int iTermOff = 0;
+
+          while( 1 ){
+            u32 iVal = 0;
+            int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal);
+            iTermOff += iVal;
+            if( (iTermIdx+nByte)>=nTermIdx ) break;
+            iTermIdx += nByte;
+          }
+          nTermIdx = iTermIdx;
+
+          memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx);
+          fts5PutU16(&pTerm->p[2], iTermOff);
+
+          fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx);
+          if( nTermIdx==0 ){
+            fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno);
+          }
+        }
+        fts5DataRelease(pTerm);
+      }
+    }
+
+    if( p->rc==SQLITE_OK ){
+      const int nMove = nPg - iNextOff;
+      int nShift = 0;
+
+      memmove(&aPg[iOff], &aPg[iNextOff], nMove);
+      iPgIdx -= (iNextOff - iOff);
+      nPg = iPgIdx;
+      fts5PutU16(&aPg[2], iPgIdx);
+
+      nShift = iNextOff - iOff;
+      for(iIdx=0, iKeyOff=0, iPrevKeyOff=0; iIdx<nIdx; /* no-op */){
+        u32 iVal = 0;
+        iIdx += fts5GetVarint32(&aIdx[iIdx], iVal);
+        iKeyOff += iVal;
+        if( iKeyOff!=iDelKeyOff ){
+          if( iKeyOff>iOff ){
+            iKeyOff -= nShift;
+            nShift = 0;
+          }
+          nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOff - iPrevKeyOff);
+          iPrevKeyOff = iKeyOff;
+        }
+      }
+
+      if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){
+        fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno);
+      }
+
+      assert_nc( nPg>4 || fts5GetU16(aPg)==0 );
+      fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg,nPg);
+    }
+    sqlite3_free(aIdx);
+}
+
+/*
+** This is called as part of flushing a delete to disk in 'secure-delete'
+** mode. It edits the segments within the database described by argument
+** pStruct to remove the entries for term zTerm, rowid iRowid.
+*/
+static void fts5FlushSecureDelete(
+  Fts5Index *p,
+  Fts5Structure *pStruct,
+  const char *zTerm,
+  i64 iRowid
+){
+  const int f = FTS5INDEX_QUERY_SKIPHASH;
+  int nTerm = (int)strlen(zTerm);
+  Fts5Iter *pIter = 0;            /* Used to find term instance */
+
+  fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter);
+  if( fts5MultiIterEof(p, pIter)==0 ){
+    i64 iThis = fts5MultiIterRowid(pIter);
+    if( iThis<iRowid ){
+      fts5MultiIterNextFrom(p, pIter, iRowid);
+    }
+
+    if( p->rc==SQLITE_OK
+     && fts5MultiIterEof(p, pIter)==0
+     && iRowid==fts5MultiIterRowid(pIter)
+    ){
+      Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst];
+      fts5DoSecureDelete(p, pSeg);
+    }
+  }
+
+  fts5MultiIterFree(pIter);
+}
+
+
+/*
 ** Flush the contents of in-memory hash table iHash to a new level-0
 ** segment on disk. Also update the corresponding structure record.
 **
@@ -235251,6 +237545,7 @@ static void fts5FlushOneHash(Fts5Index *p){
   if( iSegid ){
     const int pgsz = p->pConfig->pgsz;
     int eDetail = p->pConfig->eDetail;
+    int bSecureDelete = p->pConfig->bSecureDelete;
     Fts5StructureSegment *pSeg;   /* New segment within pStruct */
     Fts5Buffer *pBuf;             /* Buffer in which to assemble leaf page */
     Fts5Buffer *pPgidx;           /* Buffer in which to assemble pgidx */
@@ -235273,40 +237568,77 @@ static void fts5FlushOneHash(Fts5Index *p){
     }
     while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){
       const char *zTerm;          /* Buffer containing term */
+      int nTerm;                  /* Size of zTerm in bytes */
       const u8 *pDoclist;         /* Pointer to doclist for this term */
       int nDoclist;               /* Size of doclist in bytes */
 
-      /* Write the term for this entry to disk. */
+      /* Get the term and doclist for this entry. */
       sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist);
-      fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm);
-      if( p->rc!=SQLITE_OK ) break;
+      nTerm = (int)strlen(zTerm);
+      if( bSecureDelete==0 ){
+        fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+        if( p->rc!=SQLITE_OK ) break;
+        assert( writer.bFirstRowidInPage==0 );
+      }
 
-      assert( writer.bFirstRowidInPage==0 );
-      if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
+      if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){
         /* The entire doclist will fit on the current leaf. */
         fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist);
       }else{
+        int bTermWritten = !bSecureDelete;
         i64 iRowid = 0;
-        u64 iDelta = 0;
+        i64 iPrev = 0;
         int iOff = 0;
 
         /* The entire doclist will not fit on this leaf. The following
         ** loop iterates through the poslists that make up the current
         ** doclist.  */
         while( p->rc==SQLITE_OK && iOff<nDoclist ){
+          u64 iDelta = 0;
           iOff += fts5GetVarint(&pDoclist[iOff], &iDelta);
           iRowid += iDelta;
 
+          /* If in secure delete mode, and if this entry in the poslist is
+          ** in fact a delete, then edit the existing segments directly
+          ** using fts5FlushSecureDelete().  */
+          if( bSecureDelete ){
+            if( eDetail==FTS5_DETAIL_NONE ){
+              if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+                iOff++;
+                if( iOff<nDoclist && pDoclist[iOff]==0x00 ){
+                  iOff++;
+                  nDoclist = 0;
+                }else{
+                  continue;
+                }
+              }
+            }else if( (pDoclist[iOff] & 0x01) ){
+              fts5FlushSecureDelete(p, pStruct, zTerm, iRowid);
+              if( p->rc!=SQLITE_OK || pDoclist[iOff]==0x01 ){
+                iOff++;
+                continue;
+              }
+            }
+          }
+
+          if( p->rc==SQLITE_OK && bTermWritten==0 ){
+            fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm);
+            bTermWritten = 1;
+            assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 );
+          }
+
           if( writer.bFirstRowidInPage ){
             fts5PutU16(&pBuf->p[0], (u16)pBuf->n);   /* first rowid on page */
             pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid);
             writer.bFirstRowidInPage = 0;
             fts5WriteDlidxAppend(p, &writer, iRowid);
-            if( p->rc!=SQLITE_OK ) break;
           }else{
-            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta);
+            pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid-iPrev);
           }
+          if( p->rc!=SQLITE_OK ) break;
           assert( pBuf->n<=pBuf->nSpace );
+          iPrev = iRowid;
 
           if( eDetail==FTS5_DETAIL_NONE ){
             if( iOff<nDoclist && pDoclist[iOff]==0 ){
@@ -235365,20 +237697,23 @@ static void fts5FlushOneHash(Fts5Index *p){
     sqlite3Fts5HashClear(pHash);
     fts5WriteFinish(p, &writer, &pgnoLast);
 
-    /* Update the Fts5Structure. It is written back to the database by the
-    ** fts5StructureRelease() call below.  */
-    if( pStruct->nLevel==0 ){
-      fts5StructureAddLevel(&p->rc, &pStruct);
-    }
-    fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
-    if( p->rc==SQLITE_OK ){
-      pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
-      pSeg->iSegid = iSegid;
-      pSeg->pgnoFirst = 1;
-      pSeg->pgnoLast = pgnoLast;
-      pStruct->nSegment++;
+    assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 );
+    if( pgnoLast>0 ){
+      /* Update the Fts5Structure. It is written back to the database by the
+      ** fts5StructureRelease() call below.  */
+      if( pStruct->nLevel==0 ){
+        fts5StructureAddLevel(&p->rc, &pStruct);
+      }
+      fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0);
+      if( p->rc==SQLITE_OK ){
+        pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ];
+        pSeg->iSegid = iSegid;
+        pSeg->pgnoFirst = 1;
+        pSeg->pgnoLast = pgnoLast;
+        pStruct->nSegment++;
+      }
+      fts5StructurePromote(p, 0, pStruct);
     }
-    fts5StructurePromote(p, 0, pStruct);
   }
 
   fts5IndexAutomerge(p, &pStruct, pgnoLast);
@@ -236119,6 +238454,7 @@ static int sqlite3Fts5IndexClose(Fts5Index *p){
     sqlite3_finalize(p->pIdxDeleter);
     sqlite3_finalize(p->pIdxSelect);
     sqlite3_finalize(p->pDataVersion);
+    sqlite3_finalize(p->pDeleteFromIdx);
     sqlite3Fts5HashFree(p->pHash);
     sqlite3_free(p->zDataTbl);
     sqlite3_free(p);
@@ -236749,6 +239085,7 @@ static void fts5IndexIntegrityCheckSegment(
   Fts5StructureSegment *pSeg      /* Segment to check internal consistency */
 ){
   Fts5Config *pConfig = p->pConfig;
+  int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE);
   sqlite3_stmt *pStmt = 0;
   int rc2;
   int iIdxPrevLeaf = pSeg->pgnoFirst-1;
@@ -236784,7 +239121,19 @@ static void fts5IndexIntegrityCheckSegment(
     ** is also a rowid pointer within the leaf page header, it points to a
     ** location before the term.  */
     if( pLeaf->nn<=pLeaf->szLeaf ){
-      p->rc = FTS5_CORRUPT;
+
+      if( nIdxTerm==0
+       && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE
+       && pLeaf->nn==pLeaf->szLeaf
+       && pLeaf->nn==4
+      ){
+        /* special case - the very first page in a segment keeps its %_idx
+        ** entry even if all the terms are removed from it by secure-delete
+        ** operations. */
+      }else{
+        p->rc = FTS5_CORRUPT;
+      }
+
     }else{
       int iOff;                   /* Offset of first term on leaf */
       int iRowidOff;              /* Offset of first rowid on leaf */
@@ -236848,9 +239197,12 @@ static void fts5IndexIntegrityCheckSegment(
           ASSERT_SZLEAF_OK(pLeaf);
           if( iRowidOff>=pLeaf->szLeaf ){
             p->rc = FTS5_CORRUPT;
-          }else{
+          }else if( bSecureDelete==0 || iRowidOff>0 ){
+            i64 iDlRowid = fts5DlidxIterRowid(pDlidx);
             fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid);
-            if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT;
+            if( iRowid<iDlRowid || (bSecureDelete==0 && iRowid!=iDlRowid) ){
+              p->rc = FTS5_CORRUPT;
+            }
           }
           fts5DataRelease(pLeaf);
         }
@@ -239112,6 +241464,8 @@ static int fts5UpdateMethod(
   Fts5Config *pConfig = pTab->p.pConfig;
   int eType0;                     /* value_type() of apVal[0] */
   int rc = SQLITE_OK;             /* Return code */
+  int bUpdateOrDelete = 0;
+
 
   /* A transaction must be open when this is called. */
   assert( pTab->ts.eState==1 || pTab->ts.eState==2 );
@@ -239122,6 +241476,11 @@ static int fts5UpdateMethod(
        || sqlite3_value_type(apVal[0])==SQLITE_NULL
   );
   assert( pTab->p.pConfig->pzErrmsg==0 );
+  if( pConfig->pgsz==0 ){
+    rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex);
+    if( rc!=SQLITE_OK ) return rc;
+  }
+
   pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg;
 
   /* Put any active cursors into REQUIRE_SEEK state. */
@@ -239174,6 +241533,7 @@ static int fts5UpdateMethod(
     else if( nArg==1 ){
       i64 iDel = sqlite3_value_int64(apVal[0]);  /* Rowid to delete */
       rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0);
+      bUpdateOrDelete = 1;
     }
 
     /* INSERT or UPDATE */
@@ -239189,6 +241549,7 @@ static int fts5UpdateMethod(
         if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){
           i64 iNew = sqlite3_value_int64(apVal[1]);  /* Rowid to delete */
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0);
+          bUpdateOrDelete = 1;
         }
         fts5StorageInsert(&rc, pTab, apVal, pRowid);
       }
@@ -239217,10 +241578,24 @@ static int fts5UpdateMethod(
           rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0);
           fts5StorageInsert(&rc, pTab, apVal, pRowid);
         }
+        bUpdateOrDelete = 1;
       }
     }
   }
 
+  if( rc==SQLITE_OK
+   && bUpdateOrDelete
+   && pConfig->bSecureDelete
+   && pConfig->iVersion==FTS5_CURRENT_VERSION
+  ){
+    rc = sqlite3Fts5StorageConfigValue(
+        pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE
+    );
+    if( rc==SQLITE_OK ){
+      pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE;
+    }
+  }
+
   pTab->p.pConfig->pzErrmsg = 0;
   return rc;
 }
@@ -240080,6 +242455,7 @@ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){
   UNUSED_PARAM(iSavepoint);  /* Call below is a no-op for NDEBUG builds */
   fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint);
   fts5TripCursors(pTab);
+  pTab->p.pConfig->pgsz = 0;
   return sqlite3Fts5StorageRollback(pTab->pStorage);
 }
 
@@ -240282,7 +242658,7 @@ static void fts5SourceIdFunc(
 ){
   assert( nArg==0 );
   UNUSED_PARAM2(nArg, apUnused);
-  sqlite3_result_text(pCtx, "fts5: 2023-03-22 11:56:21 0d1fc92f94cb6b76bffe3ec34d69cffde2924203304e8ffc4155597af0c191da", -1, SQLITE_TRANSIENT);
+  sqlite3_result_text(pCtx, "fts5: 2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0", -1, SQLITE_TRANSIENT);
 }
 
 /*
@@ -245274,3 +247650,4 @@ SQLITE_API int sqlite3_stmt_init(
 /* Return the source-id for this library */
 SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; }
 /************************** End of sqlite3.c ******************************/
+#pragma GCC diagnostic pop
diff --git a/database/sqlite/sqlite3.h b/database/sqlite/sqlite3.h
index 7e43e1f1b..48effe202 100644
--- a/database/sqlite/sqlite3.h
+++ b/database/sqlite/sqlite3.h
@@ -146,9 +146,9 @@ extern "C" {
 ** [sqlite3_libversion_number()], [sqlite3_sourceid()],
 ** [sqlite_version()] and [sqlite_source_id()].
 */
-#define SQLITE_VERSION        "3.41.2"
-#define SQLITE_VERSION_NUMBER 3041002
-#define SQLITE_SOURCE_ID      "2023-03-22 11:56:21 0d1fc92f94cb6b76bffe3ec34d69cffde2924203304e8ffc4155597af0c191da"
+#define SQLITE_VERSION        "3.42.0"
+#define SQLITE_VERSION_NUMBER 3042000
+#define SQLITE_SOURCE_ID      "2023-05-16 12:36:15 831d0fb2836b71c9bc51067c49fee4b8f18047814f2ff22d817d25195cf350b0"
 
 /*
 ** CAPI3REF: Run-Time Library Version Numbers
@@ -1655,20 +1655,23 @@ SQLITE_API int sqlite3_os_end(void);
 ** must ensure that no other SQLite interfaces are invoked by other
 ** threads while sqlite3_config() is running.</b>
 **
-** The sqlite3_config() interface
-** may only be invoked prior to library initialization using
-** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
-** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
-** [sqlite3_shutdown()] then it will return SQLITE_MISUSE.
-** Note, however, that ^sqlite3_config() can be called as part of the
-** implementation of an application-defined [sqlite3_os_init()].
-**
 ** The first argument to sqlite3_config() is an integer
 ** [configuration option] that determines
 ** what property of SQLite is to be configured.  Subsequent arguments
 ** vary depending on the [configuration option]
 ** in the first argument.
 **
+** For most configuration options, the sqlite3_config() interface
+** may only be invoked prior to library initialization using
+** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()].
+** The exceptional configuration options that may be invoked at any time
+** are called "anytime configuration options".
+** ^If sqlite3_config() is called after [sqlite3_initialize()] and before
+** [sqlite3_shutdown()] with a first argument that is not an anytime
+** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE.
+** Note, however, that ^sqlite3_config() can be called as part of the
+** implementation of an application-defined [sqlite3_os_init()].
+**
 ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK].
 ** ^If the option is unknown or SQLite is unable to set the option
 ** then this routine returns a non-zero [error code].
@@ -1776,6 +1779,23 @@ struct sqlite3_mem_methods {
 ** These constants are the available integer configuration options that
 ** can be passed as the first argument to the [sqlite3_config()] interface.
 **
+** Most of the configuration options for sqlite3_config()
+** will only work if invoked prior to [sqlite3_initialize()] or after
+** [sqlite3_shutdown()].  The few exceptions to this rule are called
+** "anytime configuration options".
+** ^Calling [sqlite3_config()] with a first argument that is not an
+** anytime configuration option in between calls to [sqlite3_initialize()] and
+** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE.
+**
+** The set of anytime configuration options can change (by insertions
+** and/or deletions) from one release of SQLite to the next.
+** As of SQLite version 3.42.0, the complete set of anytime configuration
+** options is:
+** <ul>
+** <li> SQLITE_CONFIG_LOG
+** <li> SQLITE_CONFIG_PCACHE_HDRSZ
+** </ul>
+**
 ** New configuration options may be added in future releases of SQLite.
 ** Existing configuration options might be discontinued.  Applications
 ** should check the return code from [sqlite3_config()] to make sure that
@@ -2122,28 +2142,28 @@ struct sqlite3_mem_methods {
 ** compile-time option is not set, then the default maximum is 1073741824.
 ** </dl>
 */
-#define SQLITE_CONFIG_SINGLETHREAD  1  /* nil */
-#define SQLITE_CONFIG_MULTITHREAD   2  /* nil */
-#define SQLITE_CONFIG_SERIALIZED    3  /* nil */
-#define SQLITE_CONFIG_MALLOC        4  /* sqlite3_mem_methods* */
-#define SQLITE_CONFIG_GETMALLOC     5  /* sqlite3_mem_methods* */
-#define SQLITE_CONFIG_SCRATCH       6  /* No longer used */
-#define SQLITE_CONFIG_PAGECACHE     7  /* void*, int sz, int N */
-#define SQLITE_CONFIG_HEAP          8  /* void*, int nByte, int min */
-#define SQLITE_CONFIG_MEMSTATUS     9  /* boolean */
-#define SQLITE_CONFIG_MUTEX        10  /* sqlite3_mutex_methods* */
-#define SQLITE_CONFIG_GETMUTEX     11  /* sqlite3_mutex_methods* */
-/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */
-#define SQLITE_CONFIG_LOOKASIDE    13  /* int int */
-#define SQLITE_CONFIG_PCACHE       14  /* no-op */
-#define SQLITE_CONFIG_GETPCACHE    15  /* no-op */
-#define SQLITE_CONFIG_LOG          16  /* xFunc, void* */
-#define SQLITE_CONFIG_URI          17  /* int */
-#define SQLITE_CONFIG_PCACHE2      18  /* sqlite3_pcache_methods2* */
-#define SQLITE_CONFIG_GETPCACHE2   19  /* sqlite3_pcache_methods2* */
+#define SQLITE_CONFIG_SINGLETHREAD         1  /* nil */
+#define SQLITE_CONFIG_MULTITHREAD          2  /* nil */
+#define SQLITE_CONFIG_SERIALIZED           3  /* nil */
+#define SQLITE_CONFIG_MALLOC               4  /* sqlite3_mem_methods* */
+#define SQLITE_CONFIG_GETMALLOC            5  /* sqlite3_mem_methods* */
+#define SQLITE_CONFIG_SCRATCH              6  /* No longer used */
+#define SQLITE_CONFIG_PAGECACHE            7  /* void*, int sz, int N */
+#define SQLITE_CONFIG_HEAP                 8  /* void*, int nByte, int min */
+#define SQLITE_CONFIG_MEMSTATUS            9  /* boolean */
+#define SQLITE_CONFIG_MUTEX               10  /* sqlite3_mutex_methods* */
+#define SQLITE_CONFIG_GETMUTEX            11  /* sqlite3_mutex_methods* */
+/* previously SQLITE_CONFIG_CHUNKALLOC    12 which is now unused. */
+#define SQLITE_CONFIG_LOOKASIDE           13  /* int int */
+#define SQLITE_CONFIG_PCACHE              14  /* no-op */
+#define SQLITE_CONFIG_GETPCACHE           15  /* no-op */
+#define SQLITE_CONFIG_LOG                 16  /* xFunc, void* */
+#define SQLITE_CONFIG_URI                 17  /* int */
+#define SQLITE_CONFIG_PCACHE2             18  /* sqlite3_pcache_methods2* */
+#define SQLITE_CONFIG_GETPCACHE2          19  /* sqlite3_pcache_methods2* */
 #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20  /* int */
-#define SQLITE_CONFIG_SQLLOG       21  /* xSqllog, void* */
-#define SQLITE_CONFIG_MMAP_SIZE    22  /* sqlite3_int64, sqlite3_int64 */
+#define SQLITE_CONFIG_SQLLOG              21  /* xSqllog, void* */
+#define SQLITE_CONFIG_MMAP_SIZE           22  /* sqlite3_int64, sqlite3_int64 */
 #define SQLITE_CONFIG_WIN32_HEAPSIZE      23  /* int nByte */
 #define SQLITE_CONFIG_PCACHE_HDRSZ        24  /* int *psz */
 #define SQLITE_CONFIG_PMASZ               25  /* unsigned int szPma */
@@ -2378,7 +2398,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_DQS_DML]]
-** <dt>SQLITE_DBCONFIG_DQS_DML</td>
+** <dt>SQLITE_DBCONFIG_DQS_DML</dt>
 ** <dd>The SQLITE_DBCONFIG_DQS_DML option activates or deactivates
 ** the legacy [double-quoted string literal] misfeature for DML statements
 ** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The
@@ -2387,7 +2407,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_DQS_DDL]]
-** <dt>SQLITE_DBCONFIG_DQS_DDL</td>
+** <dt>SQLITE_DBCONFIG_DQS_DDL</dt>
 ** <dd>The SQLITE_DBCONFIG_DQS option activates or deactivates
 ** the legacy [double-quoted string literal] misfeature for DDL statements,
 ** such as CREATE TABLE and CREATE INDEX. The
@@ -2396,7 +2416,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]]
-** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</td>
+** <dt>SQLITE_DBCONFIG_TRUSTED_SCHEMA</dt>
 ** <dd>The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to
 ** assume that database schemas are untainted by malicious content.
 ** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite
@@ -2416,7 +2436,7 @@ struct sqlite3_mem_methods {
 ** </dd>
 **
 ** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]]
-** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</td>
+** <dt>SQLITE_DBCONFIG_LEGACY_FILE_FORMAT</dt>
 ** <dd>The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates
 ** the legacy file format flag.  When activated, this flag causes all newly
 ** created database file to have a schema format version number (the 4-byte
@@ -2425,7 +2445,7 @@ struct sqlite3_mem_methods {
 ** any SQLite version back to 3.0.0 ([dateof:3.0.0]).  Without this setting,
 ** newly created databases are generally not understandable by SQLite versions
 ** prior to 3.3.0 ([dateof:3.3.0]).  As these words are written, there
-** is now scarcely any need to generated database files that are compatible
+** is now scarcely any need to generate database files that are compatible
 ** all the way back to version 3.0.0, and so this setting is of little
 ** practical use, but is provided so that SQLite can continue to claim the
 ** ability to generate new database files that are compatible with  version
@@ -2436,6 +2456,38 @@ struct sqlite3_mem_methods {
 ** not considered a bug since SQLite versions 3.3.0 and earlier do not support
 ** either generated columns or decending indexes.
 ** </dd>
+**
+** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]]
+** <dt>SQLITE_DBCONFIG_STMT_SCANSTATUS</dt>
+** <dd>The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in
+** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears
+** a flag that enables collection of the sqlite3_stmt_scanstatus_v2()
+** statistics. For statistics to be collected, the flag must be set on
+** the database handle both when the SQL statement is prepared and when it
+** is stepped. The flag is set (collection of statistics is enabled)
+** by default.  This option takes two arguments: an integer and a pointer to
+** an integer..  The first argument is 1, 0, or -1 to enable, disable, or
+** leave unchanged the statement scanstatus option.  If the second argument
+** is not NULL, then the value of the statement scanstatus setting after
+** processing the first argument is written into the integer that the second
+** argument points to.
+** </dd>
+**
+** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]]
+** <dt>SQLITE_DBCONFIG_REVERSE_SCANORDER</dt>
+** <dd>The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order
+** in which tables and indexes are scanned so that the scans start at the end
+** and work toward the beginning rather than starting at the beginning and
+** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the
+** same as setting [PRAGMA reverse_unordered_selects].  This option takes
+** two arguments which are an integer and a pointer to an integer.  The first
+** argument is 1, 0, or -1 to enable, disable, or leave unchanged the
+** reverse scan order flag, respectively.  If the second argument is not NULL,
+** then 0 or 1 is written into the integer that the second argument points to
+** depending on if the reverse scan order flag is set after processing the
+** first argument.
+** </dd>
+**
 ** </dl>
 */
 #define SQLITE_DBCONFIG_MAINDBNAME            1000 /* const char* */
@@ -2456,7 +2508,9 @@ struct sqlite3_mem_methods {
 #define SQLITE_DBCONFIG_ENABLE_VIEW           1015 /* int int* */
 #define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT    1016 /* int int* */
 #define SQLITE_DBCONFIG_TRUSTED_SCHEMA        1017 /* int int* */
-#define SQLITE_DBCONFIG_MAX                   1017 /* Largest DBCONFIG */
+#define SQLITE_DBCONFIG_STMT_SCANSTATUS       1018 /* int int* */
+#define SQLITE_DBCONFIG_REVERSE_SCANORDER     1019 /* int int* */
+#define SQLITE_DBCONFIG_MAX                   1019 /* Largest DBCONFIG */
 
 /*
 ** CAPI3REF: Enable Or Disable Extended Result Codes
@@ -6201,6 +6255,13 @@ SQLITE_API void sqlite3_activate_cerod(
 ** of the default VFS is not implemented correctly, or not implemented at
 ** all, then the behavior of sqlite3_sleep() may deviate from the description
 ** in the previous paragraphs.
+**
+** If a negative argument is passed to sqlite3_sleep() the results vary by
+** VFS and operating system.  Some system treat a negative argument as an
+** instruction to sleep forever.  Others understand it to mean do not sleep
+** at all. ^In SQLite version 3.42.0 and later, a negative
+** argument passed into sqlite3_sleep() is changed to zero before it is relayed
+** down into the xSleep method of the VFS.
 */
 SQLITE_API int sqlite3_sleep(int);
 
@@ -7828,9 +7889,9 @@ SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*);
 ** is undefined if the mutex is not currently entered by the
 ** calling thread or is not currently allocated.
 **
-** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or
-** sqlite3_mutex_leave() is a NULL pointer, then all three routines
-** behave as no-ops.
+** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(),
+** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer,
+** then any of the four routines behaves as a no-op.
 **
 ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()].
 */
@@ -9564,18 +9625,28 @@ SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...);
 ** [[SQLITE_VTAB_INNOCUOUS]]<dt>SQLITE_VTAB_INNOCUOUS</dt>
 ** <dd>Calls of the form
 ** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the
-** the [xConnect] or [xCreate] methods of a [virtual table] implmentation
+** the [xConnect] or [xCreate] methods of a [virtual table] implementation
 ** identify that virtual table as being safe to use from within triggers
 ** and views.  Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the
 ** virtual table can do no serious harm even if it is controlled by a
 ** malicious hacker.  Developers should avoid setting the SQLITE_VTAB_INNOCUOUS
 ** flag unless absolutely necessary.
 ** </dd>
+**
+** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]<dt>SQLITE_VTAB_USES_ALL_SCHEMAS</dt>
+** <dd>Calls of the form
+** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the
+** the [xConnect] or [xCreate] methods of a [virtual table] implementation
+** instruct the query planner to begin at least a read transaction on
+** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the
+** virtual table is used.
+** </dd>
 ** </dl>
 */
 #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1
 #define SQLITE_VTAB_INNOCUOUS          2
 #define SQLITE_VTAB_DIRECTONLY         3
+#define SQLITE_VTAB_USES_ALL_SCHEMAS   4
 
 /*
 ** CAPI3REF: Determine The Virtual Table Conflict Policy
@@ -10750,16 +10821,20 @@ SQLITE_API int sqlite3session_create(
 SQLITE_API void sqlite3session_delete(sqlite3_session *pSession);
 
 /*
-** CAPIREF: Conigure a Session Object
+** CAPI3REF: Configure a Session Object
 ** METHOD: sqlite3_session
 **
 ** This method is used to configure a session object after it has been
-** created. At present the only valid value for the second parameter is
-** [SQLITE_SESSION_OBJCONFIG_SIZE].
+** created. At present the only valid values for the second parameter are
+** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID].
 **
-** Arguments for sqlite3session_object_config()
+*/
+SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg);
+
+/*
+** CAPI3REF: Options for sqlite3session_object_config
 **
-** The following values may passed as the the 4th parameter to
+** The following values may passed as the the 2nd parameter to
 ** sqlite3session_object_config().
 **
 ** <dt>SQLITE_SESSION_OBJCONFIG_SIZE <dd>
@@ -10775,12 +10850,21 @@ SQLITE_API void sqlite3session_delete(sqlite3_session *pSession);
 **
 **   It is an error (SQLITE_MISUSE) to attempt to modify this setting after
 **   the first table has been attached to the session object.
+**
+** <dt>SQLITE_SESSION_OBJCONFIG_ROWID <dd>
+**   This option is used to set, clear or query the flag that enables
+**   collection of data for tables with no explicit PRIMARY KEY.
+**
+**   Normally, tables with no explicit PRIMARY KEY are simply ignored
+**   by the sessions module. However, if this flag is set, it behaves
+**   as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted
+**   as their leftmost columns.
+**
+**   It is an error (SQLITE_MISUSE) to attempt to modify this setting after
+**   the first table has been attached to the session object.
 */
-SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg);
-
-/*
-*/
-#define SQLITE_SESSION_OBJCONFIG_SIZE 1
+#define SQLITE_SESSION_OBJCONFIG_SIZE  1
+#define SQLITE_SESSION_OBJCONFIG_ROWID 2
 
 /*
 ** CAPI3REF: Enable Or Disable A Session Object
@@ -11913,9 +11997,23 @@ SQLITE_API int sqlite3changeset_apply_v2(
 **   Invert the changeset before applying it. This is equivalent to inverting
 **   a changeset using sqlite3changeset_invert() before applying it. It is
 **   an error to specify this flag with a patchset.
+**
+** <dt>SQLITE_CHANGESETAPPLY_IGNORENOOP <dd>
+**   Do not invoke the conflict handler callback for any changes that
+**   would not actually modify the database even if they were applied.
+**   Specifically, this means that the conflict handler is not invoked
+**   for:
+**    <ul>
+**    <li>a delete change if the row being deleted cannot be found,
+**    <li>an update change if the modified fields are already set to
+**        their new values in the conflicting row, or
+**    <li>an insert change if all fields of the conflicting row match
+**        the row being inserted.
+**    </ul>
 */
 #define SQLITE_CHANGESETAPPLY_NOSAVEPOINT   0x0001
 #define SQLITE_CHANGESETAPPLY_INVERT        0x0002
+#define SQLITE_CHANGESETAPPLY_IGNORENOOP    0x0004
 
 /*
 ** CAPI3REF: Constants Passed To The Conflict Handler
diff --git a/database/sqlite/sqlite3recover.c b/database/sqlite/sqlite3recover.c
new file mode 100644
index 000000000..3dae0b7a9
--- /dev/null
+++ b/database/sqlite/sqlite3recover.c
@@ -0,0 +1,2872 @@
+/*
+** 2022-08-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+*/
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wsign-compare"
+#include "sqlite3recover.h"
+#include <assert.h>
+#include <string.h>
+
+#ifndef SQLITE_OMIT_VIRTUALTABLE
+
+/*
+** Declaration for public API function in file dbdata.c. This may be called
+** with NULL as the final two arguments to register the sqlite_dbptr and
+** sqlite_dbdata virtual tables with a database handle.
+*/
+#ifdef _WIN32
+__declspec(dllexport)
+#endif
+int sqlite3_dbdata_init(sqlite3*, char**, const sqlite3_api_routines*);
+
+typedef unsigned int u32;
+typedef unsigned char u8;
+typedef sqlite3_int64 i64;
+
+typedef struct RecoverTable RecoverTable;
+typedef struct RecoverColumn RecoverColumn;
+
+/*
+** When recovering rows of data that can be associated with table
+** definitions recovered from the sqlite_schema table, each table is
+** represented by an instance of the following object.
+**
+** iRoot:
+**   The root page in the original database. Not necessarily (and usually
+**   not) the same in the recovered database.
+**
+** zTab:
+**   Name of the table.
+**
+** nCol/aCol[]:
+**   aCol[] is an array of nCol columns. In the order in which they appear 
+**   in the table.
+**
+** bIntkey:
+**   Set to true for intkey tables, false for WITHOUT ROWID.
+**
+** iRowidBind:
+**   Each column in the aCol[] array has associated with it the index of
+**   the bind parameter its values will be bound to in the INSERT statement
+**   used to construct the output database. If the table does has a rowid
+**   but not an INTEGER PRIMARY KEY column, then iRowidBind contains the
+**   index of the bind paramater to which the rowid value should be bound.
+**   Otherwise, it contains -1. If the table does contain an INTEGER PRIMARY 
+**   KEY column, then the rowid value should be bound to the index associated
+**   with the column.
+**
+** pNext:
+**   All RecoverTable objects used by the recovery operation are allocated
+**   and populated as part of creating the recovered database schema in
+**   the output database, before any non-schema data are recovered. They
+**   are then stored in a singly-linked list linked by this variable beginning
+**   at sqlite3_recover.pTblList.
+*/
+struct RecoverTable {
+  u32 iRoot;                      /* Root page in original database */
+  char *zTab;                     /* Name of table */
+  int nCol;                       /* Number of columns in table */
+  RecoverColumn *aCol;            /* Array of columns */
+  int bIntkey;                    /* True for intkey, false for without rowid */
+  int iRowidBind;                 /* If >0, bind rowid to INSERT here */
+  RecoverTable *pNext;
+};
+
+/*
+** Each database column is represented by an instance of the following object
+** stored in the RecoverTable.aCol[] array of the associated table.
+**
+** iField:
+**   The index of the associated field within database records. Or -1 if
+**   there is no associated field (e.g. for virtual generated columns).
+**
+** iBind:
+**   The bind index of the INSERT statement to bind this columns values
+**   to. Or 0 if there is no such index (iff (iField<0)).
+**
+** bIPK:
+**   True if this is the INTEGER PRIMARY KEY column.
+**
+** zCol:
+**   Name of column.
+**
+** eHidden:
+**   A RECOVER_EHIDDEN_* constant value (see below for interpretation of each).
+*/
+struct RecoverColumn {
+  int iField;                     /* Field in record on disk */
+  int iBind;                      /* Binding to use in INSERT */
+  int bIPK;                       /* True for IPK column */
+  char *zCol;
+  int eHidden;
+};
+
+#define RECOVER_EHIDDEN_NONE    0      /* Normal database column */
+#define RECOVER_EHIDDEN_HIDDEN  1      /* Column is __HIDDEN__ */
+#define RECOVER_EHIDDEN_VIRTUAL 2      /* Virtual generated column */
+#define RECOVER_EHIDDEN_STORED  3      /* Stored generated column */
+
+/*
+** Bitmap object used to track pages in the input database. Allocated
+** and manipulated only by the following functions:
+**
+**     recoverBitmapAlloc()
+**     recoverBitmapFree()
+**     recoverBitmapSet()
+**     recoverBitmapQuery()
+**
+** nPg:
+**   Largest page number that may be stored in the bitmap. The range
+**   of valid keys is 1 to nPg, inclusive.
+**
+** aElem[]:
+**   Array large enough to contain a bit for each key. For key value
+**   iKey, the associated bit is the bit (iKey%32) of aElem[iKey/32].
+**   In other words, the following is true if bit iKey is set, or 
+**   false if it is clear:
+**
+**       (aElem[iKey/32] & (1 << (iKey%32))) ? 1 : 0
+*/
+typedef struct RecoverBitmap RecoverBitmap;
+struct RecoverBitmap {
+  i64 nPg;                        /* Size of bitmap */
+  u32 aElem[1];                   /* Array of 32-bit bitmasks */
+};
+
+/*
+** State variables (part of the sqlite3_recover structure) used while
+** recovering data for tables identified in the recovered schema (state
+** RECOVER_STATE_WRITING).
+*/
+typedef struct RecoverStateW1 RecoverStateW1;
+struct RecoverStateW1 {
+  sqlite3_stmt *pTbls;
+  sqlite3_stmt *pSel;
+  sqlite3_stmt *pInsert;
+  int nInsert;
+
+  RecoverTable *pTab;             /* Table currently being written */
+  int nMax;                       /* Max column count in any schema table */
+  sqlite3_value **apVal;          /* Array of nMax values */
+  int nVal;                       /* Number of valid entries in apVal[] */
+  int bHaveRowid;
+  i64 iRowid;
+  i64 iPrevPage;
+  int iPrevCell;
+};
+
+/*
+** State variables (part of the sqlite3_recover structure) used while
+** recovering data destined for the lost and found table (states
+** RECOVER_STATE_LOSTANDFOUND[123]).
+*/
+typedef struct RecoverStateLAF RecoverStateLAF;
+struct RecoverStateLAF {
+  RecoverBitmap *pUsed;
+  i64 nPg;                        /* Size of db in pages */
+  sqlite3_stmt *pAllAndParent;
+  sqlite3_stmt *pMapInsert;
+  sqlite3_stmt *pMaxField;
+  sqlite3_stmt *pUsedPages;
+  sqlite3_stmt *pFindRoot;
+  sqlite3_stmt *pInsert;          /* INSERT INTO lost_and_found ... */
+  sqlite3_stmt *pAllPage;
+  sqlite3_stmt *pPageData;
+  sqlite3_value **apVal;
+  int nMaxField;
+};
+
+/*
+** Main recover handle structure.
+*/
+struct sqlite3_recover {
+  /* Copies of sqlite3_recover_init[_sql]() parameters */
+  sqlite3 *dbIn;                  /* Input database */
+  char *zDb;                      /* Name of input db ("main" etc.) */
+  char *zUri;                     /* URI for output database */
+  void *pSqlCtx;                  /* SQL callback context */
+  int (*xSql)(void*,const char*); /* Pointer to SQL callback function */
+
+  /* Values configured by sqlite3_recover_config() */
+  char *zStateDb;                 /* State database to use (or NULL) */
+  char *zLostAndFound;            /* Name of lost-and-found table (or NULL) */
+  int bFreelistCorrupt;           /* SQLITE_RECOVER_FREELIST_CORRUPT setting */
+  int bRecoverRowid;              /* SQLITE_RECOVER_ROWIDS setting */
+  int bSlowIndexes;               /* SQLITE_RECOVER_SLOWINDEXES setting */
+
+  int pgsz;
+  int detected_pgsz;
+  int nReserve;
+  u8 *pPage1Disk;
+  u8 *pPage1Cache;
+
+  /* Error code and error message */
+  int errCode;                    /* For sqlite3_recover_errcode() */
+  char *zErrMsg;                  /* For sqlite3_recover_errmsg() */
+
+  int eState;
+  int bCloseTransaction;
+
+  /* Variables used with eState==RECOVER_STATE_WRITING */
+  RecoverStateW1 w1;
+
+  /* Variables used with states RECOVER_STATE_LOSTANDFOUND[123] */
+  RecoverStateLAF laf;
+
+  /* Fields used within sqlite3_recover_run() */
+  sqlite3 *dbOut;                 /* Output database */
+  sqlite3_stmt *pGetPage;         /* SELECT against input db sqlite_dbdata */
+  RecoverTable *pTblList;         /* List of tables recovered from schema */
+};
+
+/*
+** The various states in which an sqlite3_recover object may exist:
+**
+**   RECOVER_STATE_INIT:
+**    The object is initially created in this state. sqlite3_recover_step()
+**    has yet to be called. This is the only state in which it is permitted
+**    to call sqlite3_recover_config().
+**
+**   RECOVER_STATE_WRITING:
+**
+**   RECOVER_STATE_LOSTANDFOUND1:
+**    State to populate the bitmap of pages used by other tables or the
+**    database freelist.
+**
+**   RECOVER_STATE_LOSTANDFOUND2:
+**    Populate the recovery.map table - used to figure out a "root" page
+**    for each lost page from in the database from which records are
+**    extracted.
+**
+**   RECOVER_STATE_LOSTANDFOUND3:
+**    Populate the lost-and-found table itself.
+*/
+#define RECOVER_STATE_INIT           0
+#define RECOVER_STATE_WRITING        1
+#define RECOVER_STATE_LOSTANDFOUND1  2
+#define RECOVER_STATE_LOSTANDFOUND2  3
+#define RECOVER_STATE_LOSTANDFOUND3  4
+#define RECOVER_STATE_SCHEMA2        5
+#define RECOVER_STATE_DONE           6
+
+
+/*
+** Global variables used by this extension.
+*/
+typedef struct RecoverGlobal RecoverGlobal;
+struct RecoverGlobal {
+  const sqlite3_io_methods *pMethods;
+  sqlite3_recover *p;
+};
+static RecoverGlobal recover_g;
+
+/*
+** Use this static SQLite mutex to protect the globals during the
+** first call to sqlite3_recover_step().
+*/ 
+#define RECOVER_MUTEX_ID SQLITE_MUTEX_STATIC_APP2
+
+
+/* 
+** Default value for SQLITE_RECOVER_ROWIDS (sqlite3_recover.bRecoverRowid).
+*/
+#define RECOVER_ROWID_DEFAULT 1
+
+/*
+** Mutex handling:
+**
+**    recoverEnterMutex()       -   Enter the recovery mutex
+**    recoverLeaveMutex()       -   Leave the recovery mutex
+**    recoverAssertMutexHeld()  -   Assert that the recovery mutex is held
+*/
+#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0
+# define recoverEnterMutex()
+# define recoverLeaveMutex()
+#else
+static void recoverEnterMutex(void){
+  sqlite3_mutex_enter(sqlite3_mutex_alloc(RECOVER_MUTEX_ID));
+}
+static void recoverLeaveMutex(void){
+  sqlite3_mutex_leave(sqlite3_mutex_alloc(RECOVER_MUTEX_ID));
+}
+#endif
+#if SQLITE_THREADSAFE+0>=1 && defined(SQLITE_DEBUG)
+static void recoverAssertMutexHeld(void){
+  assert( sqlite3_mutex_held(sqlite3_mutex_alloc(RECOVER_MUTEX_ID)) );
+}
+#else
+# define recoverAssertMutexHeld()
+#endif
+
+
+/*
+** Like strlen(). But handles NULL pointer arguments.
+*/
+static int recoverStrlen(const char *zStr){
+  if( zStr==0 ) return 0;
+  return (int)(strlen(zStr)&0x7fffffff);
+}
+
+/*
+** This function is a no-op if the recover handle passed as the first 
+** argument already contains an error (if p->errCode!=SQLITE_OK). 
+**
+** Otherwise, an attempt is made to allocate, zero and return a buffer nByte
+** bytes in size. If successful, a pointer to the new buffer is returned. Or,
+** if an OOM error occurs, NULL is returned and the handle error code
+** (p->errCode) set to SQLITE_NOMEM.
+*/
+static void *recoverMalloc(sqlite3_recover *p, i64 nByte){
+  void *pRet = 0;
+  assert( nByte>0 );
+  if( p->errCode==SQLITE_OK ){
+    pRet = sqlite3_malloc64(nByte);
+    if( pRet ){
+      memset(pRet, 0, nByte);
+    }else{
+      p->errCode = SQLITE_NOMEM;
+    }
+  }
+  return pRet;
+}
+
+/*
+** Set the error code and error message for the recover handle passed as
+** the first argument. The error code is set to the value of parameter
+** errCode.
+**
+** Parameter zFmt must be a printf() style formatting string. The handle 
+** error message is set to the result of using any trailing arguments for 
+** parameter substitutions in the formatting string.
+**
+** For example:
+**
+**   recoverError(p, SQLITE_ERROR, "no such table: %s", zTablename);
+*/
+static int recoverError(
+  sqlite3_recover *p, 
+  int errCode, 
+  const char *zFmt, ...
+){
+  char *z = 0;
+  va_list ap;
+  va_start(ap, zFmt);
+  if( zFmt ){
+    z = sqlite3_vmprintf(zFmt, ap);
+    va_end(ap);
+  }
+  sqlite3_free(p->zErrMsg);
+  p->zErrMsg = z;
+  p->errCode = errCode;
+  return errCode;
+}
+
+
+/*
+** This function is a no-op if p->errCode is initially other than SQLITE_OK.
+** In this case it returns NULL.
+**
+** Otherwise, an attempt is made to allocate and return a bitmap object
+** large enough to store a bit for all page numbers between 1 and nPg,
+** inclusive. The bitmap is initially zeroed.
+*/
+static RecoverBitmap *recoverBitmapAlloc(sqlite3_recover *p, i64 nPg){
+  int nElem = (nPg+1+31) / 32;
+  int nByte = sizeof(RecoverBitmap) + nElem*sizeof(u32);
+  RecoverBitmap *pRet = (RecoverBitmap*)recoverMalloc(p, nByte);
+
+  if( pRet ){
+    pRet->nPg = nPg;
+  }
+  return pRet;
+}
+
+/*
+** Free a bitmap object allocated by recoverBitmapAlloc().
+*/
+static void recoverBitmapFree(RecoverBitmap *pMap){
+  sqlite3_free(pMap);
+}
+
+/*
+** Set the bit associated with page iPg in bitvec pMap.
+*/
+static void recoverBitmapSet(RecoverBitmap *pMap, i64 iPg){
+  if( iPg<=pMap->nPg ){
+    int iElem = (iPg / 32);
+    int iBit = (iPg % 32);
+    pMap->aElem[iElem] |= (((u32)1) << iBit);
+  }
+}
+
+/*
+** Query bitmap object pMap for the state of the bit associated with page
+** iPg. Return 1 if it is set, or 0 otherwise.
+*/
+static int recoverBitmapQuery(RecoverBitmap *pMap, i64 iPg){
+  int ret = 1;
+  if( iPg<=pMap->nPg && iPg>0 ){
+    int iElem = (iPg / 32);
+    int iBit = (iPg % 32);
+    ret = (pMap->aElem[iElem] & (((u32)1) << iBit)) ? 1 : 0;
+  }
+  return ret;
+}
+
+/*
+** Set the recover handle error to the error code and message returned by
+** calling sqlite3_errcode() and sqlite3_errmsg(), respectively, on database
+** handle db.
+*/
+static int recoverDbError(sqlite3_recover *p, sqlite3 *db){
+  return recoverError(p, sqlite3_errcode(db), "%s", sqlite3_errmsg(db));
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). 
+**
+** Otherwise, it attempts to prepare the SQL statement in zSql against
+** database handle db. If successful, the statement handle is returned.
+** Or, if an error occurs, NULL is returned and an error left in the
+** recover handle.
+*/
+static sqlite3_stmt *recoverPrepare(
+  sqlite3_recover *p,
+  sqlite3 *db, 
+  const char *zSql
+){
+  sqlite3_stmt *pStmt = 0;
+  if( p->errCode==SQLITE_OK ){
+    if( sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) ){
+      recoverDbError(p, db);
+    }
+  }
+  return pStmt;
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). 
+**
+** Otherwise, argument zFmt is used as a printf() style format string,
+** along with any trailing arguments, to create an SQL statement. This
+** SQL statement is prepared against database handle db and, if successful,
+** the statment handle returned. Or, if an error occurs - either during
+** the printf() formatting or when preparing the resulting SQL - an
+** error code and message are left in the recover handle.
+*/
+static sqlite3_stmt *recoverPreparePrintf(
+  sqlite3_recover *p,
+  sqlite3 *db, 
+  const char *zFmt, ...
+){
+  sqlite3_stmt *pStmt = 0;
+  if( p->errCode==SQLITE_OK ){
+    va_list ap;
+    char *z;
+    va_start(ap, zFmt);
+    z = sqlite3_vmprintf(zFmt, ap);
+    va_end(ap);
+    if( z==0 ){
+      p->errCode = SQLITE_NOMEM;
+    }else{
+      pStmt = recoverPrepare(p, db, z);
+      sqlite3_free(z);
+    }
+  }
+  return pStmt;
+}
+
+/*
+** Reset SQLite statement handle pStmt. If the call to sqlite3_reset() 
+** indicates that an error occurred, and there is not already an error
+** in the recover handle passed as the first argument, set the error
+** code and error message appropriately.
+**
+** This function returns a copy of the statement handle pointer passed
+** as the second argument.
+*/
+static sqlite3_stmt *recoverReset(sqlite3_recover *p, sqlite3_stmt *pStmt){
+  int rc = sqlite3_reset(pStmt);
+  if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT && p->errCode==SQLITE_OK ){
+    recoverDbError(p, sqlite3_db_handle(pStmt));
+  }
+  return pStmt;
+}
+
+/*
+** Finalize SQLite statement handle pStmt. If the call to sqlite3_reset() 
+** indicates that an error occurred, and there is not already an error
+** in the recover handle passed as the first argument, set the error
+** code and error message appropriately.
+*/
+static void recoverFinalize(sqlite3_recover *p, sqlite3_stmt *pStmt){
+  sqlite3 *db = sqlite3_db_handle(pStmt);
+  int rc = sqlite3_finalize(pStmt);
+  if( rc!=SQLITE_OK && p->errCode==SQLITE_OK ){
+    recoverDbError(p, db);
+  }
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). A copy of p->errCode is returned in this 
+** case.
+**
+** Otherwise, execute SQL script zSql. If successful, return SQLITE_OK.
+** Or, if an error occurs, leave an error code and message in the recover
+** handle and return a copy of the error code.
+*/
+static int recoverExec(sqlite3_recover *p, sqlite3 *db, const char *zSql){
+  if( p->errCode==SQLITE_OK ){
+    int rc = sqlite3_exec(db, zSql, 0, 0, 0);
+    if( rc ){
+      recoverDbError(p, db);
+    }
+  }
+  return p->errCode;
+}
+
+/*
+** Bind the value pVal to parameter iBind of statement pStmt. Leave an
+** error in the recover handle passed as the first argument if an error
+** (e.g. an OOM) occurs.
+*/
+static void recoverBindValue(
+  sqlite3_recover *p, 
+  sqlite3_stmt *pStmt, 
+  int iBind, 
+  sqlite3_value *pVal
+){
+  if( p->errCode==SQLITE_OK ){
+    int rc = sqlite3_bind_value(pStmt, iBind, pVal);
+    if( rc ) recoverError(p, rc, 0);
+  }
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). NULL is returned in this case.
+**
+** Otherwise, an attempt is made to interpret zFmt as a printf() style
+** formatting string and the result of using the trailing arguments for
+** parameter substitution with it written into a buffer obtained from
+** sqlite3_malloc(). If successful, a pointer to the buffer is returned.
+** It is the responsibility of the caller to eventually free the buffer
+** using sqlite3_free().
+**
+** Or, if an error occurs, an error code and message is left in the recover
+** handle and NULL returned.
+*/
+static char *recoverMPrintf(sqlite3_recover *p, const char *zFmt, ...){
+  va_list ap;
+  char *z;
+  va_start(ap, zFmt);
+  z = sqlite3_vmprintf(zFmt, ap);
+  va_end(ap);
+  if( p->errCode==SQLITE_OK ){
+    if( z==0 ) p->errCode = SQLITE_NOMEM;
+  }else{
+    sqlite3_free(z);
+    z = 0;
+  }
+  return z;
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). Zero is returned in this case.
+**
+** Otherwise, execute "PRAGMA page_count" against the input database. If
+** successful, return the integer result. Or, if an error occurs, leave an
+** error code and error message in the sqlite3_recover handle and return
+** zero.
+*/
+static i64 recoverPageCount(sqlite3_recover *p){
+  i64 nPg = 0;
+  if( p->errCode==SQLITE_OK ){
+    sqlite3_stmt *pStmt = 0;
+    pStmt = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.page_count", p->zDb);
+    if( pStmt ){
+      sqlite3_step(pStmt);
+      nPg = sqlite3_column_int64(pStmt, 0);
+    }
+    recoverFinalize(p, pStmt);
+  }
+  return nPg;
+}
+
+/*
+** Implementation of SQL scalar function "read_i32". The first argument to 
+** this function must be a blob. The second a non-negative integer. This 
+** function reads and returns a 32-bit big-endian integer from byte
+** offset (4*<arg2>) of the blob.
+**
+**     SELECT read_i32(<blob>, <idx>)
+*/
+static void recoverReadI32(
+  sqlite3_context *context, 
+  int argc, 
+  sqlite3_value **argv
+){
+  const unsigned char *pBlob;
+  int nBlob;
+  int iInt;
+
+  assert( argc==2 );
+  nBlob = sqlite3_value_bytes(argv[0]);
+  pBlob = (const unsigned char*)sqlite3_value_blob(argv[0]);
+  iInt = sqlite3_value_int(argv[1]) & 0xFFFF;
+
+  if( (iInt+1)*4<=nBlob ){
+    const unsigned char *a = &pBlob[iInt*4];
+    i64 iVal = ((i64)a[0]<<24)
+             + ((i64)a[1]<<16)
+             + ((i64)a[2]<< 8)
+             + ((i64)a[3]<< 0);
+    sqlite3_result_int64(context, iVal);
+  }
+}
+
+/*
+** Implementation of SQL scalar function "page_is_used". This function
+** is used as part of the procedure for locating orphan rows for the
+** lost-and-found table, and it depends on those routines having populated
+** the sqlite3_recover.laf.pUsed variable.
+**
+** The only argument to this function is a page-number. It returns true 
+** if the page has already been used somehow during data recovery, or false
+** otherwise.
+**
+**     SELECT page_is_used(<pgno>);
+*/
+static void recoverPageIsUsed(
+  sqlite3_context *pCtx,
+  int nArg,
+  sqlite3_value **apArg
+){
+  sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
+  i64 pgno = sqlite3_value_int64(apArg[0]);
+  assert( nArg==1 );
+  sqlite3_result_int(pCtx, recoverBitmapQuery(p->laf.pUsed, pgno));
+}
+
+/*
+** The implementation of a user-defined SQL function invoked by the 
+** sqlite_dbdata and sqlite_dbptr virtual table modules to access pages
+** of the database being recovered.
+**
+** This function always takes a single integer argument. If the argument
+** is zero, then the value returned is the number of pages in the db being
+** recovered. If the argument is greater than zero, it is a page number. 
+** The value returned in this case is an SQL blob containing the data for 
+** the identified page of the db being recovered. e.g.
+**
+**     SELECT getpage(0);       -- return number of pages in db
+**     SELECT getpage(4);       -- return page 4 of db as a blob of data 
+*/
+static void recoverGetPage(
+  sqlite3_context *pCtx,
+  int nArg,
+  sqlite3_value **apArg
+){
+  sqlite3_recover *p = (sqlite3_recover*)sqlite3_user_data(pCtx);
+  i64 pgno = sqlite3_value_int64(apArg[0]);
+  sqlite3_stmt *pStmt = 0;
+
+  assert( nArg==1 );
+  if( pgno==0 ){
+    i64 nPg = recoverPageCount(p);
+    sqlite3_result_int64(pCtx, nPg);
+    return;
+  }else{
+    if( p->pGetPage==0 ){
+      pStmt = p->pGetPage = recoverPreparePrintf(
+          p, p->dbIn, "SELECT data FROM sqlite_dbpage(%Q) WHERE pgno=?", p->zDb
+      );
+    }else if( p->errCode==SQLITE_OK ){
+      pStmt = p->pGetPage;
+    }
+
+    if( pStmt ){
+      sqlite3_bind_int64(pStmt, 1, pgno);
+      if( SQLITE_ROW==sqlite3_step(pStmt) ){
+        const u8 *aPg;
+        int nPg;
+        assert( p->errCode==SQLITE_OK );
+        aPg = sqlite3_column_blob(pStmt, 0);
+        nPg = sqlite3_column_bytes(pStmt, 0);
+        if( pgno==1 && nPg==p->pgsz && 0==memcmp(p->pPage1Cache, aPg, nPg) ){
+          aPg = p->pPage1Disk;
+        }
+        sqlite3_result_blob(pCtx, aPg, nPg-p->nReserve, SQLITE_TRANSIENT);
+      }
+      recoverReset(p, pStmt);
+    }
+  }
+
+  if( p->errCode ){
+    if( p->zErrMsg ) sqlite3_result_error(pCtx, p->zErrMsg, -1);
+    sqlite3_result_error_code(pCtx, p->errCode);
+  }
+}
+
+/*
+** Find a string that is not found anywhere in z[].  Return a pointer
+** to that string.
+**
+** Try to use zA and zB first.  If both of those are already found in z[]
+** then make up some string and store it in the buffer zBuf.
+*/
+static const char *recoverUnusedString(
+  const char *z,                    /* Result must not appear anywhere in z */
+  const char *zA, const char *zB,   /* Try these first */
+  char *zBuf                        /* Space to store a generated string */
+){
+  unsigned i = 0;
+  if( strstr(z, zA)==0 ) return zA;
+  if( strstr(z, zB)==0 ) return zB;
+  do{
+    sqlite3_snprintf(20,zBuf,"(%s%u)", zA, i++);
+  }while( strstr(z,zBuf)!=0 );
+  return zBuf;
+}
+
+/*
+** Implementation of scalar SQL function "escape_crnl".  The argument passed to
+** this function is the output of built-in function quote(). If the first
+** character of the input is "'", indicating that the value passed to quote()
+** was a text value, then this function searches the input for "\n" and "\r"
+** characters and adds a wrapper similar to the following:
+**
+**   replace(replace(<input>, '\n', char(10), '\r', char(13));
+**
+** Or, if the first character of the input is not "'", then a copy of the input
+** is returned.
+*/
+static void recoverEscapeCrnl(
+  sqlite3_context *context, 
+  int argc, 
+  sqlite3_value **argv
+){
+  const char *zText = (const char*)sqlite3_value_text(argv[0]);
+  (void)argc;
+  if( zText && zText[0]=='\'' ){
+    int nText = sqlite3_value_bytes(argv[0]);
+    int i;
+    char zBuf1[20];
+    char zBuf2[20];
+    const char *zNL = 0;
+    const char *zCR = 0;
+    int nCR = 0;
+    int nNL = 0;
+
+    for(i=0; zText[i]; i++){
+      if( zNL==0 && zText[i]=='\n' ){
+        zNL = recoverUnusedString(zText, "\\n", "\\012", zBuf1);
+        nNL = (int)strlen(zNL);
+      }
+      if( zCR==0 && zText[i]=='\r' ){
+        zCR = recoverUnusedString(zText, "\\r", "\\015", zBuf2);
+        nCR = (int)strlen(zCR);
+      }
+    }
+
+    if( zNL || zCR ){
+      int iOut = 0;
+      i64 nMax = (nNL > nCR) ? nNL : nCR;
+      i64 nAlloc = nMax * nText + (nMax+64)*2;
+      char *zOut = (char*)sqlite3_malloc64(nAlloc);
+      if( zOut==0 ){
+        sqlite3_result_error_nomem(context);
+        return;
+      }
+
+      if( zNL && zCR ){
+        memcpy(&zOut[iOut], "replace(replace(", 16);
+        iOut += 16;
+      }else{
+        memcpy(&zOut[iOut], "replace(", 8);
+        iOut += 8;
+      }
+      for(i=0; zText[i]; i++){
+        if( zText[i]=='\n' ){
+          memcpy(&zOut[iOut], zNL, nNL);
+          iOut += nNL;
+        }else if( zText[i]=='\r' ){
+          memcpy(&zOut[iOut], zCR, nCR);
+          iOut += nCR;
+        }else{
+          zOut[iOut] = zText[i];
+          iOut++;
+        }
+      }
+
+      if( zNL ){
+        memcpy(&zOut[iOut], ",'", 2); iOut += 2;
+        memcpy(&zOut[iOut], zNL, nNL); iOut += nNL;
+        memcpy(&zOut[iOut], "', char(10))", 12); iOut += 12;
+      }
+      if( zCR ){
+        memcpy(&zOut[iOut], ",'", 2); iOut += 2;
+        memcpy(&zOut[iOut], zCR, nCR); iOut += nCR;
+        memcpy(&zOut[iOut], "', char(13))", 12); iOut += 12;
+      }
+
+      sqlite3_result_text(context, zOut, iOut, SQLITE_TRANSIENT);
+      sqlite3_free(zOut);
+      return;
+    }
+  }
+
+  sqlite3_result_value(context, argv[0]);
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in
+** this case. 
+**
+** Otherwise, attempt to populate temporary table "recovery.schema" with the
+** parts of the database schema that can be extracted from the input database.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
+** and error message are left in the recover handle and a copy of the
+** error code returned. It is not considered an error if part of all of
+** the database schema cannot be recovered due to corruption.
+*/
+static int recoverCacheSchema(sqlite3_recover *p){
+  return recoverExec(p, p->dbOut,
+    "WITH RECURSIVE pages(p) AS ("
+    "  SELECT 1"
+    "    UNION"
+    "  SELECT child FROM sqlite_dbptr('getpage()'), pages WHERE pgno=p"
+    ")"
+    "INSERT INTO recovery.schema SELECT"
+    "  max(CASE WHEN field=0 THEN value ELSE NULL END),"
+    "  max(CASE WHEN field=1 THEN value ELSE NULL END),"
+    "  max(CASE WHEN field=2 THEN value ELSE NULL END),"
+    "  max(CASE WHEN field=3 THEN value ELSE NULL END),"
+    "  max(CASE WHEN field=4 THEN value ELSE NULL END)"
+    "FROM sqlite_dbdata('getpage()') WHERE pgno IN ("
+    "  SELECT p FROM pages"
+    ") GROUP BY pgno, cell"
+  );
+}
+
+/*
+** If this recover handle is not in SQL callback mode (i.e. was not created 
+** using sqlite3_recover_init_sql()) of if an error has already occurred, 
+** this function is a no-op. Otherwise, issue a callback with SQL statement
+** zSql as the parameter. 
+**
+** If the callback returns non-zero, set the recover handle error code to
+** the value returned (so that the caller will abandon processing).
+*/
+static void recoverSqlCallback(sqlite3_recover *p, const char *zSql){
+  if( p->errCode==SQLITE_OK && p->xSql ){
+    int res = p->xSql(p->pSqlCtx, zSql);
+    if( res ){
+      recoverError(p, SQLITE_ERROR, "callback returned an error - %d", res);
+    }
+  }
+}
+
+/*
+** Transfer the following settings from the input database to the output
+** database:
+**
+**   + page-size,
+**   + auto-vacuum settings,
+**   + database encoding,
+**   + user-version (PRAGMA user_version), and
+**   + application-id (PRAGMA application_id), and
+*/
+static void recoverTransferSettings(sqlite3_recover *p){
+  const char *aPragma[] = {
+    "encoding",
+    "page_size",
+    "auto_vacuum",
+    "user_version",
+    "application_id"
+  };
+  int ii;
+
+  /* Truncate the output database to 0 pages in size. This is done by 
+  ** opening a new, empty, temp db, then using the backup API to clobber 
+  ** any existing output db with a copy of it. */
+  if( p->errCode==SQLITE_OK ){
+    sqlite3 *db2 = 0;
+    int rc = sqlite3_open("", &db2);
+    if( rc!=SQLITE_OK ){
+      recoverDbError(p, db2);
+      return;
+    }
+
+    for(ii=0; ii<(int)(sizeof(aPragma)/sizeof(aPragma[0])); ii++){
+      const char *zPrag = aPragma[ii];
+      sqlite3_stmt *p1 = 0;
+      p1 = recoverPreparePrintf(p, p->dbIn, "PRAGMA %Q.%s", p->zDb, zPrag);
+      if( p->errCode==SQLITE_OK && sqlite3_step(p1)==SQLITE_ROW ){
+        const char *zArg = (const char*)sqlite3_column_text(p1, 0);
+        char *z2 = recoverMPrintf(p, "PRAGMA %s = %Q", zPrag, zArg);
+        recoverSqlCallback(p, z2);
+        recoverExec(p, db2, z2);
+        sqlite3_free(z2);
+        if( zArg==0 ){
+          recoverError(p, SQLITE_NOMEM, 0);
+        }
+      }
+      recoverFinalize(p, p1);
+    }
+    recoverExec(p, db2, "CREATE TABLE t1(a); DROP TABLE t1;");
+
+    if( p->errCode==SQLITE_OK ){
+      sqlite3 *db = p->dbOut;
+      sqlite3_backup *pBackup = sqlite3_backup_init(db, "main", db2, "main");
+      if( pBackup ){
+        sqlite3_backup_step(pBackup, -1);
+        p->errCode = sqlite3_backup_finish(pBackup);
+      }else{
+        recoverDbError(p, db);
+      }
+    }
+
+    sqlite3_close(db2);
+  }
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). A copy of the error code is returned in
+** this case. 
+**
+** Otherwise, an attempt is made to open the output database, attach
+** and create the schema of the temporary database used to store
+** intermediate data, and to register all required user functions and
+** virtual table modules with the output handle.
+**
+** If no error occurs, SQLITE_OK is returned. Otherwise, an error code
+** and error message are left in the recover handle and a copy of the
+** error code returned.
+*/
+static int recoverOpenOutput(sqlite3_recover *p){
+  struct Func {
+    const char *zName;
+    int nArg;
+    void (*xFunc)(sqlite3_context*,int,sqlite3_value **);
+  } aFunc[] = {
+    { "getpage", 1, recoverGetPage },
+    { "page_is_used", 1, recoverPageIsUsed },
+    { "read_i32", 2, recoverReadI32 },
+    { "escape_crnl", 1, recoverEscapeCrnl },
+  };
+
+  const int flags = SQLITE_OPEN_URI|SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE;
+  sqlite3 *db = 0;                /* New database handle */
+  int ii;                         /* For iterating through aFunc[] */
+
+  assert( p->dbOut==0 );
+
+  if( sqlite3_open_v2(p->zUri, &db, flags, 0) ){
+    recoverDbError(p, db);
+  }
+
+  /* Register the sqlite_dbdata and sqlite_dbptr virtual table modules.
+  ** These two are registered with the output database handle - this
+  ** module depends on the input handle supporting the sqlite_dbpage
+  ** virtual table only.  */
+  if( p->errCode==SQLITE_OK ){
+    p->errCode = sqlite3_dbdata_init(db, 0, 0);
+  }
+
+  /* Register the custom user-functions with the output handle. */
+  for(ii=0;
+      p->errCode==SQLITE_OK && ii<(int)(sizeof(aFunc)/sizeof(aFunc[0]));
+      ii++){
+    p->errCode = sqlite3_create_function(db, aFunc[ii].zName, 
+        aFunc[ii].nArg, SQLITE_UTF8, (void*)p, aFunc[ii].xFunc, 0, 0
+    );
+  }
+
+  p->dbOut = db;
+  return p->errCode;
+}
+
+/*
+** Attach the auxiliary database 'recovery' to the output database handle.
+** This temporary database is used during the recovery process and then 
+** discarded.
+*/
+static void recoverOpenRecovery(sqlite3_recover *p){
+  char *zSql = recoverMPrintf(p, "ATTACH %Q AS recovery;", p->zStateDb);
+  recoverExec(p, p->dbOut, zSql);
+  recoverExec(p, p->dbOut,
+      "PRAGMA writable_schema = 1;"
+      "CREATE TABLE recovery.map(pgno INTEGER PRIMARY KEY, parent INT);" 
+      "CREATE TABLE recovery.schema(type, name, tbl_name, rootpage, sql);"
+  );
+  sqlite3_free(zSql);
+}
+
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK).
+**
+** Otherwise, argument zName must be the name of a table that has just been
+** created in the output database. This function queries the output db
+** for the schema of said table, and creates a RecoverTable object to
+** store the schema in memory. The new RecoverTable object is linked into
+** the list at sqlite3_recover.pTblList.
+**
+** Parameter iRoot must be the root page of table zName in the INPUT 
+** database.
+*/
+static void recoverAddTable(
+  sqlite3_recover *p, 
+  const char *zName,              /* Name of table created in output db */
+  i64 iRoot                       /* Root page of same table in INPUT db */
+){
+  sqlite3_stmt *pStmt = recoverPreparePrintf(p, p->dbOut, 
+      "PRAGMA table_xinfo(%Q)", zName
+  );
+
+  if( pStmt ){
+    int iPk = -1;
+    int iBind = 1;
+    RecoverTable *pNew = 0;
+    int nCol = 0;
+    int nName = recoverStrlen(zName);
+    int nByte = 0;
+    while( sqlite3_step(pStmt)==SQLITE_ROW ){
+      nCol++;
+      nByte += (sqlite3_column_bytes(pStmt, 1)+1);
+    }
+    nByte += sizeof(RecoverTable) + nCol*sizeof(RecoverColumn) + nName+1;
+    recoverReset(p, pStmt);
+
+    pNew = recoverMalloc(p, nByte);
+    if( pNew ){
+      int i = 0;
+      int iField = 0;
+      char *csr = 0;
+      pNew->aCol = (RecoverColumn*)&pNew[1];
+      pNew->zTab = csr = (char*)&pNew->aCol[nCol];
+      pNew->nCol = nCol;
+      pNew->iRoot = iRoot;
+      memcpy(csr, zName, nName);
+      csr += nName+1;
+
+      for(i=0; sqlite3_step(pStmt)==SQLITE_ROW; i++){
+        int iPKF = sqlite3_column_int(pStmt, 5);
+        int n = sqlite3_column_bytes(pStmt, 1);
+        const char *z = (const char*)sqlite3_column_text(pStmt, 1);
+        const char *zType = (const char*)sqlite3_column_text(pStmt, 2);
+        int eHidden = sqlite3_column_int(pStmt, 6);
+
+        if( iPk==-1 && iPKF==1 && !sqlite3_stricmp("integer", zType) ) iPk = i;
+        if( iPKF>1 ) iPk = -2;
+        pNew->aCol[i].zCol = csr;
+        pNew->aCol[i].eHidden = eHidden;
+        if( eHidden==RECOVER_EHIDDEN_VIRTUAL ){
+          pNew->aCol[i].iField = -1;
+        }else{
+          pNew->aCol[i].iField = iField++;
+        }
+        if( eHidden!=RECOVER_EHIDDEN_VIRTUAL
+         && eHidden!=RECOVER_EHIDDEN_STORED
+        ){
+          pNew->aCol[i].iBind = iBind++;
+        }
+        memcpy(csr, z, n);
+        csr += (n+1);
+      }
+
+      pNew->pNext = p->pTblList;
+      p->pTblList = pNew;
+      pNew->bIntkey = 1;
+    }
+
+    recoverFinalize(p, pStmt);
+
+    pStmt = recoverPreparePrintf(p, p->dbOut, "PRAGMA index_xinfo(%Q)", zName);
+    while( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){
+      int iField = sqlite3_column_int(pStmt, 0);
+      int iCol = sqlite3_column_int(pStmt, 1);
+
+      assert( iCol<pNew->nCol );
+      pNew->aCol[iCol].iField = iField;
+
+      pNew->bIntkey = 0;
+      iPk = -2;
+    }
+    recoverFinalize(p, pStmt);
+
+    if( p->errCode==SQLITE_OK ){
+      if( iPk>=0 ){
+        pNew->aCol[iPk].bIPK = 1;
+      }else if( pNew->bIntkey ){
+        pNew->iRowidBind = iBind++;
+      }
+    }
+  }
+}
+
+/*
+** This function is called after recoverCacheSchema() has cached those parts
+** of the input database schema that could be recovered in temporary table
+** "recovery.schema". This function creates in the output database copies
+** of all parts of that schema that must be created before the tables can
+** be populated. Specifically, this means:
+**
+**     * all tables that are not VIRTUAL, and
+**     * UNIQUE indexes.
+**
+** If the recovery handle uses SQL callbacks, then callbacks containing
+** the associated "CREATE TABLE" and "CREATE INDEX" statements are made.
+**
+** Additionally, records are added to the sqlite_schema table of the
+** output database for any VIRTUAL tables. The CREATE VIRTUAL TABLE
+** records are written directly to sqlite_schema, not actually executed.
+** If the handle is in SQL callback mode, then callbacks are invoked 
+** with equivalent SQL statements.
+*/
+static int recoverWriteSchema1(sqlite3_recover *p){
+  sqlite3_stmt *pSelect = 0;
+  sqlite3_stmt *pTblname = 0;
+
+  pSelect = recoverPrepare(p, p->dbOut,
+      "WITH dbschema(rootpage, name, sql, tbl, isVirtual, isIndex) AS ("
+      "  SELECT rootpage, name, sql, "
+      "    type='table', "
+      "    sql LIKE 'create virtual%',"
+      "    (type='index' AND (sql LIKE '%unique%' OR ?1))"
+      "  FROM recovery.schema"
+      ")"
+      "SELECT rootpage, tbl, isVirtual, name, sql"
+      " FROM dbschema "
+      "  WHERE tbl OR isIndex"
+      "  ORDER BY tbl DESC, name=='sqlite_sequence' DESC"
+  );
+
+  pTblname = recoverPrepare(p, p->dbOut,
+      "SELECT name FROM sqlite_schema "
+      "WHERE type='table' ORDER BY rowid DESC LIMIT 1"
+  );
+
+  if( pSelect ){
+    sqlite3_bind_int(pSelect, 1, p->bSlowIndexes);
+    while( sqlite3_step(pSelect)==SQLITE_ROW ){
+      i64 iRoot = sqlite3_column_int64(pSelect, 0);
+      int bTable = sqlite3_column_int(pSelect, 1);
+      int bVirtual = sqlite3_column_int(pSelect, 2);
+      const char *zName = (const char*)sqlite3_column_text(pSelect, 3);
+      const char *zSql = (const char*)sqlite3_column_text(pSelect, 4);
+      char *zFree = 0;
+      int rc = SQLITE_OK;
+
+      if( bVirtual ){
+        zSql = (const char*)(zFree = recoverMPrintf(p,
+            "INSERT INTO sqlite_schema VALUES('table', %Q, %Q, 0, %Q)",
+            zName, zName, zSql
+        ));
+      }
+      rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0);
+      if( rc==SQLITE_OK ){
+        recoverSqlCallback(p, zSql);
+        if( bTable && !bVirtual ){
+          if( SQLITE_ROW==sqlite3_step(pTblname) ){
+            const char *zTbl = (const char*)sqlite3_column_text(pTblname, 0);
+            recoverAddTable(p, zTbl, iRoot);
+          }
+          recoverReset(p, pTblname);
+        }
+      }else if( rc!=SQLITE_ERROR ){
+        recoverDbError(p, p->dbOut);
+      }
+      sqlite3_free(zFree);
+    }
+  }
+  recoverFinalize(p, pSelect);
+  recoverFinalize(p, pTblname);
+
+  return p->errCode;
+}
+
+/*
+** This function is called after the output database has been populated. It
+** adds all recovered schema elements that were not created in the output
+** database by recoverWriteSchema1() - everything except for tables and
+** UNIQUE indexes. Specifically:
+**
+**     * views,
+**     * triggers,
+**     * non-UNIQUE indexes.
+**
+** If the recover handle is in SQL callback mode, then equivalent callbacks
+** are issued to create the schema elements.
+*/
+static int recoverWriteSchema2(sqlite3_recover *p){
+  sqlite3_stmt *pSelect = 0;
+
+  pSelect = recoverPrepare(p, p->dbOut,
+      p->bSlowIndexes ?
+      "SELECT rootpage, sql FROM recovery.schema "
+      "  WHERE type!='table' AND type!='index'"
+      :
+      "SELECT rootpage, sql FROM recovery.schema "
+      "  WHERE type!='table' AND (type!='index' OR sql NOT LIKE '%unique%')"
+  );
+
+  if( pSelect ){
+    while( sqlite3_step(pSelect)==SQLITE_ROW ){
+      const char *zSql = (const char*)sqlite3_column_text(pSelect, 1);
+      int rc = sqlite3_exec(p->dbOut, zSql, 0, 0, 0);
+      if( rc==SQLITE_OK ){
+        recoverSqlCallback(p, zSql);
+      }else if( rc!=SQLITE_ERROR ){
+        recoverDbError(p, p->dbOut);
+      }
+    }
+  }
+  recoverFinalize(p, pSelect);
+
+  return p->errCode;
+}
+
+/*
+** This function is a no-op if recover handle p already contains an error
+** (if p->errCode!=SQLITE_OK). In this case it returns NULL.
+**
+** Otherwise, if the recover handle is configured to create an output
+** database (was created by sqlite3_recover_init()), then this function
+** prepares and returns an SQL statement to INSERT a new record into table
+** pTab, assuming the first nField fields of a record extracted from disk
+** are valid.
+**
+** For example, if table pTab is:
+**
+**     CREATE TABLE name(a, b GENERATED ALWAYS AS (a+1) STORED, c, d, e);
+**
+** And nField is 4, then the SQL statement prepared and returned is:
+**
+**     INSERT INTO (a, c, d) VALUES (?1, ?2, ?3);
+**
+** In this case even though 4 values were extracted from the input db,
+** only 3 are written to the output, as the generated STORED column 
+** cannot be written.
+**
+** If the recover handle is in SQL callback mode, then the SQL statement
+** prepared is such that evaluating it returns a single row containing
+** a single text value - itself an SQL statement similar to the above,
+** except with SQL literals in place of the variables. For example:
+**
+**     SELECT 'INSERT INTO (a, c, d) VALUES (' 
+**          || quote(?1) || ', '
+**          || quote(?2) || ', '
+**          || quote(?3) || ')';
+**
+** In either case, it is the responsibility of the caller to eventually
+** free the statement handle using sqlite3_finalize().
+*/
+static sqlite3_stmt *recoverInsertStmt(
+  sqlite3_recover *p, 
+  RecoverTable *pTab,
+  int nField
+){
+  sqlite3_stmt *pRet = 0;
+  const char *zSep = "";
+  const char *zSqlSep = "";
+  char *zSql = 0;
+  char *zFinal = 0;
+  char *zBind = 0;
+  int ii;
+  int bSql = p->xSql ? 1 : 0;
+
+  if( nField<=0 ) return 0;
+
+  assert( nField<=pTab->nCol );
+
+  zSql = recoverMPrintf(p, "INSERT OR IGNORE INTO %Q(", pTab->zTab);
+
+  if( pTab->iRowidBind ){
+    assert( pTab->bIntkey );
+    zSql = recoverMPrintf(p, "%z_rowid_", zSql);
+    if( bSql ){
+      zBind = recoverMPrintf(p, "%zquote(?%d)", zBind, pTab->iRowidBind);
+    }else{
+      zBind = recoverMPrintf(p, "%z?%d", zBind, pTab->iRowidBind);
+    }
+    zSqlSep = "||', '||";
+    zSep = ", ";
+  }
+
+  for(ii=0; ii<nField; ii++){
+    int eHidden = pTab->aCol[ii].eHidden;
+    if( eHidden!=RECOVER_EHIDDEN_VIRTUAL
+     && eHidden!=RECOVER_EHIDDEN_STORED
+    ){
+      assert( pTab->aCol[ii].iField>=0 && pTab->aCol[ii].iBind>=1 );
+      zSql = recoverMPrintf(p, "%z%s%Q", zSql, zSep, pTab->aCol[ii].zCol);
+
+      if( bSql ){
+        zBind = recoverMPrintf(p, 
+            "%z%sescape_crnl(quote(?%d))", zBind, zSqlSep, pTab->aCol[ii].iBind
+        );
+        zSqlSep = "||', '||";
+      }else{
+        zBind = recoverMPrintf(p, "%z%s?%d", zBind, zSep, pTab->aCol[ii].iBind);
+      }
+      zSep = ", ";
+    }
+  }
+
+  if( bSql ){
+    zFinal = recoverMPrintf(p, "SELECT %Q || ') VALUES (' || %s || ')'", 
+        zSql, zBind
+    );
+  }else{
+    zFinal = recoverMPrintf(p, "%s) VALUES (%s)", zSql, zBind);
+  }
+
+  pRet = recoverPrepare(p, p->dbOut, zFinal);
+  sqlite3_free(zSql);
+  sqlite3_free(zBind);
+  sqlite3_free(zFinal);
+  
+  return pRet;
+}
+
+
+/*
+** Search the list of RecoverTable objects at p->pTblList for one that
+** has root page iRoot in the input database. If such an object is found,
+** return a pointer to it. Otherwise, return NULL.
+*/
+static RecoverTable *recoverFindTable(sqlite3_recover *p, u32 iRoot){
+  RecoverTable *pRet = 0;
+  for(pRet=p->pTblList; pRet && pRet->iRoot!=iRoot; pRet=pRet->pNext);
+  return pRet;
+}
+
+/*
+** This function attempts to create a lost and found table within the 
+** output db. If successful, it returns a pointer to a buffer containing
+** the name of the new table. It is the responsibility of the caller to
+** eventually free this buffer using sqlite3_free().
+**
+** If an error occurs, NULL is returned and an error code and error 
+** message left in the recover handle.
+*/
+static char *recoverLostAndFoundCreate(
+  sqlite3_recover *p,             /* Recover object */
+  int nField                      /* Number of column fields in new table */
+){
+  char *zTbl = 0;
+  sqlite3_stmt *pProbe = 0;
+  int ii = 0;
+
+  pProbe = recoverPrepare(p, p->dbOut,
+    "SELECT 1 FROM sqlite_schema WHERE name=?"
+  );
+  for(ii=-1; zTbl==0 && p->errCode==SQLITE_OK && ii<1000; ii++){
+    int bFail = 0;
+    if( ii<0 ){
+      zTbl = recoverMPrintf(p, "%s", p->zLostAndFound);
+    }else{
+      zTbl = recoverMPrintf(p, "%s_%d", p->zLostAndFound, ii);
+    }
+
+    if( p->errCode==SQLITE_OK ){
+      sqlite3_bind_text(pProbe, 1, zTbl, -1, SQLITE_STATIC);
+      if( SQLITE_ROW==sqlite3_step(pProbe) ){
+        bFail = 1;
+      }
+      recoverReset(p, pProbe);
+    }
+
+    if( bFail ){
+      sqlite3_clear_bindings(pProbe);
+      sqlite3_free(zTbl);
+      zTbl = 0;
+    }
+  }
+  recoverFinalize(p, pProbe);
+
+  if( zTbl ){
+    const char *zSep = 0;
+    char *zField = 0;
+    char *zSql = 0;
+
+    zSep = "rootpgno INTEGER, pgno INTEGER, nfield INTEGER, id INTEGER, ";
+    for(ii=0; p->errCode==SQLITE_OK && ii<nField; ii++){
+      zField = recoverMPrintf(p, "%z%sc%d", zField, zSep, ii);
+      zSep = ", ";
+    }
+
+    zSql = recoverMPrintf(p, "CREATE TABLE %s(%s)", zTbl, zField);
+    sqlite3_free(zField);
+
+    recoverExec(p, p->dbOut, zSql);
+    recoverSqlCallback(p, zSql);
+    sqlite3_free(zSql);
+  }else if( p->errCode==SQLITE_OK ){
+    recoverError(
+        p, SQLITE_ERROR, "failed to create %s output table", p->zLostAndFound
+    );
+  }
+
+  return zTbl;
+}
+
+/*
+** Synthesize and prepare an INSERT statement to write to the lost_and_found
+** table in the output database. The name of the table is zTab, and it has
+** nField c* fields.
+*/
+static sqlite3_stmt *recoverLostAndFoundInsert(
+  sqlite3_recover *p,
+  const char *zTab,
+  int nField
+){
+  int nTotal = nField + 4;
+  int ii;
+  char *zBind = 0;
+  sqlite3_stmt *pRet = 0;
+
+  if( p->xSql==0 ){
+    for(ii=0; ii<nTotal; ii++){
+      zBind = recoverMPrintf(p, "%z%s?", zBind, zBind?", ":"", ii);
+    }
+    pRet = recoverPreparePrintf(
+        p, p->dbOut, "INSERT INTO %s VALUES(%s)", zTab, zBind
+    );
+  }else{
+    const char *zSep = "";
+    for(ii=0; ii<nTotal; ii++){
+      zBind = recoverMPrintf(p, "%z%squote(?)", zBind, zSep);
+      zSep = "|| ', ' ||";
+    }
+    pRet = recoverPreparePrintf(
+        p, p->dbOut, "SELECT 'INSERT INTO %s VALUES(' || %s || ')'", zTab, zBind
+    );
+  }
+
+  sqlite3_free(zBind);
+  return pRet;
+}
+
+/*
+** Input database page iPg contains data that will be written to the
+** lost-and-found table of the output database. This function attempts
+** to identify the root page of the tree that page iPg belonged to.
+** If successful, it sets output variable (*piRoot) to the page number
+** of the root page and returns SQLITE_OK. Otherwise, if an error occurs,
+** an SQLite error code is returned and the final value of *piRoot 
+** undefined.
+*/
+static int recoverLostAndFoundFindRoot(
+  sqlite3_recover *p, 
+  i64 iPg,
+  i64 *piRoot
+){
+  RecoverStateLAF *pLaf = &p->laf;
+
+  if( pLaf->pFindRoot==0 ){
+    pLaf->pFindRoot = recoverPrepare(p, p->dbOut,
+        "WITH RECURSIVE p(pgno) AS ("
+        "  SELECT ?"
+        "    UNION"
+        "  SELECT parent FROM recovery.map AS m, p WHERE m.pgno=p.pgno"
+        ") "
+        "SELECT p.pgno FROM p, recovery.map m WHERE m.pgno=p.pgno "
+        "    AND m.parent IS NULL"
+    );
+  }
+  if( p->errCode==SQLITE_OK ){
+    sqlite3_bind_int64(pLaf->pFindRoot, 1, iPg);
+    if( sqlite3_step(pLaf->pFindRoot)==SQLITE_ROW ){
+      *piRoot = sqlite3_column_int64(pLaf->pFindRoot, 0);
+    }else{
+      *piRoot = iPg;
+    }
+    recoverReset(p, pLaf->pFindRoot);
+  }
+  return p->errCode;
+}
+
+/*
+** Recover data from page iPage of the input database and write it to
+** the lost-and-found table in the output database.
+*/
+static void recoverLostAndFoundOnePage(sqlite3_recover *p, i64 iPage){
+  RecoverStateLAF *pLaf = &p->laf;
+  sqlite3_value **apVal = pLaf->apVal;
+  sqlite3_stmt *pPageData = pLaf->pPageData;
+  sqlite3_stmt *pInsert = pLaf->pInsert;
+
+  int nVal = -1;
+  int iPrevCell = 0;
+  i64 iRoot = 0;
+  int bHaveRowid = 0;
+  i64 iRowid = 0;
+  int ii = 0;
+
+  if( recoverLostAndFoundFindRoot(p, iPage, &iRoot) ) return;
+  sqlite3_bind_int64(pPageData, 1, iPage);
+  while( p->errCode==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPageData) ){
+    int iCell = sqlite3_column_int64(pPageData, 0);
+    int iField = sqlite3_column_int64(pPageData, 1);
+
+    if( iPrevCell!=iCell && nVal>=0 ){
+      /* Insert the new row */
+      sqlite3_bind_int64(pInsert, 1, iRoot);      /* rootpgno */
+      sqlite3_bind_int64(pInsert, 2, iPage);      /* pgno */
+      sqlite3_bind_int(pInsert, 3, nVal);         /* nfield */
+      if( bHaveRowid ){
+        sqlite3_bind_int64(pInsert, 4, iRowid);   /* id */
+      }
+      for(ii=0; ii<nVal; ii++){
+        recoverBindValue(p, pInsert, 5+ii, apVal[ii]);
+      }
+      if( sqlite3_step(pInsert)==SQLITE_ROW ){
+        recoverSqlCallback(p, (const char*)sqlite3_column_text(pInsert, 0));
+      }
+      recoverReset(p, pInsert);
+
+      /* Discard the accumulated row data */
+      for(ii=0; ii<nVal; ii++){
+        sqlite3_value_free(apVal[ii]);
+        apVal[ii] = 0;
+      }
+      sqlite3_clear_bindings(pInsert);
+      bHaveRowid = 0;
+      nVal = -1;
+    }
+
+    if( iCell<0 ) break;
+
+    if( iField<0 ){
+      assert( nVal==-1 );
+      iRowid = sqlite3_column_int64(pPageData, 2);
+      bHaveRowid = 1;
+      nVal = 0;
+    }else if( iField<pLaf->nMaxField ){
+      sqlite3_value *pVal = sqlite3_column_value(pPageData, 2);
+      apVal[iField] = sqlite3_value_dup(pVal);
+      assert( iField==nVal || (nVal==-1 && iField==0) );
+      nVal = iField+1;
+      if( apVal[iField]==0 ){
+        recoverError(p, SQLITE_NOMEM, 0);
+      }
+    }
+
+    iPrevCell = iCell;
+  }
+  recoverReset(p, pPageData);
+
+  for(ii=0; ii<nVal; ii++){
+    sqlite3_value_free(apVal[ii]);
+    apVal[ii] = 0;
+  }
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection 
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_LOSTANDFOUND3 state - during which the lost-and-found 
+** table of the output database is populated with recovered data that can 
+** not be assigned to any recovered schema object.
+*/ 
+static int recoverLostAndFound3Step(sqlite3_recover *p){
+  RecoverStateLAF *pLaf = &p->laf;
+  if( p->errCode==SQLITE_OK ){
+    if( pLaf->pInsert==0 ){
+      return SQLITE_DONE;
+    }else{
+      if( p->errCode==SQLITE_OK ){
+        int res = sqlite3_step(pLaf->pAllPage);
+        if( res==SQLITE_ROW ){
+          i64 iPage = sqlite3_column_int64(pLaf->pAllPage, 0);
+          if( recoverBitmapQuery(pLaf->pUsed, iPage)==0 ){
+            recoverLostAndFoundOnePage(p, iPage);
+          }
+        }else{
+          recoverReset(p, pLaf->pAllPage);
+          return SQLITE_DONE;
+        }
+      }
+    }
+  }
+  return SQLITE_OK;
+}
+
+/*
+** Initialize resources required in RECOVER_STATE_LOSTANDFOUND3 
+** state - during which the lost-and-found table of the output database 
+** is populated with recovered data that can not be assigned to any 
+** recovered schema object.
+*/ 
+static void recoverLostAndFound3Init(sqlite3_recover *p){
+  RecoverStateLAF *pLaf = &p->laf;
+
+  if( pLaf->nMaxField>0 ){
+    char *zTab = 0;               /* Name of lost_and_found table */
+
+    zTab = recoverLostAndFoundCreate(p, pLaf->nMaxField);
+    pLaf->pInsert = recoverLostAndFoundInsert(p, zTab, pLaf->nMaxField);
+    sqlite3_free(zTab);
+
+    pLaf->pAllPage = recoverPreparePrintf(p, p->dbOut,
+        "WITH RECURSIVE seq(ii) AS ("
+        "  SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
+        ")"
+        "SELECT ii FROM seq" , p->laf.nPg
+    );
+    pLaf->pPageData = recoverPrepare(p, p->dbOut,
+        "SELECT cell, field, value "
+        "FROM sqlite_dbdata('getpage()') d WHERE d.pgno=? "
+        "UNION ALL "
+        "SELECT -1, -1, -1"
+    );
+
+    pLaf->apVal = (sqlite3_value**)recoverMalloc(p, 
+        pLaf->nMaxField*sizeof(sqlite3_value*)
+    );
+  }
+}
+
+/*
+** Initialize resources required in RECOVER_STATE_WRITING state - during which
+** tables recovered from the schema of the input database are populated with
+** recovered data.
+*/ 
+static int recoverWriteDataInit(sqlite3_recover *p){
+  RecoverStateW1 *p1 = &p->w1;
+  RecoverTable *pTbl = 0;
+  int nByte = 0;
+
+  /* Figure out the maximum number of columns for any table in the schema */
+  assert( p1->nMax==0 );
+  for(pTbl=p->pTblList; pTbl; pTbl=pTbl->pNext){
+    if( pTbl->nCol>p1->nMax ) p1->nMax = pTbl->nCol;
+  }
+
+  /* Allocate an array of (sqlite3_value*) in which to accumulate the values
+  ** that will be written to the output database in a single row. */
+  nByte = sizeof(sqlite3_value*) * (p1->nMax+1);
+  p1->apVal = (sqlite3_value**)recoverMalloc(p, nByte);
+  if( p1->apVal==0 ) return p->errCode;
+
+  /* Prepare the SELECT to loop through schema tables (pTbls) and the SELECT
+  ** to loop through cells that appear to belong to a single table (pSel). */
+  p1->pTbls = recoverPrepare(p, p->dbOut,
+      "SELECT rootpage FROM recovery.schema "
+      "  WHERE type='table' AND (sql NOT LIKE 'create virtual%')"
+      "  ORDER BY (tbl_name='sqlite_sequence') ASC"
+  );
+  p1->pSel = recoverPrepare(p, p->dbOut, 
+      "WITH RECURSIVE pages(page) AS ("
+      "  SELECT ?1"
+      "    UNION"
+      "  SELECT child FROM sqlite_dbptr('getpage()'), pages "
+      "    WHERE pgno=page"
+      ") "
+      "SELECT page, cell, field, value "
+      "FROM sqlite_dbdata('getpage()') d, pages p WHERE p.page=d.pgno "
+      "UNION ALL "
+      "SELECT 0, 0, 0, 0"
+  );
+
+  return p->errCode;
+}
+
+/*
+** Clean up resources allocated by recoverWriteDataInit() (stuff in 
+** sqlite3_recover.w1).
+*/
+static void recoverWriteDataCleanup(sqlite3_recover *p){
+  RecoverStateW1 *p1 = &p->w1;
+  int ii;
+  for(ii=0; ii<p1->nVal; ii++){
+    sqlite3_value_free(p1->apVal[ii]);
+  }
+  sqlite3_free(p1->apVal);
+  recoverFinalize(p, p1->pInsert);
+  recoverFinalize(p, p1->pTbls);
+  recoverFinalize(p, p1->pSel);
+  memset(p1, 0, sizeof(*p1));
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection 
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_WRITING state - during which tables recovered from the
+** schema of the input database are populated with recovered data.
+*/ 
+static int recoverWriteDataStep(sqlite3_recover *p){
+  RecoverStateW1 *p1 = &p->w1;
+  sqlite3_stmt *pSel = p1->pSel;
+  sqlite3_value **apVal = p1->apVal;
+
+  if( p->errCode==SQLITE_OK && p1->pTab==0 ){
+    if( sqlite3_step(p1->pTbls)==SQLITE_ROW ){
+      i64 iRoot = sqlite3_column_int64(p1->pTbls, 0);
+      p1->pTab = recoverFindTable(p, iRoot);
+
+      recoverFinalize(p, p1->pInsert);
+      p1->pInsert = 0;
+
+      /* If this table is unknown, return early. The caller will invoke this
+      ** function again and it will move on to the next table.  */
+      if( p1->pTab==0 ) return p->errCode;
+
+      /* If this is the sqlite_sequence table, delete any rows added by
+      ** earlier INSERT statements on tables with AUTOINCREMENT primary
+      ** keys before recovering its contents. The p1->pTbls SELECT statement
+      ** is rigged to deliver "sqlite_sequence" last of all, so we don't
+      ** worry about it being modified after it is recovered. */
+      if( sqlite3_stricmp("sqlite_sequence", p1->pTab->zTab)==0 ){
+        recoverExec(p, p->dbOut, "DELETE FROM sqlite_sequence");
+        recoverSqlCallback(p, "DELETE FROM sqlite_sequence");
+      }
+
+      /* Bind the root page of this table within the original database to 
+      ** SELECT statement p1->pSel. The SELECT statement will then iterate
+      ** through cells that look like they belong to table pTab.  */
+      sqlite3_bind_int64(pSel, 1, iRoot);
+
+      p1->nVal = 0;
+      p1->bHaveRowid = 0;
+      p1->iPrevPage = -1;
+      p1->iPrevCell = -1;
+    }else{
+      return SQLITE_DONE;
+    }
+  }
+  assert( p->errCode!=SQLITE_OK || p1->pTab );
+
+  if( p->errCode==SQLITE_OK && sqlite3_step(pSel)==SQLITE_ROW ){
+    RecoverTable *pTab = p1->pTab;
+
+    i64 iPage = sqlite3_column_int64(pSel, 0);
+    int iCell = sqlite3_column_int(pSel, 1);
+    int iField = sqlite3_column_int(pSel, 2);
+    sqlite3_value *pVal = sqlite3_column_value(pSel, 3);
+    int bNewCell = (p1->iPrevPage!=iPage || p1->iPrevCell!=iCell);
+
+    assert( bNewCell==0 || (iField==-1 || iField==0) );
+    assert( bNewCell || iField==p1->nVal || p1->nVal==pTab->nCol );
+
+    if( bNewCell ){
+      int ii = 0;
+      if( p1->nVal>=0 ){
+        if( p1->pInsert==0 || p1->nVal!=p1->nInsert ){
+          recoverFinalize(p, p1->pInsert);
+          p1->pInsert = recoverInsertStmt(p, pTab, p1->nVal);
+          p1->nInsert = p1->nVal;
+        }
+        if( p1->nVal>0 ){
+          sqlite3_stmt *pInsert = p1->pInsert;
+          for(ii=0; ii<pTab->nCol; ii++){
+            RecoverColumn *pCol = &pTab->aCol[ii];
+            int iBind = pCol->iBind;
+            if( iBind>0 ){
+              if( pCol->bIPK ){
+                sqlite3_bind_int64(pInsert, iBind, p1->iRowid);
+              }else if( pCol->iField<p1->nVal ){
+                recoverBindValue(p, pInsert, iBind, apVal[pCol->iField]);
+              }
+            }
+          }
+          if( p->bRecoverRowid && pTab->iRowidBind>0 && p1->bHaveRowid ){
+            sqlite3_bind_int64(pInsert, pTab->iRowidBind, p1->iRowid);
+          }
+          if( SQLITE_ROW==sqlite3_step(pInsert) ){
+            const char *z = (const char*)sqlite3_column_text(pInsert, 0);
+            recoverSqlCallback(p, z);
+          }
+          recoverReset(p, pInsert);
+          assert( p->errCode || pInsert );
+          if( pInsert ) sqlite3_clear_bindings(pInsert);
+        }
+      }
+
+      for(ii=0; ii<p1->nVal; ii++){
+        sqlite3_value_free(apVal[ii]);
+        apVal[ii] = 0;
+      }
+      p1->nVal = -1;
+      p1->bHaveRowid = 0;
+    }
+
+    if( iPage!=0 ){
+      if( iField<0 ){
+        p1->iRowid = sqlite3_column_int64(pSel, 3);
+        assert( p1->nVal==-1 );
+        p1->nVal = 0;
+        p1->bHaveRowid = 1;
+      }else if( iField<pTab->nCol ){
+        assert( apVal[iField]==0 );
+        apVal[iField] = sqlite3_value_dup( pVal );
+        if( apVal[iField]==0 ){
+          recoverError(p, SQLITE_NOMEM, 0);
+        }
+        p1->nVal = iField+1;
+      }
+      p1->iPrevCell = iCell;
+      p1->iPrevPage = iPage;
+    }
+  }else{
+    recoverReset(p, pSel);
+    p1->pTab = 0;
+  }
+
+  return p->errCode;
+}
+
+/*
+** Initialize resources required by sqlite3_recover_step() in
+** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not
+** already allocated to a recovered schema element is determined.
+*/ 
+static void recoverLostAndFound1Init(sqlite3_recover *p){
+  RecoverStateLAF *pLaf = &p->laf;
+  sqlite3_stmt *pStmt = 0;
+
+  assert( p->laf.pUsed==0 );
+  pLaf->nPg = recoverPageCount(p);
+  pLaf->pUsed = recoverBitmapAlloc(p, pLaf->nPg);
+
+  /* Prepare a statement to iterate through all pages that are part of any tree
+  ** in the recoverable part of the input database schema to the bitmap. And,
+  ** if !p->bFreelistCorrupt, add all pages that appear to be part of the
+  ** freelist.  */
+  pStmt = recoverPrepare(
+      p, p->dbOut,
+      "WITH trunk(pgno) AS ("
+      "  SELECT read_i32(getpage(1), 8) AS x WHERE x>0"
+      "    UNION"
+      "  SELECT read_i32(getpage(trunk.pgno), 0) AS x FROM trunk WHERE x>0"
+      "),"
+      "trunkdata(pgno, data) AS ("
+      "  SELECT pgno, getpage(pgno) FROM trunk"
+      "),"
+      "freelist(data, n, freepgno) AS ("
+      "  SELECT data, min(16384, read_i32(data, 1)-1), pgno FROM trunkdata"
+      "    UNION ALL"
+      "  SELECT data, n-1, read_i32(data, 2+n) FROM freelist WHERE n>=0"
+      "),"
+      ""
+      "roots(r) AS ("
+      "  SELECT 1 UNION ALL"
+      "  SELECT rootpage FROM recovery.schema WHERE rootpage>0"
+      "),"
+      "used(page) AS ("
+      "  SELECT r FROM roots"
+      "    UNION"
+      "  SELECT child FROM sqlite_dbptr('getpage()'), used "
+      "    WHERE pgno=page"
+      ") "
+      "SELECT page FROM used"
+      " UNION ALL "
+      "SELECT freepgno FROM freelist WHERE NOT ?"
+  );
+  if( pStmt ) sqlite3_bind_int(pStmt, 1, p->bFreelistCorrupt);
+  pLaf->pUsedPages = pStmt;
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection 
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_LOSTANDFOUND1 state - during which the set of pages not
+** already allocated to a recovered schema element is determined.
+*/ 
+static int recoverLostAndFound1Step(sqlite3_recover *p){
+  RecoverStateLAF *pLaf = &p->laf;
+  int rc = p->errCode;
+  if( rc==SQLITE_OK ){
+    rc = sqlite3_step(pLaf->pUsedPages);
+    if( rc==SQLITE_ROW ){
+      i64 iPg = sqlite3_column_int64(pLaf->pUsedPages, 0);
+      recoverBitmapSet(pLaf->pUsed, iPg);
+      rc = SQLITE_OK;
+    }else{
+      recoverFinalize(p, pLaf->pUsedPages);
+      pLaf->pUsedPages = 0;
+    }
+  }
+  return rc;
+}
+
+/*
+** Initialize resources required by RECOVER_STATE_LOSTANDFOUND2 
+** state - during which the pages identified in RECOVER_STATE_LOSTANDFOUND1
+** are sorted into sets that likely belonged to the same database tree.
+*/ 
+static void recoverLostAndFound2Init(sqlite3_recover *p){
+  RecoverStateLAF *pLaf = &p->laf;
+
+  assert( p->laf.pAllAndParent==0 );
+  assert( p->laf.pMapInsert==0 );
+  assert( p->laf.pMaxField==0 );
+  assert( p->laf.nMaxField==0 );
+
+  pLaf->pMapInsert = recoverPrepare(p, p->dbOut,
+      "INSERT OR IGNORE INTO recovery.map(pgno, parent) VALUES(?, ?)"
+  );
+  pLaf->pAllAndParent = recoverPreparePrintf(p, p->dbOut,
+      "WITH RECURSIVE seq(ii) AS ("
+      "  SELECT 1 UNION ALL SELECT ii+1 FROM seq WHERE ii<%lld"
+      ")"
+      "SELECT pgno, child FROM sqlite_dbptr('getpage()') "
+      " UNION ALL "
+      "SELECT NULL, ii FROM seq", p->laf.nPg
+  );
+  pLaf->pMaxField = recoverPreparePrintf(p, p->dbOut,
+      "SELECT max(field)+1 FROM sqlite_dbdata('getpage') WHERE pgno = ?"
+  );
+}
+
+/*
+** Perform one step (sqlite3_recover_step()) of work for the connection 
+** passed as the only argument, which is guaranteed to be in
+** RECOVER_STATE_LOSTANDFOUND2 state - during which the pages identified 
+** in RECOVER_STATE_LOSTANDFOUND1 are sorted into sets that likely belonged 
+** to the same database tree.
+*/ 
+static int recoverLostAndFound2Step(sqlite3_recover *p){
+  RecoverStateLAF *pLaf = &p->laf;
+  if( p->errCode==SQLITE_OK ){
+    int res = sqlite3_step(pLaf->pAllAndParent);
+    if( res==SQLITE_ROW ){
+      i64 iChild = sqlite3_column_int(pLaf->pAllAndParent, 1);
+      if( recoverBitmapQuery(pLaf->pUsed, iChild)==0 ){
+        sqlite3_bind_int64(pLaf->pMapInsert, 1, iChild);
+        sqlite3_bind_value(pLaf->pMapInsert, 2, 
+            sqlite3_column_value(pLaf->pAllAndParent, 0)
+        );
+        sqlite3_step(pLaf->pMapInsert);
+        recoverReset(p, pLaf->pMapInsert);
+        sqlite3_bind_int64(pLaf->pMaxField, 1, iChild);
+        if( SQLITE_ROW==sqlite3_step(pLaf->pMaxField) ){
+          int nMax = sqlite3_column_int(pLaf->pMaxField, 0);
+          if( nMax>pLaf->nMaxField ) pLaf->nMaxField = nMax;
+        }
+        recoverReset(p, pLaf->pMaxField);
+      }
+    }else{
+      recoverFinalize(p, pLaf->pAllAndParent);
+      pLaf->pAllAndParent =0;
+      return SQLITE_DONE;
+    }
+  }
+  return p->errCode;
+}
+
+/*
+** Free all resources allocated as part of sqlite3_recover_step() calls
+** in one of the RECOVER_STATE_LOSTANDFOUND[123] states.
+*/
+static void recoverLostAndFoundCleanup(sqlite3_recover *p){
+  recoverBitmapFree(p->laf.pUsed);
+  p->laf.pUsed = 0;
+  sqlite3_finalize(p->laf.pUsedPages);
+  sqlite3_finalize(p->laf.pAllAndParent);
+  sqlite3_finalize(p->laf.pMapInsert);
+  sqlite3_finalize(p->laf.pMaxField);
+  sqlite3_finalize(p->laf.pFindRoot);
+  sqlite3_finalize(p->laf.pInsert);
+  sqlite3_finalize(p->laf.pAllPage);
+  sqlite3_finalize(p->laf.pPageData);
+  p->laf.pUsedPages = 0;
+  p->laf.pAllAndParent = 0;
+  p->laf.pMapInsert = 0;
+  p->laf.pMaxField = 0;
+  p->laf.pFindRoot = 0;
+  p->laf.pInsert = 0;
+  p->laf.pAllPage = 0;
+  p->laf.pPageData = 0;
+  sqlite3_free(p->laf.apVal);
+  p->laf.apVal = 0;
+}
+
+/*
+** Free all resources allocated as part of sqlite3_recover_step() calls.
+*/
+static void recoverFinalCleanup(sqlite3_recover *p){
+  RecoverTable *pTab = 0;
+  RecoverTable *pNext = 0;
+
+  recoverWriteDataCleanup(p);
+  recoverLostAndFoundCleanup(p);
+
+  for(pTab=p->pTblList; pTab; pTab=pNext){
+    pNext = pTab->pNext;
+    sqlite3_free(pTab);
+  }
+  p->pTblList = 0;
+  sqlite3_finalize(p->pGetPage);
+  p->pGetPage = 0;
+  sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0);
+
+  {
+#ifndef NDEBUG
+    int res = 
+#endif
+       sqlite3_close(p->dbOut);
+    assert( res==SQLITE_OK );
+  }
+  p->dbOut = 0;
+}
+
+/*
+** Decode and return an unsigned 16-bit big-endian integer value from 
+** buffer a[].
+*/
+static u32 recoverGetU16(const u8 *a){
+  return (((u32)a[0])<<8) + ((u32)a[1]);
+}
+
+/*
+** Decode and return an unsigned 32-bit big-endian integer value from 
+** buffer a[].
+*/
+static u32 recoverGetU32(const u8 *a){
+  return (((u32)a[0])<<24) + (((u32)a[1])<<16) + (((u32)a[2])<<8) + ((u32)a[3]);
+}
+
+/*
+** Decode an SQLite varint from buffer a[]. Write the decoded value to (*pVal)
+** and return the number of bytes consumed.
+*/
+static int recoverGetVarint(const u8 *a, i64 *pVal){
+  sqlite3_uint64 u = 0;
+  int i;
+  for(i=0; i<8; i++){
+    u = (u<<7) + (a[i]&0x7f);
+    if( (a[i]&0x80)==0 ){ *pVal = (sqlite3_int64)u; return i+1; }
+  }
+  u = (u<<8) + (a[i]&0xff);
+  *pVal = (sqlite3_int64)u;
+  return 9;
+}
+
+/*
+** The second argument points to a buffer n bytes in size. If this buffer
+** or a prefix thereof appears to contain a well-formed SQLite b-tree page, 
+** return the page-size in bytes. Otherwise, if the buffer does not 
+** appear to contain a well-formed b-tree page, return 0.
+*/
+static int recoverIsValidPage(u8 *aTmp, const u8 *a, int n){
+  u8 *aUsed = aTmp;
+  int nFrag = 0;
+  int nActual = 0;
+  int iFree = 0;
+  int nCell = 0;                  /* Number of cells on page */
+  int iCellOff = 0;               /* Offset of cell array in page */
+  int iContent = 0;
+  int eType = 0;
+  int ii = 0;
+
+  eType = (int)a[0];
+  if( eType!=0x02 && eType!=0x05 && eType!=0x0A && eType!=0x0D ) return 0;
+
+  iFree = (int)recoverGetU16(&a[1]);
+  nCell = (int)recoverGetU16(&a[3]);
+  iContent = (int)recoverGetU16(&a[5]);
+  if( iContent==0 ) iContent = 65536;
+  nFrag = (int)a[7];
+
+  if( iContent>n ) return 0;
+
+  memset(aUsed, 0, n);
+  memset(aUsed, 0xFF, iContent);
+
+  /* Follow the free-list. This is the same format for all b-tree pages. */
+  if( iFree && iFree<=iContent ) return 0;
+  while( iFree ){
+    int iNext = 0;
+    int nByte = 0;
+    if( iFree>(n-4) ) return 0;
+    iNext = recoverGetU16(&a[iFree]);
+    nByte = recoverGetU16(&a[iFree+2]);
+    if( iFree+nByte>n || nByte<4 ) return 0;
+    if( iNext && iNext<iFree+nByte ) return 0;
+    memset(&aUsed[iFree], 0xFF, nByte);
+    iFree = iNext;
+  }
+
+  /* Run through the cells */
+  if( eType==0x02 || eType==0x05 ){
+    iCellOff = 12;
+  }else{
+    iCellOff = 8;
+  }
+  if( (iCellOff + 2*nCell)>iContent ) return 0;
+  for(ii=0; ii<nCell; ii++){
+    int iByte;
+    i64 nPayload = 0;
+    int nByte = 0;
+    int iOff = recoverGetU16(&a[iCellOff + 2*ii]);
+    if( iOff<iContent || iOff>n ){
+      return 0;
+    }
+    if( eType==0x05 || eType==0x02 ) nByte += 4;
+    nByte += recoverGetVarint(&a[iOff+nByte], &nPayload);
+    if( eType==0x0D ){
+      i64 dummy = 0;
+      nByte += recoverGetVarint(&a[iOff+nByte], &dummy);
+    }
+    if( eType!=0x05 ){
+      int X = (eType==0x0D) ? n-35 : (((n-12)*64/255)-23);
+      int M = ((n-12)*32/255)-23;
+      int K = M+((nPayload-M)%(n-4));
+
+      if( nPayload<X ){
+        nByte += nPayload;
+      }else if( K<=X ){
+        nByte += K+4;
+      }else{
+        nByte += M+4;
+      }
+    }
+
+    if( iOff+nByte>n ){
+      return 0;
+    }
+    for(iByte=iOff; iByte<(iOff+nByte); iByte++){
+      if( aUsed[iByte]!=0 ){
+        return 0;
+      }
+      aUsed[iByte] = 0xFF;
+    }
+  }
+
+  nActual = 0;
+  for(ii=0; ii<n; ii++){
+    if( aUsed[ii]==0 ) nActual++;
+  }
+  return (nActual==nFrag);
+}
+
+
+static int recoverVfsClose(sqlite3_file*);
+static int recoverVfsRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst);
+static int recoverVfsWrite(sqlite3_file*, const void*, int, sqlite3_int64);
+static int recoverVfsTruncate(sqlite3_file*, sqlite3_int64 size);
+static int recoverVfsSync(sqlite3_file*, int flags);
+static int recoverVfsFileSize(sqlite3_file*, sqlite3_int64 *pSize);
+static int recoverVfsLock(sqlite3_file*, int);
+static int recoverVfsUnlock(sqlite3_file*, int);
+static int recoverVfsCheckReservedLock(sqlite3_file*, int *pResOut);
+static int recoverVfsFileControl(sqlite3_file*, int op, void *pArg);
+static int recoverVfsSectorSize(sqlite3_file*);
+static int recoverVfsDeviceCharacteristics(sqlite3_file*);
+static int recoverVfsShmMap(sqlite3_file*, int, int, int, void volatile**);
+static int recoverVfsShmLock(sqlite3_file*, int offset, int n, int flags);
+static void recoverVfsShmBarrier(sqlite3_file*);
+static int recoverVfsShmUnmap(sqlite3_file*, int deleteFlag);
+static int recoverVfsFetch(sqlite3_file*, sqlite3_int64, int, void**);
+static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p);
+
+static sqlite3_io_methods recover_methods = {
+  2, /* iVersion */
+  recoverVfsClose,
+  recoverVfsRead,
+  recoverVfsWrite,
+  recoverVfsTruncate,
+  recoverVfsSync,
+  recoverVfsFileSize,
+  recoverVfsLock,
+  recoverVfsUnlock,
+  recoverVfsCheckReservedLock,
+  recoverVfsFileControl,
+  recoverVfsSectorSize,
+  recoverVfsDeviceCharacteristics,
+  recoverVfsShmMap,
+  recoverVfsShmLock,
+  recoverVfsShmBarrier,
+  recoverVfsShmUnmap,
+  recoverVfsFetch,
+  recoverVfsUnfetch
+};
+
+static int recoverVfsClose(sqlite3_file *pFd){
+  assert( pFd->pMethods!=&recover_methods );
+  return pFd->pMethods->xClose(pFd);
+}
+
+/*
+** Write value v to buffer a[] as a 16-bit big-endian unsigned integer.
+*/
+static void recoverPutU16(u8 *a, u32 v){
+  a[0] = (v>>8) & 0x00FF;
+  a[1] = (v>>0) & 0x00FF;
+}
+
+/*
+** Write value v to buffer a[] as a 32-bit big-endian unsigned integer.
+*/
+static void recoverPutU32(u8 *a, u32 v){
+  a[0] = (v>>24) & 0x00FF;
+  a[1] = (v>>16) & 0x00FF;
+  a[2] = (v>>8) & 0x00FF;
+  a[3] = (v>>0) & 0x00FF;
+}
+
+/*
+** Detect the page-size of the database opened by file-handle pFd by 
+** searching the first part of the file for a well-formed SQLite b-tree 
+** page. If parameter nReserve is non-zero, then as well as searching for
+** a b-tree page with zero reserved bytes, this function searches for one
+** with nReserve reserved bytes at the end of it.
+**
+** If successful, set variable p->detected_pgsz to the detected page-size
+** in bytes and return SQLITE_OK. Or, if no error occurs but no valid page
+** can be found, return SQLITE_OK but leave p->detected_pgsz set to 0. Or,
+** if an error occurs (e.g. an IO or OOM error), then an SQLite error code
+** is returned. The final value of p->detected_pgsz is undefined in this
+** case.
+*/
+static int recoverVfsDetectPagesize(
+  sqlite3_recover *p,             /* Recover handle */
+  sqlite3_file *pFd,              /* File-handle open on input database */
+  u32 nReserve,                   /* Possible nReserve value */
+  i64 nSz                         /* Size of database file in bytes */
+){
+  int rc = SQLITE_OK;
+  const int nMin = 512;
+  const int nMax = 65536;
+  const int nMaxBlk = 4;
+  u32 pgsz = 0;
+  int iBlk = 0;
+  u8 *aPg = 0;
+  u8 *aTmp = 0;
+  int nBlk = 0;
+
+  aPg = (u8*)sqlite3_malloc(2*nMax);
+  if( aPg==0 ) return SQLITE_NOMEM;
+  aTmp = &aPg[nMax];
+
+  nBlk = (nSz+nMax-1)/nMax;
+  if( nBlk>nMaxBlk ) nBlk = nMaxBlk;
+
+  do {
+    for(iBlk=0; rc==SQLITE_OK && iBlk<nBlk; iBlk++){
+      int nByte = (nSz>=((iBlk+1)*nMax)) ? nMax : (nSz % nMax);
+      memset(aPg, 0, nMax);
+      rc = pFd->pMethods->xRead(pFd, aPg, nByte, iBlk*nMax);
+      if( rc==SQLITE_OK ){
+        int pgsz2;
+        for(pgsz2=(pgsz ? pgsz*2 : nMin); pgsz2<=nMax; pgsz2=pgsz2*2){
+          int iOff;
+          for(iOff=0; iOff<nMax; iOff+=pgsz2){
+            if( recoverIsValidPage(aTmp, &aPg[iOff], pgsz2-nReserve) ){
+              pgsz = pgsz2;
+              break;
+            }
+          }
+        }
+      }
+    }
+    if( pgsz>(u32)p->detected_pgsz ){
+      p->detected_pgsz = pgsz;
+      p->nReserve = nReserve;
+    }
+    if( nReserve==0 ) break;
+    nReserve = 0;
+  }while( 1 );
+
+  p->detected_pgsz = pgsz;
+  sqlite3_free(aPg);
+  return rc;
+}
+
+/*
+** The xRead() method of the wrapper VFS. This is used to intercept calls
+** to read page 1 of the input database.
+*/
+static int recoverVfsRead(sqlite3_file *pFd, void *aBuf, int nByte, i64 iOff){
+  int rc = SQLITE_OK;
+  if( pFd->pMethods==&recover_methods ){
+    pFd->pMethods = recover_g.pMethods;
+    rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
+    if( nByte==16 ){
+      sqlite3_randomness(16, aBuf);
+    }else
+    if( rc==SQLITE_OK && iOff==0 && nByte>=108 ){
+      /* Ensure that the database has a valid header file. The only fields
+      ** that really matter to recovery are:
+      **
+      **   + Database page size (16-bits at offset 16)
+      **   + Size of db in pages (32-bits at offset 28)
+      **   + Database encoding (32-bits at offset 56)
+      **
+      ** Also preserved are:
+      **
+      **   + first freelist page (32-bits at offset 32)
+      **   + size of freelist (32-bits at offset 36)
+      **   + the wal-mode flags (16-bits at offset 18)
+      **
+      ** We also try to preserve the auto-vacuum, incr-value, user-version
+      ** and application-id fields - all 32 bit quantities at offsets 
+      ** 52, 60, 64 and 68. All other fields are set to known good values.
+      **
+      ** Byte offset 105 should also contain the page-size as a 16-bit 
+      ** integer.
+      */
+      const int aPreserve[] = {32, 36, 52, 60, 64, 68};
+      u8 aHdr[108] = {
+        0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66, 
+        0x6f, 0x72, 0x6d, 0x61, 0x74, 0x20, 0x33, 0x00,
+        0xFF, 0xFF, 0x01, 0x01, 0x00, 0x40, 0x20, 0x20,
+        0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
+        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+        0x00, 0x00, 0x10, 0x00, 0xFF, 0xFF, 0xFF, 0xFF,
+        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+        0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+        0x00, 0x2e, 0x5b, 0x30,
+
+        0x0D, 0x00, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00
+      };
+      u8 *a = (u8*)aBuf;
+
+      u32 pgsz = recoverGetU16(&a[16]);
+      u32 nReserve = a[20];
+      u32 enc = recoverGetU32(&a[56]);
+      u32 dbsz = 0;
+      i64 dbFileSize = 0;
+      int ii;
+      sqlite3_recover *p = recover_g.p;
+
+      if( pgsz==0x01 ) pgsz = 65536;
+      rc = pFd->pMethods->xFileSize(pFd, &dbFileSize);
+
+      if( rc==SQLITE_OK && p->detected_pgsz==0 ){
+        rc = recoverVfsDetectPagesize(p, pFd, nReserve, dbFileSize);
+      }
+      if( p->detected_pgsz ){
+        pgsz = p->detected_pgsz;
+        nReserve = p->nReserve;
+      }
+
+      if( pgsz ){
+        dbsz = dbFileSize / pgsz;
+      }
+      if( enc!=SQLITE_UTF8 && enc!=SQLITE_UTF16BE && enc!=SQLITE_UTF16LE ){
+        enc = SQLITE_UTF8;
+      }
+
+      sqlite3_free(p->pPage1Cache);
+      p->pPage1Cache = 0;
+      p->pPage1Disk = 0;
+
+      p->pgsz = nByte;
+      p->pPage1Cache = (u8*)recoverMalloc(p, nByte*2);
+      if( p->pPage1Cache ){
+        p->pPage1Disk = &p->pPage1Cache[nByte];
+        memcpy(p->pPage1Disk, aBuf, nByte);
+        aHdr[18] = a[18];
+        aHdr[19] = a[19];
+        recoverPutU32(&aHdr[28], dbsz);
+        recoverPutU32(&aHdr[56], enc);
+        recoverPutU16(&aHdr[105], pgsz-nReserve);
+        if( pgsz==65536 ) pgsz = 1;
+        recoverPutU16(&aHdr[16], pgsz);
+        aHdr[20] = nReserve;
+        for(ii=0; ii<(int)(sizeof(aPreserve)/sizeof(aPreserve[0])); ii++){
+          memcpy(&aHdr[aPreserve[ii]], &a[aPreserve[ii]], 4);
+        }
+        memcpy(aBuf, aHdr, sizeof(aHdr));
+        memset(&((u8*)aBuf)[sizeof(aHdr)], 0, nByte-sizeof(aHdr));
+
+        memcpy(p->pPage1Cache, aBuf, nByte);
+      }else{
+        rc = p->errCode;
+      }
+
+    }
+    pFd->pMethods = &recover_methods;
+  }else{
+    rc = pFd->pMethods->xRead(pFd, aBuf, nByte, iOff);
+  }
+  return rc;
+}
+
+/*
+** Used to make sqlite3_io_methods wrapper methods less verbose.
+*/
+#define RECOVER_VFS_WRAPPER(code)                         \
+  int rc = SQLITE_OK;                                     \
+  if( pFd->pMethods==&recover_methods ){                  \
+    pFd->pMethods = recover_g.pMethods;                   \
+    rc = code;                                            \
+    pFd->pMethods = &recover_methods;                     \
+  }else{                                                  \
+    rc = code;                                            \
+  }                                                       \
+  return rc;                                              
+
+/*
+** Methods of the wrapper VFS. All methods except for xRead() and xClose()
+** simply uninstall the sqlite3_io_methods wrapper, invoke the equivalent
+** method on the lower level VFS, then reinstall the wrapper before returning.
+** Those that return an integer value use the RECOVER_VFS_WRAPPER macro.
+*/
+static int recoverVfsWrite(
+  sqlite3_file *pFd, const void *aBuf, int nByte, i64 iOff
+){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xWrite(pFd, aBuf, nByte, iOff)
+  );
+}
+static int recoverVfsTruncate(sqlite3_file *pFd, sqlite3_int64 size){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xTruncate(pFd, size)
+  );
+}
+static int recoverVfsSync(sqlite3_file *pFd, int flags){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xSync(pFd, flags)
+  );
+}
+static int recoverVfsFileSize(sqlite3_file *pFd, sqlite3_int64 *pSize){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xFileSize(pFd, pSize)
+  );
+}
+static int recoverVfsLock(sqlite3_file *pFd, int eLock){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xLock(pFd, eLock)
+  );
+}
+static int recoverVfsUnlock(sqlite3_file *pFd, int eLock){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xUnlock(pFd, eLock)
+  );
+}
+static int recoverVfsCheckReservedLock(sqlite3_file *pFd, int *pResOut){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xCheckReservedLock(pFd, pResOut)
+  );
+}
+static int recoverVfsFileControl(sqlite3_file *pFd, int op, void *pArg){
+  RECOVER_VFS_WRAPPER (
+    (pFd->pMethods ?  pFd->pMethods->xFileControl(pFd, op, pArg) : SQLITE_NOTFOUND)
+  );
+}
+static int recoverVfsSectorSize(sqlite3_file *pFd){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xSectorSize(pFd)
+  );
+}
+static int recoverVfsDeviceCharacteristics(sqlite3_file *pFd){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xDeviceCharacteristics(pFd)
+  );
+}
+static int recoverVfsShmMap(
+  sqlite3_file *pFd, int iPg, int pgsz, int bExtend, void volatile **pp
+){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xShmMap(pFd, iPg, pgsz, bExtend, pp)
+  );
+}
+static int recoverVfsShmLock(sqlite3_file *pFd, int offset, int n, int flags){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xShmLock(pFd, offset, n, flags)
+  );
+}
+static void recoverVfsShmBarrier(sqlite3_file *pFd){
+  if( pFd->pMethods==&recover_methods ){
+    pFd->pMethods = recover_g.pMethods;
+    pFd->pMethods->xShmBarrier(pFd);
+    pFd->pMethods = &recover_methods;
+  }else{
+    pFd->pMethods->xShmBarrier(pFd);
+  }
+}
+static int recoverVfsShmUnmap(sqlite3_file *pFd, int deleteFlag){
+  RECOVER_VFS_WRAPPER (
+      pFd->pMethods->xShmUnmap(pFd, deleteFlag)
+  );
+}
+
+static int recoverVfsFetch(
+  sqlite3_file *pFd, 
+  sqlite3_int64 iOff, 
+  int iAmt, 
+  void **pp
+){
+  (void)pFd;
+  (void)iOff;
+  (void)iAmt;
+  *pp = 0;
+  return SQLITE_OK;
+}
+static int recoverVfsUnfetch(sqlite3_file *pFd, sqlite3_int64 iOff, void *p){
+  (void)pFd;
+  (void)iOff;
+  (void)p;
+  return SQLITE_OK;
+}
+
+/*
+** Install the VFS wrapper around the file-descriptor open on the input
+** database for recover handle p. Mutex RECOVER_MUTEX_ID must be held
+** when this function is called.
+*/
+static void recoverInstallWrapper(sqlite3_recover *p){
+  sqlite3_file *pFd = 0;
+  assert( recover_g.pMethods==0 );
+  recoverAssertMutexHeld();
+  sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_FILE_POINTER, (void*)&pFd);
+  assert( pFd==0 || pFd->pMethods!=&recover_methods );
+  if( pFd && pFd->pMethods ){
+    int iVersion = 1 + (pFd->pMethods->iVersion>1 && pFd->pMethods->xShmMap!=0);
+    recover_g.pMethods = pFd->pMethods;
+    recover_g.p = p;
+    recover_methods.iVersion = iVersion;
+    pFd->pMethods = &recover_methods;
+  }
+}
+
+/*
+** Uninstall the VFS wrapper that was installed around the file-descriptor open
+** on the input database for recover handle p. Mutex RECOVER_MUTEX_ID must be
+** held when this function is called.
+*/
+static void recoverUninstallWrapper(sqlite3_recover *p){
+  sqlite3_file *pFd = 0;
+  recoverAssertMutexHeld();
+  sqlite3_file_control(p->dbIn, p->zDb,SQLITE_FCNTL_FILE_POINTER,(void*)&pFd);
+  if( pFd && pFd->pMethods ){
+    pFd->pMethods = recover_g.pMethods;
+    recover_g.pMethods = 0;
+    recover_g.p = 0;
+  }
+}
+
+/*
+** This function does the work of a single sqlite3_recover_step() call. It
+** is guaranteed that the handle is not in an error state when this
+** function is called.
+*/
+static void recoverStep(sqlite3_recover *p){
+  assert( p && p->errCode==SQLITE_OK );
+  switch( p->eState ){
+    case RECOVER_STATE_INIT:
+      /* This is the very first call to sqlite3_recover_step() on this object.
+      */
+      recoverSqlCallback(p, "BEGIN");
+      recoverSqlCallback(p, "PRAGMA writable_schema = on");
+
+      recoverEnterMutex();
+      recoverInstallWrapper(p);
+
+      /* Open the output database. And register required virtual tables and 
+      ** user functions with the new handle. */
+      recoverOpenOutput(p);
+
+      /* Open transactions on both the input and output databases. */
+      sqlite3_file_control(p->dbIn, p->zDb, SQLITE_FCNTL_RESET_CACHE, 0);
+      recoverExec(p, p->dbIn, "PRAGMA writable_schema = on");
+      recoverExec(p, p->dbIn, "BEGIN");
+      if( p->errCode==SQLITE_OK ) p->bCloseTransaction = 1;
+      recoverExec(p, p->dbIn, "SELECT 1 FROM sqlite_schema");
+      recoverTransferSettings(p);
+      recoverOpenRecovery(p);
+      recoverCacheSchema(p);
+
+      recoverUninstallWrapper(p);
+      recoverLeaveMutex();
+
+      recoverExec(p, p->dbOut, "BEGIN");
+
+      recoverWriteSchema1(p);
+      p->eState = RECOVER_STATE_WRITING;
+      break;
+      
+    case RECOVER_STATE_WRITING: {
+      if( p->w1.pTbls==0 ){
+        recoverWriteDataInit(p);
+      }
+      if( SQLITE_DONE==recoverWriteDataStep(p) ){
+        recoverWriteDataCleanup(p);
+        if( p->zLostAndFound ){
+          p->eState = RECOVER_STATE_LOSTANDFOUND1;
+        }else{
+          p->eState = RECOVER_STATE_SCHEMA2;
+        }
+      }
+      break;
+    }
+
+    case RECOVER_STATE_LOSTANDFOUND1: {
+      if( p->laf.pUsed==0 ){
+        recoverLostAndFound1Init(p);
+      }
+      if( SQLITE_DONE==recoverLostAndFound1Step(p) ){
+        p->eState = RECOVER_STATE_LOSTANDFOUND2;
+      }
+      break;
+    }
+    case RECOVER_STATE_LOSTANDFOUND2: {
+      if( p->laf.pAllAndParent==0 ){
+        recoverLostAndFound2Init(p);
+      }
+      if( SQLITE_DONE==recoverLostAndFound2Step(p) ){
+        p->eState = RECOVER_STATE_LOSTANDFOUND3;
+      }
+      break;
+    }
+
+    case RECOVER_STATE_LOSTANDFOUND3: {
+      if( p->laf.pInsert==0 ){
+        recoverLostAndFound3Init(p);
+      }
+      if( SQLITE_DONE==recoverLostAndFound3Step(p) ){
+        p->eState = RECOVER_STATE_SCHEMA2;
+      }
+      break;
+    }
+
+    case RECOVER_STATE_SCHEMA2: {
+      int rc = SQLITE_OK;
+
+      recoverWriteSchema2(p);
+      p->eState = RECOVER_STATE_DONE;
+
+      /* If no error has occurred, commit the write transaction on the output
+      ** database. Regardless of whether or not an error has occurred, make
+      ** an attempt to end the read transaction on the input database.  */
+      recoverExec(p, p->dbOut, "COMMIT");
+      rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0);
+      if( p->errCode==SQLITE_OK ) p->errCode = rc;
+
+      recoverSqlCallback(p, "PRAGMA writable_schema = off");
+      recoverSqlCallback(p, "COMMIT");
+      p->eState = RECOVER_STATE_DONE;
+      recoverFinalCleanup(p);
+      break;
+    };
+
+    case RECOVER_STATE_DONE: {
+      /* no-op */
+      break;
+    };
+  }
+}
+
+
+/*
+** This is a worker function that does the heavy lifting for both init
+** functions:
+**
+**     sqlite3_recover_init()
+**     sqlite3_recover_init_sql()
+**
+** All this function does is allocate space for the recover handle and
+** take copies of the input parameters. All the real work is done within
+** sqlite3_recover_run().
+*/
+sqlite3_recover *recoverInit(
+  sqlite3* db, 
+  const char *zDb, 
+  const char *zUri,               /* Output URI for _recover_init() */
+  int (*xSql)(void*, const char*),/* SQL callback for _recover_init_sql() */
+  void *pSqlCtx                   /* Context arg for _recover_init_sql() */
+){
+  sqlite3_recover *pRet = 0;
+  int nDb = 0;
+  int nUri = 0;
+  int nByte = 0;
+
+  if( zDb==0 ){ zDb = "main"; }
+
+  nDb = recoverStrlen(zDb);
+  nUri = recoverStrlen(zUri);
+
+  nByte = sizeof(sqlite3_recover) + nDb+1 + nUri+1;
+  pRet = (sqlite3_recover*)sqlite3_malloc(nByte);
+  if( pRet ){
+    memset(pRet, 0, nByte);
+    pRet->dbIn = db;
+    pRet->zDb = (char*)&pRet[1];
+    pRet->zUri = &pRet->zDb[nDb+1];
+    memcpy(pRet->zDb, zDb, nDb);
+    if( nUri>0 && zUri ) memcpy(pRet->zUri, zUri, nUri);
+    pRet->xSql = xSql;
+    pRet->pSqlCtx = pSqlCtx;
+    pRet->bRecoverRowid = RECOVER_ROWID_DEFAULT;
+  }
+
+  return pRet;
+}
+
+/*
+** Initialize a recovery handle that creates a new database containing
+** the recovered data.
+*/
+sqlite3_recover *sqlite3_recover_init(
+  sqlite3* db, 
+  const char *zDb, 
+  const char *zUri
+){
+  return recoverInit(db, zDb, zUri, 0, 0);
+}
+
+/*
+** Initialize a recovery handle that returns recovered data in the
+** form of SQL statements via a callback.
+*/
+sqlite3_recover *sqlite3_recover_init_sql(
+  sqlite3* db, 
+  const char *zDb, 
+  int (*xSql)(void*, const char*),
+  void *pSqlCtx
+){
+  return recoverInit(db, zDb, 0, xSql, pSqlCtx);
+}
+
+/*
+** Return the handle error message, if any.
+*/
+const char *sqlite3_recover_errmsg(sqlite3_recover *p){
+  return (p && p->errCode!=SQLITE_NOMEM) ? p->zErrMsg : "out of memory";
+}
+
+/*
+** Return the handle error code.
+*/
+int sqlite3_recover_errcode(sqlite3_recover *p){
+  return p ? p->errCode : SQLITE_NOMEM;
+}
+
+/*
+** Configure the handle.
+*/
+int sqlite3_recover_config(sqlite3_recover *p, int op, void *pArg){
+  int rc = SQLITE_OK;
+  if( p==0 ){
+    rc = SQLITE_NOMEM;
+  }else if( p->eState!=RECOVER_STATE_INIT ){
+    rc = SQLITE_MISUSE;
+  }else{
+    switch( op ){
+      case 789:
+        /* This undocumented magic configuration option is used to set the
+        ** name of the auxiliary database that is ATTACH-ed to the database
+        ** connection and used to hold state information during the
+        ** recovery process.  This option is for debugging use only and
+        ** is subject to change or removal at any time. */
+        sqlite3_free(p->zStateDb);
+        p->zStateDb = recoverMPrintf(p, "%s", (char*)pArg);
+        break;
+
+      case SQLITE_RECOVER_LOST_AND_FOUND: {
+        const char *zArg = (const char*)pArg;
+        sqlite3_free(p->zLostAndFound);
+        if( zArg ){
+          p->zLostAndFound = recoverMPrintf(p, "%s", zArg);
+        }else{
+          p->zLostAndFound = 0;
+        }
+        break;
+      }
+
+      case SQLITE_RECOVER_FREELIST_CORRUPT:
+        p->bFreelistCorrupt = *(int*)pArg;
+        break;
+
+      case SQLITE_RECOVER_ROWIDS:
+        p->bRecoverRowid = *(int*)pArg;
+        break;
+
+      case SQLITE_RECOVER_SLOWINDEXES:
+        p->bSlowIndexes = *(int*)pArg;
+        break;
+
+      default:
+        rc = SQLITE_NOTFOUND;
+        break;
+    }
+  }
+
+  return rc;
+}
+
+/*
+** Do a unit of work towards the recovery job. Return SQLITE_OK if
+** no error has occurred but database recovery is not finished, SQLITE_DONE
+** if database recovery has been successfully completed, or an SQLite
+** error code if an error has occurred.
+*/
+int sqlite3_recover_step(sqlite3_recover *p){
+  if( p==0 ) return SQLITE_NOMEM;
+  if( p->errCode==SQLITE_OK ) recoverStep(p);
+  if( p->eState==RECOVER_STATE_DONE && p->errCode==SQLITE_OK ){
+    return SQLITE_DONE;
+  }
+  return p->errCode;
+}
+
+/*
+** Do the configured recovery operation. Return SQLITE_OK if successful, or
+** else an SQLite error code.
+*/
+int sqlite3_recover_run(sqlite3_recover *p){
+  while( SQLITE_OK==sqlite3_recover_step(p) );
+  return sqlite3_recover_errcode(p);
+}
+
+
+/*
+** Free all resources associated with the recover handle passed as the only
+** argument. The results of using a handle with any sqlite3_recover_**
+** API function after it has been passed to this function are undefined.
+**
+** A copy of the value returned by the first call made to sqlite3_recover_run()
+** on this handle is returned, or SQLITE_OK if sqlite3_recover_run() has
+** not been called on this handle.
+*/
+int sqlite3_recover_finish(sqlite3_recover *p){
+  int rc;
+  if( p==0 ){
+    rc = SQLITE_NOMEM;
+  }else{
+    recoverFinalCleanup(p);
+    if( p->bCloseTransaction && sqlite3_get_autocommit(p->dbIn)==0 ){
+      rc = sqlite3_exec(p->dbIn, "END", 0, 0, 0);
+      if( p->errCode==SQLITE_OK ) p->errCode = rc;
+    }
+    rc = p->errCode;
+    sqlite3_free(p->zErrMsg);
+    sqlite3_free(p->zStateDb);
+    sqlite3_free(p->zLostAndFound);
+    sqlite3_free(p->pPage1Cache);
+    sqlite3_free(p);
+  }
+  return rc;
+}
+
+#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */
+#pragma GCC diagnostic pop
diff --git a/database/sqlite/sqlite3recover.h b/database/sqlite/sqlite3recover.h
new file mode 100644
index 000000000..7a1cd1cd8
--- /dev/null
+++ b/database/sqlite/sqlite3recover.h
@@ -0,0 +1,249 @@
+/*
+** 2022-08-27
+**
+** The author disclaims copyright to this source code.  In place of
+** a legal notice, here is a blessing:
+**
+**    May you do good and not evil.
+**    May you find forgiveness for yourself and forgive others.
+**    May you share freely, never taking more than you give.
+**
+*************************************************************************
+**
+** This file contains the public interface to the "recover" extension -
+** an SQLite extension designed to recover data from corrupted database
+** files.
+*/
+
+/*
+** OVERVIEW:
+**
+** To use the API to recover data from a corrupted database, an
+** application:
+**
+**   1) Creates an sqlite3_recover handle by calling either
+**      sqlite3_recover_init() or sqlite3_recover_init_sql().
+**
+**   2) Configures the new handle using one or more calls to
+**      sqlite3_recover_config().
+**
+**   3) Executes the recovery by repeatedly calling sqlite3_recover_step() on
+**      the handle until it returns something other than SQLITE_OK. If it
+**      returns SQLITE_DONE, then the recovery operation completed without 
+**      error. If it returns some other non-SQLITE_OK value, then an error 
+**      has occurred.
+**
+**   4) Retrieves any error code and English language error message using the
+**      sqlite3_recover_errcode() and sqlite3_recover_errmsg() APIs,
+**      respectively.
+**
+**   5) Destroys the sqlite3_recover handle and frees all resources
+**      using sqlite3_recover_finish().
+**
+** The application may abandon the recovery operation at any point 
+** before it is finished by passing the sqlite3_recover handle to
+** sqlite3_recover_finish(). This is not an error, but the final state
+** of the output database, or the results of running the partial script
+** delivered to the SQL callback, are undefined.
+*/
+
+#ifndef _SQLITE_RECOVER_H
+#define _SQLITE_RECOVER_H
+
+#include "sqlite3.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*
+** An instance of the sqlite3_recover object represents a recovery
+** operation in progress.
+**
+** Constructors:
+**
+**    sqlite3_recover_init()
+**    sqlite3_recover_init_sql()
+**
+** Destructor:
+**
+**    sqlite3_recover_finish()
+**
+** Methods:
+**
+**    sqlite3_recover_config()
+**    sqlite3_recover_errcode()
+**    sqlite3_recover_errmsg()
+**    sqlite3_recover_run()
+**    sqlite3_recover_step()
+*/
+typedef struct sqlite3_recover sqlite3_recover;
+
+/* 
+** These two APIs attempt to create and return a new sqlite3_recover object.
+** In both cases the first two arguments identify the (possibly
+** corrupt) database to recover data from. The first argument is an open
+** database handle and the second the name of a database attached to that
+** handle (i.e. "main", "temp" or the name of an attached database).
+**
+** If sqlite3_recover_init() is used to create the new sqlite3_recover
+** handle, then data is recovered into a new database, identified by
+** string parameter zUri. zUri may be an absolute or relative file path,
+** or may be an SQLite URI. If the identified database file already exists,
+** it is overwritten.
+**
+** If sqlite3_recover_init_sql() is invoked, then any recovered data will
+** be returned to the user as a series of SQL statements. Executing these
+** SQL statements results in the same database as would have been created
+** had sqlite3_recover_init() been used. For each SQL statement in the
+** output, the callback function passed as the third argument (xSql) is 
+** invoked once. The first parameter is a passed a copy of the fourth argument
+** to this function (pCtx) as its first parameter, and a pointer to a
+** nul-terminated buffer containing the SQL statement formated as UTF-8 as 
+** the second. If the xSql callback returns any value other than SQLITE_OK,
+** then processing is immediately abandoned and the value returned used as
+** the recover handle error code (see below).
+**
+** If an out-of-memory error occurs, NULL may be returned instead of
+** a valid handle. In all other cases, it is the responsibility of the
+** application to avoid resource leaks by ensuring that
+** sqlite3_recover_finish() is called on all allocated handles.
+*/
+sqlite3_recover *sqlite3_recover_init(
+  sqlite3* db, 
+  const char *zDb, 
+  const char *zUri
+);
+sqlite3_recover *sqlite3_recover_init_sql(
+  sqlite3* db, 
+  const char *zDb, 
+  int (*xSql)(void*, const char*),
+  void *pCtx
+);
+
+/*
+** Configure an sqlite3_recover object that has just been created using
+** sqlite3_recover_init() or sqlite3_recover_init_sql(). This function
+** may only be called before the first call to sqlite3_recover_step()
+** or sqlite3_recover_run() on the object.
+**
+** The second argument passed to this function must be one of the
+** SQLITE_RECOVER_* symbols defined below. Valid values for the third argument
+** depend on the specific SQLITE_RECOVER_* symbol in use.
+**
+** SQLITE_OK is returned if the configuration operation was successful,
+** or an SQLite error code otherwise.
+*/
+int sqlite3_recover_config(sqlite3_recover*, int op, void *pArg);
+
+/*
+** SQLITE_RECOVER_LOST_AND_FOUND:
+**   The pArg argument points to a string buffer containing the name
+**   of a "lost-and-found" table in the output database, or NULL. If
+**   the argument is non-NULL and the database contains seemingly
+**   valid pages that cannot be associated with any table in the
+**   recovered part of the schema, data is extracted from these
+**   pages to add to the lost-and-found table.
+**
+** SQLITE_RECOVER_FREELIST_CORRUPT:
+**   The pArg value must actually be a pointer to a value of type
+**   int containing value 0 or 1 cast as a (void*). If this option is set
+**   (argument is 1) and a lost-and-found table has been configured using
+**   SQLITE_RECOVER_LOST_AND_FOUND, then is assumed that the freelist is 
+**   corrupt and an attempt is made to recover records from pages that
+**   appear to be linked into the freelist. Otherwise, pages on the freelist
+**   are ignored. Setting this option can recover more data from the
+**   database, but often ends up "recovering" deleted records. The default 
+**   value is 0 (clear).
+**
+** SQLITE_RECOVER_ROWIDS:
+**   The pArg value must actually be a pointer to a value of type
+**   int containing value 0 or 1 cast as a (void*). If this option is set
+**   (argument is 1), then an attempt is made to recover rowid values
+**   that are not also INTEGER PRIMARY KEY values. If this option is
+**   clear, then new rowids are assigned to all recovered rows. The
+**   default value is 1 (set).
+**
+** SQLITE_RECOVER_SLOWINDEXES:
+**   The pArg value must actually be a pointer to a value of type
+**   int containing value 0 or 1 cast as a (void*). If this option is clear
+**   (argument is 0), then when creating an output database, the recover 
+**   module creates and populates non-UNIQUE indexes right at the end of the
+**   recovery operation - after all recoverable data has been inserted
+**   into the new database. This is faster overall, but means that the
+**   final call to sqlite3_recover_step() for a recovery operation may
+**   be need to create a large number of indexes, which may be very slow.
+**
+**   Or, if this option is set (argument is 1), then non-UNIQUE indexes
+**   are created in the output database before it is populated with 
+**   recovered data. This is slower overall, but avoids the slow call
+**   to sqlite3_recover_step() at the end of the recovery operation.
+**
+**   The default option value is 0.
+*/
+#define SQLITE_RECOVER_LOST_AND_FOUND   1
+#define SQLITE_RECOVER_FREELIST_CORRUPT 2
+#define SQLITE_RECOVER_ROWIDS           3
+#define SQLITE_RECOVER_SLOWINDEXES      4
+
+/*
+** Perform a unit of work towards the recovery operation. This function 
+** must normally be called multiple times to complete database recovery.
+**
+** If no error occurs but the recovery operation is not completed, this
+** function returns SQLITE_OK. If recovery has been completed successfully
+** then SQLITE_DONE is returned. If an error has occurred, then an SQLite
+** error code (e.g. SQLITE_IOERR or SQLITE_NOMEM) is returned. It is not
+** considered an error if some or all of the data cannot be recovered
+** due to database corruption.
+**
+** Once sqlite3_recover_step() has returned a value other than SQLITE_OK,
+** all further such calls on the same recover handle are no-ops that return
+** the same non-SQLITE_OK value.
+*/
+int sqlite3_recover_step(sqlite3_recover*);
+
+/* 
+** Run the recovery operation to completion. Return SQLITE_OK if successful,
+** or an SQLite error code otherwise. Calling this function is the same
+** as executing:
+**
+**     while( SQLITE_OK==sqlite3_recover_step(p) );
+**     return sqlite3_recover_errcode(p);
+*/
+int sqlite3_recover_run(sqlite3_recover*);
+
+/*
+** If an error has been encountered during a prior call to
+** sqlite3_recover_step(), then this function attempts to return a 
+** pointer to a buffer containing an English language explanation of 
+** the error. If no error message is available, or if an out-of memory 
+** error occurs while attempting to allocate a buffer in which to format
+** the error message, NULL is returned.
+**
+** The returned buffer remains valid until the sqlite3_recover handle is
+** destroyed using sqlite3_recover_finish().
+*/
+const char *sqlite3_recover_errmsg(sqlite3_recover*);
+
+/*
+** If this function is called on an sqlite3_recover handle after
+** an error occurs, an SQLite error code is returned. Otherwise, SQLITE_OK.
+*/
+int sqlite3_recover_errcode(sqlite3_recover*);
+
+/* 
+** Clean up a recovery object created by a call to sqlite3_recover_init().
+** The results of using a recovery object with any API after it has been
+** passed to this function are undefined.
+**
+** This function returns the same value as sqlite3_recover_errcode().
+*/
+int sqlite3_recover_finish(sqlite3_recover*);
+
+
+#ifdef __cplusplus
+}  /* end of the 'extern "C"' block */
+#endif
+
+#endif /* ifndef _SQLITE_RECOVER_H */
diff --git a/database/sqlite/sqlite_aclk.c b/database/sqlite/sqlite_aclk.c
index fedce50eb..1298045c2 100644
--- a/database/sqlite/sqlite_aclk.c
+++ b/database/sqlite/sqlite_aclk.c
@@ -65,9 +65,7 @@ static void aclk_database_enq_cmd(struct aclk_database_cmd *cmd)
     uv_mutex_unlock(&aclk_sync_config.cmd_mutex);
 
     /* wake up event loop */
-    int rc = uv_async_send(&aclk_sync_config.async);
-    if (unlikely(rc))
-        netdata_log_debug(D_ACLK_SYNC, "Failed to wake up event loop");
+    (void) uv_async_send(&aclk_sync_config.async);
 }
 
 enum {
@@ -226,14 +224,8 @@ static void sql_delete_aclk_table_list(char *host_guid)
     uuid_unparse_lower(host_uuid, host_str);
     uuid_unparse_lower_fix(&host_uuid, uuid_str);
 
-    netdata_log_debug(D_ACLK_SYNC, "Checking if I should delete aclk tables for node %s", host_str);
-
-    if (is_host_available(&host_uuid)) {
-        netdata_log_debug(D_ACLK_SYNC, "Host %s exists, not deleting aclk sync tables", host_str);
+    if (is_host_available(&host_uuid))
         return;
-    }
-
-    netdata_log_debug(D_ACLK_SYNC, "Host %s does NOT exist, can delete aclk sync tables", host_str);
 
     sqlite3_stmt *res = NULL;
     BUFFER *sql = buffer_create(ACLK_SYNC_QUERY_SIZE, &netdata_buffers_statistics.buffers_sqlite);
@@ -265,7 +257,6 @@ fail:
 
 static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unused, char **argv __maybe_unused, char **column __maybe_unused)
 {
-    netdata_log_debug(D_ACLK_SYNC,"Scheduling aclk sync table check for node %s", (char *) argv[0]);
     struct aclk_database_cmd cmd;
     memset(&cmd, 0, sizeof(cmd));
     cmd.opcode = ACLK_DATABASE_DELETE_HOST;
@@ -280,7 +271,6 @@ static int sql_check_aclk_table(void *data __maybe_unused, int argc __maybe_unus
 static void sql_check_aclk_table_list(void)
 {
     char *err_msg = NULL;
-    netdata_log_debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
     int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ACTIVE_LIST, sql_check_aclk_table, NULL, &err_msg);
     if (rc != SQLITE_OK) {
         error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
@@ -305,7 +295,6 @@ static int sql_maint_aclk_sync_database(void *data __maybe_unused, int argc __ma
 static void sql_maint_aclk_sync_database_all(void)
 {
     char *err_msg = NULL;
-    netdata_log_debug(D_ACLK_SYNC,"Cleaning tables for nodes that do not exist");
     int rc = sqlite3_exec_monitored(db_meta, SQL_SELECT_ACLK_ALERT_LIST, sql_maint_aclk_sync_database, NULL, &err_msg);
     if (rc != SQLITE_OK) {
         error_report("Query failed when trying to check for obsolete ACLK sync tables, %s", err_msg);
@@ -444,7 +433,6 @@ static void aclk_synchronization(void *arg __maybe_unused)
                     sql_process_queue_removed_alerts_to_aclk(cmd.param[0]);
                     break;
                 default:
-                    netdata_log_debug(D_ACLK_SYNC, "%s: default.", __func__);
                     break;
             }
             if (cmd.completion)
@@ -499,6 +487,16 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may
         rc = db_execute(db_meta, sql);
         if (unlikely(rc))
             error_report("Failed to create ACLK alert table index for host %s", host ? string2str(host->hostname) : host_guid);
+
+        snprintfz(sql, ACLK_SYNC_QUERY_SIZE -1, INDEX_ACLK_ALERT1, uuid_str, uuid_str);
+        rc = db_execute(db_meta, sql);
+        if (unlikely(rc))
+            error_report("Failed to create ACLK alert table index 1 for host %s", host ? string2str(host->hostname) : host_guid);
+
+        snprintfz(sql, ACLK_SYNC_QUERY_SIZE -1, INDEX_ACLK_ALERT2, uuid_str, uuid_str);
+        rc = db_execute(db_meta, sql);
+        if (unlikely(rc))
+            error_report("Failed to create ACLK alert table index 2 for host %s", host ? string2str(host->hostname) : host_guid);
     }
     if (likely(host) && unlikely(host->aclk_sync_host_config))
         return;
diff --git a/database/sqlite/sqlite_aclk.h b/database/sqlite/sqlite_aclk.h
index 705102d74..850ca434e 100644
--- a/database/sqlite/sqlite_aclk.h
+++ b/database/sqlite/sqlite_aclk.h
@@ -46,6 +46,9 @@ static inline int claimed()
         "unique(alert_unique_id));"
 
 #define INDEX_ACLK_ALERT "CREATE INDEX IF NOT EXISTS aclk_alert_index_%s ON aclk_alert_%s (alert_unique_id);"
+#define INDEX_ACLK_ALERT1 "CREATE INDEX IF NOT EXISTS aclk_alert_index1_%s ON aclk_alert_%s (filtered_alert_unique_id);"
+#define INDEX_ACLK_ALERT2 "CREATE INDEX IF NOT EXISTS aclk_alert_index2_%s ON aclk_alert_%s (date_submitted);"
+
 enum aclk_database_opcode {
     ACLK_DATABASE_NOOP = 0,
 
diff --git a/database/sqlite/sqlite_aclk_alert.c b/database/sqlite/sqlite_aclk_alert.c
index 20ca0573d..e25b0f0ec 100644
--- a/database/sqlite/sqlite_aclk_alert.c
+++ b/database/sqlite/sqlite_aclk_alert.c
@@ -7,43 +7,58 @@
 #include "../../aclk/aclk_alarm_api.h"
 #endif
 
-#define SQL_UPDATE_FILTERED_ALERT "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u, date_created = unixepoch() where filtered_alert_unique_id = %u"
-void update_filtered(ALARM_ENTRY *ae, uint32_t unique_id, char *uuid_str) {
+#define SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param)                                                                     \
+    ({                                                                                                                 \
+        int _param = (param);                                                                                          \
+        sqlite3_column_bytes((res), (_param)) ? strdupz((char *)sqlite3_column_text((res), (_param))) : NULL;          \
+    })
+
+
+#define SQL_UPDATE_FILTERED_ALERT                                                                                      \
+    "UPDATE aclk_alert_%s SET filtered_alert_unique_id = %u, date_created = unixepoch() where filtered_alert_unique_id = %u"
+
+static void update_filtered(ALARM_ENTRY *ae, uint32_t unique_id, char *uuid_str)
+{
     char sql[ACLK_SYNC_QUERY_SIZE];
     snprintfz(sql, ACLK_SYNC_QUERY_SIZE-1, SQL_UPDATE_FILTERED_ALERT, uuid_str, ae->unique_id, unique_id);
     sqlite3_exec_monitored(db_meta, sql, 0, 0, NULL);
     ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
 }
 
-#define SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID "SELECT hld.unique_id FROM health_log hl, alert_hash ah, health_log_detail hld WHERE hld.unique_id = %u " \
-                            "AND hl.config_hash_id = ah.hash_id AND hld.health_log_id = hl.health_log_id AND host_id = @host_id " \
-                            "AND ah.warn IS NULL AND ah.crit IS NULL;"
-static inline bool is_event_from_alert_variable_config(uint32_t unique_id, uuid_t *host_id) {
+#define SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID                                                                         \
+    "SELECT hld.unique_id FROM health_log hl, alert_hash ah, health_log_detail hld "                                   \
+    "WHERE hld.unique_id = @unique_id AND hl.config_hash_id = ah.hash_id AND hld.health_log_id = hl.health_log_id "    \
+    "AND hl.host_id = @host_id AND ah.warn IS NULL AND ah.crit IS NULL"
+
+static inline bool is_event_from_alert_variable_config(uint32_t unique_id, uuid_t *host_id)
+{
     sqlite3_stmt *res = NULL;
     int rc = 0;
     bool ret = false;
 
-    char sql[ACLK_SYNC_QUERY_SIZE];
-    snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID, unique_id);
-
-    rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
+    rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_VARIABLE_ALERT_BY_UNIQUE_ID, -1, &res, 0);
     if (rc != SQLITE_OK) {
         error_report("Failed to prepare statement when trying to check for alert variables.");
         return false;
     }
 
-    rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
+    rc = sqlite3_bind_int(res, 1, (int) unique_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind unique_id for checking alert variable.");
+        goto fail;
+    }
+
+    rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind host_id for checking alert variable.");
-        sqlite3_finalize(res);
-        return false;
+        goto fail;
     }
 
     rc = sqlite3_step_monitored(res);
-    if (likely(rc == SQLITE_ROW)) {
+    if (likely(rc == SQLITE_ROW))
         ret = true;
-    }
 
+fail:
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
         error_report("Failed to finalize statement when trying to check for alert variables, rc = %d", rc);
@@ -54,20 +69,22 @@ static inline bool is_event_from_alert_variable_config(uint32_t unique_id, uuid_
 #define MAX_REMOVED_PERIOD 604800 //a week
 
 //decide if some events should be sent or not
-#define SQL_SELECT_ALERT_BY_ID  "SELECT hld.new_status, hl.config_hash_id, hld.unique_id FROM health_log hl, aclk_alert_%s aa, health_log_detail hld " \
-                                "WHERE hld.unique_id = aa.filtered_alert_unique_id " \
-                                "AND hld.alarm_id = %u AND hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id " \
-                                "ORDER BY hld.alarm_event_id DESC LIMIT 1;"
-int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
+#define SQL_SELECT_ALERT_BY_ID                                                                                             \
+    "SELECT hld.new_status, hl.config_hash_id, hld.unique_id FROM health_log hl, aclk_alert_%s aa, health_log_detail hld " \
+    "WHERE hl.host_id = @host_id AND hld.unique_id = aa.filtered_alert_unique_id "                                         \
+    "AND hld.alarm_id = @alarm_id AND hl.health_log_id = hld.health_log_id "                                               \
+    "ORDER BY hld.alarm_event_id DESC LIMIT 1;"
+
+static bool should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
 {
     sqlite3_stmt *res = NULL;
     char uuid_str[UUID_STR_LEN];
     uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
-    int send = 1;
 
-    if (ae->new_status == RRDCALC_STATUS_REMOVED || ae->new_status == RRDCALC_STATUS_UNINITIALIZED) {
+    bool send = false;
+
+    if (ae->new_status == RRDCALC_STATUS_REMOVED || ae->new_status == RRDCALC_STATUS_UNINITIALIZED)
         return 0;
-    }
 
     if (unlikely(uuid_is_null(ae->config_hash_id)))
         return 0;
@@ -79,92 +96,80 @@ int should_send_to_cloud(RRDHOST *host, ALARM_ENTRY *ae)
 
     //get the previous sent event of this alarm_id
     //base the search on the last filtered event
-    snprintfz(sql,ACLK_SYNC_QUERY_SIZE-1, SQL_SELECT_ALERT_BY_ID, uuid_str, ae->alarm_id);
+    snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_SELECT_ALERT_BY_ID, uuid_str);
 
     int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
     if (rc != SQLITE_OK) {
-        error_report("Failed to prepare statement when trying to filter alert events.");
-        send = 1;
-        return send;
+        error_report("Failed to prepare statement when trying should_send_to_cloud.");
+        return true;
     }
 
     rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id for checking alert variable.");
-        sqlite3_finalize(res);
-        return false;
-    }
-
-    rc = sqlite3_step_monitored(res);
-    if (likely(rc == SQLITE_ROW)) {
-        status  = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
-        if (sqlite3_column_type(res, 1) != SQLITE_NULL)
-            uuid_copy(config_hash_id, *((uuid_t *) sqlite3_column_blob(res, 1)));
-        unique_id = (uint32_t) sqlite3_column_int64(res, 2);
-    } else {
-        send = 1;
+        error_report("Failed to bind host_id for checking should_send_to_cloud");
         goto done;
     }
 
-    if (ae->new_status != (RRDCALC_STATUS)status) {
-        send = 1;
+    rc = sqlite3_bind_int(res, 2, (int) ae->alarm_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind alarm_id for checking should_send_to_cloud");
         goto done;
     }
 
-    if (uuid_memcmp(&ae->config_hash_id, &config_hash_id)) {
-        send = 1;
-        goto done;
-    }
+    rc = sqlite3_step_monitored(res);
 
-    //same status, same config
-    send = 0;
-    update_filtered(ae, unique_id, uuid_str);
+    if (likely(rc == SQLITE_ROW)) {
+        status = (RRDCALC_STATUS)sqlite3_column_int(res, 0);
+
+        if (sqlite3_column_type(res, 1) != SQLITE_NULL)
+            uuid_copy(config_hash_id, *((uuid_t *)sqlite3_column_blob(res, 1)));
+
+        unique_id = (uint32_t)sqlite3_column_int64(res, 2);
+
+        if (ae->new_status != (RRDCALC_STATUS)status || uuid_memcmp(&ae->config_hash_id, &config_hash_id))
+            send = true;
+        else
+            update_filtered(ae, unique_id, uuid_str);
+    } else
+        send = true;
 
 done:
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize statement when trying to filter alert events, rc = %d", rc);
+        error_report("Failed to finalize statement when trying should_send_to_cloud, rc = %d", rc);
 
     return send;
 }
 
-#define SQL_QUEUE_ALERT_TO_CLOUD "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
-                            "VALUES (@alert_unique_id, unixepoch(), @alert_unique_id) ON CONFLICT (alert_unique_id) do nothing;"
-int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
-{
-    if(!service_running(SERVICE_ACLK))
-        return 0;
-
-    if (!claimed())
-        return 0;
+#define SQL_QUEUE_ALERT_TO_CLOUD                                                                                       \
+    "INSERT INTO aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) "                             \
+    "VALUES (@alert_unique_id, UNIXEPOCH(), @alert_unique_id) ON CONFLICT (alert_unique_id) DO NOTHING;"
 
-    if (ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED) {
-        return 0;
-    }
+void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, bool skip_filter)
+{
+    sqlite3_stmt *res_alert = NULL;
+    char sql[ACLK_SYNC_QUERY_SIZE];
+    char uuid_str[UUID_STR_LEN];
 
-    CHECK_SQLITE_CONNECTION(db_meta);
+    if (!service_running(SERVICE_ACLK))
+        return;
 
-    if (!skip_filter) {
-        if (!should_send_to_cloud(host, ae)) {
-            return 0;
-        }
-    }
+    if (!claimed() || ae->flags & HEALTH_ENTRY_FLAG_ACLK_QUEUED)
+        return;
 
-    char uuid_str[UUID_STR_LEN];
-    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
+    if (false == skip_filter && !should_send_to_cloud(host, ae))
+            return;
 
     if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid))
-        return 0;
-
-    sqlite3_stmt *res_alert = NULL;
-    char sql[ACLK_SYNC_QUERY_SIZE];
+        return;
 
+    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
     snprintfz(sql, ACLK_SYNC_QUERY_SIZE - 1, SQL_QUEUE_ALERT_TO_CLOUD, uuid_str);
 
     int rc = sqlite3_prepare_v2(db_meta, sql, -1, &res_alert, 0);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to prepare statement to store alert event");
-        return 1;
+        return;
     }
 
     rc = sqlite3_bind_int(res_alert, 1, (int) ae->unique_id);
@@ -172,19 +177,15 @@ int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter)
         goto bind_fail;
 
     rc = execute_insert(res_alert);
-    if (unlikely(rc != SQLITE_DONE)) {
+    if (unlikely(rc == SQLITE_DONE)) {
+        ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
+        rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
+    } else
         error_report("Failed to store alert event %u, rc = %d", ae->unique_id, rc);
-        goto bind_fail;
-    }
-
-    ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
-    rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
 
 bind_fail:
     if (unlikely(sqlite3_finalize(res_alert) != SQLITE_OK))
         error_report("Failed to reset statement in store alert event, rc = %d", rc);
-
-    return 0;
 }
 
 int rrdcalc_status_to_proto_enum(RRDCALC_STATUS status)
@@ -246,7 +247,10 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc)
     int rc;
 
     if (unlikely(!wc->alert_updates)) {
-        netdata_log_access("ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A");
+        netdata_log_access(
+            "ACLK STA [%s (%s)]: Ignoring alert push event, updates have been turned off for this node.",
+            wc->node_id,
+            wc->host ? rrdhost_hostname(wc->host) : "N/A");
         return;
     }
 
@@ -265,23 +269,30 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc)
 
     sqlite3_stmt *res = NULL;
 
-    buffer_sprintf(sql, "select aa.sequence_id, hld.unique_id, hld.alarm_id, hl.config_hash_id, hld.updated_by_id, hld.when_key, " \
-        " hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name,  " \
-        " hl.chart, hl.family, hl.exec, hl.recipient, ha.source, hl.units, hld.info, hld.exec_code, hld.new_status,  " \
-        " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id, hld.alarm_event_id, hl.chart_name  " \
-        " from health_log hl, aclk_alert_%s aa, alert_hash ha, health_log_detail hld " \
-        " where hld.unique_id = aa.alert_unique_id and hl.config_hash_id = ha.hash_id and aa.date_submitted is null " \
-        " and hl.host_id = @host_id and hl.health_log_id = hld.health_log_id " \
-        " order by aa.sequence_id asc limit %d;", wc->uuid_str, limit);
+    buffer_sprintf(
+        sql,
+        "select aa.sequence_id, hld.unique_id, hld.alarm_id, hl.config_hash_id, hld.updated_by_id, hld.when_key, "
+        " hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name,  "
+        " hl.chart, hl.exec, hl.recipient, ha.source, hl.units, hld.info, hld.exec_code, hld.new_status,  "
+        " hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, hl.chart_context, hld.transition_id, "
+        " hld.alarm_event_id, hl.chart_name, hld.summary  "
+        " from health_log hl, aclk_alert_%s aa, alert_hash ha, health_log_detail hld "
+        " where hld.unique_id = aa.alert_unique_id and hl.config_hash_id = ha.hash_id and aa.date_submitted is null "
+        " and hl.host_id = @host_id and hl.health_log_id = hld.health_log_id "
+        " order by aa.sequence_id asc limit %d;",
+        wc->uuid_str,
+        limit);
 
     rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
     if (rc != SQLITE_OK) {
 
         BUFFER *sql_fix = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
         buffer_sprintf(sql_fix, TABLE_ACLK_ALERT, wc->uuid_str);
+
         rc = db_execute(db_meta, buffer_tostring(sql_fix));
         if (unlikely(rc))
             error_report("Failed to create ACLK alert table for host %s", rrdhost_hostname(wc->host));
+
         else {
             buffer_flush(sql_fix);
             buffer_sprintf(sql_fix, INDEX_ACLK_ALERT, wc->uuid_str, wc->uuid_str);
@@ -320,63 +331,52 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc)
 
         alarm_log.node_id = wc->node_id;
         alarm_log.claim_id = claim_id;
-
         alarm_log.chart = strdupz((char *)sqlite3_column_text(res, 12));
         alarm_log.name = strdupz((char *)sqlite3_column_text(res, 11));
-        alarm_log.family = sqlite3_column_bytes(res, 13) > 0 ? strdupz((char *)sqlite3_column_text(res, 13)) : NULL;
-
-        //alarm_log.batch_id = wc->alerts_batch_id;
-        //alarm_log.sequence_id = (uint64_t) sqlite3_column_int64(res, 0);
         alarm_log.when = (time_t) sqlite3_column_int64(res, 5);
-
         alarm_log.config_hash = sqlite3_uuid_unparse_strdupz(res, 3);
-
         alarm_log.utc_offset = wc->host->utc_offset;
         alarm_log.timezone = strdupz(rrdhost_abbrev_timezone(wc->host));
-        alarm_log.exec_path = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) :
+        alarm_log.exec_path = sqlite3_column_bytes(res, 13) > 0 ? strdupz((char *)sqlite3_column_text(res, 13)) :
                                                                   strdupz((char *)string2str(wc->host->health.health_default_exec));
+        alarm_log.conf_source = sqlite3_column_bytes(res, 15) > 0 ? strdupz((char *)sqlite3_column_text(res, 15)) : strdupz("");
 
-        alarm_log.conf_source = sqlite3_column_bytes(res, 16) > 0 ? strdupz((char *)sqlite3_column_text(res, 16)) : strdupz("");
-
-        char *edit_command = sqlite3_column_bytes(res, 16) > 0 ?
-                                 health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) :
+        char *edit_command = sqlite3_column_bytes(res, 15) > 0 ?
+                                 health_edit_command_from_source((char *)sqlite3_column_text(res, 15)) :
                                  strdupz("UNKNOWN=0=UNKNOWN");
         alarm_log.command = strdupz(edit_command);
 
         alarm_log.duration = (time_t) sqlite3_column_int64(res, 6);
         alarm_log.non_clear_duration = (time_t) sqlite3_column_int64(res, 7);
-        alarm_log.status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 20));
-        alarm_log.old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 21));
-        alarm_log.delay = (int) sqlite3_column_int(res, 22);
+        alarm_log.status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 19));
+        alarm_log.old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 20));
+        alarm_log.delay = (int) sqlite3_column_int(res, 21);
         alarm_log.delay_up_to_timestamp = (time_t) sqlite3_column_int64(res, 10);
-        alarm_log.last_repeat = (time_t) sqlite3_column_int64(res, 25);
-
+        alarm_log.last_repeat = (time_t) sqlite3_column_int64(res, 24);
         alarm_log.silenced = ((sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_SILENCED) ||
-                              (sqlite3_column_type(res, 15) != SQLITE_NULL &&
-                               !strncmp((char *)sqlite3_column_text(res, 15), "silent", 6))) ?
+                              (sqlite3_column_type(res, 14) != SQLITE_NULL &&
+                               !strncmp((char *)sqlite3_column_text(res, 14), "silent", 6))) ?
                                  1 :
                                  0;
-
         alarm_log.value_string =
-            sqlite3_column_type(res, 23) == SQLITE_NULL ?
+            sqlite3_column_type(res, 22) == SQLITE_NULL ?
                 strdupz((char *)"-") :
                 strdupz((char *)format_value_and_unit(
-                    new_value_string, 100, sqlite3_column_double(res, 23), (char *)sqlite3_column_text(res, 17), -1));
-
+                    new_value_string, 100, sqlite3_column_double(res, 22), (char *)sqlite3_column_text(res, 16), -1));
         alarm_log.old_value_string =
-            sqlite3_column_type(res, 24) == SQLITE_NULL ?
+            sqlite3_column_type(res, 23) == SQLITE_NULL ?
                 strdupz((char *)"-") :
                 strdupz((char *)format_value_and_unit(
-                    old_value_string, 100, sqlite3_column_double(res, 24), (char *)sqlite3_column_text(res, 17), -1));
-
-        alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);
-        alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 24);
+                    old_value_string, 100, sqlite3_column_double(res, 23), (char *)sqlite3_column_text(res, 16), -1));
+        alarm_log.value = (NETDATA_DOUBLE) sqlite3_column_double(res, 22);
+        alarm_log.old_value = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);
         alarm_log.updated = (sqlite3_column_int64(res, 8) & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;
-        alarm_log.rendered_info = sqlite3_text_strdupz_empty(res, 18);
-        alarm_log.chart_context = sqlite3_text_strdupz_empty(res, 26);
-        alarm_log.transition_id = sqlite3_uuid_unparse_strdupz(res, 27);
-        alarm_log.event_id = (time_t) sqlite3_column_int64(res, 28);
-        alarm_log.chart_name = sqlite3_text_strdupz_empty(res, 29);
+        alarm_log.rendered_info = sqlite3_text_strdupz_empty(res, 17);
+        alarm_log.chart_context = sqlite3_text_strdupz_empty(res, 25);
+        alarm_log.transition_id = sqlite3_uuid_unparse_strdupz(res, 26);
+        alarm_log.event_id = (time_t) sqlite3_column_int64(res, 27);
+        alarm_log.chart_name = sqlite3_text_strdupz_empty(res, 28);
+        alarm_log.summary = sqlite3_text_strdupz_empty(res, 29);
 
         aclk_send_alarm_log_entry(&alarm_log);
 
@@ -396,7 +396,7 @@ void aclk_push_alert_event(struct aclk_sync_host_config *wc)
     if (first_sequence_id) {
         buffer_flush(sql);
         buffer_sprintf(sql, "UPDATE aclk_alert_%s SET date_submitted=unixepoch() "
-                            "WHERE date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
+                            "WHERE +date_submitted IS NULL AND sequence_id BETWEEN %" PRIu64 " AND %" PRIu64 ";",
                        wc->uuid_str, first_sequence_id, last_sequence_id);
 
         if (unlikely(db_execute(db_meta, buffer_tostring(sql))))
@@ -431,10 +431,10 @@ void aclk_push_alert_events_for_all_hosts(void)
     RRDHOST *host;
 
     dfe_start_reentrant(rrdhost_root_index, host) {
-        if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))
+        if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) ||
+            !rrdhost_flag_check(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS))
             continue;
 
-        internal_error(true, "ACLK SYNC: Scanning host %s", rrdhost_hostname(host));
         rrdhost_flag_clear(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
 
         struct aclk_sync_host_config *wc = host->aclk_sync_host_config;
@@ -462,10 +462,13 @@ void sql_queue_existing_alerts_to_aclk(RRDHOST *host)
     }
 
     buffer_flush(sql);
-    buffer_sprintf(sql, "insert into aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) " \
-                   "select hld.unique_id alert_unique_id, unixepoch(), hld.unique_id alert_unique_id from health_log_detail hld, health_log hl " \
-                   "where hld.new_status <> 0 and hld.new_status <> -2 and hl.health_log_id = hld.health_log_id and hl.config_hash_id is not null " \
-                   "and hld.updated_by_id = 0 and hl.host_id = @host_id order by hld.unique_id asc on conflict (alert_unique_id) do nothing;", uuid_str);
+    buffer_sprintf(
+        sql,
+        "insert into aclk_alert_%s (alert_unique_id, date_created, filtered_alert_unique_id) "
+        "select hld.unique_id alert_unique_id, unixepoch(), hld.unique_id alert_unique_id from health_log_detail hld, health_log hl "
+        "where hld.new_status <> 0 and hld.new_status <> -2 and hl.health_log_id = hld.health_log_id and hl.config_hash_id is not null "
+        "and hld.updated_by_id = 0 and hl.host_id = @host_id order by hld.unique_id asc on conflict (alert_unique_id) do nothing;",
+        uuid_str);
 
     rc = sqlite3_prepare_v2(db_meta, buffer_tostring(sql), -1, &res, 0);
     if (rc != SQLITE_OK) {
@@ -485,9 +488,8 @@ void sql_queue_existing_alerts_to_aclk(RRDHOST *host)
     }
 
     rc = execute_insert(res);
-    if (unlikely(rc != SQLITE_DONE)) {
+    if (unlikely(rc != SQLITE_DONE))
         error_report("Failed to queue existing alerts, rc = %d", rc);
-    }
 
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
@@ -509,15 +511,21 @@ void aclk_send_alarm_configuration(char *config_hash)
     if (unlikely(!wc))
         return;
 
-    netdata_log_access("ACLK REQ [%s (%s)]: Request to send alert config %s.", wc->node_id, wc->host ? rrdhost_hostname(wc->host) : "N/A", config_hash);
+    netdata_log_access(
+        "ACLK REQ [%s (%s)]: Request to send alert config %s.",
+        wc->node_id,
+        wc->host ? rrdhost_hostname(wc->host) : "N/A",
+        config_hash);
 
     aclk_push_alert_config(wc->node_id, config_hash);
 }
 
-#define SQL_SELECT_ALERT_CONFIG "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin," \
-    "module, charts, families, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info," \
-    "options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after," \
-    "p_db_lookup_before, p_update_every, chart_labels FROM alert_hash WHERE hash_id = @hash_id;"
+#define SQL_SELECT_ALERT_CONFIG                                                                                        \
+    "SELECT alarm, template, on_key, class, type, component, os, hosts, plugin,"                                       \
+    "module, charts, lookup, every, units, green, red, calc, warn, crit, to_key, exec, delay, repeat, info,"           \
+    "options, host_labels, p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after,"        \
+    "p_db_lookup_before, p_update_every, chart_labels, summary FROM alert_hash WHERE hash_id = @hash_id;"
+
 int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash __maybe_unused)
 {
     int rc = 0;
@@ -564,37 +572,33 @@ int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash
 
     if (sqlite3_step_monitored(res) == SQLITE_ROW) {
 
-        alarm_config.alarm = sqlite3_column_bytes(res, 0) > 0 ? strdupz((char *)sqlite3_column_text(res, 0)) : NULL;
-        alarm_config.tmpl = sqlite3_column_bytes(res, 1) > 0 ? strdupz((char *)sqlite3_column_text(res, 1)) : NULL;
-        alarm_config.on_chart = sqlite3_column_bytes(res, 2) > 0 ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;
-        alarm_config.classification = sqlite3_column_bytes(res, 3) > 0 ? strdupz((char *)sqlite3_column_text(res, 3)) : NULL;
-        alarm_config.type = sqlite3_column_bytes(res, 4) > 0 ? strdupz((char *)sqlite3_column_text(res, 4)) : NULL;
-        alarm_config.component = sqlite3_column_bytes(res, 5) > 0 ? strdupz((char *)sqlite3_column_text(res, 5)) : NULL;
-
-        alarm_config.os = sqlite3_column_bytes(res, 6) > 0 ? strdupz((char *)sqlite3_column_text(res, 6)) : NULL;
-        alarm_config.hosts = sqlite3_column_bytes(res, 7) > 0 ? strdupz((char *)sqlite3_column_text(res, 7)) : NULL;
-        alarm_config.plugin = sqlite3_column_bytes(res, 8) > 0 ? strdupz((char *)sqlite3_column_text(res, 8)) : NULL;
-        alarm_config.module = sqlite3_column_bytes(res, 9) > 0 ? strdupz((char *)sqlite3_column_text(res, 9)) : NULL;
-        alarm_config.charts = sqlite3_column_bytes(res, 10) > 0 ? strdupz((char *)sqlite3_column_text(res, 10)) : NULL;
-        alarm_config.families = sqlite3_column_bytes(res, 11) > 0 ? strdupz((char *)sqlite3_column_text(res, 11)) : NULL;
-        alarm_config.lookup = sqlite3_column_bytes(res, 12) > 0 ? strdupz((char *)sqlite3_column_text(res, 12)) : NULL;
-        alarm_config.every = sqlite3_column_bytes(res, 13) > 0 ? strdupz((char *)sqlite3_column_text(res, 13)) : NULL;
-        alarm_config.units = sqlite3_column_bytes(res, 14) > 0 ? strdupz((char *)sqlite3_column_text(res, 14)) : NULL;
-
-        alarm_config.green = sqlite3_column_bytes(res, 15) > 0 ? strdupz((char *)sqlite3_column_text(res, 15)) : NULL;
-        alarm_config.red = sqlite3_column_bytes(res, 16) > 0 ? strdupz((char *)sqlite3_column_text(res, 16)) : NULL;
-
-        alarm_config.calculation_expr = sqlite3_column_bytes(res, 17) > 0 ? strdupz((char *)sqlite3_column_text(res, 17)) : NULL;
-        alarm_config.warning_expr = sqlite3_column_bytes(res, 18) > 0 ? strdupz((char *)sqlite3_column_text(res, 18)) : NULL;
-        alarm_config.critical_expr = sqlite3_column_bytes(res, 19) > 0 ? strdupz((char *)sqlite3_column_text(res, 19)) : NULL;
-
-        alarm_config.recipient = sqlite3_column_bytes(res, 20) > 0 ? strdupz((char *)sqlite3_column_text(res, 20)) : NULL;
-        alarm_config.exec = sqlite3_column_bytes(res, 21) > 0 ? strdupz((char *)sqlite3_column_text(res, 21)) : NULL;
-        alarm_config.delay = sqlite3_column_bytes(res, 22) > 0 ? strdupz((char *)sqlite3_column_text(res, 22)) : NULL;
-        alarm_config.repeat = sqlite3_column_bytes(res, 23) > 0 ? strdupz((char *)sqlite3_column_text(res, 23)) : NULL;
-        alarm_config.info = sqlite3_column_bytes(res, 24) > 0 ? strdupz((char *)sqlite3_column_text(res, 24)) : NULL;
-        alarm_config.options = sqlite3_column_bytes(res, 25) > 0 ? strdupz((char *)sqlite3_column_text(res, 25)) : NULL;
-        alarm_config.host_labels = sqlite3_column_bytes(res, 26) > 0 ? strdupz((char *)sqlite3_column_text(res, 26)) : NULL;
+        int param = 0;
+        alarm_config.alarm = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.tmpl = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.on_chart = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.classification = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.type = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.component = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.os = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.hosts = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.plugin = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.module = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.charts = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.lookup = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.every = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.units = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.green = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.red = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.calculation_expr = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.warning_expr = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.critical_expr = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.recipient = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.exec = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.delay = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.repeat = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.info = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.options = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);
+        alarm_config.host_labels = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);  // Current param 25
 
         alarm_config.p_db_lookup_dimensions = NULL;
         alarm_config.p_db_lookup_method = NULL;
@@ -602,23 +606,26 @@ int aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_hash
         alarm_config.p_db_lookup_after = 0;
         alarm_config.p_db_lookup_before = 0;
 
-        if (sqlite3_column_bytes(res, 30) > 0) {
+        if (sqlite3_column_bytes(res, 29) > 0) {
 
-            alarm_config.p_db_lookup_dimensions = sqlite3_column_bytes(res, 27) > 0 ? strdupz((char *)sqlite3_column_text(res, 27)) : NULL;
-            alarm_config.p_db_lookup_method = sqlite3_column_bytes(res, 28) > 0 ? strdupz((char *)sqlite3_column_text(res, 28)) : NULL;
+            alarm_config.p_db_lookup_dimensions = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);  // Current param 26
+            alarm_config.p_db_lookup_method = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, param++);      // Current param 27
+            if (param != 28)
+                netdata_log_error("aclk_push_alert_config_event: Unexpected param number %d", param);
 
             BUFFER *tmp_buf = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
-            buffer_data_options2string(tmp_buf, sqlite3_column_int(res, 29));
+            buffer_data_options2string(tmp_buf, sqlite3_column_int(res, 28));
             alarm_config.p_db_lookup_options = strdupz((char *)buffer_tostring(tmp_buf));
             buffer_free(tmp_buf);
 
-            alarm_config.p_db_lookup_after = sqlite3_column_int(res, 30);
-            alarm_config.p_db_lookup_before = sqlite3_column_int(res, 31);
+            alarm_config.p_db_lookup_after = sqlite3_column_int(res, 29);
+            alarm_config.p_db_lookup_before = sqlite3_column_int(res, 30);
         }
 
-        alarm_config.p_update_every = sqlite3_column_int(res, 32);
+        alarm_config.p_update_every = sqlite3_column_int(res, 31);
 
-        alarm_config.chart_labels = sqlite3_column_bytes(res, 33) > 0 ? strdupz((char *)sqlite3_column_text(res, 33)) : NULL;
+        alarm_config.chart_labels = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, 32);
+        alarm_config.summary = SQLITE3_COLUMN_STRDUPZ_OR_NULL(res, 33);
 
         p_alarm_config.cfg_hash = strdupz((char *) config_hash);
         p_alarm_config.cfg = alarm_config;
@@ -648,11 +655,9 @@ bind_fail:
 // Start streaming alerts
 void aclk_start_alert_streaming(char *node_id, bool resets)
 {
-    if (unlikely(!node_id))
-        return;
-
     uuid_t node_uuid;
-    if (uuid_parse(node_id, node_uuid))
+
+    if (unlikely(!node_id || uuid_parse(node_id, node_uuid)))
         return;
 
     RRDHOST *host = find_host_by_node_id(node_id);
@@ -685,6 +690,7 @@ void aclk_start_alert_streaming(char *node_id, bool resets)
     "WHERE hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id AND hld.new_status = -2 AND hld.updated_by_id = 0 " \
     "AND hld.unique_id NOT IN (SELECT alert_unique_id FROM aclk_alert_%s) " \
     "AND hl.config_hash_id NOT IN (select hash_id from alert_hash where warn is null and crit is null) " \
+    "AND hl.name || hl.chart NOT IN (select name || chart from health_log where name = hl.name and chart = hl.chart and alarm_id > hl.alarm_id and host_id = hl.host_id) " \
     "ORDER BY hld.unique_id ASC ON CONFLICT (alert_unique_id) DO NOTHING;"
 void sql_process_queue_removed_alerts_to_aclk(char *node_id)
 {
@@ -788,7 +794,6 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
 
     alarm_log->chart = strdupz(ae_chart_id(ae));
     alarm_log->name = strdupz(ae_name(ae));
-    alarm_log->family = strdupz(ae_family(ae));
 
     alarm_log->batch_id = 0;
     alarm_log->sequence_id = 0;
@@ -830,6 +835,8 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
     alarm_log->transition_id = strdupz((char *)transition_id);
     alarm_log->event_id = (uint64_t) ae->alarm_event_id;
 
+    alarm_log->summary = strdupz(ae_summary(ae));
+
     freez(edit_command);
 }
 #endif
diff --git a/database/sqlite/sqlite_aclk_alert.h b/database/sqlite/sqlite_aclk_alert.h
index d7252aad6..c92aef083 100644
--- a/database/sqlite/sqlite_aclk_alert.h
+++ b/database/sqlite/sqlite_aclk_alert.h
@@ -15,7 +15,6 @@ struct proto_alert_status {
     uint64_t last_submitted_sequence_id;
 };
 
-int aclk_add_alert_event(struct aclk_sync_host_config *wc, struct aclk_database_cmd cmd);
 void aclk_push_alert_event(struct aclk_sync_host_config *wc);
 void aclk_send_alarm_configuration (char *config_hash);
 int aclk_push_alert_config_event(char *node_id, char *config_hash);
@@ -28,8 +27,7 @@ void aclk_push_alarm_checkpoint(RRDHOST *host);
 void aclk_push_alert_snapshot_event(char *node_id);
 void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, char *snapshot_uuid);
 int get_proto_alert_status(RRDHOST *host, struct proto_alert_status *proto_alert_status);
-int sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, int skip_filter);
+void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, bool skip_filter);
 void aclk_push_alert_events_for_all_hosts(void);
 
-
 #endif //NETDATA_SQLITE_ACLK_ALERT_H
diff --git a/database/sqlite/sqlite_context.c b/database/sqlite/sqlite_context.c
index f29fe51e3..d4b15e99d 100644
--- a/database/sqlite/sqlite_context.c
+++ b/database/sqlite/sqlite_context.c
@@ -52,51 +52,20 @@ int sql_init_context_database(int memory)
     if (likely(!memory))
         target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION);
 
-    // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
-    // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
-    snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_synchronous
-    // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
-    snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list))  return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_journal_mode
-    // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
-    snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_temp_store
-    // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
-    snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
-    
-    // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
-    // PRAGMA schema.journal_size_limit = N ;
-    snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_cache_size
-    // PRAGMA schema.cache_size = pages;
-    // PRAGMA schema.cache_size = -kibibytes;
-    snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
-
-    snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
+    if (configure_sqlite_database(db_context_meta, target_version))
+        return 1;
 
     if (likely(!memory))
         snprintfz(buf, 1024, "ATTACH DATABASE \"%s/netdata-meta.db\" as meta;", netdata_configured_cache_dir);
     else
         snprintfz(buf, 1024, "ATTACH DATABASE ':memory:' as meta;");
 
-    if(init_database_batch(db_context_meta, DB_CHECK_NONE, 0, list)) return 1;
+    if(init_database_batch(db_context_meta, list)) return 1;
 
-    if (init_database_batch(db_context_meta, DB_CHECK_NONE, 0, &database_context_config[0]))
+    if (init_database_batch(db_context_meta, &database_context_config[0]))
         return 1;
 
-    if (init_database_batch(db_context_meta, DB_CHECK_NONE, 0, &database_context_cleanup[0]))
+    if (init_database_batch(db_context_meta, &database_context_cleanup[0]))
         return 1;
 
     return 0;
diff --git a/database/sqlite/sqlite_db_migration.c b/database/sqlite/sqlite_db_migration.c
index 548b7de07..a011d0fef 100644
--- a/database/sqlite/sqlite_db_migration.c
+++ b/database/sqlite/sqlite_db_migration.c
@@ -11,7 +11,40 @@ static int return_int_cb(void *data, int argc, char **argv, char **column)
     return 0;
 }
 
-int table_exists_in_database(const char *table)
+static int get_auto_vaccum(sqlite3 *database)
+{
+    char *err_msg = NULL;
+    char sql[128];
+
+    int exists = 0;
+
+    snprintf(sql, 127, "PRAGMA auto_vacuum");
+
+    int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
+    if (rc != SQLITE_OK) {
+        netdata_log_info("Error checking database auto vacuum setting; %s", err_msg);
+        sqlite3_free(err_msg);
+    }
+
+    return exists;
+}
+
+int db_table_count(sqlite3 *database)
+{
+    char *err_msg = NULL;
+    char sql[128];
+
+    int count = 0;
+    snprintf(sql, 127, "select count(1) from sqlite_schema where type = 'table'");
+    int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &count, &err_msg);
+    if (rc != SQLITE_OK) {
+        netdata_log_info("Error checking database table count; %s", err_msg);
+        sqlite3_free(err_msg);
+    }
+    return count;
+}
+
+int table_exists_in_database(sqlite3 *database, const char *table)
 {
     char *err_msg = NULL;
     char sql[128];
@@ -20,7 +53,7 @@ int table_exists_in_database(const char *table)
 
     snprintf(sql, 127, "select 1 from sqlite_schema where type = 'table' and name = '%s';", table);
 
-    int rc = sqlite3_exec_monitored(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
+    int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
     if (rc != SQLITE_OK) {
         netdata_log_info("Error checking table existence; %s", err_msg);
         sqlite3_free(err_msg);
@@ -29,7 +62,7 @@ int table_exists_in_database(const char *table)
     return exists;
 }
 
-static int column_exists_in_table(const char *table, const char *column)
+static int column_exists_in_table(sqlite3 *database, const char *table, const char *column)
 {
     char *err_msg = NULL;
     char sql[128];
@@ -38,7 +71,7 @@ static int column_exists_in_table(const char *table, const char *column)
 
     snprintf(sql, 127, "SELECT 1 FROM pragma_table_info('%s') where name = '%s';", table, column);
 
-    int rc = sqlite3_exec_monitored(db_meta, sql, return_int_cb, (void *) &exists, &err_msg);
+    int rc = sqlite3_exec_monitored(database, sql, return_int_cb, (void *) &exists, &err_msg);
     if (rc != SQLITE_OK) {
         netdata_log_info("Error checking column existence; %s", err_msg);
         sqlite3_free(err_msg);
@@ -47,6 +80,17 @@ static int column_exists_in_table(const char *table, const char *column)
     return exists;
 }
 
+static int get_database_user_version(sqlite3 *database)
+{
+    int user_version = 0;
+
+    int rc = sqlite3_exec_monitored(database, "PRAGMA user_version", return_int_cb, (void *)&user_version, NULL);
+    if (rc != SQLITE_OK)
+        netdata_log_error("Failed to get user version for database");
+
+    return user_version;
+}
+
 const char *database_migrate_v1_v2[] = {
     "ALTER TABLE host ADD hops INTEGER NOT NULL DEFAULT 0;",
     NULL
@@ -88,31 +132,43 @@ const char *database_migrate_v10_v11[] = {
     NULL
 };
 
-static int do_migration_v1_v2(sqlite3 *database, const char *name)
-{
-    UNUSED(name);
-    netdata_log_info("Running \"%s\" database migration", name);
+const char *database_migrate_v11_v12[] = {
+    "ALTER TABLE health_log_detail ADD summary TEXT;",
+    "ALTER TABLE alert_hash ADD summary TEXT;",
+    NULL
+};
 
-    if (table_exists_in_database("host") && !column_exists_in_table("host", "hops"))
-        return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v1_v2[0]);
+const char *database_migrate_v12_v13_detail[] = {
+    "ALTER TABLE health_log_detail ADD summary TEXT;",
+    NULL
+};
+
+const char *database_migrate_v12_v13_hash[] = {
+    "ALTER TABLE alert_hash ADD summary TEXT;",
+    NULL
+};
+
+const char *database_migrate_v13_v14[] = {
+    "ALTER TABLE host ADD last_connected INT NOT NULL DEFAULT 0;",
+    NULL
+};
+
+static int do_migration_v1_v2(sqlite3 *database)
+{
+    if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "hops"))
+        return init_database_batch(database, &database_migrate_v1_v2[0]);
     return 0;
 }
 
-static int do_migration_v2_v3(sqlite3 *database, const char *name)
+static int do_migration_v2_v3(sqlite3 *database)
 {
-    UNUSED(name);
-    netdata_log_info("Running \"%s\" database migration", name);
-
-    if (table_exists_in_database("host") && !column_exists_in_table("host", "memory_mode"))
-        return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v2_v3[0]);
+    if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "memory_mode"))
+        return init_database_batch(database, &database_migrate_v2_v3[0]);
     return 0;
 }
 
-static int do_migration_v3_v4(sqlite3 *database, const char *name)
+static int do_migration_v3_v4(sqlite3 *database)
 {
-    UNUSED(name);
-    netdata_log_info("Running database migration %s", name);
-
     char sql[256];
 
     int rc;
@@ -126,7 +182,7 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
 
     while (sqlite3_step_monitored(res) == SQLITE_ROW) {
          char *table = strdupz((char *) sqlite3_column_text(res, 0));
-         if (!column_exists_in_table(table, "chart_context")) {
+         if (!column_exists_in_table(database, table, "chart_context")) {
              snprintfz(sql, 255, "ALTER TABLE %s ADD chart_context text", table);
              sqlite3_exec_monitored(database, sql, 0, 0, NULL);
          }
@@ -140,27 +196,18 @@ static int do_migration_v3_v4(sqlite3 *database, const char *name)
     return 0;
 }
 
-static int do_migration_v4_v5(sqlite3 *database, const char *name)
+static int do_migration_v4_v5(sqlite3 *database)
 {
-    UNUSED(name);
-    netdata_log_info("Running \"%s\" database migration", name);
-
-    return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v4_v5[0]);
+    return init_database_batch(database, &database_migrate_v4_v5[0]);
 }
 
-static int do_migration_v5_v6(sqlite3 *database, const char *name)
+static int do_migration_v5_v6(sqlite3 *database)
 {
-    UNUSED(name);
-    netdata_log_info("Running \"%s\" database migration", name);
-
-    return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v5_v6[0]);
+    return init_database_batch(database, &database_migrate_v5_v6[0]);
 }
 
-static int do_migration_v6_v7(sqlite3 *database, const char *name)
+static int do_migration_v6_v7(sqlite3 *database)
 {
-    UNUSED(name);
-    netdata_log_info("Running \"%s\" database migration", name);
-
     char sql[256];
 
     int rc;
@@ -174,7 +221,7 @@ static int do_migration_v6_v7(sqlite3 *database, const char *name)
 
     while (sqlite3_step_monitored(res) == SQLITE_ROW) {
          char *table = strdupz((char *) sqlite3_column_text(res, 0));
-         if (!column_exists_in_table(table, "filtered_alert_unique_id")) {
+         if (!column_exists_in_table(database, table, "filtered_alert_unique_id")) {
              snprintfz(sql, 255, "ALTER TABLE %s ADD filtered_alert_unique_id", table);
              sqlite3_exec_monitored(database, sql, 0, 0, NULL);
              snprintfz(sql, 255, "UPDATE %s SET filtered_alert_unique_id = alert_unique_id", table);
@@ -190,11 +237,8 @@ static int do_migration_v6_v7(sqlite3 *database, const char *name)
     return 0;
 }
 
-static int do_migration_v7_v8(sqlite3 *database, const char *name)
+static int do_migration_v7_v8(sqlite3 *database)
 {
-    UNUSED(name);
-    netdata_log_info("Running database migration %s", name);
-
     char sql[256];
 
     int rc;
@@ -208,7 +252,7 @@ static int do_migration_v7_v8(sqlite3 *database, const char *name)
 
     while (sqlite3_step_monitored(res) == SQLITE_ROW) {
          char *table = strdupz((char *) sqlite3_column_text(res, 0));
-         if (!column_exists_in_table(table, "transition_id")) {
+         if (!column_exists_in_table(database, table, "transition_id")) {
              snprintfz(sql, 255, "ALTER TABLE %s ADD transition_id blob", table);
              sqlite3_exec_monitored(database, sql, 0, 0, NULL);
          }
@@ -222,10 +266,8 @@ static int do_migration_v7_v8(sqlite3 *database, const char *name)
     return 0;
 }
 
-static int do_migration_v8_v9(sqlite3 *database, const char *name)
+static int do_migration_v8_v9(sqlite3 *database)
 {
-    netdata_log_info("Running database migration %s", name);
-
     char sql[2048];
     int rc;
     sqlite3_stmt *res = NULL;
@@ -244,7 +286,7 @@ static int do_migration_v8_v9(sqlite3 *database, const char *name)
               "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, " \
               "flags int, exec_run_timestamp int, delay_up_to_timestamp int, " \
               "info text, exec_code int, new_status real, old_status real, delay int, " \
-              "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, host_id blob);");
+              "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text, host_id blob);");
     sqlite3_exec_monitored(database, sql, 0, 0, NULL);
 
     snprintfz(sql, 2047, "CREATE INDEX IF NOT EXISTS health_log_d_ind_1 ON health_log_detail (unique_id);");
@@ -296,36 +338,85 @@ static int do_migration_v8_v9(sqlite3 *database, const char *name)
     return 0;
 }
 
-static int do_migration_v9_v10(sqlite3 *database, const char *name)
+static int do_migration_v9_v10(sqlite3 *database)
 {
-    netdata_log_info("Running \"%s\" database migration", name);
+    if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "chart_labels"))
+        return init_database_batch(database, &database_migrate_v9_v10[0]);
+    return 0;
+}
+
+static int do_migration_v10_v11(sqlite3 *database)
+{
+    if (table_exists_in_database(database, "health_log") && !column_exists_in_table(database, "health_log", "chart_name"))
+        return init_database_batch(database, &database_migrate_v10_v11[0]);
 
-    if (table_exists_in_database("alert_hash") && !column_exists_in_table("alert_hash", "chart_labels"))
-        return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v9_v10[0]);
     return 0;
 }
 
-static int do_migration_v10_v11(sqlite3 *database, const char *name)
+#define MIGR_11_12_UPD_HEALTH_LOG_DETAIL "UPDATE health_log_detail SET summary = (select name from health_log where health_log_id = health_log_detail.health_log_id);"
+static int do_migration_v11_v12(sqlite3 *database)
+{
+    int rc = 0;
+
+    if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary") &&
+        table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary"))
+        rc = init_database_batch(database, &database_migrate_v11_v12[0]);
+
+    if (!rc)
+        sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL);
+
+    return rc;
+}
+
+static int do_migration_v12_v13(sqlite3 *database)
 {
-    netdata_log_info("Running \"%s\" database migration", name);
+    int rc = 0;
 
-    if (table_exists_in_database("health_log") && !column_exists_in_table("health_log", "chart_name"))
-        return init_database_batch(database, DB_CHECK_NONE, 0, &database_migrate_v10_v11[0]);
+    if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary")) {
+        rc = init_database_batch(database, &database_migrate_v12_v13_detail[0]);
+        sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL);
+    }
+
+    if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary"))
+        rc = init_database_batch(database, &database_migrate_v12_v13_hash[0]);
+
+    return rc;
+}
+
+static int do_migration_v13_v14(sqlite3 *database)
+{
+    if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "last_connected"))
+        return init_database_batch(database, &database_migrate_v13_v14[0]);
 
     return 0;
 }
 
-static int do_migration_noop(sqlite3 *database, const char *name)
+
+// Actions for ML migration
+const char *database_ml_migrate_v1_v2[] = {
+    "PRAGMA journal_mode=delete",
+    "PRAGMA journal_mode=WAL",
+    "PRAGMA auto_vacuum=2",
+    "VACUUM",
+    NULL
+};
+
+static int do_ml_migration_v1_v2(sqlite3 *database)
+{
+    if (get_auto_vaccum(database) != 2)
+        return init_database_batch(database, &database_ml_migrate_v1_v2[0]);
+    return 0;
+}
+
+static int do_migration_noop(sqlite3 *database)
 {
     UNUSED(database);
-    UNUSED(name);
-    netdata_log_info("Running database migration %s", name);
     return 0;
 }
 
 typedef struct database_func_migration_list {
     char *name;
-    int (*func)(sqlite3 *database, const char *name);
+    int (*func)(sqlite3 *database);
 } DATABASE_FUNC_MIGRATION_LIST;
 
 
@@ -347,7 +438,8 @@ static int migrate_database(sqlite3 *database, int target_version, char *db_name
 
     netdata_log_info("Database version is %d, current version is %d. Running migration for %s ...", user_version, target_version, db_name);
     for (int i = user_version; i < target_version && migration_list[i].func; i++) {
-        rc = (migration_list[i].func)(database, migration_list[i].name);
+        netdata_log_info("Running database \"%s\" migration %s", db_name, migration_list[i].name);
+        rc = (migration_list[i].func)(database);
         if (unlikely(rc)) {
             error_report("Database %s migration from version %d to version %d failed", db_name, i, i + 1);
             return i;
@@ -369,6 +461,9 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = {
     {.name = "v8 to v9",  .func = do_migration_v8_v9},
     {.name = "v9 to v10",  .func = do_migration_v9_v10},
     {.name = "v10 to v11",  .func = do_migration_v10_v11},
+    {.name = "v11 to v12",  .func = do_migration_v11_v12},
+    {.name = "v12 to v13",  .func = do_migration_v12_v13},
+    {.name = "v13 to v14",  .func = do_migration_v13_v14},
     // the terminator of this array
     {.name = NULL, .func = NULL}
 };
@@ -379,13 +474,34 @@ DATABASE_FUNC_MIGRATION_LIST context_migration_action[] = {
     {.name = NULL, .func = NULL}
 };
 
+DATABASE_FUNC_MIGRATION_LIST ml_migration_action[] = {
+    {.name = "v0 to v1",  .func = do_migration_noop},
+    {.name = "v1 to v2",  .func = do_ml_migration_v1_v2},
+    // the terminator of this array
+    {.name = NULL, .func = NULL}
+};
 
 int perform_database_migration(sqlite3 *database, int target_version)
 {
+    int user_version = get_database_user_version(database);
+
+    if (!user_version && !db_table_count(database))
+        return target_version;
+
     return migrate_database(database, target_version, "metadata", migration_action);
 }
 
 int perform_context_database_migration(sqlite3 *database, int target_version)
 {
+    int user_version = get_database_user_version(database);
+
+    if (!user_version && !table_exists_in_database(database, "context"))
+        return target_version;
+
     return migrate_database(database, target_version, "context", context_migration_action);
 }
+
+int perform_ml_database_migration(sqlite3 *database, int target_version)
+{
+    return migrate_database(database, target_version, "ml", ml_migration_action);
+}
diff --git a/database/sqlite/sqlite_db_migration.h b/database/sqlite/sqlite_db_migration.h
index edaac5269..e3c1be84f 100644
--- a/database/sqlite/sqlite_db_migration.h
+++ b/database/sqlite/sqlite_db_migration.h
@@ -8,6 +8,7 @@
 
 int perform_database_migration(sqlite3 *database, int target_version);
 int perform_context_database_migration(sqlite3 *database, int target_version);
-int table_exists_in_database(const char *table);
+int table_exists_in_database(sqlite3 *database, const char *table);
+int perform_ml_database_migration(sqlite3 *database, int target_version);
 
 #endif //NETDATA_SQLITE_DB_MIGRATION_H
diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c
index d976a3c6e..e081fe9ae 100644
--- a/database/sqlite/sqlite_functions.c
+++ b/database/sqlite/sqlite_functions.c
@@ -1,9 +1,10 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 
 #include "sqlite_functions.h"
+#include "sqlite3recover.h"
 #include "sqlite_db_migration.h"
 
-#define DB_METADATA_VERSION 11
+#define DB_METADATA_VERSION 14
 
 const char *database_config[] = {
     "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "
@@ -13,7 +14,7 @@ const char *database_config[] = {
     "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0,"
     "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', "
     "entries INT NOT NULL DEFAULT 0,"
-    "health_enabled INT NOT NULL DEFAULT 0);",
+    "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0);",
 
     "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "
     "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "
@@ -32,7 +33,7 @@ const char *database_config[] = {
     "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "
     "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
     "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
-    "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text);",
+    "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, summary text);",
 
     "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "
     "date_created INT, PRIMARY KEY(host_id, system_key));",
@@ -53,18 +54,17 @@ const char *database_config[] = {
     "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "
     "flags int, exec_run_timestamp int, delay_up_to_timestamp int, "
     "info text, exec_code int, new_status real, old_status real, delay int, "
-    "new_value double, old_value double, last_repeat int, transition_id blob, global_id int);",
+    "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text);",
 
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_1 ON health_log_detail (unique_id);",
     "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id);",
     "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id);",
-    "CREATE INDEX IF NOT EXISTS health_log_d_ind_4 ON health_log_detail (health_log_id);",
+    "CREATE INDEX IF NOT EXISTS health_log_d_ind_5 ON health_log_detail (health_log_id, unique_id DESC);",
+    "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)",
 
     NULL
 };
 
 const char *database_cleanup[] = {
-    "DELETE FROM chart WHERE chart_id NOT IN (SELECT chart_id FROM dimension);",
     "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart);",
     "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);",
     "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host);",
@@ -74,6 +74,8 @@ const char *database_cleanup[] = {
     "DROP INDEX IF EXISTS ind_c1;",
     "DROP INDEX IF EXISTS ind_c2;",
     "DROP INDEX IF EXISTS alert_hash_index;",
+    "DROP INDEX IF EXISTS health_log_d_ind_4;",
+    "DROP INDEX IF EXISTS health_log_d_ind_1;",
     NULL
 };
 
@@ -120,23 +122,126 @@ SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) {
     return rc;
 }
 
-int execute_insert(sqlite3_stmt *res)
+static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database)
 {
-    int rc;
-    int cnt = 0;
-    while ((rc = sqlite3_step_monitored(res)) != SQLITE_DONE && ++cnt < SQL_MAX_RETRY && likely(!netdata_exit)) {
-        if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) {
-            usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
-            error_report("Failed to insert/update, rc = %d -- attempt %d", rc, cnt);
+
+    if (!res && !database)
+        return false;
+
+    if (!database)
+        database = sqlite3_db_handle(res);
+
+    if (db_meta == database) {
+        char recover_file[FILENAME_MAX + 1];
+        snprintfz(recover_file, FILENAME_MAX, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
+        int fd = open(recover_file, O_WRONLY | O_CREAT | O_TRUNC, 444);
+        if (fd >= 0) {
+            close(fd);
+            return true;
         }
-        else {
-            error_report("SQLite error %d", rc);
-            break;
+    }
+    return false;
+}
+
+static void recover_database(const char *sqlite_database, const char *new_sqlite_database)
+{
+    sqlite3 *database;
+    int rc = sqlite3_open(sqlite_database, &database);
+    if (rc != SQLITE_OK)
+        return;
+
+    netdata_log_info("Recover %s", sqlite_database);
+    netdata_log_info("     to %s", new_sqlite_database);
+
+    // This will remove the -shm and -wal files when we close the database
+    (void) db_execute(database, "select count(*) from sqlite_master limit 0");
+
+    sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);
+    if (recover) {
+
+        rc = sqlite3_recover_run(recover);
+
+        if (rc == SQLITE_OK)
+            netdata_log_info("Recover complete");
+        else
+            netdata_log_info("Recover encountered an error but the database may be usable");
+
+        rc = sqlite3_recover_finish(recover);
+
+        (void) sqlite3_close(database);
+
+        if (rc == SQLITE_OK) {
+            rc = rename(new_sqlite_database, sqlite_database);
+            if (rc == 0) {
+                netdata_log_info("Renamed %s", new_sqlite_database);
+                netdata_log_info("     to %s", sqlite_database);
+            }
         }
     }
+    else
+        (void) sqlite3_close(database);
+}
+
+int execute_insert(sqlite3_stmt *res)
+{
+    int rc;
+    rc =  sqlite3_step_monitored(res);
+    if (rc == SQLITE_CORRUPT) {
+        (void)mark_database_to_recover(res, NULL);
+        error_report("SQLite error %d", rc);
+    }
     return rc;
 }
 
+int configure_sqlite_database(sqlite3 *database, int target_version)
+{
+    char buf[1024 + 1] = "";
+    const char *list[2] = { buf, NULL };
+
+    // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
+    // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
+    snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
+    if (init_database_batch(database, list))
+        return 1;
+
+    // https://www.sqlite.org/pragma.html#pragma_synchronous
+    // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
+    snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
+    if (init_database_batch(database, list))
+        return 1;
+
+    // https://www.sqlite.org/pragma.html#pragma_journal_mode
+    // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
+    snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
+    if (init_database_batch(database, list))
+        return 1;
+
+    // https://www.sqlite.org/pragma.html#pragma_temp_store
+    // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
+    snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
+    if (init_database_batch(database, list))
+        return 1;
+
+    // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
+    // PRAGMA schema.journal_size_limit = N ;
+    snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
+    if (init_database_batch(database, list))
+        return 1;
+
+    // https://www.sqlite.org/pragma.html#pragma_cache_size
+    // PRAGMA schema.cache_size = pages;
+    // PRAGMA schema.cache_size = -kibibytes;
+    snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
+    if (init_database_batch(database, list))
+        return 1;
+
+    snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
+    if (init_database_batch(database, list))
+        return 1;
+
+    return 0;
+}
+
 #define MAX_OPEN_STATEMENTS (512)
 
 static void add_stmt_to_list(sqlite3_stmt *res)
@@ -202,134 +307,21 @@ int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **state
     return rc;
 }
 
-static int check_table_integrity_cb(void *data, int argc, char **argv, char **column)
-{
-    int *status = data;
-    UNUSED(argc);
-    UNUSED(column);
-    netdata_log_info("---> %s", argv[0]);
-    *status = (strcmp(argv[0], "ok") != 0);
-    return 0;
-}
-
-
-static int check_table_integrity(char *table)
-{
-    int status = 0;
-    char *err_msg = NULL;
-    char wstr[255];
-
-    if (table) {
-        netdata_log_info("Checking table %s", table);
-        snprintfz(wstr, 254, "PRAGMA integrity_check(%s);", table);
-    }
-    else {
-        netdata_log_info("Checking entire database");
-        strcpy(wstr,"PRAGMA integrity_check;");
-    }
-
-    int rc = sqlite3_exec_monitored(db_meta, wstr, check_table_integrity_cb, (void *) &status, &err_msg);
-    if (rc != SQLITE_OK) {
-        error_report("SQLite error during database integrity check for %s, rc = %d (%s)",
-                     table ? table : "the entire database", rc, err_msg);
-        sqlite3_free(err_msg);
-    }
-
-    return status;
-}
-
-const char *rebuild_chart_commands[] = {
-    "BEGIN TRANSACTION; ",
-    "DROP INDEX IF EXISTS ind_c1;" ,
-    "DROP TABLE IF EXISTS chart_backup; " ,
-    "CREATE TABLE chart_backup AS SELECT * FROM chart; " ,
-    "DROP TABLE chart;  ",
-    "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, "
-       "name text, family text, context text, title text, unit text, plugin text, "
-       "module text, priority int, update_every int, chart_type int, memory_mode int, history_entries); ",
-    "INSERT INTO chart SELECT DISTINCT * FROM chart_backup; ",
-    "DROP TABLE chart_backup;  " ,
-    "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);",
-    "COMMIT TRANSACTION;",
-    NULL
-};
-
-static void rebuild_chart()
-{
-    int rc;
-    char *err_msg = NULL;
-    netdata_log_info("Rebuilding chart table");
-    for (int i = 0; rebuild_chart_commands[i]; i++) {
-        netdata_log_info("Executing %s", rebuild_chart_commands[i]);
-        rc = sqlite3_exec_monitored(db_meta, rebuild_chart_commands[i], 0, 0, &err_msg);
-        if (rc != SQLITE_OK) {
-            error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);
-            error_report("SQLite failed statement %s", rebuild_chart_commands[i]);
-            sqlite3_free(err_msg);
-        }
-    }
-}
-
-const char *rebuild_dimension_commands[] = {
-    "BEGIN TRANSACTION; ",
-    "DROP INDEX IF EXISTS ind_d1;" ,
-    "DROP TABLE IF EXISTS dimension_backup; " ,
-    "CREATE TABLE dimension_backup AS SELECT * FROM dimension; " ,
-    "DROP TABLE dimension; " ,
-    "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "
-        "multiplier int, divisor int , algorithm int, options text);" ,
-    "INSERT INTO dimension SELECT distinct * FROM dimension_backup; " ,
-    "DROP TABLE dimension_backup;  " ,
-    "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);",
-    "COMMIT TRANSACTION;",
-    NULL
-};
-
-void rebuild_dimension()
-{
-    int rc;
-    char *err_msg = NULL;
-
-    netdata_log_info("Rebuilding dimension table");
-    for (int i = 0; rebuild_dimension_commands[i]; i++) {
-        netdata_log_info("Executing %s", rebuild_dimension_commands[i]);
-        rc = sqlite3_exec_monitored(db_meta, rebuild_dimension_commands[i], 0, 0, &err_msg);
-        if (rc != SQLITE_OK) {
-            error_report("SQLite error during database setup, rc = %d (%s)", rc, err_msg);
-            error_report("SQLite failed statement %s", rebuild_dimension_commands[i]);
-            sqlite3_free(err_msg);
-        }
-    }
-}
-
-static int attempt_database_fix()
-{
-    netdata_log_info("Closing database and attempting to fix it");
-    int rc = sqlite3_close(db_meta);
-    if (rc != SQLITE_OK)
-        error_report("Failed to close database, rc = %d", rc);
-    netdata_log_info("Attempting to fix database");
-    db_meta = NULL;
-    return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT, 0);
-}
-
-int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[])
+int init_database_batch(sqlite3 *database, const char *batch[])
 {
     int rc;
     char *err_msg = NULL;
     for (int i = 0; batch[i]; i++) {
-        netdata_log_debug(D_METADATALOG, "Executing %s", batch[i]);
         rc = sqlite3_exec_monitored(database, batch[i], 0, 0, &err_msg);
         if (rc != SQLITE_OK) {
-            error_report("SQLite error during database %s, rc = %d (%s)", init_type ? "cleanup" : "setup", rc, err_msg);
+            error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg);
             error_report("SQLite failed statement %s", batch[i]);
+            analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(sqlite3_extended_errcode(database)));
             sqlite3_free(err_msg);
             if (SQLITE_CORRUPT == rc) {
-                if (!rebuild)
-                    return attempt_database_fix();
-                rc = check_table_integrity(NULL);
-                if (rc)
-                    error_report("Databse integrity errors reported");
+                if (mark_database_to_recover(NULL, database))
+                    error_report("Database is corrupted will attempt to fix");
+                return SQLITE_CORRUPT;
             }
             return 1;
         }
@@ -389,64 +381,47 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
     char sqlite_database[FILENAME_MAX + 1];
     int rc;
 
-    if (likely(!memory))
+    if (likely(!memory)) {
+        snprintfz(sqlite_database, FILENAME_MAX, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
+        rc = unlink(sqlite_database);
         snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
+
+        if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {
+            char new_sqlite_database[FILENAME_MAX + 1];
+            snprintfz(new_sqlite_database, FILENAME_MAX, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);
+            recover_database(sqlite_database, new_sqlite_database);
+            if (rebuild & DB_CHECK_RECOVER)
+                return 0;
+        }
+    }
     else
         strcpy(sqlite_database, ":memory:");
 
     rc = sqlite3_open(sqlite_database, &db_meta);
     if (rc != SQLITE_OK) {
         error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
+        analytics_set_data_str(&analytics_data.netdata_fail_reason, sqlite3_errstr(sqlite3_extended_errcode(db_meta)));
         sqlite3_close(db_meta);
         db_meta = NULL;
         return 1;
     }
 
-    if (rebuild & (DB_CHECK_INTEGRITY | DB_CHECK_FIX_DB)) {
-        int errors_detected = 0;
-        if (!(rebuild & DB_CHECK_CONT))
-            netdata_log_info("Running database check on %s", sqlite_database);
-
-        if (check_table_integrity("chart")) {
-            errors_detected++;
-            if (rebuild & DB_CHECK_FIX_DB)
-                rebuild_chart();
-            else
-                error_report("Errors reported -- run with -W sqlite-fix");
-        }
-
-        if (check_table_integrity("dimension")) {
-            errors_detected++;
-            if (rebuild & DB_CHECK_FIX_DB)
-                rebuild_dimension();
-            else
-                error_report("Errors reported -- run with -W sqlite-fix");
-        }
-
-        if (!errors_detected) {
-            if (check_table_integrity(NULL))
-                error_report("Errors reported");
-        }
-    }
-
     if (rebuild & DB_CHECK_RECLAIM_SPACE) {
-        if (!(rebuild & DB_CHECK_CONT))
-            netdata_log_info("Reclaiming space of %s", sqlite_database);
+        netdata_log_info("Reclaiming space of %s", sqlite_database);
         rc = sqlite3_exec_monitored(db_meta, "VACUUM;", 0, 0, &err_msg);
         if (rc != SQLITE_OK) {
             error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);
             sqlite3_free(err_msg);
         }
-    }
-
-    if (rebuild && !(rebuild & DB_CHECK_CONT))
+        else {
+            (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");
+            (void) sqlite3_close(db_meta);
+        }
         return 1;
+    }
 
     netdata_log_info("SQLite database %s initialization", sqlite_database);
 
-    char buf[1024 + 1] = "";
-    const char *list[2] = { buf, NULL };
-
     rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);
     if (unlikely(rc != SQLITE_OK))
         error_report("Failed to register internal u2h function");
@@ -464,44 +439,13 @@ int sql_init_database(db_check_action_type_t rebuild, int memory)
     if (likely(!memory))
         target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
 
-    // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
-    // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
-    snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_synchronous
-    // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
-    snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_journal_mode
-    // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
-    snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_temp_store
-    // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
-    snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
-    // PRAGMA schema.journal_size_limit = N ;
-    snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
-
-    // https://www.sqlite.org/pragma.html#pragma_cache_size
-    // PRAGMA schema.cache_size = pages;
-    // PRAGMA schema.cache_size = -kibibytes;
-    snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
-
-    snprintfz(buf, 1024, "PRAGMA user_version=%d;", target_version);
-    if(init_database_batch(db_meta, rebuild, 0, list)) return 1;
+   if (configure_sqlite_database(db_meta, target_version))
+        return 1;
 
-    if (init_database_batch(db_meta, rebuild, 0, &database_config[0]))
+    if (init_database_batch(db_meta, &database_config[0]))
         return 1;
 
-    if (init_database_batch(db_meta, rebuild, 0, &database_cleanup[0]))
+    if (init_database_batch(db_meta, &database_cleanup[0]))
         return 1;
 
     netdata_log_info("SQLite database initialization completed");
@@ -543,7 +487,7 @@ int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
 
     rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host parameter to %s, rc = %d", sql, rc);
+        error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc);
         goto skip;
     }
 
@@ -665,44 +609,6 @@ failed:
     return rc - 1;
 }
 
-#define SQL_SELECT_HOST_BY_NODE_ID  "select host_id from node_instance where node_id = @node_id;"
-
-int get_host_id(uuid_t *node_id, uuid_t *host_id)
-{
-    static __thread sqlite3_stmt *res = NULL;
-    int rc;
-
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            error_report("Database has not been initialized");
-        return 1;
-    }
-
-    if (unlikely(!res)) {
-        rc = prepare_statement(db_meta, SQL_SELECT_HOST_BY_NODE_ID, &res);
-        if (unlikely(rc != SQLITE_OK)) {
-            error_report("Failed to prepare statement to select node instance information for a node");
-            return 1;
-        }
-    }
-
-    rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id parameter to select node instance information");
-        goto failed;
-    }
-
-    rc = sqlite3_step_monitored(res);
-    if (likely(rc == SQLITE_ROW && host_id))
-        uuid_copy(*host_id, *((uuid_t *) sqlite3_column_blob(res, 0)));
-
-failed:
-    if (unlikely(sqlite3_reset(res) != SQLITE_OK))
-        error_report("Failed to reset the prepared statement when selecting node instance information");
-
-    return (rc == SQLITE_ROW) ? 0 : -1;
-}
-
 #define SQL_SELECT_NODE_ID  "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL;"
 
 int get_node_id(uuid_t *host_id, uuid_t *node_id)
@@ -826,6 +732,8 @@ struct node_instance_list *get_node_list(void)
             uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);
             uuid_unparse_lower(*host_id, host_guid);
             RRDHOST *host = rrdhost_find_by_guid(host_guid);
+            if (!host)
+                continue;
             if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {
                 netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));
                 continue;
@@ -856,7 +764,7 @@ failed:
 
 void sql_load_node_id(RRDHOST *host)
 {
-    static __thread sqlite3_stmt *res = NULL;
+    sqlite3_stmt *res = NULL;
     int rc;
 
     if (unlikely(!db_meta)) {
@@ -865,13 +773,11 @@ void sql_load_node_id(RRDHOST *host)
         return;
     }
 
-    if (unlikely(!res)) {
-        rc = prepare_statement(db_meta, SQL_GET_HOST_NODE_ID, &res);
-        if (unlikely(rc != SQLITE_OK)) {
-            error_report("Failed to prepare statement to fetch node id");
-            return;
-        };
-    }
+    rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to prepare statement to fetch node id");
+        return;
+    };
 
     rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
@@ -888,8 +794,8 @@ void sql_load_node_id(RRDHOST *host)
     }
 
 failed:
-    if (unlikely(sqlite3_reset(res) != SQLITE_OK))
-        error_report("Failed to reset the prepared statement when loading node instance information");
+    if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+        error_report("Failed to finalize the prepared statement when loading node instance information");
 };
 
 
@@ -926,11 +832,11 @@ skip:
 #define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \
     "AND label_key IS NOT NULL AND label_value IS NOT NULL;"
 
-DICTIONARY *sql_load_host_labels(uuid_t *host_id)
+RRDLABELS *sql_load_host_labels(uuid_t *host_id)
 {
     int rc;
 
-    DICTIONARY *labels = NULL;
+    RRDLABELS *labels = NULL;
     sqlite3_stmt *res = NULL;
 
     rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0);
@@ -948,11 +854,7 @@ DICTIONARY *sql_load_host_labels(uuid_t *host_id)
     labels = rrdlabels_create();
 
     while (sqlite3_step_monitored(res) == SQLITE_ROW) {
-        rrdlabels_add(
-            labels,
-            (const char *)sqlite3_column_text(res, 0),
-            (const char *)sqlite3_column_text(res, 1),
-            sqlite3_column_int(res, 2));
+        rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2));
     }
 
 skip:
diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h
index 407ed1eff..9cd1f7ad4 100644
--- a/database/sqlite/sqlite_functions.h
+++ b/database/sqlite/sqlite_functions.h
@@ -6,6 +6,8 @@
 #include "daemon/common.h"
 #include "sqlite3.h"
 
+void analytics_set_data_str(char **name, const char *value);
+
 // return a node list
 struct node_instance_list {
     uuid_t  node_id;
@@ -17,11 +19,10 @@ struct node_instance_list {
 };
 
 typedef enum db_check_action_type {
-    DB_CHECK_NONE  = 0x0000,
-    DB_CHECK_INTEGRITY  = 0x0001,
-    DB_CHECK_FIX_DB = 0x0002,
-    DB_CHECK_RECLAIM_SPACE = 0x0004,
-    DB_CHECK_CONT = 0x00008
+    DB_CHECK_NONE          = (1 << 0),
+    DB_CHECK_RECLAIM_SPACE = (1 << 1),
+    DB_CHECK_CONT          = (1 << 2),
+    DB_CHECK_RECOVER       = (1 << 3),
 } db_check_action_type_t;
 
 #define SQL_MAX_RETRY (100)
@@ -46,9 +47,10 @@ SQLITE_API int sqlite3_exec_monitored(
     );
 
 // Initialization and shutdown
-int init_database_batch(sqlite3 *database, int rebuild, int init_type, const char *batch[]);
+int init_database_batch(sqlite3 *database, const char *batch[]);
 int sql_init_database(db_check_action_type_t rebuild, int memory);
 void sql_close_database(void);
+int configure_sqlite_database(sqlite3 *database, int target_version);
 
 // Helpers
 int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);
@@ -60,14 +62,12 @@ void initialize_thread_key_pool(void);
 
 // Look up functions
 int get_node_id(uuid_t *host_id, uuid_t *node_id);
-int get_host_id(uuid_t *node_id, uuid_t *host_id);
 struct node_instance_list *get_node_list(void);
 void sql_load_node_id(RRDHOST *host);
-char *get_hostname_by_node_id(char *node_id);
 
 // Help build archived hosts in memory when agent starts
 void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);
-DICTIONARY *sql_load_host_labels(uuid_t *host_id);
+RRDLABELS *sql_load_host_labels(uuid_t *host_id);
 
 // TODO: move to metadata
 int update_node_id(uuid_t *host_id, uuid_t *node_id);
diff --git a/database/sqlite/sqlite_health.c b/database/sqlite/sqlite_health.c
index 9c103f098..6fc6a2e64 100644
--- a/database/sqlite/sqlite_health.c
+++ b/database/sqlite/sqlite_health.c
@@ -5,13 +5,26 @@
 #include "sqlite_db_migration.h"
 
 #define MAX_HEALTH_SQL_SIZE 2048
-#define sqlite3_bind_string_or_null(res,key,param) ((key) ? sqlite3_bind_text(res, param, string2str(key), -1, SQLITE_STATIC) : sqlite3_bind_null(res, param))
+#define SQLITE3_BIND_STRING_OR_NULL(res, key, param)                                                                   \
+    ((key) ? sqlite3_bind_text(res, param, string2str(key), -1, SQLITE_STATIC) : sqlite3_bind_null(res, param))
+
+#define SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, param)                                                                   \
+    ({                                                                                                                 \
+        int _param = (param);                                                                                          \
+        sqlite3_column_type((res), (_param)) != SQLITE_NULL ?                                                          \
+            string_strdupz((char *)sqlite3_column_text((res), (_param))) :                                             \
+            NULL;                                                                                                      \
+    })
 
 /* Health related SQL queries
    Updates an entry in the table
 */
-#define SQL_UPDATE_HEALTH_LOG "UPDATE health_log_detail set updated_by_id = ?, flags = ?, exec_run_timestamp = ?, exec_code = ? where unique_id = ? AND alarm_id = ? and transition_id = ?;"
-void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae) {
+#define SQL_UPDATE_HEALTH_LOG                                                                                          \
+    "UPDATE health_log_detail SET updated_by_id = @updated_by, flags = @flags, exec_run_timestamp = @exec_time, "      \
+    "exec_code = @exec_code WHERE unique_id = @unique_id AND alarm_id = @alarm_id AND transition_id = @transaction"
+
+static void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae)
+{
     sqlite3_stmt *res = NULL;
     int rc;
 
@@ -82,17 +95,19 @@ failed:
 /* Health related SQL queries
    Inserts an entry in the table
 */
-#define SQL_INSERT_HEALTH_LOG "INSERT INTO health_log (host_id, alarm_id, " \
-    "config_hash_id, name, chart, family, exec, recipient, units, chart_context, last_transition_id, chart_name) " \
-    "VALUES (?,?,?,?,?,?,?,?,?,?,?,?) " \
-    "ON CONFLICT (host_id, alarm_id) DO UPDATE SET last_transition_id = excluded.last_transition_id, " \
+#define SQL_INSERT_HEALTH_LOG                                                                                          \
+    "INSERT INTO health_log (host_id, alarm_id, "                                                                      \
+    "config_hash_id, name, chart, exec, recipient, units, chart_context, last_transition_id, chart_name) "     \
+    "VALUES (?,?,?,?,?,?,?,?,?,?,?) "                                                                                \
+    "ON CONFLICT (host_id, alarm_id) DO UPDATE SET last_transition_id = excluded.last_transition_id, "                 \
     "chart_name = excluded.chart_name RETURNING health_log_id; "
 
-#define SQL_INSERT_HEALTH_LOG_DETAIL "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, " \
+#define SQL_INSERT_HEALTH_LOG_DETAIL                                                                                        \
+    "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, "                                   \
     "updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, " \
-    "info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id) " \
-    "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,@global_id); "
-void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
+    "info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id, summary) " \
+    "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,@global_id,?); "
+static void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
     sqlite3_stmt *res = NULL;
     int rc;
     uint64_t health_log_id = 0;
@@ -127,55 +142,49 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->name, 4);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->name, 4);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind name parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->chart, 5);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->chart, 5);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind chart parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->family, 6);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind family parameter for SQL_INSERT_HEALTH_LOG");
-        goto failed;
-    }
-
-    rc = sqlite3_bind_string_or_null(res, ae->exec, 7);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->exec, 6);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind exec parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->recipient, 8);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->recipient, 7);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind recipient parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->units, 9);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->units, 8);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind host_id parameter to store node instance information");
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->chart_context, 10);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->chart_context, 9);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind chart_context parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
     }
 
-    rc = sqlite3_bind_blob(res, 11, &ae->transition_id, sizeof(ae->transition_id), SQLITE_STATIC);
+    rc = sqlite3_bind_blob(res, 10, &ae->transition_id, sizeof(ae->transition_id), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind transition_id parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->chart_name, 12);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->chart_name, 11);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind chart_name parameter for SQL_INSERT_HEALTH_LOG");
         goto failed;
@@ -271,7 +280,7 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
         goto failed;
     }
 
-    rc = sqlite3_bind_string_or_null(res, ae->info, 13);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->info, 13);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind info parameter for SQL_INSERT_HEALTH_LOG_DETAIL");
         goto failed;
@@ -331,6 +340,12 @@ void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae) {
         goto failed;
     }
 
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, ae->summary, 23);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind summary parameter for SQL_INSERT_HEALTH_LOG_DETAIL");
+        goto failed;
+    }
+
     rc = execute_insert(res);
     if (unlikely(rc != SQLITE_DONE)) {
         error_report("HEALTH [%s]: Failed to execute SQL_INSERT_HEALTH_LOG_DETAIL, rc = %d", rrdhost_hostname(host), rc);
@@ -353,7 +368,7 @@ void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)
         sql_health_alarm_log_insert(host, ae);
 #ifdef ENABLE_ACLK
         if (netdata_cloud_enabled) {
-            sql_queue_alarm_to_aclk(host, ae, 0);
+            sql_queue_alarm_to_aclk(host, ae, false);
         }
 #endif
     }
@@ -362,46 +377,67 @@ void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae)
 /* Health related SQL queries
    Get a count of rows from health log table
 */
-#define SQL_COUNT_HEALTH_LOG_DETAIL "SELECT count(1) FROM health_log_detail hld, health_log hl where hl.host_id = @host_id and hl.health_log_id = hld.health_log_id;"
-void sql_health_alarm_log_count(RRDHOST *host) {
+#define SQL_COUNT_HEALTH_LOG_DETAIL "SELECT count(1) FROM health_log_detail hld, health_log hl " \
+        "where hl.host_id = @host_id and hl.health_log_id = hld.health_log_id"
+
+static int sql_health_alarm_log_count(RRDHOST *host) {
     sqlite3_stmt *res = NULL;
     int rc;
 
     if (unlikely(!db_meta)) {
         if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
             error_report("Database has not been initialized");
-        return;
+        return -1;
     }
 
+    int entries_in_db = -1;
+
     rc = sqlite3_prepare_v2(db_meta, SQL_COUNT_HEALTH_LOG_DETAIL, -1, &res, 0);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to prepare statement to count health log entries from db");
-        return;
+        goto done;
     }
 
     rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind host_id for SQL_COUNT_HEALTH_LOG.");
-        sqlite3_finalize(res);
-        return;
+        goto done;
     }
 
     rc = sqlite3_step_monitored(res);
     if (likely(rc == SQLITE_ROW))
-        host->health.health_log_entries_written = (size_t) sqlite3_column_int64(res, 0);
+        entries_in_db = (int) sqlite3_column_int64(res, 0);
 
+done:
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
         error_report("Failed to finalize the prepared statement to count health log entries from db");
 
-    netdata_log_info("HEALTH [%s]: Table health_log_detail contains %lu entries.", rrdhost_hostname(host), (unsigned long int) host->health.health_log_entries_written);
+    return entries_in_db;
 }
 
-/* Health related SQL queries
-   Cleans up the health_log_detail table on a non-claimed host
-*/
-#define SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED "DELETE FROM health_log_detail WHERE health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = ?1) AND when_key + ?2 < unixepoch() AND updated_by_id <> 0 AND transition_id NOT IN (SELECT last_transition_id FROM health_log hl WHERE hl.host_id = ?3);"
-void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host) {
+/*
+ *
+ * Health related SQL queries
+ * Cleans up the health_log_detail table on a non-claimed or claimed host
+ *
+ */
+
+#define SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED "DELETE FROM health_log_detail WHERE health_log_id IN " \
+    "(SELECT health_log_id FROM health_log WHERE host_id = @host_id) AND when_key < unixepoch() - @history " \
+    "AND updated_by_id <> 0 AND transition_id NOT IN " \
+    "(SELECT last_transition_id FROM health_log hl WHERE hl.host_id = @host_id);"
+
+#define SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(guid) "DELETE from health_log_detail WHERE unique_id NOT IN " \
+    "(SELECT filtered_alert_unique_id FROM aclk_alert_%s) " \
+    "AND unique_id IN (SELECT hld.unique_id FROM health_log hl, health_log_detail hld WHERE " \
+    "hl.host_id = @host_id AND hl.health_log_id = hld.health_log_id) " \
+    "AND health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = @host_id) " \
+    "AND when_key < unixepoch() - @history " \
+    "AND updated_by_id <> 0 AND transition_id NOT IN " \
+    "(SELECT last_transition_id FROM health_log hl WHERE hl.host_id = @host_id);", guid
+
+void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed) {
     sqlite3_stmt *res = NULL;
     int rc;
     char command[MAX_HEALTH_SQL_SIZE + 1];
@@ -414,77 +450,18 @@ void sql_health_alarm_log_cleanup_not_claimed(RRDHOST *host) {
 
     char uuid_str[UUID_STR_LEN];
     uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to cleanup health log detail table (un-claimed)");
-        return;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id for SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
-    }
-
-    rc = sqlite3_bind_int64(res, 2, (sqlite3_int64)host->health_log.health_log_history);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind health log history for SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
-    }
-
-    rc = sqlite3_bind_blob(res, 3, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id for SQL_CLEANUP_HEALTH_LOG_NOT_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
-    }
-
-    rc = sqlite3_step_monitored(res);
-    if (unlikely(rc != SQLITE_DONE))
-        error_report("Failed to cleanup health log detail table, rc = %d", rc);
-
-    rc = sqlite3_finalize(res);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement to cleanup health log detail table (un-claimed)");
-
-    sql_health_alarm_log_count(host);
-
     snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);
-    if (unlikely(table_exists_in_database(command))) {
-        sql_aclk_alert_clean_dead_entries(host);
-    }
-}
 
-/* Health related SQL queries
-   Cleans up the health_log_detail table on a claimed host
-*/
-#define SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(guid) "DELETE from health_log_detail WHERE unique_id NOT IN (SELECT filtered_alert_unique_id FROM aclk_alert_%s) AND unique_id IN (SELECT hld.unique_id FROM health_log hl, health_log_detail hld WHERE hl.host_id = ?1 AND hl.health_log_id = hld.health_log_id) AND health_log_id IN (SELECT health_log_id FROM health_log WHERE host_id = ?2) AND when_key + ?3 < unixepoch() AND updated_by_id <> 0 AND transition_id NOT IN (SELECT last_transition_id FROM health_log hl WHERE hl.host_id = ?4);", guid
-void sql_health_alarm_log_cleanup_claimed(RRDHOST *host) {
-    sqlite3_stmt *res = NULL;
-    int rc;
-    char command[MAX_HEALTH_SQL_SIZE + 1];
+    bool aclk_table_exists = table_exists_in_database(db_meta, command);
 
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
-            error_report("Database has not been initialized");
-        return;
-    }
+    char *sql = SQL_CLEANUP_HEALTH_LOG_DETAIL_NOT_CLAIMED;
 
-    char uuid_str[UUID_STR_LEN];
-    uuid_unparse_lower_fix(&host->host_uuid, uuid_str);
-    snprintfz(command, MAX_HEALTH_SQL_SIZE, "aclk_alert_%s", uuid_str);
-
-    if (!table_exists_in_database(command)) {
-        sql_health_alarm_log_cleanup_not_claimed(host);
-        return;
+    if (claimed && aclk_table_exists) {
+        snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(uuid_str));
+        sql = command;
     }
 
-    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_CLEANUP_HEALTH_LOG_DETAIL_CLAIMED(uuid_str));
-
-    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to prepare statement to cleanup health log detail table (claimed)");
         return;
@@ -492,59 +469,44 @@ void sql_health_alarm_log_cleanup_claimed(RRDHOST *host) {
 
     rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind first host_id for SQL_CLEANUP_HEALTH_LOG_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
+        error_report("Failed to bind first host_id for sql_health_alarm_log_cleanup.");
+        goto done;
     }
 
-    rc = sqlite3_bind_blob(res, 2, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind second host_id for SQL_CLEANUP_HEALTH_LOG_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
-    }
-
-    rc = sqlite3_bind_int64(res, 3, (sqlite3_int64)host->health_log.health_log_history);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind health log history for SQL_CLEANUP_HEALTH_LOG_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
-    }
-
-    rc = sqlite3_bind_blob(res, 4, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
+    rc = sqlite3_bind_int64(res, 2, (sqlite3_int64)host->health_log.health_log_history);
     if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind second host_id for SQL_CLEANUP_HEALTH_LOG_CLAIMED.");
-        sqlite3_finalize(res);
-        return;
+        error_report("Failed to bind health log history for sql_health_alarm_log_cleanup.");
+        goto done;
     }
 
     rc = sqlite3_step_monitored(res);
     if (unlikely(rc != SQLITE_DONE))
         error_report("Failed to cleanup health log detail table, rc = %d", rc);
 
+    int rows = sql_health_alarm_log_count(host);
+    if (rows >= 0)
+        host->health.health_log_entries_written = rows;
+
+    if (aclk_table_exists)
+        sql_aclk_alert_clean_dead_entries(host);
+
+done:
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
         error_report("Failed to finalize the prepared statement to cleanup health log detail table (claimed)");
-
-    sql_health_alarm_log_count(host);
-
-    sql_aclk_alert_clean_dead_entries(host);
-
 }
 
-/* Health related SQL queries
-   Cleans up the health_log table.
-*/
-void sql_health_alarm_log_cleanup(RRDHOST *host) {
-    if (!claimed()) {
-        sql_health_alarm_log_cleanup_not_claimed(host);
-    } else
-        sql_health_alarm_log_cleanup_claimed(host);
-}
+#define SQL_INJECT_REMOVED                                                                                                      \
+    "insert into health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, updated_by_id, updates_id, when_key, "  \
+    "duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, info, exec_code, new_status, old_status, " \
+    "delay, new_value, old_value, last_repeat, transition_id, global_id, summary) "                                             \
+    "select health_log_id, ?1, ?2, ?3, 0, ?4, unixepoch(), 0, 0, flags, exec_run_timestamp, unixepoch(), info, exec_code, -2, " \
+    "new_status, delay, NULL, new_value, 0, ?5, now_usec(0), summary from health_log_detail where unique_id = ?6 and transition_id = ?7;"
 
-#define SQL_INJECT_REMOVED "insert into health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, updated_by_id, updates_id, when_key, duration, non_clear_duration, flags, exec_run_timestamp, delay_up_to_timestamp, info, exec_code, new_status, old_status, delay, new_value, old_value, last_repeat, transition_id, global_id) select health_log_id, ?1, ?2, ?3, 0, ?4, unixepoch(), 0, 0, flags, exec_run_timestamp, unixepoch(), info, exec_code, -2, new_status, delay, NULL, new_value, 0, ?5, now_usec(0) from health_log_detail where unique_id = ?6 and transition_id = ?7;"
 #define SQL_INJECT_REMOVED_UPDATE_DETAIL "update health_log_detail set flags = flags | ?1, updated_by_id = ?2 where unique_id = ?3 and transition_id = ?4;"
+
 #define SQL_INJECT_REMOVED_UPDATE_LOG "update health_log set last_transition_id = ?1 where alarm_id = ?2 and last_transition_id = ?3 and host_id = ?4;"
+
 void sql_inject_removed_status(RRDHOST *host, uint32_t alarm_id, uint32_t alarm_event_id, uint32_t unique_id, uint32_t max_unique_id, uuid_t *prev_transition_id)
 {
     int rc;
@@ -682,10 +644,8 @@ void sql_inject_removed_status(RRDHOST *host, uint32_t alarm_id, uint32_t alarm_
     }
 
     rc = execute_insert(res);
-    if (unlikely(rc != SQLITE_DONE)) {
+    if (unlikely(rc != SQLITE_DONE))
         error_report("HEALTH [N/A]: Failed to execute SQL_INJECT_REMOVED_UPDATE_DETAIL, rc = %d", rc);
-        goto failed;
-    }
 
 failed:
     if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
@@ -727,7 +687,10 @@ uint32_t sql_get_max_unique_id (RRDHOST *host)
      return max_unique_id;
 }
 
-#define SQL_SELECT_LAST_STATUSES "SELECT hld.new_status, hld.unique_id, hld.alarm_id, hld.alarm_event_id, hld.transition_id from health_log hl, health_log_detail hld where hl.host_id = @host_id and hl.last_transition_id = hld.transition_id;"
+#define SQL_SELECT_LAST_STATUSES                                                                                       \
+     "SELECT hld.new_status, hld.unique_id, hld.alarm_id, hld.alarm_event_id, hld.transition_id FROM health_log hl, "  \
+     "health_log_detail hld WHERE hl.host_id = @host_id AND hl.last_transition_id = hld.transition_id"
+
 void sql_check_removed_alerts_state(RRDHOST *host)
 {
     int rc;
@@ -752,21 +715,23 @@ void sql_check_removed_alerts_state(RRDHOST *host)
         uint32_t alarm_id, alarm_event_id, unique_id;
         RRDCALC_STATUS status;
 
-        status  = (RRDCALC_STATUS) sqlite3_column_int(res, 0);
-        unique_id = (uint32_t) sqlite3_column_int64(res, 1);
-        alarm_id = (uint32_t) sqlite3_column_int64(res, 2);
-        alarm_event_id = (uint32_t) sqlite3_column_int64(res, 3);
-        uuid_copy(transition_id, *((uuid_t *) sqlite3_column_blob(res, 4)));
+        status = (RRDCALC_STATUS)sqlite3_column_int(res, 0);
+        unique_id = (uint32_t)sqlite3_column_int64(res, 1);
+        alarm_id = (uint32_t)sqlite3_column_int64(res, 2);
+        alarm_event_id = (uint32_t)sqlite3_column_int64(res, 3);
+        uuid_copy(transition_id, *((uuid_t *)sqlite3_column_blob(res, 4)));
+
         if (unlikely(status != RRDCALC_STATUS_REMOVED)) {
-            if (unlikely(!max_unique_id))
-                max_unique_id = sql_get_max_unique_id (host);
-            sql_inject_removed_status (host, alarm_id, alarm_event_id, unique_id, ++max_unique_id, &transition_id);
+           if (unlikely(!max_unique_id))
+               max_unique_id = sql_get_max_unique_id(host);
+
+           sql_inject_removed_status(host, alarm_id, alarm_event_id, unique_id, ++max_unique_id, &transition_id);
         }
     }
 
-     rc = sqlite3_finalize(res);
-     if (unlikely(rc != SQLITE_OK))
-         error_report("Failed to finalize the statement");
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the statement");
 }
 
 /* Health related SQL queries
@@ -774,12 +739,14 @@ void sql_check_removed_alerts_state(RRDHOST *host)
 */
 #define SQL_LOAD_HEALTH_LOG "SELECT hld.unique_id, hld.alarm_id, hld.alarm_event_id, hl.config_hash_id, hld.updated_by_id, " \
             "hld.updates_id, hld.when_key, hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, " \
-            "hld.delay_up_to_timestamp, hl.name, hl.chart, hl.family, hl.exec, hl.recipient, ah.source, hl.units, " \
+            "hld.delay_up_to_timestamp, hl.name, hl.chart, hl.exec, hl.recipient, ah.source, hl.units, " \
             "hld.info, hld.exec_code, hld.new_status, hld.old_status, hld.delay, hld.new_value, hld.old_value, " \
-            "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.global_id, hl.chart_name " \
+            "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.global_id, hl.chart_name, hld.summary " \
             "FROM health_log hl, alert_hash ah, health_log_detail hld " \
             "WHERE hl.config_hash_id = ah.hash_id and hl.host_id = @host_id and hl.last_transition_id = hld.transition_id;"
-void sql_health_alarm_log_load(RRDHOST *host) {
+
+void sql_health_alarm_log_load(RRDHOST *host)
+{
     sqlite3_stmt *res = NULL;
     int ret;
     ssize_t errored = 0, loaded = 0;
@@ -835,7 +802,7 @@ void sql_health_alarm_log_load(RRDHOST *host) {
             continue;
         }
 
-        //need name, chart and family
+        //need name and chart
         if (sqlite3_column_type(res, 12) == SQLITE_NULL) {
             error_report("HEALTH [%s]: Got null name field. Ignoring it.", rrdhost_hostname(host));
             errored++;
@@ -848,14 +815,8 @@ void sql_health_alarm_log_load(RRDHOST *host) {
             continue;
         }
 
-        if (sqlite3_column_type(res, 14) == SQLITE_NULL) {
-            error_report("HEALTH [%s]: Got null family field. Ignoring it.", rrdhost_hostname(host));
-            errored++;
-            continue;
-        }
-
         // Check if we got last_repeat field
-        time_t last_repeat = (time_t)sqlite3_column_int64(res, 26);
+        time_t last_repeat = (time_t)sqlite3_column_int64(res, 25);
 
         rc = dictionary_get(all_rrdcalcs, (char *) sqlite3_column_text(res, 13));
         if(unlikely(rc)) {
@@ -892,73 +853,36 @@ void sql_health_alarm_log_load(RRDHOST *host) {
 
         ae->name   = string_strdupz((char *) sqlite3_column_text(res, 12));
         ae->chart  = string_strdupz((char *) sqlite3_column_text(res, 13));
-        ae->family = string_strdupz((char *) sqlite3_column_text(res, 14));
-
-        if (sqlite3_column_type(res, 15) != SQLITE_NULL)
-            ae->exec = string_strdupz((char *) sqlite3_column_text(res, 15));
-        else
-            ae->exec = NULL;
-
-        if (sqlite3_column_type(res, 16) != SQLITE_NULL)
-            ae->recipient = string_strdupz((char *) sqlite3_column_text(res, 16));
-        else
-            ae->recipient = NULL;
 
-        if (sqlite3_column_type(res, 17) != SQLITE_NULL)
-            ae->source = string_strdupz((char *) sqlite3_column_text(res, 17));
-        else
-            ae->source = NULL;
-
-        if (sqlite3_column_type(res, 18) != SQLITE_NULL)
-            ae->units = string_strdupz((char *) sqlite3_column_text(res, 18));
-        else
-            ae->units = NULL;
-
-        if (sqlite3_column_type(res, 19) != SQLITE_NULL)
-            ae->info = string_strdupz((char *) sqlite3_column_text(res, 19));
-        else
-            ae->info = NULL;
+        ae->exec = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 14);
+        ae->recipient = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 15);
+        ae->source = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 16);
+        ae->units = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 17);
+        ae->info = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 18);
 
-        ae->exec_code   = (int) sqlite3_column_int(res, 20);
-        ae->new_status  = (RRDCALC_STATUS) sqlite3_column_int(res, 21);
-        ae->old_status  = (RRDCALC_STATUS)sqlite3_column_int(res, 22);
-        ae->delay       = (int) sqlite3_column_int(res, 23);
+        ae->exec_code   = (int) sqlite3_column_int(res, 19);
+        ae->new_status  = (RRDCALC_STATUS) sqlite3_column_int(res, 20);
+        ae->old_status  = (RRDCALC_STATUS)sqlite3_column_int(res, 21);
+        ae->delay       = (int) sqlite3_column_int(res, 22);
 
-        ae->new_value   = (NETDATA_DOUBLE) sqlite3_column_double(res, 24);
-        ae->old_value   = (NETDATA_DOUBLE) sqlite3_column_double(res, 25);
+        ae->new_value   = (NETDATA_DOUBLE) sqlite3_column_double(res, 23);
+        ae->old_value   = (NETDATA_DOUBLE) sqlite3_column_double(res, 24);
 
         ae->last_repeat = last_repeat;
 
-        if (sqlite3_column_type(res, 27) != SQLITE_NULL)
-            ae->classification = string_strdupz((char *) sqlite3_column_text(res, 27));
-        else
-            ae->classification = NULL;
-
-        if (sqlite3_column_type(res, 28) != SQLITE_NULL)
-            ae->component = string_strdupz((char *) sqlite3_column_text(res, 28));
-        else
-            ae->component = NULL;
-
-        if (sqlite3_column_type(res, 29) != SQLITE_NULL)
-            ae->type = string_strdupz((char *) sqlite3_column_text(res, 29));
-        else
-            ae->type = NULL;
+        ae->classification = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 26);
+        ae->component = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 27);
+        ae->type = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 28);
+        ae->chart_context = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 29);
 
         if (sqlite3_column_type(res, 30) != SQLITE_NULL)
-            ae->chart_context = string_strdupz((char *) sqlite3_column_text(res, 30));
-        else
-            ae->chart_context = NULL;
+            uuid_copy(ae->transition_id, *((uuid_t *)sqlite3_column_blob(res, 30)));
 
         if (sqlite3_column_type(res, 31) != SQLITE_NULL)
-            uuid_copy(ae->transition_id, *((uuid_t *)sqlite3_column_blob(res, 31)));
+            ae->global_id = sqlite3_column_int64(res, 31);
 
-        if (sqlite3_column_type(res, 32) != SQLITE_NULL)
-            ae->global_id = sqlite3_column_int64(res, 32);
-
-        if (sqlite3_column_type(res, 33) != SQLITE_NULL)
-            ae->chart_name = string_strdupz((char *) sqlite3_column_text(res, 33));
-        else
-            ae->chart_name = NULL;
+        ae->chart_name = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 32);
+        ae->summary = SQLITE3_COLUMN_STRINGDUP_OR_NULL(res, 33);
 
         char value_string[100 + 1];
         string_freez(ae->old_value_string);
@@ -996,17 +920,20 @@ void sql_health_alarm_log_load(RRDHOST *host) {
     if (unlikely(ret != SQLITE_OK))
         error_report("Failed to finalize the health log read statement");
 
-    sql_health_alarm_log_count(host);
+    int rows = sql_health_alarm_log_count(host);
+
+    if (rows >= 0)
+        host->health.health_log_entries_written = rows;
 }
 
 /*
  * Store an alert config hash in the database
  */
 #define SQL_STORE_ALERT_CONFIG_HASH "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \
-    "on_key, class, component, type, os, hosts, lookup, every, units, calc, families, plugin, module, " \
+    "on_key, class, component, type, os, hosts, lookup, every, units, calc, plugin, module, " \
     "charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \
     "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \
-    "p_db_lookup_before, p_update_every, source, chart_labels) values (?1,unixepoch(),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12," \
+    "p_db_lookup_before, p_update_every, source, chart_labels, summary) values (?1,unixepoch(),?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12," \
     "?13,?14,?15,?16,?17,?18,?19,?20,?21,?22,?23,?24,?25,?26,?27,?28,?29,?30,?31,?32,?33,?34,?35,?36);"
 
 int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
@@ -1033,120 +960,116 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->alarm, ++param);
-    if (unlikely(rc != SQLITE_OK))
-        goto bind_fail;
-
-    rc = sqlite3_bind_string_or_null(res, cfg->template_key, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->alarm, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->on, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->template_key, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->classification, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->on, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->component, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->classification, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->type, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->component, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->os, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->type, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->host, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->os, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->lookup, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->host, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->every, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->lookup, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->units, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->every, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->calc, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->units, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->families, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->calc, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->plugin, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->plugin, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->module, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->module, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->charts, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->charts, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->green, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->green, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->red, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->red, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->warn, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->warn, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->crit, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->crit, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->exec, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->exec, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->to, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->to, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->info, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->info, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->delay, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->delay, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->options, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->options, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->repeat, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->repeat, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->host_labels, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->host_labels, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
     if (cfg->p_db_lookup_after) {
-        rc = sqlite3_bind_string_or_null(res, cfg->p_db_lookup_dimensions, ++param);
+        rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->p_db_lookup_dimensions, ++param);
         if (unlikely(rc != SQLITE_OK))
             goto bind_fail;
 
-        rc = sqlite3_bind_string_or_null(res, cfg->p_db_lookup_method, ++param);
+        rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->p_db_lookup_method, ++param);
         if (unlikely(rc != SQLITE_OK))
             goto bind_fail;
 
@@ -1187,11 +1110,15 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->source, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->source, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
-    rc = sqlite3_bind_string_or_null(res, cfg->chart_labels, ++param);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->chart_labels, ++param);
+    if (unlikely(rc != SQLITE_OK))
+        goto bind_fail;
+
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->summary, ++param);
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
@@ -1238,7 +1165,6 @@ int alert_hash_and_store_config(
     DIGEST_ALERT_CONFIG_VAL(cfg->os);
     DIGEST_ALERT_CONFIG_VAL(cfg->host);
     DIGEST_ALERT_CONFIG_VAL(cfg->on);
-    DIGEST_ALERT_CONFIG_VAL(cfg->families);
     DIGEST_ALERT_CONFIG_VAL(cfg->plugin);
     DIGEST_ALERT_CONFIG_VAL(cfg->module);
     DIGEST_ALERT_CONFIG_VAL(cfg->charts);
@@ -1261,6 +1187,7 @@ int alert_hash_and_store_config(
     DIGEST_ALERT_CONFIG_VAL(cfg->repeat);
     DIGEST_ALERT_CONFIG_VAL(cfg->host_labels);
     DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels);
+    DIGEST_ALERT_CONFIG_VAL(cfg->summary);
 
     EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
     EVP_MD_CTX_destroy(evpctx);
@@ -1282,16 +1209,17 @@ int alert_hash_and_store_config(
     return 1;
 }
 
-#define SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT "SELECT hld.new_status FROM health_log hl, health_log_detail hld WHERE hl.alarm_id = %u AND hld.unique_id != %u AND hld.flags & %u AND hl.host_id = @host_id and hl.health_log_id = hld.health_log_id ORDER BY hld.unique_id DESC LIMIT 1;"
+#define SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT                                                                          \
+    "SELECT hld.new_status FROM health_log hl, health_log_detail hld "                                                 \
+    "WHERE hl.host_id = @host_id AND hl.alarm_id = @alarm_id AND hld.unique_id != @unique_id AND hld.flags & @flags "  \
+    "AND hl.health_log_id = hld.health_log_id ORDER BY hld.unique_id DESC LIMIT 1;"
+
 int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status)
 {
     int rc = 0, ret = -1;
-    char command[MAX_HEALTH_SQL_SIZE + 1];
     sqlite3_stmt *res = NULL;
 
-    snprintfz(command, MAX_HEALTH_SQL_SIZE, SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT, ae->alarm_id, ae->unique_id, (uint32_t) HEALTH_ENTRY_FLAG_EXEC_RUN);
-
-    rc = sqlite3_prepare_v2(db_meta, command, -1, &res, 0);
+    rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT, -1, &res, 0);
     if (rc != SQLITE_OK) {
         error_report("Failed to prepare statement when trying to get last executed status");
         return ret;
@@ -1300,8 +1228,25 @@ int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_S
     rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind host_id parameter for SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT.");
-        sqlite3_finalize(res);
-        return ret;
+        goto done;
+    }
+
+    rc = sqlite3_bind_int(res, 2, (int) ae->alarm_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind alarm_id parameter for SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT.");
+        goto done;
+    }
+
+    rc = sqlite3_bind_int(res, 3, (int) ae->unique_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind unique_id parameter for SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT.");
+        goto done;
+    }
+
+    rc = sqlite3_bind_int(res, 4, (uint32_t) HEALTH_ENTRY_FLAG_EXEC_RUN);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind unique_id parameter for SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT.");
+        goto done;
     }
 
     ret = 0;
@@ -1310,6 +1255,7 @@ int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_S
         ret = 1;
     }
 
+done:
      rc = sqlite3_finalize(res);
      if (unlikely(rc != SQLITE_OK))
          error_report("Failed to finalize the statement.");
@@ -1317,7 +1263,15 @@ int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_S
      return ret;
 }
 
-#define SQL_SELECT_HEALTH_LOG "SELECT hld.unique_id, hld.alarm_id, hld.alarm_event_id, hl.config_hash_id, hld.updated_by_id, hld.updates_id, hld.when_key, hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, hld.delay_up_to_timestamp, hl.name, hl.chart, hl.family, hl.exec, hl.recipient, ah.source, hl.units, hld.info, hld.exec_code, hld.new_status, hld.old_status, hld.delay, hld.new_value, hld.old_value, hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id FROM health_log hl, alert_hash ah, health_log_detail hld WHERE hl.config_hash_id = ah.hash_id and hl.health_log_id = hld.health_log_id and hl.host_id = @host_id "
+#define SQL_SELECT_HEALTH_LOG                                                                                          \
+     "SELECT hld.unique_id, hld.alarm_id, hld.alarm_event_id, hl.config_hash_id, hld.updated_by_id, hld.updates_id, "  \
+     "hld.when_key, hld.duration, hld.non_clear_duration, hld.flags, hld.exec_run_timestamp, "                         \
+     "hld.delay_up_to_timestamp, hl.name, hl.chart, hl.exec, hl.recipient, ah.source, "                                \
+     "hl.units, hld.info, hld.exec_code, hld.new_status, hld.old_status, hld.delay, hld.new_value, hld.old_value, "    \
+     "hld.last_repeat, ah.class, ah.component, ah.type, hl.chart_context, hld.transition_id, hld.summary "             \
+     "FROM health_log hl, alert_hash ah, health_log_detail hld WHERE hl.config_hash_id = ah.hash_id and "              \
+     "hl.health_log_id = hld.health_log_id and hl.host_id = @host_id "
+
 void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *chart) {
 
     buffer_strcat(wb, "[");
@@ -1373,10 +1327,10 @@ void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *
         uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 3)), config_hash_id);
 
         char transition_id[UUID_STR_LEN] = {0};
-        if (sqlite3_column_type(res, 31) != SQLITE_NULL)
-            uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 31)), transition_id);
+        if (sqlite3_column_type(res, 30) != SQLITE_NULL)
+            uuid_unparse_lower(*((uuid_t *) sqlite3_column_blob(res, 30)), transition_id);
 
-        char *edit_command = sqlite3_column_bytes(res, 17) > 0 ? health_edit_command_from_source((char *)sqlite3_column_text(res, 17)) : strdupz("UNKNOWN=0=UNKNOWN");
+        char *edit_command = sqlite3_column_bytes(res, 16) > 0 ? health_edit_command_from_source((char *)sqlite3_column_text(res, 16)) : strdupz("UNKNOWN=0=UNKNOWN");
 
         if (count)
             buffer_sprintf(wb, ",");
@@ -1397,7 +1351,6 @@ void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *
             "\t\t\"name\": \"%s\",\n"
             "\t\t\"chart\": \"%s\",\n"
             "\t\t\"context\": \"%s\",\n"
-            "\t\t\"family\": \"%s\",\n"
             "\t\t\"class\": \"%s\",\n"
             "\t\t\"component\": \"%s\",\n"
             "\t\t\"type\": \"%s\",\n"
@@ -1422,7 +1375,7 @@ void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *
             "\t\t\"updates_id\": %u,\n"
             "\t\t\"value_string\": \"%s\",\n"
             "\t\t\"old_value_string\": \"%s\",\n"
-            "\t\t\"last_repeat\": \"%lu\",\n"
+            "\t\t\"last_repeat\": %lu,\n"
             "\t\t\"silenced\": \"%s\",\n",
             rrdhost_hostname(host),
             host->utc_offset,
@@ -1434,53 +1387,53 @@ void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, uint32_t after, char *
             transition_id,
             sqlite3_column_text(res, 12),
             sqlite3_column_text(res, 13),
-            sqlite3_column_text(res, 30),
-            sqlite3_column_text(res, 14),
+            sqlite3_column_text(res, 29),
+            sqlite3_column_text(res, 26) ? (const char *) sqlite3_column_text(res, 26) : (char *) "Unknown",
             sqlite3_column_text(res, 27) ? (const char *) sqlite3_column_text(res, 27) : (char *) "Unknown",
             sqlite3_column_text(res, 28) ? (const char *) sqlite3_column_text(res, 28) : (char *) "Unknown",
-            sqlite3_column_text(res, 29) ? (const char *) sqlite3_column_text(res, 29) : (char *) "Unknown",
             (sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_PROCESSED)?"true":"false",
             (sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_UPDATED)?"true":"false",
             (long unsigned int)sqlite3_column_int64(res, 10),
             (sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_EXEC_FAILED)?"true":"false",
-            sqlite3_column_text(res, 15) ? (const char *) sqlite3_column_text(res, 15) : string2str(host->health.health_default_exec),
-            sqlite3_column_text(res, 16) ? (const char *) sqlite3_column_text(res, 16) : string2str(host->health.health_default_recipient),
-            sqlite3_column_int(res, 20),
-            sqlite3_column_text(res, 17) ? (const char *) sqlite3_column_text(res, 17) : (char *) "Unknown",
+            sqlite3_column_text(res, 14) ? (const char *) sqlite3_column_text(res, 14) : string2str(host->health.health_default_exec),
+            sqlite3_column_text(res, 15) ? (const char *) sqlite3_column_text(res, 15) : string2str(host->health.health_default_recipient),
+            sqlite3_column_int(res, 19),
+            sqlite3_column_text(res, 16) ? (const char *) sqlite3_column_text(res, 16) : (char *) "Unknown",
             edit_command,
-            sqlite3_column_text(res, 18),
+            sqlite3_column_text(res, 17),
             (long unsigned int)sqlite3_column_int64(res, 6),
             (long unsigned int)sqlite3_column_int64(res, 7),
             (long unsigned int)sqlite3_column_int64(res, 8),
+            rrdcalc_status2string(sqlite3_column_int(res, 20)),
             rrdcalc_status2string(sqlite3_column_int(res, 21)),
-            rrdcalc_status2string(sqlite3_column_int(res, 22)),
-            sqlite3_column_int(res, 23),
+            sqlite3_column_int(res, 22),
             (long unsigned int)sqlite3_column_int64(res, 11),
             (unsigned int)sqlite3_column_int64(res, 4),
             (unsigned int)sqlite3_column_int64(res, 5),
-            sqlite3_column_type(res, 24) == SQLITE_NULL ? "-" : format_value_and_unit(new_value_string, 100, sqlite3_column_double(res, 24), (char *) sqlite3_column_text(res, 18), -1),
-            sqlite3_column_type(res, 25) == SQLITE_NULL ? "-" : format_value_and_unit(old_value_string, 100, sqlite3_column_double(res, 25), (char *) sqlite3_column_text(res, 18), -1),
-            (long unsigned int)sqlite3_column_int64(res, 26),
+            sqlite3_column_type(res, 23) == SQLITE_NULL ? "-" : format_value_and_unit(new_value_string, 100, sqlite3_column_double(res, 23), (char *) sqlite3_column_text(res, 17), -1),
+            sqlite3_column_type(res, 24) == SQLITE_NULL ? "-" : format_value_and_unit(old_value_string, 100, sqlite3_column_double(res, 24), (char *) sqlite3_column_text(res, 17), -1),
+            (long unsigned int)sqlite3_column_int64(res, 25),
             (sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_SILENCED)?"true":"false");
 
-        health_string2json(wb, "\t\t", "info", (char *) sqlite3_column_text(res, 19), ",\n");
+        health_string2json(wb, "\t\t", "summary", (char *) sqlite3_column_text(res, 31), ",\n");
+        health_string2json(wb, "\t\t", "info", (char *) sqlite3_column_text(res, 18), ",\n");
 
         if(unlikely(sqlite3_column_int64(res, 9) & HEALTH_ENTRY_FLAG_NO_CLEAR_NOTIFICATION)) {
             buffer_strcat(wb, "\t\t\"no_clear_notification\": true,\n");
         }
 
         buffer_strcat(wb, "\t\t\"value\":");
-        if (sqlite3_column_type(res, 24) == SQLITE_NULL)
+        if (sqlite3_column_type(res, 23) == SQLITE_NULL)
             buffer_strcat(wb, "null");
         else
-            buffer_print_netdata_double(wb, sqlite3_column_double(res, 24));
+            buffer_print_netdata_double(wb, sqlite3_column_double(res, 23));
         buffer_strcat(wb, ",\n");
 
         buffer_strcat(wb, "\t\t\"old_value\":");
-        if (sqlite3_column_type(res, 25) == SQLITE_NULL)
+        if (sqlite3_column_type(res, 24) == SQLITE_NULL)
             buffer_strcat(wb, "null");
         else
-            buffer_print_netdata_double(wb, sqlite3_column_double(res, 25));
+            buffer_print_netdata_double(wb, sqlite3_column_double(res, 24));
         buffer_strcat(wb, "\n");
 
         buffer_strcat(wb, "\t}");
@@ -1647,8 +1600,50 @@ int health_migrate_old_health_log_table(char *table) {
     return 1;
 }
 
-#define SQL_GET_ALARM_ID "select alarm_id, health_log_id from health_log where host_id = @host_id and chart = @chart and name = @name and config_hash_id = @config_hash_id"
-#define SQL_GET_EVENT_ID "select max(alarm_event_id) + 1 from health_log_detail where health_log_id = @health_log_id and alarm_id = @alarm_id"
+#define SQL_GET_EVENT_ID                                                                                               \
+    "SELECT MAX(alarm_event_id)+1 FROM health_log_detail WHERE health_log_id = @health_log_id AND alarm_id = @alarm_id"
+
+static uint32_t get_next_alarm_event_id(uint64_t health_log_id, uint32_t alarm_id)
+{
+    int rc;
+    sqlite3_stmt *res = NULL;
+    uint32_t next_event_id = 0;
+
+    rc = sqlite3_prepare_v2(db_meta, SQL_GET_EVENT_ID, -1, &res, 0);
+    if (rc != SQLITE_OK) {
+        error_report("Failed to prepare statement when trying to get an event id");
+        return alarm_id;
+    }
+
+    rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) health_log_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind host_id parameter for SQL_GET_EVENT_ID.");
+        sqlite3_finalize(res);
+        return alarm_id;
+    }
+
+    rc = sqlite3_bind_int64(res, 2, (sqlite3_int64) alarm_id);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind char parameter for SQL_GET_EVENT_ID.");
+        sqlite3_finalize(res);
+        return alarm_id;
+    }
+
+    while (sqlite3_step_monitored(res) == SQLITE_ROW) {
+        next_event_id = (uint32_t) sqlite3_column_int64(res, 0);
+    }
+
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the statement while getting an alarm id.");
+
+    return next_event_id;
+}
+
+#define SQL_GET_ALARM_ID                                                                                               \
+    "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart "                      \
+    "AND name = @name AND config_hash_id = @config_hash_id"
+
 uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id)
 {
     int rc = 0;
@@ -1669,14 +1664,14 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *
         return alarm_id;
     }
 
-    rc = sqlite3_bind_string_or_null(res, chart, 2);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, chart, 2);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind char parameter for SQL_GET_ALARM_ID.");
         sqlite3_finalize(res);
         return alarm_id;
     }
 
-    rc = sqlite3_bind_string_or_null(res, name, 3);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, name, 3);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind name parameter for SQL_GET_ALARM_ID.");
         sqlite3_finalize(res);
@@ -1699,40 +1694,16 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *
      if (unlikely(rc != SQLITE_OK))
          error_report("Failed to finalize the statement while getting an alarm id.");
 
-     if (alarm_id) {
-         rc = sqlite3_prepare_v2(db_meta, SQL_GET_EVENT_ID, -1, &res, 0);
-         if (rc != SQLITE_OK) {
-             error_report("Failed to prepare statement when trying to get an event id");
-             return alarm_id;
-         }
-
-         rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) health_log_id);
-         if (unlikely(rc != SQLITE_OK)) {
-             error_report("Failed to bind host_id parameter for SQL_GET_EVENT_ID.");
-             sqlite3_finalize(res);
-             return alarm_id;
-         }
-
-         rc = sqlite3_bind_int64(res, 2, (sqlite3_int64) alarm_id);
-         if (unlikely(rc != SQLITE_OK)) {
-             error_report("Failed to bind char parameter for SQL_GET_EVENT_ID.");
-             sqlite3_finalize(res);
-             return alarm_id;
-         }
-
-         while (sqlite3_step_monitored(res) == SQLITE_ROW) {
-             *next_event_id = (uint32_t) sqlite3_column_int64(res, 0);
-         }
-
-         rc = sqlite3_finalize(res);
-         if (unlikely(rc != SQLITE_OK))
-             error_report("Failed to finalize the statement while getting an alarm id.");
-     }
+     if (alarm_id)
+           *next_event_id = get_next_alarm_event_id(health_log_id, alarm_id);
 
      return alarm_id;
 }
 
-#define SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH "update health_log set config_hash_id = @config_hash_id where host_id = @host_id and alarm_id = @alarm_id and health_log_id = @health_log_id"
+#define SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH                                                                           \
+     "UPDATE health_log SET config_hash_id = @config_hash_id WHERE host_id = @host_id AND alarm_id = @alarm_id "       \
+     "AND health_log_id = @health_log_id"
+
 void sql_update_alarm_with_config_hash(RRDHOST *host, uint32_t alarm_id, uint64_t health_log_id, uuid_t *config_hash_id)
 {
     int rc = 0;
@@ -1747,42 +1718,42 @@ void sql_update_alarm_with_config_hash(RRDHOST *host, uint32_t alarm_id, uint64_
     rc = sqlite3_bind_blob(res, 1, config_hash_id, sizeof(*config_hash_id), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind config_hash_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH.");
-        sqlite3_finalize(res);
-        return;
+        goto done;
     }
 
     rc = sqlite3_bind_blob(res, 2, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind host_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH.");
-        sqlite3_finalize(res);
-        return;
+        goto done;
     }
 
     rc = sqlite3_bind_int64(res, 3, (sqlite3_int64) alarm_id);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID.");
-        sqlite3_finalize(res);
-        return;
+        goto done;
     }
 
     rc = sqlite3_bind_int64(res, 4, (sqlite3_int64) health_log_id);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID.");
-        sqlite3_finalize(res);
-        return;
+        goto done;
     }
 
     rc = execute_insert(res);
-    if (unlikely(rc != SQLITE_DONE)) {
+    if (unlikely(rc != SQLITE_DONE))
         error_report("Failed to execute SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, rc = %d", rc);
-        rc = sqlite3_finalize(res);
-        if (unlikely(rc != SQLITE_OK))
-            error_report("Failed to reset statement to update health log detail table with config hash ids, rc = %d", rc);
-        return;
-    }
+
+done:
+     rc = sqlite3_finalize(res);
+     if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to reset statement to update health log detail table with config hash ids, rc = %d", rc);
+
 }
 
-#define SQL_GET_ALARM_ID_CHECK_ZERO_HASH "select alarm_id, health_log_id from health_log where host_id = @host_id and chart = @chart and name = @name and (config_hash_id is null or config_hash_id = zeroblob(16))"
+#define SQL_GET_ALARM_ID_CHECK_ZERO_HASH                                                                               \
+     "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart "                     \
+     "AND name = @name AND (config_hash_id IS NULL OR config_hash_id = ZEROBLOB(16))"
+
 uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id)
 {
     int rc = 0;
@@ -1803,14 +1774,14 @@ uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *
         return alarm_id;
     }
 
-    rc = sqlite3_bind_string_or_null(res, chart, 2);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, chart, 2);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind char parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH.");
         sqlite3_finalize(res);
         return alarm_id;
     }
 
-    rc = sqlite3_bind_string_or_null(res, name, 3);
+    rc = SQLITE3_BIND_STRING_OR_NULL(res, name, 3);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind name parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH.");
         sqlite3_finalize(res);
@@ -1828,44 +1799,21 @@ uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *
 
      if (alarm_id) {
          sql_update_alarm_with_config_hash(host, alarm_id, health_log_id, config_hash_id);
-
-         rc = sqlite3_prepare_v2(db_meta, SQL_GET_EVENT_ID, -1, &res, 0);
-         if (rc != SQLITE_OK) {
-             error_report("Failed to prepare statement when trying to get an event id");
-             return alarm_id;
-         }
-
-         rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) health_log_id);
-         if (unlikely(rc != SQLITE_OK)) {
-             error_report("Failed to bind host_id parameter for SQL_GET_EVENT_ID.");
-             sqlite3_finalize(res);
-             return alarm_id;
-         }
-
-         rc = sqlite3_bind_int64(res, 2, (sqlite3_int64) alarm_id);
-         if (unlikely(rc != SQLITE_OK)) {
-             error_report("Failed to bind char parameter for SQL_GET_EVENT_ID.");
-             sqlite3_finalize(res);
-             return alarm_id;
-         }
-
-         while (sqlite3_step_monitored(res) == SQLITE_ROW) {
-             *next_event_id = (uint32_t) sqlite3_column_int64(res, 0);
-         }
-
-         rc = sqlite3_finalize(res);
-         if (unlikely(rc != SQLITE_OK))
-             error_report("Failed to finalize the statement while getting an alarm id.");
+         *next_event_id = get_next_alarm_event_id(health_log_id, alarm_id);
      }
 
      return alarm_id;
 }
 
-#define SQL_GET_ALARM_ID_FROM_TRANSITION_ID "SELECT hld.alarm_id, hl.host_id, hl.chart_context FROM " \
-        "health_log_detail hld, health_log hl WHERE hld.transition_id = @transition_id " \
-        "and hld.health_log_id = hl.health_log_id"
+#define SQL_GET_ALARM_ID_FROM_TRANSITION_ID                                                                            \
+     "SELECT hld.alarm_id, hl.host_id, hl.chart_context FROM health_log_detail hld, health_log hl "                    \
+     "WHERE hld.transition_id = @transition_id "                                                                       \
+     "AND hld.health_log_id = hl.health_log_id"
 
-bool sql_find_alert_transition(const char *transition, void (*cb)(const char *machine_guid, const char *context, time_t alert_id, void *data), void *data)
+bool sql_find_alert_transition(
+    const char *transition,
+    void (*cb)(const char *machine_guid, const char *context, time_t alert_id, void *data),
+    void *data)
 {
     static __thread sqlite3_stmt *res = NULL;
 
@@ -1889,7 +1837,7 @@ bool sql_find_alert_transition(const char *transition, void (*cb)(const char *ma
     rc = sqlite3_bind_blob(res, 1, &transition_uuid, sizeof(transition_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind transition");
-        goto fail;
+        goto done;
     }
 
     while (sqlite3_step_monitored(res) == SQLITE_ROW) {
@@ -1898,7 +1846,7 @@ bool sql_find_alert_transition(const char *transition, void (*cb)(const char *ma
         cb(machine_guid, (const char *) sqlite3_column_text(res, 2), sqlite3_column_int(res, 0), data);
     }
 
-fail:
+done:
     rc = sqlite3_reset(res);
     if (unlikely(rc != SQLITE_OK))
         error_report("Failed to reset the statement when trying to find transition");
@@ -1910,20 +1858,24 @@ fail:
 
 #define SQL_POPULATE_TEMP_ALERT_TRANSITION_TABLE "INSERT INTO v_%p (host_id) VALUES (@host_id)"
 
-#define SQL_SEARCH_ALERT_TRANSITION_SELECT "SELECT " \
-    "h.host_id, h.alarm_id, h.config_hash_id, h.name, h.chart, h.chart_name, h.family, h.recipient, h.units, h.exec, " \
-    "h.chart_context,  d.when_key, d.duration, d.non_clear_duration, d.flags, d.delay_up_to_timestamp, " \
-    "d.info, d.exec_code, d.new_status, d.old_status, d.delay, d.new_value, d.old_value, d.last_repeat, " \
-    "d.transition_id, d.global_id, ah.class, ah.type, ah.component, d.exec_run_timestamp"
+#define SQL_SEARCH_ALERT_TRANSITION_SELECT                                                                                    \
+    "SELECT h.host_id, h.alarm_id, h.config_hash_id, h.name, h.chart, h.chart_name, h.family, h.recipient, h.units, h.exec, " \
+    "h.chart_context,  d.when_key, d.duration, d.non_clear_duration, d.flags, d.delay_up_to_timestamp, "                      \
+    "d.info, d.exec_code, d.new_status, d.old_status, d.delay, d.new_value, d.old_value, d.last_repeat, "                     \
+    "d.transition_id, d.global_id, ah.class, ah.type, ah.component, d.exec_run_timestamp, d.summary"
 
-#define SQL_SEARCH_ALERT_TRANSITION_COMMON_WHERE \
-    "h.config_hash_id = ah.hash_id AND h.health_log_id = d.health_log_id"
+#define SQL_SEARCH_ALERT_TRANSITION_COMMON_WHERE "h.config_hash_id = ah.hash_id AND h.health_log_id = d.health_log_id"
 
-#define SQL_SEARCH_ALERT_TRANSITION SQL_SEARCH_ALERT_TRANSITION_SELECT " FROM health_log h, health_log_detail d, v_%p t, alert_hash ah " \
-    " WHERE h.host_id = t.host_id AND " SQL_SEARCH_ALERT_TRANSITION_COMMON_WHERE " AND ( d.new_status > 2 OR d.old_status > 2 ) AND d.global_id BETWEEN @after AND @before "
+#define SQL_SEARCH_ALERT_TRANSITION                                                                                    \
+    SQL_SEARCH_ALERT_TRANSITION_SELECT                                                                                 \
+        " FROM health_log h, health_log_detail d, v_%p t, alert_hash ah "                                              \
+        " WHERE h.host_id = t.host_id AND " SQL_SEARCH_ALERT_TRANSITION_COMMON_WHERE                                   \
+        " AND ( d.new_status > 2 OR d.old_status > 2 ) AND d.global_id BETWEEN @after AND @before "
 
-#define SQL_SEARCH_ALERT_TRANSITION_DIRECT SQL_SEARCH_ALERT_TRANSITION_SELECT " FROM health_log h, health_log_detail d, alert_hash ah " \
-    " WHERE " SQL_SEARCH_ALERT_TRANSITION_COMMON_WHERE " AND transition_id = @transition "
+#define SQL_SEARCH_ALERT_TRANSITION_DIRECT                                                                             \
+    SQL_SEARCH_ALERT_TRANSITION_SELECT " FROM health_log h, health_log_detail d, alert_hash ah "                       \
+                                       " WHERE " SQL_SEARCH_ALERT_TRANSITION_COMMON_WHERE                              \
+                                       " AND transition_id = @transition "
 
 void sql_alert_transitions(
     DICTIONARY *nodes,
@@ -1956,7 +1908,7 @@ void sql_alert_transitions(
         rc = sqlite3_bind_blob(res, 1, &transition_uuid, sizeof(transition_uuid), SQLITE_STATIC);
         if (unlikely(rc != SQLITE_OK)) {
             error_report("Failed to bind transition_id parameter");
-            goto fail;
+            goto done;
         }
         goto run_query;
     }
@@ -1972,7 +1924,7 @@ void sql_alert_transitions(
     rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to prepare statement to INSERT into v_%p", nodes);
-        goto fail_only_drop;
+        goto done_only_drop;
     }
 
     void *t;
@@ -2015,27 +1967,27 @@ void sql_alert_transitions(
     rc = sqlite3_prepare_v2(db_meta, buffer_tostring(command), -1, &res, 0);
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to prepare statement sql_alert_transitions");
-        goto fail_only_drop;
+        goto done_only_drop;
     }
 
     int param = 1;
     rc = sqlite3_bind_int64(res, param++, (sqlite3_int64)(after * USEC_PER_SEC));
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind after parameter");
-        goto fail;
+        goto done;
     }
 
     rc = sqlite3_bind_int64(res, param++, (sqlite3_int64)(before * USEC_PER_SEC));
     if (unlikely(rc != SQLITE_OK)) {
         error_report("Failed to bind before parameter");
-        goto fail;
+        goto done;
     }
 
     if (context) {
         rc = sqlite3_bind_text(res, param++, context, -1, SQLITE_STATIC);
         if (unlikely(rc != SQLITE_OK)) {
             error_report("Failed to bind context parameter");
-            goto fail;
+            goto done;
         }
     }
 
@@ -2043,7 +1995,7 @@ void sql_alert_transitions(
         rc = sqlite3_bind_text(res, param++, alert_name, -1, SQLITE_STATIC);
         if (unlikely(rc != SQLITE_OK)) {
             error_report("Failed to bind alert_name parameter");
-            goto fail;
+            goto done;
         }
     }
 
@@ -2082,16 +2034,17 @@ run_query:;
         atd.type = (const char *) sqlite3_column_text(res, 27);
         atd.component = (const char *) sqlite3_column_text(res, 28);
         atd.exec_run_timestamp = sqlite3_column_int64(res, 29);
+        atd.summary = (const char *) sqlite3_column_text(res, 30);
 
         cb(&atd, data);
     }
 
-fail:
+done:
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
         error_report("Failed to finalize statement for sql_alert_transitions");
 
-fail_only_drop:
+done_only_drop:
     if (likely(!transition)) {
         (void)snprintfz(sql, 511, "DROP TABLE IF EXISTS v_%p", nodes);
         (void)db_execute(db_meta, sql);
@@ -2103,10 +2056,11 @@ fail_only_drop:
 
 #define SQL_POPULATE_TEMP_CONFIG_TARGET_TABLE "INSERT INTO c_%p (hash_id) VALUES (@hash_id)"
 
-#define SQL_SEARCH_CONFIG_LIST "SELECT ah.hash_id, alarm, template, on_key, class, component, type, os, hosts, lookup, every, " \
-    " units, calc, families, plugin, module, charts, green, red, warn, crit, " \
-    " exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, " \
-    " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels " \
+#define SQL_SEARCH_CONFIG_LIST                                                                                         \
+    "SELECT ah.hash_id, alarm, template, on_key, class, component, type, os, hosts, lookup, every, "                   \
+    " units, calc, families, plugin, module, charts, green, red, warn, crit, "                                         \
+    " exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, "           \
+    " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary "      \
     " FROM alert_hash ah, c_%p t where ah.hash_id = t.hash_id"
 
 int sql_get_alert_configuration(
@@ -2215,6 +2169,7 @@ int sql_get_alert_configuration(
         acd.value.update_every = (int32_t) sqlite3_column_int(res, param++);
         acd.source = (const char *) sqlite3_column_text(res, param++);
         acd.selectors.chart_labels = (const char *) sqlite3_column_text(res, param++);
+        acd.summary = (const char *) sqlite3_column_text(res, param++);
 
         cb(&acd, data);
         added++;
@@ -2230,69 +2185,3 @@ fail_only_drop:
     buffer_free(command);
     return added;
 }
-
-#define SQL_FETCH_CHART_NAME "SELECT chart_name FROM health_log where host_id = @host_id LIMIT 1;"
-bool is_chart_name_populated(uuid_t  *host_uuid)
-{
-    sqlite3_stmt *res = NULL;
-    int rc;
-
-    bool status = true;
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_FETCH_CHART_NAME, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to check health_log chart_name");
-        return true;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id for health_log chart_name check");
-        goto fail;
-    }
-
-    rc = sqlite3_step_monitored(res);
-    if (likely(rc == SQLITE_ROW))
-        status = sqlite3_column_type(res, 0) != SQLITE_NULL;
-fail:
-
-    rc = sqlite3_finalize(res);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement for health_log chart_name check");
-
-    return status;
-}
-
-#define SQL_POPULATE_CHART_NAME " UPDATE health_log SET chart_name = upd.chart_name FROM " \
-    "(SELECT c.type || '.' || IFNULL(c.name, c.id) AS chart_name, hl.host_id, hl.health_log_id FROM " \
-    "chart c, health_log hl WHERE (c.type || '.' || c.id) = hl.chart AND c.host_id = hl.host_id " \
-    "AND hl.host_id = @host_id) AS upd WHERE health_log.host_id = upd.host_id " \
-    "AND health_log.health_log_id = upd.health_log_id"
-
-void chart_name_populate(uuid_t *host_uuid)
-{
-    sqlite3_stmt *res = NULL;
-    int rc;
-
-    rc = sqlite3_prepare_v2(db_meta, SQL_POPULATE_CHART_NAME, -1, &res, 0);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to prepare statement to update health_log chart_name");
-        return;
-    }
-
-    rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
-    if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to bind host_id for health_log chart_name update");
-        goto fail;
-    }
-
-    rc = execute_insert(res);
-    if (unlikely(rc != SQLITE_DONE))
-        error_report("Failed to update chart name in health_log, rc = %d", rc);
-
-fail:
-
-    rc = sqlite3_finalize(res);
-    if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement for health_log chart_name update");
-}
diff --git a/database/sqlite/sqlite_health.h b/database/sqlite/sqlite_health.h
index 3aebb94b7..e21912368 100644
--- a/database/sqlite/sqlite_health.h
+++ b/database/sqlite/sqlite_health.h
@@ -7,12 +7,9 @@
 
 struct sql_alert_transition_data;
 struct sql_alert_config_data;
-extern sqlite3 *db_meta;
 void sql_health_alarm_log_load(RRDHOST *host);
-void sql_health_alarm_log_update(RRDHOST *host, ALARM_ENTRY *ae);
-void sql_health_alarm_log_insert(RRDHOST *host, ALARM_ENTRY *ae);
 void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
-void sql_health_alarm_log_cleanup(RRDHOST *host);
+void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed);
 int alert_hash_and_store_config(uuid_t hash_id, struct alert_config *cfg, int store_hash);
 void sql_aclk_alert_clean_dead_entries(RRDHOST *host);
 int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status);
@@ -38,6 +35,4 @@ int sql_get_alert_configuration(
     bool debug __maybe_unused);
 
 bool sql_find_alert_transition(const char *transition, void (*cb)(const char *machine_guid, const char *context, time_t alert_id, void *data), void *data);
-bool is_chart_name_populated(uuid_t  *host_uuid);
-void chart_name_populate(uuid_t *host_uuid);
 #endif //NETDATA_SQLITE_HEALTH_H
diff --git a/database/sqlite/sqlite_metadata.c b/database/sqlite/sqlite_metadata.c
index 697772bf5..143783163 100644
--- a/database/sqlite/sqlite_metadata.c
+++ b/database/sqlite/sqlite_metadata.c
@@ -11,52 +11,68 @@
 #define SQL_DELETE_HOST_LABELS  "DELETE FROM host_label WHERE host_id = @uuid;"
 
 #define STORE_HOST_LABEL                                                                                               \
-    "INSERT OR REPLACE INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "
+    "INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "
 
 #define STORE_CHART_LABEL                                                                                              \
-    "INSERT OR REPLACE INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "
+    "INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "
 
 #define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"
 
 #define DELETE_DIMENSION_UUID   "DELETE FROM dimension WHERE dim_id = @uuid;"
 
-#define SQL_STORE_HOST_INFO "INSERT OR REPLACE INTO host " \
-        "(host_id, hostname, registry_hostname, update_every, os, timezone," \
-        "tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, program_version," \
-        "entries, health_enabled) " \
-        "values (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, @memory_mode, " \
-        "@abbrev_timezone, @utc_offset, @program_name, @program_version, " \
-        "@entries, @health_enabled);"
-
-#define SQL_STORE_CHART "insert or replace into chart (chart_id, host_id, type, id, " \
-    "name, family, context, title, unit, plugin, module, priority, update_every , chart_type , memory_mode , " \
-    "history_entries) values (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16);"
-
-#define SQL_STORE_DIMENSION "INSERT OR REPLACE INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \
-        "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options);"
+#define SQL_STORE_HOST_INFO                                                                                              \
+    "INSERT OR REPLACE INTO host (host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, "        \
+    "memory_mode, abbrev_timezone, utc_offset, program_name, program_version, entries, health_enabled, last_connected) " \
+    "VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, "                     \
+    "@memory_mode, @abbrev_tz, @utc_offset, @prog_name, @prog_version, @entries, @health_enabled, @last_connected);"
+
+#define SQL_STORE_CHART                                                                                                \
+    "INSERT INTO chart (chart_id, host_id, type, id, name, family, context, title, unit, plugin, module, priority, "   \
+    "update_every, chart_type, memory_mode, history_entries) "                                                         \
+    "values (@chart_id, @host_id, @type, @id, @name, @family, @context, @title, @unit, @plugin, @module, @priority, "  \
+    "@update_every, @chart_type, @memory_mode, @history_entries) "                                                     \
+    "ON CONFLICT(chart_id) DO UPDATE SET type=excluded.type, id=excluded.id, name=excluded.name, "                     \
+    "family=excluded.family, context=excluded.context, title=excluded.title, unit=excluded.unit, "                     \
+    "plugin=excluded.plugin, module=excluded.module, priority=excluded.priority, update_every=excluded.update_every, " \
+    "chart_type=excluded.chart_type, memory_mode = excluded.memory_mode, history_entries = excluded.history_entries"
+
+#define SQL_STORE_DIMENSION                                                                                            \
+    "INSERT INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) "                    \
+    "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options) "                            \
+    "ON CONFLICT(dim_id) DO UPDATE SET id=excluded.id, name=excluded.name, multiplier=excluded.multiplier, "           \
+    "divisor=excluded.divisor, algorithm=excluded.algorithm, options=excluded.options"
 
 #define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id"
+#define SELECT_CHART_LIST "SELECT chart_id, rowid FROM chart WHERE rowid > @row_id"
+#define SELECT_CHART_LABEL_LIST "SELECT chart_id, rowid FROM chart_label WHERE rowid > @row_id"
 
-#define SQL_STORE_HOST_SYSTEM_INFO_VALUES "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \
-    "(@uuid, @name, @value, unixepoch())"
+#define SQL_STORE_HOST_SYSTEM_INFO_VALUES                                                                              \
+    "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES "                       \
+    "(@uuid, @name, @value, UNIXEPOCH())"
 
 #define MIGRATE_LOCALHOST_TO_NEW_MACHINE_GUID                                                                          \
     "UPDATE chart SET host_id = @host_id WHERE host_id in (SELECT host_id FROM host where host_id <> @host_id and hops = 0);"
 #define DELETE_NON_EXISTING_LOCALHOST "DELETE FROM host WHERE hops = 0 AND host_id <> @host_id;"
 #define DELETE_MISSING_NODE_INSTANCES "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);"
 
-#define METADATA_CMD_Q_MAX_SIZE (1024)              // Max queue size; callers will block until there is room
+#define METADATA_CMD_Q_MAX_SIZE (2048)              // Max queue size; callers will block until there is room
 #define METADATA_MAINTENANCE_FIRST_CHECK (1800)     // Maintenance first run after agent startup in seconds
-#define METADATA_MAINTENANCE_RETRY (60)             // Retry run if already running or last run did actual work
-#define METADATA_MAINTENANCE_INTERVAL (3600)        // Repeat maintenance after latest successful
+#define METADATA_MAINTENANCE_REPEAT (60)            // Repeat if last run for dimensions, charts, labels needs more work
+#define METADATA_HEALTH_LOG_INTERVAL (3600)         // Repeat maintenance for health
+#define METADATA_DIM_CHECK_INTERVAL (3600)          // Repeat maintenance for dimensions
+#define METADATA_CHART_CHECK_INTERVAL (3600)        // Repeat maintenance for charts
+#define METADATA_LABEL_CHECK_INTERVAL (3600)        // Repeat maintenance for labels
+#define METADATA_RUNTIME_THRESHOLD (5)              // Run time threshold for cleanup task
 
 #define METADATA_HOST_CHECK_FIRST_CHECK (5)         // First check for pending metadata
 #define METADATA_HOST_CHECK_INTERVAL (30)           // Repeat check for pending metadata
 #define METADATA_HOST_CHECK_IMMEDIATE (5)           // Repeat immediate run because we have more metadata to write
-
 #define MAX_METADATA_CLEANUP (500)                  // Maximum metadata write operations (e.g  deletes before retrying)
 #define METADATA_MAX_BATCH_SIZE (512)               // Maximum commands to execute before running the event loop
 
+#define DATABASE_FREE_PAGES_THRESHOLD_PC (5)        // Percentage of free pages to trigger vacuum
+#define DATABASE_FREE_PAGES_VACUUM_PC (10)          // Percentage of free pages to vacuum
+
 enum metadata_opcode {
     METADATA_DATABASE_NOOP = 0,
     METADATA_DATABASE_TIMER,
@@ -79,35 +95,31 @@ struct metadata_cmd {
     enum metadata_opcode opcode;
     struct completion *completion;
     const void *param[MAX_PARAM_LIST];
+    struct metadata_cmd *prev, *next;
 };
 
 struct metadata_database_cmdqueue {
-    unsigned head, tail;
-    struct metadata_cmd cmd_array[METADATA_CMD_Q_MAX_SIZE];
+    struct metadata_cmd *cmd_base;
 };
 
 typedef enum {
-    METADATA_FLAG_CLEANUP           = (1 << 0), // Cleanup is running
-    METADATA_FLAG_SCANNING_HOSTS    = (1 << 1), // Scanning of hosts in worker thread
-    METADATA_FLAG_SHUTDOWN          = (1 << 2), // Shutting down
+    METADATA_FLAG_PROCESSING    = (1 << 0), // store or cleanup
+    METADATA_FLAG_SHUTDOWN      = (1 << 1), // Shutting down
+    METADATA_FLAG_ML_LOADING    = (1 << 2), // ML model load in progress
 } METADATA_FLAG;
 
-#define METADATA_WORKER_BUSY    (METADATA_FLAG_CLEANUP | METADATA_FLAG_SCANNING_HOSTS)
-
 struct metadata_wc {
     uv_thread_t thread;
     uv_loop_t *loop;
     uv_async_t async;
     uv_timer_t timer_req;
-    time_t check_metadata_after;
-    time_t check_hosts_after;
+    time_t metadata_check_after;
     volatile unsigned queue_size;
     METADATA_FLAG flags;
-    uint64_t row_id;
     struct completion init_complete;
+    struct completion *scan_complete;
     /* FIFO command queue */
     uv_mutex_t cmd_mutex;
-    uv_cond_t cmd_cond;
     struct metadata_database_cmdqueue cmd_queue;
 };
 
@@ -140,7 +152,7 @@ static int host_label_store_to_sql_callback(const char *name, const char *value,
         buffer_sprintf(lb->sql, STORE_HOST_LABEL);
     else
         buffer_strcat(lb->sql, ", ");
-    buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int)ls & ~(RRDLABEL_FLAG_INTERNAL), name, value);
+    buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);
     lb->count++;
     return 1;
 }
@@ -151,7 +163,7 @@ static int chart_label_store_to_sql_callback(const char *name, const char *value
         buffer_sprintf(lb->sql, STORE_CHART_LABEL);
     else
         buffer_strcat(lb->sql, ", ");
-    buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, ls, name, value);
+    buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);
     lb->count++;
     return 1;
 }
@@ -177,7 +189,7 @@ static void clean_old_chart_labels(RRDSET *st)
 static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t *query_counter)
 {
     size_t old_version = st->rrdlabels_last_saved_version;
-    size_t new_version = dictionary_version(st->rrdlabels);
+    size_t new_version = rrdlabels_version(st->rrdlabels);
 
     if (new_version == old_version)
         return 0;
@@ -185,6 +197,7 @@ static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t
     struct query_build tmp = {.sql = work_buffer, .count = 0};
     uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
     rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
+    buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
     int rc = db_execute(db_meta, buffer_tostring(work_buffer));
     if (likely(!rc)) {
         st->rrdlabels_last_saved_version = new_version;
@@ -252,7 +265,7 @@ failed:
     return rc != SQLITE_DONE;
 }
 
-static void delete_dimension_uuid(uuid_t *dimension_uuid)
+static void delete_dimension_uuid(uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused)
 {
     static __thread sqlite3_stmt *res = NULL;
     int rc;
@@ -265,7 +278,7 @@ static void delete_dimension_uuid(uuid_t *dimension_uuid)
         }
     }
 
-    rc = sqlite3_bind_blob(res, 1, dimension_uuid,  sizeof(*dimension_uuid), SQLITE_STATIC);
+    rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);
     if (unlikely(rc != SQLITE_OK))
         goto skip_execution;
 
@@ -286,13 +299,6 @@ static int store_host_metadata(RRDHOST *host)
     static __thread sqlite3_stmt *res = NULL;
     int rc, param = 0;
 
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
-            return 0;
-        error_report("Database has not been initialized");
-        return 1;
-    }
-
     if (unlikely((!res))) {
         rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res);
         if (unlikely(rc != SQLITE_OK)) {
@@ -361,6 +367,10 @@ static int store_host_metadata(RRDHOST *host)
     if (unlikely(rc != SQLITE_OK))
         goto bind_fail;
 
+    rc = sqlite3_bind_int64(res, ++param, (sqlite3_int64) host->last_connected);
+    if (unlikely(rc != SQLITE_OK))
+        goto bind_fail;
+
     int store_rc = sqlite3_step_monitored(res);
     if (unlikely(store_rc != SQLITE_DONE))
         error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc);
@@ -474,13 +484,6 @@ static int store_chart_metadata(RRDSET *st)
     static __thread sqlite3_stmt *res = NULL;
     int rc, param = 0, store_rc = 0;
 
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
-            return 0;
-        error_report("Database has not been initialized");
-        return 1;
-    }
-
     if (unlikely(!res)) {
         rc = prepare_statement(db_meta, SQL_STORE_CHART, &res);
         if (unlikely(rc != SQLITE_OK)) {
@@ -583,13 +586,6 @@ static int store_dimension_metadata(RRDDIM *rd)
     static __thread sqlite3_stmt *res = NULL;
     int rc, param = 0;
 
-    if (unlikely(!db_meta)) {
-        if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
-            return 0;
-        error_report("Database has not been initialized");
-        return 1;
-    }
-
     if (unlikely(!res)) {
         rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res);
         if (unlikely(rc != SQLITE_OK)) {
@@ -650,7 +646,7 @@ bind_fail:
     return 1;
 }
 
-static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused)
+static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused)
 {
 #ifdef ENABLE_DBENGINE
     if(dbengine_enabled) {
@@ -675,8 +671,173 @@ static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused)
 #endif
 }
 
+int get_pragma_value(sqlite3 *database, const char *sql)
+{
+    sqlite3_stmt *res = NULL;
+    int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK))
+        return -1;
+
+    int result = -1;
+    rc = sqlite3_step_monitored(res);
+    if (likely(rc == SQLITE_ROW))
+        result = sqlite3_column_int(res, 0);
+
+    rc = sqlite3_finalize(res);
+    (void) rc;
+
+    return result;
+}
+
+
+int get_free_page_count(sqlite3 *database)
+{
+    return get_pragma_value(database, "PRAGMA freelist_count");
+}
+
+int get_database_page_count(sqlite3 *database)
+{
+    return get_pragma_value(database, "PRAGMA page_count");
+}
+
+static bool run_cleanup_loop(
+    sqlite3_stmt *res,
+    struct metadata_wc *wc,
+    bool (*check_cb)(uuid_t *, sqlite3_stmt **, bool),
+    void (*action_cb)(uuid_t *, sqlite3_stmt **, bool),
+    uint32_t *total_checked,
+    uint32_t *total_deleted,
+    uint64_t *row_id,
+    sqlite3_stmt **check_stmt,
+    sqlite3_stmt **action_stmt,
+    bool check_flag,
+    bool action_flag)
+{
+    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+        return true;
+
+    int rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) *row_id);
+    if (unlikely(rc != SQLITE_OK))
+        return true;
+
+    time_t start_running = now_monotonic_sec();
+    bool time_expired = false;
+    while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW &&
+           (*total_deleted < MAX_METADATA_CLEANUP && *total_checked < MAX_METADATA_CLEANUP)) {
+        if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+            break;
+
+        *row_id = sqlite3_column_int64(res, 1);
+        rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag);
+
+        if (rc == true) {
+            action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag);
+            (*total_deleted)++;
+        }
+
+        (*total_checked)++;
+        time_expired = ((now_monotonic_sec() - start_running) > METADATA_RUNTIME_THRESHOLD);
+    }
+    return time_expired || (*total_checked == MAX_METADATA_CLEANUP) || (*total_deleted == MAX_METADATA_CLEANUP);
+}
+
+
+#define SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION "SELECT count(1) FROM dimension WHERE chart_id = @chart_id"
+#define SQL_CHECK_CHART_EXISTENCE_IN_CHART "SELECT count(1) FROM chart WHERE chart_id = @chart_id"
+
+static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension)
+{
+    int rc, result = 1;
+    sqlite3_stmt *res = check_res ? *check_res : NULL;
+
+    if (!res) {
+        if (check_in_dimension)
+            rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION, -1, &res, 0);
+        else
+            rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_CHART, -1, &res, 0);
+        if (unlikely(rc != SQLITE_OK)) {
+            error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
+            return 0;
+        }
+        if (check_res)
+            *check_res = res;
+    }
+
+    rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind chart uuid parameter, rc = %d", rc);
+        goto skip;
+    }
+
+    rc = sqlite3_step_monitored(res);
+    if (likely(rc == SQLITE_ROW))
+        result = sqlite3_column_int(res, 0);
+
+skip:
+    if (check_res)
+        rc = sqlite3_reset(res);
+    else
+        rc = sqlite3_finalize(res);
+
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to %s statement that checks chart uuid existence rc = %d", check_res ? "reset" : "finalize", rc);
+    return result == 0;
+}
+
+#define SQL_DELETE_CHART_BY_UUID        "DELETE FROM chart WHERE chart_id = @chart_id"
+#define SQL_DELETE_CHART_LABEL_BY_UUID  "DELETE FROM chart_label WHERE chart_id = @chart_id"
+
+static void delete_chart_uuid(uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only)
+{
+    int rc;
+    sqlite3_stmt *res = action_res ? *action_res : NULL;
+
+    if (!res) {
+        if (label_only)
+            rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABEL_BY_UUID, -1, &res, 0);
+        else
+            rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_BY_UUID, -1, &res, 0);
+        if (unlikely(rc != SQLITE_OK)) {
+            error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
+            return;
+        }
+        if (action_res)
+            *action_res = res;
+    }
+
+    rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to bind chart uuid parameter, rc = %d", rc);
+        goto skip;
+    }
+
+    rc = sqlite3_step_monitored(res);
+    if (unlikely(rc != SQLITE_DONE))
+        error_report("Failed to delete a chart uuid from the %s table, rc = %d", label_only ? "labels" : "chart", rc);
+
+skip:
+    if (action_res)
+        rc = sqlite3_reset(res);
+    else
+        rc = sqlite3_finalize(res);
+
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to %s statement that deletes a chart uuid rc = %d", action_res ? "reset" : "finalize", rc);
+}
+
 static void check_dimension_metadata(struct metadata_wc *wc)
 {
+    static time_t next_execution_t = 0;
+    static uint64_t last_row_id = 0;
+
+    time_t now = now_realtime_sec();
+
+    if (!next_execution_t)
+        next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+    if (next_execution_t && next_execution_t > now)
+        return;
+
     int rc;
     sqlite3_stmt *res = NULL;
 
@@ -686,54 +847,212 @@ static void check_dimension_metadata(struct metadata_wc *wc)
         return;
     }
 
-    rc = sqlite3_bind_int64(res, 1,  (sqlite3_int64) wc->row_id);
+    uint32_t total_checked = 0;
+    uint32_t total_deleted = 0;
+
+    internal_error(true, "METADATA: Checking dimensions starting after row %"PRIu64, last_row_id);
+
+    bool more_to_do = run_cleanup_loop(
+        res,
+        wc,
+        dimension_can_be_deleted,
+        delete_dimension_uuid,
+        &total_checked,
+        &total_deleted,
+        &last_row_id,
+        NULL,
+        NULL,
+        false,
+        false);
+
+    now = now_realtime_sec();
+    if (more_to_do)
+        next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
+    else {
+        last_row_id = 0;
+        next_execution_t = now + METADATA_DIM_CHECK_INTERVAL;
+    }
+
+    netdata_log_info(
+        "METADATA: Dimensions checked %u, deleted %u. Checks will %s in %lld seconds",
+        total_checked,
+        total_deleted,
+        last_row_id ? "resume" : "restart",
+        (long long)(next_execution_t - now));
+
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the prepared statement to check dimensions");
+}
+
+static void check_chart_metadata(struct metadata_wc *wc)
+{
+    static time_t next_execution_t = 0;
+    static uint64_t last_row_id = 0;
+
+    time_t now = now_realtime_sec();
+
+    if (!next_execution_t)
+        next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+    if (next_execution_t && next_execution_t > now)
+        return;
+
+    sqlite3_stmt *res = NULL;
+
+    int rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LIST, -1, &res, 0);
     if (unlikely(rc != SQLITE_OK)) {
-        error_report("Failed to row parameter");
-        goto skip_run;
+        error_report("Failed to prepare statement to fetch charts");
+        return;
     }
 
     uint32_t total_checked = 0;
-    uint32_t total_deleted= 0;
-    uint64_t last_row_id = wc->row_id;
+    uint32_t total_deleted = 0;
+
+    internal_error(true, "METADATA: Checking charts starting after row %"PRIu64, last_row_id);
+
+    sqlite3_stmt *check_res = NULL;
+    sqlite3_stmt *action_res = NULL;
+    bool more_to_do = run_cleanup_loop(
+        res,
+        wc,
+        chart_can_be_deleted,
+        delete_chart_uuid,
+        &total_checked,
+        &total_deleted,
+        &last_row_id,
+        &check_res,
+        &action_res,
+        true,
+        false);
+
+    if (check_res)
+        sqlite3_finalize(check_res);
+
+    if (action_res)
+        sqlite3_finalize(action_res);
+
+    now = now_realtime_sec();
+    if (more_to_do)
+        next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
+    else {
+        last_row_id = 0;
+        next_execution_t = now + METADATA_CHART_CHECK_INTERVAL;
+    }
 
-    netdata_log_info("METADATA: Checking dimensions starting after row %"PRIu64, wc->row_id);
+    netdata_log_info(
+        "METADATA: Charts checked %u, deleted %u. Checks will %s in %lld seconds",
+        total_checked,
+        total_deleted,
+        last_row_id ? "resume" : "restart",
+        (long long)(next_execution_t - now));
 
-    while (sqlite3_step_monitored(res) == SQLITE_ROW && total_deleted < MAX_METADATA_CLEANUP) {
-        if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
-            break;
+    rc = sqlite3_finalize(res);
+    if (unlikely(rc != SQLITE_OK))
+        error_report("Failed to finalize the prepared statement when reading charts");
+}
+
+static void check_label_metadata(struct metadata_wc *wc)
+{
+    static time_t next_execution_t = 0;
+    static uint64_t last_row_id = 0;
 
-        last_row_id = sqlite3_column_int64(res, 1);
-        rc = dimension_can_be_deleted((uuid_t *)sqlite3_column_blob(res, 0));
-        if (rc == true) {
-            delete_dimension_uuid((uuid_t *)sqlite3_column_blob(res, 0));
-            total_deleted++;
-        }
-        total_checked++;
-    }
-    wc->row_id = last_row_id;
     time_t now = now_realtime_sec();
-    if (total_deleted > 0) {
-        wc->check_metadata_after = now + METADATA_MAINTENANCE_RETRY;
-    } else
-        wc->row_id = 0;
-    netdata_log_info("METADATA: Checked %u, deleted %u -- will resume after row %"PRIu64" in %lld seconds", total_checked, total_deleted, wc->row_id,
-         (long long)(wc->check_metadata_after - now));
-
-skip_run:
+
+    if (!next_execution_t)
+        next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+    if (next_execution_t && next_execution_t > now)
+        return;
+
+    int rc;
+    sqlite3_stmt *res = NULL;
+
+    rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LABEL_LIST, -1, &res, 0);
+    if (unlikely(rc != SQLITE_OK)) {
+        error_report("Failed to prepare statement to fetch charts");
+        return;
+    }
+
+    uint32_t total_checked = 0;
+    uint32_t total_deleted = 0;
+
+    internal_error(true,"METADATA: Checking charts labels starting after row %"PRIu64, last_row_id);
+
+    sqlite3_stmt *check_res = NULL;
+    sqlite3_stmt *action_res = NULL;
+
+    bool more_to_do = run_cleanup_loop(
+        res,
+        wc,
+        chart_can_be_deleted,
+        delete_chart_uuid,
+        &total_checked,
+        &total_deleted,
+        &last_row_id,
+        &check_res,
+        &action_res,
+        false,
+        true);
+
+    if (check_res)
+        sqlite3_finalize(check_res);
+
+    if (action_res)
+        sqlite3_finalize(action_res);
+
+    now = now_realtime_sec();
+    if (more_to_do)
+        next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
+    else {
+        last_row_id = 0;
+        next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL;
+    }
+
+    netdata_log_info(
+        "METADATA: Chart labels checked %u, deleted %u. Checks will %s in %lld seconds",
+        total_checked,
+        total_deleted,
+        last_row_id ? "resume" : "restart",
+        (long long)(next_execution_t - now));
+
     rc = sqlite3_finalize(res);
     if (unlikely(rc != SQLITE_OK))
-        error_report("Failed to finalize the prepared statement when reading dimensions");
+        error_report("Failed to finalize the prepared statement when checking charts");
 }
 
-static void cleanup_health_log(void)
+
+static void cleanup_health_log(struct metadata_wc *wc)
 {
+    static time_t next_execution_t = 0;
+
+    time_t now = now_realtime_sec();
+
+    if (!next_execution_t)
+        next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+    if (next_execution_t && next_execution_t > now)
+        return;
+
+    next_execution_t = now + METADATA_HEALTH_LOG_INTERVAL;
+
     RRDHOST *host;
-    dfe_start_reentrant(rrdhost_root_index, host) {
+
+    bool is_claimed = claimed();
+    dfe_start_reentrant(rrdhost_root_index, host){
         if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
             continue;
-        sql_health_alarm_log_cleanup(host);
+        sql_health_alarm_log_cleanup(host, is_claimed);
+        if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+            break;
     }
     dfe_done(host);
+
+    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+        return;
+
+    (void) db_execute(db_meta,"DELETE FROM health_log WHERE host_id NOT IN (SELECT host_id FROM host)");
+    (void) db_execute(db_meta,"DELETE FROM health_log_detail WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)");
 }
 
 //
@@ -742,103 +1061,57 @@ static void cleanup_health_log(void)
 
 static void metadata_init_cmd_queue(struct metadata_wc *wc)
 {
-    wc->cmd_queue.head = wc->cmd_queue.tail = 0;
-    wc->queue_size = 0;
-    fatal_assert(0 == uv_cond_init(&wc->cmd_cond));
+    wc->cmd_queue.cmd_base = NULL;
     fatal_assert(0 == uv_mutex_init(&wc->cmd_mutex));
 }
 
-int metadata_enq_cmd_noblock(struct metadata_wc *wc, struct metadata_cmd *cmd)
+static void metadata_free_cmd_queue(struct metadata_wc *wc)
 {
-    unsigned queue_size;
-
-    /* wait for free space in queue */
     uv_mutex_lock(&wc->cmd_mutex);
-
-    if (cmd->opcode == METADATA_SYNC_SHUTDOWN) {
-        metadata_flag_set(wc, METADATA_FLAG_SHUTDOWN);
-        uv_mutex_unlock(&wc->cmd_mutex);
-        return 0;
-    }
-
-    if (unlikely((queue_size = wc->queue_size) == METADATA_CMD_Q_MAX_SIZE ||
-        metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
-        uv_mutex_unlock(&wc->cmd_mutex);
-        return 1;
+    while(wc->cmd_queue.cmd_base) {
+        struct metadata_cmd *t = wc->cmd_queue.cmd_base;
+        DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wc->cmd_queue.cmd_base, t, prev, next);
+        freez(t);
     }
-
-    fatal_assert(queue_size < METADATA_CMD_Q_MAX_SIZE);
-    /* enqueue command */
-    wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
-    wc->cmd_queue.tail = wc->cmd_queue.tail != METADATA_CMD_Q_MAX_SIZE - 1 ?
-                             wc->cmd_queue.tail + 1 : 0;
-    wc->queue_size = queue_size + 1;
     uv_mutex_unlock(&wc->cmd_mutex);
-    return 0;
 }
 
 static void metadata_enq_cmd(struct metadata_wc *wc, struct metadata_cmd *cmd)
 {
-    unsigned queue_size;
-
-    /* wait for free space in queue */
-    uv_mutex_lock(&wc->cmd_mutex);
-    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
-        uv_mutex_unlock(&wc->cmd_mutex);
-        (void) uv_async_send(&wc->async);
-        return;
-    }
-
     if (cmd->opcode == METADATA_SYNC_SHUTDOWN) {
         metadata_flag_set(wc, METADATA_FLAG_SHUTDOWN);
-        uv_mutex_unlock(&wc->cmd_mutex);
-        (void) uv_async_send(&wc->async);
-        return;
+        goto wakeup_event_loop;
     }
 
-    while ((queue_size = wc->queue_size) == METADATA_CMD_Q_MAX_SIZE) {
-        if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
-            uv_mutex_unlock(&wc->cmd_mutex);
-            return;
-        }
-        uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
-    }
-    fatal_assert(queue_size < METADATA_CMD_Q_MAX_SIZE);
-    /* enqueue command */
-    wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
-    wc->cmd_queue.tail = wc->cmd_queue.tail != METADATA_CMD_Q_MAX_SIZE - 1 ?
-                             wc->cmd_queue.tail + 1 : 0;
-    wc->queue_size = queue_size + 1;
+    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+        goto wakeup_event_loop;
+
+    struct metadata_cmd *t = mallocz(sizeof(*t));
+    *t = *cmd;
+    t->prev = t->next = NULL;
+
+    uv_mutex_lock(&wc->cmd_mutex);
+    DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wc->cmd_queue.cmd_base, t, prev, next);
     uv_mutex_unlock(&wc->cmd_mutex);
 
-    /* wake up event loop */
+wakeup_event_loop:
     (void) uv_async_send(&wc->async);
 }
 
 static struct metadata_cmd metadata_deq_cmd(struct metadata_wc *wc)
 {
     struct metadata_cmd ret;
-    unsigned queue_size;
 
     uv_mutex_lock(&wc->cmd_mutex);
-    queue_size = wc->queue_size;
-    if (queue_size == 0) {
-        memset(&ret, 0, sizeof(ret));
+    if(wc->cmd_queue.cmd_base) {
+        struct metadata_cmd *t = wc->cmd_queue.cmd_base;
+        DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wc->cmd_queue.cmd_base, t, prev, next);
+        ret = *t;
+        freez(t);
+    }
+    else {
         ret.opcode = METADATA_DATABASE_NOOP;
         ret.completion = NULL;
-    } else {
-        /* dequeue command */
-        ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];
-
-        if (queue_size == 1) {
-            wc->cmd_queue.head = wc->cmd_queue.tail = 0;
-        } else {
-            wc->cmd_queue.head = wc->cmd_queue.head != METADATA_CMD_Q_MAX_SIZE - 1 ?
-                                     wc->cmd_queue.head + 1 : 0;
-        }
-        wc->queue_size = queue_size - 1;
-        /* wake up producers */
-        uv_cond_signal(&wc->cmd_cond);
     }
     uv_mutex_unlock(&wc->cmd_mutex);
 
@@ -865,43 +1138,62 @@ static void timer_cb(uv_timer_t* handle)
 
    time_t now = now_realtime_sec();
 
-   if (wc->check_metadata_after && wc->check_metadata_after < now) {
-       cmd.opcode = METADATA_MAINTENANCE;
-       if (!metadata_enq_cmd_noblock(wc, &cmd))
-           wc->check_metadata_after = now + METADATA_MAINTENANCE_INTERVAL;
-   }
-
-   if (wc->check_hosts_after && wc->check_hosts_after < now) {
+   if (wc->metadata_check_after && wc->metadata_check_after < now) {
        cmd.opcode = METADATA_SCAN_HOSTS;
-       if (!metadata_enq_cmd_noblock(wc, &cmd))
-           wc->check_hosts_after = now + METADATA_HOST_CHECK_INTERVAL;
+       metadata_enq_cmd(wc, &cmd);
    }
 }
 
-static void after_metadata_cleanup(uv_work_t *req, int status)
+void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc)
 {
-    UNUSED(status);
+   int free_pages = get_free_page_count(database);
+   int total_pages = get_database_page_count(database);
+
+   if (!threshold)
+       threshold = DATABASE_FREE_PAGES_THRESHOLD_PC;
+
+   if (!vacuum_pc)
+       vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC;
+
+   if (free_pages > (total_pages * threshold / 100)) {
 
-    struct metadata_wc *wc = req->data;
-    metadata_flag_clear(wc, METADATA_FLAG_CLEANUP);
+       int do_free_pages = (int) (free_pages * vacuum_pc / 100);
+       netdata_log_info("%s: Freeing %d database pages", db_alias, do_free_pages);
+
+       char sql[128];
+       snprintfz(sql, 127, "PRAGMA incremental_vacuum(%d)", do_free_pages);
+       (void) db_execute(database, sql);
+   }
 }
 
-static void start_metadata_cleanup(uv_work_t *req)
+void run_metadata_cleanup(struct metadata_wc *wc)
 {
-    register_libuv_worker_jobs();
+    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+       return;
 
-    worker_is_busy(UV_EVENT_METADATA_CLEANUP);
-    struct metadata_wc *wc = req->data;
     check_dimension_metadata(wc);
-    cleanup_health_log();
+    check_chart_metadata(wc);
+    check_label_metadata(wc);
+    cleanup_health_log(wc);
+
+    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+       return;
+
+    vacuum_database(db_meta, "METADATA", DATABASE_FREE_PAGES_THRESHOLD_PC, DATABASE_FREE_PAGES_VACUUM_PC);
+
     (void) sqlite3_wal_checkpoint(db_meta, NULL);
-    worker_is_idle();
 }
 
+struct ml_model_payload {
+    uv_work_t request;
+    struct metadata_wc *wc;
+    Pvoid_t JudyL;
+    size_t count;
+};
+
 struct scan_metadata_payload {
     uv_work_t request;
     struct metadata_wc *wc;
-    struct completion *completion;
     BUFFER *work_buffer;
     uint32_t max_count;
 };
@@ -1027,10 +1319,10 @@ static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)
     struct scan_metadata_payload *data = req->data;
     struct metadata_wc *wc = data->wc;
 
-    metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
+    metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);
     internal_error(true, "METADATA: scanning hosts complete");
-    if (unlikely(data->completion)) {
-        completion_mark_complete(data->completion);
+    if (unlikely(wc->scan_complete)) {
+        completion_mark_complete(wc->scan_complete);
         internal_error(true, "METADATA: Sending completion done");
     }
     freez(data);
@@ -1044,7 +1336,7 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_trans
     uint32_t scan_count = 1;
 
     if (use_transaction)
-        (void)db_execute(db_meta, "BEGIN TRANSACTION;");
+        (void)db_execute(db_meta, "BEGIN TRANSACTION");
 
     rrdset_foreach_reentrant(st, host) {
         if (scan_count == max_count) {
@@ -1093,7 +1385,7 @@ static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_trans
     rrdset_foreach_done(st);
 
     if (use_transaction)
-        (void)db_execute(db_meta, "COMMIT TRANSACTION;");
+        (void)db_execute(db_meta, "COMMIT TRANSACTION");
 
     return more_to_do;
 }
@@ -1160,6 +1452,7 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
                 struct query_build tmp = {.sql = work_buffer, .count = 0};
                 uuid_unparse_lower(host->host_uuid, tmp.uuid_str);
                 rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
+                buffer_strcat(work_buffer, " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
                 rc = db_execute(db_meta, buffer_tostring(work_buffer));
 
                 if (unlikely(rc)) {
@@ -1215,12 +1508,50 @@ static void start_metadata_hosts(uv_work_t *req __maybe_unused)
                    (double)(all_ended_ut - all_started_ut) / USEC_PER_MS);
 
     if (unlikely(run_again))
-        wc->check_hosts_after = now_realtime_sec() + METADATA_HOST_CHECK_IMMEDIATE;
-    else
-        wc->check_hosts_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL;
+        wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_IMMEDIATE;
+    else {
+        wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL;
+        run_metadata_cleanup(wc);
+    }
+    worker_is_idle();
+}
+
+// Callback after scan of hosts is done
+static void after_start_ml_model_load(uv_work_t *req, int status __maybe_unused)
+{
+    struct ml_model_payload *ml_data = req->data;
+    struct metadata_wc *wc = ml_data->wc;
+    metadata_flag_clear(wc, METADATA_FLAG_ML_LOADING);
+    JudyLFreeArray(&ml_data->JudyL, PJE0);
+    freez(ml_data);
+}
+
+static void start_ml_model_load(uv_work_t *req __maybe_unused)
+{
+    register_libuv_worker_jobs();
+
+    struct ml_model_payload *ml_data = req->data;
+
+    worker_is_busy(UV_EVENT_METADATA_ML_LOAD);
+
+    Pvoid_t *PValue;
+    Word_t Index = 0;
+    bool first = true;
+    RRDDIM *rd;
+    RRDDIM_ACQUIRED *rda;
+    internal_error(true, "Batch ML load loader, %zu items", ml_data->count);
+    while((PValue = JudyLFirstThenNext(ml_data->JudyL, &Index, &first))) {
+        UNUSED(PValue);
+        rda = (RRDDIM_ACQUIRED *) Index;
+        rd = rrddim_acquired_to_rrddim(rda);
+        ml_dimension_load_models(rd);
+        rrddim_acquired_release(rda);
+    }
     worker_is_idle();
 }
 
+
+
 static void metadata_event_loop(void *arg)
 {
     worker_register("METASYNC");
@@ -1237,10 +1568,8 @@ static void metadata_event_loop(void *arg)
     unsigned cmd_batch_size;
     struct metadata_wc *wc = arg;
     enum metadata_opcode opcode;
-    uv_work_t metadata_cleanup_worker;
 
     uv_thread_set_name_np(wc->thread, "METASYNC");
-//    service_register(SERVICE_THREAD_TYPE_EVENT_LOOP, NULL, NULL, NULL, true);
     loop = wc->loop = mallocz(sizeof(uv_loop_t));
     ret = uv_loop_init(loop);
     if (ret) {
@@ -1268,19 +1597,17 @@ static void metadata_event_loop(void *arg)
 
     struct metadata_cmd cmd;
     memset(&cmd, 0, sizeof(cmd));
-    metadata_flag_clear(wc, METADATA_FLAG_CLEANUP);
-    metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
+    metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);
 
-    wc->check_metadata_after = now_realtime_sec() + METADATA_MAINTENANCE_FIRST_CHECK;
-    wc->check_hosts_after    = now_realtime_sec() + METADATA_HOST_CHECK_FIRST_CHECK;
+    wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_FIRST_CHECK;
 
     int shutdown = 0;
-    wc->row_id = 0;
     completion_mark_complete(&wc->init_complete);
     BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
     struct scan_metadata_payload *data;
 
-    while (shutdown == 0 || (wc->flags & METADATA_WORKER_BUSY)) {
+    struct ml_model_payload *ml_data = NULL;
+    while (shutdown == 0 || (wc->flags & METADATA_FLAG_PROCESSING)) {
         uuid_t  *uuid;
         RRDHOST *host = NULL;
 
@@ -1306,6 +1633,24 @@ static void metadata_event_loop(void *arg)
             if (likely(opcode != METADATA_DATABASE_NOOP))
                 worker_is_busy(opcode);
 
+            // Have pending ML models to load?
+            if (opcode != METADATA_ML_LOAD_MODELS && ml_data && ml_data->count) {
+                static usec_t ml_submit_last = 0;
+                usec_t now = now_monotonic_usec();
+                if (!ml_submit_last)
+                    ml_submit_last = now;
+
+                if (!metadata_flag_check(wc, METADATA_FLAG_ML_LOADING) && (now - ml_submit_last > 150 * USEC_PER_MS)) {
+                    metadata_flag_set(wc, METADATA_FLAG_ML_LOADING);
+                    if (unlikely(uv_queue_work(loop, &ml_data->request, start_ml_model_load, after_start_ml_model_load)))
+                        metadata_flag_clear(wc, METADATA_FLAG_ML_LOADING);
+                    else {
+                        ml_submit_last = now;
+                        ml_data = NULL;
+                    }
+                }
+            }
+
             switch (opcode) {
                 case METADATA_DATABASE_NOOP:
                 case METADATA_DATABASE_TIMER:
@@ -1313,13 +1658,22 @@ static void metadata_event_loop(void *arg)
 
                 case METADATA_ML_LOAD_MODELS: {
                     RRDDIM *rd = (RRDDIM *) cmd.param[0];
-                    ml_dimension_load_models(rd);
+                    RRDDIM_ACQUIRED *rda = rrddim_find_and_acquire(rd->rrdset, rrddim_id(rd));
+                    if (likely(rda)) {
+                        if (!ml_data) {
+                            ml_data = callocz(1,sizeof(*ml_data));
+                            ml_data->request.data = ml_data;
+                            ml_data->wc = wc;
+                        }
+                        JudyLIns(&ml_data->JudyL, (Word_t)rda, PJE0);
+                        ml_data->count++;
+                    }
                     break;
                 }
                 case METADATA_DEL_DIMENSION:
                     uuid = (uuid_t *) cmd.param[0];
-                    if (likely(dimension_can_be_deleted(uuid)))
-                        delete_dimension_uuid(uuid);
+                    if (likely(dimension_can_be_deleted(uuid, NULL, false)))
+                        delete_dimension_uuid(uuid, NULL, false);
                     freez(uuid);
                     break;
                 case METADATA_STORE_CLAIM_ID:
@@ -1332,7 +1686,7 @@ static void metadata_event_loop(void *arg)
                     store_host_and_system_info(host, NULL);
                     break;
                 case METADATA_SCAN_HOSTS:
-                    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SCANNING_HOSTS)))
+                    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_PROCESSING)))
                         break;
 
                     if (unittest_running)
@@ -1341,7 +1695,6 @@ static void metadata_event_loop(void *arg)
                     data = mallocz(sizeof(*data));
                     data->request.data = data;
                     data->wc = wc;
-                    data->completion = cmd.completion;  // Completion by the worker
                     data->work_buffer = work_buffer;
 
                     if (unlikely(cmd.completion)) {
@@ -1351,15 +1704,15 @@ static void metadata_event_loop(void *arg)
                     else
                         data->max_count = 5000;
 
-                    metadata_flag_set(wc, METADATA_FLAG_SCANNING_HOSTS);
+                    metadata_flag_set(wc, METADATA_FLAG_PROCESSING);
                     if (unlikely(
                             uv_queue_work(loop,&data->request,
                                           start_metadata_hosts,
                                           after_metadata_hosts))) {
                         // Failed to launch worker -- let the event loop handle completion
-                        cmd.completion = data->completion;
+                        cmd.completion = wc->scan_complete;
                         freez(data);
-                        metadata_flag_clear(wc, METADATA_FLAG_SCANNING_HOSTS);
+                        metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);
                     }
                     break;
                 case METADATA_LOAD_HOST_CONTEXT:;
@@ -1375,17 +1728,6 @@ static void metadata_event_loop(void *arg)
                         freez(data);
                     }
                     break;
-                case METADATA_MAINTENANCE:
-                    if (unlikely(metadata_flag_check(wc, METADATA_FLAG_CLEANUP)))
-                        break;
-
-                    metadata_cleanup_worker.data = wc;
-                    metadata_flag_set(wc, METADATA_FLAG_CLEANUP);
-                    if (unlikely(
-                            uv_queue_work(loop, &metadata_cleanup_worker, start_metadata_cleanup, after_metadata_cleanup))) {
-                        metadata_flag_clear(wc, METADATA_FLAG_CLEANUP);
-                    }
-                    break;
                 case METADATA_UNITTEST:;
                     struct thread_unittest *tu = (struct thread_unittest *) cmd.param[0];
                     sleep_usec(1000); // processing takes 1ms
@@ -1404,7 +1746,6 @@ static void metadata_event_loop(void *arg)
         uv_close((uv_handle_t *)&wc->timer_req, NULL);
 
     uv_close((uv_handle_t *)&wc->async, NULL);
-    uv_cond_destroy(&wc->cmd_cond);
     int rc;
     do {
         rc = uv_loop_close(loop);
@@ -1416,6 +1757,9 @@ static void metadata_event_loop(void *arg)
 
     netdata_log_info("METADATA: Shutting down event loop");
     completion_mark_complete(&wc->init_complete);
+    completion_destroy(wc->scan_complete);
+    freez(wc->scan_complete);
+    metadata_free_cmd_queue(wc);
     return;
 
 error_after_timer_init:
@@ -1454,24 +1798,25 @@ void metadata_sync_shutdown_prepare(void)
     struct metadata_cmd cmd;
     memset(&cmd, 0, sizeof(cmd));
 
-    struct completion compl;
-    completion_init(&compl);
+    struct metadata_wc *wc = &metasync_worker;
+
+    struct completion *compl = mallocz(sizeof(*compl));
+    completion_init(compl);
+    __atomic_store_n(&wc->scan_complete, compl, __ATOMIC_RELAXED);
 
     netdata_log_info("METADATA: Sending a scan host command");
     uint32_t max_wait_iterations = 2000;
-    while (unlikely(metadata_flag_check(&metasync_worker, METADATA_FLAG_SCANNING_HOSTS)) && max_wait_iterations--) {
+    while (unlikely(metadata_flag_check(&metasync_worker, METADATA_FLAG_PROCESSING)) && max_wait_iterations--) {
         if (max_wait_iterations == 1999)
             netdata_log_info("METADATA: Current worker is running; waiting to finish");
         sleep_usec(1000);
     }
 
     cmd.opcode = METADATA_SCAN_HOSTS;
-    cmd.completion = &compl;
     metadata_enq_cmd(&metasync_worker, &cmd);
 
     netdata_log_info("METADATA: Waiting for host scan completion");
-    completion_wait_for(&compl);
-    completion_destroy(&compl);
+    completion_wait_for(wc->scan_complete);
     netdata_log_info("METADATA: Host scan complete; can continue with shutdown");
 }
 
@@ -1631,7 +1976,6 @@ int metadata_unittest(void)
     // Queue items for a specific period of time
     metadata_unittest_threads();
 
-    fprintf(stderr, "Items still in queue %u\n", metasync_worker.queue_size);
     metadata_sync_shutdown();
 
     return 0;
diff --git a/database/sqlite/sqlite_metadata.h b/database/sqlite/sqlite_metadata.h
index 6b0676ee7..f75a9ab00 100644
--- a/database/sqlite/sqlite_metadata.h
+++ b/database/sqlite/sqlite_metadata.h
@@ -17,6 +17,7 @@ void metaqueue_host_update_info(RRDHOST *host);
 void metaqueue_ml_load_models(RRDDIM *rd);
 void migrate_localhost(uuid_t *host_uuid);
 void metadata_queue_load_host_context(RRDHOST *host);
+void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc);
 
 // UNIT TEST
 int metadata_unittest(void);