diff options
Diffstat (limited to '')
-rw-r--r-- | src/database/sqlite/dbdata.c (renamed from database/sqlite/dbdata.c) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite3.c (renamed from database/sqlite/sqlite3.c) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite3.h (renamed from database/sqlite/sqlite3.h) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite3recover.c (renamed from database/sqlite/sqlite3recover.c) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite3recover.h (renamed from database/sqlite/sqlite3recover.h) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_aclk.c (renamed from database/sqlite/sqlite_aclk.c) | 22 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_aclk.h (renamed from database/sqlite/sqlite_aclk.h) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_aclk_alert.c (renamed from database/sqlite/sqlite_aclk_alert.c) | 15 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_aclk_alert.h (renamed from database/sqlite/sqlite_aclk_alert.h) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_aclk_node.c (renamed from database/sqlite/sqlite_aclk_node.c) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_aclk_node.h (renamed from database/sqlite/sqlite_aclk_node.h) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_context.c (renamed from database/sqlite/sqlite_context.c) | 37 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_context.h (renamed from database/sqlite/sqlite_context.h) | 1 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_db_migration.c (renamed from database/sqlite/sqlite_db_migration.c) | 60 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_db_migration.h (renamed from database/sqlite/sqlite_db_migration.h) | 0 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_functions.c | 373 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_functions.h | 47 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_health.c (renamed from database/sqlite/sqlite_health.c) | 365 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_health.h (renamed from database/sqlite/sqlite_health.h) | 9 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_metadata.c | 2645 | ||||
-rw-r--r-- | src/database/sqlite/sqlite_metadata.h | 59 |
21 files changed, 3328 insertions, 305 deletions
diff --git a/database/sqlite/dbdata.c b/src/database/sqlite/dbdata.c index 1ad742e04..1ad742e04 100644 --- a/database/sqlite/dbdata.c +++ b/src/database/sqlite/dbdata.c diff --git a/database/sqlite/sqlite3.c b/src/database/sqlite/sqlite3.c index da8c38d09..da8c38d09 100644 --- a/database/sqlite/sqlite3.c +++ b/src/database/sqlite/sqlite3.c diff --git a/database/sqlite/sqlite3.h b/src/database/sqlite/sqlite3.h index 48effe202..48effe202 100644 --- a/database/sqlite/sqlite3.h +++ b/src/database/sqlite/sqlite3.h diff --git a/database/sqlite/sqlite3recover.c b/src/database/sqlite/sqlite3recover.c index 3dae0b7a9..3dae0b7a9 100644 --- a/database/sqlite/sqlite3recover.c +++ b/src/database/sqlite/sqlite3recover.c diff --git a/database/sqlite/sqlite3recover.h b/src/database/sqlite/sqlite3recover.h index 7a1cd1cd8..7a1cd1cd8 100644 --- a/database/sqlite/sqlite3recover.h +++ b/src/database/sqlite/sqlite3recover.h diff --git a/database/sqlite/sqlite_aclk.c b/src/database/sqlite/sqlite_aclk.c index ac574879c..c410406b2 100644 --- a/database/sqlite/sqlite_aclk.c +++ b/src/database/sqlite/sqlite_aclk.c @@ -61,7 +61,6 @@ enum { IDX_UPDATE_EVERY, IDX_OS, IDX_TIMEZONE, - IDX_TAGS, IDX_HOPS, IDX_MEMORY_MODE, IDX_ABBREV_TIMEZONE, @@ -72,6 +71,7 @@ enum { IDX_HEALTH_ENABLED, IDX_LAST_CONNECTED, IDX_IS_EPHEMERAL, + IDX_IS_REGISTERED, }; static int create_host_callback(void *data, int argc, char **argv, char **column) @@ -88,21 +88,27 @@ static int create_host_callback(void *data, int argc, char **argv, char **column time_t age = now_realtime_sec() - last_connected; int is_ephemeral = 0; + int is_registered = 0; if (argv[IDX_IS_EPHEMERAL]) is_ephemeral = str2i(argv[IDX_IS_EPHEMERAL]); + if (argv[IDX_IS_REGISTERED]) + is_registered = str2i(argv[IDX_IS_REGISTERED]); + char guid[UUID_STR_LEN]; uuid_unparse_lower(*(uuid_t *)argv[IDX_HOST_ID], guid); if (is_ephemeral && age > rrdhost_free_ephemeral_time_s) { netdata_log_info( - "Skipping ephemeral hostname \"%s\" with GUID \"%s\", age = %ld seconds (limit %ld seconds)", + "%s ephemeral hostname \"%s\" with GUID \"%s\", age = %ld seconds (limit %ld seconds)", + is_registered ? "Loading registered" : "Skipping unregistered", (const char *)argv[IDX_HOSTNAME], guid, age, rrdhost_free_ephemeral_time_s); - return 0; + if (!is_registered) + return 0; } struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info)); @@ -120,7 +126,6 @@ static int create_host_callback(void *data, int argc, char **argv, char **column (const char *)argv[IDX_TIMEZONE], (const char *)argv[IDX_ABBREV_TIMEZONE], (int32_t)(argv[IDX_UTC_OFFSET] ? str2uint32_t(argv[IDX_UTC_OFFSET], NULL) : 0), - (const char *)argv[IDX_TAGS], (const char *)(argv[IDX_PROGRAM_NAME] ? argv[IDX_PROGRAM_NAME] : "unknown"), (const char *)(argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown"), argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1, @@ -557,11 +562,12 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may #define SQL_FETCH_ALL_HOSTS \ "SELECT host_id, hostname, registry_hostname, update_every, os, " \ - "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \ + "timezone, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \ "program_version, entries, health_enabled, last_connected, " \ "(SELECT CASE WHEN hl.label_value = 'true' THEN 1 ELSE 0 END FROM " \ - "host_label hl WHERE hl.host_id = h.host_id AND hl.label_key = '_is_ephemeral') " \ - "FROM host h WHERE hops > 0" + "host_label hl WHERE hl.host_id = h.host_id AND hl.label_key = '_is_ephemeral'), " \ + "(SELECT CASE WHEN ni.node_id is NULL THEN 0 ELSE 1 END FROM " \ + "node_instance ni WHERE ni.host_id = h.host_id) FROM host h WHERE hops > 0" #define SQL_FETCH_ALL_INSTANCES \ "SELECT ni.host_id, ni.node_id FROM host h, node_instance ni " \ @@ -675,4 +681,4 @@ void unregister_node(const char *machine_guid) cmd.completion = NULL; aclk_database_enq_cmd(&cmd); } -#endif
\ No newline at end of file +#endif diff --git a/database/sqlite/sqlite_aclk.h b/src/database/sqlite/sqlite_aclk.h index 0db2647bf..0db2647bf 100644 --- a/database/sqlite/sqlite_aclk.h +++ b/src/database/sqlite/sqlite_aclk.h diff --git a/database/sqlite/sqlite_aclk_alert.c b/src/database/sqlite/sqlite_aclk_alert.c index 9bd060f96..c96f0eef8 100644 --- a/database/sqlite/sqlite_aclk_alert.c +++ b/src/database/sqlite/sqlite_aclk_alert.c @@ -201,7 +201,7 @@ void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, bool skip_filter) ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED; rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS); } else - error_report("Failed to store alert event %"PRId64", rc = %d", ae->unique_id, rc); + error_report("Failed to store alert event %"PRIu32", rc = %d", ae->unique_id, rc); done: if (unlikely(sqlite3_finalize(res_alert) != SQLITE_OK)) @@ -351,7 +351,8 @@ static void aclk_push_alert_event(struct aclk_sync_cfg_t *wc __maybe_unused) strdupz("UNKNOWN=0=UNKNOWN"); alarm_log.command = strdupz(edit_command); - alarm_log.duration = (time_t) sqlite3_column_int64(res, 6); + time_t duration = (time_t) sqlite3_column_int64(res, 6); + alarm_log.duration = (duration > 0) ? duration : 0; alarm_log.non_clear_duration = (time_t) sqlite3_column_int64(res, 7); alarm_log.status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 19)); alarm_log.old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 20)); @@ -609,7 +610,7 @@ void aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_has netdata_log_error("aclk_push_alert_config_event: Unexpected param number %d", param); BUFFER *tmp_buf = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite); - buffer_data_options2string(tmp_buf, sqlite3_column_int(res, 28)); + rrdr_options_to_buffer(tmp_buf, sqlite3_column_int(res, 28)); alarm_config.p_db_lookup_options = strdupz((char *)buffer_tostring(tmp_buf)); buffer_free(tmp_buf); @@ -821,7 +822,7 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN #endif #ifdef ENABLE_ACLK -static bool have_recent_alarm(RRDHOST *host, int64_t alarm_id, int64_t mark) +static bool have_recent_alarm_unsafe(RRDHOST *host, int64_t alarm_id, int64_t mark) { ALARM_ENTRY *ae = host->health_log.alarms; @@ -882,7 +883,7 @@ void aclk_push_alert_snapshot_event(char *node_id __maybe_unused) if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED)) continue; - if (have_recent_alarm(host, ae->alarm_id, ae->unique_id)) + if (have_recent_alarm_unsafe(host, ae->alarm_id, ae->unique_id)) continue; if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid)) @@ -911,7 +912,7 @@ void aclk_push_alert_snapshot_event(char *node_id __maybe_unused) if (likely(ae->updated_by_id) || unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED)) continue; - if (have_recent_alarm(host, ae->alarm_id, ae->unique_id)) + if (have_recent_alarm_unsafe(host, ae->alarm_id, ae->unique_id)) continue; if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid)) @@ -1090,7 +1091,7 @@ void aclk_push_alarm_checkpoint(RRDHOST *host __maybe_unused) } active_alerts[cnt].name = (char *)rrdcalc_name(rc); - len += string_strlen(rc->name); + len += string_strlen(rc->config.name); active_alerts[cnt].chart = (char *)rrdcalc_chart_name(rc); len += string_strlen(rc->chart); active_alerts[cnt].status = rc->status; diff --git a/database/sqlite/sqlite_aclk_alert.h b/src/database/sqlite/sqlite_aclk_alert.h index cfb3468b9..cfb3468b9 100644 --- a/database/sqlite/sqlite_aclk_alert.h +++ b/src/database/sqlite/sqlite_aclk_alert.h diff --git a/database/sqlite/sqlite_aclk_node.c b/src/database/sqlite/sqlite_aclk_node.c index dcc8c375c..dcc8c375c 100644 --- a/database/sqlite/sqlite_aclk_node.c +++ b/src/database/sqlite/sqlite_aclk_node.c diff --git a/database/sqlite/sqlite_aclk_node.h b/src/database/sqlite/sqlite_aclk_node.h index 6afdf8d78..6afdf8d78 100644 --- a/database/sqlite/sqlite_aclk_node.h +++ b/src/database/sqlite/sqlite_aclk_node.h diff --git a/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c index 26ed8a96a..ad76a1ee2 100644 --- a/database/sqlite/sqlite_context.c +++ b/src/database/sqlite/sqlite_context.c @@ -52,7 +52,7 @@ int sql_init_context_database(int memory) if (likely(!memory)) target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION); - if (configure_sqlite_database(db_context_meta, target_version)) + if (configure_sqlite_database(db_context_meta, target_version, "context_config")) return 1; if (likely(!memory)) @@ -60,34 +60,17 @@ int sql_init_context_database(int memory) else snprintfz(buf, sizeof(buf) - 1, "ATTACH DATABASE ':memory:' as meta"); - if(init_database_batch(db_context_meta, list)) return 1; + if(init_database_batch(db_context_meta, list, "context")) return 1; - if (init_database_batch(db_context_meta, &database_context_config[0])) + if (init_database_batch(db_context_meta, &database_context_config[0], "context_init")) return 1; - if (init_database_batch(db_context_meta, &database_context_cleanup[0])) + if (init_database_batch(db_context_meta, &database_context_cleanup[0], "context_cleanup")) return 1; return 0; } -/* - * Close the sqlite database - */ - -void sql_close_context_database(void) -{ - int rc; - if (unlikely(!db_context_meta)) - return; - - netdata_log_info("Closing context SQLite database"); - - rc = sqlite3_close_v2(db_context_meta); - if (unlikely(rc != SQLITE_OK)) - error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc)); -} - // // Fetching data // @@ -421,6 +404,12 @@ int sql_context_cache_stats(int op) return count; } + +uint64_t sqlite_get_context_space(void) +{ + return sqlite_get_db_space(db_context_meta); +} + // // TESTING FUNCTIONS // @@ -455,7 +444,8 @@ int ctx_unittest(void) uuid_t host_uuid; uuid_generate(host_uuid); - initialize_thread_key_pool(); + if (sqlite_library_init()) + return 1; int rc = sql_init_context_database(1); @@ -531,7 +521,8 @@ int ctx_unittest(void) ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL); netdata_log_info("List context end after delete"); - sql_close_context_database(); + sql_close_database(db_context_meta, "CONTEXT"); + sqlite_library_shutdown(); return 0; } diff --git a/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h index 2586916ea..92d02fdd2 100644 --- a/database/sqlite/sqlite_context.h +++ b/src/database/sqlite/sqlite_context.h @@ -65,6 +65,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data); int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data); int sql_init_context_database(int memory); +uint64_t sqlite_get_context_space(void); void sql_close_context_database(void); int ctx_unittest(void); #endif //NETDATA_SQLITE_CONTEXT_H diff --git a/database/sqlite/sqlite_db_migration.c b/src/database/sqlite/sqlite_db_migration.c index 29da6c249..0131c4bf6 100644 --- a/database/sqlite/sqlite_db_migration.c +++ b/src/database/sqlite/sqlite_db_migration.c @@ -153,17 +153,36 @@ const char *database_migrate_v13_v14[] = { NULL }; +const char *database_migrate_v16_v17[] = { + "ALTER TABLE alert_hash ADD time_group_condition INT", + "ALTER TABLE alert_hash ADD time_group_value DOUBLE", + "ALTER TABLE alert_hash ADD dims_group INT", + "ALTER TABLE alert_hash ADD data_source INT", + NULL +}; + +// Note: Same as database_migrate_v16_v17. This is not wrong +// Do additional migration to handle agents that created wrong alert_hash table +const char *database_migrate_v17_v18[] = { + "ALTER TABLE alert_hash ADD time_group_condition INT", + "ALTER TABLE alert_hash ADD time_group_value DOUBLE", + "ALTER TABLE alert_hash ADD dims_group INT", + "ALTER TABLE alert_hash ADD data_source INT", + NULL +}; + + static int do_migration_v1_v2(sqlite3 *database) { if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "hops")) - return init_database_batch(database, &database_migrate_v1_v2[0]); + return init_database_batch(database, &database_migrate_v1_v2[0], "meta_migrate"); return 0; } static int do_migration_v2_v3(sqlite3 *database) { if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "memory_mode")) - return init_database_batch(database, &database_migrate_v2_v3[0]); + return init_database_batch(database, &database_migrate_v2_v3[0], "meta_migrate"); return 0; } @@ -198,12 +217,12 @@ static int do_migration_v3_v4(sqlite3 *database) static int do_migration_v4_v5(sqlite3 *database) { - return init_database_batch(database, &database_migrate_v4_v5[0]); + return init_database_batch(database, &database_migrate_v4_v5[0], "meta_migrate"); } static int do_migration_v5_v6(sqlite3 *database) { - return init_database_batch(database, &database_migrate_v5_v6[0]); + return init_database_batch(database, &database_migrate_v5_v6[0], "meta_migrate"); } static int do_migration_v6_v7(sqlite3 *database) @@ -341,14 +360,14 @@ static int do_migration_v8_v9(sqlite3 *database) static int do_migration_v9_v10(sqlite3 *database) { if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "chart_labels")) - return init_database_batch(database, &database_migrate_v9_v10[0]); + return init_database_batch(database, &database_migrate_v9_v10[0], "meta_migrate"); return 0; } static int do_migration_v10_v11(sqlite3 *database) { if (table_exists_in_database(database, "health_log") && !column_exists_in_table(database, "health_log", "chart_name")) - return init_database_batch(database, &database_migrate_v10_v11[0]); + return init_database_batch(database, &database_migrate_v10_v11[0], "meta_migrate"); return 0; } @@ -360,7 +379,7 @@ static int do_migration_v11_v12(sqlite3 *database) if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary") && table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary")) - rc = init_database_batch(database, &database_migrate_v11_v12[0]); + rc = init_database_batch(database, &database_migrate_v11_v12[0], "meta_migrate"); if (!rc) sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL); @@ -430,17 +449,34 @@ static int do_migration_v15_v16(sqlite3 *database) return 0; } +static int do_migration_v16_v17(sqlite3 *database) +{ + if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "time_group_condition")) + return init_database_batch(database, &database_migrate_v16_v17[0], "meta_migrate"); + + return 0; +} + +static int do_migration_v17_v18(sqlite3 *database) +{ + if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "time_group_condition")) + return init_database_batch(database, &database_migrate_v17_v18[0], "meta_migrate"); + + return 0; +} + + static int do_migration_v12_v13(sqlite3 *database) { int rc = 0; if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary")) { - rc = init_database_batch(database, &database_migrate_v12_v13_detail[0]); + rc = init_database_batch(database, &database_migrate_v12_v13_detail[0], "meta_migrate"); sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL); } if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary")) - rc = init_database_batch(database, &database_migrate_v12_v13_hash[0]); + rc = init_database_batch(database, &database_migrate_v12_v13_hash[0], "meta_migrate"); return rc; } @@ -448,7 +484,7 @@ static int do_migration_v12_v13(sqlite3 *database) static int do_migration_v13_v14(sqlite3 *database) { if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "last_connected")) - return init_database_batch(database, &database_migrate_v13_v14[0]); + return init_database_batch(database, &database_migrate_v13_v14[0], "meta_migrate"); return 0; } @@ -466,7 +502,7 @@ const char *database_ml_migrate_v1_v2[] = { static int do_ml_migration_v1_v2(sqlite3 *database) { if (get_auto_vaccum(database) != 2) - return init_database_batch(database, &database_ml_migrate_v1_v2[0]); + return init_database_batch(database, &database_ml_migrate_v1_v2[0], "ml_migrate"); return 0; } @@ -527,6 +563,8 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = { {.name = "v13 to v14", .func = do_migration_v13_v14}, {.name = "v14 to v15", .func = do_migration_v14_v15}, {.name = "v15 to v16", .func = do_migration_v15_v16}, + {.name = "v16 to v17", .func = do_migration_v16_v17}, + {.name = "v17 to v18", .func = do_migration_v17_v18}, // the terminator of this array {.name = NULL, .func = NULL} }; diff --git a/database/sqlite/sqlite_db_migration.h b/src/database/sqlite/sqlite_db_migration.h index e3c1be84f..e3c1be84f 100644 --- a/database/sqlite/sqlite_db_migration.h +++ b/src/database/sqlite/sqlite_db_migration.h diff --git a/src/database/sqlite/sqlite_functions.c b/src/database/sqlite/sqlite_functions.c new file mode 100644 index 000000000..1dc2022b3 --- /dev/null +++ b/src/database/sqlite/sqlite_functions.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "sqlite_functions.h" + +#define MAX_PREPARED_STATEMENTS (32) +pthread_key_t key_pool[MAX_PREPARED_STATEMENTS]; + +SQLITE_API int sqlite3_exec_monitored( + sqlite3 *db, /* An open database */ + const char *sql, /* SQL to be evaluated */ + int (*callback)(void*,int,char**,char**), /* Callback function */ + void *data, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ +) { + int rc = sqlite3_exec(db, sql, callback, data, errmsg); + global_statistics_sqlite3_query_completed(rc == SQLITE_OK, rc == SQLITE_BUSY, rc == SQLITE_LOCKED); + return rc; +} + +SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) { + int rc; + int cnt = 0; + + while (cnt++ < SQL_MAX_RETRY) { + rc = sqlite3_step(stmt); + switch (rc) { + case SQLITE_DONE: + global_statistics_sqlite3_query_completed(1, 0, 0); + break; + case SQLITE_ROW: + global_statistics_sqlite3_row_completed(); + break; + case SQLITE_BUSY: + case SQLITE_LOCKED: + global_statistics_sqlite3_query_completed(false, rc == SQLITE_BUSY, rc == SQLITE_LOCKED); + usleep(SQLITE_INSERT_DELAY * USEC_PER_MS); + continue; + default: + break; + } + break; + } + return rc; +} + +static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database) +{ + + if (!res && !database) + return false; + + if (!database) + database = sqlite3_db_handle(res); + + if (db_meta == database) { + char recover_file[FILENAME_MAX + 1]; + snprintfz(recover_file, FILENAME_MAX, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir); + int fd = open(recover_file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 444); + if (fd >= 0) { + close(fd); + return true; + } + } + return false; +} + +int execute_insert(sqlite3_stmt *res) +{ + int rc; + rc = sqlite3_step_monitored(res); + if (rc == SQLITE_CORRUPT) { + (void)mark_database_to_recover(res, NULL); + error_report("SQLite error %d", rc); + } + return rc; +} + +int configure_sqlite_database(sqlite3 *database, int target_version, const char *description) +{ + char buf[1024 + 1] = ""; + const char *list[2] = { buf, NULL }; + + // https://www.sqlite.org/pragma.html#pragma_auto_vacuum + // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL; + snprintfz(buf, sizeof(buf) - 1, "PRAGMA auto_vacuum=%s", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL")); + if (init_database_batch(database, list, description)) + return 1; + + // https://www.sqlite.org/pragma.html#pragma_synchronous + // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA; + snprintfz(buf, sizeof(buf) - 1, "PRAGMA synchronous=%s", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL")); + if (init_database_batch(database, list, description)) + return 1; + + // https://www.sqlite.org/pragma.html#pragma_journal_mode + // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF + snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_mode=%s", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL")); + if (init_database_batch(database, list, description)) + return 1; + + // https://www.sqlite.org/pragma.html#pragma_temp_store + // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY; + snprintfz(buf, sizeof(buf) - 1, "PRAGMA temp_store=%s", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY")); + if (init_database_batch(database, list, description)) + return 1; + + // https://www.sqlite.org/pragma.html#pragma_journal_size_limit + // PRAGMA schema.journal_size_limit = N ; + snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_size_limit=%lld", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216)); + if (init_database_batch(database, list, description)) + return 1; + + // https://www.sqlite.org/pragma.html#pragma_cache_size + // PRAGMA schema.cache_size = pages; + // PRAGMA schema.cache_size = -kibibytes; + snprintfz(buf, sizeof(buf) - 1, "PRAGMA cache_size=%lld", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000)); + if (init_database_batch(database, list, description)) + return 1; + + snprintfz(buf, sizeof(buf) - 1, "PRAGMA user_version=%d", target_version); + if (init_database_batch(database, list, description)) + return 1; + + return 0; +} + +#define MAX_OPEN_STATEMENTS (512) + +static void add_stmt_to_list(sqlite3_stmt *res) +{ + static int idx = 0; + static sqlite3_stmt *statements[MAX_OPEN_STATEMENTS]; + + if (unlikely(!res)) { + if (idx) + netdata_log_info("Finilizing %d statements", idx); + else + netdata_log_info("No statements pending to finalize"); + while (idx > 0) { + int rc; + rc = sqlite3_finalize(statements[--idx]); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize statement during shutdown, rc = %d", rc); + } + return; + } + + if (unlikely(idx == MAX_OPEN_STATEMENTS)) + return; +} + +static void release_statement(void *statement) +{ + int rc; + if (unlikely(rc = sqlite3_finalize((sqlite3_stmt *) statement) != SQLITE_OK)) + error_report("Failed to finalize statement, rc = %d", rc); +} + +static void initialize_thread_key_pool(void) +{ + for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++) + (void)pthread_key_create(&key_pool[i], release_statement); +} + +int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement) +{ + static __thread uint32_t keys_used = 0; + + pthread_key_t *key = NULL; + int ret = 1; + + if (likely(keys_used < MAX_PREPARED_STATEMENTS)) + key = &key_pool[keys_used++]; + + int rc = sqlite3_prepare_v2(database, query, -1, statement, 0); + if (rc == SQLITE_OK) { + if (key) + ret = pthread_setspecific(*key, *statement); + if (ret) + add_stmt_to_list(*statement); + } + return rc; +} + +char *get_database_extented_error(sqlite3 *database, int i, const char *description) +{ + const char *err = sqlite3_errstr(sqlite3_extended_errcode(database)); + + if (!err) + return NULL; + + size_t len = strlen(err)+ strlen(description) + 32; + char *full_err = mallocz(len); + + snprintfz(full_err, len - 1, "%s: %d: %s", description, i, err); + return full_err; +} + +int init_database_batch(sqlite3 *database, const char *batch[], const char *description) +{ + int rc; + char *err_msg = NULL; + for (int i = 0; batch[i]; i++) { + rc = sqlite3_exec_monitored(database, batch[i], 0, 0, &err_msg); + if (rc != SQLITE_OK) { + error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg); + error_report("SQLite failed statement %s", batch[i]); + char *error_str = get_database_extented_error(database, i, description); + if (error_str) + analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str); + sqlite3_free(err_msg); + freez(error_str); + if (SQLITE_CORRUPT == rc) { + if (mark_database_to_recover(NULL, database)) + error_report("Database is corrupted will attempt to fix"); + return SQLITE_CORRUPT; + } + return 1; + } + } + return 0; +} + +// Return 0 OK +// Return 1 Failed +int db_execute(sqlite3 *db, const char *cmd) +{ + int rc; + int cnt = 0; + + while (cnt < SQL_MAX_RETRY) { + char *err_msg; + rc = sqlite3_exec_monitored(db, cmd, 0, 0, &err_msg); + if (likely(rc == SQLITE_OK)) + break; + + ++cnt; + error_report("Failed to execute '%s', rc = %d (%s) -- attempt %d", cmd, rc, err_msg, cnt); + sqlite3_free(err_msg); + + if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) { + usleep(SQLITE_INSERT_DELAY * USEC_PER_MS); + continue; + } + + if (rc == SQLITE_CORRUPT) + mark_database_to_recover(NULL, db); + break; + } + return (rc != SQLITE_OK); +} + +// Utils +int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null) +{ + if (likely(text)) + return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC); + if (!can_be_null) + return 1; + return sqlite3_bind_null(res, position); +} + +#define SQL_DROP_TABLE "DROP table %s" + +void sql_drop_table(const char *table) +{ + if (!table) + return; + + char wstr[255]; + snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table); + + int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL); + if (rc != SQLITE_OK) { + error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc); + } +} + +static int get_pragma_value(sqlite3 *database, const char *sql) +{ + sqlite3_stmt *res = NULL; + int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) + return -1; + + int result = -1; + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW)) + result = sqlite3_column_int(res, 0); + + rc = sqlite3_finalize(res); + (void) rc; + + return result; +} + +int get_free_page_count(sqlite3 *database) +{ + return get_pragma_value(database, "PRAGMA freelist_count"); +} + +int get_database_page_count(sqlite3 *database) +{ + return get_pragma_value(database, "PRAGMA page_count"); +} + +uint64_t sqlite_get_db_space(sqlite3 *db) +{ + if (!db) + return 0; + + uint64_t page_size = (uint64_t) get_pragma_value(db, "PRAGMA page_size"); + uint64_t page_count = (uint64_t) get_pragma_value(db, "PRAGMA page_count"); + + return page_size * page_count; +} + +/* + * Close the sqlite database + */ + +void sql_close_database(sqlite3 *database, const char *database_name) +{ + int rc; + if (unlikely(!database)) + return; + + (void) db_execute(database, "PRAGMA analysis_limit=10000"); + (void) db_execute(database, "PRAGMA optimize"); + + netdata_log_info("%s: Closing sqlite database", database_name); + +#ifdef NETDATA_DEV_MODE + int t_count_used,t_count_hit,t_count_miss,t_count_full, dummy; + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_USED, &dummy, &t_count_used, 0); + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_HIT, &dummy,&t_count_hit, 0); + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &dummy,&t_count_miss, 0); + (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &dummy,&t_count_full, 0); + + netdata_log_info("%s: Database lookaside allocation statistics: Used slots %d, Hit %d, Misses due to small slot size %d, Misses due to slots full %d", database_name, + t_count_used,t_count_hit, t_count_miss, t_count_full); + + (void) sqlite3_db_release_memory(database); +#endif + + rc = sqlite3_close_v2(database); + if (unlikely(rc != SQLITE_OK)) + error_report("%s: Error while closing the sqlite database: rc %d, error \"%s\"", database_name, rc, sqlite3_errstr(rc)); +} + +extern sqlite3 *db_context_meta; + +void sqlite_close_databases(void) +{ + add_stmt_to_list(NULL); + + sql_close_database(db_context_meta, "CONTEXT"); + sql_close_database(db_meta, "METADATA"); +} + +int sqlite_library_init(void) +{ + initialize_thread_key_pool(); + + int rc = sqlite3_initialize(); + + return (SQLITE_OK != rc); +} + +void sqlite_library_shutdown(void) +{ + (void) sqlite3_shutdown(); +} diff --git a/src/database/sqlite/sqlite_functions.h b/src/database/sqlite/sqlite_functions.h new file mode 100644 index 000000000..2841b27f6 --- /dev/null +++ b/src/database/sqlite/sqlite_functions.h @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SQLITE_FUNCTIONS_H +#define NETDATA_SQLITE_FUNCTIONS_H + +#include "daemon/common.h" +#include "sqlite3.h" + +void analytics_set_data_str(char **name, const char *value); + +#define SQL_MAX_RETRY (100) +#define SQLITE_INSERT_DELAY (10) // Insert delay in case of lock + +SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt); +SQLITE_API int sqlite3_exec_monitored( + sqlite3 *db, /* An open database */ + const char *sql, /* SQL to be evaluated */ + int (*callback)(void*,int,char**,char**), /* Callback function */ + void *data, /* 1st argument to callback */ + char **errmsg /* Error msg written here */ + ); + +// Initialization and shutdown +int init_database_batch(sqlite3 *database, const char *batch[], const char *description); +int configure_sqlite_database(sqlite3 *database, int target_version, const char *description); + +// Helpers +int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null); +int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement); +int execute_insert(sqlite3_stmt *res); +int db_execute(sqlite3 *database, const char *cmd); +char *get_database_extented_error(sqlite3 *database, int i, const char *description); + +void sql_drop_table(const char *table); +void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv); + +uint64_t sqlite_get_db_space(sqlite3 *db); + +int get_free_page_count(sqlite3 *database); +int get_database_page_count(sqlite3 *database); + +int sqlite_library_init(void); +void sqlite_library_shutdown(void); + +void sql_close_database(sqlite3 *database, const char *database_name); +void sqlite_close_databases(void); +#endif //NETDATA_SQLITE_FUNCTIONS_H diff --git a/database/sqlite/sqlite_health.c b/src/database/sqlite/sqlite_health.c index 7d79ff70b..ea883c51b 100644 --- a/database/sqlite/sqlite_health.c +++ b/src/database/sqlite/sqlite_health.c @@ -3,6 +3,7 @@ #include "sqlite_health.h" #include "sqlite_functions.h" #include "sqlite_db_migration.h" +#include "health/health_internals.h" #define MAX_HEALTH_SQL_SIZE 2048 #define SQLITE3_BIND_STRING_OR_NULL(res, key, param) \ @@ -101,7 +102,8 @@ failed: "config_hash_id, name, chart, exec, recipient, units, chart_context, last_transition_id, chart_name) " \ "VALUES (@host_id,@alarm_id, @config_hash_id,@name,@chart,@exec,@recipient,@units,@chart_context," \ "@last_transition_id,@chart_name) ON CONFLICT (host_id, alarm_id) DO UPDATE " \ - "SET last_transition_id = excluded.last_transition_id, chart_name = excluded.chart_name RETURNING health_log_id" + "SET last_transition_id = excluded.last_transition_id, chart_name = excluded.chart_name, " \ + "config_hash_id=excluded.config_hash_id RETURNING health_log_id" #define SQL_INSERT_HEALTH_LOG_DETAIL \ "INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, " \ @@ -893,18 +895,21 @@ void sql_health_alarm_log_load(RRDHOST *host) /* * Store an alert config hash in the database */ -#define SQL_STORE_ALERT_CONFIG_HASH \ - "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \ - "on_key, class, component, type, os, hosts, lookup, every, units, calc, plugin, module, " \ - "charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \ - "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \ - "p_db_lookup_before, p_update_every, source, chart_labels, summary) values (@hash_id,UNIXEPOCH(),@alarm,@template," \ - "@on_key,@class,@component,@type,@os,@hosts,@lookup,@every,@units,@calc,@plugin,@module," \ - "@charts,@green,@red,@warn,@crit,@exec,@to_key,@info,@delay,@options,@repeat,@host_labels," \ - "@p_db_lookup_dimensions,@p_db_lookup_method,@p_db_lookup_options,@p_db_lookup_after," \ - "@p_db_lookup_before,@p_update_every,@source,@chart_labels,@summary)" - -int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg) +#define SQL_STORE_ALERT_CONFIG_HASH \ + "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \ + "on_key, class, component, type, lookup, every, units, calc, " \ + "green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \ + "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \ + "p_db_lookup_before, p_update_every, source, chart_labels, summary, time_group_condition, " \ + "time_group_value, dims_group, data_source) " \ + "values (@hash_id,UNIXEPOCH(),@alarm,@template," \ + "@on_key,@class,@component,@type,@lookup,@every,@units,@calc," \ + "@green,@red,@warn,@crit,@exec,@to_key,@info,@delay,@options,@repeat,@host_labels," \ + "@p_db_lookup_dimensions,@p_db_lookup_method,@p_db_lookup_options,@p_db_lookup_after," \ + "@p_db_lookup_before,@p_update_every,@source,@chart_labels,@summary, @time_group_condition, " \ + "@time_group_value, @dims_group, @data_source)" + +int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused) { static __thread sqlite3_stmt *res = NULL; int rc, param = 0; @@ -923,133 +928,153 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg) return 1; } } + BUFFER *buf = buffer_create(128, NULL); - rc = sqlite3_bind_blob(res, ++param, hash_id, sizeof(*hash_id), SQLITE_STATIC); + rc = sqlite3_bind_blob(res, ++param, &ap->config.hash_id, sizeof(ap->config.hash_id), SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->alarm, ++param); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; + if (ap->match.is_template) + rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param); + else + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.name, ++param); - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->template_key, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->on, ++param); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; + if (ap->match.is_template) + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.name, ++param); + else + rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param); - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->classification, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->component, ++param); + if (ap->match.is_template) + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.on.context, ++param); + else + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.on.chart, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->type, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.classification, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->os, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.component, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->host, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.type, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->lookup, ++param); + // Rebuild lookup + rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param); // lookup line if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->every, ++param); + rc = sqlite3_bind_int(res, ++param, ap->config.update_every); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->units, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.units, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->calc, ++param); + if (ap->config.calculation) + rc = sqlite3_bind_text(res, ++param, expression_source(ap->config.calculation), -1, SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->plugin, ++param); + NETDATA_DOUBLE green = NAN; + rc = sqlite3_bind_double(res, ++param, green); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->module, ++param); + NETDATA_DOUBLE red = NAN; + rc = sqlite3_bind_double(res, ++param, red); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->charts, ++param); + if (ap->config.warning) + rc = sqlite3_bind_text(res, ++param, expression_source(ap->config.warning), -1, SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->green, ++param); + if (ap->config.critical) + rc = sqlite3_bind_text(res, ++param, expression_source(ap->config.critical), -1, SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->red, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.exec, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->warn, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.recipient, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->crit, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.info, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->exec, ++param); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; + if (ap->config.delay_up_duration) + buffer_sprintf(buf, "up %ds ", ap->config.delay_up_duration); - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->to, ++param); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; + if (ap->config.delay_down_duration) + buffer_sprintf(buf, "down %ds ", ap->config.delay_down_duration); - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->info, ++param); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; + if (ap->config.delay_multiplier) + buffer_sprintf(buf, "multiplier %.1f ", ap->config.delay_multiplier); + + if (ap->config.delay_max_duration) + buffer_sprintf(buf, "max %ds", ap->config.delay_max_duration); - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->delay, ++param); + // delay + rc = sqlite3_bind_text(res, ++param, buffer_tostring(buf), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->options, ++param); + if (ap->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION) + rc = sqlite3_bind_text(res, ++param, "no-clear-notification", -1, SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->repeat, ++param); + rc = sqlite3_bind_int(res, ++param, ap->config.update_every); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->host_labels, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.host_labels, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - if (cfg->p_db_lookup_after) { - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->p_db_lookup_dimensions, ++param); + if (ap->config.after) { + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.dimensions, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->p_db_lookup_method, ++param); + rc = sqlite3_bind_text(res, ++param, time_grouping_id2txt(ap->config.time_group), -1, SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_options); + rc = sqlite3_bind_int(res, ++param, (int) RRDR_OPTIONS_REMOVE_OVERLAPPING(ap->config.options)); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_after); + rc = sqlite3_bind_int64(res, ++param, (int) ap->config.after); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_before); + rc = sqlite3_bind_int64(res, ++param, (int) ap->config.before); if (unlikely(rc != SQLITE_OK)) goto bind_fail; } else { @@ -1074,19 +1099,35 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg) goto bind_fail; } - rc = sqlite3_bind_int(res, ++param, cfg->p_update_every); + rc = sqlite3_bind_int(res, ++param, ap->config.update_every); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->source, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.source, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->chart_labels, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.chart_labels, ++param); if (unlikely(rc != SQLITE_OK)) goto bind_fail; - rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->summary, ++param); + rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.summary, ++param); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, ap->config.time_group_condition); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_double(res, ++param, ap->config.time_group_value); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, ap->config.dims_group); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, ap->config.data_source); if (unlikely(rc != SQLITE_OK)) goto bind_fail; @@ -1098,9 +1139,11 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg) if (unlikely(rc != SQLITE_OK)) error_report("Failed to reset statement in alert hash_id store function, rc = %d", rc); + buffer_free(buf); return 0; bind_fail: + buffer_free(buf); error_report("Failed to bind parameter %d to store alert hash_id, rc = %d", param, rc); rc = sqlite3_reset(res); if (unlikely(rc != SQLITE_OK)) @@ -1108,75 +1151,6 @@ bind_fail: return 1; } -/* - alert hashes are used for cloud communication. - if cloud is disabled or openssl is not available (which will prevent cloud connectivity) - skip hash calculations -*/ -#if defined ENABLE_HTTPS -#define DIGEST_ALERT_CONFIG_VAL(v) ((v) ? EVP_DigestUpdate(evpctx, (string2str(v)), string_strlen((v))) : EVP_DigestUpdate(evpctx, "", 1)) -#endif -int alert_hash_and_store_config( - uuid_t hash_id, - struct alert_config *cfg, - int store_hash) -{ -#if defined ENABLE_HTTPS - EVP_MD_CTX *evpctx; - unsigned char hash_value[EVP_MAX_MD_SIZE]; - unsigned int hash_len; - evpctx = EVP_MD_CTX_create(); - EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL); - - DIGEST_ALERT_CONFIG_VAL(cfg->alarm); - DIGEST_ALERT_CONFIG_VAL(cfg->template_key); - DIGEST_ALERT_CONFIG_VAL(cfg->os); - DIGEST_ALERT_CONFIG_VAL(cfg->host); - DIGEST_ALERT_CONFIG_VAL(cfg->on); - DIGEST_ALERT_CONFIG_VAL(cfg->plugin); - DIGEST_ALERT_CONFIG_VAL(cfg->module); - DIGEST_ALERT_CONFIG_VAL(cfg->charts); - DIGEST_ALERT_CONFIG_VAL(cfg->lookup); - DIGEST_ALERT_CONFIG_VAL(cfg->calc); - DIGEST_ALERT_CONFIG_VAL(cfg->every); - DIGEST_ALERT_CONFIG_VAL(cfg->green); - DIGEST_ALERT_CONFIG_VAL(cfg->red); - DIGEST_ALERT_CONFIG_VAL(cfg->warn); - DIGEST_ALERT_CONFIG_VAL(cfg->crit); - DIGEST_ALERT_CONFIG_VAL(cfg->exec); - DIGEST_ALERT_CONFIG_VAL(cfg->to); - DIGEST_ALERT_CONFIG_VAL(cfg->units); - DIGEST_ALERT_CONFIG_VAL(cfg->info); - DIGEST_ALERT_CONFIG_VAL(cfg->classification); - DIGEST_ALERT_CONFIG_VAL(cfg->component); - DIGEST_ALERT_CONFIG_VAL(cfg->type); - DIGEST_ALERT_CONFIG_VAL(cfg->delay); - DIGEST_ALERT_CONFIG_VAL(cfg->options); - DIGEST_ALERT_CONFIG_VAL(cfg->repeat); - DIGEST_ALERT_CONFIG_VAL(cfg->host_labels); - DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels); - DIGEST_ALERT_CONFIG_VAL(cfg->summary); - - EVP_DigestFinal_ex(evpctx, hash_value, &hash_len); - EVP_MD_CTX_destroy(evpctx); - fatal_assert(hash_len > sizeof(uuid_t)); - - char uuid_str[UUID_STR_LEN]; - uuid_unparse_lower(*((uuid_t *)&hash_value), uuid_str); - uuid_copy(hash_id, *((uuid_t *)&hash_value)); - - /* store everything, so it can be recreated when not in memory or just a subset ? */ - if (store_hash) - (void)sql_store_alert_config_hash( (uuid_t *)&hash_value, cfg); -#else - UNUSED(hash_id); - UNUSED(cfg); - UNUSED(store_hash); -#endif - - return 1; -} - #define SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT \ "SELECT hld.new_status FROM health_log hl, health_log_detail hld " \ "WHERE hl.host_id = @host_id AND hl.alarm_id = @alarm_id AND hld.unique_id != @unique_id AND hld.flags & @flags " \ @@ -1580,10 +1554,9 @@ static uint32_t get_next_alarm_event_id(uint64_t health_log_id, uint32_t alarm_i } #define SQL_GET_ALARM_ID \ - "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart " \ - "AND name = @name AND config_hash_id = @config_hash_id" + "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart AND name = @name" -uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id) +uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id) { int rc = 0; sqlite3_stmt *res = NULL; @@ -1617,13 +1590,6 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t * return alarm_id; } - rc = sqlite3_bind_blob(res, 4, config_hash_id, sizeof(*config_hash_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind config_hash_id parameter for SQL_GET_ALARM_ID."); - sqlite3_finalize(res); - return alarm_id; - } - while (sqlite3_step_monitored(res) == SQLITE_ROW) { alarm_id = (uint32_t) sqlite3_column_int64(res, 0); health_log_id = (uint64_t) sqlite3_column_int64(res, 1); @@ -1639,111 +1605,6 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t * return alarm_id; } -#define SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH \ - "UPDATE health_log SET config_hash_id = @config_hash_id WHERE host_id = @host_id AND alarm_id = @alarm_id " \ - "AND health_log_id = @health_log_id" - -void sql_update_alarm_with_config_hash(RRDHOST *host, uint32_t alarm_id, uint64_t health_log_id, uuid_t *config_hash_id) -{ - int rc = 0; - sqlite3_stmt *res = NULL; - - rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, -1, &res, 0); - if (rc != SQLITE_OK) { - error_report("Failed to prepare statement when trying to update an alarm id with a config hash."); - return; - } - - rc = sqlite3_bind_blob(res, 1, config_hash_id, sizeof(*config_hash_id), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind config_hash_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH."); - goto done; - } - - rc = sqlite3_bind_blob(res, 2, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH."); - goto done; - } - - rc = sqlite3_bind_int64(res, 3, (sqlite3_int64) alarm_id); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID."); - goto done; - } - - rc = sqlite3_bind_int64(res, 4, (sqlite3_int64) health_log_id); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID."); - goto done; - } - - rc = execute_insert(res); - if (unlikely(rc != SQLITE_DONE)) - error_report("Failed to execute SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, rc = %d", rc); - -done: - rc = sqlite3_finalize(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to reset statement to update health log detail table with config hash ids, rc = %d", rc); - -} - -#define SQL_GET_ALARM_ID_CHECK_ZERO_HASH \ - "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart " \ - "AND name = @name AND (config_hash_id IS NULL OR config_hash_id = ZEROBLOB(16))" - -uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id) -{ - int rc = 0; - sqlite3_stmt *res = NULL; - uint32_t alarm_id = 0; - uint64_t health_log_id = 0; - - rc = sqlite3_prepare_v2(db_meta, SQL_GET_ALARM_ID_CHECK_ZERO_HASH, -1, &res, 0); - if (rc != SQLITE_OK) { - error_report("Failed to prepare statement when trying to get an alarm id with zero hash"); - return alarm_id; - } - - rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind host_id parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH."); - sqlite3_finalize(res); - return alarm_id; - } - - rc = SQLITE3_BIND_STRING_OR_NULL(res, chart, 2); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind char parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH."); - sqlite3_finalize(res); - return alarm_id; - } - - rc = SQLITE3_BIND_STRING_OR_NULL(res, name, 3); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind name parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH."); - sqlite3_finalize(res); - return alarm_id; - } - - while (sqlite3_step_monitored(res) == SQLITE_ROW) { - alarm_id = (uint32_t) sqlite3_column_int64(res, 0); - health_log_id = (uint64_t) sqlite3_column_int64(res, 1); - } - - rc = sqlite3_finalize(res); - if (unlikely(rc != SQLITE_OK)) - error_report("Failed to finalize the statement while getting an alarm id."); - - if (alarm_id) { - sql_update_alarm_with_config_hash(host, alarm_id, health_log_id, config_hash_id); - *next_event_id = get_next_alarm_event_id(health_log_id, alarm_id); - } - - return alarm_id; -} - #define SQL_GET_ALARM_ID_FROM_TRANSITION_ID \ "SELECT hld.alarm_id, hl.host_id, hl.chart_context FROM health_log_detail hld, health_log hl " \ "WHERE hld.transition_id = @transition_id " \ @@ -1996,10 +1857,11 @@ done_only_drop: #define SQL_POPULATE_TEMP_CONFIG_TARGET_TABLE "INSERT INTO c_%p (hash_id) VALUES (@hash_id)" #define SQL_SEARCH_CONFIG_LIST \ - "SELECT ah.hash_id, alarm, template, on_key, class, component, type, os, hosts, lookup, every, " \ - " units, calc, families, plugin, module, charts, green, red, warn, crit, " \ + "SELECT ah.hash_id, alarm, template, on_key, class, component, type, lookup, every, " \ + " units, calc, families, green, red, warn, crit, " \ " exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, " \ - " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary " \ + " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary, " \ + " time_group_condition, time_group_value, dims_group, data_source " \ " FROM alert_hash ah, c_%p t where ah.hash_id = t.hash_id" int sql_get_alert_configuration( @@ -2079,16 +1941,11 @@ int sql_get_alert_configuration( acd.classification = (const char *) sqlite3_column_text(res, param++); acd.component = (const char *) sqlite3_column_text(res, param++); acd.type = (const char *) sqlite3_column_text(res, param++); - acd.selectors.os = (const char *) sqlite3_column_text(res, param++); - acd.selectors.hosts = (const char *) sqlite3_column_text(res, param++); acd.value.db.lookup = (const char *) sqlite3_column_text(res, param++); acd.value.every = (const char *) sqlite3_column_text(res, param++); acd.value.units = (const char *) sqlite3_column_text(res, param++); acd.value.calc = (const char *) sqlite3_column_text(res, param++); acd.selectors.families = (const char *) sqlite3_column_text(res, param++); - acd.selectors.plugin = (const char *) sqlite3_column_text(res, param++); - acd.selectors.module = (const char *) sqlite3_column_text(res, param++); - acd.selectors.charts = (const char *) sqlite3_column_text(res, param++); acd.status.green = (const char *) sqlite3_column_text(res, param++); acd.status.red = (const char *) sqlite3_column_text(res, param++); acd.status.warn = (const char *) sqlite3_column_text(res, param++); @@ -2109,6 +1966,10 @@ int sql_get_alert_configuration( acd.source = (const char *) sqlite3_column_text(res, param++); acd.selectors.chart_labels = (const char *) sqlite3_column_text(res, param++); acd.summary = (const char *) sqlite3_column_text(res, param++); + acd.value.db.time_group_condition =(int32_t) sqlite3_column_int(res, param++); + acd.value.db.time_group_value = sqlite3_column_double(res, param++); + acd.value.db.dims_group = (int32_t) sqlite3_column_int(res, param++); + acd.value.db.data_source = (int32_t) sqlite3_column_int(res, param++); cb(&acd, data); added++; diff --git a/database/sqlite/sqlite_health.h b/src/database/sqlite/sqlite_health.h index 5549b7525..1b889436e 100644 --- a/database/sqlite/sqlite_health.h +++ b/src/database/sqlite/sqlite_health.h @@ -2,21 +2,22 @@ #ifndef NETDATA_SQLITE_HEALTH_H #define NETDATA_SQLITE_HEALTH_H -#include "../../daemon/common.h" + +#include "daemon/common.h" #include "sqlite3.h" struct sql_alert_transition_data; struct sql_alert_config_data; +struct rrd_alert_prototype; void sql_health_alarm_log_load(RRDHOST *host); void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae); void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed); -int alert_hash_and_store_config(uuid_t hash_id, struct alert_config *cfg, int store_hash); +int sql_alert_store_config(struct rrd_alert_prototype *ap); void sql_aclk_alert_clean_dead_entries(RRDHOST *host); int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status); void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, time_t after, const char *chart); int health_migrate_old_health_log_table(char *table); -uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id); -uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id); +uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id); void sql_alert_transitions( DICTIONARY *nodes, time_t after, diff --git a/src/database/sqlite/sqlite_metadata.c b/src/database/sqlite/sqlite_metadata.c new file mode 100644 index 000000000..86ecee1f6 --- /dev/null +++ b/src/database/sqlite/sqlite_metadata.c @@ -0,0 +1,2645 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "sqlite_metadata.h" +#include "sqlite3recover.h" +//#include "sqlite_db_migration.h" + +#define DB_METADATA_VERSION 18 + +const char *database_config[] = { + "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, " + "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, " + "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default ''," + "hops INT NOT NULL DEFAULT 0," + "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0," + "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', " + "entries INT NOT NULL DEFAULT 0," + "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)", + + "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, " + "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, " + "chart_type int, memory_mode int, history_entries)", + + "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, " + "multiplier int, divisor int , algorithm int, options text)", + + "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)", + + "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, " + "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))", + + "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)", + + "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, " + "on_key text, class text, component text, type text, os text, hosts text, lookup text, " + "every text, units text, calc text, families text, plugin text, module text, charts text, green text, " + "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, " + "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, " + "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, " + "summary text, time_group_condition INT, time_group_value DOUBLE, dims_group INT, data_source INT)", + + "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, " + "date_created INT, PRIMARY KEY(host_id, system_key))", + + "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, " + "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))", + + "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)" + " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END", + + "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, " + "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, " + "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))", + + "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, " + "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, " + "flags int, exec_run_timestamp int, delay_up_to_timestamp int, " + "info text, exec_code int, new_status real, old_status real, delay int, " + "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)", + + "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)", + "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)", + "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)", + "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)", + + NULL +}; + +const char *database_cleanup[] = { + "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)", + "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)", + "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)", + "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)", + "DROP TRIGGER IF EXISTS tr_dim_del", + "DROP INDEX IF EXISTS ind_d1", + "DROP INDEX IF EXISTS ind_c1", + "DROP INDEX IF EXISTS ind_c2", + "DROP INDEX IF EXISTS alert_hash_index", + "DROP INDEX IF EXISTS health_log_d_ind_4", + "DROP INDEX IF EXISTS health_log_d_ind_1", + "DROP INDEX IF EXISTS health_log_d_ind_5", + NULL +}; + +sqlite3 *db_meta = NULL; + +// SQL statements + +#define SQL_STORE_CLAIM_ID \ + "INSERT INTO node_instance " \ + "(host_id, claim_id, date_created) VALUES (@host_id, @claim_id, UNIXEPOCH()) " \ + "ON CONFLICT(host_id) DO UPDATE SET claim_id = excluded.claim_id" + +#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid" + +#define STORE_HOST_LABEL \ + "INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES " + +#define STORE_CHART_LABEL \ + "INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES " + +#define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())" + +#define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid" + +#define SQL_STORE_HOST_INFO \ + "INSERT OR REPLACE INTO host (host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, " \ + "memory_mode, abbrev_timezone, utc_offset, program_name, program_version, entries, health_enabled, last_connected) " \ + "VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, " \ + "@memory_mode, @abbrev_tz, @utc_offset, @prog_name, @prog_version, @entries, @health_enabled, @last_connected)" + +#define SQL_STORE_CHART \ + "INSERT INTO chart (chart_id, host_id, type, id, name, family, context, title, unit, plugin, module, priority, " \ + "update_every, chart_type, memory_mode, history_entries) " \ + "values (@chart_id, @host_id, @type, @id, @name, @family, @context, @title, @unit, @plugin, @module, @priority, " \ + "@update_every, @chart_type, @memory_mode, @history_entries) " \ + "ON CONFLICT(chart_id) DO UPDATE SET type=excluded.type, id=excluded.id, name=excluded.name, " \ + "family=excluded.family, context=excluded.context, title=excluded.title, unit=excluded.unit, " \ + "plugin=excluded.plugin, module=excluded.module, priority=excluded.priority, update_every=excluded.update_every, " \ + "chart_type=excluded.chart_type, memory_mode = excluded.memory_mode, history_entries = excluded.history_entries" + +#define SQL_STORE_DIMENSION \ + "INSERT INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \ + "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options) " \ + "ON CONFLICT(dim_id) DO UPDATE SET id=excluded.id, name=excluded.name, multiplier=excluded.multiplier, " \ + "divisor=excluded.divisor, algorithm=excluded.algorithm, options=excluded.options" + +#define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id" +#define SELECT_CHART_LIST "SELECT chart_id, rowid FROM chart WHERE rowid > @row_id" +#define SELECT_CHART_LABEL_LIST "SELECT chart_id, rowid FROM chart_label WHERE rowid > @row_id" + +#define SQL_STORE_HOST_SYSTEM_INFO_VALUES \ + "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \ + "(@uuid, @name, @value, UNIXEPOCH())" + +#define CONVERT_EXISTING_LOCALHOST "UPDATE host SET hops = 1 WHERE hops = 0 AND host_id <> @host_id" +#define DELETE_MISSING_NODE_INSTANCES "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)" + +#define METADATA_MAINTENANCE_FIRST_CHECK (1800) // Maintenance first run after agent startup in seconds +#define METADATA_MAINTENANCE_REPEAT (60) // Repeat if last run for dimensions, charts, labels needs more work +#define METADATA_HEALTH_LOG_INTERVAL (3600) // Repeat maintenance for health +#define METADATA_DIM_CHECK_INTERVAL (3600) // Repeat maintenance for dimensions +#define METADATA_CHART_CHECK_INTERVAL (3600) // Repeat maintenance for charts +#define METADATA_LABEL_CHECK_INTERVAL (3600) // Repeat maintenance for labels +#define METADATA_RUNTIME_THRESHOLD (5) // Run time threshold for cleanup task + +#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata +#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata +#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write +#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying) +#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop + +#define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum +#define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum + +enum metadata_opcode { + METADATA_DATABASE_NOOP = 0, + METADATA_DATABASE_TIMER, + METADATA_DEL_DIMENSION, + METADATA_STORE_CLAIM_ID, + METADATA_ADD_HOST_INFO, + METADATA_SCAN_HOSTS, + METADATA_LOAD_HOST_CONTEXT, + METADATA_DELETE_HOST_CHART_LABELS, + METADATA_MAINTENANCE, + METADATA_SYNC_SHUTDOWN, + METADATA_UNITTEST, + // leave this last + // we need it to check for worker utilization + METADATA_MAX_ENUMERATIONS_DEFINED +}; + +#define MAX_PARAM_LIST (2) +struct metadata_cmd { + enum metadata_opcode opcode; + struct completion *completion; + const void *param[MAX_PARAM_LIST]; + struct metadata_cmd *prev, *next; +}; + +typedef enum { + METADATA_FLAG_PROCESSING = (1 << 0), // store or cleanup + METADATA_FLAG_SHUTDOWN = (1 << 1), // Shutting down +} METADATA_FLAG; + +struct metadata_wc { + uv_thread_t thread; + uv_loop_t *loop; + uv_async_t async; + uv_timer_t timer_req; + time_t metadata_check_after; + METADATA_FLAG flags; + struct completion start_stop_complete; + struct completion *scan_complete; + /* FIFO command queue */ + SPINLOCK cmd_queue_lock; + struct metadata_cmd *cmd_base; +}; + +#define metadata_flag_check(target_flags, flag) (__atomic_load_n(&((target_flags)->flags), __ATOMIC_SEQ_CST) & (flag)) +#define metadata_flag_set(target_flags, flag) __atomic_or_fetch(&((target_flags)->flags), (flag), __ATOMIC_SEQ_CST) +#define metadata_flag_clear(target_flags, flag) __atomic_and_fetch(&((target_flags)->flags), ~(flag), __ATOMIC_SEQ_CST) + +struct metadata_wc metasync_worker = {.loop = NULL}; + +// +// For unittest +// +struct thread_unittest { + int join; + unsigned added; + unsigned processed; + unsigned *done; +}; + +int sql_metadata_cache_stats(int op) +{ + int count, dummy; + + if (unlikely(!db_meta)) + return 0; + + netdata_thread_disable_cancelability(); + sqlite3_db_status(db_meta, op, &count, &dummy, 0); + netdata_thread_enable_cancelability(); + return count; +} + +static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id) +{ + if (unlikely(!host)) + return; + + if (unlikely(!node_id)) { + freez(host->node_id); + __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED); + return; + } + + struct aclk_sync_cfg_t *wc = host->aclk_config; + + if (unlikely(!host->node_id)) { + uuid_t *t = mallocz(sizeof(*host->node_id)); + uuid_copy(*t, *node_id); + __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED); + } + else { + uuid_copy(*(host->node_id), *node_id); + } + + if (unlikely(!wc)) + sql_create_aclk_table(host, &host->host_uuid, node_id); + else + uuid_unparse_lower(*node_id, wc->node_id); +} + +#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id" + +int update_node_id(uuid_t *host_id, uuid_t *node_id) +{ + sqlite3_stmt *res = NULL; + RRDHOST *host = NULL; + int rc = 2; + + char host_guid[GUID_LEN + 1]; + uuid_unparse_lower(*host_id, host_guid); + rrd_wrlock(); + host = rrdhost_find_by_guid(host_guid); + if (likely(host)) + set_host_node_id(host, node_id); + rrd_unlock(); + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return 1; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store node instance information"); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to store node instance information"); + goto failed; + } + + rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to store node instance information"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store node instance information, rc = %d", rc); + rc = sqlite3_changes(db_meta); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when storing node instance information"); + + return rc - 1; +} + +#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL" + +int get_node_id(uuid_t *host_id, uuid_t *node_id) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return 1; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to select node instance information for a host"); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to select node instance information"); + goto failed; + } + + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW && node_id)) + uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0))); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when selecting node instance information"); + + return (rc == SQLITE_ROW) ? 0 : -1; +} + +#define SQL_INVALIDATE_NODE_INSTANCES \ + "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \ + "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))" + +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to invalidate node instance ids"); + return; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to invalidate node instance information"); + goto failed; + } + + if (claim_id) + rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, 2); + + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind claim_id parameter to invalidate node instance information"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to invalidate node instance information, rc = %d", rc); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when invalidating node instance information"); +} + +#define SQL_GET_NODE_INSTANCE_LIST \ + "SELECT ni.node_id, ni.host_id, h.hostname " \ + "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0" + +struct node_instance_list *get_node_list(void) +{ + struct node_instance_list *node_list = NULL; + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return NULL; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to get node instance information"); + return NULL; + } + + int row = 0; + char host_guid[UUID_STR_LEN]; + while (sqlite3_step_monitored(res) == SQLITE_ROW) + row++; + + if (sqlite3_reset(res) != SQLITE_OK) { + error_report("Failed to reset the prepared statement while fetching node instance information"); + goto failed; + } + node_list = callocz(row + 1, sizeof(*node_list)); + int max_rows = row; + row = 0; + // TODO: Check to remove lock + rrd_rdlock(); + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t)) + uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0))); + if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) { + uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1); + uuid_unparse_lower(*host_id, host_guid); + RRDHOST *host = rrdhost_find_by_guid(host_guid); + if (!host) + continue; + if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) { + netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host)); + continue; + } + uuid_copy(node_list[row].host_id, *host_id); + node_list[row].queryable = 1; + node_list[row].live = (host && (host == localhost || host->receiver + || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0; + node_list[row].hops = (host && host->system_info) ? host->system_info->hops : + uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0; + node_list[row].hostname = + sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL; + } + row++; + if (row == max_rows) + break; + } + rrd_unlock(); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when fetching node instance information"); + + return node_list; +} + +#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id" + +void sql_load_node_id(RRDHOST *host) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch node id"); + return; + } + + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to load node instance information"); + goto failed; + } + + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW)) { + if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t))) + set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0)); + else + set_host_node_id(host, NULL); + } + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when loading node instance information"); +} + +#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id" + +void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info) +{ + int rc; + + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_INFO, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to read host information"); + return; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter host information"); + goto skip; + } + + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0), + (char *) sqlite3_column_text(res, 1)); + } + +skip: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading host information"); +} + +#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \ + "AND label_key IS NOT NULL AND label_value IS NOT NULL" + +RRDLABELS *sql_load_host_labels(uuid_t *host_id) +{ + int rc; + + RRDLABELS *labels = NULL; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to read host information"); + return NULL; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter host information"); + goto skip; + } + + labels = rrdlabels_create(); + + while (sqlite3_step_monitored(res) == SQLITE_ROW) { + rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2)); + } + +skip: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading host information"); + return labels; +} + +static int exec_statement_with_uuid(const char *sql, uuid_t *uuid) +{ + int rc, result = 1; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement %s, rc = %d", sql, rc); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc); + goto skip; + } + + rc = execute_insert(res); + if (likely(rc == SQLITE_DONE)) + result = SQLITE_OK; + else + error_report("Failed to execute %s, rc = %d", sql, rc); + +skip: + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize statement %s, rc = %d", sql, rc); + return result; +} + +static void recover_database(const char *sqlite_database, const char *new_sqlite_database) +{ + sqlite3 *database; + int rc = sqlite3_open(sqlite_database, &database); + if (rc != SQLITE_OK) + return; + + netdata_log_info("Recover %s", sqlite_database); + netdata_log_info(" to %s", new_sqlite_database); + + // This will remove the -shm and -wal files when we close the database + (void) db_execute(database, "select count(*) from sqlite_master limit 0"); + + sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database); + if (recover) { + + rc = sqlite3_recover_run(recover); + + if (rc == SQLITE_OK) + netdata_log_info("Recover complete"); + else + netdata_log_error("Recover encountered an error but the database may be usable"); + + rc = sqlite3_recover_finish(recover); + + (void) sqlite3_close(database); + + if (rc == SQLITE_OK) { + rc = rename(new_sqlite_database, sqlite_database); + if (rc == 0) { + netdata_log_info("Renamed %s", new_sqlite_database); + netdata_log_info(" to %s", sqlite_database); + } + } + else + netdata_log_error("Recover failed to free resources"); + } + else + (void) sqlite3_close(database); +} + + +static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + uuid_t uuid; + + if ( argc != 1 ){ + sqlite3_result_null(context); + return ; + } + int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid); + if (rc == -1) { + sqlite3_result_null(context); + return ; + } + + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); +} + +void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + if (argc != 1 ){ + sqlite3_result_null(context); + return ; + } + + if (sqlite3_value_int(argv[0]) != 0) { + struct timespec req = {.tv_sec = 0, .tv_nsec = 1}; + nanosleep(&req, NULL); + } + + sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec()); +} + +void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv) +{ + (void)argc; + (void)argv; + + uuid_t uuid; + uuid_generate_random(uuid); + sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT); +} + +// Init +/* + * Initialize the SQLite database + * Return 0 on success + */ +int sql_init_meta_database(db_check_action_type_t rebuild, int memory) +{ + char *err_msg = NULL; + char sqlite_database[FILENAME_MAX + 1]; + int rc; + + if (likely(!memory)) { + snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir); + rc = unlink(sqlite_database); + snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir); + + if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) { + char new_sqlite_database[FILENAME_MAX + 1]; + snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir); + recover_database(sqlite_database, new_sqlite_database); + if (rebuild & DB_CHECK_RECOVER) + return 0; + } + } + else + strncpyz(sqlite_database, ":memory:", sizeof(sqlite_database) - 1); + + rc = sqlite3_open(sqlite_database, &db_meta); + if (rc != SQLITE_OK) { + error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); + char *error_str = get_database_extented_error(db_meta, 0, "meta_open"); + if (error_str) + analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str); + freez(error_str); + sqlite3_close(db_meta); + db_meta = NULL; + return 1; + } + + if (rebuild & DB_CHECK_RECLAIM_SPACE) { + netdata_log_info("Reclaiming space of %s", sqlite_database); + rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg); + if (rc != SQLITE_OK) { + error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg); + sqlite3_free(err_msg); + } + else { + (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0"); + (void) sqlite3_close(db_meta); + } + return 1; + } + + if (rebuild & DB_CHECK_ANALYZE) { + errno = 0; + netdata_log_info("Running ANALYZE on %s", sqlite_database); + rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg); + if (rc != SQLITE_OK) { + error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg); + sqlite3_free(err_msg); + } + else { + (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0"); + (void) sqlite3_close(db_meta); + } + return 1; + } + + netdata_log_info("SQLite database %s initialization", sqlite_database); + + rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal u2h function"); + + rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal now_usec function"); + + rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to register internal uuid_random function"); + + int target_version = DB_METADATA_VERSION; + + if (likely(!memory)) + target_version = perform_database_migration(db_meta, DB_METADATA_VERSION); + + if (configure_sqlite_database(db_meta, target_version, "meta_config")) + return 1; + + if (init_database_batch(db_meta, &database_config[0], "meta_init")) + return 1; + + if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup")) + return 1; + + netdata_log_info("SQLite database initialization completed"); + + return 0; +} + +// Metadata functions + +struct query_build { + BUFFER *sql; + int count; + char uuid_str[UUID_STR_LEN]; +}; + +#define SQL_DELETE_CHART_LABELS_BY_HOST \ + "DELETE FROM chart_label WHERE chart_id in (SELECT chart_id FROM chart WHERE host_id = @host_id)" + +static void delete_host_chart_labels(uuid_t *host_uuid) +{ + sqlite3_stmt *res = NULL; + + int rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABELS_BY_HOST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to delete chart labels by host"); + return; + } + + rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to host chart labels"); + goto failed; + } + rc = sqlite3_step_monitored(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to execute command to remove host chart labels"); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize statement to remove host chart labels"); +} + +static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + struct query_build *lb = data; + if (unlikely(!lb->count)) + buffer_sprintf(lb->sql, STORE_HOST_LABEL); + else + buffer_strcat(lb->sql, ", "); + buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value); + lb->count++; + return 1; +} + +static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) { + struct query_build *lb = data; + if (unlikely(!lb->count)) + buffer_sprintf(lb->sql, STORE_CHART_LABEL); + else + buffer_strcat(lb->sql, ", "); + buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value); + lb->count++; + return 1; +} + +#define SQL_DELETE_CHART_LABEL "DELETE FROM chart_label WHERE chart_id = @chart_id" +#define SQL_DELETE_CHART_LABEL_HISTORY "DELETE FROM chart_label WHERE date_created < %ld AND chart_id = @chart_id" + +static void clean_old_chart_labels(RRDSET *st) +{ + char sql[512]; + time_t first_time_s = rrdset_first_entry_s(st); + + if (unlikely(!first_time_s)) + snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL); + else + snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL_HISTORY, first_time_s); + + int rc = exec_statement_with_uuid(sql, &st->chart_uuid); + if (unlikely(rc)) + error_report("METADATA: 'host:%s' Failed to clean old labels for chart %s", rrdhost_hostname(st->rrdhost), rrdset_name(st)); +} + +static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t *query_counter) +{ + size_t old_version = st->rrdlabels_last_saved_version; + size_t new_version = rrdlabels_version(st->rrdlabels); + + if (new_version == old_version) + return 0; + + struct query_build tmp = {.sql = work_buffer, .count = 0}; + uuid_unparse_lower(st->chart_uuid, tmp.uuid_str); + rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp); + buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()"); + int rc = db_execute(db_meta, buffer_tostring(work_buffer)); + if (likely(!rc)) { + st->rrdlabels_last_saved_version = new_version; + (*query_counter)++; + } + + clean_old_chart_labels(st); + return rc; +} + +// If the machine guid has changed, then existing one with hops 0 will be marked as hops 1 (child) +void detect_machine_guid_change(uuid_t *host_uuid) +{ + int rc; + + rc = exec_statement_with_uuid(CONVERT_EXISTING_LOCALHOST, host_uuid); + if (!rc) { + if (unlikely(db_execute(db_meta, DELETE_MISSING_NODE_INSTANCES))) + error_report("Failed to remove deleted hosts from node instances"); + } +} + +static int store_claim_id(uuid_t *host_id, uuid_t *claim_id) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return 1; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CLAIM_ID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store host claim id"); + return 1; + } + + rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to store claim id"); + goto failed; + } + + if (claim_id) + rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, 2); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind claim_id parameter to host claim id"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store host claim id rc = %d", rc); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when storing a host claim id"); + + return rc != SQLITE_DONE; +} + +static void delete_dimension_uuid(uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused) +{ + static __thread sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!res)) { + rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res); + if (rc != SQLITE_OK) { + error_report("Failed to prepare statement to delete a dimension uuid"); + return; + } + } + + rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto skip_execution; + + rc = sqlite3_step_monitored(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to delete dimension uuid, rc = %d", rc); + +skip_execution: + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement when deleting dimension UUID, rc = %d", rc); +} + +// +// Store host and host system info information in the database +static int store_host_metadata(RRDHOST *host) +{ + static __thread sqlite3_stmt *res = NULL; + int rc, param = 0; + + if (unlikely((!res))) { + rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store host, rc = %d", rc); + return 1; + } + } + + rc = sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_hostname(host), 0); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_registry_hostname(host), 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, host->rrd_update_every); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_os(host), 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_timezone(host), 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + // TODO: remove/migrate + rc = bind_text_null(res, ++param, "", 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, host->system_info ? host->system_info->hops : 0); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, host->rrd_memory_mode); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_abbrev_timezone(host), 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, host->utc_offset); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_program_name(host), 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, rrdhost_program_version(host), 1); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int64(res, ++param, host->rrd_history_entries); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, (int ) host->health.health_enabled); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int64(res, ++param, (sqlite3_int64) host->last_connected); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + int store_rc = sqlite3_step_monitored(res); + if (unlikely(store_rc != SQLITE_DONE)) + error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc); + + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc); + + return store_rc != SQLITE_DONE; +bind_fail: + error_report("Failed to bind %d parameter to store host %s, rc = %d", param, rrdhost_hostname(host), rc); + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc); + return 1; +} + +static int add_host_sysinfo_key_value(const char *name, const char *value, uuid_t *uuid) +{ + static __thread sqlite3_stmt *res = NULL; + int rc, param = 0; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; + error_report("Database has not been initialized"); + return 0; + } + + if (unlikely((!res))) { + rc = prepare_statement(db_meta, SQL_STORE_HOST_SYSTEM_INFO_VALUES, &res); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store host info values, rc = %d", rc); + return 0; + } + } + + rc = sqlite3_bind_blob(res, ++param, uuid, sizeof(*uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, name, 0); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = bind_text_null(res, ++param, value ? value : "unknown", 0); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + int store_rc = sqlite3_step_monitored(res); + if (unlikely(store_rc != SQLITE_DONE)) + error_report("Failed to store host info value %s, rc = %d", name, rc); + + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement to store host info value %s, rc = %d", name, rc); + + return store_rc == SQLITE_DONE; +bind_fail: + error_report("Failed to bind %d parameter to store host info values %s, rc = %d", param, name, rc); + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement to store host info values %s, rc = %d", name, rc); + return 0; +} + +static bool store_host_systeminfo(RRDHOST *host) +{ + struct rrdhost_system_info *system_info = host->system_info; + + if (unlikely(!system_info)) + return false; + + int ret = 0; + + ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER", system_info->container, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection, &host->host_uuid); + ret += add_host_sysinfo_key_value("NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node, &host->host_uuid); + + return !(24 == ret); +} + + +/* + * Store a chart in the database + */ + +static int store_chart_metadata(RRDSET *st) +{ + static __thread sqlite3_stmt *res = NULL; + int rc, param = 0, store_rc = 0; + + if (unlikely(!res)) { + rc = prepare_statement(db_meta, SQL_STORE_CHART, &res); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store chart, rc = %d", rc); + return 1; + } + } + + rc = sqlite3_bind_blob(res, ++param, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_blob(res, ++param, &st->rrdhost->host_uuid, sizeof(st->rrdhost->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, string2str(st->parts.type), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, string2str(st->parts.id), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + const char *name = string2str(st->parts.name); + if (name && *name) + rc = sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, ++param); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, rrdset_family(st), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, rrdset_context(st), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, rrdset_title(st), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, rrdset_units(st), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, rrdset_plugin_name(st), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, rrdset_module_name(st), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, (int) st->priority); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, st->update_every); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, st->chart_type); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, st->rrd_memory_mode); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, (int) st->db.entries); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + store_rc = execute_insert(res); + if (unlikely(store_rc != SQLITE_DONE)) + error_report("Failed to store chart, rc = %d", store_rc); + + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement in chart store function, rc = %d", rc); + + return store_rc != SQLITE_DONE; + +bind_fail: + error_report("Failed to bind parameter %d to store chart, rc = %d", param, rc); + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement in chart store function, rc = %d", rc); + return 1; +} + +/* + * Store a dimension + */ +static int store_dimension_metadata(RRDDIM *rd) +{ + static __thread sqlite3_stmt *res = NULL; + int rc, param = 0; + + if (unlikely(!res)) { + rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to store dimension, rc = %d", rc); + return 1; + } + } + + rc = sqlite3_bind_blob(res, ++param, &rd->metric_uuid, sizeof(rd->metric_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_blob(res, ++param, &rd->rrdset->chart_uuid, sizeof(rd->rrdset->chart_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, string2str(rd->id), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_text(res, ++param, string2str(rd->name), -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, (int) rd->multiplier); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, (int ) rd->divisor); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = sqlite3_bind_int(res, ++param, rd->algorithm); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)) + rc = sqlite3_bind_text(res, ++param, "hidden", -1, SQLITE_STATIC); + else + rc = sqlite3_bind_null(res, ++param); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store dimension, rc = %d", rc); + + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement in store dimension, rc = %d", rc); + return 0; + +bind_fail: + error_report("Failed to bind parameter %d to store dimension, rc = %d", param, rc); + rc = sqlite3_reset(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to reset statement in store dimension, rc = %d", rc); + return 1; +} + +static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused) +{ +#ifdef ENABLE_DBENGINE + if(dbengine_enabled) { + bool no_retention = true; + for (size_t tier = 0; tier < storage_tiers; tier++) { + if (!multidb_ctx[tier]) + continue; + time_t first_time_t = 0, last_time_t = 0; + if (rrdeng_metric_retention_by_uuid((void *) multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) { + if (first_time_t > 0) { + no_retention = false; + break; + } + } + } + return no_retention; + } + else + return false; +#else + return false; +#endif +} + +static bool run_cleanup_loop( + sqlite3_stmt *res, + struct metadata_wc *wc, + bool (*check_cb)(uuid_t *, sqlite3_stmt **, bool), + void (*action_cb)(uuid_t *, sqlite3_stmt **, bool), + uint32_t *total_checked, + uint32_t *total_deleted, + uint64_t *row_id, + sqlite3_stmt **check_stmt, + sqlite3_stmt **action_stmt, + bool check_flag, + bool action_flag) +{ + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + return true; + + int rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) *row_id); + if (unlikely(rc != SQLITE_OK)) + return true; + + time_t start_running = now_monotonic_sec(); + bool time_expired = false; + while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW && + (*total_deleted < MAX_METADATA_CLEANUP && *total_checked < MAX_METADATA_CLEANUP)) { + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + break; + + *row_id = sqlite3_column_int64(res, 1); + rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag); + + if (rc == true) { + action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag); + (*total_deleted)++; + } + + (*total_checked)++; + time_expired = ((now_monotonic_sec() - start_running) > METADATA_RUNTIME_THRESHOLD); + } + return time_expired || (*total_checked == MAX_METADATA_CLEANUP) || (*total_deleted == MAX_METADATA_CLEANUP); +} + + +#define SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION "SELECT count(1) FROM dimension WHERE chart_id = @chart_id" +#define SQL_CHECK_CHART_EXISTENCE_IN_CHART "SELECT count(1) FROM chart WHERE chart_id = @chart_id" + +static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension) +{ + int rc, result = 1; + sqlite3_stmt *res = check_res ? *check_res : NULL; + + if (!res) { + if (check_in_dimension) + rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION, -1, &res, 0); + else + rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_CHART, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to check for chart existence, rc = %d", rc); + return 0; + } + if (check_res) + *check_res = res; + } + + rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind chart uuid parameter, rc = %d", rc); + goto skip; + } + + rc = sqlite3_step_monitored(res); + if (likely(rc == SQLITE_ROW)) + result = sqlite3_column_int(res, 0); + +skip: + if (check_res) + rc = sqlite3_reset(res); + else + rc = sqlite3_finalize(res); + + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to %s statement that checks chart uuid existence rc = %d", check_res ? "reset" : "finalize", rc); + return result == 0; +} + +#define SQL_DELETE_CHART_BY_UUID "DELETE FROM chart WHERE chart_id = @chart_id" +#define SQL_DELETE_CHART_LABEL_BY_UUID "DELETE FROM chart_label WHERE chart_id = @chart_id" + +static void delete_chart_uuid(uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only) +{ + int rc; + sqlite3_stmt *res = action_res ? *action_res : NULL; + + if (!res) { + if (label_only) + rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABEL_BY_UUID, -1, &res, 0); + else + rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_BY_UUID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to check for chart existence, rc = %d", rc); + return; + } + if (action_res) + *action_res = res; + } + + rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind chart uuid parameter, rc = %d", rc); + goto skip; + } + + rc = sqlite3_step_monitored(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to delete a chart uuid from the %s table, rc = %d", label_only ? "labels" : "chart", rc); + +skip: + if (action_res) + rc = sqlite3_reset(res); + else + rc = sqlite3_finalize(res); + + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to %s statement that deletes a chart uuid rc = %d", action_res ? "reset" : "finalize", rc); +} + +static void check_dimension_metadata(struct metadata_wc *wc) +{ + static time_t next_execution_t = 0; + static uint64_t last_row_id = 0; + + time_t now = now_realtime_sec(); + + if (!next_execution_t) + next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK; + + if (next_execution_t && next_execution_t > now) + return; + + int rc; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION_LIST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host dimensions"); + return; + } + + uint32_t total_checked = 0; + uint32_t total_deleted = 0; + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking dimensions starting after row %" PRIu64, last_row_id); + + bool more_to_do = run_cleanup_loop( + res, + wc, + dimension_can_be_deleted, + delete_dimension_uuid, + &total_checked, + &total_deleted, + &last_row_id, + NULL, + NULL, + false, + false); + + now = now_realtime_sec(); + if (more_to_do) + next_execution_t = now + METADATA_MAINTENANCE_REPEAT; + else { + last_row_id = 0; + next_execution_t = now + METADATA_DIM_CHECK_INTERVAL; + } + + nd_log( + NDLS_DAEMON, + NDLP_DEBUG, + "Dimensions checked %u, deleted %u. Checks will %s in %lld seconds", + total_checked, + total_deleted, + last_row_id ? "resume" : "restart", + (long long)(next_execution_t - now)); + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement to check dimensions"); +} + +static void check_chart_metadata(struct metadata_wc *wc) +{ + static time_t next_execution_t = 0; + static uint64_t last_row_id = 0; + + time_t now = now_realtime_sec(); + + if (!next_execution_t) + next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK; + + if (next_execution_t && next_execution_t > now) + return; + + sqlite3_stmt *res = NULL; + + int rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LIST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch charts"); + return; + } + + uint32_t total_checked = 0; + uint32_t total_deleted = 0; + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts starting after row %" PRIu64, last_row_id); + + sqlite3_stmt *check_res = NULL; + sqlite3_stmt *action_res = NULL; + bool more_to_do = run_cleanup_loop( + res, + wc, + chart_can_be_deleted, + delete_chart_uuid, + &total_checked, + &total_deleted, + &last_row_id, + &check_res, + &action_res, + true, + false); + + if (check_res) + sqlite3_finalize(check_res); + + if (action_res) + sqlite3_finalize(action_res); + + now = now_realtime_sec(); + if (more_to_do) + next_execution_t = now + METADATA_MAINTENANCE_REPEAT; + else { + last_row_id = 0; + next_execution_t = now + METADATA_CHART_CHECK_INTERVAL; + } + + nd_log( + NDLS_DAEMON, + NDLP_DEBUG, + "Charts checked %u, deleted %u. Checks will %s in %lld seconds", + total_checked, + total_deleted, + last_row_id ? "resume" : "restart", + (long long)(next_execution_t - now)); + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading charts"); +} + +static void check_label_metadata(struct metadata_wc *wc) +{ + static time_t next_execution_t = 0; + static uint64_t last_row_id = 0; + + time_t now = now_realtime_sec(); + + if (!next_execution_t) + next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK; + + if (next_execution_t && next_execution_t > now) + return; + + int rc; + sqlite3_stmt *res = NULL; + + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LABEL_LIST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch charts"); + return; + } + + uint32_t total_checked = 0; + uint32_t total_deleted = 0; + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts labels starting after row %" PRIu64, last_row_id); + + sqlite3_stmt *check_res = NULL; + sqlite3_stmt *action_res = NULL; + + bool more_to_do = run_cleanup_loop( + res, + wc, + chart_can_be_deleted, + delete_chart_uuid, + &total_checked, + &total_deleted, + &last_row_id, + &check_res, + &action_res, + false, + true); + + if (check_res) + sqlite3_finalize(check_res); + + if (action_res) + sqlite3_finalize(action_res); + + now = now_realtime_sec(); + if (more_to_do) + next_execution_t = now + METADATA_MAINTENANCE_REPEAT; + else { + last_row_id = 0; + next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL; + } + + nd_log( + NDLS_DAEMON, + NDLP_DEBUG, + "Chart labels checked %u, deleted %u. Checks will %s in %lld seconds", + total_checked, + total_deleted, + last_row_id ? "resume" : "restart", + (long long)(next_execution_t - now)); + + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when checking charts"); +} + + +static void cleanup_health_log(struct metadata_wc *wc) +{ + static time_t next_execution_t = 0; + + time_t now = now_realtime_sec(); + + if (!next_execution_t) + next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK; + + if (next_execution_t && next_execution_t > now) + return; + + next_execution_t = now + METADATA_HEALTH_LOG_INTERVAL; + + RRDHOST *host; + + bool is_claimed = claimed(); + dfe_start_reentrant(rrdhost_root_index, host){ + if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) + continue; + sql_health_alarm_log_cleanup(host, is_claimed); + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + break; + } + dfe_done(host); + + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + return; + + (void) db_execute(db_meta,"DELETE FROM health_log WHERE host_id NOT IN (SELECT host_id FROM host)"); + (void) db_execute(db_meta,"DELETE FROM health_log_detail WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)"); +} + +// +// EVENT LOOP STARTS HERE +// + +static void metadata_free_cmd_queue(struct metadata_wc *wc) +{ + spinlock_lock(&wc->cmd_queue_lock); + while(wc->cmd_base) { + struct metadata_cmd *t = wc->cmd_base; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wc->cmd_base, t, prev, next); + freez(t); + } + spinlock_unlock(&wc->cmd_queue_lock); +} + +static void metadata_enq_cmd(struct metadata_wc *wc, struct metadata_cmd *cmd) +{ + if (cmd->opcode == METADATA_SYNC_SHUTDOWN) { + metadata_flag_set(wc, METADATA_FLAG_SHUTDOWN); + goto wakeup_event_loop; + } + + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + goto wakeup_event_loop; + + struct metadata_cmd *t = mallocz(sizeof(*t)); + *t = *cmd; + t->prev = t->next = NULL; + + spinlock_lock(&wc->cmd_queue_lock); + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wc->cmd_base, t, prev, next); + spinlock_unlock(&wc->cmd_queue_lock); + +wakeup_event_loop: + (void) uv_async_send(&wc->async); +} + +static struct metadata_cmd metadata_deq_cmd(struct metadata_wc *wc) +{ + struct metadata_cmd ret; + + spinlock_lock(&wc->cmd_queue_lock); + if(wc->cmd_base) { + struct metadata_cmd *t = wc->cmd_base; + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wc->cmd_base, t, prev, next); + ret = *t; + freez(t); + } + else { + ret.opcode = METADATA_DATABASE_NOOP; + ret.completion = NULL; + } + spinlock_unlock(&wc->cmd_queue_lock); + + return ret; +} + +static void async_cb(uv_async_t *handle) +{ + uv_stop(handle->loop); + uv_update_time(handle->loop); +} + +#define TIMER_INITIAL_PERIOD_MS (1000) +#define TIMER_REPEAT_PERIOD_MS (1000) + +static void timer_cb(uv_timer_t* handle) +{ + uv_stop(handle->loop); + uv_update_time(handle->loop); + + struct metadata_wc *wc = handle->data; + struct metadata_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); + + if (wc->metadata_check_after < now_realtime_sec()) { + cmd.opcode = METADATA_SCAN_HOSTS; + metadata_enq_cmd(wc, &cmd); + } +} + +void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc) +{ + int free_pages = get_free_page_count(database); + int total_pages = get_database_page_count(database); + + if (!threshold) + threshold = DATABASE_FREE_PAGES_THRESHOLD_PC; + + if (!vacuum_pc) + vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC; + + if (free_pages > (total_pages * threshold / 100)) { + + int do_free_pages = (int) (free_pages * vacuum_pc / 100); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "%s: Freeing %d database pages", db_alias, do_free_pages); + + char sql[128]; + snprintfz(sql, sizeof(sql) - 1, "PRAGMA incremental_vacuum(%d)", do_free_pages); + (void) db_execute(database, sql); + } +} + +void run_metadata_cleanup(struct metadata_wc *wc) +{ + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + return; + + check_dimension_metadata(wc); + check_chart_metadata(wc); + check_label_metadata(wc); + cleanup_health_log(wc); + + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) + return; + + vacuum_database(db_meta, "METADATA", DATABASE_FREE_PAGES_THRESHOLD_PC, DATABASE_FREE_PAGES_VACUUM_PC); + + (void) sqlite3_wal_checkpoint(db_meta, NULL); +} + +struct scan_metadata_payload { + uv_work_t request; + struct metadata_wc *wc; + void *data; + BUFFER *work_buffer; + uint32_t max_count; +}; + +struct host_context_load_thread { + uv_thread_t thread; + RRDHOST *host; + bool busy; + bool finished; +}; + +static void restore_host_context(void *arg) +{ + struct host_context_load_thread *hclt = arg; + RRDHOST *host = hclt->host; + + usec_t started_ut = now_monotonic_usec(); (void)started_ut; + rrdhost_load_rrdcontext_data(host); + usec_t ended_ut = now_monotonic_usec(); (void)ended_ut; + + rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD); + +#ifdef ENABLE_ACLK + aclk_queue_node_info(host, false); +#endif + + nd_log( + NDLS_DAEMON, + NDLP_DEBUG, + "Contexts for host %s loaded in %0.2f ms", + rrdhost_hostname(host), + (double)(ended_ut - started_ut) / USEC_PER_MS); + + __atomic_store_n(&hclt->finished, true, __ATOMIC_RELEASE); +} + +// Callback after scan of hosts is done +static void after_start_host_load_context(uv_work_t *req, int status __maybe_unused) +{ + struct scan_metadata_payload *data = req->data; + freez(data); +} + +#define MAX_FIND_THREAD_RETRIES (10) + +static void cleanup_finished_threads(struct host_context_load_thread *hclt, size_t max_thread_slots, bool wait) +{ + if (!hclt) + return; + + for (size_t index = 0; index < max_thread_slots; index++) { + if (__atomic_load_n(&(hclt[index].finished), __ATOMIC_RELAXED) + || (wait && __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE))) { + int rc = uv_thread_join(&(hclt[index].thread)); + if (rc) + nd_log(NDLS_DAEMON, NDLP_WARNING, "Failed to join thread, rc = %d", rc); + __atomic_store_n(&(hclt[index].busy), false, __ATOMIC_RELEASE); + __atomic_store_n(&(hclt[index].finished), false, __ATOMIC_RELEASE); + } + } +} + +static size_t find_available_thread_slot(struct host_context_load_thread *hclt, size_t max_thread_slots, size_t *found_index) +{ + size_t retries = MAX_FIND_THREAD_RETRIES; + while (retries--) { + size_t index = 0; + while (index < max_thread_slots) { + if (false == __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE)) { + *found_index = index; + return true; + } + index++; + } + sleep_usec(10 * USEC_PER_MS); + } + return false; +} + +static void start_all_host_load_context(uv_work_t *req __maybe_unused) +{ + register_libuv_worker_jobs(); + + struct scan_metadata_payload *data = req->data; + struct metadata_wc *wc = data->wc; + + worker_is_busy(UV_EVENT_HOST_CONTEXT_LOAD); + usec_t started_ut = now_monotonic_usec(); (void)started_ut; + + RRDHOST *host; + + size_t max_threads = MIN(get_netdata_cpus() / 2, 6); + if (max_threads < 1) + max_threads = 1; + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Using %zu threads for context loading", max_threads); + struct host_context_load_thread *hclt = max_threads > 1 ? callocz(max_threads, sizeof(*hclt)) : NULL; + + size_t thread_index = 0; + dfe_start_reentrant(rrdhost_root_index, host) { + if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) + continue; + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host)); + + int rc = 0; + if (hclt) { + bool found_slot = false; + do { + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)) + break; + + cleanup_finished_threads(hclt, max_threads, false); + found_slot = find_available_thread_slot(hclt, max_threads, &thread_index); + } while (!found_slot); + + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)) + break; + + __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED); + hclt[thread_index].host = host; + rc = uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]); + } + // if single thread or thread creation failed + if (rc || !hclt) { + struct host_context_load_thread hclt_sync = {.host = host}; + restore_host_context(&hclt_sync); + + if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)) + break; + } + } + dfe_done(host); + + cleanup_finished_threads(hclt, max_threads, true); + freez(hclt); + usec_t ended_ut = now_monotonic_usec(); (void)ended_ut; + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS); + + worker_is_idle(); +} + +// Callback after scan of hosts is done +static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused) +{ + struct scan_metadata_payload *data = req->data; + struct metadata_wc *wc = data->wc; + + metadata_flag_clear(wc, METADATA_FLAG_PROCESSING); + + if (unlikely(wc->scan_complete)) + completion_mark_complete(wc->scan_complete); + + freez(data); +} + +static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_transaction, BUFFER *work_buffer, size_t *query_counter) { + RRDSET *st; + int rc; + + bool more_to_do = false; + uint32_t scan_count = 1; + + sqlite3_stmt *ml_load_stmt = NULL; + + bool load_ml_models = max_count; + + if (use_transaction) + (void)db_execute(db_meta, "BEGIN TRANSACTION"); + + rrdset_foreach_reentrant(st, host) { + if (scan_count == max_count) { + more_to_do = true; + break; + } + if(rrdset_flag_check(st, RRDSET_FLAG_METADATA_UPDATE)) { + (*query_counter)++; + + rrdset_flag_clear(st, RRDSET_FLAG_METADATA_UPDATE); + scan_count++; + + buffer_flush(work_buffer); + rc = check_and_update_chart_labels(st, work_buffer, query_counter); + if (unlikely(rc)) + error_report("METADATA: 'host:%s': Failed to update labels for chart %s", rrdhost_hostname(host), rrdset_name(st)); + else + (*query_counter)++; + + rc = store_chart_metadata(st); + if (unlikely(rc)) + error_report("METADATA: 'host:%s': Failed to store metadata for chart %s", rrdhost_hostname(host), rrdset_name(st)); + } + + RRDDIM *rd; + rrddim_foreach_read(rd, st) { + if(rrddim_flag_check(rd, RRDDIM_FLAG_METADATA_UPDATE)) { + (*query_counter)++; + + rrddim_flag_clear(rd, RRDDIM_FLAG_METADATA_UPDATE); + + if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)) + rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN); + else + rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN); + + rc = store_dimension_metadata(rd); + if (unlikely(rc)) + error_report("METADATA: 'host:%s': Failed to dimension metadata for chart %s. dimension %s", + rrdhost_hostname(host), rrdset_name(st), + rrddim_name(rd)); + } + + if(rrddim_flag_check(rd, RRDDIM_FLAG_ML_MODEL_LOAD)) { + rrddim_flag_clear(rd, RRDDIM_FLAG_ML_MODEL_LOAD); + if (likely(load_ml_models)) + (void) ml_dimension_load_models(rd, &ml_load_stmt); + } + + worker_is_idle(); + } + rrddim_foreach_done(rd); + } + rrdset_foreach_done(st); + + if (use_transaction) + (void)db_execute(db_meta, "COMMIT TRANSACTION"); + + if (ml_load_stmt) { + sqlite3_finalize(ml_load_stmt); + ml_load_stmt = NULL; + } + + return more_to_do; +} + +static void store_host_and_system_info(RRDHOST *host, size_t *query_counter) +{ + if (unlikely(store_host_systeminfo(host))) { + error_report("METADATA: 'host:%s': Failed to store host updated system information in the database", rrdhost_hostname(host)); + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE); + } + else { + if (likely(query_counter)) + (*query_counter)++; + } + + if (unlikely(store_host_metadata(host))) { + error_report("METADATA: 'host:%s': Failed to store host info in the database", rrdhost_hostname(host)); + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE); + } + else { + if (likely(query_counter)) + (*query_counter)++; + } +} + +struct host_chart_label_cleanup { + Pvoid_t JudyL; + Word_t count; +}; + +static void do_chart_label_cleanup(struct host_chart_label_cleanup *cl_cleanup_data) +{ + if (!cl_cleanup_data) + return; + + Word_t Index = 0; + bool first = true; + Pvoid_t *PValue; + while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) { + char *machine_guid = *PValue; + + RRDHOST *host = rrdhost_find_by_guid(machine_guid); + if (likely(!host)) { + uuid_t host_uuid; + if (!uuid_parse(machine_guid, host_uuid)) + delete_host_chart_labels(&host_uuid); + } + + freez(machine_guid); + } + JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0); + freez(cl_cleanup_data); +} + +// Worker thread to scan hosts for pending metadata to store +static void start_metadata_hosts(uv_work_t *req __maybe_unused) +{ + register_libuv_worker_jobs(); + + RRDHOST *host; + int transaction_started = 0; + + struct scan_metadata_payload *data = req->data; + struct metadata_wc *wc = data->wc; + + BUFFER *work_buffer = data->work_buffer; + usec_t all_started_ut = now_monotonic_usec(); (void)all_started_ut; + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking all hosts started"); + usec_t started_ut = now_monotonic_usec(); (void)started_ut; + + do_chart_label_cleanup((struct host_chart_label_cleanup *) data->data); + + bool run_again = false; + worker_is_busy(UV_EVENT_METADATA_STORE); + + if (!data->max_count) + transaction_started = !db_execute(db_meta, "BEGIN TRANSACTION"); + + dfe_start_reentrant(rrdhost_root_index, host) { + if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE)) + continue; + + size_t query_counter = 0; (void)query_counter; + + rrdhost_flag_clear(host,RRDHOST_FLAG_METADATA_UPDATE); + + if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_LABELS))) { + rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_LABELS); + + int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid); + if (likely(!rc)) { + query_counter++; + + buffer_flush(work_buffer); + struct query_build tmp = {.sql = work_buffer, .count = 0}; + uuid_unparse_lower(host->host_uuid, tmp.uuid_str); + rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp); + buffer_strcat(work_buffer, " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()"); + rc = db_execute(db_meta, buffer_tostring(work_buffer)); + + if (unlikely(rc)) { + error_report("METADATA: 'host:%s': failed to update metadata host labels", rrdhost_hostname(host)); + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE); + } + else + query_counter++; + } else { + error_report("METADATA: 'host:%s': failed to delete old host labels", rrdhost_hostname(host)); + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE); + } + } + + if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_CLAIMID))) { + rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_CLAIMID); + uuid_t uuid; + int rc; + if (likely(host->aclk_state.claimed_id && !uuid_parse(host->aclk_state.claimed_id, uuid))) + rc = store_claim_id(&host->host_uuid, &uuid); + else + rc = store_claim_id(&host->host_uuid, NULL); + + if (unlikely(rc)) + rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID | RRDHOST_FLAG_METADATA_UPDATE); + else + query_counter++; + } + if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_INFO))) { + rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_INFO); + store_host_and_system_info(host, &query_counter); + } + + // For clarity + bool use_transaction = data->max_count; + if (unlikely(metadata_scan_host(host, data->max_count, use_transaction, work_buffer, &query_counter))) { + run_again = true; + rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE); + } + usec_t ended_ut = now_monotonic_usec(); (void)ended_ut; + nd_log( + NDLS_DAEMON, + NDLP_DEBUG, + "Host %s saved metadata with %zu SQL statements, in %0.2f ms", + rrdhost_hostname(host), + query_counter, + (double)(ended_ut - started_ut) / USEC_PER_MS); + } + dfe_done(host); + + if (!data->max_count && transaction_started) + transaction_started = db_execute(db_meta, "COMMIT TRANSACTION"); + + usec_t all_ended_ut = now_monotonic_usec(); (void)all_ended_ut; + nd_log( + NDLS_DAEMON, + NDLP_DEBUG, + "Checking all hosts completed in %0.2f ms", + (double)(all_ended_ut - all_started_ut) / USEC_PER_MS); + + if (unlikely(run_again)) + wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_IMMEDIATE; + else { + wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL; + run_metadata_cleanup(wc); + } + worker_is_idle(); +} + +static void metadata_event_loop(void *arg) +{ + worker_register("METASYNC"); + worker_register_job_name(METADATA_DATABASE_NOOP, "noop"); + worker_register_job_name(METADATA_DATABASE_TIMER, "timer"); + worker_register_job_name(METADATA_DEL_DIMENSION, "delete dimension"); + worker_register_job_name(METADATA_STORE_CLAIM_ID, "add claim id"); + worker_register_job_name(METADATA_ADD_HOST_INFO, "add host info"); + worker_register_job_name(METADATA_MAINTENANCE, "maintenance"); + + int ret; + uv_loop_t *loop; + unsigned cmd_batch_size; + struct metadata_wc *wc = arg; + enum metadata_opcode opcode; + + uv_thread_set_name_np(wc->thread, "METASYNC"); + loop = wc->loop = mallocz(sizeof(uv_loop_t)); + ret = uv_loop_init(loop); + if (ret) { + netdata_log_error("uv_loop_init(): %s", uv_strerror(ret)); + goto error_after_loop_init; + } + loop->data = wc; + + ret = uv_async_init(wc->loop, &wc->async, async_cb); + if (ret) { + netdata_log_error("uv_async_init(): %s", uv_strerror(ret)); + goto error_after_async_init; + } + wc->async.data = wc; + + ret = uv_timer_init(loop, &wc->timer_req); + if (ret) { + netdata_log_error("uv_timer_init(): %s", uv_strerror(ret)); + goto error_after_timer_init; + } + wc->timer_req.data = wc; + fatal_assert(0 == uv_timer_start(&wc->timer_req, timer_cb, TIMER_INITIAL_PERIOD_MS, TIMER_REPEAT_PERIOD_MS)); + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Starting metadata sync thread"); + + struct metadata_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); + metadata_flag_clear(wc, METADATA_FLAG_PROCESSING); + + wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_FIRST_CHECK; + + int shutdown = 0; + completion_mark_complete(&wc->start_stop_complete); + BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite); + struct scan_metadata_payload *data; + struct host_chart_label_cleanup *cl_cleanup_data = NULL; + + while (shutdown == 0 || (wc->flags & METADATA_FLAG_PROCESSING)) { + uuid_t *uuid; + RRDHOST *host = NULL; + + worker_is_idle(); + uv_run(loop, UV_RUN_DEFAULT); + + /* wait for commands */ + cmd_batch_size = 0; + do { + if (unlikely(cmd_batch_size >= METADATA_MAX_BATCH_SIZE)) + break; + + cmd = metadata_deq_cmd(wc); + opcode = cmd.opcode; + + if (unlikely(opcode == METADATA_DATABASE_NOOP && metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) { + shutdown = 1; + continue; + } + + ++cmd_batch_size; + + if (likely(opcode != METADATA_DATABASE_NOOP)) + worker_is_busy(opcode); + + switch (opcode) { + case METADATA_DATABASE_NOOP: + case METADATA_DATABASE_TIMER: + break; + case METADATA_DEL_DIMENSION: + uuid = (uuid_t *) cmd.param[0]; + if (likely(dimension_can_be_deleted(uuid, NULL, false))) + delete_dimension_uuid(uuid, NULL, false); + freez(uuid); + break; + case METADATA_STORE_CLAIM_ID: + store_claim_id((uuid_t *) cmd.param[0], (uuid_t *) cmd.param[1]); + freez((void *) cmd.param[0]); + freez((void *) cmd.param[1]); + break; + case METADATA_ADD_HOST_INFO: + host = (RRDHOST *) cmd.param[0]; + store_host_and_system_info(host, NULL); + break; + case METADATA_SCAN_HOSTS: + if (unlikely(metadata_flag_check(wc, METADATA_FLAG_PROCESSING))) + break; + + if (unittest_running) + break; + + data = mallocz(sizeof(*data)); + data->request.data = data; + data->wc = wc; + data->data = cl_cleanup_data; + data->work_buffer = work_buffer; + cl_cleanup_data = NULL; + + if (unlikely(cmd.completion)) { + data->max_count = 0; // 0 will process all pending updates + cmd.completion = NULL; // Do not complete after launching worker (worker will do) + } + else + data->max_count = 5000; + + metadata_flag_set(wc, METADATA_FLAG_PROCESSING); + if (unlikely( + uv_queue_work(loop,&data->request, + start_metadata_hosts, + after_metadata_hosts))) { + // Failed to launch worker -- let the event loop handle completion + cmd.completion = wc->scan_complete; + cl_cleanup_data = data->data; + freez(data); + metadata_flag_clear(wc, METADATA_FLAG_PROCESSING); + } + break; + case METADATA_LOAD_HOST_CONTEXT:; + if (unittest_running) + break; + + data = callocz(1,sizeof(*data)); + data->request.data = data; + data->wc = wc; + if (unlikely( + uv_queue_work(loop,&data->request, start_all_host_load_context, + after_start_host_load_context))) { + freez(data); + } + break; + case METADATA_DELETE_HOST_CHART_LABELS:; + if (!cl_cleanup_data) + cl_cleanup_data = callocz(1,sizeof(*cl_cleanup_data)); + + Pvoid_t *PValue = JudyLIns(&cl_cleanup_data->JudyL, (Word_t) ++cl_cleanup_data->count, PJE0); + if (PValue) + *PValue = (void *) cmd.param[0]; + + break; + case METADATA_UNITTEST:; + struct thread_unittest *tu = (struct thread_unittest *) cmd.param[0]; + sleep_usec(1000); // processing takes 1ms + __atomic_fetch_add(&tu->processed, 1, __ATOMIC_SEQ_CST); + break; + default: + break; + } + + if (cmd.completion) + completion_mark_complete(cmd.completion); + } while (opcode != METADATA_DATABASE_NOOP); + } + + if (!uv_timer_stop(&wc->timer_req)) + uv_close((uv_handle_t *)&wc->timer_req, NULL); + + uv_close((uv_handle_t *)&wc->async, NULL); + int rc; + do { + rc = uv_loop_close(loop); + } while (rc != UV_EBUSY); + + buffer_free(work_buffer); + freez(loop); + worker_unregister(); + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Shutting down metadata thread"); + completion_mark_complete(&wc->start_stop_complete); + if (wc->scan_complete) { + completion_destroy(wc->scan_complete); + freez(wc->scan_complete); + } + metadata_free_cmd_queue(wc); + return; + +error_after_timer_init: + uv_close((uv_handle_t *)&wc->async, NULL); +error_after_async_init: + fatal_assert(0 == uv_loop_close(loop)); +error_after_loop_init: + freez(loop); + worker_unregister(); +} + +void metadata_sync_shutdown(void) +{ + completion_init(&metasync_worker.start_stop_complete); + + struct metadata_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Sending a shutdown command"); + cmd.opcode = METADATA_SYNC_SHUTDOWN; + metadata_enq_cmd(&metasync_worker, &cmd); + + /* wait for metadata thread to shut down */ + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Waiting for shutdown ACK"); + completion_wait_for(&metasync_worker.start_stop_complete); + completion_destroy(&metasync_worker.start_stop_complete); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Shutdown complete"); +} + +void metadata_sync_shutdown_prepare(void) +{ + static bool running = false; + if (unlikely(!metasync_worker.loop || running)) + return; + + running = true; + + struct metadata_cmd cmd; + memset(&cmd, 0, sizeof(cmd)); + + struct metadata_wc *wc = &metasync_worker; + + struct completion *compl = mallocz(sizeof(*compl)); + completion_init(compl); + __atomic_store_n(&wc->scan_complete, compl, __ATOMIC_RELAXED); + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Sending a scan host command"); + uint32_t max_wait_iterations = 2000; + while (unlikely(metadata_flag_check(&metasync_worker, METADATA_FLAG_PROCESSING)) && max_wait_iterations--) { + if (max_wait_iterations == 1999) + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Current worker is running; waiting to finish"); + sleep_usec(1000); + } + + cmd.opcode = METADATA_SCAN_HOSTS; + metadata_enq_cmd(&metasync_worker, &cmd); + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Waiting for host scan completion"); + completion_wait_for(wc->scan_complete); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Host scan complete; can continue with shutdown"); +} + +// ------------------------------------------------------------- +// Init function called on agent startup + +void metadata_sync_init(void) +{ + struct metadata_wc *wc = &metasync_worker; + + memset(wc, 0, sizeof(*wc)); + completion_init(&wc->start_stop_complete); + + fatal_assert(0 == uv_thread_create(&(wc->thread), metadata_event_loop, wc)); + + completion_wait_for(&wc->start_stop_complete); + completion_destroy(&wc->start_stop_complete); + + nd_log(NDLS_DAEMON, NDLP_DEBUG, "SQLite metadata sync initialization complete"); +} + + +// Helpers + +static inline void queue_metadata_cmd(enum metadata_opcode opcode, const void *param0, const void *param1) +{ + struct metadata_cmd cmd; + cmd.opcode = opcode; + cmd.param[0] = param0; + cmd.param[1] = param1; + cmd.completion = NULL; + metadata_enq_cmd(&metasync_worker, &cmd); +} + +// Public +void metaqueue_delete_dimension_uuid(uuid_t *uuid) +{ + if (unlikely(!metasync_worker.loop)) + return; + uuid_t *use_uuid = mallocz(sizeof(*uuid)); + uuid_copy(*use_uuid, *uuid); + queue_metadata_cmd(METADATA_DEL_DIMENSION, use_uuid, NULL); +} + +void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid) +{ + if (unlikely(!host_uuid)) + return; + + uuid_t *local_host_uuid = mallocz(sizeof(*host_uuid)); + uuid_t *local_claim_uuid = NULL; + + uuid_copy(*local_host_uuid, *host_uuid); + if (likely(claim_uuid)) { + local_claim_uuid = mallocz(sizeof(*claim_uuid)); + uuid_copy(*local_claim_uuid, *claim_uuid); + } + queue_metadata_cmd(METADATA_STORE_CLAIM_ID, local_host_uuid, local_claim_uuid); +} + +void metaqueue_host_update_info(RRDHOST *host) +{ + if (unlikely(!metasync_worker.loop)) + return; + queue_metadata_cmd(METADATA_ADD_HOST_INFO, host, NULL); +} + +void metaqueue_ml_load_models(RRDDIM *rd) +{ + rrddim_flag_set(rd, RRDDIM_FLAG_ML_MODEL_LOAD); +} + +void metadata_queue_load_host_context(RRDHOST *host) +{ + if (unlikely(!metasync_worker.loop)) + return; + queue_metadata_cmd(METADATA_LOAD_HOST_CONTEXT, host, NULL); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command to load host contexts"); +} + +void metadata_delete_host_chart_labels(char *machine_guid) +{ + if (unlikely(!metasync_worker.loop)) { + freez(machine_guid); + return; + } + + // Node machine guid is already strdup-ed + queue_metadata_cmd(METADATA_DELETE_HOST_CHART_LABELS, machine_guid, NULL); + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command delete chart labels for host %s", machine_guid); +} + + +// +// unitests +// + +static void *unittest_queue_metadata(void *arg) { + struct thread_unittest *tu = arg; + + struct metadata_cmd cmd; + cmd.opcode = METADATA_UNITTEST; + cmd.param[0] = tu; + cmd.param[1] = NULL; + cmd.completion = NULL; + metadata_enq_cmd(&metasync_worker, &cmd); + + do { + __atomic_fetch_add(&tu->added, 1, __ATOMIC_SEQ_CST); + metadata_enq_cmd(&metasync_worker, &cmd); + sleep_usec(10000); + } while (!__atomic_load_n(&tu->join, __ATOMIC_RELAXED)); + return arg; +} + +static void *metadata_unittest_threads(void) +{ + + unsigned done; + + struct thread_unittest tu = { + .join = 0, + .added = 0, + .processed = 0, + .done = &done, + }; + + // Queue messages / Time it + time_t seconds_to_run = 5; + int threads_to_create = 4; + fprintf( + stderr, + "\nChecking metadata queue using %d threads for %lld seconds...\n", + threads_to_create, + (long long)seconds_to_run); + + netdata_thread_t threads[threads_to_create]; + tu.join = 0; + for (int i = 0; i < threads_to_create; i++) { + char buf[100 + 1]; + snprintf(buf, sizeof(buf) - 1, "META[%d]", i); + netdata_thread_create( + &threads[i], + buf, + NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE, + unittest_queue_metadata, + &tu); + } + (void) uv_async_send(&metasync_worker.async); + sleep_usec(seconds_to_run * USEC_PER_SEC); + + __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED); + for (int i = 0; i < threads_to_create; i++) { + void *retval; + netdata_thread_join(threads[i], &retval); + } + sleep_usec(5 * USEC_PER_SEC); + + fprintf(stderr, "Added %u elements, processed %u\n", tu.added, tu.processed); + + return 0; +} + +int metadata_unittest(void) +{ + metadata_sync_init(); + + // Queue items for a specific period of time + metadata_unittest_threads(); + + metadata_sync_shutdown(); + + return 0; +} diff --git a/src/database/sqlite/sqlite_metadata.h b/src/database/sqlite/sqlite_metadata.h new file mode 100644 index 000000000..a276f5c4d --- /dev/null +++ b/src/database/sqlite/sqlite_metadata.h @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_SQLITE_METADATA_H +#define NETDATA_SQLITE_METADATA_H + +#include "sqlite3.h" +#include "sqlite_functions.h" + +// return a node list +struct node_instance_list { + uuid_t node_id; + uuid_t host_id; + char *hostname; + int live; + int queryable; + int hops; +}; + +typedef enum db_check_action_type { + DB_CHECK_NONE = (1 << 0), + DB_CHECK_RECLAIM_SPACE = (1 << 1), + DB_CHECK_ANALYZE = (1 << 2), + DB_CHECK_CONT = (1 << 3), + DB_CHECK_RECOVER = (1 << 4), +} db_check_action_type_t; + +// To initialize and shutdown +void metadata_sync_init(void); +void metadata_sync_shutdown(void); +void metadata_sync_shutdown_prepare(void); + +void metaqueue_delete_dimension_uuid(uuid_t *uuid); +void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid); +void metaqueue_host_update_info(RRDHOST *host); +void metaqueue_ml_load_models(RRDDIM *rd); +void detect_machine_guid_change(uuid_t *host_uuid); +void metadata_queue_load_host_context(RRDHOST *host); +void metadata_delete_host_chart_labels(char *machine_guid); +void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc); + +int sql_metadata_cache_stats(int op); + +int get_node_id(uuid_t *host_id, uuid_t *node_id); +int update_node_id(uuid_t *host_id, uuid_t *node_id); +struct node_instance_list *get_node_list(void); +void sql_load_node_id(RRDHOST *host); + +// Help build archived hosts in memory when agent starts +void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info); +void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id); +RRDLABELS *sql_load_host_labels(uuid_t *host_id); + +uint64_t sqlite_get_meta_space(void); +int sql_init_meta_database(db_check_action_type_t rebuild, int memory); +void sql_close_meta_database(void); + +// UNIT TEST +int metadata_unittest(void); +#endif //NETDATA_SQLITE_METADATA_H |