summaryrefslogtreecommitdiffstats
path: root/src/database/sqlite
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/database/sqlite/dbdata.c (renamed from database/sqlite/dbdata.c)0
-rw-r--r--src/database/sqlite/sqlite3.c (renamed from database/sqlite/sqlite3.c)0
-rw-r--r--src/database/sqlite/sqlite3.h (renamed from database/sqlite/sqlite3.h)0
-rw-r--r--src/database/sqlite/sqlite3recover.c (renamed from database/sqlite/sqlite3recover.c)0
-rw-r--r--src/database/sqlite/sqlite3recover.h (renamed from database/sqlite/sqlite3recover.h)0
-rw-r--r--src/database/sqlite/sqlite_aclk.c (renamed from database/sqlite/sqlite_aclk.c)22
-rw-r--r--src/database/sqlite/sqlite_aclk.h (renamed from database/sqlite/sqlite_aclk.h)0
-rw-r--r--src/database/sqlite/sqlite_aclk_alert.c (renamed from database/sqlite/sqlite_aclk_alert.c)15
-rw-r--r--src/database/sqlite/sqlite_aclk_alert.h (renamed from database/sqlite/sqlite_aclk_alert.h)0
-rw-r--r--src/database/sqlite/sqlite_aclk_node.c (renamed from database/sqlite/sqlite_aclk_node.c)0
-rw-r--r--src/database/sqlite/sqlite_aclk_node.h (renamed from database/sqlite/sqlite_aclk_node.h)0
-rw-r--r--src/database/sqlite/sqlite_context.c (renamed from database/sqlite/sqlite_context.c)37
-rw-r--r--src/database/sqlite/sqlite_context.h (renamed from database/sqlite/sqlite_context.h)1
-rw-r--r--src/database/sqlite/sqlite_db_migration.c (renamed from database/sqlite/sqlite_db_migration.c)60
-rw-r--r--src/database/sqlite/sqlite_db_migration.h (renamed from database/sqlite/sqlite_db_migration.h)0
-rw-r--r--src/database/sqlite/sqlite_functions.c373
-rw-r--r--src/database/sqlite/sqlite_functions.h47
-rw-r--r--src/database/sqlite/sqlite_health.c (renamed from database/sqlite/sqlite_health.c)365
-rw-r--r--src/database/sqlite/sqlite_health.h (renamed from database/sqlite/sqlite_health.h)9
-rw-r--r--src/database/sqlite/sqlite_metadata.c2645
-rw-r--r--src/database/sqlite/sqlite_metadata.h59
21 files changed, 3328 insertions, 305 deletions
diff --git a/database/sqlite/dbdata.c b/src/database/sqlite/dbdata.c
index 1ad742e04..1ad742e04 100644
--- a/database/sqlite/dbdata.c
+++ b/src/database/sqlite/dbdata.c
diff --git a/database/sqlite/sqlite3.c b/src/database/sqlite/sqlite3.c
index da8c38d09..da8c38d09 100644
--- a/database/sqlite/sqlite3.c
+++ b/src/database/sqlite/sqlite3.c
diff --git a/database/sqlite/sqlite3.h b/src/database/sqlite/sqlite3.h
index 48effe202..48effe202 100644
--- a/database/sqlite/sqlite3.h
+++ b/src/database/sqlite/sqlite3.h
diff --git a/database/sqlite/sqlite3recover.c b/src/database/sqlite/sqlite3recover.c
index 3dae0b7a9..3dae0b7a9 100644
--- a/database/sqlite/sqlite3recover.c
+++ b/src/database/sqlite/sqlite3recover.c
diff --git a/database/sqlite/sqlite3recover.h b/src/database/sqlite/sqlite3recover.h
index 7a1cd1cd8..7a1cd1cd8 100644
--- a/database/sqlite/sqlite3recover.h
+++ b/src/database/sqlite/sqlite3recover.h
diff --git a/database/sqlite/sqlite_aclk.c b/src/database/sqlite/sqlite_aclk.c
index ac574879c..c410406b2 100644
--- a/database/sqlite/sqlite_aclk.c
+++ b/src/database/sqlite/sqlite_aclk.c
@@ -61,7 +61,6 @@ enum {
IDX_UPDATE_EVERY,
IDX_OS,
IDX_TIMEZONE,
- IDX_TAGS,
IDX_HOPS,
IDX_MEMORY_MODE,
IDX_ABBREV_TIMEZONE,
@@ -72,6 +71,7 @@ enum {
IDX_HEALTH_ENABLED,
IDX_LAST_CONNECTED,
IDX_IS_EPHEMERAL,
+ IDX_IS_REGISTERED,
};
static int create_host_callback(void *data, int argc, char **argv, char **column)
@@ -88,21 +88,27 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
time_t age = now_realtime_sec() - last_connected;
int is_ephemeral = 0;
+ int is_registered = 0;
if (argv[IDX_IS_EPHEMERAL])
is_ephemeral = str2i(argv[IDX_IS_EPHEMERAL]);
+ if (argv[IDX_IS_REGISTERED])
+ is_registered = str2i(argv[IDX_IS_REGISTERED]);
+
char guid[UUID_STR_LEN];
uuid_unparse_lower(*(uuid_t *)argv[IDX_HOST_ID], guid);
if (is_ephemeral && age > rrdhost_free_ephemeral_time_s) {
netdata_log_info(
- "Skipping ephemeral hostname \"%s\" with GUID \"%s\", age = %ld seconds (limit %ld seconds)",
+ "%s ephemeral hostname \"%s\" with GUID \"%s\", age = %ld seconds (limit %ld seconds)",
+ is_registered ? "Loading registered" : "Skipping unregistered",
(const char *)argv[IDX_HOSTNAME],
guid,
age,
rrdhost_free_ephemeral_time_s);
- return 0;
+ if (!is_registered)
+ return 0;
}
struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));
@@ -120,7 +126,6 @@ static int create_host_callback(void *data, int argc, char **argv, char **column
(const char *)argv[IDX_TIMEZONE],
(const char *)argv[IDX_ABBREV_TIMEZONE],
(int32_t)(argv[IDX_UTC_OFFSET] ? str2uint32_t(argv[IDX_UTC_OFFSET], NULL) : 0),
- (const char *)argv[IDX_TAGS],
(const char *)(argv[IDX_PROGRAM_NAME] ? argv[IDX_PROGRAM_NAME] : "unknown"),
(const char *)(argv[IDX_PROGRAM_VERSION] ? argv[IDX_PROGRAM_VERSION] : "unknown"),
argv[IDX_UPDATE_EVERY] ? str2i(argv[IDX_UPDATE_EVERY]) : 1,
@@ -557,11 +562,12 @@ void sql_create_aclk_table(RRDHOST *host __maybe_unused, uuid_t *host_uuid __may
#define SQL_FETCH_ALL_HOSTS \
"SELECT host_id, hostname, registry_hostname, update_every, os, " \
- "timezone, tags, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \
+ "timezone, hops, memory_mode, abbrev_timezone, utc_offset, program_name, " \
"program_version, entries, health_enabled, last_connected, " \
"(SELECT CASE WHEN hl.label_value = 'true' THEN 1 ELSE 0 END FROM " \
- "host_label hl WHERE hl.host_id = h.host_id AND hl.label_key = '_is_ephemeral') " \
- "FROM host h WHERE hops > 0"
+ "host_label hl WHERE hl.host_id = h.host_id AND hl.label_key = '_is_ephemeral'), " \
+ "(SELECT CASE WHEN ni.node_id is NULL THEN 0 ELSE 1 END FROM " \
+ "node_instance ni WHERE ni.host_id = h.host_id) FROM host h WHERE hops > 0"
#define SQL_FETCH_ALL_INSTANCES \
"SELECT ni.host_id, ni.node_id FROM host h, node_instance ni " \
@@ -675,4 +681,4 @@ void unregister_node(const char *machine_guid)
cmd.completion = NULL;
aclk_database_enq_cmd(&cmd);
}
-#endif \ No newline at end of file
+#endif
diff --git a/database/sqlite/sqlite_aclk.h b/src/database/sqlite/sqlite_aclk.h
index 0db2647bf..0db2647bf 100644
--- a/database/sqlite/sqlite_aclk.h
+++ b/src/database/sqlite/sqlite_aclk.h
diff --git a/database/sqlite/sqlite_aclk_alert.c b/src/database/sqlite/sqlite_aclk_alert.c
index 9bd060f96..c96f0eef8 100644
--- a/database/sqlite/sqlite_aclk_alert.c
+++ b/src/database/sqlite/sqlite_aclk_alert.c
@@ -201,7 +201,7 @@ void sql_queue_alarm_to_aclk(RRDHOST *host, ALARM_ENTRY *ae, bool skip_filter)
ae->flags |= HEALTH_ENTRY_FLAG_ACLK_QUEUED;
rrdhost_flag_set(host, RRDHOST_FLAG_ACLK_STREAM_ALERTS);
} else
- error_report("Failed to store alert event %"PRId64", rc = %d", ae->unique_id, rc);
+ error_report("Failed to store alert event %"PRIu32", rc = %d", ae->unique_id, rc);
done:
if (unlikely(sqlite3_finalize(res_alert) != SQLITE_OK))
@@ -351,7 +351,8 @@ static void aclk_push_alert_event(struct aclk_sync_cfg_t *wc __maybe_unused)
strdupz("UNKNOWN=0=UNKNOWN");
alarm_log.command = strdupz(edit_command);
- alarm_log.duration = (time_t) sqlite3_column_int64(res, 6);
+ time_t duration = (time_t) sqlite3_column_int64(res, 6);
+ alarm_log.duration = (duration > 0) ? duration : 0;
alarm_log.non_clear_duration = (time_t) sqlite3_column_int64(res, 7);
alarm_log.status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 19));
alarm_log.old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS) sqlite3_column_int(res, 20));
@@ -609,7 +610,7 @@ void aclk_push_alert_config_event(char *node_id __maybe_unused, char *config_has
netdata_log_error("aclk_push_alert_config_event: Unexpected param number %d", param);
BUFFER *tmp_buf = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
- buffer_data_options2string(tmp_buf, sqlite3_column_int(res, 28));
+ rrdr_options_to_buffer(tmp_buf, sqlite3_column_int(res, 28));
alarm_config.p_db_lookup_options = strdupz((char *)buffer_tostring(tmp_buf));
buffer_free(tmp_buf);
@@ -821,7 +822,7 @@ void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_EN
#endif
#ifdef ENABLE_ACLK
-static bool have_recent_alarm(RRDHOST *host, int64_t alarm_id, int64_t mark)
+static bool have_recent_alarm_unsafe(RRDHOST *host, int64_t alarm_id, int64_t mark)
{
ALARM_ENTRY *ae = host->health_log.alarms;
@@ -882,7 +883,7 @@ void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)
if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))
continue;
- if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))
+ if (have_recent_alarm_unsafe(host, ae->alarm_id, ae->unique_id))
continue;
if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid))
@@ -911,7 +912,7 @@ void aclk_push_alert_snapshot_event(char *node_id __maybe_unused)
if (likely(ae->updated_by_id) || unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))
continue;
- if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))
+ if (have_recent_alarm_unsafe(host, ae->alarm_id, ae->unique_id))
continue;
if (is_event_from_alert_variable_config(ae->unique_id, &host->host_uuid))
@@ -1090,7 +1091,7 @@ void aclk_push_alarm_checkpoint(RRDHOST *host __maybe_unused)
}
active_alerts[cnt].name = (char *)rrdcalc_name(rc);
- len += string_strlen(rc->name);
+ len += string_strlen(rc->config.name);
active_alerts[cnt].chart = (char *)rrdcalc_chart_name(rc);
len += string_strlen(rc->chart);
active_alerts[cnt].status = rc->status;
diff --git a/database/sqlite/sqlite_aclk_alert.h b/src/database/sqlite/sqlite_aclk_alert.h
index cfb3468b9..cfb3468b9 100644
--- a/database/sqlite/sqlite_aclk_alert.h
+++ b/src/database/sqlite/sqlite_aclk_alert.h
diff --git a/database/sqlite/sqlite_aclk_node.c b/src/database/sqlite/sqlite_aclk_node.c
index dcc8c375c..dcc8c375c 100644
--- a/database/sqlite/sqlite_aclk_node.c
+++ b/src/database/sqlite/sqlite_aclk_node.c
diff --git a/database/sqlite/sqlite_aclk_node.h b/src/database/sqlite/sqlite_aclk_node.h
index 6afdf8d78..6afdf8d78 100644
--- a/database/sqlite/sqlite_aclk_node.h
+++ b/src/database/sqlite/sqlite_aclk_node.h
diff --git a/database/sqlite/sqlite_context.c b/src/database/sqlite/sqlite_context.c
index 26ed8a96a..ad76a1ee2 100644
--- a/database/sqlite/sqlite_context.c
+++ b/src/database/sqlite/sqlite_context.c
@@ -52,7 +52,7 @@ int sql_init_context_database(int memory)
if (likely(!memory))
target_version = perform_context_database_migration(db_context_meta, DB_CONTEXT_METADATA_VERSION);
- if (configure_sqlite_database(db_context_meta, target_version))
+ if (configure_sqlite_database(db_context_meta, target_version, "context_config"))
return 1;
if (likely(!memory))
@@ -60,34 +60,17 @@ int sql_init_context_database(int memory)
else
snprintfz(buf, sizeof(buf) - 1, "ATTACH DATABASE ':memory:' as meta");
- if(init_database_batch(db_context_meta, list)) return 1;
+ if(init_database_batch(db_context_meta, list, "context")) return 1;
- if (init_database_batch(db_context_meta, &database_context_config[0]))
+ if (init_database_batch(db_context_meta, &database_context_config[0], "context_init"))
return 1;
- if (init_database_batch(db_context_meta, &database_context_cleanup[0]))
+ if (init_database_batch(db_context_meta, &database_context_cleanup[0], "context_cleanup"))
return 1;
return 0;
}
-/*
- * Close the sqlite database
- */
-
-void sql_close_context_database(void)
-{
- int rc;
- if (unlikely(!db_context_meta))
- return;
-
- netdata_log_info("Closing context SQLite database");
-
- rc = sqlite3_close_v2(db_context_meta);
- if (unlikely(rc != SQLITE_OK))
- error_report("Error %d while closing the context SQLite database, %s", rc, sqlite3_errstr(rc));
-}
-
//
// Fetching data
//
@@ -421,6 +404,12 @@ int sql_context_cache_stats(int op)
return count;
}
+
+uint64_t sqlite_get_context_space(void)
+{
+ return sqlite_get_db_space(db_context_meta);
+}
+
//
// TESTING FUNCTIONS
//
@@ -455,7 +444,8 @@ int ctx_unittest(void)
uuid_t host_uuid;
uuid_generate(host_uuid);
- initialize_thread_key_pool();
+ if (sqlite_library_init())
+ return 1;
int rc = sql_init_context_database(1);
@@ -531,7 +521,8 @@ int ctx_unittest(void)
ctx_get_context_list(&host_uuid, dict_ctx_get_context_list_cb, NULL);
netdata_log_info("List context end after delete");
- sql_close_context_database();
+ sql_close_database(db_context_meta, "CONTEXT");
+ sqlite_library_shutdown();
return 0;
}
diff --git a/database/sqlite/sqlite_context.h b/src/database/sqlite/sqlite_context.h
index 2586916ea..92d02fdd2 100644
--- a/database/sqlite/sqlite_context.h
+++ b/src/database/sqlite/sqlite_context.h
@@ -65,6 +65,7 @@ int ctx_store_context(uuid_t *host_uuid, VERSIONED_CONTEXT_DATA *context_data);
int ctx_delete_context(uuid_t *host_id, VERSIONED_CONTEXT_DATA *context_data);
int sql_init_context_database(int memory);
+uint64_t sqlite_get_context_space(void);
void sql_close_context_database(void);
int ctx_unittest(void);
#endif //NETDATA_SQLITE_CONTEXT_H
diff --git a/database/sqlite/sqlite_db_migration.c b/src/database/sqlite/sqlite_db_migration.c
index 29da6c249..0131c4bf6 100644
--- a/database/sqlite/sqlite_db_migration.c
+++ b/src/database/sqlite/sqlite_db_migration.c
@@ -153,17 +153,36 @@ const char *database_migrate_v13_v14[] = {
NULL
};
+const char *database_migrate_v16_v17[] = {
+ "ALTER TABLE alert_hash ADD time_group_condition INT",
+ "ALTER TABLE alert_hash ADD time_group_value DOUBLE",
+ "ALTER TABLE alert_hash ADD dims_group INT",
+ "ALTER TABLE alert_hash ADD data_source INT",
+ NULL
+};
+
+// Note: Same as database_migrate_v16_v17. This is not wrong
+// Do additional migration to handle agents that created wrong alert_hash table
+const char *database_migrate_v17_v18[] = {
+ "ALTER TABLE alert_hash ADD time_group_condition INT",
+ "ALTER TABLE alert_hash ADD time_group_value DOUBLE",
+ "ALTER TABLE alert_hash ADD dims_group INT",
+ "ALTER TABLE alert_hash ADD data_source INT",
+ NULL
+};
+
+
static int do_migration_v1_v2(sqlite3 *database)
{
if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "hops"))
- return init_database_batch(database, &database_migrate_v1_v2[0]);
+ return init_database_batch(database, &database_migrate_v1_v2[0], "meta_migrate");
return 0;
}
static int do_migration_v2_v3(sqlite3 *database)
{
if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "memory_mode"))
- return init_database_batch(database, &database_migrate_v2_v3[0]);
+ return init_database_batch(database, &database_migrate_v2_v3[0], "meta_migrate");
return 0;
}
@@ -198,12 +217,12 @@ static int do_migration_v3_v4(sqlite3 *database)
static int do_migration_v4_v5(sqlite3 *database)
{
- return init_database_batch(database, &database_migrate_v4_v5[0]);
+ return init_database_batch(database, &database_migrate_v4_v5[0], "meta_migrate");
}
static int do_migration_v5_v6(sqlite3 *database)
{
- return init_database_batch(database, &database_migrate_v5_v6[0]);
+ return init_database_batch(database, &database_migrate_v5_v6[0], "meta_migrate");
}
static int do_migration_v6_v7(sqlite3 *database)
@@ -341,14 +360,14 @@ static int do_migration_v8_v9(sqlite3 *database)
static int do_migration_v9_v10(sqlite3 *database)
{
if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "chart_labels"))
- return init_database_batch(database, &database_migrate_v9_v10[0]);
+ return init_database_batch(database, &database_migrate_v9_v10[0], "meta_migrate");
return 0;
}
static int do_migration_v10_v11(sqlite3 *database)
{
if (table_exists_in_database(database, "health_log") && !column_exists_in_table(database, "health_log", "chart_name"))
- return init_database_batch(database, &database_migrate_v10_v11[0]);
+ return init_database_batch(database, &database_migrate_v10_v11[0], "meta_migrate");
return 0;
}
@@ -360,7 +379,7 @@ static int do_migration_v11_v12(sqlite3 *database)
if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary") &&
table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary"))
- rc = init_database_batch(database, &database_migrate_v11_v12[0]);
+ rc = init_database_batch(database, &database_migrate_v11_v12[0], "meta_migrate");
if (!rc)
sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL);
@@ -430,17 +449,34 @@ static int do_migration_v15_v16(sqlite3 *database)
return 0;
}
+static int do_migration_v16_v17(sqlite3 *database)
+{
+ if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "time_group_condition"))
+ return init_database_batch(database, &database_migrate_v16_v17[0], "meta_migrate");
+
+ return 0;
+}
+
+static int do_migration_v17_v18(sqlite3 *database)
+{
+ if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "time_group_condition"))
+ return init_database_batch(database, &database_migrate_v17_v18[0], "meta_migrate");
+
+ return 0;
+}
+
+
static int do_migration_v12_v13(sqlite3 *database)
{
int rc = 0;
if (table_exists_in_database(database, "health_log_detail") && !column_exists_in_table(database, "health_log_detail", "summary")) {
- rc = init_database_batch(database, &database_migrate_v12_v13_detail[0]);
+ rc = init_database_batch(database, &database_migrate_v12_v13_detail[0], "meta_migrate");
sqlite3_exec_monitored(database, MIGR_11_12_UPD_HEALTH_LOG_DETAIL, 0, 0, NULL);
}
if (table_exists_in_database(database, "alert_hash") && !column_exists_in_table(database, "alert_hash", "summary"))
- rc = init_database_batch(database, &database_migrate_v12_v13_hash[0]);
+ rc = init_database_batch(database, &database_migrate_v12_v13_hash[0], "meta_migrate");
return rc;
}
@@ -448,7 +484,7 @@ static int do_migration_v12_v13(sqlite3 *database)
static int do_migration_v13_v14(sqlite3 *database)
{
if (table_exists_in_database(database, "host") && !column_exists_in_table(database, "host", "last_connected"))
- return init_database_batch(database, &database_migrate_v13_v14[0]);
+ return init_database_batch(database, &database_migrate_v13_v14[0], "meta_migrate");
return 0;
}
@@ -466,7 +502,7 @@ const char *database_ml_migrate_v1_v2[] = {
static int do_ml_migration_v1_v2(sqlite3 *database)
{
if (get_auto_vaccum(database) != 2)
- return init_database_batch(database, &database_ml_migrate_v1_v2[0]);
+ return init_database_batch(database, &database_ml_migrate_v1_v2[0], "ml_migrate");
return 0;
}
@@ -527,6 +563,8 @@ DATABASE_FUNC_MIGRATION_LIST migration_action[] = {
{.name = "v13 to v14", .func = do_migration_v13_v14},
{.name = "v14 to v15", .func = do_migration_v14_v15},
{.name = "v15 to v16", .func = do_migration_v15_v16},
+ {.name = "v16 to v17", .func = do_migration_v16_v17},
+ {.name = "v17 to v18", .func = do_migration_v17_v18},
// the terminator of this array
{.name = NULL, .func = NULL}
};
diff --git a/database/sqlite/sqlite_db_migration.h b/src/database/sqlite/sqlite_db_migration.h
index e3c1be84f..e3c1be84f 100644
--- a/database/sqlite/sqlite_db_migration.h
+++ b/src/database/sqlite/sqlite_db_migration.h
diff --git a/src/database/sqlite/sqlite_functions.c b/src/database/sqlite/sqlite_functions.c
new file mode 100644
index 000000000..1dc2022b3
--- /dev/null
+++ b/src/database/sqlite/sqlite_functions.c
@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "sqlite_functions.h"
+
+#define MAX_PREPARED_STATEMENTS (32)
+pthread_key_t key_pool[MAX_PREPARED_STATEMENTS];
+
+SQLITE_API int sqlite3_exec_monitored(
+ sqlite3 *db, /* An open database */
+ const char *sql, /* SQL to be evaluated */
+ int (*callback)(void*,int,char**,char**), /* Callback function */
+ void *data, /* 1st argument to callback */
+ char **errmsg /* Error msg written here */
+) {
+ int rc = sqlite3_exec(db, sql, callback, data, errmsg);
+ global_statistics_sqlite3_query_completed(rc == SQLITE_OK, rc == SQLITE_BUSY, rc == SQLITE_LOCKED);
+ return rc;
+}
+
+SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt) {
+ int rc;
+ int cnt = 0;
+
+ while (cnt++ < SQL_MAX_RETRY) {
+ rc = sqlite3_step(stmt);
+ switch (rc) {
+ case SQLITE_DONE:
+ global_statistics_sqlite3_query_completed(1, 0, 0);
+ break;
+ case SQLITE_ROW:
+ global_statistics_sqlite3_row_completed();
+ break;
+ case SQLITE_BUSY:
+ case SQLITE_LOCKED:
+ global_statistics_sqlite3_query_completed(false, rc == SQLITE_BUSY, rc == SQLITE_LOCKED);
+ usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
+ continue;
+ default:
+ break;
+ }
+ break;
+ }
+ return rc;
+}
+
+static bool mark_database_to_recover(sqlite3_stmt *res, sqlite3 *database)
+{
+
+ if (!res && !database)
+ return false;
+
+ if (!database)
+ database = sqlite3_db_handle(res);
+
+ if (db_meta == database) {
+ char recover_file[FILENAME_MAX + 1];
+ snprintfz(recover_file, FILENAME_MAX, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
+ int fd = open(recover_file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 444);
+ if (fd >= 0) {
+ close(fd);
+ return true;
+ }
+ }
+ return false;
+}
+
+int execute_insert(sqlite3_stmt *res)
+{
+ int rc;
+ rc = sqlite3_step_monitored(res);
+ if (rc == SQLITE_CORRUPT) {
+ (void)mark_database_to_recover(res, NULL);
+ error_report("SQLite error %d", rc);
+ }
+ return rc;
+}
+
+int configure_sqlite_database(sqlite3 *database, int target_version, const char *description)
+{
+ char buf[1024 + 1] = "";
+ const char *list[2] = { buf, NULL };
+
+ // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
+ // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA auto_vacuum=%s", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_synchronous
+ // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA synchronous=%s", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_journal_mode
+ // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_mode=%s", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_temp_store
+ // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA temp_store=%s", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
+ // PRAGMA schema.journal_size_limit = N ;
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA journal_size_limit=%lld", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_cache_size
+ // PRAGMA schema.cache_size = pages;
+ // PRAGMA schema.cache_size = -kibibytes;
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA cache_size=%lld", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ snprintfz(buf, sizeof(buf) - 1, "PRAGMA user_version=%d", target_version);
+ if (init_database_batch(database, list, description))
+ return 1;
+
+ return 0;
+}
+
+#define MAX_OPEN_STATEMENTS (512)
+
+static void add_stmt_to_list(sqlite3_stmt *res)
+{
+ static int idx = 0;
+ static sqlite3_stmt *statements[MAX_OPEN_STATEMENTS];
+
+ if (unlikely(!res)) {
+ if (idx)
+ netdata_log_info("Finilizing %d statements", idx);
+ else
+ netdata_log_info("No statements pending to finalize");
+ while (idx > 0) {
+ int rc;
+ rc = sqlite3_finalize(statements[--idx]);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to finalize statement during shutdown, rc = %d", rc);
+ }
+ return;
+ }
+
+ if (unlikely(idx == MAX_OPEN_STATEMENTS))
+ return;
+}
+
+static void release_statement(void *statement)
+{
+ int rc;
+ if (unlikely(rc = sqlite3_finalize((sqlite3_stmt *) statement) != SQLITE_OK))
+ error_report("Failed to finalize statement, rc = %d", rc);
+}
+
+static void initialize_thread_key_pool(void)
+{
+ for (int i = 0; i < MAX_PREPARED_STATEMENTS; i++)
+ (void)pthread_key_create(&key_pool[i], release_statement);
+}
+
+int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement)
+{
+ static __thread uint32_t keys_used = 0;
+
+ pthread_key_t *key = NULL;
+ int ret = 1;
+
+ if (likely(keys_used < MAX_PREPARED_STATEMENTS))
+ key = &key_pool[keys_used++];
+
+ int rc = sqlite3_prepare_v2(database, query, -1, statement, 0);
+ if (rc == SQLITE_OK) {
+ if (key)
+ ret = pthread_setspecific(*key, *statement);
+ if (ret)
+ add_stmt_to_list(*statement);
+ }
+ return rc;
+}
+
+char *get_database_extented_error(sqlite3 *database, int i, const char *description)
+{
+ const char *err = sqlite3_errstr(sqlite3_extended_errcode(database));
+
+ if (!err)
+ return NULL;
+
+ size_t len = strlen(err)+ strlen(description) + 32;
+ char *full_err = mallocz(len);
+
+ snprintfz(full_err, len - 1, "%s: %d: %s", description, i, err);
+ return full_err;
+}
+
+int init_database_batch(sqlite3 *database, const char *batch[], const char *description)
+{
+ int rc;
+ char *err_msg = NULL;
+ for (int i = 0; batch[i]; i++) {
+ rc = sqlite3_exec_monitored(database, batch[i], 0, 0, &err_msg);
+ if (rc != SQLITE_OK) {
+ error_report("SQLite error during database initialization, rc = %d (%s)", rc, err_msg);
+ error_report("SQLite failed statement %s", batch[i]);
+ char *error_str = get_database_extented_error(database, i, description);
+ if (error_str)
+ analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str);
+ sqlite3_free(err_msg);
+ freez(error_str);
+ if (SQLITE_CORRUPT == rc) {
+ if (mark_database_to_recover(NULL, database))
+ error_report("Database is corrupted will attempt to fix");
+ return SQLITE_CORRUPT;
+ }
+ return 1;
+ }
+ }
+ return 0;
+}
+
+// Return 0 OK
+// Return 1 Failed
+int db_execute(sqlite3 *db, const char *cmd)
+{
+ int rc;
+ int cnt = 0;
+
+ while (cnt < SQL_MAX_RETRY) {
+ char *err_msg;
+ rc = sqlite3_exec_monitored(db, cmd, 0, 0, &err_msg);
+ if (likely(rc == SQLITE_OK))
+ break;
+
+ ++cnt;
+ error_report("Failed to execute '%s', rc = %d (%s) -- attempt %d", cmd, rc, err_msg, cnt);
+ sqlite3_free(err_msg);
+
+ if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) {
+ usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
+ continue;
+ }
+
+ if (rc == SQLITE_CORRUPT)
+ mark_database_to_recover(NULL, db);
+ break;
+ }
+ return (rc != SQLITE_OK);
+}
+
+// Utils
+int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null)
+{
+ if (likely(text))
+ return sqlite3_bind_text(res, position, text, -1, SQLITE_STATIC);
+ if (!can_be_null)
+ return 1;
+ return sqlite3_bind_null(res, position);
+}
+
+#define SQL_DROP_TABLE "DROP table %s"
+
+void sql_drop_table(const char *table)
+{
+ if (!table)
+ return;
+
+ char wstr[255];
+ snprintfz(wstr, sizeof(wstr) - 1, SQL_DROP_TABLE, table);
+
+ int rc = sqlite3_exec_monitored(db_meta, wstr, 0, 0, NULL);
+ if (rc != SQLITE_OK) {
+ error_report("DES SQLite error during drop table operation for %s, rc = %d", table, rc);
+ }
+}
+
+static int get_pragma_value(sqlite3 *database, const char *sql)
+{
+ sqlite3_stmt *res = NULL;
+ int rc = sqlite3_prepare_v2(database, sql, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK))
+ return -1;
+
+ int result = -1;
+ rc = sqlite3_step_monitored(res);
+ if (likely(rc == SQLITE_ROW))
+ result = sqlite3_column_int(res, 0);
+
+ rc = sqlite3_finalize(res);
+ (void) rc;
+
+ return result;
+}
+
+int get_free_page_count(sqlite3 *database)
+{
+ return get_pragma_value(database, "PRAGMA freelist_count");
+}
+
+int get_database_page_count(sqlite3 *database)
+{
+ return get_pragma_value(database, "PRAGMA page_count");
+}
+
+uint64_t sqlite_get_db_space(sqlite3 *db)
+{
+ if (!db)
+ return 0;
+
+ uint64_t page_size = (uint64_t) get_pragma_value(db, "PRAGMA page_size");
+ uint64_t page_count = (uint64_t) get_pragma_value(db, "PRAGMA page_count");
+
+ return page_size * page_count;
+}
+
+/*
+ * Close the sqlite database
+ */
+
+void sql_close_database(sqlite3 *database, const char *database_name)
+{
+ int rc;
+ if (unlikely(!database))
+ return;
+
+ (void) db_execute(database, "PRAGMA analysis_limit=10000");
+ (void) db_execute(database, "PRAGMA optimize");
+
+ netdata_log_info("%s: Closing sqlite database", database_name);
+
+#ifdef NETDATA_DEV_MODE
+ int t_count_used,t_count_hit,t_count_miss,t_count_full, dummy;
+ (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_USED, &dummy, &t_count_used, 0);
+ (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_HIT, &dummy,&t_count_hit, 0);
+ (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE, &dummy,&t_count_miss, 0);
+ (void) sqlite3_db_status(database, SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL, &dummy,&t_count_full, 0);
+
+ netdata_log_info("%s: Database lookaside allocation statistics: Used slots %d, Hit %d, Misses due to small slot size %d, Misses due to slots full %d", database_name,
+ t_count_used,t_count_hit, t_count_miss, t_count_full);
+
+ (void) sqlite3_db_release_memory(database);
+#endif
+
+ rc = sqlite3_close_v2(database);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("%s: Error while closing the sqlite database: rc %d, error \"%s\"", database_name, rc, sqlite3_errstr(rc));
+}
+
+extern sqlite3 *db_context_meta;
+
+void sqlite_close_databases(void)
+{
+ add_stmt_to_list(NULL);
+
+ sql_close_database(db_context_meta, "CONTEXT");
+ sql_close_database(db_meta, "METADATA");
+}
+
+int sqlite_library_init(void)
+{
+ initialize_thread_key_pool();
+
+ int rc = sqlite3_initialize();
+
+ return (SQLITE_OK != rc);
+}
+
+void sqlite_library_shutdown(void)
+{
+ (void) sqlite3_shutdown();
+}
diff --git a/src/database/sqlite/sqlite_functions.h b/src/database/sqlite/sqlite_functions.h
new file mode 100644
index 000000000..2841b27f6
--- /dev/null
+++ b/src/database/sqlite/sqlite_functions.h
@@ -0,0 +1,47 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SQLITE_FUNCTIONS_H
+#define NETDATA_SQLITE_FUNCTIONS_H
+
+#include "daemon/common.h"
+#include "sqlite3.h"
+
+void analytics_set_data_str(char **name, const char *value);
+
+#define SQL_MAX_RETRY (100)
+#define SQLITE_INSERT_DELAY (10) // Insert delay in case of lock
+
+SQLITE_API int sqlite3_step_monitored(sqlite3_stmt *stmt);
+SQLITE_API int sqlite3_exec_monitored(
+ sqlite3 *db, /* An open database */
+ const char *sql, /* SQL to be evaluated */
+ int (*callback)(void*,int,char**,char**), /* Callback function */
+ void *data, /* 1st argument to callback */
+ char **errmsg /* Error msg written here */
+ );
+
+// Initialization and shutdown
+int init_database_batch(sqlite3 *database, const char *batch[], const char *description);
+int configure_sqlite_database(sqlite3 *database, int target_version, const char *description);
+
+// Helpers
+int bind_text_null(sqlite3_stmt *res, int position, const char *text, bool can_be_null);
+int prepare_statement(sqlite3 *database, const char *query, sqlite3_stmt **statement);
+int execute_insert(sqlite3_stmt *res);
+int db_execute(sqlite3 *database, const char *cmd);
+char *get_database_extented_error(sqlite3 *database, int i, const char *description);
+
+void sql_drop_table(const char *table);
+void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv);
+
+uint64_t sqlite_get_db_space(sqlite3 *db);
+
+int get_free_page_count(sqlite3 *database);
+int get_database_page_count(sqlite3 *database);
+
+int sqlite_library_init(void);
+void sqlite_library_shutdown(void);
+
+void sql_close_database(sqlite3 *database, const char *database_name);
+void sqlite_close_databases(void);
+#endif //NETDATA_SQLITE_FUNCTIONS_H
diff --git a/database/sqlite/sqlite_health.c b/src/database/sqlite/sqlite_health.c
index 7d79ff70b..ea883c51b 100644
--- a/database/sqlite/sqlite_health.c
+++ b/src/database/sqlite/sqlite_health.c
@@ -3,6 +3,7 @@
#include "sqlite_health.h"
#include "sqlite_functions.h"
#include "sqlite_db_migration.h"
+#include "health/health_internals.h"
#define MAX_HEALTH_SQL_SIZE 2048
#define SQLITE3_BIND_STRING_OR_NULL(res, key, param) \
@@ -101,7 +102,8 @@ failed:
"config_hash_id, name, chart, exec, recipient, units, chart_context, last_transition_id, chart_name) " \
"VALUES (@host_id,@alarm_id, @config_hash_id,@name,@chart,@exec,@recipient,@units,@chart_context," \
"@last_transition_id,@chart_name) ON CONFLICT (host_id, alarm_id) DO UPDATE " \
- "SET last_transition_id = excluded.last_transition_id, chart_name = excluded.chart_name RETURNING health_log_id"
+ "SET last_transition_id = excluded.last_transition_id, chart_name = excluded.chart_name, " \
+ "config_hash_id=excluded.config_hash_id RETURNING health_log_id"
#define SQL_INSERT_HEALTH_LOG_DETAIL \
"INSERT INTO health_log_detail (health_log_id, unique_id, alarm_id, alarm_event_id, " \
@@ -893,18 +895,21 @@ void sql_health_alarm_log_load(RRDHOST *host)
/*
* Store an alert config hash in the database
*/
-#define SQL_STORE_ALERT_CONFIG_HASH \
- "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \
- "on_key, class, component, type, os, hosts, lookup, every, units, calc, plugin, module, " \
- "charts, green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \
- "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \
- "p_db_lookup_before, p_update_every, source, chart_labels, summary) values (@hash_id,UNIXEPOCH(),@alarm,@template," \
- "@on_key,@class,@component,@type,@os,@hosts,@lookup,@every,@units,@calc,@plugin,@module," \
- "@charts,@green,@red,@warn,@crit,@exec,@to_key,@info,@delay,@options,@repeat,@host_labels," \
- "@p_db_lookup_dimensions,@p_db_lookup_method,@p_db_lookup_options,@p_db_lookup_after," \
- "@p_db_lookup_before,@p_update_every,@source,@chart_labels,@summary)"
-
-int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
+#define SQL_STORE_ALERT_CONFIG_HASH \
+ "insert or replace into alert_hash (hash_id, date_updated, alarm, template, " \
+ "on_key, class, component, type, lookup, every, units, calc, " \
+ "green, red, warn, crit, exec, to_key, info, delay, options, repeat, host_labels, " \
+ "p_db_lookup_dimensions, p_db_lookup_method, p_db_lookup_options, p_db_lookup_after, " \
+ "p_db_lookup_before, p_update_every, source, chart_labels, summary, time_group_condition, " \
+ "time_group_value, dims_group, data_source) " \
+ "values (@hash_id,UNIXEPOCH(),@alarm,@template," \
+ "@on_key,@class,@component,@type,@lookup,@every,@units,@calc," \
+ "@green,@red,@warn,@crit,@exec,@to_key,@info,@delay,@options,@repeat,@host_labels," \
+ "@p_db_lookup_dimensions,@p_db_lookup_method,@p_db_lookup_options,@p_db_lookup_after," \
+ "@p_db_lookup_before,@p_update_every,@source,@chart_labels,@summary, @time_group_condition, " \
+ "@time_group_value, @dims_group, @data_source)"
+
+int sql_alert_store_config(RRD_ALERT_PROTOTYPE *ap __maybe_unused)
{
static __thread sqlite3_stmt *res = NULL;
int rc, param = 0;
@@ -923,133 +928,153 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
return 1;
}
}
+ BUFFER *buf = buffer_create(128, NULL);
- rc = sqlite3_bind_blob(res, ++param, hash_id, sizeof(*hash_id), SQLITE_STATIC);
+ rc = sqlite3_bind_blob(res, ++param, &ap->config.hash_id, sizeof(ap->config.hash_id), SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->alarm, ++param);
- if (unlikely(rc != SQLITE_OK))
- goto bind_fail;
+ if (ap->match.is_template)
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param);
+ else
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.name, ++param);
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->template_key, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->on, ++param);
- if (unlikely(rc != SQLITE_OK))
- goto bind_fail;
+ if (ap->match.is_template)
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.name, ++param);
+ else
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param);
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->classification, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->component, ++param);
+ if (ap->match.is_template)
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.on.context, ++param);
+ else
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.on.chart, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->type, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.classification, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->os, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.component, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->host, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.type, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->lookup, ++param);
+ // Rebuild lookup
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, NULL, ++param); // lookup line
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->every, ++param);
+ rc = sqlite3_bind_int(res, ++param, ap->config.update_every);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->units, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.units, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->calc, ++param);
+ if (ap->config.calculation)
+ rc = sqlite3_bind_text(res, ++param, expression_source(ap->config.calculation), -1, SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->plugin, ++param);
+ NETDATA_DOUBLE green = NAN;
+ rc = sqlite3_bind_double(res, ++param, green);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->module, ++param);
+ NETDATA_DOUBLE red = NAN;
+ rc = sqlite3_bind_double(res, ++param, red);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->charts, ++param);
+ if (ap->config.warning)
+ rc = sqlite3_bind_text(res, ++param, expression_source(ap->config.warning), -1, SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->green, ++param);
+ if (ap->config.critical)
+ rc = sqlite3_bind_text(res, ++param, expression_source(ap->config.critical), -1, SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->red, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.exec, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->warn, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.recipient, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->crit, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.info, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->exec, ++param);
- if (unlikely(rc != SQLITE_OK))
- goto bind_fail;
+ if (ap->config.delay_up_duration)
+ buffer_sprintf(buf, "up %ds ", ap->config.delay_up_duration);
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->to, ++param);
- if (unlikely(rc != SQLITE_OK))
- goto bind_fail;
+ if (ap->config.delay_down_duration)
+ buffer_sprintf(buf, "down %ds ", ap->config.delay_down_duration);
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->info, ++param);
- if (unlikely(rc != SQLITE_OK))
- goto bind_fail;
+ if (ap->config.delay_multiplier)
+ buffer_sprintf(buf, "multiplier %.1f ", ap->config.delay_multiplier);
+
+ if (ap->config.delay_max_duration)
+ buffer_sprintf(buf, "max %ds", ap->config.delay_max_duration);
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->delay, ++param);
+ // delay
+ rc = sqlite3_bind_text(res, ++param, buffer_tostring(buf), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->options, ++param);
+ if (ap->config.alert_action_options & ALERT_ACTION_OPTION_NO_CLEAR_NOTIFICATION)
+ rc = sqlite3_bind_text(res, ++param, "no-clear-notification", -1, SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->repeat, ++param);
+ rc = sqlite3_bind_int(res, ++param, ap->config.update_every);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->host_labels, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.host_labels, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- if (cfg->p_db_lookup_after) {
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->p_db_lookup_dimensions, ++param);
+ if (ap->config.after) {
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.dimensions, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->p_db_lookup_method, ++param);
+ rc = sqlite3_bind_text(res, ++param, time_grouping_id2txt(ap->config.time_group), -1, SQLITE_STATIC);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_options);
+ rc = sqlite3_bind_int(res, ++param, (int) RRDR_OPTIONS_REMOVE_OVERLAPPING(ap->config.options));
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_after);
+ rc = sqlite3_bind_int64(res, ++param, (int) ap->config.after);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = sqlite3_bind_int(res, ++param, (int) cfg->p_db_lookup_before);
+ rc = sqlite3_bind_int64(res, ++param, (int) ap->config.before);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
} else {
@@ -1074,19 +1099,35 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
goto bind_fail;
}
- rc = sqlite3_bind_int(res, ++param, cfg->p_update_every);
+ rc = sqlite3_bind_int(res, ++param, ap->config.update_every);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->source, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.source, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->chart_labels, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->match.chart_labels, ++param);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
- rc = SQLITE3_BIND_STRING_OR_NULL(res, cfg->summary, ++param);
+ rc = SQLITE3_BIND_STRING_OR_NULL(res, ap->config.summary, ++param);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, ap->config.time_group_condition);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_double(res, ++param, ap->config.time_group_value);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, ap->config.dims_group);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, ap->config.data_source);
if (unlikely(rc != SQLITE_OK))
goto bind_fail;
@@ -1098,9 +1139,11 @@ int sql_store_alert_config_hash(uuid_t *hash_id, struct alert_config *cfg)
if (unlikely(rc != SQLITE_OK))
error_report("Failed to reset statement in alert hash_id store function, rc = %d", rc);
+ buffer_free(buf);
return 0;
bind_fail:
+ buffer_free(buf);
error_report("Failed to bind parameter %d to store alert hash_id, rc = %d", param, rc);
rc = sqlite3_reset(res);
if (unlikely(rc != SQLITE_OK))
@@ -1108,75 +1151,6 @@ bind_fail:
return 1;
}
-/*
- alert hashes are used for cloud communication.
- if cloud is disabled or openssl is not available (which will prevent cloud connectivity)
- skip hash calculations
-*/
-#if defined ENABLE_HTTPS
-#define DIGEST_ALERT_CONFIG_VAL(v) ((v) ? EVP_DigestUpdate(evpctx, (string2str(v)), string_strlen((v))) : EVP_DigestUpdate(evpctx, "", 1))
-#endif
-int alert_hash_and_store_config(
- uuid_t hash_id,
- struct alert_config *cfg,
- int store_hash)
-{
-#if defined ENABLE_HTTPS
- EVP_MD_CTX *evpctx;
- unsigned char hash_value[EVP_MAX_MD_SIZE];
- unsigned int hash_len;
- evpctx = EVP_MD_CTX_create();
- EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
-
- DIGEST_ALERT_CONFIG_VAL(cfg->alarm);
- DIGEST_ALERT_CONFIG_VAL(cfg->template_key);
- DIGEST_ALERT_CONFIG_VAL(cfg->os);
- DIGEST_ALERT_CONFIG_VAL(cfg->host);
- DIGEST_ALERT_CONFIG_VAL(cfg->on);
- DIGEST_ALERT_CONFIG_VAL(cfg->plugin);
- DIGEST_ALERT_CONFIG_VAL(cfg->module);
- DIGEST_ALERT_CONFIG_VAL(cfg->charts);
- DIGEST_ALERT_CONFIG_VAL(cfg->lookup);
- DIGEST_ALERT_CONFIG_VAL(cfg->calc);
- DIGEST_ALERT_CONFIG_VAL(cfg->every);
- DIGEST_ALERT_CONFIG_VAL(cfg->green);
- DIGEST_ALERT_CONFIG_VAL(cfg->red);
- DIGEST_ALERT_CONFIG_VAL(cfg->warn);
- DIGEST_ALERT_CONFIG_VAL(cfg->crit);
- DIGEST_ALERT_CONFIG_VAL(cfg->exec);
- DIGEST_ALERT_CONFIG_VAL(cfg->to);
- DIGEST_ALERT_CONFIG_VAL(cfg->units);
- DIGEST_ALERT_CONFIG_VAL(cfg->info);
- DIGEST_ALERT_CONFIG_VAL(cfg->classification);
- DIGEST_ALERT_CONFIG_VAL(cfg->component);
- DIGEST_ALERT_CONFIG_VAL(cfg->type);
- DIGEST_ALERT_CONFIG_VAL(cfg->delay);
- DIGEST_ALERT_CONFIG_VAL(cfg->options);
- DIGEST_ALERT_CONFIG_VAL(cfg->repeat);
- DIGEST_ALERT_CONFIG_VAL(cfg->host_labels);
- DIGEST_ALERT_CONFIG_VAL(cfg->chart_labels);
- DIGEST_ALERT_CONFIG_VAL(cfg->summary);
-
- EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
- EVP_MD_CTX_destroy(evpctx);
- fatal_assert(hash_len > sizeof(uuid_t));
-
- char uuid_str[UUID_STR_LEN];
- uuid_unparse_lower(*((uuid_t *)&hash_value), uuid_str);
- uuid_copy(hash_id, *((uuid_t *)&hash_value));
-
- /* store everything, so it can be recreated when not in memory or just a subset ? */
- if (store_hash)
- (void)sql_store_alert_config_hash( (uuid_t *)&hash_value, cfg);
-#else
- UNUSED(hash_id);
- UNUSED(cfg);
- UNUSED(store_hash);
-#endif
-
- return 1;
-}
-
#define SQL_SELECT_HEALTH_LAST_EXECUTED_EVENT \
"SELECT hld.new_status FROM health_log hl, health_log_detail hld " \
"WHERE hl.host_id = @host_id AND hl.alarm_id = @alarm_id AND hld.unique_id != @unique_id AND hld.flags & @flags " \
@@ -1580,10 +1554,9 @@ static uint32_t get_next_alarm_event_id(uint64_t health_log_id, uint32_t alarm_i
}
#define SQL_GET_ALARM_ID \
- "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart " \
- "AND name = @name AND config_hash_id = @config_hash_id"
+ "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart AND name = @name"
-uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id)
+uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id)
{
int rc = 0;
sqlite3_stmt *res = NULL;
@@ -1617,13 +1590,6 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *
return alarm_id;
}
- rc = sqlite3_bind_blob(res, 4, config_hash_id, sizeof(*config_hash_id), SQLITE_STATIC);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind config_hash_id parameter for SQL_GET_ALARM_ID.");
- sqlite3_finalize(res);
- return alarm_id;
- }
-
while (sqlite3_step_monitored(res) == SQLITE_ROW) {
alarm_id = (uint32_t) sqlite3_column_int64(res, 0);
health_log_id = (uint64_t) sqlite3_column_int64(res, 1);
@@ -1639,111 +1605,6 @@ uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *
return alarm_id;
}
-#define SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH \
- "UPDATE health_log SET config_hash_id = @config_hash_id WHERE host_id = @host_id AND alarm_id = @alarm_id " \
- "AND health_log_id = @health_log_id"
-
-void sql_update_alarm_with_config_hash(RRDHOST *host, uint32_t alarm_id, uint64_t health_log_id, uuid_t *config_hash_id)
-{
- int rc = 0;
- sqlite3_stmt *res = NULL;
-
- rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, -1, &res, 0);
- if (rc != SQLITE_OK) {
- error_report("Failed to prepare statement when trying to update an alarm id with a config hash.");
- return;
- }
-
- rc = sqlite3_bind_blob(res, 1, config_hash_id, sizeof(*config_hash_id), SQLITE_STATIC);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind config_hash_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH.");
- goto done;
- }
-
- rc = sqlite3_bind_blob(res, 2, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind host_id parameter for SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH.");
- goto done;
- }
-
- rc = sqlite3_bind_int64(res, 3, (sqlite3_int64) alarm_id);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID.");
- goto done;
- }
-
- rc = sqlite3_bind_int64(res, 4, (sqlite3_int64) health_log_id);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind alarm_id parameter for SQL_GET_ALARM_ID.");
- goto done;
- }
-
- rc = execute_insert(res);
- if (unlikely(rc != SQLITE_DONE))
- error_report("Failed to execute SQL_UPDATE_ALARM_ID_WITH_CONFIG_HASH, rc = %d", rc);
-
-done:
- rc = sqlite3_finalize(res);
- if (unlikely(rc != SQLITE_OK))
- error_report("Failed to reset statement to update health log detail table with config hash ids, rc = %d", rc);
-
-}
-
-#define SQL_GET_ALARM_ID_CHECK_ZERO_HASH \
- "SELECT alarm_id, health_log_id FROM health_log WHERE host_id = @host_id AND chart = @chart " \
- "AND name = @name AND (config_hash_id IS NULL OR config_hash_id = ZEROBLOB(16))"
-
-uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id)
-{
- int rc = 0;
- sqlite3_stmt *res = NULL;
- uint32_t alarm_id = 0;
- uint64_t health_log_id = 0;
-
- rc = sqlite3_prepare_v2(db_meta, SQL_GET_ALARM_ID_CHECK_ZERO_HASH, -1, &res, 0);
- if (rc != SQLITE_OK) {
- error_report("Failed to prepare statement when trying to get an alarm id with zero hash");
- return alarm_id;
- }
-
- rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind host_id parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH.");
- sqlite3_finalize(res);
- return alarm_id;
- }
-
- rc = SQLITE3_BIND_STRING_OR_NULL(res, chart, 2);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind char parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH.");
- sqlite3_finalize(res);
- return alarm_id;
- }
-
- rc = SQLITE3_BIND_STRING_OR_NULL(res, name, 3);
- if (unlikely(rc != SQLITE_OK)) {
- error_report("Failed to bind name parameter for SQL_GET_ALARM_ID_CHECK_ZERO_HASH.");
- sqlite3_finalize(res);
- return alarm_id;
- }
-
- while (sqlite3_step_monitored(res) == SQLITE_ROW) {
- alarm_id = (uint32_t) sqlite3_column_int64(res, 0);
- health_log_id = (uint64_t) sqlite3_column_int64(res, 1);
- }
-
- rc = sqlite3_finalize(res);
- if (unlikely(rc != SQLITE_OK))
- error_report("Failed to finalize the statement while getting an alarm id.");
-
- if (alarm_id) {
- sql_update_alarm_with_config_hash(host, alarm_id, health_log_id, config_hash_id);
- *next_event_id = get_next_alarm_event_id(health_log_id, alarm_id);
- }
-
- return alarm_id;
-}
-
#define SQL_GET_ALARM_ID_FROM_TRANSITION_ID \
"SELECT hld.alarm_id, hl.host_id, hl.chart_context FROM health_log_detail hld, health_log hl " \
"WHERE hld.transition_id = @transition_id " \
@@ -1996,10 +1857,11 @@ done_only_drop:
#define SQL_POPULATE_TEMP_CONFIG_TARGET_TABLE "INSERT INTO c_%p (hash_id) VALUES (@hash_id)"
#define SQL_SEARCH_CONFIG_LIST \
- "SELECT ah.hash_id, alarm, template, on_key, class, component, type, os, hosts, lookup, every, " \
- " units, calc, families, plugin, module, charts, green, red, warn, crit, " \
+ "SELECT ah.hash_id, alarm, template, on_key, class, component, type, lookup, every, " \
+ " units, calc, families, green, red, warn, crit, " \
" exec, to_key, info, delay, options, repeat, host_labels, p_db_lookup_dimensions, p_db_lookup_method, " \
- " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary " \
+ " p_db_lookup_options, p_db_lookup_after, p_db_lookup_before, p_update_every, source, chart_labels, summary, " \
+ " time_group_condition, time_group_value, dims_group, data_source " \
" FROM alert_hash ah, c_%p t where ah.hash_id = t.hash_id"
int sql_get_alert_configuration(
@@ -2079,16 +1941,11 @@ int sql_get_alert_configuration(
acd.classification = (const char *) sqlite3_column_text(res, param++);
acd.component = (const char *) sqlite3_column_text(res, param++);
acd.type = (const char *) sqlite3_column_text(res, param++);
- acd.selectors.os = (const char *) sqlite3_column_text(res, param++);
- acd.selectors.hosts = (const char *) sqlite3_column_text(res, param++);
acd.value.db.lookup = (const char *) sqlite3_column_text(res, param++);
acd.value.every = (const char *) sqlite3_column_text(res, param++);
acd.value.units = (const char *) sqlite3_column_text(res, param++);
acd.value.calc = (const char *) sqlite3_column_text(res, param++);
acd.selectors.families = (const char *) sqlite3_column_text(res, param++);
- acd.selectors.plugin = (const char *) sqlite3_column_text(res, param++);
- acd.selectors.module = (const char *) sqlite3_column_text(res, param++);
- acd.selectors.charts = (const char *) sqlite3_column_text(res, param++);
acd.status.green = (const char *) sqlite3_column_text(res, param++);
acd.status.red = (const char *) sqlite3_column_text(res, param++);
acd.status.warn = (const char *) sqlite3_column_text(res, param++);
@@ -2109,6 +1966,10 @@ int sql_get_alert_configuration(
acd.source = (const char *) sqlite3_column_text(res, param++);
acd.selectors.chart_labels = (const char *) sqlite3_column_text(res, param++);
acd.summary = (const char *) sqlite3_column_text(res, param++);
+ acd.value.db.time_group_condition =(int32_t) sqlite3_column_int(res, param++);
+ acd.value.db.time_group_value = sqlite3_column_double(res, param++);
+ acd.value.db.dims_group = (int32_t) sqlite3_column_int(res, param++);
+ acd.value.db.data_source = (int32_t) sqlite3_column_int(res, param++);
cb(&acd, data);
added++;
diff --git a/database/sqlite/sqlite_health.h b/src/database/sqlite/sqlite_health.h
index 5549b7525..1b889436e 100644
--- a/database/sqlite/sqlite_health.h
+++ b/src/database/sqlite/sqlite_health.h
@@ -2,21 +2,22 @@
#ifndef NETDATA_SQLITE_HEALTH_H
#define NETDATA_SQLITE_HEALTH_H
-#include "../../daemon/common.h"
+
+#include "daemon/common.h"
#include "sqlite3.h"
struct sql_alert_transition_data;
struct sql_alert_config_data;
+struct rrd_alert_prototype;
void sql_health_alarm_log_load(RRDHOST *host);
void sql_health_alarm_log_save(RRDHOST *host, ALARM_ENTRY *ae);
void sql_health_alarm_log_cleanup(RRDHOST *host, bool claimed);
-int alert_hash_and_store_config(uuid_t hash_id, struct alert_config *cfg, int store_hash);
+int sql_alert_store_config(struct rrd_alert_prototype *ap);
void sql_aclk_alert_clean_dead_entries(RRDHOST *host);
int sql_health_get_last_executed_event(RRDHOST *host, ALARM_ENTRY *ae, RRDCALC_STATUS *last_executed_status);
void sql_health_alarm_log2json(RRDHOST *host, BUFFER *wb, time_t after, const char *chart);
int health_migrate_old_health_log_table(char *table);
-uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id);
-uint32_t sql_get_alarm_id_check_zero_hash(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id, uuid_t *config_hash_id);
+uint32_t sql_get_alarm_id(RRDHOST *host, STRING *chart, STRING *name, uint32_t *next_event_id);
void sql_alert_transitions(
DICTIONARY *nodes,
time_t after,
diff --git a/src/database/sqlite/sqlite_metadata.c b/src/database/sqlite/sqlite_metadata.c
new file mode 100644
index 000000000..86ecee1f6
--- /dev/null
+++ b/src/database/sqlite/sqlite_metadata.c
@@ -0,0 +1,2645 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "sqlite_metadata.h"
+#include "sqlite3recover.h"
+//#include "sqlite_db_migration.h"
+
+#define DB_METADATA_VERSION 18
+
+const char *database_config[] = {
+ "CREATE TABLE IF NOT EXISTS host(host_id BLOB PRIMARY KEY, hostname TEXT NOT NULL, "
+ "registry_hostname TEXT NOT NULL default 'unknown', update_every INT NOT NULL default 1, "
+ "os TEXT NOT NULL default 'unknown', timezone TEXT NOT NULL default 'unknown', tags TEXT NOT NULL default '',"
+ "hops INT NOT NULL DEFAULT 0,"
+ "memory_mode INT DEFAULT 0, abbrev_timezone TEXT DEFAULT '', utc_offset INT NOT NULL DEFAULT 0,"
+ "program_name TEXT NOT NULL DEFAULT 'unknown', program_version TEXT NOT NULL DEFAULT 'unknown', "
+ "entries INT NOT NULL DEFAULT 0,"
+ "health_enabled INT NOT NULL DEFAULT 0, last_connected INT NOT NULL DEFAULT 0)",
+
+ "CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "
+ "family text, context text, title text, unit text, plugin text, module text, priority int, update_every int, "
+ "chart_type int, memory_mode int, history_entries)",
+
+ "CREATE TABLE IF NOT EXISTS dimension(dim_id blob PRIMARY KEY, chart_id blob, id text, name text, "
+ "multiplier int, divisor int , algorithm int, options text)",
+
+ "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int)",
+
+ "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, "
+ "label_value text, date_created int, PRIMARY KEY (chart_id, label_key))",
+
+ "CREATE TABLE IF NOT EXISTS node_instance (host_id blob PRIMARY KEY, claim_id, node_id, date_created)",
+
+ "CREATE TABLE IF NOT EXISTS alert_hash(hash_id blob PRIMARY KEY, date_updated int, alarm text, template text, "
+ "on_key text, class text, component text, type text, os text, hosts text, lookup text, "
+ "every text, units text, calc text, families text, plugin text, module text, charts text, green text, "
+ "red text, warn text, crit text, exec text, to_key text, info text, delay text, options text, "
+ "repeat text, host_labels text, p_db_lookup_dimensions text, p_db_lookup_method text, p_db_lookup_options int, "
+ "p_db_lookup_after int, p_db_lookup_before int, p_update_every int, source text, chart_labels text, "
+ "summary text, time_group_condition INT, time_group_value DOUBLE, dims_group INT, data_source INT)",
+
+ "CREATE TABLE IF NOT EXISTS host_info(host_id blob, system_key text NOT NULL, system_value text NOT NULL, "
+ "date_created INT, PRIMARY KEY(host_id, system_key))",
+
+ "CREATE TABLE IF NOT EXISTS host_label(host_id blob, source_type int, label_key text NOT NULL, "
+ "label_value text NOT NULL, date_created INT, PRIMARY KEY (host_id, label_key))",
+
+ "CREATE TRIGGER IF NOT EXISTS ins_host AFTER INSERT ON host BEGIN INSERT INTO node_instance (host_id, date_created)"
+ " SELECT new.host_id, unixepoch() WHERE new.host_id NOT IN (SELECT host_id FROM node_instance); END",
+
+ "CREATE TABLE IF NOT EXISTS health_log (health_log_id INTEGER PRIMARY KEY, host_id blob, alarm_id int, "
+ "config_hash_id blob, name text, chart text, family text, recipient text, units text, exec text, "
+ "chart_context text, last_transition_id blob, chart_name text, UNIQUE (host_id, alarm_id))",
+
+ "CREATE TABLE IF NOT EXISTS health_log_detail (health_log_id int, unique_id int, alarm_id int, alarm_event_id int, "
+ "updated_by_id int, updates_id int, when_key int, duration int, non_clear_duration int, "
+ "flags int, exec_run_timestamp int, delay_up_to_timestamp int, "
+ "info text, exec_code int, new_status real, old_status real, delay int, "
+ "new_value double, old_value double, last_repeat int, transition_id blob, global_id int, summary text)",
+
+ "CREATE INDEX IF NOT EXISTS ind_d2 on dimension (chart_id)",
+ "CREATE INDEX IF NOT EXISTS ind_c3 on chart (host_id)",
+ "CREATE INDEX IF NOT EXISTS health_log_ind_1 ON health_log (host_id)",
+ "CREATE INDEX IF NOT EXISTS health_log_d_ind_2 ON health_log_detail (global_id)",
+ "CREATE INDEX IF NOT EXISTS health_log_d_ind_3 ON health_log_detail (transition_id)",
+ "CREATE INDEX IF NOT EXISTS health_log_d_ind_9 ON health_log_detail (unique_id DESC, health_log_id)",
+ "CREATE INDEX IF NOT EXISTS health_log_d_ind_6 on health_log_detail (health_log_id, when_key)",
+ "CREATE INDEX IF NOT EXISTS health_log_d_ind_7 on health_log_detail (alarm_id)",
+ "CREATE INDEX IF NOT EXISTS health_log_d_ind_8 on health_log_detail (new_status, updated_by_id)",
+
+ NULL
+};
+
+const char *database_cleanup[] = {
+ "DELETE FROM host WHERE host_id NOT IN (SELECT host_id FROM chart)",
+ "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)",
+ "DELETE FROM host_info WHERE host_id NOT IN (SELECT host_id FROM host)",
+ "DELETE FROM host_label WHERE host_id NOT IN (SELECT host_id FROM host)",
+ "DROP TRIGGER IF EXISTS tr_dim_del",
+ "DROP INDEX IF EXISTS ind_d1",
+ "DROP INDEX IF EXISTS ind_c1",
+ "DROP INDEX IF EXISTS ind_c2",
+ "DROP INDEX IF EXISTS alert_hash_index",
+ "DROP INDEX IF EXISTS health_log_d_ind_4",
+ "DROP INDEX IF EXISTS health_log_d_ind_1",
+ "DROP INDEX IF EXISTS health_log_d_ind_5",
+ NULL
+};
+
+sqlite3 *db_meta = NULL;
+
+// SQL statements
+
+#define SQL_STORE_CLAIM_ID \
+ "INSERT INTO node_instance " \
+ "(host_id, claim_id, date_created) VALUES (@host_id, @claim_id, UNIXEPOCH()) " \
+ "ON CONFLICT(host_id) DO UPDATE SET claim_id = excluded.claim_id"
+
+#define SQL_DELETE_HOST_LABELS "DELETE FROM host_label WHERE host_id = @uuid"
+
+#define STORE_HOST_LABEL \
+ "INSERT INTO host_label (host_id, source_type, label_key, label_value, date_created) VALUES "
+
+#define STORE_CHART_LABEL \
+ "INSERT INTO chart_label (chart_id, source_type, label_key, label_value, date_created) VALUES "
+
+#define STORE_HOST_OR_CHART_LABEL_VALUE "(u2h('%s'), %d,'%s','%s', unixepoch())"
+
+#define DELETE_DIMENSION_UUID "DELETE FROM dimension WHERE dim_id = @uuid"
+
+#define SQL_STORE_HOST_INFO \
+ "INSERT OR REPLACE INTO host (host_id, hostname, registry_hostname, update_every, os, timezone, tags, hops, " \
+ "memory_mode, abbrev_timezone, utc_offset, program_name, program_version, entries, health_enabled, last_connected) " \
+ "VALUES (@host_id, @hostname, @registry_hostname, @update_every, @os, @timezone, @tags, @hops, " \
+ "@memory_mode, @abbrev_tz, @utc_offset, @prog_name, @prog_version, @entries, @health_enabled, @last_connected)"
+
+#define SQL_STORE_CHART \
+ "INSERT INTO chart (chart_id, host_id, type, id, name, family, context, title, unit, plugin, module, priority, " \
+ "update_every, chart_type, memory_mode, history_entries) " \
+ "values (@chart_id, @host_id, @type, @id, @name, @family, @context, @title, @unit, @plugin, @module, @priority, " \
+ "@update_every, @chart_type, @memory_mode, @history_entries) " \
+ "ON CONFLICT(chart_id) DO UPDATE SET type=excluded.type, id=excluded.id, name=excluded.name, " \
+ "family=excluded.family, context=excluded.context, title=excluded.title, unit=excluded.unit, " \
+ "plugin=excluded.plugin, module=excluded.module, priority=excluded.priority, update_every=excluded.update_every, " \
+ "chart_type=excluded.chart_type, memory_mode = excluded.memory_mode, history_entries = excluded.history_entries"
+
+#define SQL_STORE_DIMENSION \
+ "INSERT INTO dimension (dim_id, chart_id, id, name, multiplier, divisor , algorithm, options) " \
+ "VALUES (@dim_id, @chart_id, @id, @name, @multiplier, @divisor, @algorithm, @options) " \
+ "ON CONFLICT(dim_id) DO UPDATE SET id=excluded.id, name=excluded.name, multiplier=excluded.multiplier, " \
+ "divisor=excluded.divisor, algorithm=excluded.algorithm, options=excluded.options"
+
+#define SELECT_DIMENSION_LIST "SELECT dim_id, rowid FROM dimension WHERE rowid > @row_id"
+#define SELECT_CHART_LIST "SELECT chart_id, rowid FROM chart WHERE rowid > @row_id"
+#define SELECT_CHART_LABEL_LIST "SELECT chart_id, rowid FROM chart_label WHERE rowid > @row_id"
+
+#define SQL_STORE_HOST_SYSTEM_INFO_VALUES \
+ "INSERT OR REPLACE INTO host_info (host_id, system_key, system_value, date_created) VALUES " \
+ "(@uuid, @name, @value, UNIXEPOCH())"
+
+#define CONVERT_EXISTING_LOCALHOST "UPDATE host SET hops = 1 WHERE hops = 0 AND host_id <> @host_id"
+#define DELETE_MISSING_NODE_INSTANCES "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host)"
+
+#define METADATA_MAINTENANCE_FIRST_CHECK (1800) // Maintenance first run after agent startup in seconds
+#define METADATA_MAINTENANCE_REPEAT (60) // Repeat if last run for dimensions, charts, labels needs more work
+#define METADATA_HEALTH_LOG_INTERVAL (3600) // Repeat maintenance for health
+#define METADATA_DIM_CHECK_INTERVAL (3600) // Repeat maintenance for dimensions
+#define METADATA_CHART_CHECK_INTERVAL (3600) // Repeat maintenance for charts
+#define METADATA_LABEL_CHECK_INTERVAL (3600) // Repeat maintenance for labels
+#define METADATA_RUNTIME_THRESHOLD (5) // Run time threshold for cleanup task
+
+#define METADATA_HOST_CHECK_FIRST_CHECK (5) // First check for pending metadata
+#define METADATA_HOST_CHECK_INTERVAL (30) // Repeat check for pending metadata
+#define METADATA_HOST_CHECK_IMMEDIATE (5) // Repeat immediate run because we have more metadata to write
+#define MAX_METADATA_CLEANUP (500) // Maximum metadata write operations (e.g deletes before retrying)
+#define METADATA_MAX_BATCH_SIZE (512) // Maximum commands to execute before running the event loop
+
+#define DATABASE_FREE_PAGES_THRESHOLD_PC (5) // Percentage of free pages to trigger vacuum
+#define DATABASE_FREE_PAGES_VACUUM_PC (10) // Percentage of free pages to vacuum
+
+enum metadata_opcode {
+ METADATA_DATABASE_NOOP = 0,
+ METADATA_DATABASE_TIMER,
+ METADATA_DEL_DIMENSION,
+ METADATA_STORE_CLAIM_ID,
+ METADATA_ADD_HOST_INFO,
+ METADATA_SCAN_HOSTS,
+ METADATA_LOAD_HOST_CONTEXT,
+ METADATA_DELETE_HOST_CHART_LABELS,
+ METADATA_MAINTENANCE,
+ METADATA_SYNC_SHUTDOWN,
+ METADATA_UNITTEST,
+ // leave this last
+ // we need it to check for worker utilization
+ METADATA_MAX_ENUMERATIONS_DEFINED
+};
+
+#define MAX_PARAM_LIST (2)
+struct metadata_cmd {
+ enum metadata_opcode opcode;
+ struct completion *completion;
+ const void *param[MAX_PARAM_LIST];
+ struct metadata_cmd *prev, *next;
+};
+
+typedef enum {
+ METADATA_FLAG_PROCESSING = (1 << 0), // store or cleanup
+ METADATA_FLAG_SHUTDOWN = (1 << 1), // Shutting down
+} METADATA_FLAG;
+
+struct metadata_wc {
+ uv_thread_t thread;
+ uv_loop_t *loop;
+ uv_async_t async;
+ uv_timer_t timer_req;
+ time_t metadata_check_after;
+ METADATA_FLAG flags;
+ struct completion start_stop_complete;
+ struct completion *scan_complete;
+ /* FIFO command queue */
+ SPINLOCK cmd_queue_lock;
+ struct metadata_cmd *cmd_base;
+};
+
+#define metadata_flag_check(target_flags, flag) (__atomic_load_n(&((target_flags)->flags), __ATOMIC_SEQ_CST) & (flag))
+#define metadata_flag_set(target_flags, flag) __atomic_or_fetch(&((target_flags)->flags), (flag), __ATOMIC_SEQ_CST)
+#define metadata_flag_clear(target_flags, flag) __atomic_and_fetch(&((target_flags)->flags), ~(flag), __ATOMIC_SEQ_CST)
+
+struct metadata_wc metasync_worker = {.loop = NULL};
+
+//
+// For unittest
+//
+struct thread_unittest {
+ int join;
+ unsigned added;
+ unsigned processed;
+ unsigned *done;
+};
+
+int sql_metadata_cache_stats(int op)
+{
+ int count, dummy;
+
+ if (unlikely(!db_meta))
+ return 0;
+
+ netdata_thread_disable_cancelability();
+ sqlite3_db_status(db_meta, op, &count, &dummy, 0);
+ netdata_thread_enable_cancelability();
+ return count;
+}
+
+static inline void set_host_node_id(RRDHOST *host, uuid_t *node_id)
+{
+ if (unlikely(!host))
+ return;
+
+ if (unlikely(!node_id)) {
+ freez(host->node_id);
+ __atomic_store_n(&host->node_id, NULL, __ATOMIC_RELAXED);
+ return;
+ }
+
+ struct aclk_sync_cfg_t *wc = host->aclk_config;
+
+ if (unlikely(!host->node_id)) {
+ uuid_t *t = mallocz(sizeof(*host->node_id));
+ uuid_copy(*t, *node_id);
+ __atomic_store_n(&host->node_id, t, __ATOMIC_RELAXED);
+ }
+ else {
+ uuid_copy(*(host->node_id), *node_id);
+ }
+
+ if (unlikely(!wc))
+ sql_create_aclk_table(host, &host->host_uuid, node_id);
+ else
+ uuid_unparse_lower(*node_id, wc->node_id);
+}
+
+#define SQL_UPDATE_NODE_ID "UPDATE node_instance SET node_id = @node_id WHERE host_id = @host_id"
+
+int update_node_id(uuid_t *host_id, uuid_t *node_id)
+{
+ sqlite3_stmt *res = NULL;
+ RRDHOST *host = NULL;
+ int rc = 2;
+
+ char host_guid[GUID_LEN + 1];
+ uuid_unparse_lower(*host_id, host_guid);
+ rrd_wrlock();
+ host = rrdhost_find_by_guid(host_guid);
+ if (likely(host))
+ set_host_node_id(host, node_id);
+ rrd_unlock();
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ error_report("Database has not been initialized");
+ return 1;
+ }
+
+ rc = sqlite3_prepare_v2(db_meta, SQL_UPDATE_NODE_ID, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to store node instance information");
+ return 1;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, node_id, sizeof(*node_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to store node instance information");
+ goto failed;
+ }
+
+ rc = sqlite3_bind_blob(res, 2, host_id, sizeof(*host_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to store node instance information");
+ goto failed;
+ }
+
+ rc = execute_insert(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to store node instance information, rc = %d", rc);
+ rc = sqlite3_changes(db_meta);
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when storing node instance information");
+
+ return rc - 1;
+}
+
+#define SQL_SELECT_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id AND node_id IS NOT NULL"
+
+int get_node_id(uuid_t *host_id, uuid_t *node_id)
+{
+ sqlite3_stmt *res = NULL;
+ int rc;
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ error_report("Database has not been initialized");
+ return 1;
+ }
+
+ rc = sqlite3_prepare_v2(db_meta, SQL_SELECT_NODE_ID, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to select node instance information for a host");
+ return 1;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to select node instance information");
+ goto failed;
+ }
+
+ rc = sqlite3_step_monitored(res);
+ if (likely(rc == SQLITE_ROW && node_id))
+ uuid_copy(*node_id, *((uuid_t *) sqlite3_column_blob(res, 0)));
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when selecting node instance information");
+
+ return (rc == SQLITE_ROW) ? 0 : -1;
+}
+
+#define SQL_INVALIDATE_NODE_INSTANCES \
+ "UPDATE node_instance SET node_id = NULL WHERE EXISTS " \
+ "(SELECT host_id FROM node_instance WHERE host_id = @host_id AND (@claim_id IS NULL OR claim_id <> @claim_id))"
+
+void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id)
+{
+ sqlite3_stmt *res = NULL;
+ int rc;
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ error_report("Database has not been initialized");
+ return;
+ }
+
+ rc = sqlite3_prepare_v2(db_meta, SQL_INVALIDATE_NODE_INSTANCES, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to invalidate node instance ids");
+ return;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to invalidate node instance information");
+ goto failed;
+ }
+
+ if (claim_id)
+ rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, 2);
+
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind claim_id parameter to invalidate node instance information");
+ goto failed;
+ }
+
+ rc = execute_insert(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to invalidate node instance information, rc = %d", rc);
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when invalidating node instance information");
+}
+
+#define SQL_GET_NODE_INSTANCE_LIST \
+ "SELECT ni.node_id, ni.host_id, h.hostname " \
+ "FROM node_instance ni, host h WHERE ni.host_id = h.host_id AND h.hops >=0"
+
+struct node_instance_list *get_node_list(void)
+{
+ struct node_instance_list *node_list = NULL;
+ sqlite3_stmt *res = NULL;
+ int rc;
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ error_report("Database has not been initialized");
+ return NULL;
+ }
+
+ rc = sqlite3_prepare_v2(db_meta, SQL_GET_NODE_INSTANCE_LIST, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to get node instance information");
+ return NULL;
+ }
+
+ int row = 0;
+ char host_guid[UUID_STR_LEN];
+ while (sqlite3_step_monitored(res) == SQLITE_ROW)
+ row++;
+
+ if (sqlite3_reset(res) != SQLITE_OK) {
+ error_report("Failed to reset the prepared statement while fetching node instance information");
+ goto failed;
+ }
+ node_list = callocz(row + 1, sizeof(*node_list));
+ int max_rows = row;
+ row = 0;
+ // TODO: Check to remove lock
+ rrd_rdlock();
+ while (sqlite3_step_monitored(res) == SQLITE_ROW) {
+ if (sqlite3_column_bytes(res, 0) == sizeof(uuid_t))
+ uuid_copy(node_list[row].node_id, *((uuid_t *)sqlite3_column_blob(res, 0)));
+ if (sqlite3_column_bytes(res, 1) == sizeof(uuid_t)) {
+ uuid_t *host_id = (uuid_t *)sqlite3_column_blob(res, 1);
+ uuid_unparse_lower(*host_id, host_guid);
+ RRDHOST *host = rrdhost_find_by_guid(host_guid);
+ if (!host)
+ continue;
+ if (rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD)) {
+ netdata_log_info("ACLK: 'host:%s' skipping get node list because context is initializing", rrdhost_hostname(host));
+ continue;
+ }
+ uuid_copy(node_list[row].host_id, *host_id);
+ node_list[row].queryable = 1;
+ node_list[row].live = (host && (host == localhost || host->receiver
+ || !(rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)))) ? 1 : 0;
+ node_list[row].hops = (host && host->system_info) ? host->system_info->hops :
+ uuid_memcmp(host_id, &localhost->host_uuid) ? 1 : 0;
+ node_list[row].hostname =
+ sqlite3_column_bytes(res, 2) ? strdupz((char *)sqlite3_column_text(res, 2)) : NULL;
+ }
+ row++;
+ if (row == max_rows)
+ break;
+ }
+ rrd_unlock();
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when fetching node instance information");
+
+ return node_list;
+}
+
+#define SQL_GET_HOST_NODE_ID "SELECT node_id FROM node_instance WHERE host_id = @host_id"
+
+void sql_load_node_id(RRDHOST *host)
+{
+ sqlite3_stmt *res = NULL;
+ int rc;
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ error_report("Database has not been initialized");
+ return;
+ }
+
+ rc = sqlite3_prepare_v2(db_meta, SQL_GET_HOST_NODE_ID, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to fetch node id");
+ return;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to load node instance information");
+ goto failed;
+ }
+
+ rc = sqlite3_step_monitored(res);
+ if (likely(rc == SQLITE_ROW)) {
+ if (likely(sqlite3_column_bytes(res, 0) == sizeof(uuid_t)))
+ set_host_node_id(host, (uuid_t *)sqlite3_column_blob(res, 0));
+ else
+ set_host_node_id(host, NULL);
+ }
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when loading node instance information");
+}
+
+#define SELECT_HOST_INFO "SELECT system_key, system_value FROM host_info WHERE host_id = @host_id"
+
+void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info)
+{
+ int rc;
+
+ sqlite3_stmt *res = NULL;
+
+ rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_INFO, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to read host information");
+ return;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host parameter host information");
+ goto skip;
+ }
+
+ while (sqlite3_step_monitored(res) == SQLITE_ROW) {
+ rrdhost_set_system_info_variable(system_info, (char *) sqlite3_column_text(res, 0),
+ (char *) sqlite3_column_text(res, 1));
+ }
+
+skip:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when reading host information");
+}
+
+#define SELECT_HOST_LABELS "SELECT label_key, label_value, source_type FROM host_label WHERE host_id = @host_id " \
+ "AND label_key IS NOT NULL AND label_value IS NOT NULL"
+
+RRDLABELS *sql_load_host_labels(uuid_t *host_id)
+{
+ int rc;
+
+ RRDLABELS *labels = NULL;
+ sqlite3_stmt *res = NULL;
+
+ rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_LABELS, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to read host information");
+ return NULL;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host parameter host information");
+ goto skip;
+ }
+
+ labels = rrdlabels_create();
+
+ while (sqlite3_step_monitored(res) == SQLITE_ROW) {
+ rrdlabels_add(labels, (const char *)sqlite3_column_text(res, 0), (const char *)sqlite3_column_text(res, 1), sqlite3_column_int(res, 2));
+ }
+
+skip:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when reading host information");
+ return labels;
+}
+
+static int exec_statement_with_uuid(const char *sql, uuid_t *uuid)
+{
+ int rc, result = 1;
+ sqlite3_stmt *res = NULL;
+
+ rc = sqlite3_prepare_v2(db_meta, sql, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement %s, rc = %d", sql, rc);
+ return 1;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, uuid, sizeof(*uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind UUID parameter to %s, rc = %d", sql, rc);
+ goto skip;
+ }
+
+ rc = execute_insert(res);
+ if (likely(rc == SQLITE_DONE))
+ result = SQLITE_OK;
+ else
+ error_report("Failed to execute %s, rc = %d", sql, rc);
+
+skip:
+ rc = sqlite3_finalize(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to finalize statement %s, rc = %d", sql, rc);
+ return result;
+}
+
+static void recover_database(const char *sqlite_database, const char *new_sqlite_database)
+{
+ sqlite3 *database;
+ int rc = sqlite3_open(sqlite_database, &database);
+ if (rc != SQLITE_OK)
+ return;
+
+ netdata_log_info("Recover %s", sqlite_database);
+ netdata_log_info(" to %s", new_sqlite_database);
+
+ // This will remove the -shm and -wal files when we close the database
+ (void) db_execute(database, "select count(*) from sqlite_master limit 0");
+
+ sqlite3_recover *recover = sqlite3_recover_init(database, "main", new_sqlite_database);
+ if (recover) {
+
+ rc = sqlite3_recover_run(recover);
+
+ if (rc == SQLITE_OK)
+ netdata_log_info("Recover complete");
+ else
+ netdata_log_error("Recover encountered an error but the database may be usable");
+
+ rc = sqlite3_recover_finish(recover);
+
+ (void) sqlite3_close(database);
+
+ if (rc == SQLITE_OK) {
+ rc = rename(new_sqlite_database, sqlite_database);
+ if (rc == 0) {
+ netdata_log_info("Renamed %s", new_sqlite_database);
+ netdata_log_info(" to %s", sqlite_database);
+ }
+ }
+ else
+ netdata_log_error("Recover failed to free resources");
+ }
+ else
+ (void) sqlite3_close(database);
+}
+
+
+static void sqlite_uuid_parse(sqlite3_context *context, int argc, sqlite3_value **argv)
+{
+ uuid_t uuid;
+
+ if ( argc != 1 ){
+ sqlite3_result_null(context);
+ return ;
+ }
+ int rc = uuid_parse((const char *) sqlite3_value_text(argv[0]), uuid);
+ if (rc == -1) {
+ sqlite3_result_null(context);
+ return ;
+ }
+
+ sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);
+}
+
+void sqlite_now_usec(sqlite3_context *context, int argc, sqlite3_value **argv)
+{
+ if (argc != 1 ){
+ sqlite3_result_null(context);
+ return ;
+ }
+
+ if (sqlite3_value_int(argv[0]) != 0) {
+ struct timespec req = {.tv_sec = 0, .tv_nsec = 1};
+ nanosleep(&req, NULL);
+ }
+
+ sqlite3_result_int64(context, (sqlite_int64) now_realtime_usec());
+}
+
+void sqlite_uuid_random(sqlite3_context *context, int argc, sqlite3_value **argv)
+{
+ (void)argc;
+ (void)argv;
+
+ uuid_t uuid;
+ uuid_generate_random(uuid);
+ sqlite3_result_blob(context, &uuid, sizeof(uuid_t), SQLITE_TRANSIENT);
+}
+
+// Init
+/*
+ * Initialize the SQLite database
+ * Return 0 on success
+ */
+int sql_init_meta_database(db_check_action_type_t rebuild, int memory)
+{
+ char *err_msg = NULL;
+ char sqlite_database[FILENAME_MAX + 1];
+ int rc;
+
+ if (likely(!memory)) {
+ snprintfz(sqlite_database, sizeof(sqlite_database) - 1, "%s/.netdata-meta.db.recover", netdata_configured_cache_dir);
+ rc = unlink(sqlite_database);
+ snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
+
+ if (rc == 0 || (rebuild & DB_CHECK_RECOVER)) {
+ char new_sqlite_database[FILENAME_MAX + 1];
+ snprintfz(new_sqlite_database, sizeof(new_sqlite_database) - 1, "%s/netdata-meta-recover.db", netdata_configured_cache_dir);
+ recover_database(sqlite_database, new_sqlite_database);
+ if (rebuild & DB_CHECK_RECOVER)
+ return 0;
+ }
+ }
+ else
+ strncpyz(sqlite_database, ":memory:", sizeof(sqlite_database) - 1);
+
+ rc = sqlite3_open(sqlite_database, &db_meta);
+ if (rc != SQLITE_OK) {
+ error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
+ char *error_str = get_database_extented_error(db_meta, 0, "meta_open");
+ if (error_str)
+ analytics_set_data_str(&analytics_data.netdata_fail_reason, error_str);
+ freez(error_str);
+ sqlite3_close(db_meta);
+ db_meta = NULL;
+ return 1;
+ }
+
+ if (rebuild & DB_CHECK_RECLAIM_SPACE) {
+ netdata_log_info("Reclaiming space of %s", sqlite_database);
+ rc = sqlite3_exec_monitored(db_meta, "VACUUM", 0, 0, &err_msg);
+ if (rc != SQLITE_OK) {
+ error_report("Failed to execute VACUUM rc = %d (%s)", rc, err_msg);
+ sqlite3_free(err_msg);
+ }
+ else {
+ (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");
+ (void) sqlite3_close(db_meta);
+ }
+ return 1;
+ }
+
+ if (rebuild & DB_CHECK_ANALYZE) {
+ errno = 0;
+ netdata_log_info("Running ANALYZE on %s", sqlite_database);
+ rc = sqlite3_exec_monitored(db_meta, "ANALYZE", 0, 0, &err_msg);
+ if (rc != SQLITE_OK) {
+ error_report("Failed to execute ANALYZE rc = %d (%s)", rc, err_msg);
+ sqlite3_free(err_msg);
+ }
+ else {
+ (void) db_execute(db_meta, "select count(*) from sqlite_master limit 0");
+ (void) sqlite3_close(db_meta);
+ }
+ return 1;
+ }
+
+ netdata_log_info("SQLite database %s initialization", sqlite_database);
+
+ rc = sqlite3_create_function(db_meta, "u2h", 1, SQLITE_ANY | SQLITE_DETERMINISTIC, 0, sqlite_uuid_parse, 0, 0);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to register internal u2h function");
+
+ rc = sqlite3_create_function(db_meta, "now_usec", 1, SQLITE_ANY, 0, sqlite_now_usec, 0, 0);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to register internal now_usec function");
+
+ rc = sqlite3_create_function(db_meta, "uuid_random", 0, SQLITE_ANY, 0, sqlite_uuid_random, 0, 0);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to register internal uuid_random function");
+
+ int target_version = DB_METADATA_VERSION;
+
+ if (likely(!memory))
+ target_version = perform_database_migration(db_meta, DB_METADATA_VERSION);
+
+ if (configure_sqlite_database(db_meta, target_version, "meta_config"))
+ return 1;
+
+ if (init_database_batch(db_meta, &database_config[0], "meta_init"))
+ return 1;
+
+ if (init_database_batch(db_meta, &database_cleanup[0], "meta_cleanup"))
+ return 1;
+
+ netdata_log_info("SQLite database initialization completed");
+
+ return 0;
+}
+
+// Metadata functions
+
+struct query_build {
+ BUFFER *sql;
+ int count;
+ char uuid_str[UUID_STR_LEN];
+};
+
+#define SQL_DELETE_CHART_LABELS_BY_HOST \
+ "DELETE FROM chart_label WHERE chart_id in (SELECT chart_id FROM chart WHERE host_id = @host_id)"
+
+static void delete_host_chart_labels(uuid_t *host_uuid)
+{
+ sqlite3_stmt *res = NULL;
+
+ int rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABELS_BY_HOST, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to delete chart labels by host");
+ return;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, host_uuid, sizeof(*host_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to host chart labels");
+ goto failed;
+ }
+ rc = sqlite3_step_monitored(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to execute command to remove host chart labels");
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize statement to remove host chart labels");
+}
+
+static int host_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+ struct query_build *lb = data;
+ if (unlikely(!lb->count))
+ buffer_sprintf(lb->sql, STORE_HOST_LABEL);
+ else
+ buffer_strcat(lb->sql, ", ");
+ buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);
+ lb->count++;
+ return 1;
+}
+
+static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
+ struct query_build *lb = data;
+ if (unlikely(!lb->count))
+ buffer_sprintf(lb->sql, STORE_CHART_LABEL);
+ else
+ buffer_strcat(lb->sql, ", ");
+ buffer_sprintf(lb->sql, STORE_HOST_OR_CHART_LABEL_VALUE, lb->uuid_str, (int) (ls & ~(RRDLABEL_FLAG_INTERNAL)), name, value);
+ lb->count++;
+ return 1;
+}
+
+#define SQL_DELETE_CHART_LABEL "DELETE FROM chart_label WHERE chart_id = @chart_id"
+#define SQL_DELETE_CHART_LABEL_HISTORY "DELETE FROM chart_label WHERE date_created < %ld AND chart_id = @chart_id"
+
+static void clean_old_chart_labels(RRDSET *st)
+{
+ char sql[512];
+ time_t first_time_s = rrdset_first_entry_s(st);
+
+ if (unlikely(!first_time_s))
+ snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL);
+ else
+ snprintfz(sql, sizeof(sql) - 1, SQL_DELETE_CHART_LABEL_HISTORY, first_time_s);
+
+ int rc = exec_statement_with_uuid(sql, &st->chart_uuid);
+ if (unlikely(rc))
+ error_report("METADATA: 'host:%s' Failed to clean old labels for chart %s", rrdhost_hostname(st->rrdhost), rrdset_name(st));
+}
+
+static int check_and_update_chart_labels(RRDSET *st, BUFFER *work_buffer, size_t *query_counter)
+{
+ size_t old_version = st->rrdlabels_last_saved_version;
+ size_t new_version = rrdlabels_version(st->rrdlabels);
+
+ if (new_version == old_version)
+ return 0;
+
+ struct query_build tmp = {.sql = work_buffer, .count = 0};
+ uuid_unparse_lower(st->chart_uuid, tmp.uuid_str);
+ rrdlabels_walkthrough_read(st->rrdlabels, chart_label_store_to_sql_callback, &tmp);
+ buffer_strcat(work_buffer, " ON CONFLICT (chart_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
+ int rc = db_execute(db_meta, buffer_tostring(work_buffer));
+ if (likely(!rc)) {
+ st->rrdlabels_last_saved_version = new_version;
+ (*query_counter)++;
+ }
+
+ clean_old_chart_labels(st);
+ return rc;
+}
+
+// If the machine guid has changed, then existing one with hops 0 will be marked as hops 1 (child)
+void detect_machine_guid_change(uuid_t *host_uuid)
+{
+ int rc;
+
+ rc = exec_statement_with_uuid(CONVERT_EXISTING_LOCALHOST, host_uuid);
+ if (!rc) {
+ if (unlikely(db_execute(db_meta, DELETE_MISSING_NODE_INSTANCES)))
+ error_report("Failed to remove deleted hosts from node instances");
+ }
+}
+
+static int store_claim_id(uuid_t *host_id, uuid_t *claim_id)
+{
+ sqlite3_stmt *res = NULL;
+ int rc;
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
+ error_report("Database has not been initialized");
+ return 1;
+ }
+
+ rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CLAIM_ID, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to store host claim id");
+ return 1;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, host_id, sizeof(*host_id), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind host_id parameter to store claim id");
+ goto failed;
+ }
+
+ if (claim_id)
+ rc = sqlite3_bind_blob(res, 2, claim_id, sizeof(*claim_id), SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, 2);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind claim_id parameter to host claim id");
+ goto failed;
+ }
+
+ rc = execute_insert(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to store host claim id rc = %d", rc);
+
+failed:
+ if (unlikely(sqlite3_finalize(res) != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when storing a host claim id");
+
+ return rc != SQLITE_DONE;
+}
+
+static void delete_dimension_uuid(uuid_t *dimension_uuid, sqlite3_stmt **action_res __maybe_unused, bool flag __maybe_unused)
+{
+ static __thread sqlite3_stmt *res = NULL;
+ int rc;
+
+ if (unlikely(!res)) {
+ rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res);
+ if (rc != SQLITE_OK) {
+ error_report("Failed to prepare statement to delete a dimension uuid");
+ return;
+ }
+ }
+
+ rc = sqlite3_bind_blob(res, 1, dimension_uuid, sizeof(*dimension_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto skip_execution;
+
+ rc = sqlite3_step_monitored(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to delete dimension uuid, rc = %d", rc);
+
+skip_execution:
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement when deleting dimension UUID, rc = %d", rc);
+}
+
+//
+// Store host and host system info information in the database
+static int store_host_metadata(RRDHOST *host)
+{
+ static __thread sqlite3_stmt *res = NULL;
+ int rc, param = 0;
+
+ if (unlikely((!res))) {
+ rc = prepare_statement(db_meta, SQL_STORE_HOST_INFO, &res);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to store host, rc = %d", rc);
+ return 1;
+ }
+ }
+
+ rc = sqlite3_bind_blob(res, ++param, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_hostname(host), 0);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_registry_hostname(host), 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, host->rrd_update_every);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_os(host), 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_timezone(host), 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ // TODO: remove/migrate
+ rc = bind_text_null(res, ++param, "", 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, host->system_info ? host->system_info->hops : 0);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, host->rrd_memory_mode);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_abbrev_timezone(host), 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, host->utc_offset);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_program_name(host), 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, rrdhost_program_version(host), 1);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int64(res, ++param, host->rrd_history_entries);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, (int ) host->health.health_enabled);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int64(res, ++param, (sqlite3_int64) host->last_connected);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ int store_rc = sqlite3_step_monitored(res);
+ if (unlikely(store_rc != SQLITE_DONE))
+ error_report("Failed to store host %s, rc = %d", rrdhost_hostname(host), rc);
+
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
+
+ return store_rc != SQLITE_DONE;
+bind_fail:
+ error_report("Failed to bind %d parameter to store host %s, rc = %d", param, rrdhost_hostname(host), rc);
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement to store host %s, rc = %d", rrdhost_hostname(host), rc);
+ return 1;
+}
+
+static int add_host_sysinfo_key_value(const char *name, const char *value, uuid_t *uuid)
+{
+ static __thread sqlite3_stmt *res = NULL;
+ int rc, param = 0;
+
+ if (unlikely(!db_meta)) {
+ if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)
+ return 0;
+ error_report("Database has not been initialized");
+ return 0;
+ }
+
+ if (unlikely((!res))) {
+ rc = prepare_statement(db_meta, SQL_STORE_HOST_SYSTEM_INFO_VALUES, &res);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to store host info values, rc = %d", rc);
+ return 0;
+ }
+ }
+
+ rc = sqlite3_bind_blob(res, ++param, uuid, sizeof(*uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, name, 0);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = bind_text_null(res, ++param, value ? value : "unknown", 0);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ int store_rc = sqlite3_step_monitored(res);
+ if (unlikely(store_rc != SQLITE_DONE))
+ error_report("Failed to store host info value %s, rc = %d", name, rc);
+
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement to store host info value %s, rc = %d", name, rc);
+
+ return store_rc == SQLITE_DONE;
+bind_fail:
+ error_report("Failed to bind %d parameter to store host info values %s, rc = %d", param, name, rc);
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement to store host info values %s, rc = %d", name, rc);
+ return 0;
+}
+
+static bool store_host_systeminfo(RRDHOST *host)
+{
+ struct rrdhost_system_info *system_info = host->system_info;
+
+ if (unlikely(!system_info))
+ return false;
+
+ int ret = 0;
+
+ ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_NAME", system_info->container_os_name, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID", system_info->container_os_id, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_ID_LIKE", system_info->container_os_id_like, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION", system_info->container_os_version, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_VERSION_ID", system_info->container_os_version_id, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_CONTAINER_OS_DETECTION", system_info->host_os_detection, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_NAME", system_info->host_os_name, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID", system_info->host_os_id, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_ID_LIKE", system_info->host_os_id_like, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION", system_info->host_os_version, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_VERSION_ID", system_info->host_os_version_id, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_OS_DETECTION", system_info->host_os_detection, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_NAME", system_info->kernel_name, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT", system_info->host_cores, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CPU_FREQ", system_info->host_cpu_freq, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_RAM", system_info->host_ram_total, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_TOTAL_DISK_SIZE", system_info->host_disk_space, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_KERNEL_VERSION", system_info->kernel_version, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_ARCHITECTURE", system_info->architecture, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRTUALIZATION", system_info->virtualization, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_VIRT_DETECTION", system_info->virt_detection, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER", system_info->container, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_SYSTEM_CONTAINER_DETECTION", system_info->container_detection, &host->host_uuid);
+ ret += add_host_sysinfo_key_value("NETDATA_HOST_IS_K8S_NODE", system_info->is_k8s_node, &host->host_uuid);
+
+ return !(24 == ret);
+}
+
+
+/*
+ * Store a chart in the database
+ */
+
+static int store_chart_metadata(RRDSET *st)
+{
+ static __thread sqlite3_stmt *res = NULL;
+ int rc, param = 0, store_rc = 0;
+
+ if (unlikely(!res)) {
+ rc = prepare_statement(db_meta, SQL_STORE_CHART, &res);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to store chart, rc = %d", rc);
+ return 1;
+ }
+ }
+
+ rc = sqlite3_bind_blob(res, ++param, &st->chart_uuid, sizeof(st->chart_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_blob(res, ++param, &st->rrdhost->host_uuid, sizeof(st->rrdhost->host_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, string2str(st->parts.type), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, string2str(st->parts.id), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ const char *name = string2str(st->parts.name);
+ if (name && *name)
+ rc = sqlite3_bind_text(res, ++param, name, -1, SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, ++param);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, rrdset_family(st), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, rrdset_context(st), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, rrdset_title(st), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, rrdset_units(st), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, rrdset_plugin_name(st), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, rrdset_module_name(st), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, (int) st->priority);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, st->update_every);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, st->chart_type);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, st->rrd_memory_mode);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, (int) st->db.entries);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ store_rc = execute_insert(res);
+ if (unlikely(store_rc != SQLITE_DONE))
+ error_report("Failed to store chart, rc = %d", store_rc);
+
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement in chart store function, rc = %d", rc);
+
+ return store_rc != SQLITE_DONE;
+
+bind_fail:
+ error_report("Failed to bind parameter %d to store chart, rc = %d", param, rc);
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement in chart store function, rc = %d", rc);
+ return 1;
+}
+
+/*
+ * Store a dimension
+ */
+static int store_dimension_metadata(RRDDIM *rd)
+{
+ static __thread sqlite3_stmt *res = NULL;
+ int rc, param = 0;
+
+ if (unlikely(!res)) {
+ rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to store dimension, rc = %d", rc);
+ return 1;
+ }
+ }
+
+ rc = sqlite3_bind_blob(res, ++param, &rd->metric_uuid, sizeof(rd->metric_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_blob(res, ++param, &rd->rrdset->chart_uuid, sizeof(rd->rrdset->chart_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, string2str(rd->id), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_text(res, ++param, string2str(rd->name), -1, SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, (int) rd->multiplier);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, (int ) rd->divisor);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = sqlite3_bind_int(res, ++param, rd->algorithm);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
+ rc = sqlite3_bind_text(res, ++param, "hidden", -1, SQLITE_STATIC);
+ else
+ rc = sqlite3_bind_null(res, ++param);
+ if (unlikely(rc != SQLITE_OK))
+ goto bind_fail;
+
+ rc = execute_insert(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to store dimension, rc = %d", rc);
+
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement in store dimension, rc = %d", rc);
+ return 0;
+
+bind_fail:
+ error_report("Failed to bind parameter %d to store dimension, rc = %d", param, rc);
+ rc = sqlite3_reset(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to reset statement in store dimension, rc = %d", rc);
+ return 1;
+}
+
+static bool dimension_can_be_deleted(uuid_t *dim_uuid __maybe_unused, sqlite3_stmt **res __maybe_unused, bool flag __maybe_unused)
+{
+#ifdef ENABLE_DBENGINE
+ if(dbengine_enabled) {
+ bool no_retention = true;
+ for (size_t tier = 0; tier < storage_tiers; tier++) {
+ if (!multidb_ctx[tier])
+ continue;
+ time_t first_time_t = 0, last_time_t = 0;
+ if (rrdeng_metric_retention_by_uuid((void *) multidb_ctx[tier], dim_uuid, &first_time_t, &last_time_t)) {
+ if (first_time_t > 0) {
+ no_retention = false;
+ break;
+ }
+ }
+ }
+ return no_retention;
+ }
+ else
+ return false;
+#else
+ return false;
+#endif
+}
+
+static bool run_cleanup_loop(
+ sqlite3_stmt *res,
+ struct metadata_wc *wc,
+ bool (*check_cb)(uuid_t *, sqlite3_stmt **, bool),
+ void (*action_cb)(uuid_t *, sqlite3_stmt **, bool),
+ uint32_t *total_checked,
+ uint32_t *total_deleted,
+ uint64_t *row_id,
+ sqlite3_stmt **check_stmt,
+ sqlite3_stmt **action_stmt,
+ bool check_flag,
+ bool action_flag)
+{
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ return true;
+
+ int rc = sqlite3_bind_int64(res, 1, (sqlite3_int64) *row_id);
+ if (unlikely(rc != SQLITE_OK))
+ return true;
+
+ time_t start_running = now_monotonic_sec();
+ bool time_expired = false;
+ while (!time_expired && sqlite3_step_monitored(res) == SQLITE_ROW &&
+ (*total_deleted < MAX_METADATA_CLEANUP && *total_checked < MAX_METADATA_CLEANUP)) {
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ break;
+
+ *row_id = sqlite3_column_int64(res, 1);
+ rc = check_cb((uuid_t *)sqlite3_column_blob(res, 0), check_stmt, check_flag);
+
+ if (rc == true) {
+ action_cb((uuid_t *)sqlite3_column_blob(res, 0), action_stmt, action_flag);
+ (*total_deleted)++;
+ }
+
+ (*total_checked)++;
+ time_expired = ((now_monotonic_sec() - start_running) > METADATA_RUNTIME_THRESHOLD);
+ }
+ return time_expired || (*total_checked == MAX_METADATA_CLEANUP) || (*total_deleted == MAX_METADATA_CLEANUP);
+}
+
+
+#define SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION "SELECT count(1) FROM dimension WHERE chart_id = @chart_id"
+#define SQL_CHECK_CHART_EXISTENCE_IN_CHART "SELECT count(1) FROM chart WHERE chart_id = @chart_id"
+
+static bool chart_can_be_deleted(uuid_t *chart_uuid, sqlite3_stmt **check_res, bool check_in_dimension)
+{
+ int rc, result = 1;
+ sqlite3_stmt *res = check_res ? *check_res : NULL;
+
+ if (!res) {
+ if (check_in_dimension)
+ rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_DIMENSION, -1, &res, 0);
+ else
+ rc = sqlite3_prepare_v2(db_meta, SQL_CHECK_CHART_EXISTENCE_IN_CHART, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
+ return 0;
+ }
+ if (check_res)
+ *check_res = res;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind chart uuid parameter, rc = %d", rc);
+ goto skip;
+ }
+
+ rc = sqlite3_step_monitored(res);
+ if (likely(rc == SQLITE_ROW))
+ result = sqlite3_column_int(res, 0);
+
+skip:
+ if (check_res)
+ rc = sqlite3_reset(res);
+ else
+ rc = sqlite3_finalize(res);
+
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to %s statement that checks chart uuid existence rc = %d", check_res ? "reset" : "finalize", rc);
+ return result == 0;
+}
+
+#define SQL_DELETE_CHART_BY_UUID "DELETE FROM chart WHERE chart_id = @chart_id"
+#define SQL_DELETE_CHART_LABEL_BY_UUID "DELETE FROM chart_label WHERE chart_id = @chart_id"
+
+static void delete_chart_uuid(uuid_t *chart_uuid, sqlite3_stmt **action_res, bool label_only)
+{
+ int rc;
+ sqlite3_stmt *res = action_res ? *action_res : NULL;
+
+ if (!res) {
+ if (label_only)
+ rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_LABEL_BY_UUID, -1, &res, 0);
+ else
+ rc = sqlite3_prepare_v2(db_meta, SQL_DELETE_CHART_BY_UUID, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to check for chart existence, rc = %d", rc);
+ return;
+ }
+ if (action_res)
+ *action_res = res;
+ }
+
+ rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to bind chart uuid parameter, rc = %d", rc);
+ goto skip;
+ }
+
+ rc = sqlite3_step_monitored(res);
+ if (unlikely(rc != SQLITE_DONE))
+ error_report("Failed to delete a chart uuid from the %s table, rc = %d", label_only ? "labels" : "chart", rc);
+
+skip:
+ if (action_res)
+ rc = sqlite3_reset(res);
+ else
+ rc = sqlite3_finalize(res);
+
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to %s statement that deletes a chart uuid rc = %d", action_res ? "reset" : "finalize", rc);
+}
+
+static void check_dimension_metadata(struct metadata_wc *wc)
+{
+ static time_t next_execution_t = 0;
+ static uint64_t last_row_id = 0;
+
+ time_t now = now_realtime_sec();
+
+ if (!next_execution_t)
+ next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+ if (next_execution_t && next_execution_t > now)
+ return;
+
+ int rc;
+ sqlite3_stmt *res = NULL;
+
+ rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION_LIST, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to fetch host dimensions");
+ return;
+ }
+
+ uint32_t total_checked = 0;
+ uint32_t total_deleted = 0;
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking dimensions starting after row %" PRIu64, last_row_id);
+
+ bool more_to_do = run_cleanup_loop(
+ res,
+ wc,
+ dimension_can_be_deleted,
+ delete_dimension_uuid,
+ &total_checked,
+ &total_deleted,
+ &last_row_id,
+ NULL,
+ NULL,
+ false,
+ false);
+
+ now = now_realtime_sec();
+ if (more_to_do)
+ next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
+ else {
+ last_row_id = 0;
+ next_execution_t = now + METADATA_DIM_CHECK_INTERVAL;
+ }
+
+ nd_log(
+ NDLS_DAEMON,
+ NDLP_DEBUG,
+ "Dimensions checked %u, deleted %u. Checks will %s in %lld seconds",
+ total_checked,
+ total_deleted,
+ last_row_id ? "resume" : "restart",
+ (long long)(next_execution_t - now));
+
+ rc = sqlite3_finalize(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement to check dimensions");
+}
+
+static void check_chart_metadata(struct metadata_wc *wc)
+{
+ static time_t next_execution_t = 0;
+ static uint64_t last_row_id = 0;
+
+ time_t now = now_realtime_sec();
+
+ if (!next_execution_t)
+ next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+ if (next_execution_t && next_execution_t > now)
+ return;
+
+ sqlite3_stmt *res = NULL;
+
+ int rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LIST, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to fetch charts");
+ return;
+ }
+
+ uint32_t total_checked = 0;
+ uint32_t total_deleted = 0;
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts starting after row %" PRIu64, last_row_id);
+
+ sqlite3_stmt *check_res = NULL;
+ sqlite3_stmt *action_res = NULL;
+ bool more_to_do = run_cleanup_loop(
+ res,
+ wc,
+ chart_can_be_deleted,
+ delete_chart_uuid,
+ &total_checked,
+ &total_deleted,
+ &last_row_id,
+ &check_res,
+ &action_res,
+ true,
+ false);
+
+ if (check_res)
+ sqlite3_finalize(check_res);
+
+ if (action_res)
+ sqlite3_finalize(action_res);
+
+ now = now_realtime_sec();
+ if (more_to_do)
+ next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
+ else {
+ last_row_id = 0;
+ next_execution_t = now + METADATA_CHART_CHECK_INTERVAL;
+ }
+
+ nd_log(
+ NDLS_DAEMON,
+ NDLP_DEBUG,
+ "Charts checked %u, deleted %u. Checks will %s in %lld seconds",
+ total_checked,
+ total_deleted,
+ last_row_id ? "resume" : "restart",
+ (long long)(next_execution_t - now));
+
+ rc = sqlite3_finalize(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when reading charts");
+}
+
+static void check_label_metadata(struct metadata_wc *wc)
+{
+ static time_t next_execution_t = 0;
+ static uint64_t last_row_id = 0;
+
+ time_t now = now_realtime_sec();
+
+ if (!next_execution_t)
+ next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+ if (next_execution_t && next_execution_t > now)
+ return;
+
+ int rc;
+ sqlite3_stmt *res = NULL;
+
+ rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_LABEL_LIST, -1, &res, 0);
+ if (unlikely(rc != SQLITE_OK)) {
+ error_report("Failed to prepare statement to fetch charts");
+ return;
+ }
+
+ uint32_t total_checked = 0;
+ uint32_t total_deleted = 0;
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking charts labels starting after row %" PRIu64, last_row_id);
+
+ sqlite3_stmt *check_res = NULL;
+ sqlite3_stmt *action_res = NULL;
+
+ bool more_to_do = run_cleanup_loop(
+ res,
+ wc,
+ chart_can_be_deleted,
+ delete_chart_uuid,
+ &total_checked,
+ &total_deleted,
+ &last_row_id,
+ &check_res,
+ &action_res,
+ false,
+ true);
+
+ if (check_res)
+ sqlite3_finalize(check_res);
+
+ if (action_res)
+ sqlite3_finalize(action_res);
+
+ now = now_realtime_sec();
+ if (more_to_do)
+ next_execution_t = now + METADATA_MAINTENANCE_REPEAT;
+ else {
+ last_row_id = 0;
+ next_execution_t = now + METADATA_LABEL_CHECK_INTERVAL;
+ }
+
+ nd_log(
+ NDLS_DAEMON,
+ NDLP_DEBUG,
+ "Chart labels checked %u, deleted %u. Checks will %s in %lld seconds",
+ total_checked,
+ total_deleted,
+ last_row_id ? "resume" : "restart",
+ (long long)(next_execution_t - now));
+
+ rc = sqlite3_finalize(res);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to finalize the prepared statement when checking charts");
+}
+
+
+static void cleanup_health_log(struct metadata_wc *wc)
+{
+ static time_t next_execution_t = 0;
+
+ time_t now = now_realtime_sec();
+
+ if (!next_execution_t)
+ next_execution_t = now + METADATA_MAINTENANCE_FIRST_CHECK;
+
+ if (next_execution_t && next_execution_t > now)
+ return;
+
+ next_execution_t = now + METADATA_HEALTH_LOG_INTERVAL;
+
+ RRDHOST *host;
+
+ bool is_claimed = claimed();
+ dfe_start_reentrant(rrdhost_root_index, host){
+ if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))
+ continue;
+ sql_health_alarm_log_cleanup(host, is_claimed);
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ break;
+ }
+ dfe_done(host);
+
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ return;
+
+ (void) db_execute(db_meta,"DELETE FROM health_log WHERE host_id NOT IN (SELECT host_id FROM host)");
+ (void) db_execute(db_meta,"DELETE FROM health_log_detail WHERE health_log_id NOT IN (SELECT health_log_id FROM health_log)");
+}
+
+//
+// EVENT LOOP STARTS HERE
+//
+
+static void metadata_free_cmd_queue(struct metadata_wc *wc)
+{
+ spinlock_lock(&wc->cmd_queue_lock);
+ while(wc->cmd_base) {
+ struct metadata_cmd *t = wc->cmd_base;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wc->cmd_base, t, prev, next);
+ freez(t);
+ }
+ spinlock_unlock(&wc->cmd_queue_lock);
+}
+
+static void metadata_enq_cmd(struct metadata_wc *wc, struct metadata_cmd *cmd)
+{
+ if (cmd->opcode == METADATA_SYNC_SHUTDOWN) {
+ metadata_flag_set(wc, METADATA_FLAG_SHUTDOWN);
+ goto wakeup_event_loop;
+ }
+
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ goto wakeup_event_loop;
+
+ struct metadata_cmd *t = mallocz(sizeof(*t));
+ *t = *cmd;
+ t->prev = t->next = NULL;
+
+ spinlock_lock(&wc->cmd_queue_lock);
+ DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(wc->cmd_base, t, prev, next);
+ spinlock_unlock(&wc->cmd_queue_lock);
+
+wakeup_event_loop:
+ (void) uv_async_send(&wc->async);
+}
+
+static struct metadata_cmd metadata_deq_cmd(struct metadata_wc *wc)
+{
+ struct metadata_cmd ret;
+
+ spinlock_lock(&wc->cmd_queue_lock);
+ if(wc->cmd_base) {
+ struct metadata_cmd *t = wc->cmd_base;
+ DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(wc->cmd_base, t, prev, next);
+ ret = *t;
+ freez(t);
+ }
+ else {
+ ret.opcode = METADATA_DATABASE_NOOP;
+ ret.completion = NULL;
+ }
+ spinlock_unlock(&wc->cmd_queue_lock);
+
+ return ret;
+}
+
+static void async_cb(uv_async_t *handle)
+{
+ uv_stop(handle->loop);
+ uv_update_time(handle->loop);
+}
+
+#define TIMER_INITIAL_PERIOD_MS (1000)
+#define TIMER_REPEAT_PERIOD_MS (1000)
+
+static void timer_cb(uv_timer_t* handle)
+{
+ uv_stop(handle->loop);
+ uv_update_time(handle->loop);
+
+ struct metadata_wc *wc = handle->data;
+ struct metadata_cmd cmd;
+ memset(&cmd, 0, sizeof(cmd));
+
+ if (wc->metadata_check_after < now_realtime_sec()) {
+ cmd.opcode = METADATA_SCAN_HOSTS;
+ metadata_enq_cmd(wc, &cmd);
+ }
+}
+
+void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc)
+{
+ int free_pages = get_free_page_count(database);
+ int total_pages = get_database_page_count(database);
+
+ if (!threshold)
+ threshold = DATABASE_FREE_PAGES_THRESHOLD_PC;
+
+ if (!vacuum_pc)
+ vacuum_pc = DATABASE_FREE_PAGES_VACUUM_PC;
+
+ if (free_pages > (total_pages * threshold / 100)) {
+
+ int do_free_pages = (int) (free_pages * vacuum_pc / 100);
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "%s: Freeing %d database pages", db_alias, do_free_pages);
+
+ char sql[128];
+ snprintfz(sql, sizeof(sql) - 1, "PRAGMA incremental_vacuum(%d)", do_free_pages);
+ (void) db_execute(database, sql);
+ }
+}
+
+void run_metadata_cleanup(struct metadata_wc *wc)
+{
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ return;
+
+ check_dimension_metadata(wc);
+ check_chart_metadata(wc);
+ check_label_metadata(wc);
+ cleanup_health_log(wc);
+
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN)))
+ return;
+
+ vacuum_database(db_meta, "METADATA", DATABASE_FREE_PAGES_THRESHOLD_PC, DATABASE_FREE_PAGES_VACUUM_PC);
+
+ (void) sqlite3_wal_checkpoint(db_meta, NULL);
+}
+
+struct scan_metadata_payload {
+ uv_work_t request;
+ struct metadata_wc *wc;
+ void *data;
+ BUFFER *work_buffer;
+ uint32_t max_count;
+};
+
+struct host_context_load_thread {
+ uv_thread_t thread;
+ RRDHOST *host;
+ bool busy;
+ bool finished;
+};
+
+static void restore_host_context(void *arg)
+{
+ struct host_context_load_thread *hclt = arg;
+ RRDHOST *host = hclt->host;
+
+ usec_t started_ut = now_monotonic_usec(); (void)started_ut;
+ rrdhost_load_rrdcontext_data(host);
+ usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
+
+ rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD);
+
+#ifdef ENABLE_ACLK
+ aclk_queue_node_info(host, false);
+#endif
+
+ nd_log(
+ NDLS_DAEMON,
+ NDLP_DEBUG,
+ "Contexts for host %s loaded in %0.2f ms",
+ rrdhost_hostname(host),
+ (double)(ended_ut - started_ut) / USEC_PER_MS);
+
+ __atomic_store_n(&hclt->finished, true, __ATOMIC_RELEASE);
+}
+
+// Callback after scan of hosts is done
+static void after_start_host_load_context(uv_work_t *req, int status __maybe_unused)
+{
+ struct scan_metadata_payload *data = req->data;
+ freez(data);
+}
+
+#define MAX_FIND_THREAD_RETRIES (10)
+
+static void cleanup_finished_threads(struct host_context_load_thread *hclt, size_t max_thread_slots, bool wait)
+{
+ if (!hclt)
+ return;
+
+ for (size_t index = 0; index < max_thread_slots; index++) {
+ if (__atomic_load_n(&(hclt[index].finished), __ATOMIC_RELAXED)
+ || (wait && __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE))) {
+ int rc = uv_thread_join(&(hclt[index].thread));
+ if (rc)
+ nd_log(NDLS_DAEMON, NDLP_WARNING, "Failed to join thread, rc = %d", rc);
+ __atomic_store_n(&(hclt[index].busy), false, __ATOMIC_RELEASE);
+ __atomic_store_n(&(hclt[index].finished), false, __ATOMIC_RELEASE);
+ }
+ }
+}
+
+static size_t find_available_thread_slot(struct host_context_load_thread *hclt, size_t max_thread_slots, size_t *found_index)
+{
+ size_t retries = MAX_FIND_THREAD_RETRIES;
+ while (retries--) {
+ size_t index = 0;
+ while (index < max_thread_slots) {
+ if (false == __atomic_load_n(&(hclt[index].busy), __ATOMIC_ACQUIRE)) {
+ *found_index = index;
+ return true;
+ }
+ index++;
+ }
+ sleep_usec(10 * USEC_PER_MS);
+ }
+ return false;
+}
+
+static void start_all_host_load_context(uv_work_t *req __maybe_unused)
+{
+ register_libuv_worker_jobs();
+
+ struct scan_metadata_payload *data = req->data;
+ struct metadata_wc *wc = data->wc;
+
+ worker_is_busy(UV_EVENT_HOST_CONTEXT_LOAD);
+ usec_t started_ut = now_monotonic_usec(); (void)started_ut;
+
+ RRDHOST *host;
+
+ size_t max_threads = MIN(get_netdata_cpus() / 2, 6);
+ if (max_threads < 1)
+ max_threads = 1;
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Using %zu threads for context loading", max_threads);
+ struct host_context_load_thread *hclt = max_threads > 1 ? callocz(max_threads, sizeof(*hclt)) : NULL;
+
+ size_t thread_index = 0;
+ dfe_start_reentrant(rrdhost_root_index, host) {
+ if (!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))
+ continue;
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Loading context for host %s", rrdhost_hostname(host));
+
+ int rc = 0;
+ if (hclt) {
+ bool found_slot = false;
+ do {
+ if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
+ break;
+
+ cleanup_finished_threads(hclt, max_threads, false);
+ found_slot = find_available_thread_slot(hclt, max_threads, &thread_index);
+ } while (!found_slot);
+
+ if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
+ break;
+
+ __atomic_store_n(&hclt[thread_index].busy, true, __ATOMIC_RELAXED);
+ hclt[thread_index].host = host;
+ rc = uv_thread_create(&hclt[thread_index].thread, restore_host_context, &hclt[thread_index]);
+ }
+ // if single thread or thread creation failed
+ if (rc || !hclt) {
+ struct host_context_load_thread hclt_sync = {.host = host};
+ restore_host_context(&hclt_sync);
+
+ if (metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))
+ break;
+ }
+ }
+ dfe_done(host);
+
+ cleanup_finished_threads(hclt, max_threads, true);
+ freez(hclt);
+ usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Host contexts loaded in %0.2f ms", (double)(ended_ut - started_ut) / USEC_PER_MS);
+
+ worker_is_idle();
+}
+
+// Callback after scan of hosts is done
+static void after_metadata_hosts(uv_work_t *req, int status __maybe_unused)
+{
+ struct scan_metadata_payload *data = req->data;
+ struct metadata_wc *wc = data->wc;
+
+ metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);
+
+ if (unlikely(wc->scan_complete))
+ completion_mark_complete(wc->scan_complete);
+
+ freez(data);
+}
+
+static bool metadata_scan_host(RRDHOST *host, uint32_t max_count, bool use_transaction, BUFFER *work_buffer, size_t *query_counter) {
+ RRDSET *st;
+ int rc;
+
+ bool more_to_do = false;
+ uint32_t scan_count = 1;
+
+ sqlite3_stmt *ml_load_stmt = NULL;
+
+ bool load_ml_models = max_count;
+
+ if (use_transaction)
+ (void)db_execute(db_meta, "BEGIN TRANSACTION");
+
+ rrdset_foreach_reentrant(st, host) {
+ if (scan_count == max_count) {
+ more_to_do = true;
+ break;
+ }
+ if(rrdset_flag_check(st, RRDSET_FLAG_METADATA_UPDATE)) {
+ (*query_counter)++;
+
+ rrdset_flag_clear(st, RRDSET_FLAG_METADATA_UPDATE);
+ scan_count++;
+
+ buffer_flush(work_buffer);
+ rc = check_and_update_chart_labels(st, work_buffer, query_counter);
+ if (unlikely(rc))
+ error_report("METADATA: 'host:%s': Failed to update labels for chart %s", rrdhost_hostname(host), rrdset_name(st));
+ else
+ (*query_counter)++;
+
+ rc = store_chart_metadata(st);
+ if (unlikely(rc))
+ error_report("METADATA: 'host:%s': Failed to store metadata for chart %s", rrdhost_hostname(host), rrdset_name(st));
+ }
+
+ RRDDIM *rd;
+ rrddim_foreach_read(rd, st) {
+ if(rrddim_flag_check(rd, RRDDIM_FLAG_METADATA_UPDATE)) {
+ (*query_counter)++;
+
+ rrddim_flag_clear(rd, RRDDIM_FLAG_METADATA_UPDATE);
+
+ if (rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN))
+ rrddim_flag_set(rd, RRDDIM_FLAG_META_HIDDEN);
+ else
+ rrddim_flag_clear(rd, RRDDIM_FLAG_META_HIDDEN);
+
+ rc = store_dimension_metadata(rd);
+ if (unlikely(rc))
+ error_report("METADATA: 'host:%s': Failed to dimension metadata for chart %s. dimension %s",
+ rrdhost_hostname(host), rrdset_name(st),
+ rrddim_name(rd));
+ }
+
+ if(rrddim_flag_check(rd, RRDDIM_FLAG_ML_MODEL_LOAD)) {
+ rrddim_flag_clear(rd, RRDDIM_FLAG_ML_MODEL_LOAD);
+ if (likely(load_ml_models))
+ (void) ml_dimension_load_models(rd, &ml_load_stmt);
+ }
+
+ worker_is_idle();
+ }
+ rrddim_foreach_done(rd);
+ }
+ rrdset_foreach_done(st);
+
+ if (use_transaction)
+ (void)db_execute(db_meta, "COMMIT TRANSACTION");
+
+ if (ml_load_stmt) {
+ sqlite3_finalize(ml_load_stmt);
+ ml_load_stmt = NULL;
+ }
+
+ return more_to_do;
+}
+
+static void store_host_and_system_info(RRDHOST *host, size_t *query_counter)
+{
+ if (unlikely(store_host_systeminfo(host))) {
+ error_report("METADATA: 'host:%s': Failed to store host updated system information in the database", rrdhost_hostname(host));
+ rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
+ }
+ else {
+ if (likely(query_counter))
+ (*query_counter)++;
+ }
+
+ if (unlikely(store_host_metadata(host))) {
+ error_report("METADATA: 'host:%s': Failed to store host info in the database", rrdhost_hostname(host));
+ rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_INFO | RRDHOST_FLAG_METADATA_UPDATE);
+ }
+ else {
+ if (likely(query_counter))
+ (*query_counter)++;
+ }
+}
+
+struct host_chart_label_cleanup {
+ Pvoid_t JudyL;
+ Word_t count;
+};
+
+static void do_chart_label_cleanup(struct host_chart_label_cleanup *cl_cleanup_data)
+{
+ if (!cl_cleanup_data)
+ return;
+
+ Word_t Index = 0;
+ bool first = true;
+ Pvoid_t *PValue;
+ while ((PValue = JudyLFirstThenNext(cl_cleanup_data->JudyL, &Index, &first))) {
+ char *machine_guid = *PValue;
+
+ RRDHOST *host = rrdhost_find_by_guid(machine_guid);
+ if (likely(!host)) {
+ uuid_t host_uuid;
+ if (!uuid_parse(machine_guid, host_uuid))
+ delete_host_chart_labels(&host_uuid);
+ }
+
+ freez(machine_guid);
+ }
+ JudyLFreeArray(&cl_cleanup_data->JudyL, PJE0);
+ freez(cl_cleanup_data);
+}
+
+// Worker thread to scan hosts for pending metadata to store
+static void start_metadata_hosts(uv_work_t *req __maybe_unused)
+{
+ register_libuv_worker_jobs();
+
+ RRDHOST *host;
+ int transaction_started = 0;
+
+ struct scan_metadata_payload *data = req->data;
+ struct metadata_wc *wc = data->wc;
+
+ BUFFER *work_buffer = data->work_buffer;
+ usec_t all_started_ut = now_monotonic_usec(); (void)all_started_ut;
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Checking all hosts started");
+ usec_t started_ut = now_monotonic_usec(); (void)started_ut;
+
+ do_chart_label_cleanup((struct host_chart_label_cleanup *) data->data);
+
+ bool run_again = false;
+ worker_is_busy(UV_EVENT_METADATA_STORE);
+
+ if (!data->max_count)
+ transaction_started = !db_execute(db_meta, "BEGIN TRANSACTION");
+
+ dfe_start_reentrant(rrdhost_root_index, host) {
+ if (rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED) || !rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_UPDATE))
+ continue;
+
+ size_t query_counter = 0; (void)query_counter;
+
+ rrdhost_flag_clear(host,RRDHOST_FLAG_METADATA_UPDATE);
+
+ if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_LABELS))) {
+ rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_LABELS);
+
+ int rc = exec_statement_with_uuid(SQL_DELETE_HOST_LABELS, &host->host_uuid);
+ if (likely(!rc)) {
+ query_counter++;
+
+ buffer_flush(work_buffer);
+ struct query_build tmp = {.sql = work_buffer, .count = 0};
+ uuid_unparse_lower(host->host_uuid, tmp.uuid_str);
+ rrdlabels_walkthrough_read(host->rrdlabels, host_label_store_to_sql_callback, &tmp);
+ buffer_strcat(work_buffer, " ON CONFLICT (host_id, label_key) DO UPDATE SET source_type = excluded.source_type, label_value=excluded.label_value, date_created=UNIXEPOCH()");
+ rc = db_execute(db_meta, buffer_tostring(work_buffer));
+
+ if (unlikely(rc)) {
+ error_report("METADATA: 'host:%s': failed to update metadata host labels", rrdhost_hostname(host));
+ rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
+ }
+ else
+ query_counter++;
+ } else {
+ error_report("METADATA: 'host:%s': failed to delete old host labels", rrdhost_hostname(host));
+ rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);
+ }
+ }
+
+ if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_CLAIMID))) {
+ rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_CLAIMID);
+ uuid_t uuid;
+ int rc;
+ if (likely(host->aclk_state.claimed_id && !uuid_parse(host->aclk_state.claimed_id, uuid)))
+ rc = store_claim_id(&host->host_uuid, &uuid);
+ else
+ rc = store_claim_id(&host->host_uuid, NULL);
+
+ if (unlikely(rc))
+ rrdhost_flag_set(host, RRDHOST_FLAG_METADATA_CLAIMID | RRDHOST_FLAG_METADATA_UPDATE);
+ else
+ query_counter++;
+ }
+ if (unlikely(rrdhost_flag_check(host, RRDHOST_FLAG_METADATA_INFO))) {
+ rrdhost_flag_clear(host, RRDHOST_FLAG_METADATA_INFO);
+ store_host_and_system_info(host, &query_counter);
+ }
+
+ // For clarity
+ bool use_transaction = data->max_count;
+ if (unlikely(metadata_scan_host(host, data->max_count, use_transaction, work_buffer, &query_counter))) {
+ run_again = true;
+ rrdhost_flag_set(host,RRDHOST_FLAG_METADATA_UPDATE);
+ }
+ usec_t ended_ut = now_monotonic_usec(); (void)ended_ut;
+ nd_log(
+ NDLS_DAEMON,
+ NDLP_DEBUG,
+ "Host %s saved metadata with %zu SQL statements, in %0.2f ms",
+ rrdhost_hostname(host),
+ query_counter,
+ (double)(ended_ut - started_ut) / USEC_PER_MS);
+ }
+ dfe_done(host);
+
+ if (!data->max_count && transaction_started)
+ transaction_started = db_execute(db_meta, "COMMIT TRANSACTION");
+
+ usec_t all_ended_ut = now_monotonic_usec(); (void)all_ended_ut;
+ nd_log(
+ NDLS_DAEMON,
+ NDLP_DEBUG,
+ "Checking all hosts completed in %0.2f ms",
+ (double)(all_ended_ut - all_started_ut) / USEC_PER_MS);
+
+ if (unlikely(run_again))
+ wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_IMMEDIATE;
+ else {
+ wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_INTERVAL;
+ run_metadata_cleanup(wc);
+ }
+ worker_is_idle();
+}
+
+static void metadata_event_loop(void *arg)
+{
+ worker_register("METASYNC");
+ worker_register_job_name(METADATA_DATABASE_NOOP, "noop");
+ worker_register_job_name(METADATA_DATABASE_TIMER, "timer");
+ worker_register_job_name(METADATA_DEL_DIMENSION, "delete dimension");
+ worker_register_job_name(METADATA_STORE_CLAIM_ID, "add claim id");
+ worker_register_job_name(METADATA_ADD_HOST_INFO, "add host info");
+ worker_register_job_name(METADATA_MAINTENANCE, "maintenance");
+
+ int ret;
+ uv_loop_t *loop;
+ unsigned cmd_batch_size;
+ struct metadata_wc *wc = arg;
+ enum metadata_opcode opcode;
+
+ uv_thread_set_name_np(wc->thread, "METASYNC");
+ loop = wc->loop = mallocz(sizeof(uv_loop_t));
+ ret = uv_loop_init(loop);
+ if (ret) {
+ netdata_log_error("uv_loop_init(): %s", uv_strerror(ret));
+ goto error_after_loop_init;
+ }
+ loop->data = wc;
+
+ ret = uv_async_init(wc->loop, &wc->async, async_cb);
+ if (ret) {
+ netdata_log_error("uv_async_init(): %s", uv_strerror(ret));
+ goto error_after_async_init;
+ }
+ wc->async.data = wc;
+
+ ret = uv_timer_init(loop, &wc->timer_req);
+ if (ret) {
+ netdata_log_error("uv_timer_init(): %s", uv_strerror(ret));
+ goto error_after_timer_init;
+ }
+ wc->timer_req.data = wc;
+ fatal_assert(0 == uv_timer_start(&wc->timer_req, timer_cb, TIMER_INITIAL_PERIOD_MS, TIMER_REPEAT_PERIOD_MS));
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Starting metadata sync thread");
+
+ struct metadata_cmd cmd;
+ memset(&cmd, 0, sizeof(cmd));
+ metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);
+
+ wc->metadata_check_after = now_realtime_sec() + METADATA_HOST_CHECK_FIRST_CHECK;
+
+ int shutdown = 0;
+ completion_mark_complete(&wc->start_stop_complete);
+ BUFFER *work_buffer = buffer_create(1024, &netdata_buffers_statistics.buffers_sqlite);
+ struct scan_metadata_payload *data;
+ struct host_chart_label_cleanup *cl_cleanup_data = NULL;
+
+ while (shutdown == 0 || (wc->flags & METADATA_FLAG_PROCESSING)) {
+ uuid_t *uuid;
+ RRDHOST *host = NULL;
+
+ worker_is_idle();
+ uv_run(loop, UV_RUN_DEFAULT);
+
+ /* wait for commands */
+ cmd_batch_size = 0;
+ do {
+ if (unlikely(cmd_batch_size >= METADATA_MAX_BATCH_SIZE))
+ break;
+
+ cmd = metadata_deq_cmd(wc);
+ opcode = cmd.opcode;
+
+ if (unlikely(opcode == METADATA_DATABASE_NOOP && metadata_flag_check(wc, METADATA_FLAG_SHUTDOWN))) {
+ shutdown = 1;
+ continue;
+ }
+
+ ++cmd_batch_size;
+
+ if (likely(opcode != METADATA_DATABASE_NOOP))
+ worker_is_busy(opcode);
+
+ switch (opcode) {
+ case METADATA_DATABASE_NOOP:
+ case METADATA_DATABASE_TIMER:
+ break;
+ case METADATA_DEL_DIMENSION:
+ uuid = (uuid_t *) cmd.param[0];
+ if (likely(dimension_can_be_deleted(uuid, NULL, false)))
+ delete_dimension_uuid(uuid, NULL, false);
+ freez(uuid);
+ break;
+ case METADATA_STORE_CLAIM_ID:
+ store_claim_id((uuid_t *) cmd.param[0], (uuid_t *) cmd.param[1]);
+ freez((void *) cmd.param[0]);
+ freez((void *) cmd.param[1]);
+ break;
+ case METADATA_ADD_HOST_INFO:
+ host = (RRDHOST *) cmd.param[0];
+ store_host_and_system_info(host, NULL);
+ break;
+ case METADATA_SCAN_HOSTS:
+ if (unlikely(metadata_flag_check(wc, METADATA_FLAG_PROCESSING)))
+ break;
+
+ if (unittest_running)
+ break;
+
+ data = mallocz(sizeof(*data));
+ data->request.data = data;
+ data->wc = wc;
+ data->data = cl_cleanup_data;
+ data->work_buffer = work_buffer;
+ cl_cleanup_data = NULL;
+
+ if (unlikely(cmd.completion)) {
+ data->max_count = 0; // 0 will process all pending updates
+ cmd.completion = NULL; // Do not complete after launching worker (worker will do)
+ }
+ else
+ data->max_count = 5000;
+
+ metadata_flag_set(wc, METADATA_FLAG_PROCESSING);
+ if (unlikely(
+ uv_queue_work(loop,&data->request,
+ start_metadata_hosts,
+ after_metadata_hosts))) {
+ // Failed to launch worker -- let the event loop handle completion
+ cmd.completion = wc->scan_complete;
+ cl_cleanup_data = data->data;
+ freez(data);
+ metadata_flag_clear(wc, METADATA_FLAG_PROCESSING);
+ }
+ break;
+ case METADATA_LOAD_HOST_CONTEXT:;
+ if (unittest_running)
+ break;
+
+ data = callocz(1,sizeof(*data));
+ data->request.data = data;
+ data->wc = wc;
+ if (unlikely(
+ uv_queue_work(loop,&data->request, start_all_host_load_context,
+ after_start_host_load_context))) {
+ freez(data);
+ }
+ break;
+ case METADATA_DELETE_HOST_CHART_LABELS:;
+ if (!cl_cleanup_data)
+ cl_cleanup_data = callocz(1,sizeof(*cl_cleanup_data));
+
+ Pvoid_t *PValue = JudyLIns(&cl_cleanup_data->JudyL, (Word_t) ++cl_cleanup_data->count, PJE0);
+ if (PValue)
+ *PValue = (void *) cmd.param[0];
+
+ break;
+ case METADATA_UNITTEST:;
+ struct thread_unittest *tu = (struct thread_unittest *) cmd.param[0];
+ sleep_usec(1000); // processing takes 1ms
+ __atomic_fetch_add(&tu->processed, 1, __ATOMIC_SEQ_CST);
+ break;
+ default:
+ break;
+ }
+
+ if (cmd.completion)
+ completion_mark_complete(cmd.completion);
+ } while (opcode != METADATA_DATABASE_NOOP);
+ }
+
+ if (!uv_timer_stop(&wc->timer_req))
+ uv_close((uv_handle_t *)&wc->timer_req, NULL);
+
+ uv_close((uv_handle_t *)&wc->async, NULL);
+ int rc;
+ do {
+ rc = uv_loop_close(loop);
+ } while (rc != UV_EBUSY);
+
+ buffer_free(work_buffer);
+ freez(loop);
+ worker_unregister();
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Shutting down metadata thread");
+ completion_mark_complete(&wc->start_stop_complete);
+ if (wc->scan_complete) {
+ completion_destroy(wc->scan_complete);
+ freez(wc->scan_complete);
+ }
+ metadata_free_cmd_queue(wc);
+ return;
+
+error_after_timer_init:
+ uv_close((uv_handle_t *)&wc->async, NULL);
+error_after_async_init:
+ fatal_assert(0 == uv_loop_close(loop));
+error_after_loop_init:
+ freez(loop);
+ worker_unregister();
+}
+
+void metadata_sync_shutdown(void)
+{
+ completion_init(&metasync_worker.start_stop_complete);
+
+ struct metadata_cmd cmd;
+ memset(&cmd, 0, sizeof(cmd));
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Sending a shutdown command");
+ cmd.opcode = METADATA_SYNC_SHUTDOWN;
+ metadata_enq_cmd(&metasync_worker, &cmd);
+
+ /* wait for metadata thread to shut down */
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Waiting for shutdown ACK");
+ completion_wait_for(&metasync_worker.start_stop_complete);
+ completion_destroy(&metasync_worker.start_stop_complete);
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Shutdown complete");
+}
+
+void metadata_sync_shutdown_prepare(void)
+{
+ static bool running = false;
+ if (unlikely(!metasync_worker.loop || running))
+ return;
+
+ running = true;
+
+ struct metadata_cmd cmd;
+ memset(&cmd, 0, sizeof(cmd));
+
+ struct metadata_wc *wc = &metasync_worker;
+
+ struct completion *compl = mallocz(sizeof(*compl));
+ completion_init(compl);
+ __atomic_store_n(&wc->scan_complete, compl, __ATOMIC_RELAXED);
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Sending a scan host command");
+ uint32_t max_wait_iterations = 2000;
+ while (unlikely(metadata_flag_check(&metasync_worker, METADATA_FLAG_PROCESSING)) && max_wait_iterations--) {
+ if (max_wait_iterations == 1999)
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Current worker is running; waiting to finish");
+ sleep_usec(1000);
+ }
+
+ cmd.opcode = METADATA_SCAN_HOSTS;
+ metadata_enq_cmd(&metasync_worker, &cmd);
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Waiting for host scan completion");
+ completion_wait_for(wc->scan_complete);
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "METADATA: Host scan complete; can continue with shutdown");
+}
+
+// -------------------------------------------------------------
+// Init function called on agent startup
+
+void metadata_sync_init(void)
+{
+ struct metadata_wc *wc = &metasync_worker;
+
+ memset(wc, 0, sizeof(*wc));
+ completion_init(&wc->start_stop_complete);
+
+ fatal_assert(0 == uv_thread_create(&(wc->thread), metadata_event_loop, wc));
+
+ completion_wait_for(&wc->start_stop_complete);
+ completion_destroy(&wc->start_stop_complete);
+
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "SQLite metadata sync initialization complete");
+}
+
+
+// Helpers
+
+static inline void queue_metadata_cmd(enum metadata_opcode opcode, const void *param0, const void *param1)
+{
+ struct metadata_cmd cmd;
+ cmd.opcode = opcode;
+ cmd.param[0] = param0;
+ cmd.param[1] = param1;
+ cmd.completion = NULL;
+ metadata_enq_cmd(&metasync_worker, &cmd);
+}
+
+// Public
+void metaqueue_delete_dimension_uuid(uuid_t *uuid)
+{
+ if (unlikely(!metasync_worker.loop))
+ return;
+ uuid_t *use_uuid = mallocz(sizeof(*uuid));
+ uuid_copy(*use_uuid, *uuid);
+ queue_metadata_cmd(METADATA_DEL_DIMENSION, use_uuid, NULL);
+}
+
+void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid)
+{
+ if (unlikely(!host_uuid))
+ return;
+
+ uuid_t *local_host_uuid = mallocz(sizeof(*host_uuid));
+ uuid_t *local_claim_uuid = NULL;
+
+ uuid_copy(*local_host_uuid, *host_uuid);
+ if (likely(claim_uuid)) {
+ local_claim_uuid = mallocz(sizeof(*claim_uuid));
+ uuid_copy(*local_claim_uuid, *claim_uuid);
+ }
+ queue_metadata_cmd(METADATA_STORE_CLAIM_ID, local_host_uuid, local_claim_uuid);
+}
+
+void metaqueue_host_update_info(RRDHOST *host)
+{
+ if (unlikely(!metasync_worker.loop))
+ return;
+ queue_metadata_cmd(METADATA_ADD_HOST_INFO, host, NULL);
+}
+
+void metaqueue_ml_load_models(RRDDIM *rd)
+{
+ rrddim_flag_set(rd, RRDDIM_FLAG_ML_MODEL_LOAD);
+}
+
+void metadata_queue_load_host_context(RRDHOST *host)
+{
+ if (unlikely(!metasync_worker.loop))
+ return;
+ queue_metadata_cmd(METADATA_LOAD_HOST_CONTEXT, host, NULL);
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command to load host contexts");
+}
+
+void metadata_delete_host_chart_labels(char *machine_guid)
+{
+ if (unlikely(!metasync_worker.loop)) {
+ freez(machine_guid);
+ return;
+ }
+
+ // Node machine guid is already strdup-ed
+ queue_metadata_cmd(METADATA_DELETE_HOST_CHART_LABELS, machine_guid, NULL);
+ nd_log(NDLS_DAEMON, NDLP_DEBUG, "Queued command delete chart labels for host %s", machine_guid);
+}
+
+
+//
+// unitests
+//
+
+static void *unittest_queue_metadata(void *arg) {
+ struct thread_unittest *tu = arg;
+
+ struct metadata_cmd cmd;
+ cmd.opcode = METADATA_UNITTEST;
+ cmd.param[0] = tu;
+ cmd.param[1] = NULL;
+ cmd.completion = NULL;
+ metadata_enq_cmd(&metasync_worker, &cmd);
+
+ do {
+ __atomic_fetch_add(&tu->added, 1, __ATOMIC_SEQ_CST);
+ metadata_enq_cmd(&metasync_worker, &cmd);
+ sleep_usec(10000);
+ } while (!__atomic_load_n(&tu->join, __ATOMIC_RELAXED));
+ return arg;
+}
+
+static void *metadata_unittest_threads(void)
+{
+
+ unsigned done;
+
+ struct thread_unittest tu = {
+ .join = 0,
+ .added = 0,
+ .processed = 0,
+ .done = &done,
+ };
+
+ // Queue messages / Time it
+ time_t seconds_to_run = 5;
+ int threads_to_create = 4;
+ fprintf(
+ stderr,
+ "\nChecking metadata queue using %d threads for %lld seconds...\n",
+ threads_to_create,
+ (long long)seconds_to_run);
+
+ netdata_thread_t threads[threads_to_create];
+ tu.join = 0;
+ for (int i = 0; i < threads_to_create; i++) {
+ char buf[100 + 1];
+ snprintf(buf, sizeof(buf) - 1, "META[%d]", i);
+ netdata_thread_create(
+ &threads[i],
+ buf,
+ NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_JOINABLE,
+ unittest_queue_metadata,
+ &tu);
+ }
+ (void) uv_async_send(&metasync_worker.async);
+ sleep_usec(seconds_to_run * USEC_PER_SEC);
+
+ __atomic_store_n(&tu.join, 1, __ATOMIC_RELAXED);
+ for (int i = 0; i < threads_to_create; i++) {
+ void *retval;
+ netdata_thread_join(threads[i], &retval);
+ }
+ sleep_usec(5 * USEC_PER_SEC);
+
+ fprintf(stderr, "Added %u elements, processed %u\n", tu.added, tu.processed);
+
+ return 0;
+}
+
+int metadata_unittest(void)
+{
+ metadata_sync_init();
+
+ // Queue items for a specific period of time
+ metadata_unittest_threads();
+
+ metadata_sync_shutdown();
+
+ return 0;
+}
diff --git a/src/database/sqlite/sqlite_metadata.h b/src/database/sqlite/sqlite_metadata.h
new file mode 100644
index 000000000..a276f5c4d
--- /dev/null
+++ b/src/database/sqlite/sqlite_metadata.h
@@ -0,0 +1,59 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#ifndef NETDATA_SQLITE_METADATA_H
+#define NETDATA_SQLITE_METADATA_H
+
+#include "sqlite3.h"
+#include "sqlite_functions.h"
+
+// return a node list
+struct node_instance_list {
+ uuid_t node_id;
+ uuid_t host_id;
+ char *hostname;
+ int live;
+ int queryable;
+ int hops;
+};
+
+typedef enum db_check_action_type {
+ DB_CHECK_NONE = (1 << 0),
+ DB_CHECK_RECLAIM_SPACE = (1 << 1),
+ DB_CHECK_ANALYZE = (1 << 2),
+ DB_CHECK_CONT = (1 << 3),
+ DB_CHECK_RECOVER = (1 << 4),
+} db_check_action_type_t;
+
+// To initialize and shutdown
+void metadata_sync_init(void);
+void metadata_sync_shutdown(void);
+void metadata_sync_shutdown_prepare(void);
+
+void metaqueue_delete_dimension_uuid(uuid_t *uuid);
+void metaqueue_store_claim_id(uuid_t *host_uuid, uuid_t *claim_uuid);
+void metaqueue_host_update_info(RRDHOST *host);
+void metaqueue_ml_load_models(RRDDIM *rd);
+void detect_machine_guid_change(uuid_t *host_uuid);
+void metadata_queue_load_host_context(RRDHOST *host);
+void metadata_delete_host_chart_labels(char *machine_guid);
+void vacuum_database(sqlite3 *database, const char *db_alias, int threshold, int vacuum_pc);
+
+int sql_metadata_cache_stats(int op);
+
+int get_node_id(uuid_t *host_id, uuid_t *node_id);
+int update_node_id(uuid_t *host_id, uuid_t *node_id);
+struct node_instance_list *get_node_list(void);
+void sql_load_node_id(RRDHOST *host);
+
+// Help build archived hosts in memory when agent starts
+void sql_build_host_system_info(uuid_t *host_id, struct rrdhost_system_info *system_info);
+void invalidate_node_instances(uuid_t *host_id, uuid_t *claim_id);
+RRDLABELS *sql_load_host_labels(uuid_t *host_id);
+
+uint64_t sqlite_get_meta_space(void);
+int sql_init_meta_database(db_check_action_type_t rebuild, int memory);
+void sql_close_meta_database(void);
+
+// UNIT TEST
+int metadata_unittest(void);
+#endif //NETDATA_SQLITE_METADATA_H
generated by cgit v1.2.3 (git 2.39.1) at 2024-09-15 22:09:27 +0000