diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2021-03-31 12:58:11 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2021-03-31 12:58:11 +0000 |
commit | f99c4526d94d3e04124c5c48ab4a3da6ca53a458 (patch) | |
tree | a2ed8860030cc49f492b09b3222d593c65619800 /database/sqlite | |
parent | Adding upstream version 1.29.3. (diff) | |
download | netdata-f99c4526d94d3e04124c5c48ab4a3da6ca53a458.tar.xz netdata-f99c4526d94d3e04124c5c48ab4a3da6ca53a458.zip |
Adding upstream version 1.30.0.upstream/1.30.0
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'database/sqlite')
-rw-r--r-- | database/sqlite/Makefile.am | 4 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.c | 380 | ||||
-rw-r--r-- | database/sqlite/sqlite_functions.h | 3 |
3 files changed, 355 insertions, 32 deletions
diff --git a/database/sqlite/Makefile.am b/database/sqlite/Makefile.am new file mode 100644 index 000000000..babdcf0df --- /dev/null +++ b/database/sqlite/Makefile.am @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-3.0-or-later + +AUTOMAKE_OPTIONS = subdir-objects +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c index ab6c59cfa..694b86330 100644 --- a/database/sqlite/sqlite_functions.c +++ b/database/sqlite/sqlite_functions.c @@ -17,12 +17,14 @@ const char *database_config[] = { "CREATE TABLE IF NOT EXISTS metadata_migration(filename text, file_size, date_created int);", "CREATE INDEX IF NOT EXISTS ind_d1 on dimension (chart_id, id, name);", "CREATE INDEX IF NOT EXISTS ind_c1 on chart (host_id, id, type, name);", + "CREATE TABLE IF NOT EXISTS chart_label(chart_id blob, source_type int, label_key text, " + "label_value text, date_created int, PRIMARY KEY (chart_id, label_key));", "delete from chart_active;", "delete from dimension_active;", - "delete from chart where chart_id not in (select chart_id from dimension);", "delete from host where host_id not in (select host_id from chart);", + "delete from chart_label where chart_id not in (select chart_id from chart);", NULL }; @@ -46,6 +48,31 @@ static int execute_insert(sqlite3_stmt *res) return rc; } +#define MAX_OPEN_STATEMENTS (512) + +static void add_stmt_to_list(sqlite3_stmt *res) +{ + static int idx = 0; + static sqlite3_stmt *statements[MAX_OPEN_STATEMENTS]; + + if (unlikely(!res)) { + while (idx > 0) + sqlite3_finalize(statements[--idx]); + return; + } + + if (unlikely(idx == MAX_OPEN_STATEMENTS)) + return; + statements[idx++] = res; +} + +static int prepare_statement(sqlite3 *database, char *query, sqlite3_stmt **statement) { + int rc = sqlite3_prepare_v2(database, query, -1, statement, 0); + if (likely(rc == SQLITE_OK)) + add_stmt_to_list(*statement); + return rc; +} + /* * Store a chart or dimension UUID in chart_active or dimension_active * The statement that will be prepared determines that @@ -82,7 +109,8 @@ void store_active_chart(uuid_t *chart_uuid) int rc; if (unlikely(!db_meta)) { - error_report("Database has not been initialized"); + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); return; } @@ -109,7 +137,8 @@ void store_active_dimension(uuid_t *dimension_uuid) int rc; if (unlikely(!db_meta)) { - error_report("Database has not been initialized"); + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); return; } @@ -141,7 +170,9 @@ int sql_init_database(void) snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir); rc = sqlite3_open(sqlite_database, &db_meta); if (rc != SQLITE_OK) { - error_report("Failed to initialize database at %s", sqlite_database); + error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc)); + sqlite3_close(db_meta); + db_meta = NULL; return 1; } @@ -172,9 +203,12 @@ void sql_close_database(void) return; info("Closing SQLite database"); - rc = sqlite3_close(db_meta); + + add_stmt_to_list(NULL); + + rc = sqlite3_close_v2(db_meta); if (unlikely(rc != SQLITE_OK)) - error_report("Error %d while closing the SQLite database", rc); + error_report("Error %d while closing the SQLite database, %s", rc, sqlite3_errstr(rc)); return; } @@ -187,7 +221,7 @@ int find_uuid_type(uuid_t *uuid) int uuid_type = 3; if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, FIND_UUID_TYPE, -1, &res, 0); + rc = prepare_statement(db_meta, FIND_UUID_TYPE, &res); if (rc != SQLITE_OK) { error_report("Failed to bind prepare statement to find UUID type in the database"); return 0; @@ -218,8 +252,11 @@ uuid_t *find_dimension_uuid(RRDSET *st, RRDDIM *rd) uuid_t *uuid = NULL; int rc; + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return NULL; + if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_FIND_DIMENSION_UUID, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_FIND_DIMENSION_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to bind prepare statement to lookup dimension UUID in the database"); return NULL; @@ -299,7 +336,7 @@ void delete_dimension_uuid(uuid_t *dimension_uuid) #endif if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, DELETE_DIMENSION_UUID, -1, &res, 0); + rc = prepare_statement(db_meta, DELETE_DIMENSION_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to prepare statement to delete a dimension uuid"); return; @@ -331,8 +368,11 @@ uuid_t *find_chart_uuid(RRDHOST *host, const char *type, const char *id, const c uuid_t *uuid = NULL; int rc; + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return NULL; + if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_FIND_CHART_UUID, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_FIND_CHART_UUID, &res); if (rc != SQLITE_OK) { error_report("Failed to prepare statement to lookup chart UUID in the database"); return NULL; @@ -388,6 +428,9 @@ int update_chart_metadata(uuid_t *chart_uuid, RRDSET *st, const char *id, const { int rc; + if (unlikely(!db_meta) && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; + rc = sql_store_chart( chart_uuid, &st->rrdhost->host_uuid, st->type, id, name, st->family, st->context, st->title, st->units, st->plugin_name, st->module_name, st->priority, st->update_every, st->chart_type, st->rrd_memory_mode, st->entries); @@ -427,12 +470,14 @@ int sql_store_host( int rc; if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; error_report("Database has not been initialized"); return 1; } if (unlikely((!res))) { - rc = sqlite3_prepare_v2(db_meta, SQL_STORE_HOST, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_STORE_HOST, &res); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to prepare statement to store host, rc = %d", rc); return 1; @@ -493,16 +538,18 @@ int sql_store_chart( const char *context, const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every, int chart_type, int memory_mode, long history_entries) { - static __thread sqlite3_stmt *res; + static __thread sqlite3_stmt *res = NULL; int rc, param = 0; if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; error_report("Database has not been initialized"); return 1; } if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_STORE_CHART, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_STORE_CHART, &res); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to prepare statement to store chart, rc = %d", rc); return 1; @@ -530,11 +577,12 @@ int sql_store_chart( goto bind_fail; param++; - if (name) { + if (name && *name) rc = sqlite3_bind_text(res, 5, name, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) - goto bind_fail; - } + else + rc = sqlite3_bind_null(res, 5); + if (unlikely(rc != SQLITE_OK)) + goto bind_fail; param++; rc = sqlite3_bind_text(res, 6, family, -1, SQLITE_STATIC); @@ -620,12 +668,14 @@ int sql_store_dimension( int rc; if (unlikely(!db_meta)) { + if (default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return 0; error_report("Database has not been initialized"); return 1; } if (unlikely(!res)) { - rc = sqlite3_prepare_v2(db_meta, SQL_STORE_DIMENSION, -1, &res, 0); + rc = prepare_statement(db_meta, SQL_STORE_DIMENSION, &res); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to prepare statement to store dimension, rc = %d", rc); return 1; @@ -733,7 +783,7 @@ void sql_rrdset2json(RRDHOST *host, BUFFER *wb) rc = sqlite3_bind_blob(res_chart, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); if (unlikely(rc != SQLITE_OK)) { error_report("Failed to bind host parameter to fetch archived charts"); - return; + goto failed; } rc = sqlite3_prepare_v2(db_meta, SELECT_DIMENSION, -1, &res_dim, 0); @@ -883,25 +933,41 @@ failed: return; } -#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname;" +#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;" +#define SELECT_HOST_BY_UUID "select host_id, registry_hostname, update_every, os, timezone, tags from host where host_id = @host_id ;" RRDHOST *sql_create_host_by_uuid(char *hostname) { int rc; RRDHOST *host = NULL; + uuid_t host_uuid; sqlite3_stmt *res = NULL; - rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to prepare statement to fetch host"); - return NULL; + rc = uuid_parse(hostname, host_uuid); + if (!rc) { + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST_BY_UUID, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host by uuid"); + return NULL; + } + rc = sqlite3_bind_blob(res, 1, &host_uuid, sizeof(host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host_id parameter to fetch host information"); + goto failed; + } } - - rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC); - if (unlikely(rc != SQLITE_OK)) { - error_report("Failed to bind hostname parameter to fetch host information"); - return NULL; + else { + rc = sqlite3_prepare_v2(db_meta, SELECT_HOST, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host by hostname"); + return NULL; + } + rc = sqlite3_bind_text(res, 1, hostname, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind hostname parameter to fetch host information"); + goto failed; + } } rc = sqlite3_step(res); @@ -916,13 +982,17 @@ RRDHOST *sql_create_host_by_uuid(char *hostname) host = callocz(1, sizeof(RRDHOST)); set_host_properties(host, sqlite3_column_int(res, 2), RRD_MEMORY_MODE_DBENGINE, hostname, - (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, + (char *) sqlite3_column_text(res, 1), (const char *) uuid_str, (char *) sqlite3_column_text(res, 3), (char *) sqlite3_column_text(res, 5), (char *) sqlite3_column_text(res, 4), NULL, NULL); uuid_copy(host->host_uuid, *((uuid_t *) sqlite3_column_blob(res, 0))); - host->system_info = NULL; + host->system_info = callocz(1, sizeof(*host->system_info));; + rrdhost_flag_set(host, RRDHOST_FLAG_ARCHIVED); +#ifdef ENABLE_DBENGINE + host->rrdeng_ctx = &multidb_ctx; +#endif failed: rc = sqlite3_finalize(res); @@ -1020,3 +1090,251 @@ void add_migrated_file(char *path, uint64_t file_size) return; } + +#define SQL_INS_CHART_LABEL "insert or replace into chart_label " \ + "(chart_id, source_type, label_key, label_value, date_created) " \ + "values (@chart, @source, @label, @value, strftime('%s'));" + +void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value) +{ + sqlite3_stmt *res = NULL; + int rc; + + if (unlikely(!db_meta)) { + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) + error_report("Database has not been initialized"); + return; + } + + rc = sqlite3_prepare_v2(db_meta, SQL_INS_CHART_LABEL, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement store chart labels"); + return; + } + + rc = sqlite3_bind_blob(res, 1, chart_uuid, sizeof(*chart_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind chart_id parameter to store label information"); + goto failed; + } + + rc = sqlite3_bind_int(res, 2, source_type); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind type parameter to store label information"); + goto failed; + } + + rc = sqlite3_bind_text(res, 3, label, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind label parameter to store label information"); + goto failed; + } + + rc = sqlite3_bind_text(res, 4, value, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind value parameter to store label information"); + goto failed; + } + + rc = execute_insert(res); + if (unlikely(rc != SQLITE_DONE)) + error_report("Failed to store chart label entry, rc = %d", rc); + +failed: + if (unlikely(sqlite3_finalize(res) != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when storing chart label information"); + + return; +} + +int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id, + uuid_t *uuid, time_t *first_entry_t, time_t *last_entry_t, uuid_t *rrdeng_uuid) +{ +#ifdef ENABLE_DBENGINE + int rc; + uuid_t legacy_uuid; + uuid_t multihost_legacy_uuid; + time_t dim_first_entry_t, dim_last_entry_t; + + rc = rrdeng_metric_latest_time_by_uuid(uuid, &dim_first_entry_t, &dim_last_entry_t); + if (unlikely(rc)) { + rrdeng_generate_legacy_uuid(dim_id, chart_id, &legacy_uuid); + rc = rrdeng_metric_latest_time_by_uuid(&legacy_uuid, &dim_first_entry_t, &dim_last_entry_t); + if (likely(rc)) { + rrdeng_convert_legacy_uuid_to_multihost(machine_guid, &legacy_uuid, &multihost_legacy_uuid); + rc = rrdeng_metric_latest_time_by_uuid(&multihost_legacy_uuid, &dim_first_entry_t, &dim_last_entry_t); + if (likely(!rc)) + uuid_copy(*rrdeng_uuid, multihost_legacy_uuid); + } + else + uuid_copy(*rrdeng_uuid, legacy_uuid); + } + else + uuid_copy(*rrdeng_uuid, *uuid); + + if (likely(!rc)) { + *first_entry_t = MIN(*first_entry_t, dim_first_entry_t); + *last_entry_t = MAX(*last_entry_t, dim_last_entry_t); + } + return rc; +#else + UNUSED(machine_guid); + UNUSED(chart_id); + UNUSED(dim_id); + UNUSED(uuid); + UNUSED(first_entry_t); + UNUSED(last_entry_t); + UNUSED(rrdeng_uuid); + return 1; +#endif +} + +#ifdef ENABLE_DBENGINE +static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid) +{ + RRDDIM *rd = callocz(1, sizeof(*rd)); + rd->rrdset = st; + rd->last_stored_value = NAN; + rrddim_flag_set(rd, RRDDIM_FLAG_NONE); + rd->state = mallocz(sizeof(*rd->state)); + rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE; + rd->state->query_ops.init = rrdeng_load_metric_init; + rd->state->query_ops.next_metric = rrdeng_load_metric_next; + rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished; + rd->state->query_ops.finalize = rrdeng_load_metric_finalize; + rd->state->query_ops.latest_time = rrdeng_metric_latest_time; + rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time; + rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t)); + uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid); + rd->state->metric_uuid = rd->state->rrdeng_uuid; + rd->id = strdupz(id); + rd->name = strdupz(name); + return rd; +} +#endif + +#define SELECT_CHART_CONTEXT "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id from chart c, " \ + "dimension d, host h " \ + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.context = @context " \ + "order by c.chart_id asc, c.type||c.id desc;" + +#define SELECT_CHART_SINGLE "select d.dim_id, d.id, d.name, c.id, c.type, c.name, c.update_every, c.chart_id, c.context from chart c, " \ + "dimension d, host h " \ + "where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \ + "order by c.chart_id asc, c.type||'.'||c.id desc;" + +void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart) +{ +#ifdef ENABLE_DBENGINE + int rc; + + if (unlikely(!param_list) || host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) + return; + + if (unlikely(!(*param_list))) { + *param_list = mallocz(sizeof(struct context_param)); + (*param_list)->first_entry_t = LONG_MAX; + (*param_list)->last_entry_t = 0; + (*param_list)->rd = NULL; + (*param_list)->flags = CONTEXT_FLAGS_ARCHIVE; + if (chart) + (*param_list)->flags |= CONTEXT_FLAGS_CHART; + else + (*param_list)->flags |= CONTEXT_FLAGS_CONTEXT; + } + + sqlite3_stmt *res = NULL; + + if (context) + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_CONTEXT, -1, &res, 0); + else + rc = sqlite3_prepare_v2(db_meta, SELECT_CHART_SINGLE, -1, &res, 0); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to prepare statement to fetch host archived charts"); + return; + } + + rc = sqlite3_bind_blob(res, 1, &host->host_uuid, sizeof(host->host_uuid), SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter to fetch archived charts"); + goto failed; + } + + if (context) + rc = sqlite3_bind_text(res, 2, context, -1, SQLITE_STATIC); + else + rc = sqlite3_bind_text(res, 2, chart, -1, SQLITE_STATIC); + if (unlikely(rc != SQLITE_OK)) { + error_report("Failed to bind host parameter to fetch archived charts"); + goto failed; + } + + RRDSET *st = NULL; + char machine_guid[GUID_LEN + 1]; + uuid_unparse_lower(host->host_uuid, machine_guid); + uuid_t rrdeng_uuid; + uuid_t chart_id; + + while (sqlite3_step(res) == SQLITE_ROW) { + char id[512]; + sprintf(id, "%s.%s", sqlite3_column_text(res, 3), sqlite3_column_text(res, 1)); + + if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) { + if (unlikely(st && !st->counter)) { + freez(st->context); + freez((char *) st->name); + freez(st); + } + st = callocz(1, sizeof(*st)); + char n[RRD_ID_LENGTH_MAX + 1]; + + snprintfz( + n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4), + (char *)sqlite3_column_text(res, 3)); + st->name = strdupz(n); + st->update_every = sqlite3_column_int(res, 6); + st->counter = 0; + if (chart) { + st->context = strdupz((char *)sqlite3_column_text(res, 8)); + strncpyz(st->id, chart, RRD_ID_LENGTH_MAX); + } + uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7)); + st->last_entry_t = 0; + st->rrdhost = host; + } + + if (unlikely(find_dimension_first_last_t(machine_guid, (char *)st->name, (char *)sqlite3_column_text(res, 1), + (uuid_t *)sqlite3_column_blob(res, 0), &(*param_list)->first_entry_t, &(*param_list)->last_entry_t, + &rrdeng_uuid))) + continue; + + st->counter++; + st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t); + + RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid); + rd->next = (*param_list)->rd; + (*param_list)->rd = rd; + } + if (st) { + if (!st->counter) { + freez(st->context); + freez((char *)st->name); + freez(st); + } + else + if (!st->context && context) + st->context = strdupz(context); + } + +failed: + rc = sqlite3_finalize(res); + if (unlikely(rc != SQLITE_OK)) + error_report("Failed to finalize the prepared statement when reading archived charts"); +#else + UNUSED(param_list); + UNUSED(host); + UNUSED(context); + UNUSED(chart); +#endif + return; +} diff --git a/database/sqlite/sqlite_functions.h b/database/sqlite/sqlite_functions.h index f0b4b775e..d2bee75d2 100644 --- a/database/sqlite/sqlite_functions.h +++ b/database/sqlite/sqlite_functions.h @@ -58,5 +58,6 @@ extern void add_migrated_file(char *path, uint64_t file_size); extern void db_unlock(void); extern void db_lock(void); extern void delete_dimension_uuid(uuid_t *dimension_uuid); - +extern void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value); +extern void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart); #endif //NETDATA_SQLITE_FUNCTIONS_H |