summaryrefslogtreecommitdiffstats
path: root/database/sqlite/sqlite_functions.c
diff options
context:
space:
mode:
Diffstat (limited to 'database/sqlite/sqlite_functions.c')
-rw-r--r--database/sqlite/sqlite_functions.c123
1 files changed, 91 insertions, 32 deletions
diff --git a/database/sqlite/sqlite_functions.c b/database/sqlite/sqlite_functions.c
index 1e1d2a74..502633c6 100644
--- a/database/sqlite/sqlite_functions.c
+++ b/database/sqlite/sqlite_functions.c
@@ -5,8 +5,6 @@
#define DB_METADATA_VERSION "1"
const char *database_config[] = {
- "PRAGMA auto_vacuum=incremental; PRAGMA synchronous=1 ; PRAGMA journal_mode=WAL; PRAGMA temp_store=MEMORY;",
- "PRAGMA journal_size_limit=16777216;",
"CREATE TABLE IF NOT EXISTS host(host_id blob PRIMARY KEY, hostname text, "
"registry_hostname text, update_every int, os text, timezone text, tags text);",
"CREATE TABLE IF NOT EXISTS chart(chart_id blob PRIMARY KEY, host_id blob, type text, id text, name text, "
@@ -62,6 +60,9 @@ const char *database_cleanup[] = {
"delete from chart where chart_id not in (select chart_id from dimension);",
"delete from host where host_id not in (select host_id from chart);",
"delete from chart_label where chart_id not in (select chart_id from chart);",
+ "DELETE FROM chart_hash_map WHERE chart_id NOT IN (SELECT chart_id FROM chart);",
+ "DELETE FROM chart_hash WHERE hash_id NOT IN (SELECT hash_id FROM chart_hash_map);",
+ "DELETE FROM node_instance WHERE host_id NOT IN (SELECT host_id FROM host);",
NULL
};
@@ -72,10 +73,12 @@ static uv_mutex_t sqlite_transaction_lock;
int execute_insert(sqlite3_stmt *res)
{
int rc;
-
- while ((rc = sqlite3_step(res)) != SQLITE_DONE && unlikely(netdata_exit)) {
- if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED))
+ int cnt = 0;
+ while ((rc = sqlite3_step(res)) != SQLITE_DONE && ++cnt < SQL_MAX_RETRY && likely(!netdata_exit)) {
+ if (likely(rc == SQLITE_BUSY || rc == SQLITE_LOCKED)) {
usleep(SQLITE_INSERT_DELAY * USEC_PER_MS);
+ error_report("Failed to insert/update, rc = %d -- attempt %d", rc, cnt);
+ }
else {
error_report("SQLite error %d", rc);
break;
@@ -93,8 +96,12 @@ static void add_stmt_to_list(sqlite3_stmt *res)
static sqlite3_stmt *statements[MAX_OPEN_STATEMENTS];
if (unlikely(!res)) {
- while (idx > 0)
- sqlite3_finalize(statements[--idx]);
+ while (idx > 0) {
+ int rc;
+ rc = sqlite3_finalize(statements[--idx]);
+ if (unlikely(rc != SQLITE_OK))
+ error_report("Failed to finalize statement during shutdown, rc = %d", rc);
+ }
return;
}
@@ -302,7 +309,7 @@ static int attempt_database_fix()
error_report("Failed to close database, rc = %d", rc);
info("Attempting to fix database");
db_meta = NULL;
- return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT);
+ return sql_init_database(DB_CHECK_FIX_DB | DB_CHECK_CONT, 0);
}
static int init_database_batch(int rebuild, int init_type, const char *batch[])
@@ -333,13 +340,17 @@ static int init_database_batch(int rebuild, int init_type, const char *batch[])
* Initialize the SQLite database
* Return 0 on success
*/
-int sql_init_database(db_check_action_type_t rebuild)
+int sql_init_database(db_check_action_type_t rebuild, int memory)
{
char *err_msg = NULL;
char sqlite_database[FILENAME_MAX + 1];
int rc;
- snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
+ if (likely(!memory))
+ snprintfz(sqlite_database, FILENAME_MAX, "%s/netdata-meta.db", netdata_configured_cache_dir);
+ else
+ strcpy(sqlite_database, ":memory:");
+
rc = sqlite3_open(sqlite_database, &db_meta);
if (rc != SQLITE_OK) {
error_report("Failed to initialize database at %s, due to \"%s\"", sqlite_database, sqlite3_errstr(rc));
@@ -390,6 +401,40 @@ int sql_init_database(db_check_action_type_t rebuild)
info("SQLite database %s initialization", sqlite_database);
+ char buf[1024 + 1] = "";
+ const char *list[2] = { buf, NULL };
+
+ // https://www.sqlite.org/pragma.html#pragma_auto_vacuum
+ // PRAGMA schema.auto_vacuum = 0 | NONE | 1 | FULL | 2 | INCREMENTAL;
+ snprintfz(buf, 1024, "PRAGMA auto_vacuum=%s;", config_get(CONFIG_SECTION_SQLITE, "auto vacuum", "INCREMENTAL"));
+ if(init_database_batch(rebuild, 0, list)) return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_synchronous
+ // PRAGMA schema.synchronous = 0 | OFF | 1 | NORMAL | 2 | FULL | 3 | EXTRA;
+ snprintfz(buf, 1024, "PRAGMA synchronous=%s;", config_get(CONFIG_SECTION_SQLITE, "synchronous", "NORMAL"));
+ if(init_database_batch(rebuild, 0, list)) return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_journal_mode
+ // PRAGMA schema.journal_mode = DELETE | TRUNCATE | PERSIST | MEMORY | WAL | OFF
+ snprintfz(buf, 1024, "PRAGMA journal_mode=%s;", config_get(CONFIG_SECTION_SQLITE, "journal mode", "WAL"));
+ if(init_database_batch(rebuild, 0, list)) return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_temp_store
+ // PRAGMA temp_store = 0 | DEFAULT | 1 | FILE | 2 | MEMORY;
+ snprintfz(buf, 1024, "PRAGMA temp_store=%s;", config_get(CONFIG_SECTION_SQLITE, "temp store", "MEMORY"));
+ if(init_database_batch(rebuild, 0, list)) return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_journal_size_limit
+ // PRAGMA schema.journal_size_limit = N ;
+ snprintfz(buf, 1024, "PRAGMA journal_size_limit=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "journal size limit", 16777216));
+ if(init_database_batch(rebuild, 0, list)) return 1;
+
+ // https://www.sqlite.org/pragma.html#pragma_cache_size
+ // PRAGMA schema.cache_size = pages;
+ // PRAGMA schema.cache_size = -kibibytes;
+ snprintfz(buf, 1024, "PRAGMA cache_size=%lld;", config_get_number(CONFIG_SECTION_SQLITE, "cache size", -2000));
+ if(init_database_batch(rebuild, 0, list)) return 1;
+
if (init_database_batch(rebuild, 0, &database_config[0]))
return 1;
@@ -1160,8 +1205,24 @@ failed:
return;
}
+void free_temporary_host(RRDHOST *host)
+{
+ if (host) {
+ freez(host->hostname);
+ freez((char *)host->os);
+ freez((char *)host->tags);
+ freez((char *)host->timezone);
+ freez(host->program_name);
+ freez(host->program_version);
+ freez(host->registry_hostname);
+ freez(host->system_info);
+ freez(host);
+ }
+}
+
#define SELECT_HOST "select host_id, registry_hostname, update_every, os, timezone, tags from host where hostname = @hostname order by rowid desc;"
-#define SELECT_HOST_BY_UUID "select host_id, registry_hostname, update_every, os, timezone, tags from host where host_id = @host_id ;"
+#define SELECT_HOST_BY_UUID "select h.host_id, h.registry_hostname, h.update_every, h.os, h.timezone, h.tags from host h, node_instance ni " \
+ "where (ni.host_id = @host_id or ni.node_id = @host_id) AND ni.host_id = h.host_id;"
RRDHOST *sql_create_host_by_uuid(char *hostname)
{
@@ -1229,8 +1290,6 @@ failed:
return host;
}
-#define SQL_MAX_RETRY 100
-
void db_execute(const char *cmd)
{
int rc;
@@ -1430,13 +1489,13 @@ int find_dimension_first_last_t(char *machine_guid, char *chart_id, char *dim_id
}
#ifdef ENABLE_DBENGINE
-static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
+static RRDDIM *create_rrdim_entry(ONEWAYALLOC *owa, RRDSET *st, char *id, char *name, uuid_t *metric_uuid)
{
- RRDDIM *rd = callocz(1, sizeof(*rd));
+ RRDDIM *rd = onewayalloc_callocz(owa, 1, sizeof(*rd));
rd->rrdset = st;
rd->last_stored_value = NAN;
rrddim_flag_set(rd, RRDDIM_FLAG_NONE);
- rd->state = mallocz(sizeof(*rd->state));
+ rd->state = onewayalloc_mallocz(owa, sizeof(*rd->state));
rd->rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;
rd->state->query_ops.init = rrdeng_load_metric_init;
rd->state->query_ops.next_metric = rrdeng_load_metric_next;
@@ -1444,11 +1503,11 @@ static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metr
rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
- rd->state->rrdeng_uuid = mallocz(sizeof(uuid_t));
+ rd->state->rrdeng_uuid = onewayalloc_mallocz(owa, sizeof(uuid_t));
uuid_copy(*rd->state->rrdeng_uuid, *metric_uuid);
uuid_copy(rd->state->metric_uuid, *metric_uuid);
- rd->id = strdupz(id);
- rd->name = strdupz(name);
+ rd->id = onewayalloc_strdupz(owa, id);
+ rd->name = onewayalloc_strdupz(owa, name);
return rd;
}
#endif
@@ -1465,7 +1524,7 @@ static RRDDIM *create_rrdim_entry(RRDSET *st, char *id, char *name, uuid_t *metr
"where d.chart_id = c.chart_id and c.host_id = h.host_id and c.host_id = @host_id and c.type||'.'||c.id = @chart " \
"order by c.chart_id asc, c.type||'.'||c.id desc;"
-void sql_build_context_param_list(struct context_param **param_list, RRDHOST *host, char *context, char *chart)
+void sql_build_context_param_list(ONEWAYALLOC *owa, struct context_param **param_list, RRDHOST *host, char *context, char *chart)
{
#ifdef ENABLE_DBENGINE
int rc;
@@ -1474,7 +1533,7 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
return;
if (unlikely(!(*param_list))) {
- *param_list = mallocz(sizeof(struct context_param));
+ *param_list = onewayalloc_mallocz(owa, sizeof(struct context_param));
(*param_list)->first_entry_t = LONG_MAX;
(*param_list)->last_entry_t = 0;
(*param_list)->rd = NULL;
@@ -1523,21 +1582,21 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
if (!st || uuid_compare(*(uuid_t *)sqlite3_column_blob(res, 7), chart_id)) {
if (unlikely(st && !st->counter)) {
- freez(st->context);
- freez((char *) st->name);
- freez(st);
+ onewayalloc_freez(owa, st->context);
+ onewayalloc_freez(owa, (char *) st->name);
+ onewayalloc_freez(owa, st);
}
- st = callocz(1, sizeof(*st));
+ st = onewayalloc_callocz(owa, 1, sizeof(*st));
char n[RRD_ID_LENGTH_MAX + 1];
snprintfz(
n, RRD_ID_LENGTH_MAX, "%s.%s", (char *)sqlite3_column_text(res, 4),
(char *)sqlite3_column_text(res, 3));
- st->name = strdupz(n);
+ st->name = onewayalloc_strdupz(owa, n);
st->update_every = sqlite3_column_int(res, 6);
st->counter = 0;
if (chart) {
- st->context = strdupz((char *)sqlite3_column_text(res, 8));
+ st->context = onewayalloc_strdupz(owa, (char *)sqlite3_column_text(res, 8));
strncpyz(st->id, chart, RRD_ID_LENGTH_MAX);
}
uuid_copy(chart_id, *(uuid_t *)sqlite3_column_blob(res, 7));
@@ -1553,7 +1612,7 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
st->counter++;
st->last_entry_t = MAX(st->last_entry_t, (*param_list)->last_entry_t);
- RRDDIM *rd = create_rrdim_entry(st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
+ RRDDIM *rd = create_rrdim_entry(owa, st, (char *)sqlite3_column_text(res, 1), (char *)sqlite3_column_text(res, 2), &rrdeng_uuid);
if (sqlite3_column_int(res, 9) == 1)
rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
rd->next = (*param_list)->rd;
@@ -1561,13 +1620,13 @@ void sql_build_context_param_list(struct context_param **param_list, RRDHOST *ho
}
if (st) {
if (!st->counter) {
- freez(st->context);
- freez((char *)st->name);
- freez(st);
+ onewayalloc_freez(owa,st->context);
+ onewayalloc_freez(owa,(char *)st->name);
+ onewayalloc_freez(owa,st);
}
else
if (!st->context && context)
- st->context = strdupz(context);
+ st->context = onewayalloc_strdupz(owa,context);
}
failed:
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-03 03:56:55 +0000