summaryrefslogtreecommitdiffstats
path: root/database/engine/rrdengineapi.h
blob: feb79b977638527a451f6b3ea19ee2e850ecd973 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
// SPDX-License-Identifier: GPL-3.0-or-later

#ifndef NETDATA_RRDENGINEAPI_H
#define NETDATA_RRDENGINEAPI_H

#include "rrdengine.h"

#define RRDENG_MIN_PAGE_CACHE_SIZE_MB (8)
#define RRDENG_MIN_DISK_SPACE_MB (64)

#define RRDENG_NR_STATS (38)

#define RRDENG_FD_BUDGET_PER_INSTANCE (50)

extern int db_engine_use_malloc;
extern int default_rrdeng_page_fetch_timeout;
extern int default_rrdeng_page_fetch_retries;
extern int default_rrdeng_page_cache_mb;
extern int db_engine_journal_indexing;
extern int db_engine_journal_check;
extern int default_rrdeng_disk_quota_mb;
extern int default_multidb_disk_quota_mb;
extern struct rrdengine_instance *multidb_ctx[RRD_STORAGE_TIERS];
extern size_t page_type_size[];
extern size_t tier_page_size[];

#define CTX_POINT_SIZE_BYTES(ctx) page_type_size[(ctx)->config.page_type]

void rrdeng_generate_legacy_uuid(const char *dim_id, const char *chart_id, uuid_t *ret_uuid);
void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid,
                                                    uuid_t *ret_uuid);


STORAGE_METRIC_HANDLE *rrdeng_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *db_instance);
STORAGE_METRIC_HANDLE *rrdeng_metric_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
void rrdeng_metric_release(STORAGE_METRIC_HANDLE *db_metric_handle);
STORAGE_METRIC_HANDLE *rrdeng_metric_dup(STORAGE_METRIC_HANDLE *db_metric_handle);

STORAGE_COLLECT_HANDLE *rrdeng_store_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, uint32_t update_every, STORAGE_METRICS_GROUP *smg);
void rrdeng_store_metric_flush_current_page(STORAGE_COLLECT_HANDLE *collection_handle);
void rrdeng_store_metric_change_collection_frequency(STORAGE_COLLECT_HANDLE *collection_handle, int update_every);
void rrdeng_store_metric_next(STORAGE_COLLECT_HANDLE *collection_handle, usec_t point_in_time_ut, NETDATA_DOUBLE n,
                                     NETDATA_DOUBLE min_value,
                                     NETDATA_DOUBLE max_value,
                                     uint16_t count,
                                     uint16_t anomaly_count,
                                     SN_FLAGS flags);
int rrdeng_store_metric_finalize(STORAGE_COLLECT_HANDLE *collection_handle);

void rrdeng_load_metric_init(STORAGE_METRIC_HANDLE *db_metric_handle, struct storage_engine_query_handle *rrddim_handle,
                                    time_t start_time_s, time_t end_time_s, STORAGE_PRIORITY priority);
STORAGE_POINT rrdeng_load_metric_next(struct storage_engine_query_handle *rrddim_handle);


int rrdeng_load_metric_is_finished(struct storage_engine_query_handle *rrddim_handle);
void rrdeng_load_metric_finalize(struct storage_engine_query_handle *rrddim_handle);
time_t rrdeng_metric_latest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
time_t rrdeng_metric_oldest_time(STORAGE_METRIC_HANDLE *db_metric_handle);
time_t rrdeng_load_align_to_optimal_before(struct storage_engine_query_handle *rrddim_handle);

void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);

/* must call once before using anything */
int rrdeng_init(struct rrdengine_instance **ctxp, const char *dbfiles_path,
                       unsigned disk_space_mb, size_t tier);

void rrdeng_readiness_wait(struct rrdengine_instance *ctx);
void rrdeng_exit_mode(struct rrdengine_instance *ctx);

int rrdeng_exit(struct rrdengine_instance *ctx);
void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
bool rrdeng_metric_retention_by_uuid(STORAGE_INSTANCE *db_instance, uuid_t *dim_uuid, time_t *first_entry_s, time_t *last_entry_s);

extern STORAGE_METRICS_GROUP *rrdeng_metrics_group_get(STORAGE_INSTANCE *db_instance, uuid_t *uuid);
extern void rrdeng_metrics_group_release(STORAGE_INSTANCE *db_instance, STORAGE_METRICS_GROUP *smg);

typedef struct rrdengine_size_statistics {
    size_t default_granularity_secs;

    size_t sizeof_datafile;
    size_t sizeof_page_in_cache;
    size_t sizeof_point_data;
    size_t sizeof_page_data;

    size_t pages_per_extent;

    size_t datafiles;
    size_t extents;
    size_t extents_pages;
    size_t points;
    size_t metrics;
    size_t metrics_pages;

    size_t extents_compressed_bytes;
    size_t pages_uncompressed_bytes;
    time_t pages_duration_secs;

    struct {
        size_t pages;
        size_t pages_uncompressed_bytes;
        time_t pages_duration_secs;
        size_t points;
    } page_types[256];

    size_t single_point_pages;

    time_t first_time_s;
    time_t last_time_s;

    size_t currently_collected_metrics;
    size_t estimated_concurrently_collected_metrics;

    size_t disk_space;
    size_t max_disk_space;

    time_t database_retention_secs;
    double average_compression_savings;
    double average_point_duration_secs;
    double average_metric_retention_secs;

    double ephemeral_metrics_per_day_percent;

    double average_page_size_bytes;
} RRDENG_SIZE_STATS;

struct rrdeng_cache_efficiency_stats {
    size_t queries;
    size_t queries_planned_with_gaps;
    size_t queries_executed_with_gaps;
    size_t queries_open;
    size_t queries_journal_v2;

    size_t currently_running_queries;

    // query planner output of the queries
    size_t pages_total;
    size_t pages_to_load_from_disk;
    size_t extents_loaded_from_disk;

    // pages metadata sources
    size_t pages_meta_source_main_cache;
    size_t pages_meta_source_open_cache;
    size_t pages_meta_source_journal_v2;

    // preloading
    size_t page_next_wait_failed;
    size_t page_next_wait_loaded;
    size_t page_next_nowait_failed;
    size_t page_next_nowait_loaded;

    // pages data sources
    size_t pages_data_source_main_cache;
    size_t pages_data_source_main_cache_at_pass4;
    size_t pages_data_source_disk;
    size_t pages_data_source_extent_cache;              // loaded by a cached extent

    // cache hits at different points
    size_t pages_load_ok_loaded_but_cache_hit_while_inserting; // found in cache while inserting it (conflict)

    // loading
    size_t pages_load_extent_merged;
    size_t pages_load_ok_uncompressed;
    size_t pages_load_ok_compressed;
    size_t pages_load_fail_invalid_page_in_extent;
    size_t pages_load_fail_cant_mmap_extent;
    size_t pages_load_fail_datafile_not_available;
    size_t pages_load_fail_unroutable;
    size_t pages_load_fail_not_found;
    size_t pages_load_fail_invalid_extent;
    size_t pages_load_fail_cancelled;

    // timings for query preparation
    size_t prep_time_to_route;
    size_t prep_time_in_main_cache_lookup;
    size_t prep_time_in_open_cache_lookup;
    size_t prep_time_in_journal_v2_lookup;
    size_t prep_time_in_pass4_lookup;

    // timings the query thread experiences
    size_t query_time_init;
    size_t query_time_wait_for_prep;
    size_t query_time_to_slow_disk_next_page;
    size_t query_time_to_fast_disk_next_page;
    size_t query_time_to_slow_preload_next_page;
    size_t query_time_to_fast_preload_next_page;

    // query issues
    size_t pages_zero_time_skipped;
    size_t pages_past_time_skipped;
    size_t pages_overlapping_skipped;
    size_t pages_invalid_size_skipped;
    size_t pages_invalid_update_every_fixed;
    size_t pages_invalid_entries_fixed;

    // database events
    size_t journal_v2_mapped;
    size_t journal_v2_unmapped;
    size_t datafile_creation_started;
    size_t datafile_deletion_started;
    size_t datafile_deletion_spin;
    size_t journal_v2_indexing_started;
    size_t metrics_retention_started;
};

struct rrdeng_buffer_sizes {
    size_t workers;
    size_t pdc;
    size_t wal;
    size_t descriptors;
    size_t xt_io;
    size_t xt_buf;
    size_t handles;
    size_t opcodes;
    size_t epdl;
    size_t deol;
    size_t pd;
    size_t pgc;
    size_t mrg;
#ifdef PDC_USE_JULYL
    size_t julyl;
#endif
};

struct rrdeng_buffer_sizes rrdeng_get_buffer_sizes(void);
struct rrdeng_cache_efficiency_stats rrdeng_get_cache_efficiency_stats(void);

RRDENG_SIZE_STATS rrdeng_size_statistics(struct rrdengine_instance *ctx);
size_t rrdeng_collectors_running(struct rrdengine_instance *ctx);
bool rrdeng_is_legacy(STORAGE_INSTANCE *db_instance);

#endif /* NETDATA_RRDENGINEAPI_H */