1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
|
// SPDX-License-Identifier: GPL-3.0-or-later
#ifndef NETDATA_JOURNALFILE_H
#define NETDATA_JOURNALFILE_H
#include "rrdengine.h"
/* Forward declarations */
struct rrdengine_instance;
struct rrdengine_datafile;
struct rrdengine_journalfile;
#define WALFILE_PREFIX "journalfile-"
#define WALFILE_EXTENSION ".njf"
#define WALFILE_EXTENSION_V2 ".njfv2"
#define is_descr_journal_v2(descr) ((descr)->extent_entry != NULL)
typedef enum __attribute__ ((__packed__)) {
JOURNALFILE_FLAG_IS_AVAILABLE = (1 << 0),
JOURNALFILE_FLAG_IS_MOUNTED = (1 << 1),
JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION = (1 << 2),
JOURNALFILE_FLAG_METRIC_CRC_CHECK = (1 << 3),
} JOURNALFILE_FLAGS;
/* only one event loop is supported for now */
struct rrdengine_journalfile {
struct {
SPINLOCK spinlock;
void *data; // MMAPed file of journal v2
uint32_t size; // Total file size mapped
int fd;
} mmap;
struct {
SPINLOCK spinlock;
JOURNALFILE_FLAGS flags;
int32_t refcount;
time_t first_time_s;
time_t last_time_s;
time_t not_needed_since_s;
uint32_t size_of_directory;
} v2;
struct {
Word_t indexed_as;
} njfv2idx;
struct {
SPINLOCK spinlock;
uint64_t pos;
} unsafe;
uv_file file;
struct rrdengine_datafile *datafile;
};
static inline uint64_t journalfile_current_size(struct rrdengine_journalfile *journalfile) {
spinlock_lock(&journalfile->unsafe.spinlock);
uint64_t size = journalfile->unsafe.pos;
spinlock_unlock(&journalfile->unsafe.spinlock);
return size;
}
// Journal v2 structures
#define JOURVAL_V2_MAGIC (0x01230317)
#define JOURVAL_V2_REBUILD_MAGIC (0x00230317)
#define JOURVAL_V2_SKIP_MAGIC (0x02230317)
struct journal_v2_block_trailer {
union {
uint8_t checksum[CHECKSUM_SZ]; /* CRC32 */
uint32_t crc;
};
};
// Journal V2
// 28 bytes
struct journal_page_header {
union {
uint8_t checksum[CHECKSUM_SZ]; // CRC check
uint32_t crc;
};
uint32_t uuid_offset; // Points back to the UUID list which should point here (UUIDs should much)
uint32_t entries; // Entries
nd_uuid_t uuid; // Which UUID this is
};
// 20 bytes
struct journal_page_list {
uint32_t delta_start_s; // relative to the start time of journal
uint32_t delta_end_s; // relative to delta_start
uint32_t extent_index; // Index to the extent (extent list) (bytes from BASE)
uint32_t update_every_s;
uint16_t page_length;
uint8_t type;
};
// UUID_LIST
// 36 bytes
struct journal_metric_list {
nd_uuid_t uuid;
uint32_t entries; // Number of entries
uint32_t page_offset; // OFFSET that contains entries * struct( journal_page_list )
uint32_t delta_start_s; // Min time of metric
uint32_t delta_end_s; // Max time of metric (to be used to populate page_index)
uint32_t update_every_s; // Last update every for this metric in this journal (last page collected)
};
// 16 bytes
struct journal_extent_list {
uint64_t datafile_offset; // Datafile offset to find the extent
uint32_t datafile_size; // Size of the extent
uint16_t file_index; // which file index is this datafile[index]
uint8_t pages; // number of pages (not all are necesssarily valid)
};
// 72 bytes
struct journal_v2_header {
uint32_t magic;
usec_t start_time_ut; // Min start time of journal
usec_t end_time_ut; // Maximum end time of journal
uint32_t extent_count; // Count of extents
uint32_t extent_offset;
uint32_t metric_count; // Count of metrics (unique UUIDS)
uint32_t metric_offset;
uint32_t page_count; // Total count of pages (descriptors @ time)
uint32_t page_offset;
uint32_t extent_trailer_offset; // CRC for entent list
uint32_t metric_trailer_offset; // CRC for metric list
uint32_t journal_v1_file_size; // This is the original journal file
uint32_t journal_v2_file_size; // This is the total file size
void *data; // Used when building the index
};
#define JOURNAL_V2_HEADER_PADDING_SZ (RRDENG_BLOCK_SIZE - (sizeof(struct journal_v2_header)))
struct wal;
void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen);
void journalfile_v2_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen);
struct rrdengine_journalfile *journalfile_alloc_and_init(struct rrdengine_datafile *datafile);
void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, struct wal *wal, uv_loop_t *loop);
int journalfile_close(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);
int journalfile_unlink(struct rrdengine_journalfile *journalfile);
int journalfile_destroy_unsafe(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);
int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile);
int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile,
struct rrdengine_datafile *datafile);
void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile);
void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno __maybe_unused, uint8_t type __maybe_unused,
Pvoid_t JudyL_metrics, Pvoid_t JudyL_extents_pos,
size_t number_of_extents, size_t number_of_metrics, size_t number_of_pages, void *user_data);
bool journalfile_v2_data_available(struct rrdengine_journalfile *journalfile);
size_t journalfile_v2_data_size_get(struct rrdengine_journalfile *journalfile);
void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd, void *journal_data, uint32_t journal_data_size);
struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s);
void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile);
void journalfile_v2_data_unmount_cleanup(time_t now_s);
typedef struct {
bool init;
Word_t last;
time_t wanted_start_time_s;
time_t wanted_end_time_s;
struct rrdengine_instance *ctx;
struct journal_v2_header *j2_header_acquired;
} NJFV2IDX_FIND_STATE;
struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJFV2IDX_FIND_STATE *s);
#endif /* NETDATA_JOURNALFILE_H */
|
