diff options
Diffstat (limited to 'database/engine/journalfile.h')
| -rw-r--r-- | database/engine/journalfile.h | 150 |
1 files changed, 130 insertions, 20 deletions
diff --git a/database/engine/journalfile.h b/database/engine/journalfile.h index 011c5065f..5fbcc90fa 100644 --- a/database/engine/journalfile.h +++ b/database/engine/journalfile.h @@ -13,37 +13,147 @@ struct rrdengine_journalfile; #define WALFILE_PREFIX "journalfile-" #define WALFILE_EXTENSION ".njf" +#define WALFILE_EXTENSION_V2 ".njfv2" +#define is_descr_journal_v2(descr) ((descr)->extent_entry != NULL) + +typedef enum __attribute__ ((__packed__)) { + JOURNALFILE_FLAG_IS_AVAILABLE = (1 << 0), + JOURNALFILE_FLAG_IS_MOUNTED = (1 << 1), + JOURNALFILE_FLAG_MOUNTED_FOR_RETENTION = (1 << 2), +} JOURNALFILE_FLAGS; /* only one event loop is supported for now */ struct rrdengine_journalfile { + struct { + SPINLOCK spinlock; + void *data; // MMAPed file of journal v2 + uint32_t size; // Total file size mapped + int fd; + } mmap; + + struct { + SPINLOCK spinlock; + JOURNALFILE_FLAGS flags; + int32_t refcount; + time_t first_time_s; + time_t last_time_s; + time_t not_needed_since_s; + } v2; + + struct { + SPINLOCK spinlock; + uint64_t pos; + } unsafe; + uv_file file; - uint64_t pos; - void *data; struct rrdengine_datafile *datafile; }; -/* only one event loop is supported for now */ -struct transaction_commit_log { - uint64_t transaction_id; +static inline uint64_t journalfile_current_size(struct rrdengine_journalfile *journalfile) { + netdata_spinlock_lock(&journalfile->unsafe.spinlock); + uint64_t size = journalfile->unsafe.pos; + netdata_spinlock_unlock(&journalfile->unsafe.spinlock); + return size; +} + +// Journal v2 structures + +#define JOURVAL_V2_MAGIC (0x01221019) +#define JOURVAL_V2_REBUILD_MAGIC (0x00221019) +#define JOURVAL_V2_SKIP_MAGIC (0x02221019) + +struct journal_v2_block_trailer { + union { + uint8_t checksum[CHECKSUM_SZ]; /* CRC32 */ + uint32_t crc; + }; +}; + +// Journal V2 +// 28 bytes +struct journal_page_header { + union { + uint8_t checksum[CHECKSUM_SZ]; // CRC check + uint32_t crc; + }; + uint32_t uuid_offset; // Points back to the UUID list which should point here (UUIDs should much) + uint32_t entries; // Entries + uuid_t uuid; // Which UUID this is +}; - /* outstanding transaction buffer */ - void *buf; - unsigned buf_pos; - unsigned buf_size; +// 20 bytes +struct journal_page_list { + uint32_t delta_start_s; // relative to the start time of journal + uint32_t delta_end_s; // relative to delta_start + uint32_t extent_index; // Index to the extent (extent list) (bytes from BASE) + uint32_t update_every_s; + uint16_t page_length; + uint8_t type; }; -void generate_journalfilepath(struct rrdengine_datafile *datafile, char *str, size_t maxlen); -void journalfile_init(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); -void *wal_get_transaction_buffer(struct rrdengine_worker_config* wc, unsigned size); -void wal_flush_transaction_buffer(struct rrdengine_worker_config* wc); -int close_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); -int unlink_journal_file(struct rrdengine_journalfile *journalfile); -int destroy_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); -int create_journal_file(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); -int load_journal_file(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile, - struct rrdengine_datafile *datafile); -void init_commit_log(struct rrdengine_instance *ctx); +// UUID_LIST +// 32 bytes +struct journal_metric_list { + uuid_t uuid; + uint32_t entries; // Number of entries + uint32_t page_offset; // OFFSET that contains entries * struct( journal_page_list ) + uint32_t delta_start_s; // Min time of metric + uint32_t delta_end_s; // Max time of metric (to be used to populate page_index) +}; + +// 16 bytes +struct journal_extent_list { + uint64_t datafile_offset; // Datafile offset to find the extent + uint32_t datafile_size; // Size of the extent + uint16_t file_index; // which file index is this datafile[index] + uint8_t pages; // number of pages (not all are necesssarily valid) +}; + +// 72 bytes +struct journal_v2_header { + uint32_t magic; + usec_t start_time_ut; // Min start time of journal + usec_t end_time_ut; // Maximum end time of journal + uint32_t extent_count; // Count of extents + uint32_t extent_offset; + uint32_t metric_count; // Count of metrics (unique UUIDS) + uint32_t metric_offset; + uint32_t page_count; // Total count of pages (descriptors @ time) + uint32_t page_offset; + uint32_t extent_trailer_offset; // CRC for entent list + uint32_t metric_trailer_offset; // CRC for metric list + uint32_t journal_v1_file_size; // This is the original journal file + uint32_t journal_v2_file_size; // This is the total file size + void *data; // Used when building the index +}; + +#define JOURNAL_V2_HEADER_PADDING_SZ (RRDENG_BLOCK_SIZE - (sizeof(struct journal_v2_header))) + +struct wal; + +void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen); +void journalfile_v2_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen); +struct rrdengine_journalfile *journalfile_alloc_and_init(struct rrdengine_datafile *datafile); +void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengine_datafile *datafile, struct wal *wal, uv_loop_t *loop); +int journalfile_close(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); +int journalfile_unlink(struct rrdengine_journalfile *journalfile); +int journalfile_destroy_unsafe(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); +int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile); +int journalfile_load(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile, + struct rrdengine_datafile *datafile); +void journalfile_v2_populate_retention_to_mrg(struct rrdengine_instance *ctx, struct rrdengine_journalfile *journalfile); + +void journalfile_migrate_to_v2_callback(Word_t section, unsigned datafile_fileno __maybe_unused, uint8_t type __maybe_unused, + Pvoid_t JudyL_metrics, Pvoid_t JudyL_extents_pos, + size_t number_of_extents, size_t number_of_metrics, size_t number_of_pages, void *user_data); + +bool journalfile_v2_data_available(struct rrdengine_journalfile *journalfile); +size_t journalfile_v2_data_size_get(struct rrdengine_journalfile *journalfile); +void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd, void *journal_data, uint32_t journal_data_size); +struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s); +void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile); +void journalfile_v2_data_unmount_cleanup(time_t now_s); #endif /* NETDATA_JOURNALFILE_H */
\ No newline at end of file |