diff options
Diffstat (limited to 'src/lib-index/mail-index-private.h')
| -rw-r--r-- | src/lib-index/mail-index-private.h | 437 |
1 files changed, 437 insertions, 0 deletions
diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h new file mode 100644 index 0000000..26f7e39 --- /dev/null +++ b/src/lib-index/mail-index-private.h @@ -0,0 +1,437 @@ +#ifndef MAIL_INDEX_PRIVATE_H +#define MAIL_INDEX_PRIVATE_H + +#include "file-lock.h" +#include "mail-index.h" +#include "mail-index-util.h" +#include "mail-index-view-private.h" +#include "mail-index-transaction-private.h" + +#include <sys/stat.h> + +struct mail_transaction_header; +struct mail_transaction_log_view; +struct mail_index_sync_map_ctx; + +/* How large index files to mmap() instead of reading to memory. */ +#define MAIL_INDEX_MMAP_MIN_SIZE (1024*64) +/* How many times to retry opening index files if read/fstat returns ESTALE. + This happens with NFS when the file has been deleted (ie. index file was + rewritten by another computer than us). */ +#define MAIL_INDEX_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT +/* Large extension header sizes are probably caused by file corruption, so + try to catch them by limiting the header size. */ +#define MAIL_INDEX_EXT_HEADER_MAX_SIZE (1024*1024*16-1) + +#define MAIL_INDEX_IS_IN_MEMORY(index) \ + ((index)->dir == NULL) + +#define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \ + ((map)->rec_map->mmap_base == NULL) + +#define MAIL_INDEX_MAP_IDX(map, idx) \ + ((struct mail_index_record *) \ + PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size)) +#define MAIL_INDEX_REC_AT_SEQ(map, seq) \ + ((struct mail_index_record *) \ + PTR_OFFSET((map)->rec_map->records, ((seq)-1) * (map)->hdr.record_size)) + +#define MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u) \ + ((((u)->add_flags | (u)->remove_flags) & MAIL_INDEX_FLAGS_MASK) == 0 && \ + (u)->modseq_inc_flag == 0) + +#define MAIL_INDEX_EXT_KEYWORDS "keywords" +#define MAIL_INDEX_EXT_NAME_MAX_LENGTH 64 + +typedef int mail_index_expunge_handler_t(struct mail_index_sync_map_ctx *ctx, + const void *data, void **sync_context); + +#define MAIL_INDEX_HEADER_SIZE_ALIGN(size) \ + (((size) + 7) & ~7U) + +/* In-memory copy of struct mail_index_ext_header */ +struct mail_index_ext { + const char *name; + uint32_t index_idx; /* index ext_id */ + uint32_t reset_id; + uint32_t ext_offset; /* points to beginning of mail_index_ext_header */ + uint32_t hdr_offset; /* points to mail_index_ext_header.data[] */ + uint32_t hdr_size; /* size of mail_index_ext_header.data[] */ + uint16_t record_offset; + uint16_t record_size; + uint16_t record_align; +}; + +struct mail_index_ext_header { + /* Size of data[], i.e. the extension size in header */ + uint32_t hdr_size; + /* If reset_id changes, all of the extension record data is + invalidated. For example with cache files reset_id must match the + cache header's file_seq or the cache offsets aren't valid. */ + uint32_t reset_id; + /* Offset of this extension in struct mail_index_record. */ + uint16_t record_offset; + /* Size of this extension in struct mail_index_record. */ + uint16_t record_size; + /* Required alignment of this extension in struct mail_index_record. + It's expected that record_offset is correctly aligned. This is used + only when rearranging fields due to adding/removing other + extensions. */ + uint16_t record_align; + /* Size of name[], which contains the extension's unique name. */ + uint16_t name_size; + /* unsigned char name[name_size]; */ + /* Extension header data, if any. This starts from the next 64-bit + aligned offset after name[]. */ + /* unsigned char data[hdr_size]; */ +}; + +struct mail_index_keyword_header { + uint32_t keywords_count; + /* struct mail_index_keyword_header_rec[] */ + /* char name[][] */ +}; + +struct mail_index_keyword_header_rec { + uint32_t unused; /* for backwards compatibility */ + uint32_t name_offset; /* relative to beginning of name[] */ +}; + +enum mail_index_sync_handler_type { + MAIL_INDEX_SYNC_HANDLER_FILE = 0x01, + MAIL_INDEX_SYNC_HANDLER_HEAD = 0x02, + MAIL_INDEX_SYNC_HANDLER_VIEW = 0x04 +}; + +struct mail_index_registered_ext { + const char *name; + uint32_t index_idx; /* index ext_id */ + uint32_t hdr_size; /* size of mail_index_ext_header.data[] */ + uint16_t record_size; + uint16_t record_align; + + mail_index_expunge_handler_t *expunge_handler; +}; + +struct mail_index_modseq_header { + /* highest used modseq */ + uint64_t highest_modseq; + /* last tracked log file position */ + uint32_t log_seq; + uint32_t log_offset; +}; + +struct mail_index_record_map { + ARRAY(struct mail_index_map *) maps; + + void *mmap_base; + size_t mmap_size, mmap_used_size; + + buffer_t *buffer; + + void *records; /* struct mail_index_record[] */ + unsigned int records_count; + + struct mail_index_map_modseq *modseq; + uint32_t last_appended_uid; +}; + +#define MAIL_INDEX_MAP_HDR_OFFSET(map, hdr_offset) \ + CONST_PTR_OFFSET((map)->hdr_copy_buf->data, hdr_offset) +struct mail_index_map { + struct mail_index *index; + int refcount; + + /* Copy of the base header for convenience. Note that base_header_size + may be smaller or larger than this struct. If it's smaller, the last + fields in the struct are filled with zeroes. */ + struct mail_index_header hdr; + /* Copy of the full header. */ + buffer_t *hdr_copy_buf; + + pool_t extension_pool; + ARRAY(struct mail_index_ext) extensions; + ARRAY(uint32_t) ext_id_map; /* index -> file */ + + ARRAY(unsigned int) keyword_idx_map; /* file -> index */ + + struct mail_index_modseq_header modseq_hdr_snapshot; + + struct mail_index_record_map *rec_map; +}; + +struct mail_index_module_register { + unsigned int id; +}; + +union mail_index_module_context { + struct mail_index_module_register *reg; +}; + +struct mail_index_settings { + /* Directory path for .cache file. Set via + mail_index_set_cache_dir(). */ + char *cache_dir; + + /* fsyncing behavior. Set via mail_index_set_fsync_mode(). */ + enum fsync_mode fsync_mode; + enum mail_index_fsync_mask fsync_mask; + + /* Index file permissions. Set via mail_index_set_permissions(). */ + mode_t mode; + gid_t gid; + char *gid_origin; + + /* Lock settings. Set via mail_index_set_lock_method(). */ + enum file_lock_method lock_method; + unsigned int max_lock_timeout_secs; + + /* Initial extension added to newly created indexes. Set via + mail_index_set_ext_init_data(). */ + uint32_t ext_hdr_init_id; + void *ext_hdr_init_data; +}; + +struct mail_index_error { + /* Human-readable error text */ + char *text; + + /* Error happened because there's no disk space, i.e. syscall failed + with ENOSPC or EDQUOT. */ + bool nodiskspace:1; +}; + +struct mail_index { + /* Directory path for the index, or NULL for in-memory indexes. */ + char *dir; + /* Filename prefix for the index, e.g. "dovecot.index." */ + char *prefix; + struct event *event; + enum mail_index_open_flags flags; + struct mail_index_settings set; + struct mail_index_optimization_settings optimization_set; + + struct mail_cache *cache; + struct mail_transaction_log *log; + + char *filepath; + int fd; + /* Linked list of currently opened views */ + struct mail_index_view *views; + /* Latest map */ + struct mail_index_map *map; + + /* ID number that permanently identifies the index. This is stored in + the index files' headers. If the indexids suddenly changes, it means + that the index has been completely recreated and needs to be + reopened (e.g. the mailbox was deleted and recreated while it + was open). */ + uint32_t indexid; + /* Views initially use this same ID value. This ID is incremented + whenever something unexpected happens to the index that prevents + syncing existing views. When the view's inconsistency_id doesn't + match this one, the view is marked as inconsistent. */ + unsigned int inconsistency_id; + /* How many times this index has been opened with mail_index_open(). */ + unsigned int open_count; + + /* These contain the log_file_seq and log_file_tail_offset that exists + in dovecot.index file's header. These are used to figure out if it's + time to rewrite the dovecot.index file. Note that these aren't + available in index->map->hdr, because it gets updated when + transaction log file is read. */ + uint32_t main_index_hdr_log_file_seq; + uint32_t main_index_hdr_log_file_tail_offset; + + /* log file which last updated index_deleted */ + uint32_t index_delete_changed_file_seq; + + /* transaction log head seq/offset when we last fscked */ + uint32_t fsck_log_head_file_seq; + uoff_t fsck_log_head_file_offset; + + /* syncing will update this if non-NULL */ + struct mail_index_transaction_commit_result *sync_commit_result; + /* Delayed log2_rotate_time update to mail_index_header. This is set + and unset within the same sync. */ + uint32_t hdr_log2_rotate_time_delayed_update; + + /* Registered extensions */ + pool_t extension_pool; + ARRAY(struct mail_index_registered_ext) extensions; + + /* All keywords that have ever been used in this index. Keywords are + only added here, never removed. */ + pool_t keywords_pool; + ARRAY_TYPE(keywords) keywords; + HASH_TABLE(char *, void *) keywords_hash; /* name -> unsigned int idx */ + + /* Registered extension IDs */ + uint32_t keywords_ext_id; + uint32_t modseq_ext_id; + + /* Module-specific contexts. */ + ARRAY(union mail_index_module_context *) module_contexts; + + /* Last error returned by mail_index_get_error_message(). + Cleared by mail_index_reset_error(). */ + struct mail_index_error last_error; + /* Timestamp when mmap() failure was logged the last time. This is used + to prevent logging the same error too rapidly. This could happen + e.g. if mmap()ing a large cache file that exceeeds process's + VSZ limit. */ + time_t last_mmap_error_time; + /* If non-NULL, dovecot.index should be recreated as soon as possible. + The reason for why the recreation is wanted is stored as human- + readable text. */ + char *need_recreate; + + /* Mapping has noticed non-external MAIL_TRANSACTION_INDEX_DELETED + record, i.e. a request to mark the index deleted. The next sync + will finish the deletion by writing external + MAIL_TRANSACTION_INDEX_DELETED record. */ + bool index_delete_requested:1; + /* Mapping has noticed external MAIL_TRANSACTION_INDEX_DELETED record, + or index was unexpectedly deleted under us. No more changes are + allowed to the index, except undeletion. */ + bool index_deleted:1; + /* .log is locked for syncing. This is the main exclusive lock for + indexes. */ + bool log_sync_locked:1; + /* Main index or .log couldn't be opened read-write */ + bool readonly:1; + /* mail_index_map() is running */ + bool mapping:1; + /* mail_index_sync_*() is running */ + bool syncing:1; + /* Mapping has read more from .log than it preferred. Use + mail_index_base_optimization_settings.rewrite_min_log_bytes the next + time when checking if index needs a rewrite. */ + bool index_min_write:1; + /* mail_index_modseq_enable() has been called. Track per-flag + modseq numbers in memory (global modseqs are tracked anyway). */ + bool modseqs_enabled:1; + /* mail_index_open() is creating new index files */ + bool initial_create:1; + /* TRUE after mail_index_map() has succeeded */ + bool initial_mapped:1; + /* The next mail_index_map() must reopen the main index, because the + currently opened one is too old. */ + bool reopen_main_index:1; + /* Index has been fsck'd, but mail_index_reset_fscked() hasn't been + called yet. */ + bool fscked:1; +}; + +extern struct mail_index_module_register mail_index_module_register; +extern struct event_category event_category_mail_index; + +/* Add/replace expunge handler for specified extension. */ +void mail_index_register_expunge_handler(struct mail_index *index, + uint32_t ext_id, + mail_index_expunge_handler_t *callback); +void mail_index_unregister_expunge_handler(struct mail_index *index, + uint32_t ext_id); + +int mail_index_create_tmp_file(struct mail_index *index, + const char *path_prefix, const char **path_r); + +int mail_index_try_open_only(struct mail_index *index); +void mail_index_close_file(struct mail_index *index); +/* Returns 1 if index was successfully (re-)opened, 0 if the index no longer + exists, -1 if I/O error. If 1 is returned, reopened_r=TRUE if a new index + was actually reopened (or if index wasn't even open before this call). */ +int mail_index_reopen_if_changed(struct mail_index *index, bool *reopened_r, + const char **reason_r); +/* Update/rewrite the main index file from index->map */ +void mail_index_write(struct mail_index *index, bool want_rotate, + const char *reason); + +void mail_index_flush_read_cache(struct mail_index *index, const char *path, + int fd, bool locked); + +int mail_index_lock_fd(struct mail_index *index, const char *path, int fd, + int lock_type, unsigned int timeout_secs, + struct file_lock **lock_r); + +/* Allocate a new empty map. */ +struct mail_index_map *mail_index_map_alloc(struct mail_index *index); +/* Replace index->map with the latest index changes. This may reopen the index + file and/or it may read the latest changes from transaction log. The log is + read up to EOF, but non-synced expunges are skipped. + + If we mmap()ed the index file, the map is returned locked. + + Returns 1 = ok, 0 = corrupted, -1 = error. */ +int mail_index_map(struct mail_index *index, + enum mail_index_sync_handler_type type); +/* Unreference given mapping and unmap it if it's dropped to zero. */ +void mail_index_unmap(struct mail_index_map **map); +/* Clone a map. It still points to the original rec_map. */ +struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map); +/* Make sure the map has its own private rec_map, cloning it if necessary. */ +void mail_index_record_map_move_to_private(struct mail_index_map *map); +/* If map points to mmap()ed index, copy it to the memory. */ +void mail_index_map_move_to_memory(struct mail_index_map *map); + +void mail_index_fchown(struct mail_index *index, int fd, const char *path); + +bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name, + uint32_t *idx_r); +bool mail_index_ext_name_is_valid(const char *name); +uint32_t +mail_index_map_register_ext(struct mail_index_map *map, + const char *name, uint32_t ext_offset, + const struct mail_index_ext_header *ext_hdr); +bool mail_index_map_get_ext_idx(struct mail_index_map *map, + uint32_t ext_id, uint32_t *idx_r); +const struct mail_index_ext * +mail_index_view_get_ext(struct mail_index_view *view, uint32_t ext_id); + +void mail_index_map_lookup_seq_range(struct mail_index_map *map, + uint32_t first_uid, uint32_t last_uid, + uint32_t *first_seq_r, + uint32_t *last_seq_r); + +/* Returns 1 on success, 0 on non-critical errors we want to silently fix, + -1 if map isn't usable. The caller is responsible for logging the errors + if -1 is returned. */ +int mail_index_map_check_header(struct mail_index_map *map, + const char **error_r); +/* Returns 1 if header is usable, 0 or -1 if not. The caller should log an + error if -1 is returned, but not if 0 is returned. */ +bool mail_index_check_header_compat(struct mail_index *index, + const struct mail_index_header *hdr, + uoff_t file_size, const char **error_r); +int mail_index_map_parse_extensions(struct mail_index_map *map); +int mail_index_map_parse_keywords(struct mail_index_map *map); + +void mail_index_map_init_extbufs(struct mail_index_map *map, + unsigned int initial_count); +int mail_index_map_ext_get_next(struct mail_index_map *map, + unsigned int *offset, + const struct mail_index_ext_header **ext_hdr_r, + const char **name_r); +int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr, + const struct mail_index_ext_header *ext_hdr, + const char *name, const char **error_r); +unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len); + +void mail_index_fsck_locked(struct mail_index *index); + +/* Log an error and set it as the index's current error that is available + with mail_index_get_error_message(). */ +void mail_index_set_error(struct mail_index *index, const char *fmt, ...) + ATTR_FORMAT(2, 3) ATTR_COLD; +/* Same as mail_index_set_error(), but don't log the error. */ +void mail_index_set_error_nolog(struct mail_index *index, const char *str) + ATTR_COLD; +/* "%s failed with index file %s: %m" */ +void mail_index_set_syscall_error(struct mail_index *index, + const char *function) ATTR_COLD; +/* "%s failed with file %s: %m" */ +void mail_index_file_set_syscall_error(struct mail_index *index, + const char *filepath, + const char *function) ATTR_COLD; + +#endif |