Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 437 insertions, 0 deletions

diff --git a/src/lib-index/mail-index-private.h b/src/lib-index/mail-index-private.h
new file mode 100644
index 0000000..26f7e39
--- /dev/null
+++ b/src/lib-index/mail-index-private.h
@@ -0,0 +1,437 @@
+#ifndef MAIL_INDEX_PRIVATE_H
+#define MAIL_INDEX_PRIVATE_H
+
+#include "file-lock.h"
+#include "mail-index.h"
+#include "mail-index-util.h"
+#include "mail-index-view-private.h"
+#include "mail-index-transaction-private.h"
+
+#include <sys/stat.h>
+
+struct mail_transaction_header;
+struct mail_transaction_log_view;
+struct mail_index_sync_map_ctx;
+
+/* How large index files to mmap() instead of reading to memory. */
+#define MAIL_INDEX_MMAP_MIN_SIZE (1024*64)
+/* How many times to retry opening index files if read/fstat returns ESTALE.
+   This happens with NFS when the file has been deleted (ie. index file was
+   rewritten by another computer than us). */
+#define MAIL_INDEX_ESTALE_RETRY_COUNT NFS_ESTALE_RETRY_COUNT
+/* Large extension header sizes are probably caused by file corruption, so
+   try to catch them by limiting the header size. */
+#define MAIL_INDEX_EXT_HEADER_MAX_SIZE (1024*1024*16-1)
+
+#define MAIL_INDEX_IS_IN_MEMORY(index) \
+	((index)->dir == NULL)
+
+#define MAIL_INDEX_MAP_IS_IN_MEMORY(map) \
+	((map)->rec_map->mmap_base == NULL)
+
+#define MAIL_INDEX_MAP_IDX(map, idx) \
+	((struct mail_index_record *) \
+	 PTR_OFFSET((map)->rec_map->records, (idx) * (map)->hdr.record_size))
+#define MAIL_INDEX_REC_AT_SEQ(map, seq)					\
+	((struct mail_index_record *)					\
+	 PTR_OFFSET((map)->rec_map->records, ((seq)-1) * (map)->hdr.record_size))
+
+#define MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(u) \
+	((((u)->add_flags | (u)->remove_flags) & MAIL_INDEX_FLAGS_MASK) == 0 && \
+	 (u)->modseq_inc_flag == 0)
+
+#define MAIL_INDEX_EXT_KEYWORDS "keywords"
+#define MAIL_INDEX_EXT_NAME_MAX_LENGTH 64
+
+typedef int mail_index_expunge_handler_t(struct mail_index_sync_map_ctx *ctx,
+					 const void *data, void **sync_context);
+
+#define MAIL_INDEX_HEADER_SIZE_ALIGN(size) \
+	(((size) + 7) & ~7U)
+
+/* In-memory copy of struct mail_index_ext_header */
+struct mail_index_ext {
+	const char *name;
+	uint32_t index_idx; /* index ext_id */
+	uint32_t reset_id;
+	uint32_t ext_offset; /* points to beginning of mail_index_ext_header */
+	uint32_t hdr_offset; /* points to mail_index_ext_header.data[] */
+	uint32_t hdr_size; /* size of mail_index_ext_header.data[] */
+	uint16_t record_offset;
+	uint16_t record_size;
+	uint16_t record_align;
+};
+
+struct mail_index_ext_header {
+	/* Size of data[], i.e. the extension size in header */
+	uint32_t hdr_size;
+	/* If reset_id changes, all of the extension record data is
+	   invalidated. For example with cache files reset_id must match the
+	   cache header's file_seq or the cache offsets aren't valid. */
+	uint32_t reset_id;
+	/* Offset of this extension in struct mail_index_record. */
+	uint16_t record_offset;
+	/* Size of this extension in struct mail_index_record. */
+	uint16_t record_size;
+	/* Required alignment of this extension in struct mail_index_record.
+	   It's expected that record_offset is correctly aligned. This is used
+	   only when rearranging fields due to adding/removing other
+	   extensions. */
+	uint16_t record_align;
+	/* Size of name[], which contains the extension's unique name. */
+	uint16_t name_size;
+	/* unsigned char name[name_size]; */
+	/* Extension header data, if any. This starts from the next 64-bit
+	   aligned offset after name[]. */
+	/* unsigned char data[hdr_size]; */
+};
+
+struct mail_index_keyword_header {
+	uint32_t keywords_count;
+	/* struct mail_index_keyword_header_rec[] */
+	/* char name[][] */
+};
+
+struct mail_index_keyword_header_rec {
+	uint32_t unused; /* for backwards compatibility */
+	uint32_t name_offset; /* relative to beginning of name[] */
+};
+
+enum mail_index_sync_handler_type {
+	MAIL_INDEX_SYNC_HANDLER_FILE	= 0x01,
+	MAIL_INDEX_SYNC_HANDLER_HEAD	= 0x02,
+	MAIL_INDEX_SYNC_HANDLER_VIEW	= 0x04
+};
+
+struct mail_index_registered_ext {
+	const char *name;
+	uint32_t index_idx; /* index ext_id */
+	uint32_t hdr_size; /* size of mail_index_ext_header.data[] */
+	uint16_t record_size;
+	uint16_t record_align;
+
+	mail_index_expunge_handler_t *expunge_handler;
+};
+
+struct mail_index_modseq_header {
+	/* highest used modseq */
+	uint64_t highest_modseq;
+	/* last tracked log file position */
+	uint32_t log_seq;
+	uint32_t log_offset;
+};
+
+struct mail_index_record_map {
+	ARRAY(struct mail_index_map *) maps;
+
+	void *mmap_base;
+	size_t mmap_size, mmap_used_size;
+
+	buffer_t *buffer;
+
+	void *records; /* struct mail_index_record[] */
+	unsigned int records_count;
+
+	struct mail_index_map_modseq *modseq;
+	uint32_t last_appended_uid;
+};
+
+#define MAIL_INDEX_MAP_HDR_OFFSET(map, hdr_offset) \
+	CONST_PTR_OFFSET((map)->hdr_copy_buf->data, hdr_offset)
+struct mail_index_map {
+	struct mail_index *index;
+	int refcount;
+
+	/* Copy of the base header for convenience. Note that base_header_size
+	   may be smaller or larger than this struct. If it's smaller, the last
+	   fields in the struct are filled with zeroes. */
+	struct mail_index_header hdr;
+	/* Copy of the full header. */
+	buffer_t *hdr_copy_buf;
+
+	pool_t extension_pool;
+	ARRAY(struct mail_index_ext) extensions;
+	ARRAY(uint32_t) ext_id_map; /* index -> file */
+
+	ARRAY(unsigned int) keyword_idx_map; /* file -> index */
+
+	struct mail_index_modseq_header modseq_hdr_snapshot;
+
+	struct mail_index_record_map *rec_map;
+};
+
+struct mail_index_module_register {
+	unsigned int id;
+};
+
+union mail_index_module_context {
+	struct mail_index_module_register *reg;
+};
+
+struct mail_index_settings {
+	/* Directory path for .cache file. Set via
+	   mail_index_set_cache_dir(). */
+	char *cache_dir;
+
+	/* fsyncing behavior. Set via mail_index_set_fsync_mode(). */
+	enum fsync_mode fsync_mode;
+	enum mail_index_fsync_mask fsync_mask;
+
+	/* Index file permissions. Set via mail_index_set_permissions(). */
+	mode_t mode;
+	gid_t gid;
+	char *gid_origin;
+
+	/* Lock settings. Set via mail_index_set_lock_method(). */
+	enum file_lock_method lock_method;
+	unsigned int max_lock_timeout_secs;
+
+	/* Initial extension added to newly created indexes. Set via
+	   mail_index_set_ext_init_data(). */
+	uint32_t ext_hdr_init_id;
+	void *ext_hdr_init_data;
+};
+
+struct mail_index_error {
+	/* Human-readable error text */
+	char *text;
+
+	/* Error happened because there's no disk space, i.e. syscall failed
+	   with ENOSPC or EDQUOT. */
+	bool nodiskspace:1;
+};
+
+struct mail_index {
+	/* Directory path for the index, or NULL for in-memory indexes. */
+	char *dir;
+	/* Filename prefix for the index, e.g. "dovecot.index." */
+	char *prefix;
+	struct event *event;
+	enum mail_index_open_flags flags;
+	struct mail_index_settings set;
+	struct mail_index_optimization_settings optimization_set;
+
+	struct mail_cache *cache;
+	struct mail_transaction_log *log;
+
+	char *filepath;
+	int fd;
+	/* Linked list of currently opened views */
+	struct mail_index_view *views;
+	/* Latest map */
+	struct mail_index_map *map;
+
+	/* ID number that permanently identifies the index. This is stored in
+	   the index files' headers. If the indexids suddenly changes, it means
+	   that the index has been completely recreated and needs to be
+	   reopened (e.g. the mailbox was deleted and recreated while it
+	   was open). */
+	uint32_t indexid;
+	/* Views initially use this same ID value. This ID is incremented
+	   whenever something unexpected happens to the index that prevents
+	   syncing existing views. When the view's inconsistency_id doesn't
+	   match this one, the view is marked as inconsistent. */
+	unsigned int inconsistency_id;
+	/* How many times this index has been opened with mail_index_open(). */
+	unsigned int open_count;
+
+	/* These contain the log_file_seq and log_file_tail_offset that exists
+	   in dovecot.index file's header. These are used to figure out if it's
+	   time to rewrite the dovecot.index file. Note that these aren't
+	   available in index->map->hdr, because it gets updated when
+	   transaction log file is read. */
+	uint32_t main_index_hdr_log_file_seq;
+	uint32_t main_index_hdr_log_file_tail_offset;
+
+	/* log file which last updated index_deleted */
+	uint32_t index_delete_changed_file_seq;
+
+	/* transaction log head seq/offset when we last fscked */
+	uint32_t fsck_log_head_file_seq;
+	uoff_t fsck_log_head_file_offset;
+
+	/* syncing will update this if non-NULL */
+	struct mail_index_transaction_commit_result *sync_commit_result;
+	/* Delayed log2_rotate_time update to mail_index_header. This is set
+	   and unset within the same sync. */
+	uint32_t hdr_log2_rotate_time_delayed_update;
+
+	/* Registered extensions */
+	pool_t extension_pool;
+	ARRAY(struct mail_index_registered_ext) extensions;
+
+	/* All keywords that have ever been used in this index. Keywords are
+	   only added here, never removed. */
+	pool_t keywords_pool;
+	ARRAY_TYPE(keywords) keywords;
+	HASH_TABLE(char *, void *) keywords_hash; /* name -> unsigned int idx */
+
+	/* Registered extension IDs */
+	uint32_t keywords_ext_id;
+	uint32_t modseq_ext_id;
+
+	/* Module-specific contexts. */
+	ARRAY(union mail_index_module_context *) module_contexts;
+
+	/* Last error returned by mail_index_get_error_message().
+	   Cleared by mail_index_reset_error(). */
+	struct mail_index_error last_error;
+	/* Timestamp when mmap() failure was logged the last time. This is used
+	   to prevent logging the same error too rapidly. This could happen
+	   e.g. if mmap()ing a large cache file that exceeeds process's
+	   VSZ limit. */
+	time_t last_mmap_error_time;
+	/* If non-NULL, dovecot.index should be recreated as soon as possible.
+	   The reason for why the recreation is wanted is stored as human-
+	   readable text. */
+	char *need_recreate;
+
+	/* Mapping has noticed non-external MAIL_TRANSACTION_INDEX_DELETED
+	   record, i.e. a request to mark the index deleted. The next sync
+	   will finish the deletion by writing external
+	   MAIL_TRANSACTION_INDEX_DELETED record. */
+	bool index_delete_requested:1;
+	/* Mapping has noticed external MAIL_TRANSACTION_INDEX_DELETED record,
+	   or index was unexpectedly deleted under us. No more changes are
+	   allowed to the index, except undeletion. */
+	bool index_deleted:1;
+	/* .log is locked for syncing. This is the main exclusive lock for
+	   indexes. */
+	bool log_sync_locked:1;
+	/* Main index or .log couldn't be opened read-write */
+	bool readonly:1;
+	/* mail_index_map() is running */
+	bool mapping:1;
+	/* mail_index_sync_*() is running */
+	bool syncing:1;
+	/* Mapping has read more from .log than it preferred. Use
+	   mail_index_base_optimization_settings.rewrite_min_log_bytes the next
+	   time when checking if index needs a rewrite. */
+	bool index_min_write:1;
+	/* mail_index_modseq_enable() has been called. Track per-flag
+	   modseq numbers in memory (global modseqs are tracked anyway). */
+	bool modseqs_enabled:1;
+	/* mail_index_open() is creating new index files */
+	bool initial_create:1;
+	/* TRUE after mail_index_map() has succeeded */
+	bool initial_mapped:1;
+	/* The next mail_index_map() must reopen the main index, because the
+	   currently opened one is too old. */
+	bool reopen_main_index:1;
+	/* Index has been fsck'd, but mail_index_reset_fscked() hasn't been
+	   called yet. */
+	bool fscked:1;
+};
+
+extern struct mail_index_module_register mail_index_module_register;
+extern struct event_category event_category_mail_index;
+
+/* Add/replace expunge handler for specified extension. */
+void mail_index_register_expunge_handler(struct mail_index *index,
+					 uint32_t ext_id,
+					 mail_index_expunge_handler_t *callback);
+void mail_index_unregister_expunge_handler(struct mail_index *index,
+					   uint32_t ext_id);
+
+int mail_index_create_tmp_file(struct mail_index *index,
+			       const char *path_prefix, const char **path_r);
+
+int mail_index_try_open_only(struct mail_index *index);
+void mail_index_close_file(struct mail_index *index);
+/* Returns 1 if index was successfully (re-)opened, 0 if the index no longer
+   exists, -1 if I/O error. If 1 is returned, reopened_r=TRUE if a new index
+   was actually reopened (or if index wasn't even open before this call). */
+int mail_index_reopen_if_changed(struct mail_index *index, bool *reopened_r,
+				 const char **reason_r);
+/* Update/rewrite the main index file from index->map */
+void mail_index_write(struct mail_index *index, bool want_rotate,
+		      const char *reason);
+
+void mail_index_flush_read_cache(struct mail_index *index, const char *path,
+				 int fd, bool locked);
+
+int mail_index_lock_fd(struct mail_index *index, const char *path, int fd,
+		       int lock_type, unsigned int timeout_secs,
+		       struct file_lock **lock_r);
+
+/* Allocate a new empty map. */
+struct mail_index_map *mail_index_map_alloc(struct mail_index *index);
+/* Replace index->map with the latest index changes. This may reopen the index
+   file and/or it may read the latest changes from transaction log. The log is
+   read up to EOF, but non-synced expunges are skipped.
+
+   If we mmap()ed the index file, the map is returned locked.
+
+   Returns 1 = ok, 0 = corrupted, -1 = error. */
+int mail_index_map(struct mail_index *index,
+		   enum mail_index_sync_handler_type type);
+/* Unreference given mapping and unmap it if it's dropped to zero. */
+void mail_index_unmap(struct mail_index_map **map);
+/* Clone a map. It still points to the original rec_map. */
+struct mail_index_map *mail_index_map_clone(const struct mail_index_map *map);
+/* Make sure the map has its own private rec_map, cloning it if necessary. */
+void mail_index_record_map_move_to_private(struct mail_index_map *map);
+/* If map points to mmap()ed index, copy it to the memory. */
+void mail_index_map_move_to_memory(struct mail_index_map *map);
+
+void mail_index_fchown(struct mail_index *index, int fd, const char *path);
+
+bool mail_index_map_lookup_ext(struct mail_index_map *map, const char *name,
+			       uint32_t *idx_r);
+bool mail_index_ext_name_is_valid(const char *name);
+uint32_t
+mail_index_map_register_ext(struct mail_index_map *map,
+			    const char *name, uint32_t ext_offset,
+			    const struct mail_index_ext_header *ext_hdr);
+bool mail_index_map_get_ext_idx(struct mail_index_map *map,
+				uint32_t ext_id, uint32_t *idx_r);
+const struct mail_index_ext *
+mail_index_view_get_ext(struct mail_index_view *view, uint32_t ext_id);
+
+void mail_index_map_lookup_seq_range(struct mail_index_map *map,
+				     uint32_t first_uid, uint32_t last_uid,
+				     uint32_t *first_seq_r,
+				     uint32_t *last_seq_r);
+
+/* Returns 1 on success, 0 on non-critical errors we want to silently fix,
+   -1 if map isn't usable. The caller is responsible for logging the errors
+   if -1 is returned. */
+int mail_index_map_check_header(struct mail_index_map *map,
+				const char **error_r);
+/* Returns 1 if header is usable, 0 or -1 if not. The caller should log an
+   error if -1 is returned, but not if 0 is returned. */
+bool mail_index_check_header_compat(struct mail_index *index,
+				    const struct mail_index_header *hdr,
+				    uoff_t file_size, const char **error_r);
+int mail_index_map_parse_extensions(struct mail_index_map *map);
+int mail_index_map_parse_keywords(struct mail_index_map *map);
+
+void mail_index_map_init_extbufs(struct mail_index_map *map,
+				 unsigned int initial_count);
+int mail_index_map_ext_get_next(struct mail_index_map *map,
+				unsigned int *offset,
+				const struct mail_index_ext_header **ext_hdr_r,
+				const char **name_r);
+int mail_index_map_ext_hdr_check(const struct mail_index_header *hdr,
+				 const struct mail_index_ext_header *ext_hdr,
+				 const char *name, const char **error_r);
+unsigned int mail_index_map_ext_hdr_offset(unsigned int name_len);
+
+void mail_index_fsck_locked(struct mail_index *index);
+
+/* Log an error and set it as the index's current error that is available
+   with mail_index_get_error_message(). */
+void mail_index_set_error(struct mail_index *index, const char *fmt, ...)
+	ATTR_FORMAT(2, 3) ATTR_COLD;
+/* Same as mail_index_set_error(), but don't log the error. */
+void mail_index_set_error_nolog(struct mail_index *index, const char *str)
+	ATTR_COLD;
+/* "%s failed with index file %s: %m" */
+void mail_index_set_syscall_error(struct mail_index *index,
+				  const char *function) ATTR_COLD;
+/* "%s failed with file %s: %m" */
+void mail_index_file_set_syscall_error(struct mail_index *index,
+				       const char *filepath,
+				       const char *function) ATTR_COLD;
+
+#endif