summaryrefslogtreecommitdiffstats
path: root/src/lib-index/mail-index.h
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 09:51:24 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 09:51:24 +0000
commitf7548d6d28c313cf80e6f3ef89aed16a19815df1 (patch)
treea3f6f2a3f247293bee59ecd28e8cd8ceb6ca064a /src/lib-index/mail-index.h
parentInitial commit. (diff)
downloaddovecot-upstream.tar.xz
dovecot-upstream.zip
Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src/lib-index/mail-index.h')
-rw-r--r--src/lib-index/mail-index.h817
1 files changed, 817 insertions, 0 deletions
diff --git a/src/lib-index/mail-index.h b/src/lib-index/mail-index.h
new file mode 100644
index 0000000..c1947cf
--- /dev/null
+++ b/src/lib-index/mail-index.h
@@ -0,0 +1,817 @@
+#ifndef MAIL_INDEX_H
+#define MAIL_INDEX_H
+
+#include "file-lock.h"
+#include "fsync-mode.h"
+#include "guid.h"
+#include "mail-types.h"
+#include "seq-range-array.h"
+
+#define MAIL_INDEX_MAJOR_VERSION 7
+#define MAIL_INDEX_MINOR_VERSION 3
+
+#define MAIL_INDEX_HEADER_MIN_SIZE 120
+
+/* Log a warning when transaction log has been locked for this many seconds.
+ This lock is held also between mail_index_sync_begin()..commit(). */
+#define MAIL_TRANSACTION_LOG_LOCK_WARN_SECS 30
+
+enum mail_index_open_flags {
+ /* Create index if it doesn't exist */
+ MAIL_INDEX_OPEN_FLAG_CREATE = 0x01,
+ /* Don't try to mmap() index files */
+ MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE = 0x04,
+ /* Rely on O_EXCL when creating dotlocks */
+ MAIL_INDEX_OPEN_FLAG_DOTLOCK_USE_EXCL = 0x10,
+ /* Flush NFS attr/data/write cache when necessary */
+ MAIL_INDEX_OPEN_FLAG_NFS_FLUSH = 0x40,
+ /* Open the index read-only */
+ MAIL_INDEX_OPEN_FLAG_READONLY = 0x80,
+ /* Create backups of dovecot.index files once in a while */
+ MAIL_INDEX_OPEN_FLAG_KEEP_BACKUPS = 0x100,
+ /* If we run out of disk space, fail modifications instead of moving
+ indexes to memory. */
+ MAIL_INDEX_OPEN_FLAG_NEVER_IN_MEMORY = 0x200,
+ /* We're only going to save new messages to the index.
+ Avoid unnecessary reads. */
+ MAIL_INDEX_OPEN_FLAG_SAVEONLY = 0x400,
+ /* Enable debug logging */
+ MAIL_INDEX_OPEN_FLAG_DEBUG = 0x800,
+ /* MAIL_INDEX_MAIL_FLAG_DIRTY can be used as a backend-specific flag.
+ All special handling of the flag is disabled by this. */
+ MAIL_INDEX_OPEN_FLAG_NO_DIRTY = 0x1000,
+};
+
+enum mail_index_header_compat_flags {
+ /* All fields in these index files are in little-endian format.
+ If the current CPU endianess doesn't match this, the indexes can't
+ be used. There is currently no support to translate endianess. */
+ MAIL_INDEX_COMPAT_LITTLE_ENDIAN = 0x01
+};
+
+enum mail_index_header_flag {
+ /* mail_index_mark_corrupted() was just called by this process.
+ Reopen or recreate it. This flag is never actually written to
+ disk. */
+ MAIL_INDEX_HDR_FLAG_CORRUPTED = 0x0001,
+ /* There are messages with MAIL_INDEX_MAIL_FLAG_DIRTY flag. */
+ MAIL_INDEX_HDR_FLAG_HAVE_DIRTY = 0x0002,
+ /* Index has been fsck'd. The caller may want to resync the index
+ to make sure it's valid and drop this flag. */
+ MAIL_INDEX_HDR_FLAG_FSCKD = 0x0004,
+};
+
+enum mail_index_mail_flags {
+ /* This flag used to contain MAIL_RECENT flag, but is always zero
+ with the current index file format. */
+ MAIL_INDEX_MAIL_FLAG_UNUSED = 0x20,
+ /* For private use by backend. Replacing flags doesn't change this. */
+ MAIL_INDEX_MAIL_FLAG_BACKEND = 0x40,
+ /* Message flags haven't been written to backend. If
+ MAIL_INDEX_OPEN_FLAG_NO_DIRTY is set, this is treated as a
+ backend-specific flag with no special internal handling. */
+ MAIL_INDEX_MAIL_FLAG_DIRTY = 0x80,
+
+ /* Force updating this message's modseq via a flag update record.
+ Note that this flag isn't saved to disk. */
+ MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ = 0x100
+};
+
+#define MAIL_INDEX_FLAGS_MASK \
+ (MAIL_ANSWERED | MAIL_FLAGGED | MAIL_DELETED | MAIL_SEEN | MAIL_DRAFT)
+
+struct mail_index_header {
+ /* Major version is increased only when you can't have backwards
+ compatibility. If the field doesn't match MAIL_INDEX_MAJOR_VERSION,
+ don't even try to read it. */
+ uint8_t major_version;
+ /* Minor version is increased when the file format changes in a
+ backwards compatible way. If the field is smaller than
+ MAIL_INDEX_MINOR_VERSION, upgrade the file format and update the
+ minor_version field as well. If minor_version is higher than
+ MAIL_INDEX_MINOR_VERSION, leave it as it is. It likely means that a
+ new Dovecot version is currently being upgraded to, but the file was
+ still accessed by an old version. */
+ uint8_t minor_version;
+
+ /* sizeof(struct mail_index_header) when creating a new index. If the
+ header is smaller, fill the missing fields with 0. If the header is
+ larger, preserve the size and unknown fields. */
+ uint16_t base_header_size;
+ uint32_t header_size; /* base + extended header size */
+ /* sizeof(struct mail_index_record) + extensions */
+ uint32_t record_size;
+
+ uint8_t compat_flags; /* enum mail_index_header_compat_flags */
+ uint8_t unused[3];
+
+ /* Unique index file ID. Initialized with the current UNIX timestamp.
+ This is used to make sure that the main index, transaction log and
+ cache file are all part of the same index. */
+ uint32_t indexid;
+ uint32_t flags; /* enum mail_index_header_flag */
+
+ /* IMAP UIDVALIDITY. Initially can be 0, but must be non-0 after the
+ first mailbox sync. The UIDVALIDITY shouldn't normally change after
+ the mailbox is created. */
+ uint32_t uid_validity;
+ /* UID for the next saved message (must not be lower than this). This
+ value can only increase. */
+ uint32_t next_uid;
+
+ /* Number of messages in the index */
+ uint32_t messages_count;
+ uint32_t unused_old_recent_messages_count;
+ /* Number of messages with MAIL_SEEN flag */
+ uint32_t seen_messages_count;
+ /* Number of messages with MAIL_DELETED flag */
+ uint32_t deleted_messages_count;
+
+ /* The specified UID and all mails after it have MAIL_RECENT flag */
+ uint32_t first_recent_uid;
+ /* There are no UIDs lower than this without MAIL_SEEN flag. There are
+ no guarantees whether this UID has MAIL_SEEN flag, or whether the it
+ even exists. Used to optimize finding the first unseen message. */
+ uint32_t first_unseen_uid_lowwater;
+ /* Similarly to above, used to optimize finding the first deleted
+ message. */
+ uint32_t first_deleted_uid_lowwater;
+
+ /* The index is synced up to this log_file_seq and
+ log_file_head_offset. However, non-external transaction records
+ between tail_offset..head_offset haven't been synced to the
+ mailbox yet. For example there may be pending expunges or flag
+ changes, which will be synced on the next mail_index_sync_*()
+ calls. */
+ uint32_t log_file_seq;
+ uint32_t log_file_tail_offset;
+ uint32_t log_file_head_offset;
+
+ uint32_t unused_old_sync_size_part1;
+ /* Timestamp of when .log was rotated into .log.2. This can be used to
+ optimize checking when it's time to unlink it without stat()ing it.
+ 0 = unknown, -1 = .log.2 doesn't exists. */
+ uint32_t log2_rotate_time;
+ /* Timestamp when the mailbox backend-specific code last checked
+ whether there are old temporary files (left by crashes) that should
+ be deleted. 0 = unknown. */
+ uint32_t last_temp_file_scan;
+
+ /* UNIX timestamp to the beginning of the day (in server's local
+ timezone) when new messages were last added to the index file. */
+ uint32_t day_stamp;
+ /* These fields are updated when day_stamp < today. The [0..6] are
+ first moved to [1..7], then [0] is set to the first appended UID. So
+ they contain the first UID of the day for last 8 days when messages
+ were appended.
+
+ These are used by cache purging to decide when to drop
+ MAIL_CACHE_DECISION_TEMP fields. */
+ uint32_t day_first_uid[8];
+};
+
+#define MAIL_INDEX_RECORD_MIN_SIZE (sizeof(uint32_t) + sizeof(uint8_t))
+struct mail_index_record {
+ uint32_t uid;
+ uint8_t flags; /* enum mail_flags | enum mail_index_mail_flags */
+};
+
+struct mail_keywords {
+ struct mail_index *index;
+ unsigned int count;
+ int refcount;
+
+ /* variable sized list of keyword indexes */
+ unsigned int idx[FLEXIBLE_ARRAY_MEMBER];
+};
+
+enum mail_index_transaction_flags {
+ /* If transaction is marked as hidden, the changes are marked with
+ hidden=TRUE when the view is synchronized. */
+ MAIL_INDEX_TRANSACTION_FLAG_HIDE = 0x01,
+ /* External transactions describe changes to mailbox that have already
+ happened. */
+ MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL = 0x02,
+ /* Don't add flag updates unless they actually change something.
+ This is reliable only when syncing, otherwise someone else might
+ have already committed a transaction that had changed the flags. */
+ MAIL_INDEX_TRANSACTION_FLAG_AVOID_FLAG_UPDATES = 0x04,
+ /* fsync() this transaction (unless fsyncs are disabled) */
+ MAIL_INDEX_TRANSACTION_FLAG_FSYNC = 0x08,
+ /* Sync transaction describes changes to mailbox that already happened
+ to another mailbox with whom we're syncing with (dsync) */
+ MAIL_INDEX_TRANSACTION_FLAG_SYNC = 0x10
+};
+
+enum mail_index_sync_type {
+ MAIL_INDEX_SYNC_TYPE_EXPUNGE = 0x02,
+ MAIL_INDEX_SYNC_TYPE_FLAGS = 0x04,
+ MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD = 0x08,
+ MAIL_INDEX_SYNC_TYPE_KEYWORD_REMOVE = 0x10
+};
+
+enum mail_index_fsync_mask {
+ MAIL_INDEX_FSYNC_MASK_APPENDS = 0x01,
+ MAIL_INDEX_FSYNC_MASK_EXPUNGES = 0x02,
+ MAIL_INDEX_FSYNC_MASK_FLAGS = 0x04,
+ MAIL_INDEX_FSYNC_MASK_KEYWORDS = 0x08
+};
+
+enum mail_index_sync_flags {
+ /* Resync all dirty messages' flags. */
+ MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY = 0x01,
+ /* Drop recent flags from all messages */
+ MAIL_INDEX_SYNC_FLAG_DROP_RECENT = 0x02,
+ /* Create the transaction with AVOID_FLAG_UPDATES flag */
+ MAIL_INDEX_SYNC_FLAG_AVOID_FLAG_UPDATES = 0x04,
+ /* If there are no new transactions and nothing else to do,
+ return 0 in mail_index_sync_begin() */
+ MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES = 0x08,
+ /* Create the transaction with FSYNC flag */
+ MAIL_INDEX_SYNC_FLAG_FSYNC = 0x10,
+ /* If we see "delete index" request transaction, finish it.
+ This flag also allows committing more changes to a deleted index. */
+ MAIL_INDEX_SYNC_FLAG_DELETING_INDEX = 0x20,
+ /* Same as MAIL_INDEX_SYNC_FLAG_DELETING_INDEX, but finish index
+ deletion only once and fail the rest (= avoid race conditions when
+ multiple processes try to mark the index deleted) */
+ MAIL_INDEX_SYNC_FLAG_TRY_DELETING_INDEX = 0x40,
+ /* Update header's tail_offset to head_offset, even if it's the only
+ thing we do and there's no strict need for it. */
+ MAIL_INDEX_SYNC_FLAG_UPDATE_TAIL_OFFSET = 0x80
+};
+
+enum mail_index_view_sync_flags {
+ /* Don't sync expunges */
+ MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES = 0x01,
+ /* Make sure view isn't inconsistent after syncing. This also means
+ that you don't care about view_sync_next()'s output, so it won't
+ return anything. */
+ MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT = 0x02,
+ /* Indicate that this is a secondary view, this can be used to indicate
+ that inconsistencies can be expected and if found should be fixed
+ by fully syncing. */
+ MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX = 0x04,
+};
+
+struct mail_index_sync_rec {
+ uint32_t uid1, uid2;
+ enum mail_index_sync_type type;
+
+ /* MAIL_INDEX_SYNC_TYPE_FLAGS: */
+ uint8_t add_flags;
+ uint8_t remove_flags;
+
+ /* MAIL_INDEX_SYNC_TYPE_KEYWORD_ADD, .._REMOVE: */
+ unsigned int keyword_idx;
+
+ /* MAIL_INDEX_SYNC_TYPE_EXPUNGE: */
+ guid_128_t guid_128;
+};
+
+enum mail_index_view_sync_type {
+ /* Flags or keywords changed */
+ MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS = 0x01,
+ MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ = 0x02
+};
+
+struct mail_index_view_sync_rec {
+ uint32_t uid1, uid2;
+ enum mail_index_view_sync_type type;
+
+ /* TRUE if this was a hidden transaction. */
+ bool hidden:1;
+};
+
+enum mail_index_transaction_change {
+ MAIL_INDEX_TRANSACTION_CHANGE_APPEND = BIT(0),
+ MAIL_INDEX_TRANSACTION_CHANGE_EXPUNGE = BIT(1),
+ MAIL_INDEX_TRANSACTION_CHANGE_FLAGS = BIT(2),
+ MAIL_INDEX_TRANSACTION_CHANGE_KEYWORDS = BIT(3),
+ MAIL_INDEX_TRANSACTION_CHANGE_MODSEQ = BIT(4),
+ MAIL_INDEX_TRANSACTION_CHANGE_ATTRIBUTE = BIT(5),
+
+ MAIL_INDEX_TRANSACTION_CHANGE_OTHERS = BIT(30),
+};
+
+struct mail_index_transaction_commit_result {
+ /* seq/offset points to end of transaction */
+ uint32_t log_file_seq;
+ uoff_t log_file_offset;
+ /* number of bytes in the written transaction.
+ all of it was written to the same file. */
+ uoff_t commit_size;
+
+ enum mail_index_transaction_change changes_mask;
+ unsigned int ignored_modseq_changes;
+};
+
+struct mail_index_base_optimization_settings {
+ /* Rewrite the index when the number of bytes that needs to be read
+ from the .log on refresh is between these min/max values. */
+ uoff_t rewrite_min_log_bytes;
+ uoff_t rewrite_max_log_bytes;
+};
+
+struct mail_index_log_optimization_settings {
+ /* Rotate transaction log after it's a) min_size or larger and it was
+ created at least min_age_secs or b) larger than max_size. */
+ uoff_t min_size;
+ uoff_t max_size;
+ unsigned int min_age_secs;
+
+ /* Delete .log.2 when it's older than log2_stale_secs. Don't be too
+ eager, because older files are useful for QRESYNC and dsync. */
+ unsigned int log2_max_age_secs;
+};
+
+struct mail_index_cache_optimization_settings {
+ /* Drop fields that haven't been accessed for n seconds */
+ unsigned int unaccessed_field_drop_secs;
+ /* If cache record becomes larger than this, don't add it. */
+ unsigned int record_max_size;
+
+ /* Maximum size for the cache file. Internally the limit is 1 GB. */
+ uoff_t max_size;
+ /* Never purge the file if it's smaller than this */
+ uoff_t purge_min_size;
+ /* Purge the file when n% of records are deleted */
+ unsigned int purge_delete_percentage;
+ /* Purge the file when n% of rows contain continued rows.
+ For example 200% means that the record has 2 continued rows, i.e.
+ it exists in 3 separate segments in the cache file. */
+ unsigned int purge_continued_percentage;
+ /* Purge the file when we need to follow more than n next_offsets to
+ find the latest cache header. */
+ unsigned int purge_header_continue_count;
+};
+
+struct mail_index_optimization_settings {
+ struct mail_index_base_optimization_settings index;
+ struct mail_index_log_optimization_settings log;
+ struct mail_index_cache_optimization_settings cache;
+};
+
+struct mail_index;
+struct mail_index_map;
+struct mail_index_view;
+struct mail_index_transaction;
+struct mail_index_sync_ctx;
+struct mail_index_view_sync_ctx;
+
+struct mail_index *mail_index_alloc(struct event *parent_event,
+ const char *dir, const char *prefix);
+void mail_index_free(struct mail_index **index);
+
+/* Change .cache file's directory. */
+void mail_index_set_cache_dir(struct mail_index *index, const char *dir);
+/* Specify how often to do fsyncs. If mode is FSYNC_MODE_OPTIMIZED, the mask
+ can be used to specify which transaction types to fsync. */
+void mail_index_set_fsync_mode(struct mail_index *index, enum fsync_mode mode,
+ enum mail_index_fsync_mask mask);
+/* Try to set the index's permissions based on its index directory. Returns
+ TRUE if successful (directory existed), FALSE if mail_index_set_permissions()
+ should be called. */
+bool mail_index_use_existing_permissions(struct mail_index *index);
+void mail_index_set_permissions(struct mail_index *index,
+ mode_t mode, gid_t gid, const char *gid_origin);
+/* Set locking method and maximum time to wait for a lock
+ (UINT_MAX = default). */
+void mail_index_set_lock_method(struct mail_index *index,
+ enum file_lock_method lock_method,
+ unsigned int max_timeout_secs);
+/* Override the default optimization-related settings. Anything set to 0 will
+ use the default. */
+void mail_index_set_optimization_settings(struct mail_index *index,
+ const struct mail_index_optimization_settings *set);
+/* When creating a new index file or reseting an existing one, add the given
+ extension header data immediately to it. */
+void mail_index_set_ext_init_data(struct mail_index *index, uint32_t ext_id,
+ const void *data, size_t size);
+
+/* Open index. Returns 1 if ok, 0 if index doesn't exist and CREATE flags
+ wasn't given, -1 if error. */
+int mail_index_open(struct mail_index *index, enum mail_index_open_flags flags);
+/* Open or create index. Returns 0 if ok, -1 if error. */
+int mail_index_open_or_create(struct mail_index *index,
+ enum mail_index_open_flags flags);
+void mail_index_close(struct mail_index *index);
+/* unlink() all the index files. */
+int mail_index_unlink(struct mail_index *index);
+
+/* Returns TRUE if index is currently in memory. */
+bool mail_index_is_in_memory(struct mail_index *index);
+/* Move the index into memory. Returns 0 if ok, -1 if error occurred. */
+int mail_index_move_to_memory(struct mail_index *index);
+
+struct mail_cache *mail_index_get_cache(struct mail_index *index);
+
+/* Refresh index so mail_index_lookup*() will return latest values. Note that
+ immediately after this call there may already be changes, so if you need to
+ rely on validity of the returned values, use some external locking for it. */
+int ATTR_NOWARN_UNUSED_RESULT
+mail_index_refresh(struct mail_index *index);
+
+/* View can be used to look into index. Sequence numbers inside view change
+ only when you synchronize it. The view acquires required locks
+ automatically, but you'll have to drop them manually. */
+struct mail_index_view *
+mail_index_view_open(struct mail_index *index,
+ const char *source_filename, unsigned int source_linenum);
+#define mail_index_view_open(index) \
+ mail_index_view_open(index, __FILE__, __LINE__)
+void mail_index_view_close(struct mail_index_view **view);
+
+/* Returns the index for given view. */
+struct mail_index *mail_index_view_get_index(struct mail_index_view *view);
+/* Returns number of mails in view. */
+uint32_t mail_index_view_get_messages_count(struct mail_index_view *view);
+/* Returns TRUE if we lost track of changes for some reason. */
+bool mail_index_view_is_inconsistent(struct mail_index_view *view);
+/* Returns TRUE if there are open transactions open for the view. */
+bool mail_index_view_have_transactions(struct mail_index_view *view);
+
+/* Transaction has to be opened to be able to modify index. You can have
+ multiple transactions open simultaneously. Committed transactions won't
+ show up until you've synchronized the view. Expunges won't show up until
+ you've synchronized the mailbox (mail_index_sync_begin). */
+struct mail_index_transaction *
+mail_index_transaction_begin(struct mail_index_view *view,
+ enum mail_index_transaction_flags flags);
+int mail_index_transaction_commit(struct mail_index_transaction **t);
+int mail_index_transaction_commit_full(struct mail_index_transaction **t,
+ struct mail_index_transaction_commit_result *result_r);
+void mail_index_transaction_rollback(struct mail_index_transaction **t);
+/* Discard all changes in the transaction. */
+void mail_index_transaction_reset(struct mail_index_transaction *t);
+/* When committing transaction, drop flag/keyword updates for messages whose
+ mdoseq is larger than max_modseq. Save those messages' sequences to the
+ given array. */
+void mail_index_transaction_set_max_modseq(struct mail_index_transaction *t,
+ uint64_t max_modseq,
+ ARRAY_TYPE(seq_range) *seqs);
+
+/* Returns the view transaction was created for. */
+struct mail_index_view *
+mail_index_transaction_get_view(struct mail_index_transaction *t);
+/* Returns TRUE if the given sequence is being expunged in this transaction. */
+bool mail_index_transaction_is_expunged(struct mail_index_transaction *t,
+ uint32_t seq);
+
+/* Returns a view containing the mailbox state after changes in transaction
+ are applied. The view can still be used after transaction has been
+ committed. */
+struct mail_index_view *
+mail_index_transaction_open_updated_view(struct mail_index_transaction *t);
+
+/* Begin synchronizing mailbox with index file. Returns 1 if ok,
+ 0 if MAIL_INDEX_SYNC_FLAG_REQUIRE_CHANGES is set and there's nothing to
+ sync, -1 if error.
+
+ mail_index_sync_next() returns all changes from previously committed
+ transactions which haven't yet been committed to the actual mailbox.
+ They're returned in ascending order and they never overlap (if we add more
+ sync types, then they might). You must go through all of them and update
+ the mailbox accordingly.
+
+ Changes done to the returned transaction are expected to describe the
+ mailbox's current state.
+
+ The returned view already contains all the changes (except expunge
+ requests). After applying sync records on top of backend flags they should
+ match flags in the view. If they don't, there have been external changes.
+
+ Returned expunges are treated as expunge requests. They're not really
+ removed from the index until you mark them expunged to the returned
+ transaction. If it's not possible to expunge the message (e.g. permission
+ denied), simply don't mark them expunged.
+
+ Returned sequence numbers describe the mailbox state at the beginning of
+ synchronization, ie. expunges don't affect them. */
+int mail_index_sync_begin(struct mail_index *index,
+ struct mail_index_sync_ctx **ctx_r,
+ struct mail_index_view **view_r,
+ struct mail_index_transaction **trans_r,
+ enum mail_index_sync_flags flags);
+/* Like mail_index_sync_begin(), but returns 1 if OK and if index is already
+ synchronized up to the given log_file_seq+offset, the synchronization isn't
+ started and this function returns 0. This should be done when you wish to
+ sync your committed transaction instead of doing a full mailbox
+ synchronization. */
+int mail_index_sync_begin_to(struct mail_index *index,
+ struct mail_index_sync_ctx **ctx_r,
+ struct mail_index_view **view_r,
+ struct mail_index_transaction **trans_r,
+ uint32_t log_file_seq, uoff_t log_file_offset,
+ enum mail_index_sync_flags flags);
+/* Returns TRUE if it currently looks like syncing would return changes. */
+bool mail_index_sync_have_any(struct mail_index *index,
+ enum mail_index_sync_flags flags);
+/* Returns TRUE if it currently looks like syncing would return expunges. */
+bool mail_index_sync_have_any_expunges(struct mail_index *index);
+/* Returns the log file seq+offsets for the area which this sync is handling. */
+void mail_index_sync_get_offsets(struct mail_index_sync_ctx *ctx,
+ uint32_t *seq1_r, uoff_t *offset1_r,
+ uint32_t *seq2_r, uoff_t *offset2_r);
+/* Returns -1 if error, 0 if sync is finished, 1 if record was filled. */
+bool mail_index_sync_next(struct mail_index_sync_ctx *ctx,
+ struct mail_index_sync_rec *sync_rec);
+/* Returns TRUE if there's more to sync. */
+bool mail_index_sync_have_more(struct mail_index_sync_ctx *ctx);
+/* Returns TRUE if sync has any expunges to handle. */
+bool mail_index_sync_has_expunges(struct mail_index_sync_ctx *ctx);
+/* Reset syncing to initial state after mail_index_sync_begin(), so you can
+ go through all the sync records again with mail_index_sync_next(). */
+void mail_index_sync_reset(struct mail_index_sync_ctx *ctx);
+/* Update result when refreshing index at the end of sync. */
+void mail_index_sync_set_commit_result(struct mail_index_sync_ctx *ctx,
+ struct mail_index_transaction_commit_result *result);
+/* Don't log a warning even if syncing took over
+ MAIL_TRANSACTION_LOG_LOCK_WARN_SECS seconds. Usually this is called because
+ the caller itself already logged a warning about it. */
+void mail_index_sync_no_warning(struct mail_index_sync_ctx *ctx);
+/* If a warning is logged because syncing took over
+ MAIL_TRANSACTION_LOG_LOCK_WARN_SECS seconds, log this as the reason for the
+ syncing. */
+void mail_index_sync_set_reason(struct mail_index_sync_ctx *ctx,
+ const char *reason);
+/* Commit synchronization by writing all changes to mail index file. */
+int mail_index_sync_commit(struct mail_index_sync_ctx **ctx);
+/* Rollback synchronization - none of the changes listed by sync_next() are
+ actually written to index file. */
+void mail_index_sync_rollback(struct mail_index_sync_ctx **ctx);
+
+/* Lock the index exclusively. This is the same locking as what happens when
+ syncing the index. It's not necessary to normally call this function, unless
+ doing something special such as rebuilding the index outside syncing.
+ Returns 0 on success, -1 if locking failed for any reason. */
+int mail_index_lock_sync(struct mail_index *index, const char *lock_reason);
+/* Unlock the locked index. The index must have been locked previously with
+ mail_index_lock_sync(). If the lock had been kept for excessively long,
+ a warning is logged with the long_lock_reason. */
+void mail_index_unlock(struct mail_index *index, const char *long_lock_reason);
+/* Returns TRUE if index is currently exclusively locked. */
+bool mail_index_is_locked(struct mail_index *index);
+
+/* Mark index file corrupted in memory and delete it from disk.
+ Invalidates all views. This should be called only for index files that can
+ safely be recreated without any data loss. */
+void mail_index_mark_corrupted(struct mail_index *index) ATTR_COLD;
+/* Check and fix any found problems. Returns -1 if we couldn't lock for sync,
+ 0 if everything went ok. */
+int mail_index_fsck(struct mail_index *index) ATTR_COLD;
+/* Returns TRUE if mail_index_fsck() has been called since the last
+ mail_index_reset_fscked() call. */
+bool mail_index_reset_fscked(struct mail_index *index);
+
+/* Synchronize changes in view. You have to go through all records, or view
+ will be marked inconsistent. Only sync_mask type records are
+ synchronized. */
+struct mail_index_view_sync_ctx *
+mail_index_view_sync_begin(struct mail_index_view *view,
+ enum mail_index_view_sync_flags flags);
+bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
+ struct mail_index_view_sync_rec *sync_rec);
+void
+mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
+ const ARRAY_TYPE(seq_range) **expunges_r);
+int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **ctx,
+ bool *delayed_expunges_r);
+
+/* Returns the index header. */
+const struct mail_index_header *
+mail_index_get_header(struct mail_index_view *view);
+/* Returns the wanted message record. */
+const struct mail_index_record *
+mail_index_lookup(struct mail_index_view *view, uint32_t seq);
+const struct mail_index_record *
+mail_index_lookup_full(struct mail_index_view *view, uint32_t seq,
+ struct mail_index_map **map_r, bool *expunged_r);
+/* Returns TRUE if the given message has already been expunged from index. */
+bool mail_index_is_expunged(struct mail_index_view *view, uint32_t seq);
+/* Note that returned keyword indexes aren't sorted. */
+void mail_index_lookup_keywords(struct mail_index_view *view, uint32_t seq,
+ ARRAY_TYPE(keyword_indexes) *keyword_idx);
+/* Return keywords from given map. */
+void mail_index_map_lookup_keywords(struct mail_index_map *map, uint32_t seq,
+ ARRAY_TYPE(keyword_indexes) *keyword_idx);
+/* mail_index_lookup[_keywords]() returns the latest flag changes.
+ This function instead attempts to return the flags and keywords done by the
+ last view sync. */
+void mail_index_lookup_view_flags(struct mail_index_view *view, uint32_t seq,
+ enum mail_flags *flags_r,
+ ARRAY_TYPE(keyword_indexes) *keyword_idx);
+/* Returns the UID for given message. May be slightly faster than
+ mail_index_lookup()->uid. */
+void mail_index_lookup_uid(struct mail_index_view *view, uint32_t seq,
+ uint32_t *uid_r);
+/* Convert UID range to sequence range. If no UIDs are found, returns FALSE and
+ sequences are set to 0. Note that any of the returned sequences may have
+ been expunged already. */
+bool mail_index_lookup_seq_range(struct mail_index_view *view,
+ uint32_t first_uid, uint32_t last_uid,
+ uint32_t *first_seq_r, uint32_t *last_seq_r);
+bool mail_index_lookup_seq(struct mail_index_view *view,
+ uint32_t uid, uint32_t *seq_r);
+/* Find first mail with (mail->flags & flags_mask) == flags. Useful mostly for
+ taking advantage of lowwater-fields in headers. */
+void mail_index_lookup_first(struct mail_index_view *view,
+ enum mail_flags flags, uint8_t flags_mask,
+ uint32_t *seq_r);
+
+/* Append a new record to index. */
+void mail_index_append(struct mail_index_transaction *t, uint32_t uid,
+ uint32_t *seq_r);
+/* Assign new UIDs for mails with uid=0 or uid<min_allowed_uid. All the new
+ UIDs are >= first_new_uid, an also higher than the highest seen uid (i.e. it
+ doesn't try to fill UID gaps). Assumes that mailbox is locked in a way that
+ UIDs can be safely assigned. Returns UIDs for all assigned messages, in
+ their sequence order (so UIDs are not necessary ascending). */
+void mail_index_append_finish_uids_full(struct mail_index_transaction *t,
+ uint32_t min_allowed_uid,
+ uint32_t first_new_uid,
+ ARRAY_TYPE(seq_range) *uids_r);
+/* Call mail_index_append_finish_uids_full() with first_uid used for both
+ min_allowed_uid and first_new_uid. */
+void mail_index_append_finish_uids(struct mail_index_transaction *t,
+ uint32_t first_uid,
+ ARRAY_TYPE(seq_range) *uids_r);
+/* Expunge record from index. Note that this doesn't affect sequence numbers
+ until transaction is committed and mailbox is synced. */
+void mail_index_expunge(struct mail_index_transaction *t, uint32_t seq);
+/* Like mail_index_expunge(), but also write message GUID to transaction log. */
+void mail_index_expunge_guid(struct mail_index_transaction *t, uint32_t seq,
+ const guid_128_t guid_128);
+/* Revert all changes done in this transaction to the given existing mail. */
+void mail_index_revert_changes(struct mail_index_transaction *t, uint32_t seq);
+/* Update flags in index. */
+void mail_index_update_flags(struct mail_index_transaction *t, uint32_t seq,
+ enum modify_type modify_type,
+ enum mail_flags flags);
+void mail_index_update_flags_range(struct mail_index_transaction *t,
+ uint32_t seq1, uint32_t seq2,
+ enum modify_type modify_type,
+ enum mail_flags flags);
+/* Specified attribute's value was changed. This is just a notification so the
+ change gets assigned its own modseq and any log readers can find out about
+ this change. */
+void mail_index_attribute_set(struct mail_index_transaction *t,
+ bool pvt, const char *key,
+ time_t timestamp, uint32_t value_len);
+/* Attribute was deleted. */
+void mail_index_attribute_unset(struct mail_index_transaction *t,
+ bool pvt, const char *key, time_t timestamp);
+/* Update message's modseq to be at least min_modseq. */
+void mail_index_update_modseq(struct mail_index_transaction *t, uint32_t seq,
+ uint64_t min_modseq);
+/* Update highest modseq to be at least min_modseq. */
+void mail_index_update_highest_modseq(struct mail_index_transaction *t,
+ uint64_t min_modseq);
+/* Reset the index before committing this transaction. This is usually done
+ only when UIDVALIDITY changes. */
+void mail_index_reset(struct mail_index_transaction *t);
+/* Remove MAIL_INDEX_HDR_FLAG_FSCKD from header if it exists. This must be
+ called only during syncing so that the mailbox is locked. */
+void mail_index_unset_fscked(struct mail_index_transaction *t);
+/* Mark index deleted. No further changes will be possible after the
+ transaction has been committed. */
+void mail_index_set_deleted(struct mail_index_transaction *t);
+/* Mark a deleted index as undeleted. Afterwards index can be changed again. */
+void mail_index_set_undeleted(struct mail_index_transaction *t);
+/* Returns TRUE if index has been set deleted. This gets set only after
+ index has been opened/refreshed and the transaction has been seen. */
+bool mail_index_is_deleted(struct mail_index *index);
+/* Returns the last time the index was modified. This can be called even if the
+ index isn't open. If the index doesn't exist, sets mtime to 0. */
+int mail_index_get_modification_time(struct mail_index *index, time_t *mtime_r);
+
+/* Lookup a keyword, returns TRUE if found, FALSE if not. */
+bool mail_index_keyword_lookup(struct mail_index *index,
+ const char *keyword, unsigned int *idx_r);
+void mail_index_keyword_lookup_or_create(struct mail_index *index,
+ const char *keyword,
+ unsigned int *idx_r);
+/* Return a pointer to array of NULL-terminated list of keywords. Note that
+ the array contents (and thus pointers inside it) may change after calling
+ mail_index_keywords_create() or mail_index_sync_begin(). */
+const ARRAY_TYPE(keywords) *mail_index_get_keywords(struct mail_index *index);
+
+/* Create a keyword list structure. */
+struct mail_keywords *
+mail_index_keywords_create(struct mail_index *index,
+ const char *const keywords[]) ATTR_NULL(2);
+struct mail_keywords *
+mail_index_keywords_create_from_indexes(struct mail_index *index,
+ const ARRAY_TYPE(keyword_indexes)
+ *keyword_indexes);
+void mail_index_keywords_ref(struct mail_keywords *keywords);
+void mail_index_keywords_unref(struct mail_keywords **keywords);
+
+/* Update keywords for given message. */
+void mail_index_update_keywords(struct mail_index_transaction *t, uint32_t seq,
+ enum modify_type modify_type,
+ struct mail_keywords *keywords);
+
+/* Update field in header. If prepend is TRUE, the header change is visible
+ before message syncing begins. */
+void mail_index_update_header(struct mail_index_transaction *t,
+ size_t offset, const void *data, size_t size,
+ bool prepend);
+
+/* Returns the full error message for last error. This message may
+ contain paths etc. so it shouldn't be shown to users. */
+const char *mail_index_get_error_message(struct mail_index *index);
+/* Reset the error message. */
+void mail_index_reset_error(struct mail_index *index);
+
+/* Apply changes in MAIL_INDEX_SYNC_TYPE_FLAGS typed sync records to given
+ flags variable. */
+void mail_index_sync_flags_apply(const struct mail_index_sync_rec *sync_rec,
+ uint8_t *flags);
+/* Apply changes in MAIL_INDEX_SYNC_TYPE_KEYWORD_* typed sync records to given
+ keywords array. Returns TRUE If something was changed. */
+bool mail_index_sync_keywords_apply(const struct mail_index_sync_rec *sync_rec,
+ ARRAY_TYPE(keyword_indexes) *keywords);
+
+/* register index extension. name is a unique identifier for the extension.
+ returns unique identifier for the name. */
+uint32_t mail_index_ext_register(struct mail_index *index, const char *name,
+ uint32_t default_hdr_size,
+ uint16_t default_record_size,
+ uint16_t default_record_align);
+/* Change an already registered extension's default sizes. */
+void mail_index_ext_register_resize_defaults(struct mail_index *index,
+ uint32_t ext_id,
+ uint32_t default_hdr_size,
+ uint16_t default_record_size,
+ uint16_t default_record_align);
+/* Returns TRUE and sets ext_id_r if extension with given name is registered. */
+bool mail_index_ext_lookup(struct mail_index *index, const char *name,
+ uint32_t *ext_id_r);
+/* Resize existing extension data. If size is grown, the new data will be
+ zero-filled. If size is shrinked, the data is simply dropped. */
+void mail_index_ext_resize(struct mail_index_transaction *t, uint32_t ext_id,
+ uint32_t hdr_size, uint16_t record_size,
+ uint16_t record_align);
+/* Resize header, keeping the old record size. */
+void mail_index_ext_resize_hdr(struct mail_index_transaction *t,
+ uint32_t ext_id, uint32_t hdr_size);
+
+/* Reset extension. Any updates for this extension which were issued before the
+ writer had seen this reset are discarded. reset_id is used to figure this
+ out, so it must be different every time. If clear_data=TRUE, records and
+ header is zeroed. */
+void mail_index_ext_reset(struct mail_index_transaction *t, uint32_t ext_id,
+ uint32_t reset_id, bool clear_data);
+/* Like mail_index_ext_reset(), but increase extension's reset_id atomically
+ when the transaction is being committed. If prev_reset_id doesn't match the
+ latest reset_id, the reset_id isn't increased and all extension changes are
+ ignored. */
+void mail_index_ext_reset_inc(struct mail_index_transaction *t, uint32_t ext_id,
+ uint32_t prev_reset_id, bool clear_data);
+/* Discard existing extension updates in this transaction and write new updates
+ using the given reset_id. The difference to mail_index_ext_reset() is that
+ this doesn't clear any existing record or header data. */
+void mail_index_ext_set_reset_id(struct mail_index_transaction *t,
+ uint32_t ext_id, uint32_t reset_id);
+/* Get the current reset_id for given extension. Returns TRUE if it exists. */
+bool mail_index_ext_get_reset_id(struct mail_index_view *view,
+ struct mail_index_map *map,
+ uint32_t ext_id, uint32_t *reset_id_r);
+
+/* Returns extension header. */
+void mail_index_get_header_ext(struct mail_index_view *view, uint32_t ext_id,
+ const void **data_r, size_t *data_size_r);
+void mail_index_map_get_header_ext(struct mail_index_view *view,
+ struct mail_index_map *map, uint32_t ext_id,
+ const void **data_r, size_t *data_size_r);
+/* Returns the wanted extension record for given message. If it doesn't exist,
+ *data_r is set to NULL. expunged_r is TRUE if the message has already been
+ expunged from the index. */
+void mail_index_lookup_ext(struct mail_index_view *view, uint32_t seq,
+ uint32_t ext_id, const void **data_r,
+ bool *expunged_r);
+void mail_index_lookup_ext_full(struct mail_index_view *view, uint32_t seq,
+ uint32_t ext_id, struct mail_index_map **map_r,
+ const void **data_r, bool *expunged_r);
+/* Get current extension sizes. Returns 1 if ok, 0 if extension doesn't exist
+ in view. Any of the _r parameters may be NULL. */
+void mail_index_ext_get_size(struct mail_index_map *map, uint32_t ext_id,
+ uint32_t *hdr_size_r, uint16_t *record_size_r,
+ uint16_t *record_align_r);
+/* Update extension header field. */
+void mail_index_update_header_ext(struct mail_index_transaction *t,
+ uint32_t ext_id, size_t offset,
+ const void *data, size_t size);
+/* Update extension record. If old_data_r is non-NULL and the record extension
+ was already updated in this transaction, it's set to contain the data it's
+ now overwriting. */
+void mail_index_update_ext(struct mail_index_transaction *t, uint32_t seq,
+ uint32_t ext_id, const void *data, void *old_data)
+ ATTR_NULL(5);
+/* Increase/decrease number in extension atomically. Returns the sum of the
+ diffs for this seq. */
+int mail_index_atomic_inc_ext(struct mail_index_transaction *t,
+ uint32_t seq, uint32_t ext_id, int diff);
+
+#endif