1 files changed, 733 insertions, 0 deletions

diff --git a/src/lib-index/mail-index-modseq.c b/src/lib-index/mail-index-modseq.c
new file mode 100644
index 0000000..4285983
--- /dev/null
+++ b/src/lib-index/mail-index-modseq.c
@@ -0,0 +1,733 @@
+/* Copyright (c) 2008-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-transaction-log-private.h"
+#include "mail-index-private.h"
+#include "mail-index-sync-private.h"
+#include "mail-index-modseq.h"
+
+ARRAY_DEFINE_TYPE(modseqs, uint64_t);
+
+enum modseq_metadata_idx {
+	/* must be in the same order as enum mail_flags */
+	METADATA_MODSEQ_IDX_ANSWERED = 0,
+	METADATA_MODSEQ_IDX_FLAGGED,
+	METADATA_MODSEQ_IDX_DELETED,
+	METADATA_MODSEQ_IDX_SEEN,
+	METADATA_MODSEQ_IDX_DRAFT,
+
+	METADATA_MODSEQ_IDX_KEYWORD_START
+};
+
+struct metadata_modseqs {
+	ARRAY_TYPE(modseqs) modseqs;
+};
+
+struct mail_index_map_modseq {
+	/* indexes use enum modseq_metadata_idx */
+	ARRAY(struct metadata_modseqs) metadata_modseqs;
+};
+
+struct mail_index_modseq_sync {
+	struct mail_index_sync_map_ctx *sync_map_ctx;
+	struct mail_index_view *view;
+	struct mail_transaction_log_view *log_view;
+	struct mail_index_map_modseq *mmap;
+};
+
+void mail_index_modseq_init(struct mail_index *index)
+{
+	index->modseq_ext_id =
+		mail_index_ext_register(index, MAIL_INDEX_MODSEQ_EXT_NAME,
+					sizeof(struct mail_index_modseq_header),
+					sizeof(uint64_t), sizeof(uint64_t));
+}
+
+static uint64_t mail_index_modseq_get_head(struct mail_index *index)
+{
+	return index->log->head == NULL ? 1 :
+		I_MAX(index->log->head->sync_highest_modseq, 1);
+}
+
+void mail_index_modseq_enable(struct mail_index *index)
+{
+	struct mail_index_transaction *trans;
+	struct mail_index_view *view;
+	struct mail_index_modseq_header hdr;
+	uint32_t ext_map_idx;
+
+	if (index->modseqs_enabled)
+		return;
+
+	if (!mail_index_map_get_ext_idx(index->map, index->modseq_ext_id,
+					&ext_map_idx)) {
+		/* modseqs not enabled to the index yet, add them. */
+		view = mail_index_view_open(index);
+		trans = mail_index_transaction_begin(view, 0);
+
+		i_zero(&hdr);
+		hdr.highest_modseq = mail_index_modseq_get_head(index);
+		mail_index_update_header_ext(trans, index->modseq_ext_id,
+					     0, &hdr, sizeof(hdr));
+
+		/* commit also refreshes the index, which syncs the modseqs */
+		(void)mail_index_transaction_commit(&trans);
+		mail_index_view_close(&view);
+
+		/* get the modseq extension to index map */
+		if (!mail_index_map_get_ext_idx(index->map,
+						index->modseq_ext_id,
+						&ext_map_idx)) {
+			/* didn't work for some reason */
+			return;
+		}
+	}
+	index->modseqs_enabled = TRUE;
+}
+
+bool mail_index_have_modseq_tracking(struct mail_index *index)
+{
+	return mail_index_map_get_modseq_header(index->map) != NULL;
+}
+
+void mail_index_modseq_hdr_snapshot_update(struct mail_index_map *map)
+{
+	const struct mail_index_modseq_header *hdr =
+		mail_index_map_get_modseq_header(map);
+	if (hdr != NULL)
+		map->modseq_hdr_snapshot = *hdr;
+	else
+		i_zero(&map->modseq_hdr_snapshot);
+}
+
+const struct mail_index_modseq_header *
+mail_index_map_get_modseq_header(struct mail_index_map *map)
+{
+	const struct mail_index_ext *ext;
+	uint32_t idx;
+
+	if (!mail_index_map_get_ext_idx(map, map->index->modseq_ext_id, &idx))
+		return NULL;
+
+	ext = array_idx(&map->extensions, idx);
+	if (ext->hdr_size != sizeof(struct mail_index_modseq_header))
+		return NULL;
+
+	return MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
+}
+
+uint64_t mail_index_map_modseq_get_highest(struct mail_index_map *map)
+{
+	const struct mail_index_modseq_header *modseq_hdr;
+
+	modseq_hdr = mail_index_map_get_modseq_header(map);
+	if (modseq_hdr != NULL && modseq_hdr->highest_modseq != 0)
+		return modseq_hdr->highest_modseq;
+	else {
+		/* fallback to returning the log head. if modseqs aren't
+		   enabled, we return 0. */
+		return map->index->log->head == NULL ? 0 :
+			map->index->log->head->sync_highest_modseq;
+	}
+}
+
+uint64_t mail_index_modseq_get_highest(struct mail_index_view *view)
+{
+	return mail_index_map_modseq_get_highest(view->map);
+}
+
+static struct mail_index_map_modseq *
+mail_index_map_modseq(struct mail_index_view *view)
+{
+	struct mail_index_map_modseq *mmap = view->map->rec_map->modseq;
+	uint32_t ext_map_idx;
+
+	if (mmap != NULL)
+		return mmap;
+
+	/* don't start tracking until we've seen modseq extension intro */
+	if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id,
+					&ext_map_idx))
+		return NULL;
+
+	mmap = i_new(struct mail_index_map_modseq, 1);
+	i_array_init(&mmap->metadata_modseqs,
+		     METADATA_MODSEQ_IDX_KEYWORD_START +
+		     array_count(&view->index->keywords));
+	view->map->rec_map->modseq = mmap;
+	return mmap;
+}
+
+uint64_t mail_index_modseq_lookup(struct mail_index_view *view, uint32_t seq)
+{
+	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
+	struct mail_index_map *map;
+	const struct mail_index_ext *ext;
+	const struct mail_index_record *rec;
+	const uint64_t *modseqp;
+	uint32_t ext_map_idx;
+
+	if (mmap == NULL)
+		return mail_index_modseq_get_head(view->index);
+
+	rec = mail_index_lookup_full(view, seq, &map, NULL);
+	if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id,
+					&ext_map_idx)) {
+		/* not enabled yet */
+		return mail_index_modseq_get_head(view->index);
+	}
+
+	ext = array_idx(&map->extensions, ext_map_idx);
+	modseqp = CONST_PTR_OFFSET(rec, ext->record_offset);
+	if (*modseqp == 0) {
+		/* If we're here because we just enabled modseqs, we'll return
+		   the same modseq (initial highestmodseq) for all messages.
+		   The next sync will change these zeros to initial
+		   highestmodseq or higher.
+
+		   If we're here because a message got appended but modseq
+		   wasn't set (older Dovecot?), we'll again use the current
+		   highest modseq. This isn't exactly correct, but it gets
+		   fixed after the next sync and this situation shouldn't
+		   normally happen anyway. */
+		return mail_index_modseq_get_highest(view);
+	}
+	return *modseqp;
+}
+
+int mail_index_modseq_set(struct mail_index_view *view,
+			  uint32_t seq, uint64_t min_modseq)
+{
+	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
+	const struct mail_index_ext *ext;
+	struct mail_index_record *rec;
+	uint64_t *modseqp;
+	uint32_t ext_map_idx;
+
+	if (mmap == NULL)
+		return -1;
+
+	rec = MAIL_INDEX_REC_AT_SEQ(view->map, seq);
+	if (!mail_index_map_get_ext_idx(view->map, view->index->modseq_ext_id,
+					&ext_map_idx))
+		return -1;
+
+	ext = array_idx(&view->map->extensions, ext_map_idx);
+	modseqp = PTR_OFFSET(rec, ext->record_offset);
+	if (*modseqp > min_modseq)
+		return 0;
+	else {
+		*modseqp = min_modseq;
+		return 1;
+	}
+}
+
+static uint64_t
+modseq_idx_lookup(struct mail_index_map_modseq *mmap,
+		  unsigned int idx, uint32_t seq)
+{
+	const struct metadata_modseqs *metadata;
+	const uint64_t *modseqs;
+	unsigned int count;
+
+	metadata = array_get(&mmap->metadata_modseqs, &count);
+	if (idx >= count || !array_is_created(&metadata[idx].modseqs))
+		return 0;
+
+	modseqs = array_get(&metadata[idx].modseqs, &count);
+	return seq > count ? 0 : modseqs[seq-1];
+}
+
+uint64_t mail_index_modseq_lookup_flags(struct mail_index_view *view,
+					enum mail_flags flags_mask,
+					uint32_t seq)
+{
+	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
+	unsigned int i;
+	uint64_t modseq, highest_modseq = 0;
+
+	if (mmap != NULL) {
+		/* first try to find a specific match */
+		for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) {
+			if ((flags_mask & (1 << i)) != 0) {
+				modseq = modseq_idx_lookup(mmap, i, seq);
+				if (highest_modseq < modseq)
+					highest_modseq = modseq;
+			}
+		}
+	}
+
+	if (highest_modseq == 0) {
+		/* no specific matches, fallback to using the highest */
+		highest_modseq = mail_index_modseq_lookup(view, seq);
+	}
+	return highest_modseq;
+}
+
+uint64_t mail_index_modseq_lookup_keywords(struct mail_index_view *view,
+					   const struct mail_keywords *keywords,
+					   uint32_t seq)
+{
+	struct mail_index_map_modseq *mmap = mail_index_map_modseq(view);
+	unsigned int i, metadata_idx;
+	uint64_t modseq, highest_modseq = 0;
+
+	if (mmap != NULL) {
+		/* first try to find a specific match */
+		for (i = 0; i < keywords->count; i++) {
+			metadata_idx = METADATA_MODSEQ_IDX_KEYWORD_START +
+				keywords->idx[i];
+
+			modseq = modseq_idx_lookup(mmap, metadata_idx, seq);
+			if (highest_modseq < modseq)
+				highest_modseq = modseq;
+		}
+	}
+
+	if (highest_modseq == 0) {
+		/* no specific matches, fallback to using the highest */
+		highest_modseq = mail_index_modseq_lookup(view, seq);
+	}
+	return highest_modseq;
+}
+
+static void
+mail_index_modseq_update(struct mail_index_modseq_sync *ctx,
+			 uint64_t modseq, bool nonzeros,
+			 uint32_t seq1, uint32_t seq2)
+{
+	const struct mail_index_ext *ext;
+	struct mail_index_record *rec;
+	uint32_t ext_map_idx;
+	uint64_t *modseqp;
+
+	if (!mail_index_map_get_ext_idx(ctx->view->map,
+					ctx->view->index->modseq_ext_id,
+					&ext_map_idx))
+		return;
+
+	ext = array_idx(&ctx->view->map->extensions, ext_map_idx);
+	for (; seq1 <= seq2; seq1++) {
+		rec = MAIL_INDEX_REC_AT_SEQ(ctx->view->map, seq1);
+		modseqp = PTR_OFFSET(rec, ext->record_offset);
+		if (*modseqp == 0 || (nonzeros && *modseqp < modseq))
+			*modseqp = modseq;
+	}
+}
+
+static bool
+mail_index_modseq_update_to_highest(struct mail_index_modseq_sync *ctx,
+				    uint32_t seq1, uint32_t seq2)
+{
+	uint64_t modseq;
+
+	if (ctx->mmap == NULL)
+		return FALSE;
+
+	modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view);
+	mail_index_modseq_update(ctx, modseq, TRUE, seq1, seq2);
+	return TRUE;
+}
+
+static void
+mail_index_modseq_update_old_rec(struct mail_index_modseq_sync *ctx,
+				 const struct mail_transaction_header *thdr,
+				 const void *tdata)
+{
+	ARRAY_TYPE(seq_range) uids = ARRAY_INIT;
+	const struct seq_range *rec;
+	buffer_t uid_buf;
+	unsigned int i, count;
+	uint32_t seq1, seq2;
+
+	switch (thdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+	case MAIL_TRANSACTION_APPEND: {
+		const struct mail_index_record *appends = tdata;
+
+		count = thdr->size / sizeof(*appends);
+		for (i = 0; i < count; i++) {
+			if (mail_index_lookup_seq(ctx->view,
+						  appends[i].uid, &seq1)) {
+				(void)mail_index_modseq_update_to_highest(ctx, seq1, seq1);
+			}
+		}
+		return;
+	}
+	case MAIL_TRANSACTION_FLAG_UPDATE: {
+		buffer_create_from_const_data(&uid_buf, tdata, thdr->size);
+		array_create_from_buffer(&uids, &uid_buf,
+			sizeof(struct mail_transaction_flag_update));
+		break;
+	}
+	case MAIL_TRANSACTION_KEYWORD_UPDATE: {
+		const struct mail_transaction_keyword_update *rec = tdata;
+		unsigned int seqset_offset;
+
+		seqset_offset = sizeof(*rec) + rec->name_size;
+		if ((seqset_offset % 4) != 0)
+			seqset_offset += 4 - (seqset_offset % 4);
+
+		buffer_create_from_const_data(&uid_buf,
+					 CONST_PTR_OFFSET(tdata, seqset_offset),
+					 thdr->size - seqset_offset);
+		array_create_from_buffer(&uids, &uid_buf, sizeof(uint32_t)*2);
+		break;
+	}
+	case MAIL_TRANSACTION_KEYWORD_RESET:
+		buffer_create_from_const_data(&uid_buf, tdata, thdr->size);
+		array_create_from_buffer(&uids, &uid_buf,
+			sizeof(struct mail_transaction_keyword_reset));
+		break;
+	case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
+		break;
+	default:
+		return;
+	}
+
+	/* update modseqs */
+	count = array_is_created(&uids) ? array_count(&uids) : 0;
+	for (i = 0; i < count; i++) {
+		rec = array_idx(&uids, i);
+		if (mail_index_lookup_seq_range(ctx->view, rec->seq1, rec->seq2,
+						&seq1, &seq2))
+			(void)mail_index_modseq_update_to_highest(ctx, seq1, seq2);
+	}
+}
+
+static void mail_index_modseq_sync_init(struct mail_index_modseq_sync *ctx)
+{
+	struct mail_index_map *map = ctx->view->map;
+	const struct mail_index_ext *ext;
+	const struct mail_index_modseq_header *hdr;
+	const struct mail_transaction_header *thdr;
+	const void *tdata;
+	const char *reason;
+	uint32_t ext_map_idx;
+	uint32_t end_seq;
+	uoff_t end_offset;
+	uint64_t cur_modseq;
+	bool reset;
+	int ret;
+
+	if (!mail_index_map_get_ext_idx(map, ctx->view->index->modseq_ext_id,
+					&ext_map_idx))
+		i_unreached();
+	ext = array_idx(&map->extensions, ext_map_idx);
+
+	/* get the current highest_modseq. don't change any modseq below it. */
+	hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
+
+	/* Scan logs for updates between ext_hdr.log_* .. view position.
+	   There are two reasons why there could be any:
+
+	   1) We just enabled modseqs and we're filling the initial values.
+	   2) A non-modseq-aware Dovecot version added new messages and wrote
+	      dovecot.index file. */
+	mail_transaction_log_view_get_prev_pos(ctx->view->log_view,
+					       &end_seq, &end_offset);
+	if (end_seq < hdr->log_seq ||
+	    (end_seq == hdr->log_seq && end_offset <= hdr->log_offset)) {
+		/* modseqs are up to date */
+		return;
+	}
+
+	ctx->log_view = mail_transaction_log_view_open(ctx->view->index->log);
+	ret = mail_transaction_log_view_set(ctx->log_view,
+					    I_MAX(1, hdr->log_seq),
+					    hdr->log_offset,
+					    end_seq, end_offset, &reset, &reason);
+	if (ret <= 0) {
+		/* missing files / error - try with only the last file */
+		ret = mail_transaction_log_view_set(ctx->log_view, end_seq, 0,
+						    end_seq, end_offset,
+						    &reset, &reason);
+		/* since we don't know if we skipped some changes, set all
+		   modseqs to beginning of the latest file. */
+		cur_modseq = mail_transaction_log_view_get_prev_modseq(
+								ctx->log_view);
+		if (cur_modseq < hdr->highest_modseq) {
+			/* should happen only when setting initial modseqs.
+			   we may already have returned highest_modseq as
+			   some messages' modseq value. don't shrink it. */
+			cur_modseq = hdr->highest_modseq;
+		}
+		mail_index_modseq_update(ctx, cur_modseq, TRUE, 1,
+					 map->hdr.messages_count);
+	} else {
+		/* we have all the logs. replace zero modseqs with the current
+		   highest modseq (we may have already returned it for them). */
+		mail_index_modseq_update(ctx, hdr->highest_modseq, FALSE, 1,
+					 map->hdr.messages_count);
+	}
+	if (ret > 0) {
+		while (mail_transaction_log_view_next(ctx->log_view,
+						      &thdr, &tdata) > 0) {
+			T_BEGIN {
+				mail_index_modseq_update_old_rec(ctx, thdr,
+								 tdata);
+			} T_END;
+		}
+	}
+	mail_transaction_log_view_close(&ctx->log_view);
+}
+
+struct mail_index_modseq_sync *
+mail_index_modseq_sync_begin(struct mail_index_sync_map_ctx *sync_map_ctx)
+{
+	struct mail_index_modseq_sync *ctx;
+
+	ctx = i_new(struct mail_index_modseq_sync, 1);
+	ctx->sync_map_ctx = sync_map_ctx;
+	ctx->view = sync_map_ctx->view;
+	ctx->mmap = mail_index_map_modseq(ctx->view);
+	if (ctx->mmap != NULL) {
+		mail_index_modseq_sync_init(ctx);
+		ctx->log_view = ctx->view->log_view;
+	}
+	return ctx;
+}
+
+static void mail_index_modseq_update_header(struct mail_index_modseq_sync *ctx)
+{
+	struct mail_index_view *view = ctx->view;
+	struct mail_index_map *map = view->map;
+	const struct mail_index_ext *ext;
+	const struct mail_index_modseq_header *old_modseq_hdr;
+	struct mail_index_modseq_header new_modseq_hdr;
+	uint32_t ext_map_idx, log_seq;
+	uoff_t log_offset;
+	uint64_t highest_modseq;
+
+	if (!mail_index_map_get_ext_idx(map, view->index->modseq_ext_id,
+					&ext_map_idx))
+		return;
+
+	mail_transaction_log_view_get_prev_pos(view->log_view,
+					       &log_seq, &log_offset);
+	highest_modseq = mail_transaction_log_view_get_prev_modseq(view->log_view);
+
+	ext = array_idx(&map->extensions, ext_map_idx);
+	old_modseq_hdr = MAIL_INDEX_MAP_HDR_OFFSET(map, ext->hdr_offset);
+
+	if (old_modseq_hdr->log_seq < log_seq ||
+	    (old_modseq_hdr->log_seq == log_seq &&
+	     old_modseq_hdr->log_offset < log_offset)) {
+		i_zero(&new_modseq_hdr);
+		new_modseq_hdr.highest_modseq = highest_modseq;
+		new_modseq_hdr.log_seq = log_seq;
+		new_modseq_hdr.log_offset = log_offset;
+
+		buffer_write(map->hdr_copy_buf, ext->hdr_offset,
+			     &new_modseq_hdr, sizeof(new_modseq_hdr));
+		i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
+	}
+}
+
+void mail_index_modseq_sync_end(struct mail_index_modseq_sync **_ctx)
+{
+	struct mail_index_modseq_sync *ctx = *_ctx;
+
+	*_ctx = NULL;
+	if (ctx->mmap != NULL) {
+		i_assert(ctx->mmap == ctx->view->map->rec_map->modseq);
+		mail_index_modseq_update_header(ctx);
+	}
+	i_free(ctx);
+}
+
+void mail_index_modseq_sync_map_replaced(struct mail_index_modseq_sync *ctx)
+{
+	ctx->mmap = mail_index_map_modseq(ctx->view);
+}
+
+void mail_index_modseq_hdr_update(struct mail_index_modseq_sync *ctx)
+{
+	if (ctx->mmap == NULL) {
+		ctx->mmap = mail_index_map_modseq(ctx->view);
+		i_assert(ctx->mmap != NULL);
+		mail_index_modseq_sync_init(ctx);
+		ctx->log_view = ctx->view->log_view;
+	}
+}
+
+void mail_index_modseq_append(struct mail_index_modseq_sync *ctx, uint32_t seq)
+{
+	(void)mail_index_modseq_update_to_highest(ctx, seq, seq);
+}
+
+void mail_index_modseq_expunge(struct mail_index_modseq_sync *ctx,
+			       uint32_t seq1, uint32_t seq2)
+{
+	struct metadata_modseqs *metadata;
+
+	if (ctx->mmap == NULL)
+		return;
+
+	seq1--;
+	array_foreach_modifiable(&ctx->mmap->metadata_modseqs, metadata) {
+		if (array_is_created(&metadata->modseqs))
+			array_delete(&metadata->modseqs, seq1, seq2-seq1);
+	}
+}
+
+static void
+modseqs_update(ARRAY_TYPE(modseqs) *array, uint32_t seq1, uint32_t seq2,
+	       uint64_t value)
+{
+	uint64_t *modseqp;
+
+	for (; seq1 <= seq2; seq1++) {
+		modseqp = array_idx_get_space(array, seq1-1);
+		if (*modseqp < value)
+			*modseqp = value;
+	}
+}
+
+static void
+modseqs_idx_update(struct mail_index_modseq_sync *ctx, unsigned int idx,
+		   uint32_t seq1, uint32_t seq2)
+{
+	struct metadata_modseqs *metadata;
+	uint64_t modseq;
+
+	if (!ctx->view->index->modseqs_enabled) {
+		/* we want to keep permanent modseqs updated, but don't bother
+		   updating in-memory per-flag updates */
+		return;
+	}
+
+	modseq = mail_transaction_log_view_get_prev_modseq(ctx->log_view);
+	metadata = array_idx_get_space(&ctx->mmap->metadata_modseqs, idx);
+	if (!array_is_created(&metadata->modseqs))
+		i_array_init(&metadata->modseqs, seq2 + 16);
+	modseqs_update(&metadata->modseqs, seq1, seq2, modseq);
+}
+
+void mail_index_modseq_update_flags(struct mail_index_modseq_sync *ctx,
+				    enum mail_flags flags_mask,
+				    uint32_t seq1, uint32_t seq2)
+{
+	unsigned int i;
+
+	if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
+		return;
+
+	for (i = 0; i < METADATA_MODSEQ_IDX_KEYWORD_START; i++) {
+		if ((flags_mask & (1 << i)) != 0)
+			modseqs_idx_update(ctx, i, seq1, seq2);
+	}
+}
+
+void mail_index_modseq_update_keyword(struct mail_index_modseq_sync *ctx,
+				      unsigned int keyword_idx,
+				      uint32_t seq1, uint32_t seq2)
+{
+	if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
+		return;
+
+	modseqs_idx_update(ctx, METADATA_MODSEQ_IDX_KEYWORD_START + keyword_idx,
+			   seq1, seq2);
+}
+
+void mail_index_modseq_reset_keywords(struct mail_index_modseq_sync *ctx,
+				      uint32_t seq1, uint32_t seq2)
+{
+	unsigned int i, count;
+
+	if (!mail_index_modseq_update_to_highest(ctx, seq1, seq2))
+		return;
+
+	count = array_count(&ctx->mmap->metadata_modseqs);
+	for (i = METADATA_MODSEQ_IDX_KEYWORD_START; i < count; i++)
+		modseqs_idx_update(ctx, i, seq1, seq2);
+}
+
+struct mail_index_map_modseq *
+mail_index_map_modseq_clone(const struct mail_index_map_modseq *mmap)
+{
+	struct mail_index_map_modseq *new_mmap;
+	const struct metadata_modseqs *src_metadata;
+	struct metadata_modseqs *dest_metadata;
+	unsigned int i, count;
+
+	src_metadata = array_get(&mmap->metadata_modseqs, &count);
+
+	new_mmap = i_new(struct mail_index_map_modseq, 1);
+	i_array_init(&new_mmap->metadata_modseqs, count + 16);
+
+	for (i = 0; i < count; i++) {
+		dest_metadata = array_append_space(&new_mmap->metadata_modseqs);
+		if (array_is_created(&src_metadata[i].modseqs)) {
+			i_array_init(&dest_metadata->modseqs,
+				     array_count(&src_metadata[i].modseqs));
+			array_append_array(&dest_metadata->modseqs,
+					   &src_metadata[i].modseqs);
+		}
+	}
+	return new_mmap;
+}
+
+void mail_index_map_modseq_free(struct mail_index_map_modseq **_mmap)
+{
+	struct mail_index_map_modseq *mmap = *_mmap;
+	struct metadata_modseqs *metadata;
+
+	*_mmap = NULL;
+
+	array_foreach_modifiable(&mmap->metadata_modseqs, metadata) {
+		if (array_is_created(&metadata->modseqs))
+			array_free(&metadata->modseqs);
+	}
+	array_free(&mmap->metadata_modseqs);
+	i_free(mmap);
+}
+
+bool mail_index_modseq_get_next_log_offset(struct mail_index_view *view,
+					   uint64_t modseq, uint32_t *log_seq_r,
+					   uoff_t *log_offset_r)
+{
+	struct mail_transaction_log *log = view->index->log;
+	struct mail_transaction_log_file *file, *prev_file;
+	const char *reason;
+	int ret;
+
+	if (log->files == NULL) {
+		/* we shouldn't normally get here */
+		return FALSE;
+	}
+	while (modseq < log->files->hdr.initial_modseq) {
+		/* try to find the previous log file if it still exists */
+		ret = mail_transaction_log_find_file(log,
+			log->files->hdr.file_seq - 1, FALSE, &file, &reason);
+		if (ret <= 0)
+			return FALSE;
+	}
+
+	prev_file = NULL;
+	for (file = log->files; file != NULL; file = file->next) {
+		if (modseq < file->hdr.initial_modseq)
+			break;
+		prev_file = file;
+	}
+
+	if (prev_file == NULL) {
+		/* the log file has been deleted already */
+		return FALSE;
+	}
+
+	*log_seq_r = prev_file->hdr.file_seq;
+	if (mail_transaction_log_file_get_modseq_next_offset(prev_file, modseq,
+							     log_offset_r) < 0)
+		return FALSE;
+
+	if (*log_seq_r > view->log_file_head_seq ||
+	    (*log_seq_r == view->log_file_head_seq &&
+	     *log_offset_r > view->log_file_head_offset)) {
+		/* modseq is already beyond our view. move it back so the
+		   caller won't be confused. */
+		*log_seq_r = view->log_file_head_seq;
+		*log_offset_r = view->log_file_head_offset;
+	}
+	return TRUE;
+}