Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1045 insertions, 0 deletions

diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
new file mode 100644
index 0000000..d8a5793
--- /dev/null
+++ b/src/lib-index/mail-index-view-sync.c
@@ -0,0 +1,1045 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "mail-index-view-private.h"
+#include "mail-index-sync-private.h"
+#include "mail-index-modseq.h"
+#include "mail-transaction-log.h"
+
+
+struct mail_index_view_sync_ctx {
+	struct mail_index_view *view;
+	enum mail_index_view_sync_flags flags;
+	struct mail_index_sync_map_ctx sync_map_ctx;
+
+	/* After syncing view, map is replaced with sync_new_map. */
+	struct mail_index_map *sync_new_map;
+
+	ARRAY_TYPE(seq_range) expunges;
+	unsigned int finish_min_msg_count;
+
+	const struct mail_transaction_header *hdr;
+	const void *data;
+
+	/* temporary variables while handling lost transaction logs: */
+	ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
+	buffer_t *lost_kw_buf;
+	uint32_t lost_new_ext_idx;
+	/* result of lost transaction logs: */
+	ARRAY_TYPE(seq_range) lost_flags;
+	unsigned int lost_flag_idx;
+
+	size_t data_offset;
+	bool failed:1;
+	bool sync_map_update:1;
+	bool skipped_expunges:1;
+	bool last_read:1;
+	bool log_was_lost:1;
+	bool hidden:1;
+};
+
+static int
+view_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
+			     bool *reset_r, bool *partial_sync_r,
+			     const char **error_r)
+{
+	const struct mail_index_header *hdr = &view->index->map->hdr;
+	uint32_t start_seq, end_seq;
+	uoff_t start_offset, end_offset;
+	const char *reason;
+	int ret;
+
+	*partial_sync_r = FALSE;
+
+	if (sync_expunges) {
+		/* Sync everything after the last expunge syncing position.
+		   We'll just skip over the non-expunge transaction records
+		   that have already been synced previously. */
+		start_seq = view->log_file_expunge_seq;
+		start_offset = view->log_file_expunge_offset;
+	} else {
+		/* Sync only new changes since the last view sync. */
+		start_seq = view->log_file_head_seq;
+		start_offset = view->log_file_head_offset;
+	}
+	/* Sync the view up to the (already refreshed) index map. */
+        end_seq = hdr->log_file_seq;
+        end_offset = hdr->log_file_head_offset;
+
+	if (end_seq < view->log_file_head_seq ||
+	    (end_seq == view->log_file_head_seq &&
+	     end_offset < view->log_file_head_offset)) {
+		*error_r = t_strdup_printf(
+			"%s log position went backwards "
+			"(%u,%"PRIuUOFF_T" < %u,%"PRIuUOFF_T")",
+			view->index->filepath, end_seq, end_offset,
+			view->log_file_head_seq, view->log_file_head_offset);
+		return -1;
+	}
+
+	for (;;) {
+		/* the view begins from the first non-synced transaction */
+		ret = mail_transaction_log_view_set(view->log_view,
+						    start_seq, start_offset,
+						    end_seq, end_offset,
+						    reset_r, &reason);
+		if (ret <= 0) {
+			*error_r = t_strdup_printf(
+				"Failed to map view for %s: %s",
+				view->index->filepath, reason);
+			return ret;
+		}
+
+		if (!*reset_r || sync_expunges)
+			break;
+
+		/* log was reset, but we don't want to sync expunges.
+		   we can't do this, so sync only up to the reset. */
+		mail_transaction_log_view_get_prev_pos(view->log_view,
+						       &end_seq, &end_offset);
+		end_seq--; end_offset = UOFF_T_MAX;
+		if (end_seq < start_seq) {
+			/* we have only this reset log */
+			mail_transaction_log_view_clear(view->log_view,
+				view->log_file_expunge_seq);
+			break;
+		}
+		*partial_sync_r = TRUE;
+	}
+	return 1;
+}
+
+static unsigned int
+view_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
+{
+	struct mail_index_view *view = ctx->view;
+	struct seq_range *src, *src_end, *dest;
+	unsigned int count, expunge_count = 0;
+	uint32_t prev_seq = 0;
+
+	/* convert UIDs to sequences */
+	src = dest = array_get_modifiable(&ctx->expunges, &count);
+	src_end = src + count;
+	for (; src != src_end; src++) {
+		if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
+						 &dest->seq1, &dest->seq2))
+			count--;
+		else {
+			i_assert(dest->seq1 > prev_seq);
+			prev_seq = dest->seq2;
+
+			expunge_count += dest->seq2 - dest->seq1 + 1;
+			dest++;
+		}
+	}
+	array_delete(&ctx->expunges, count,
+		     array_count(&ctx->expunges) - count);
+	return expunge_count;
+}
+
+static void
+view_sync_add_expunge_range(ARRAY_TYPE(seq_range) *dest,
+			    const struct seq_range *src, size_t src_size)
+{
+	unsigned int i, src_count;
+
+	i_assert(src_size % sizeof(*src) == 0);
+
+	src_count = src_size / sizeof(*src);
+	for (i = 0; i < src_count; i++)
+		seq_range_array_add_range(dest, src[i].seq1, src[i].seq2);
+}
+
+static void
+view_sync_add_expunge_guids(ARRAY_TYPE(seq_range) *dest,
+			    const struct mail_transaction_expunge_guid *src,
+			    size_t src_size)
+{
+	unsigned int i, src_count;
+
+	i_assert(src_size % sizeof(*src) == 0);
+
+	src_count = src_size / sizeof(*src);
+	for (i = 0; i < src_count; i++)
+		seq_range_array_add(dest, src[i].uid);
+}
+
+static int
+view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
+		       unsigned int *expunge_count_r)
+{
+	struct mail_index_view *view = ctx->view;
+	const struct mail_transaction_header *hdr;
+	const void *data;
+	int ret;
+
+	/* get a list of expunge transactions. there may be some that we have
+	   already synced, but it doesn't matter because they'll get dropped
+	   out when converting to sequences. the uid ranges' validity has
+	   already been verified, so we can use them directly. */
+	mail_transaction_log_view_mark(view->log_view);
+	while ((ret = mail_transaction_log_view_next(view->log_view,
+						     &hdr, &data)) > 0) {
+		if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+			/* skip expunge requests */
+			continue;
+		}
+		if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
+			view_sync_add_expunge_guids(&ctx->expunges,
+						    data, hdr->size);
+		} else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
+			view_sync_add_expunge_range(&ctx->expunges,
+						    data, hdr->size);
+		}
+	}
+	mail_transaction_log_view_rewind(view->log_view);
+
+	*expunge_count_r = view_sync_expunges2seqs(ctx);
+	return ret;
+}
+
+static bool have_existing_expunges(struct mail_index_view *view,
+				   const struct seq_range *range, size_t size)
+{
+	const struct seq_range *range_end;
+	uint32_t seq1, seq2;
+
+	range_end = CONST_PTR_OFFSET(range, size);
+	for (; range != range_end; range++) {
+		if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
+						&seq1, &seq2))
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static bool
+have_existing_guid_expunge(struct mail_index_view *view,
+			   const struct mail_transaction_expunge_guid *expunges,
+			   size_t size)
+{
+	const struct mail_transaction_expunge_guid *expunges_end;
+	uint32_t seq;
+
+	expunges_end = CONST_PTR_OFFSET(expunges, size);
+	for (; expunges != expunges_end; expunges++) {
+		if (mail_index_lookup_seq(view, expunges->uid, &seq))
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static bool view_sync_have_expunges(struct mail_index_view *view)
+{
+	const struct mail_transaction_header *hdr;
+	const void *data;
+	bool have_expunges = FALSE;
+	int ret;
+
+	if (mail_transaction_log_view_is_last(view->log_view))
+		return FALSE;
+
+	mail_transaction_log_view_mark(view->log_view);
+
+	while ((ret = mail_transaction_log_view_next(view->log_view,
+						     &hdr, &data)) > 0) {
+		if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+			/* skip expunge requests */
+			continue;
+		}
+		if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
+			/* we have an expunge. see if it still exists. */
+			if (have_existing_guid_expunge(view, data, hdr->size)) {
+				have_expunges = TRUE;
+				break;
+			}
+		} else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
+			/* we have an expunge. see if it still exists. */
+			if (have_existing_expunges(view, data, hdr->size)) {
+				have_expunges = TRUE;
+				break;
+			}
+		}
+	}
+
+	mail_transaction_log_view_rewind(view->log_view);
+
+	/* handle failures as having expunges (which is safer).
+	   we'll probably fail later. */
+	return ret < 0 || have_expunges;
+}
+
+static int uint_cmp(const void *p1, const void *p2)
+{
+	const unsigned int *u1 = p1, *u2 = p2;
+
+	if (*u1 < *u2)
+		return -1;
+	if (*u1 > *u2)
+		return 1;
+	return 0;
+}
+
+static bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
+{
+	unsigned int *old_idx, *new_idx;
+	unsigned int old_count, new_count;
+
+	old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
+	new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
+	if (old_count != new_count)
+		return FALSE;
+
+	qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
+	qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
+	return memcmp(old_idx, new_idx, old_count * sizeof(*old_idx)) == 0;
+}
+
+static int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
+				     uint32_t uid)
+{
+	struct mail_transaction_header thdr;
+	struct mail_transaction_keyword_update kw_up;
+	const unsigned int *kw_idx;
+	const char *const *kw_names;
+	unsigned int i, count;
+
+	kw_idx = array_get(&ctx->lost_new_kw, &count);
+	if (count == 0)
+		return 0;
+	kw_names = array_front(&ctx->view->index->keywords);
+
+	i_zero(&thdr);
+	thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
+	i_zero(&kw_up);
+	kw_up.modify_type = MODIFY_ADD;
+	/* add new flags one by one */
+	for (i = 0; i < count; i++) {
+		kw_up.name_size = strlen(kw_names[kw_idx[i]]);
+		buffer_set_used_size(ctx->lost_kw_buf, 0);
+		buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
+		buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
+			      kw_up.name_size);
+		if (ctx->lost_kw_buf->used % 4 != 0) {
+			buffer_append_zero(ctx->lost_kw_buf,
+					   4 - ctx->lost_kw_buf->used % 4);
+		}
+		buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
+		buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
+
+		thdr.size = ctx->lost_kw_buf->used;
+		if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+					   ctx->lost_kw_buf->data) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
+					uint32_t old_seq, uint32_t new_seq)
+{
+	struct mail_index_map *old_map = ctx->view->map;
+	struct mail_index_map *new_map = ctx->view->index->map;
+	const struct mail_index_record *old_rec, *new_rec;
+	struct mail_transaction_header thdr;
+	const struct mail_index_ext *ext;
+	const uint64_t *modseqp;
+	uint64_t new_modseq;
+	bool changed = FALSE;
+
+	old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, old_seq);
+	new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, new_seq);
+
+	i_zero(&thdr);
+	if (old_rec->flags != new_rec->flags) {
+		struct mail_transaction_flag_update flag_update;
+
+		/* check this before syncing the record, since it updates
+		   old_rec. */
+		if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
+		    (new_rec->flags & MAIL_INDEX_FLAGS_MASK))
+			changed = TRUE;
+
+		thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
+			MAIL_TRANSACTION_EXTERNAL;
+		thdr.size = sizeof(flag_update);
+
+		i_zero(&flag_update);
+		flag_update.uid1 = flag_update.uid2 = new_rec->uid;
+		flag_update.add_flags = new_rec->flags;
+		flag_update.remove_flags = ENUM_NEGATE(new_rec->flags) & 0xff;
+		if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+					   &flag_update) < 0)
+			return -1;
+	}
+
+	mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw);
+	mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw);
+	if (!view_sync_lost_keywords_equal(ctx)) {
+		struct mail_transaction_keyword_reset kw_reset;
+
+		thdr.type = MAIL_TRANSACTION_KEYWORD_RESET |
+			MAIL_TRANSACTION_EXTERNAL;
+		thdr.size = sizeof(kw_reset);
+
+		/* remove all old flags by resetting them */
+		i_zero(&kw_reset);
+		kw_reset.uid1 = kw_reset.uid2 = new_rec->uid;
+		if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+					   &kw_reset) < 0)
+			return -1;
+
+		if (view_sync_update_keywords(ctx, new_rec->uid) < 0)
+			return -1;
+		changed = TRUE;
+	}
+
+	if (changed) {
+		/* flags or keywords changed */
+	} else if (ctx->view->highest_modseq != 0 &&
+		   ctx->lost_new_ext_idx != (uint32_t)-1) {
+		/* if modseq has changed include this message in changed flags
+		   list, even if we didn't see any changes above. */
+		ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
+		modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
+		new_modseq = *modseqp;
+
+		if (new_modseq > ctx->view->highest_modseq)
+			changed = TRUE;
+	}
+
+	/* without modseqs lost_flags isn't updated perfectly correctly, because
+	   by the time we're comparing old flags it may have changed from what
+	   we last sent to the client (because the map is shared). This could
+	   be avoided by always keeping a private copy of the map in the view,
+	   but that's a waste of memory for as rare of a problem as this. */
+	if (changed)
+		seq_range_array_add(&ctx->lost_flags, new_rec->uid);
+	return 0;
+}
+
+static int
+view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
+			       unsigned int *expunge_count_r)
+{
+	struct mail_index_view *view = ctx->view;
+	struct mail_index_map *old_map = view->map;
+	struct mail_index_map *new_map = view->index->map;
+	const unsigned int old_count = old_map->hdr.messages_count;
+	const unsigned int new_count = new_map->hdr.messages_count;
+	const struct mail_index_record *old_rec, *new_rec;
+	struct mail_transaction_header thdr;
+	uint32_t seqi, seqj;
+
+	/* we don't update the map in the same order as it's typically done.
+	   map->rec_map may already have some messages appended that we don't
+	   want. get an atomic map to make sure these get removed. */
+	(void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
+
+	if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
+					&ctx->lost_new_ext_idx))
+		ctx->lost_new_ext_idx = (uint32_t)-1;
+
+	i_array_init(&ctx->lost_flags, 64);
+	t_array_init(&ctx->lost_old_kw, 32);
+	t_array_init(&ctx->lost_new_kw, 32);
+	ctx->lost_kw_buf = t_buffer_create(128);
+
+	/* handle expunges and sync flags */
+	seqi = seqj = 1;
+	while (seqi <= old_count && seqj <= new_count) {
+		old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi);
+		new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj);
+		if (old_rec->uid == new_rec->uid) {
+			/* message found - check if flags have changed */
+			if (view_sync_apply_lost_changes(ctx, seqi, seqj) < 0)
+				return -1;
+			seqi++; seqj++;
+		} else if (old_rec->uid < new_rec->uid) {
+			/* message expunged */
+			seq_range_array_add(&ctx->expunges, old_rec->uid);
+			seqi++;
+		} else {
+			/* new message appeared out of nowhere */
+			mail_index_set_error(view->index,
+				"%s view is inconsistent: "
+				"uid=%u inserted in the middle of mailbox",
+				view->index->filepath, new_rec->uid);
+			return -1;
+		}
+	}
+	/* if there are old messages left, they're all expunged */
+	for (; seqi <= old_count; seqi++) {
+		old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi);
+		seq_range_array_add(&ctx->expunges, old_rec->uid);
+	}
+	/* if there are new messages left, they're all new messages */
+	thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL;
+	thdr.size = sizeof(*new_rec);
+	for (; seqj <= new_count; seqj++) {
+		new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj);
+		if (mail_index_sync_record(&ctx->sync_map_ctx,
+					   &thdr, new_rec) < 0)
+			return -1;
+		mail_index_map_lookup_keywords(new_map, seqj,
+					       &ctx->lost_new_kw);
+		if (view_sync_update_keywords(ctx, new_rec->uid) < 0)
+			return -1;
+	}
+	*expunge_count_r = view_sync_expunges2seqs(ctx);
+
+	/* we have no idea how far we've synced - make sure these aren't used */
+	old_map->hdr.log_file_seq = 0;
+	old_map->hdr.log_file_head_offset = 0;
+	old_map->hdr.log_file_tail_offset = 0;
+
+	if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
+		/* Expunges aren't wanted to be synced. Remember if we skipped
+		   over any expunges. If yes, we must not update
+		   log_file_expunge_seq/offset at the end of the view sync
+		   so that a later sync can finish the expunges. */
+		array_clear(&ctx->expunges);
+		ctx->skipped_expunges = *expunge_count_r > 0;
+	}
+	/* After the view sync is finished, update
+	   log_file_head_seq/offset, since we've synced everything
+	   (except maybe the expunges) up to this point. */
+	view->log_file_head_seq = new_map->hdr.log_file_seq;
+	view->log_file_head_offset = new_map->hdr.log_file_head_offset;
+	return 0;
+}
+
+static int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx)
+{
+	struct mail_index_view *view = ctx->view;
+	uint32_t seq;
+	uoff_t offset;
+	const char *reason;
+	bool reset;
+	int ret;
+
+	/* replace the view's map */
+	view->index->map->refcount++;
+	mail_index_unmap(&view->map);
+	view->map = view->index->map;
+
+	/* update log positions */
+	view->log_file_head_seq = seq = view->map->hdr.log_file_seq;
+	view->log_file_head_offset = offset =
+		view->map->hdr.log_file_head_offset;
+
+	ret = mail_transaction_log_view_set(view->log_view, seq, offset,
+					    seq, offset, &reset, &reason);
+	if (ret <= 0) {
+		mail_index_set_error(view->index, "Failed to fix view for %s: %s",
+				     view->index->filepath, reason);
+		return ret;
+	}
+	view->inconsistent = FALSE;
+	return 0;
+}
+
+struct mail_index_view_sync_ctx *
+mail_index_view_sync_begin(struct mail_index_view *view,
+			   enum mail_index_view_sync_flags flags)
+{
+	struct mail_index_view_sync_ctx *ctx;
+	struct mail_index_map *tmp_map;
+	unsigned int expunge_count = 0;
+	bool reset, partial_sync, sync_expunges, have_expunges;
+	const char *error;
+	int ret;
+
+	i_assert(!view->syncing);
+	i_assert(view->transactions_list == NULL);
+
+	view->syncing = TRUE;
+
+	/* Syncing the view invalidates all previous looked up records.
+	   Unreference the mappings this view keeps because of them. */
+	mail_index_view_unref_maps(view);
+
+	ctx = i_new(struct mail_index_view_sync_ctx, 1);
+	ctx->view = view;
+	ctx->flags = flags;
+
+	sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
+	if (sync_expunges)
+		i_array_init(&ctx->expunges, 64);
+	if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
+		/* just get this view synced - don't return anything */
+		i_assert(sync_expunges);
+		if (mail_index_view_sync_init_fix(ctx) < 0)
+			ctx->failed = TRUE;
+		return ctx;
+	}
+	if (mail_index_view_is_inconsistent(view)) {
+		mail_index_set_error(view->index, "%s view is inconsistent",
+				     view->index->filepath);
+		ctx->failed = TRUE;
+		return ctx;
+	}
+
+	ret = view_sync_set_log_view_range(view, sync_expunges, &reset,
+					   &partial_sync, &error);
+	if (ret < 0) {
+		mail_index_set_error(view->index, "%s", error);
+		ctx->failed = TRUE;
+		return ctx;
+	}
+
+	if (ret == 0) {
+		/* Log the warning only when all expunges have been synced
+		   by previous syncs. This way when there's a _FLAG_NOEXPUNGES
+		   sync, there's no second warning logged when the expunges
+		   finally are synced. */
+		if (view->log_file_expunge_seq == view->log_file_head_seq &&
+		    view->log_file_expunge_offset == view->log_file_head_offset) {
+			e_warning(view->index->event,
+				  "%s - generating missing logs", error);
+		}
+		ctx->log_was_lost = TRUE;
+		if (!sync_expunges)
+			i_array_init(&ctx->expunges, 64);
+		mail_index_sync_map_init(&ctx->sync_map_ctx, view,
+					 MAIL_INDEX_SYNC_HANDLER_VIEW);
+		ret = view_sync_get_log_lost_changes(ctx, &expunge_count);
+		mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
+		mail_index_sync_map_deinit(&ctx->sync_map_ctx);
+		if (ret < 0) {
+			mail_index_set_error(view->index,
+				"%s view syncing failed to apply changes",
+				view->index->filepath);
+			view->inconsistent = TRUE;
+			ctx->failed = TRUE;
+			return ctx;
+		}
+		have_expunges = expunge_count > 0;
+	} else if (sync_expunges) {
+		/* get list of all expunges first */
+		if (view_sync_get_expunges(ctx, &expunge_count) < 0) {
+			ctx->failed = TRUE;
+			return ctx;
+		}
+		have_expunges = expunge_count > 0;
+	} else if (view->log_file_expunge_seq == view->log_file_head_seq &&
+		   view->log_file_expunge_offset == view->log_file_head_offset) {
+		/* Previous syncs haven't left any pending expunges. See if
+		   this sync will. */
+		have_expunges = view_sync_have_expunges(view);
+	} else {
+		/* Expunges weren't synced in the previous sync either.
+		   We already know there are missing expunges. */
+		ctx->skipped_expunges = TRUE;
+		have_expunges = TRUE;
+	}
+
+	ctx->finish_min_msg_count = reset ? 0 :
+		view->map->hdr.messages_count - expunge_count;
+	if (!reset)
+		;
+	else if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX) != 0 &&
+		 view->map->hdr.messages_count == 0) {
+			/* The secondary index is still empty, so it may have
+			   just been created for the first time. This is
+			   expected, so it shouldn't cause the view to become
+			   inconsistent. */
+			if (mail_index_view_sync_init_fix(ctx) < 0)
+				ctx->failed = TRUE;
+			return ctx;
+	} else {
+		view->inconsistent = TRUE;
+		mail_index_set_error(view->index,
+				     "%s reset, view is now inconsistent",
+				     view->index->filepath);
+		ctx->failed = TRUE;
+		return ctx;
+	}
+
+	if (!have_expunges && !partial_sync) {
+		/* no expunges, we can just replace the map */
+		if (view->index->map->hdr.messages_count <
+		    ctx->finish_min_msg_count) {
+			mail_index_set_error(view->index,
+				"Index %s lost messages without expunging "
+				"(%u -> %u)", view->index->filepath,
+				view->map->hdr.messages_count,
+				view->index->map->hdr.messages_count);
+			ctx->finish_min_msg_count = 0;
+			view->inconsistent = TRUE;
+		}
+
+		view->index->map->refcount++;
+		mail_index_unmap(&view->map);
+		view->map = view->index->map;
+	} else {
+		/* a) expunges seen. b) doing a partial sync because we saw
+		   a reset.
+
+		   Create a private map which we update. If we're syncing
+		   expunges the map will finally be replaced with the head map
+		   to remove the expunged messages. */
+		ctx->sync_map_update = TRUE;
+
+		if (view->map->refcount > 1) {
+			tmp_map = mail_index_map_clone(view->map);
+			mail_index_unmap(&view->map);
+			view->map = tmp_map;
+		}
+
+		if (sync_expunges) {
+			ctx->sync_new_map = view->index->map;
+			ctx->sync_new_map->refcount++;
+		}
+	}
+	mail_index_sync_map_init(&ctx->sync_map_ctx, view,
+				 MAIL_INDEX_SYNC_HANDLER_VIEW);
+
+#ifdef DEBUG
+	mail_index_map_check(view->map);
+#endif
+	return ctx;
+}
+
+static bool
+view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
+{
+	const struct mail_index_view_log_sync_area *sync;
+
+	if (!array_is_created(&view->syncs_hidden))
+		return FALSE;
+
+	array_foreach(&view->syncs_hidden, sync) {
+		if (sync->log_file_offset <= offset &&
+		    offset - sync->log_file_offset < sync->length &&
+		    sync->log_file_seq == seq)
+			return TRUE;
+	}
+
+	return FALSE;
+}
+
+static bool
+mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
+			  const struct mail_transaction_header *hdr)
+{
+        struct mail_index_view *view = ctx->view;
+	uint32_t seq;
+	uoff_t offset, next_offset;
+
+	mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
+	next_offset = offset + sizeof(*hdr) + hdr->size;
+
+	if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
+			  MAIL_TRANSACTION_EXPUNGE_GUID)) != 0 &&
+	    (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
+		if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
+			i_assert(!LOG_IS_BEFORE(seq, offset,
+						view->log_file_expunge_seq,
+						view->log_file_expunge_offset));
+			if (!ctx->skipped_expunges) {
+				view->log_file_expunge_seq = seq;
+				view->log_file_expunge_offset = offset;
+				ctx->skipped_expunges = TRUE;
+			}
+			return FALSE;
+		}
+		if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
+				  view->log_file_expunge_offset)) {
+			/* already synced */
+			return FALSE;
+		}
+	}
+
+	if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
+			  view->log_file_head_offset)) {
+		/* already synced */
+		return FALSE;
+	}
+
+	view->log_file_head_seq = seq;
+	view->log_file_head_offset = next_offset;
+	return TRUE;
+}
+
+static int
+mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
+{
+        struct mail_transaction_log_view *log_view = ctx->view->log_view;
+	struct mail_index_view *view = ctx->view;
+	const struct mail_transaction_header *hdr;
+	uint32_t seq;
+	uoff_t offset;
+	int ret;
+	bool synced_to_map;
+
+	do {
+		/* Get the next transaction from log. */
+		ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
+						     &ctx->data);
+		if (ret <= 0) {
+			if (ret < 0)
+				return -1;
+
+			ctx->hdr = NULL;
+			ctx->last_read = TRUE;
+			return 0;
+		}
+
+		hdr = ctx->hdr;
+		/* skip records we've already synced */
+	} while (!mail_index_view_sync_want(ctx, hdr));
+
+	mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
+
+	/* If we started from a map that we didn't create ourself,
+	   some of the transactions may already be synced. at the end
+	   of this view sync we'll update file_seq=0 so that this check
+	   always becomes FALSE for subsequent syncs. */
+	synced_to_map = view->map->hdr.log_file_seq != 0 &&
+		LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq,
+			      view->map->hdr.log_file_head_offset);
+
+	/* Apply transaction to view's mapping if needed (meaning we
+	   didn't just re-map the view to head mapping). */
+	if (ctx->sync_map_update && !synced_to_map) {
+		if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
+				  MAIL_TRANSACTION_EXPUNGE_GUID)) == 0) {
+			ret = mail_index_sync_record(&ctx->sync_map_ctx,
+						     hdr, ctx->data);
+		}
+		if (ret < 0)
+			return -1;
+	}
+
+	ctx->hidden = view_sync_is_hidden(view, seq, offset);
+	return 1;
+}
+
+static bool
+mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
+			     struct mail_index_view_sync_rec *rec)
+{
+	const struct mail_transaction_header *hdr = ctx->hdr;
+	const void *data = ctx->data;
+
+	switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+	case MAIL_TRANSACTION_FLAG_UPDATE: {
+		const struct mail_transaction_flag_update *update =
+			CONST_PTR_OFFSET(data, ctx->data_offset);
+
+		/* data contains mail_transaction_flag_update[] */
+		for (;;) {
+			ctx->data_offset += sizeof(*update);
+			if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update))
+				break;
+
+			/* skip internal flag changes */
+			if (ctx->data_offset == ctx->hdr->size)
+				return FALSE;
+
+			update = CONST_PTR_OFFSET(data, ctx->data_offset);
+		}
+
+		if (update->add_flags != 0 || update->remove_flags != 0)
+			rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+		else
+			rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ;
+		rec->uid1 = update->uid1;
+		rec->uid2 = update->uid2;
+		break;
+	}
+	case MAIL_TRANSACTION_KEYWORD_UPDATE: {
+		const struct mail_transaction_keyword_update *update = data;
+		const uint32_t *uids;
+
+		/* data contains mail_transaction_keyword_update header,
+		   the keyword name and an array of { uint32_t uid1, uid2; } */
+
+		if (ctx->data_offset == 0) {
+			/* skip over the header and name */
+			ctx->data_offset = sizeof(*update) + update->name_size;
+			if ((ctx->data_offset % 4) != 0)
+				ctx->data_offset += 4 - (ctx->data_offset % 4);
+		}
+
+		uids = CONST_PTR_OFFSET(data, ctx->data_offset);
+		rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+		rec->uid1 = uids[0];
+		rec->uid2 = uids[1];
+
+		ctx->data_offset += sizeof(uint32_t) * 2;
+		break;
+	}
+	case MAIL_TRANSACTION_KEYWORD_RESET: {
+		const struct mail_transaction_keyword_reset *reset =
+			CONST_PTR_OFFSET(data, ctx->data_offset);
+
+		/* data contains mail_transaction_keyword_reset[] */
+		rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+		rec->uid1 = reset->uid1;
+		rec->uid2 = reset->uid2;
+		ctx->data_offset += sizeof(*reset);
+		break;
+	}
+	default:
+		ctx->hdr = NULL;
+		return FALSE;
+	}
+
+	rec->hidden = ctx->hidden;
+	return TRUE;
+}
+
+static bool
+mail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx,
+			       struct mail_index_view_sync_rec *sync_rec)
+{
+	const struct seq_range *range;
+	unsigned int count;
+
+	range = array_get(&ctx->lost_flags, &count);
+	if (ctx->lost_flag_idx == count) {
+		ctx->last_read = TRUE;
+		return FALSE;
+	}
+
+	sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+	sync_rec->uid1 = range[ctx->lost_flag_idx].seq1;
+	sync_rec->uid2 = range[ctx->lost_flag_idx].seq2;
+	sync_rec->hidden = FALSE;
+	ctx->lost_flag_idx++;
+	return TRUE;
+}
+
+bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
+			       struct mail_index_view_sync_rec *sync_rec)
+{
+	int ret;
+
+	if (ctx->log_was_lost)
+		return mail_index_view_sync_next_lost(ctx, sync_rec);
+
+	do {
+		if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
+			ret = mail_index_view_sync_get_next_transaction(ctx);
+			if (ret <= 0) {
+				if (ret < 0)
+					ctx->failed = TRUE;
+				return FALSE;
+			}
+
+			ctx->data_offset = 0;
+		}
+	} while (!mail_index_view_sync_get_rec(ctx, sync_rec));
+
+	return TRUE;
+}
+
+void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
+				       const ARRAY_TYPE(seq_range) **expunges_r)
+{
+	*expunges_r = &ctx->expunges;
+}
+
+static void
+mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
+{
+	const struct mail_index_view_log_sync_area *syncs;
+	unsigned int i, count;
+
+	if (!array_is_created(&view->syncs_hidden))
+		return;
+
+	/* Clean up to view's tail */
+	syncs = array_get(&view->syncs_hidden, &count);
+	for (i = 0; i < count; i++) {
+		if ((syncs[i].log_file_offset +
+		     syncs[i].length > view->log_file_expunge_offset &&
+                     syncs[i].log_file_seq == view->log_file_expunge_seq) ||
+		    syncs[i].log_file_seq > view->log_file_expunge_seq)
+			break;
+	}
+	if (i > 0)
+		array_delete(&view->syncs_hidden, 0, i);
+}
+
+int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
+				bool *delayed_expunges_r)
+{
+        struct mail_index_view_sync_ctx *ctx = *_ctx;
+        struct mail_index_view *view = ctx->view;
+	int ret = ctx->failed ? -1 : 0;
+
+	i_assert(view->syncing);
+
+	*_ctx = NULL;
+	*delayed_expunges_r = ctx->skipped_expunges;
+
+	if ((!ctx->last_read || view->inconsistent) &&
+	    (ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
+		/* we didn't sync everything */
+		view->inconsistent = TRUE;
+		ret = -1;
+	}
+	if (ctx->sync_map_ctx.modseq_ctx != NULL)
+		mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
+
+	if (ctx->sync_new_map != NULL) {
+		mail_index_unmap(&view->map);
+		view->map = ctx->sync_new_map;
+	} else if (ctx->sync_map_update) {
+		/* log offsets have no meaning in views. make sure they're not
+		   tried to be used wrong by setting them to zero. */
+		view->map->hdr.log_file_seq = 0;
+		view->map->hdr.log_file_head_offset = 0;
+		view->map->hdr.log_file_tail_offset = 0;
+	}
+
+	i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
+
+	if (!ctx->skipped_expunges) {
+		view->log_file_expunge_seq = view->log_file_head_seq;
+		view->log_file_expunge_offset = view->log_file_head_offset;
+	}
+
+	if (ctx->sync_map_ctx.view != NULL)
+		mail_index_sync_map_deinit(&ctx->sync_map_ctx);
+	mail_index_view_sync_clean_log_syncs(ctx->view);
+
+#ifdef DEBUG
+	mail_index_map_check(view->map);
+#endif
+
+	/* set log view to empty range so unneeded memory gets freed */
+	mail_transaction_log_view_clear(view->log_view,
+					view->log_file_expunge_seq);
+
+	if (array_is_created(&ctx->expunges))
+		array_free(&ctx->expunges);
+	if (array_is_created(&ctx->lost_flags))
+		array_free(&ctx->lost_flags);
+
+	view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
+	view->syncing = FALSE;
+	i_free(ctx);
+	return ret;
+}
+
+void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
+					    uint32_t log_file_seq,
+					    uoff_t log_file_offset,
+					    unsigned int length)
+{
+	struct mail_index_view_log_sync_area *area;
+
+	if (!array_is_created(&view->syncs_hidden))
+		i_array_init(&view->syncs_hidden, 32);
+
+	area = array_append_space(&view->syncs_hidden);
+	area->log_file_seq = log_file_seq;
+	area->log_file_offset = log_file_offset;
+	area->length = length;
+}