summaryrefslogtreecommitdiffstats
path: root/src/lib-index/mail-index-view-sync.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/lib-index/mail-index-view-sync.c1045
1 files changed, 1045 insertions, 0 deletions
diff --git a/src/lib-index/mail-index-view-sync.c b/src/lib-index/mail-index-view-sync.c
new file mode 100644
index 0000000..d8a5793
--- /dev/null
+++ b/src/lib-index/mail-index-view-sync.c
@@ -0,0 +1,1045 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "mail-index-view-private.h"
+#include "mail-index-sync-private.h"
+#include "mail-index-modseq.h"
+#include "mail-transaction-log.h"
+
+
+struct mail_index_view_sync_ctx {
+ struct mail_index_view *view;
+ enum mail_index_view_sync_flags flags;
+ struct mail_index_sync_map_ctx sync_map_ctx;
+
+ /* After syncing view, map is replaced with sync_new_map. */
+ struct mail_index_map *sync_new_map;
+
+ ARRAY_TYPE(seq_range) expunges;
+ unsigned int finish_min_msg_count;
+
+ const struct mail_transaction_header *hdr;
+ const void *data;
+
+ /* temporary variables while handling lost transaction logs: */
+ ARRAY_TYPE(keyword_indexes) lost_old_kw, lost_new_kw;
+ buffer_t *lost_kw_buf;
+ uint32_t lost_new_ext_idx;
+ /* result of lost transaction logs: */
+ ARRAY_TYPE(seq_range) lost_flags;
+ unsigned int lost_flag_idx;
+
+ size_t data_offset;
+ bool failed:1;
+ bool sync_map_update:1;
+ bool skipped_expunges:1;
+ bool last_read:1;
+ bool log_was_lost:1;
+ bool hidden:1;
+};
+
+static int
+view_sync_set_log_view_range(struct mail_index_view *view, bool sync_expunges,
+ bool *reset_r, bool *partial_sync_r,
+ const char **error_r)
+{
+ const struct mail_index_header *hdr = &view->index->map->hdr;
+ uint32_t start_seq, end_seq;
+ uoff_t start_offset, end_offset;
+ const char *reason;
+ int ret;
+
+ *partial_sync_r = FALSE;
+
+ if (sync_expunges) {
+ /* Sync everything after the last expunge syncing position.
+ We'll just skip over the non-expunge transaction records
+ that have already been synced previously. */
+ start_seq = view->log_file_expunge_seq;
+ start_offset = view->log_file_expunge_offset;
+ } else {
+ /* Sync only new changes since the last view sync. */
+ start_seq = view->log_file_head_seq;
+ start_offset = view->log_file_head_offset;
+ }
+ /* Sync the view up to the (already refreshed) index map. */
+ end_seq = hdr->log_file_seq;
+ end_offset = hdr->log_file_head_offset;
+
+ if (end_seq < view->log_file_head_seq ||
+ (end_seq == view->log_file_head_seq &&
+ end_offset < view->log_file_head_offset)) {
+ *error_r = t_strdup_printf(
+ "%s log position went backwards "
+ "(%u,%"PRIuUOFF_T" < %u,%"PRIuUOFF_T")",
+ view->index->filepath, end_seq, end_offset,
+ view->log_file_head_seq, view->log_file_head_offset);
+ return -1;
+ }
+
+ for (;;) {
+ /* the view begins from the first non-synced transaction */
+ ret = mail_transaction_log_view_set(view->log_view,
+ start_seq, start_offset,
+ end_seq, end_offset,
+ reset_r, &reason);
+ if (ret <= 0) {
+ *error_r = t_strdup_printf(
+ "Failed to map view for %s: %s",
+ view->index->filepath, reason);
+ return ret;
+ }
+
+ if (!*reset_r || sync_expunges)
+ break;
+
+ /* log was reset, but we don't want to sync expunges.
+ we can't do this, so sync only up to the reset. */
+ mail_transaction_log_view_get_prev_pos(view->log_view,
+ &end_seq, &end_offset);
+ end_seq--; end_offset = UOFF_T_MAX;
+ if (end_seq < start_seq) {
+ /* we have only this reset log */
+ mail_transaction_log_view_clear(view->log_view,
+ view->log_file_expunge_seq);
+ break;
+ }
+ *partial_sync_r = TRUE;
+ }
+ return 1;
+}
+
+static unsigned int
+view_sync_expunges2seqs(struct mail_index_view_sync_ctx *ctx)
+{
+ struct mail_index_view *view = ctx->view;
+ struct seq_range *src, *src_end, *dest;
+ unsigned int count, expunge_count = 0;
+ uint32_t prev_seq = 0;
+
+ /* convert UIDs to sequences */
+ src = dest = array_get_modifiable(&ctx->expunges, &count);
+ src_end = src + count;
+ for (; src != src_end; src++) {
+ if (!mail_index_lookup_seq_range(view, src->seq1, src->seq2,
+ &dest->seq1, &dest->seq2))
+ count--;
+ else {
+ i_assert(dest->seq1 > prev_seq);
+ prev_seq = dest->seq2;
+
+ expunge_count += dest->seq2 - dest->seq1 + 1;
+ dest++;
+ }
+ }
+ array_delete(&ctx->expunges, count,
+ array_count(&ctx->expunges) - count);
+ return expunge_count;
+}
+
+static void
+view_sync_add_expunge_range(ARRAY_TYPE(seq_range) *dest,
+ const struct seq_range *src, size_t src_size)
+{
+ unsigned int i, src_count;
+
+ i_assert(src_size % sizeof(*src) == 0);
+
+ src_count = src_size / sizeof(*src);
+ for (i = 0; i < src_count; i++)
+ seq_range_array_add_range(dest, src[i].seq1, src[i].seq2);
+}
+
+static void
+view_sync_add_expunge_guids(ARRAY_TYPE(seq_range) *dest,
+ const struct mail_transaction_expunge_guid *src,
+ size_t src_size)
+{
+ unsigned int i, src_count;
+
+ i_assert(src_size % sizeof(*src) == 0);
+
+ src_count = src_size / sizeof(*src);
+ for (i = 0; i < src_count; i++)
+ seq_range_array_add(dest, src[i].uid);
+}
+
+static int
+view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
+ unsigned int *expunge_count_r)
+{
+ struct mail_index_view *view = ctx->view;
+ const struct mail_transaction_header *hdr;
+ const void *data;
+ int ret;
+
+ /* get a list of expunge transactions. there may be some that we have
+ already synced, but it doesn't matter because they'll get dropped
+ out when converting to sequences. the uid ranges' validity has
+ already been verified, so we can use them directly. */
+ mail_transaction_log_view_mark(view->log_view);
+ while ((ret = mail_transaction_log_view_next(view->log_view,
+ &hdr, &data)) > 0) {
+ if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+ /* skip expunge requests */
+ continue;
+ }
+ if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
+ view_sync_add_expunge_guids(&ctx->expunges,
+ data, hdr->size);
+ } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
+ view_sync_add_expunge_range(&ctx->expunges,
+ data, hdr->size);
+ }
+ }
+ mail_transaction_log_view_rewind(view->log_view);
+
+ *expunge_count_r = view_sync_expunges2seqs(ctx);
+ return ret;
+}
+
+static bool have_existing_expunges(struct mail_index_view *view,
+ const struct seq_range *range, size_t size)
+{
+ const struct seq_range *range_end;
+ uint32_t seq1, seq2;
+
+ range_end = CONST_PTR_OFFSET(range, size);
+ for (; range != range_end; range++) {
+ if (mail_index_lookup_seq_range(view, range->seq1, range->seq2,
+ &seq1, &seq2))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static bool
+have_existing_guid_expunge(struct mail_index_view *view,
+ const struct mail_transaction_expunge_guid *expunges,
+ size_t size)
+{
+ const struct mail_transaction_expunge_guid *expunges_end;
+ uint32_t seq;
+
+ expunges_end = CONST_PTR_OFFSET(expunges, size);
+ for (; expunges != expunges_end; expunges++) {
+ if (mail_index_lookup_seq(view, expunges->uid, &seq))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static bool view_sync_have_expunges(struct mail_index_view *view)
+{
+ const struct mail_transaction_header *hdr;
+ const void *data;
+ bool have_expunges = FALSE;
+ int ret;
+
+ if (mail_transaction_log_view_is_last(view->log_view))
+ return FALSE;
+
+ mail_transaction_log_view_mark(view->log_view);
+
+ while ((ret = mail_transaction_log_view_next(view->log_view,
+ &hdr, &data)) > 0) {
+ if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+ /* skip expunge requests */
+ continue;
+ }
+ if ((hdr->type & MAIL_TRANSACTION_EXPUNGE_GUID) != 0) {
+ /* we have an expunge. see if it still exists. */
+ if (have_existing_guid_expunge(view, data, hdr->size)) {
+ have_expunges = TRUE;
+ break;
+ }
+ } else if ((hdr->type & MAIL_TRANSACTION_EXPUNGE) != 0) {
+ /* we have an expunge. see if it still exists. */
+ if (have_existing_expunges(view, data, hdr->size)) {
+ have_expunges = TRUE;
+ break;
+ }
+ }
+ }
+
+ mail_transaction_log_view_rewind(view->log_view);
+
+ /* handle failures as having expunges (which is safer).
+ we'll probably fail later. */
+ return ret < 0 || have_expunges;
+}
+
+static int uint_cmp(const void *p1, const void *p2)
+{
+ const unsigned int *u1 = p1, *u2 = p2;
+
+ if (*u1 < *u2)
+ return -1;
+ if (*u1 > *u2)
+ return 1;
+ return 0;
+}
+
+static bool view_sync_lost_keywords_equal(struct mail_index_view_sync_ctx *ctx)
+{
+ unsigned int *old_idx, *new_idx;
+ unsigned int old_count, new_count;
+
+ old_idx = array_get_modifiable(&ctx->lost_old_kw, &old_count);
+ new_idx = array_get_modifiable(&ctx->lost_new_kw, &new_count);
+ if (old_count != new_count)
+ return FALSE;
+
+ qsort(old_idx, old_count, sizeof(*old_idx), uint_cmp);
+ qsort(new_idx, new_count, sizeof(*new_idx), uint_cmp);
+ return memcmp(old_idx, new_idx, old_count * sizeof(*old_idx)) == 0;
+}
+
+static int view_sync_update_keywords(struct mail_index_view_sync_ctx *ctx,
+ uint32_t uid)
+{
+ struct mail_transaction_header thdr;
+ struct mail_transaction_keyword_update kw_up;
+ const unsigned int *kw_idx;
+ const char *const *kw_names;
+ unsigned int i, count;
+
+ kw_idx = array_get(&ctx->lost_new_kw, &count);
+ if (count == 0)
+ return 0;
+ kw_names = array_front(&ctx->view->index->keywords);
+
+ i_zero(&thdr);
+ thdr.type = MAIL_TRANSACTION_KEYWORD_UPDATE | MAIL_TRANSACTION_EXTERNAL;
+ i_zero(&kw_up);
+ kw_up.modify_type = MODIFY_ADD;
+ /* add new flags one by one */
+ for (i = 0; i < count; i++) {
+ kw_up.name_size = strlen(kw_names[kw_idx[i]]);
+ buffer_set_used_size(ctx->lost_kw_buf, 0);
+ buffer_append(ctx->lost_kw_buf, &kw_up, sizeof(kw_up));
+ buffer_append(ctx->lost_kw_buf, kw_names[kw_idx[i]],
+ kw_up.name_size);
+ if (ctx->lost_kw_buf->used % 4 != 0) {
+ buffer_append_zero(ctx->lost_kw_buf,
+ 4 - ctx->lost_kw_buf->used % 4);
+ }
+ buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
+ buffer_append(ctx->lost_kw_buf, &uid, sizeof(uid));
+
+ thdr.size = ctx->lost_kw_buf->used;
+ if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+ ctx->lost_kw_buf->data) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static int view_sync_apply_lost_changes(struct mail_index_view_sync_ctx *ctx,
+ uint32_t old_seq, uint32_t new_seq)
+{
+ struct mail_index_map *old_map = ctx->view->map;
+ struct mail_index_map *new_map = ctx->view->index->map;
+ const struct mail_index_record *old_rec, *new_rec;
+ struct mail_transaction_header thdr;
+ const struct mail_index_ext *ext;
+ const uint64_t *modseqp;
+ uint64_t new_modseq;
+ bool changed = FALSE;
+
+ old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, old_seq);
+ new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, new_seq);
+
+ i_zero(&thdr);
+ if (old_rec->flags != new_rec->flags) {
+ struct mail_transaction_flag_update flag_update;
+
+ /* check this before syncing the record, since it updates
+ old_rec. */
+ if ((old_rec->flags & MAIL_INDEX_FLAGS_MASK) !=
+ (new_rec->flags & MAIL_INDEX_FLAGS_MASK))
+ changed = TRUE;
+
+ thdr.type = MAIL_TRANSACTION_FLAG_UPDATE |
+ MAIL_TRANSACTION_EXTERNAL;
+ thdr.size = sizeof(flag_update);
+
+ i_zero(&flag_update);
+ flag_update.uid1 = flag_update.uid2 = new_rec->uid;
+ flag_update.add_flags = new_rec->flags;
+ flag_update.remove_flags = ENUM_NEGATE(new_rec->flags) & 0xff;
+ if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+ &flag_update) < 0)
+ return -1;
+ }
+
+ mail_index_map_lookup_keywords(old_map, old_seq, &ctx->lost_old_kw);
+ mail_index_map_lookup_keywords(new_map, new_seq, &ctx->lost_new_kw);
+ if (!view_sync_lost_keywords_equal(ctx)) {
+ struct mail_transaction_keyword_reset kw_reset;
+
+ thdr.type = MAIL_TRANSACTION_KEYWORD_RESET |
+ MAIL_TRANSACTION_EXTERNAL;
+ thdr.size = sizeof(kw_reset);
+
+ /* remove all old flags by resetting them */
+ i_zero(&kw_reset);
+ kw_reset.uid1 = kw_reset.uid2 = new_rec->uid;
+ if (mail_index_sync_record(&ctx->sync_map_ctx, &thdr,
+ &kw_reset) < 0)
+ return -1;
+
+ if (view_sync_update_keywords(ctx, new_rec->uid) < 0)
+ return -1;
+ changed = TRUE;
+ }
+
+ if (changed) {
+ /* flags or keywords changed */
+ } else if (ctx->view->highest_modseq != 0 &&
+ ctx->lost_new_ext_idx != (uint32_t)-1) {
+ /* if modseq has changed include this message in changed flags
+ list, even if we didn't see any changes above. */
+ ext = array_idx(&new_map->extensions, ctx->lost_new_ext_idx);
+ modseqp = CONST_PTR_OFFSET(new_rec, ext->record_offset);
+ new_modseq = *modseqp;
+
+ if (new_modseq > ctx->view->highest_modseq)
+ changed = TRUE;
+ }
+
+ /* without modseqs lost_flags isn't updated perfectly correctly, because
+ by the time we're comparing old flags it may have changed from what
+ we last sent to the client (because the map is shared). This could
+ be avoided by always keeping a private copy of the map in the view,
+ but that's a waste of memory for as rare of a problem as this. */
+ if (changed)
+ seq_range_array_add(&ctx->lost_flags, new_rec->uid);
+ return 0;
+}
+
+static int
+view_sync_get_log_lost_changes(struct mail_index_view_sync_ctx *ctx,
+ unsigned int *expunge_count_r)
+{
+ struct mail_index_view *view = ctx->view;
+ struct mail_index_map *old_map = view->map;
+ struct mail_index_map *new_map = view->index->map;
+ const unsigned int old_count = old_map->hdr.messages_count;
+ const unsigned int new_count = new_map->hdr.messages_count;
+ const struct mail_index_record *old_rec, *new_rec;
+ struct mail_transaction_header thdr;
+ uint32_t seqi, seqj;
+
+ /* we don't update the map in the same order as it's typically done.
+ map->rec_map may already have some messages appended that we don't
+ want. get an atomic map to make sure these get removed. */
+ (void)mail_index_sync_get_atomic_map(&ctx->sync_map_ctx);
+
+ if (!mail_index_map_get_ext_idx(new_map, view->index->modseq_ext_id,
+ &ctx->lost_new_ext_idx))
+ ctx->lost_new_ext_idx = (uint32_t)-1;
+
+ i_array_init(&ctx->lost_flags, 64);
+ t_array_init(&ctx->lost_old_kw, 32);
+ t_array_init(&ctx->lost_new_kw, 32);
+ ctx->lost_kw_buf = t_buffer_create(128);
+
+ /* handle expunges and sync flags */
+ seqi = seqj = 1;
+ while (seqi <= old_count && seqj <= new_count) {
+ old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi);
+ new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj);
+ if (old_rec->uid == new_rec->uid) {
+ /* message found - check if flags have changed */
+ if (view_sync_apply_lost_changes(ctx, seqi, seqj) < 0)
+ return -1;
+ seqi++; seqj++;
+ } else if (old_rec->uid < new_rec->uid) {
+ /* message expunged */
+ seq_range_array_add(&ctx->expunges, old_rec->uid);
+ seqi++;
+ } else {
+ /* new message appeared out of nowhere */
+ mail_index_set_error(view->index,
+ "%s view is inconsistent: "
+ "uid=%u inserted in the middle of mailbox",
+ view->index->filepath, new_rec->uid);
+ return -1;
+ }
+ }
+ /* if there are old messages left, they're all expunged */
+ for (; seqi <= old_count; seqi++) {
+ old_rec = MAIL_INDEX_REC_AT_SEQ(old_map, seqi);
+ seq_range_array_add(&ctx->expunges, old_rec->uid);
+ }
+ /* if there are new messages left, they're all new messages */
+ thdr.type = MAIL_TRANSACTION_APPEND | MAIL_TRANSACTION_EXTERNAL;
+ thdr.size = sizeof(*new_rec);
+ for (; seqj <= new_count; seqj++) {
+ new_rec = MAIL_INDEX_REC_AT_SEQ(new_map, seqj);
+ if (mail_index_sync_record(&ctx->sync_map_ctx,
+ &thdr, new_rec) < 0)
+ return -1;
+ mail_index_map_lookup_keywords(new_map, seqj,
+ &ctx->lost_new_kw);
+ if (view_sync_update_keywords(ctx, new_rec->uid) < 0)
+ return -1;
+ }
+ *expunge_count_r = view_sync_expunges2seqs(ctx);
+
+ /* we have no idea how far we've synced - make sure these aren't used */
+ old_map->hdr.log_file_seq = 0;
+ old_map->hdr.log_file_head_offset = 0;
+ old_map->hdr.log_file_tail_offset = 0;
+
+ if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
+ /* Expunges aren't wanted to be synced. Remember if we skipped
+ over any expunges. If yes, we must not update
+ log_file_expunge_seq/offset at the end of the view sync
+ so that a later sync can finish the expunges. */
+ array_clear(&ctx->expunges);
+ ctx->skipped_expunges = *expunge_count_r > 0;
+ }
+ /* After the view sync is finished, update
+ log_file_head_seq/offset, since we've synced everything
+ (except maybe the expunges) up to this point. */
+ view->log_file_head_seq = new_map->hdr.log_file_seq;
+ view->log_file_head_offset = new_map->hdr.log_file_head_offset;
+ return 0;
+}
+
+static int mail_index_view_sync_init_fix(struct mail_index_view_sync_ctx *ctx)
+{
+ struct mail_index_view *view = ctx->view;
+ uint32_t seq;
+ uoff_t offset;
+ const char *reason;
+ bool reset;
+ int ret;
+
+ /* replace the view's map */
+ view->index->map->refcount++;
+ mail_index_unmap(&view->map);
+ view->map = view->index->map;
+
+ /* update log positions */
+ view->log_file_head_seq = seq = view->map->hdr.log_file_seq;
+ view->log_file_head_offset = offset =
+ view->map->hdr.log_file_head_offset;
+
+ ret = mail_transaction_log_view_set(view->log_view, seq, offset,
+ seq, offset, &reset, &reason);
+ if (ret <= 0) {
+ mail_index_set_error(view->index, "Failed to fix view for %s: %s",
+ view->index->filepath, reason);
+ return ret;
+ }
+ view->inconsistent = FALSE;
+ return 0;
+}
+
+struct mail_index_view_sync_ctx *
+mail_index_view_sync_begin(struct mail_index_view *view,
+ enum mail_index_view_sync_flags flags)
+{
+ struct mail_index_view_sync_ctx *ctx;
+ struct mail_index_map *tmp_map;
+ unsigned int expunge_count = 0;
+ bool reset, partial_sync, sync_expunges, have_expunges;
+ const char *error;
+ int ret;
+
+ i_assert(!view->syncing);
+ i_assert(view->transactions_list == NULL);
+
+ view->syncing = TRUE;
+
+ /* Syncing the view invalidates all previous looked up records.
+ Unreference the mappings this view keeps because of them. */
+ mail_index_view_unref_maps(view);
+
+ ctx = i_new(struct mail_index_view_sync_ctx, 1);
+ ctx->view = view;
+ ctx->flags = flags;
+
+ sync_expunges = (flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) == 0;
+ if (sync_expunges)
+ i_array_init(&ctx->expunges, 64);
+ if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) != 0) {
+ /* just get this view synced - don't return anything */
+ i_assert(sync_expunges);
+ if (mail_index_view_sync_init_fix(ctx) < 0)
+ ctx->failed = TRUE;
+ return ctx;
+ }
+ if (mail_index_view_is_inconsistent(view)) {
+ mail_index_set_error(view->index, "%s view is inconsistent",
+ view->index->filepath);
+ ctx->failed = TRUE;
+ return ctx;
+ }
+
+ ret = view_sync_set_log_view_range(view, sync_expunges, &reset,
+ &partial_sync, &error);
+ if (ret < 0) {
+ mail_index_set_error(view->index, "%s", error);
+ ctx->failed = TRUE;
+ return ctx;
+ }
+
+ if (ret == 0) {
+ /* Log the warning only when all expunges have been synced
+ by previous syncs. This way when there's a _FLAG_NOEXPUNGES
+ sync, there's no second warning logged when the expunges
+ finally are synced. */
+ if (view->log_file_expunge_seq == view->log_file_head_seq &&
+ view->log_file_expunge_offset == view->log_file_head_offset) {
+ e_warning(view->index->event,
+ "%s - generating missing logs", error);
+ }
+ ctx->log_was_lost = TRUE;
+ if (!sync_expunges)
+ i_array_init(&ctx->expunges, 64);
+ mail_index_sync_map_init(&ctx->sync_map_ctx, view,
+ MAIL_INDEX_SYNC_HANDLER_VIEW);
+ ret = view_sync_get_log_lost_changes(ctx, &expunge_count);
+ mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
+ mail_index_sync_map_deinit(&ctx->sync_map_ctx);
+ if (ret < 0) {
+ mail_index_set_error(view->index,
+ "%s view syncing failed to apply changes",
+ view->index->filepath);
+ view->inconsistent = TRUE;
+ ctx->failed = TRUE;
+ return ctx;
+ }
+ have_expunges = expunge_count > 0;
+ } else if (sync_expunges) {
+ /* get list of all expunges first */
+ if (view_sync_get_expunges(ctx, &expunge_count) < 0) {
+ ctx->failed = TRUE;
+ return ctx;
+ }
+ have_expunges = expunge_count > 0;
+ } else if (view->log_file_expunge_seq == view->log_file_head_seq &&
+ view->log_file_expunge_offset == view->log_file_head_offset) {
+ /* Previous syncs haven't left any pending expunges. See if
+ this sync will. */
+ have_expunges = view_sync_have_expunges(view);
+ } else {
+ /* Expunges weren't synced in the previous sync either.
+ We already know there are missing expunges. */
+ ctx->skipped_expunges = TRUE;
+ have_expunges = TRUE;
+ }
+
+ ctx->finish_min_msg_count = reset ? 0 :
+ view->map->hdr.messages_count - expunge_count;
+ if (!reset)
+ ;
+ else if ((flags & MAIL_INDEX_VIEW_SYNC_FLAG_2ND_INDEX) != 0 &&
+ view->map->hdr.messages_count == 0) {
+ /* The secondary index is still empty, so it may have
+ just been created for the first time. This is
+ expected, so it shouldn't cause the view to become
+ inconsistent. */
+ if (mail_index_view_sync_init_fix(ctx) < 0)
+ ctx->failed = TRUE;
+ return ctx;
+ } else {
+ view->inconsistent = TRUE;
+ mail_index_set_error(view->index,
+ "%s reset, view is now inconsistent",
+ view->index->filepath);
+ ctx->failed = TRUE;
+ return ctx;
+ }
+
+ if (!have_expunges && !partial_sync) {
+ /* no expunges, we can just replace the map */
+ if (view->index->map->hdr.messages_count <
+ ctx->finish_min_msg_count) {
+ mail_index_set_error(view->index,
+ "Index %s lost messages without expunging "
+ "(%u -> %u)", view->index->filepath,
+ view->map->hdr.messages_count,
+ view->index->map->hdr.messages_count);
+ ctx->finish_min_msg_count = 0;
+ view->inconsistent = TRUE;
+ }
+
+ view->index->map->refcount++;
+ mail_index_unmap(&view->map);
+ view->map = view->index->map;
+ } else {
+ /* a) expunges seen. b) doing a partial sync because we saw
+ a reset.
+
+ Create a private map which we update. If we're syncing
+ expunges the map will finally be replaced with the head map
+ to remove the expunged messages. */
+ ctx->sync_map_update = TRUE;
+
+ if (view->map->refcount > 1) {
+ tmp_map = mail_index_map_clone(view->map);
+ mail_index_unmap(&view->map);
+ view->map = tmp_map;
+ }
+
+ if (sync_expunges) {
+ ctx->sync_new_map = view->index->map;
+ ctx->sync_new_map->refcount++;
+ }
+ }
+ mail_index_sync_map_init(&ctx->sync_map_ctx, view,
+ MAIL_INDEX_SYNC_HANDLER_VIEW);
+
+#ifdef DEBUG
+ mail_index_map_check(view->map);
+#endif
+ return ctx;
+}
+
+static bool
+view_sync_is_hidden(struct mail_index_view *view, uint32_t seq, uoff_t offset)
+{
+ const struct mail_index_view_log_sync_area *sync;
+
+ if (!array_is_created(&view->syncs_hidden))
+ return FALSE;
+
+ array_foreach(&view->syncs_hidden, sync) {
+ if (sync->log_file_offset <= offset &&
+ offset - sync->log_file_offset < sync->length &&
+ sync->log_file_seq == seq)
+ return TRUE;
+ }
+
+ return FALSE;
+}
+
+static bool
+mail_index_view_sync_want(struct mail_index_view_sync_ctx *ctx,
+ const struct mail_transaction_header *hdr)
+{
+ struct mail_index_view *view = ctx->view;
+ uint32_t seq;
+ uoff_t offset, next_offset;
+
+ mail_transaction_log_view_get_prev_pos(view->log_view, &seq, &offset);
+ next_offset = offset + sizeof(*hdr) + hdr->size;
+
+ if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
+ MAIL_TRANSACTION_EXPUNGE_GUID)) != 0 &&
+ (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0) {
+ if ((ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_NOEXPUNGES) != 0) {
+ i_assert(!LOG_IS_BEFORE(seq, offset,
+ view->log_file_expunge_seq,
+ view->log_file_expunge_offset));
+ if (!ctx->skipped_expunges) {
+ view->log_file_expunge_seq = seq;
+ view->log_file_expunge_offset = offset;
+ ctx->skipped_expunges = TRUE;
+ }
+ return FALSE;
+ }
+ if (LOG_IS_BEFORE(seq, offset, view->log_file_expunge_seq,
+ view->log_file_expunge_offset)) {
+ /* already synced */
+ return FALSE;
+ }
+ }
+
+ if (LOG_IS_BEFORE(seq, offset, view->log_file_head_seq,
+ view->log_file_head_offset)) {
+ /* already synced */
+ return FALSE;
+ }
+
+ view->log_file_head_seq = seq;
+ view->log_file_head_offset = next_offset;
+ return TRUE;
+}
+
+static int
+mail_index_view_sync_get_next_transaction(struct mail_index_view_sync_ctx *ctx)
+{
+ struct mail_transaction_log_view *log_view = ctx->view->log_view;
+ struct mail_index_view *view = ctx->view;
+ const struct mail_transaction_header *hdr;
+ uint32_t seq;
+ uoff_t offset;
+ int ret;
+ bool synced_to_map;
+
+ do {
+ /* Get the next transaction from log. */
+ ret = mail_transaction_log_view_next(log_view, &ctx->hdr,
+ &ctx->data);
+ if (ret <= 0) {
+ if (ret < 0)
+ return -1;
+
+ ctx->hdr = NULL;
+ ctx->last_read = TRUE;
+ return 0;
+ }
+
+ hdr = ctx->hdr;
+ /* skip records we've already synced */
+ } while (!mail_index_view_sync_want(ctx, hdr));
+
+ mail_transaction_log_view_get_prev_pos(log_view, &seq, &offset);
+
+ /* If we started from a map that we didn't create ourself,
+ some of the transactions may already be synced. at the end
+ of this view sync we'll update file_seq=0 so that this check
+ always becomes FALSE for subsequent syncs. */
+ synced_to_map = view->map->hdr.log_file_seq != 0 &&
+ LOG_IS_BEFORE(seq, offset, view->map->hdr.log_file_seq,
+ view->map->hdr.log_file_head_offset);
+
+ /* Apply transaction to view's mapping if needed (meaning we
+ didn't just re-map the view to head mapping). */
+ if (ctx->sync_map_update && !synced_to_map) {
+ if ((hdr->type & (MAIL_TRANSACTION_EXPUNGE |
+ MAIL_TRANSACTION_EXPUNGE_GUID)) == 0) {
+ ret = mail_index_sync_record(&ctx->sync_map_ctx,
+ hdr, ctx->data);
+ }
+ if (ret < 0)
+ return -1;
+ }
+
+ ctx->hidden = view_sync_is_hidden(view, seq, offset);
+ return 1;
+}
+
+static bool
+mail_index_view_sync_get_rec(struct mail_index_view_sync_ctx *ctx,
+ struct mail_index_view_sync_rec *rec)
+{
+ const struct mail_transaction_header *hdr = ctx->hdr;
+ const void *data = ctx->data;
+
+ switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+ case MAIL_TRANSACTION_FLAG_UPDATE: {
+ const struct mail_transaction_flag_update *update =
+ CONST_PTR_OFFSET(data, ctx->data_offset);
+
+ /* data contains mail_transaction_flag_update[] */
+ for (;;) {
+ ctx->data_offset += sizeof(*update);
+ if (!MAIL_TRANSACTION_FLAG_UPDATE_IS_INTERNAL(update))
+ break;
+
+ /* skip internal flag changes */
+ if (ctx->data_offset == ctx->hdr->size)
+ return FALSE;
+
+ update = CONST_PTR_OFFSET(data, ctx->data_offset);
+ }
+
+ if (update->add_flags != 0 || update->remove_flags != 0)
+ rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+ else
+ rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_MODSEQ;
+ rec->uid1 = update->uid1;
+ rec->uid2 = update->uid2;
+ break;
+ }
+ case MAIL_TRANSACTION_KEYWORD_UPDATE: {
+ const struct mail_transaction_keyword_update *update = data;
+ const uint32_t *uids;
+
+ /* data contains mail_transaction_keyword_update header,
+ the keyword name and an array of { uint32_t uid1, uid2; } */
+
+ if (ctx->data_offset == 0) {
+ /* skip over the header and name */
+ ctx->data_offset = sizeof(*update) + update->name_size;
+ if ((ctx->data_offset % 4) != 0)
+ ctx->data_offset += 4 - (ctx->data_offset % 4);
+ }
+
+ uids = CONST_PTR_OFFSET(data, ctx->data_offset);
+ rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+ rec->uid1 = uids[0];
+ rec->uid2 = uids[1];
+
+ ctx->data_offset += sizeof(uint32_t) * 2;
+ break;
+ }
+ case MAIL_TRANSACTION_KEYWORD_RESET: {
+ const struct mail_transaction_keyword_reset *reset =
+ CONST_PTR_OFFSET(data, ctx->data_offset);
+
+ /* data contains mail_transaction_keyword_reset[] */
+ rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+ rec->uid1 = reset->uid1;
+ rec->uid2 = reset->uid2;
+ ctx->data_offset += sizeof(*reset);
+ break;
+ }
+ default:
+ ctx->hdr = NULL;
+ return FALSE;
+ }
+
+ rec->hidden = ctx->hidden;
+ return TRUE;
+}
+
+static bool
+mail_index_view_sync_next_lost(struct mail_index_view_sync_ctx *ctx,
+ struct mail_index_view_sync_rec *sync_rec)
+{
+ const struct seq_range *range;
+ unsigned int count;
+
+ range = array_get(&ctx->lost_flags, &count);
+ if (ctx->lost_flag_idx == count) {
+ ctx->last_read = TRUE;
+ return FALSE;
+ }
+
+ sync_rec->type = MAIL_INDEX_VIEW_SYNC_TYPE_FLAGS;
+ sync_rec->uid1 = range[ctx->lost_flag_idx].seq1;
+ sync_rec->uid2 = range[ctx->lost_flag_idx].seq2;
+ sync_rec->hidden = FALSE;
+ ctx->lost_flag_idx++;
+ return TRUE;
+}
+
+bool mail_index_view_sync_next(struct mail_index_view_sync_ctx *ctx,
+ struct mail_index_view_sync_rec *sync_rec)
+{
+ int ret;
+
+ if (ctx->log_was_lost)
+ return mail_index_view_sync_next_lost(ctx, sync_rec);
+
+ do {
+ if (ctx->hdr == NULL || ctx->data_offset == ctx->hdr->size) {
+ ret = mail_index_view_sync_get_next_transaction(ctx);
+ if (ret <= 0) {
+ if (ret < 0)
+ ctx->failed = TRUE;
+ return FALSE;
+ }
+
+ ctx->data_offset = 0;
+ }
+ } while (!mail_index_view_sync_get_rec(ctx, sync_rec));
+
+ return TRUE;
+}
+
+void mail_index_view_sync_get_expunges(struct mail_index_view_sync_ctx *ctx,
+ const ARRAY_TYPE(seq_range) **expunges_r)
+{
+ *expunges_r = &ctx->expunges;
+}
+
+static void
+mail_index_view_sync_clean_log_syncs(struct mail_index_view *view)
+{
+ const struct mail_index_view_log_sync_area *syncs;
+ unsigned int i, count;
+
+ if (!array_is_created(&view->syncs_hidden))
+ return;
+
+ /* Clean up to view's tail */
+ syncs = array_get(&view->syncs_hidden, &count);
+ for (i = 0; i < count; i++) {
+ if ((syncs[i].log_file_offset +
+ syncs[i].length > view->log_file_expunge_offset &&
+ syncs[i].log_file_seq == view->log_file_expunge_seq) ||
+ syncs[i].log_file_seq > view->log_file_expunge_seq)
+ break;
+ }
+ if (i > 0)
+ array_delete(&view->syncs_hidden, 0, i);
+}
+
+int mail_index_view_sync_commit(struct mail_index_view_sync_ctx **_ctx,
+ bool *delayed_expunges_r)
+{
+ struct mail_index_view_sync_ctx *ctx = *_ctx;
+ struct mail_index_view *view = ctx->view;
+ int ret = ctx->failed ? -1 : 0;
+
+ i_assert(view->syncing);
+
+ *_ctx = NULL;
+ *delayed_expunges_r = ctx->skipped_expunges;
+
+ if ((!ctx->last_read || view->inconsistent) &&
+ (ctx->flags & MAIL_INDEX_VIEW_SYNC_FLAG_FIX_INCONSISTENT) == 0) {
+ /* we didn't sync everything */
+ view->inconsistent = TRUE;
+ ret = -1;
+ }
+ if (ctx->sync_map_ctx.modseq_ctx != NULL)
+ mail_index_modseq_sync_end(&ctx->sync_map_ctx.modseq_ctx);
+
+ if (ctx->sync_new_map != NULL) {
+ mail_index_unmap(&view->map);
+ view->map = ctx->sync_new_map;
+ } else if (ctx->sync_map_update) {
+ /* log offsets have no meaning in views. make sure they're not
+ tried to be used wrong by setting them to zero. */
+ view->map->hdr.log_file_seq = 0;
+ view->map->hdr.log_file_head_offset = 0;
+ view->map->hdr.log_file_tail_offset = 0;
+ }
+
+ i_assert(view->map->hdr.messages_count >= ctx->finish_min_msg_count);
+
+ if (!ctx->skipped_expunges) {
+ view->log_file_expunge_seq = view->log_file_head_seq;
+ view->log_file_expunge_offset = view->log_file_head_offset;
+ }
+
+ if (ctx->sync_map_ctx.view != NULL)
+ mail_index_sync_map_deinit(&ctx->sync_map_ctx);
+ mail_index_view_sync_clean_log_syncs(ctx->view);
+
+#ifdef DEBUG
+ mail_index_map_check(view->map);
+#endif
+
+ /* set log view to empty range so unneeded memory gets freed */
+ mail_transaction_log_view_clear(view->log_view,
+ view->log_file_expunge_seq);
+
+ if (array_is_created(&ctx->expunges))
+ array_free(&ctx->expunges);
+ if (array_is_created(&ctx->lost_flags))
+ array_free(&ctx->lost_flags);
+
+ view->highest_modseq = mail_index_map_modseq_get_highest(view->map);
+ view->syncing = FALSE;
+ i_free(ctx);
+ return ret;
+}
+
+void mail_index_view_add_hidden_transaction(struct mail_index_view *view,
+ uint32_t log_file_seq,
+ uoff_t log_file_offset,
+ unsigned int length)
+{
+ struct mail_index_view_log_sync_area *area;
+
+ if (!array_is_created(&view->syncs_hidden))
+ i_array_init(&view->syncs_hidden, 32);
+
+ area = array_append_space(&view->syncs_hidden);
+ area->log_file_seq = log_file_seq;
+ area->log_file_offset = log_file_offset;
+ area->length = length;
+}