diff options
Diffstat (limited to 'src/lib-index/mail-index-transaction-export.c')
| -rw-r--r-- | src/lib-index/mail-index-transaction-export.c | 533 |
1 files changed, 533 insertions, 0 deletions
diff --git a/src/lib-index/mail-index-transaction-export.c b/src/lib-index/mail-index-transaction-export.c new file mode 100644 index 0000000..2aced67 --- /dev/null +++ b/src/lib-index/mail-index-transaction-export.c @@ -0,0 +1,533 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index-private.h" +#include "mail-index-modseq.h" +#include "mail-transaction-log-private.h" +#include "mail-index-transaction-private.h" + +struct mail_index_export_context { + struct mail_index_transaction *trans; + struct mail_transaction_log_append_ctx *append_ctx; +}; + +static void +log_append_buffer(struct mail_index_export_context *ctx, + const buffer_t *buf, enum mail_transaction_type type) +{ + mail_transaction_log_append_add(ctx->append_ctx, type, + buf->data, buf->used); +} + +static void log_append_flag_updates(struct mail_index_export_context *ctx, + struct mail_index_transaction *t) +{ + ARRAY(struct mail_transaction_flag_update) log_updates; + const struct mail_index_flag_update *updates; + struct mail_transaction_flag_update *log_update; + unsigned int i, count; + + updates = array_get(&t->updates, &count); + if (count == 0) + return; + + i_array_init(&log_updates, count); + + for (i = 0; i < count; i++) { + log_update = array_append_space(&log_updates); + log_update->uid1 = updates[i].uid1; + log_update->uid2 = updates[i].uid2; + log_update->add_flags = updates[i].add_flags & 0xff; + log_update->remove_flags = updates[i].remove_flags & 0xff; + if ((updates[i].add_flags & MAIL_INDEX_MAIL_FLAG_UPDATE_MODSEQ) != 0) + log_update->modseq_inc_flag = 1; + } + log_append_buffer(ctx, log_updates.arr.buffer, + MAIL_TRANSACTION_FLAG_UPDATE); + array_free(&log_updates); +} + +static const buffer_t * +log_get_hdr_update_buffer(struct mail_index_transaction *t, bool prepend) +{ + buffer_t *buf; + const unsigned char *data, *mask; + struct mail_transaction_header_update u; + uint16_t offset; + int state = 0; + + i_zero(&u); + + data = prepend ? t->pre_hdr_change : t->post_hdr_change; + mask = prepend ? t->pre_hdr_mask : t->post_hdr_mask; + + buf = t_buffer_create(256); + for (offset = 0; offset <= sizeof(t->pre_hdr_change); offset++) { + if (offset < sizeof(t->pre_hdr_change) && mask[offset] != 0) { + if (state == 0) { + u.offset = offset; + state++; + } + } else { + if (state > 0) { + u.size = offset - u.offset; + buffer_append(buf, &u, sizeof(u)); + buffer_append(buf, data + u.offset, u.size); + state = 0; + } + } + } + return buf; +} + +static unsigned int +ext_hdr_update_get_size(const struct mail_index_transaction_ext_hdr_update *hu) +{ + unsigned int i; + + for (i = hu->alloc_size; i > 0; i--) { + if (hu->mask[i-1] != 0) + return i; + } + return 0; +} + +static void log_append_ext_intro(struct mail_index_export_context *ctx, + uint32_t ext_id, uint32_t reset_id, + unsigned int *hdr_size_r) +{ + struct mail_index_transaction *t = ctx->trans; + const struct mail_index_registered_ext *rext; + const struct mail_index_ext *ext; + struct mail_transaction_ext_intro *intro, *resizes; + buffer_t *buf; + uint32_t idx; + unsigned int count; + + i_assert(ext_id != (uint32_t)-1); + + if (t->reset || + !mail_index_map_get_ext_idx(t->view->index->map, ext_id, &idx)) { + /* new extension */ + idx = (uint32_t)-1; + } + + rext = array_idx(&t->view->index->extensions, ext_id); + if (!array_is_created(&t->ext_resizes)) { + resizes = NULL; + count = 0; + } else { + resizes = array_get_modifiable(&t->ext_resizes, &count); + } + + buf = t_buffer_create(128); + if (ext_id < count && resizes[ext_id].name_size != 0) { + /* we're resizing the extension. use the resize struct. */ + intro = &resizes[ext_id]; + + if (idx != (uint32_t)-1) { + intro->ext_id = idx; + intro->name_size = 0; + } else { + intro->ext_id = (uint32_t)-1; + intro->name_size = strlen(rext->name); + } + buffer_append(buf, intro, sizeof(*intro)); + } else { + /* generate a new intro structure */ + intro = buffer_append_space_unsafe(buf, sizeof(*intro)); + intro->ext_id = idx; + intro->record_size = rext->record_size; + intro->record_align = rext->record_align; + if (idx == (uint32_t)-1) { + intro->hdr_size = rext->hdr_size; + intro->name_size = strlen(rext->name); + } else { + ext = array_idx(&t->view->index->map->extensions, idx); + intro->hdr_size = ext->hdr_size; + intro->name_size = 0; + } + intro->flags = MAIL_TRANSACTION_EXT_INTRO_FLAG_NO_SHRINK; + + /* handle increasing header size automatically */ + if (array_is_created(&t->ext_hdr_updates) && + ext_id < array_count(&t->ext_hdr_updates)) { + const struct mail_index_transaction_ext_hdr_update *hu; + unsigned int hdr_update_size; + + hu = array_idx(&t->ext_hdr_updates, ext_id); + hdr_update_size = ext_hdr_update_get_size(hu); + if (intro->hdr_size < hdr_update_size) + intro->hdr_size = hdr_update_size; + } + } + i_assert(intro->record_size != 0 || intro->hdr_size != 0); + if (reset_id != 0) { + /* we're going to reset this extension in this transaction */ + intro->reset_id = reset_id; + } else if (idx != (uint32_t)-1) { + /* use the existing reset_id */ + const struct mail_index_ext *map_ext = + array_idx(&t->view->index->map->extensions, idx); + intro->reset_id = map_ext->reset_id; + } else { + /* new extension, reset_id defaults to 0 */ + } + buffer_append(buf, rext->name, intro->name_size); + if ((buf->used % 4) != 0) + buffer_append_zero(buf, 4 - (buf->used % 4)); + + if (ctx->append_ctx->new_highest_modseq == 0 && + strcmp(rext->name, MAIL_INDEX_MODSEQ_EXT_NAME) == 0) { + /* modseq tracking started */ + ctx->append_ctx->new_highest_modseq = 1; + } + + log_append_buffer(ctx, buf, MAIL_TRANSACTION_EXT_INTRO); + *hdr_size_r = intro->hdr_size; +} + +static void +log_append_ext_hdr_update(struct mail_index_export_context *ctx, + const struct mail_index_transaction_ext_hdr_update *hdr, + unsigned int ext_hdr_size) +{ + buffer_t *buf; + const unsigned char *data, *mask; + struct mail_transaction_ext_hdr_update u; + struct mail_transaction_ext_hdr_update32 u32; + size_t offset; + bool started = FALSE, use_32 = hdr->alloc_size >= 65536; + + i_zero(&u); + i_zero(&u32); + + data = hdr->data; + mask = hdr->mask; + + buf = buffer_create_dynamic(default_pool, 256); + for (offset = 0; offset <= hdr->alloc_size; offset++) { + if (offset < hdr->alloc_size && mask[offset] != 0) { + if (!started) { + u32.offset = offset; + started = TRUE; + } + } else { + if (started) { + u32.size = offset - u32.offset; + if (use_32) + buffer_append(buf, &u32, sizeof(u32)); + else { + u.offset = u32.offset; + u.size = u32.size; + buffer_append(buf, &u, sizeof(u)); + } + i_assert(u32.offset + u32.size <= ext_hdr_size); + buffer_append(buf, data + u32.offset, u32.size); + started = FALSE; + } + } + } + if (buf->used % 4 != 0) + buffer_append_zero(buf, 4 - buf->used % 4); + log_append_buffer(ctx, buf, use_32 ? MAIL_TRANSACTION_EXT_HDR_UPDATE32 : + MAIL_TRANSACTION_EXT_HDR_UPDATE); + buffer_free(&buf); +} + +static void +mail_transaction_log_append_ext_intros(struct mail_index_export_context *ctx) +{ + struct mail_index_transaction *t = ctx->trans; + const struct mail_transaction_ext_intro *resize; + const struct mail_index_transaction_ext_hdr_update *hdrs; + struct mail_transaction_ext_reset ext_reset; + unsigned int resize_count, ext_count = 0; + unsigned int hdrs_count, reset_id_count, reset_count, hdr_size; + uint32_t ext_id, reset_id; + const struct mail_transaction_ext_reset *reset; + const uint32_t *reset_ids; + buffer_t reset_buf; + + if (!array_is_created(&t->ext_resizes)) { + resize = NULL; + resize_count = 0; + } else { + resize = array_get(&t->ext_resizes, &resize_count); + if (ext_count < resize_count) + ext_count = resize_count; + } + + if (!array_is_created(&t->ext_reset_ids)) { + reset_ids = NULL; + reset_id_count = 0; + } else { + reset_ids = array_get(&t->ext_reset_ids, &reset_id_count); + } + + if (!array_is_created(&t->ext_resets)) { + reset = NULL; + reset_count = 0; + } else { + reset = array_get(&t->ext_resets, &reset_count); + if (ext_count < reset_count) + ext_count = reset_count; + } + + if (!array_is_created(&t->ext_hdr_updates)) { + hdrs = NULL; + hdrs_count = 0; + } else { + hdrs = array_get(&t->ext_hdr_updates, &hdrs_count); + if (ext_count < hdrs_count) + ext_count = hdrs_count; + } + + i_zero(&ext_reset); + buffer_create_from_data(&reset_buf, &ext_reset, sizeof(ext_reset)); + buffer_set_used_size(&reset_buf, sizeof(ext_reset)); + + for (ext_id = 0; ext_id < ext_count; ext_id++) { + if (ext_id < reset_count) + ext_reset = reset[ext_id]; + else + ext_reset.new_reset_id = 0; + if ((ext_id < resize_count && resize[ext_id].name_size > 0) || + ext_reset.new_reset_id != 0 || + (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0)) { + if (ext_reset.new_reset_id != 0) { + /* we're going to reset this extension + immediately after the intro */ + reset_id = 0; + } else { + reset_id = ext_id < reset_id_count ? + reset_ids[ext_id] : 0; + } + log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size); + } else { + hdr_size = 0; + } + if (ext_reset.new_reset_id != 0) { + i_assert(ext_id < reset_id_count && + ext_reset.new_reset_id == reset_ids[ext_id]); + log_append_buffer(ctx, &reset_buf, + MAIL_TRANSACTION_EXT_RESET); + } + if (ext_id < hdrs_count && hdrs[ext_id].alloc_size > 0) { + T_BEGIN { + log_append_ext_hdr_update(ctx, &hdrs[ext_id], + hdr_size); + } T_END; + } + } +} + +static void log_append_ext_recs(struct mail_index_export_context *ctx, + const ARRAY_TYPE(seq_array_array) *arr, + enum mail_transaction_type type) +{ + struct mail_index_transaction *t = ctx->trans; + const ARRAY_TYPE(seq_array) *updates; + const uint32_t *reset_ids; + unsigned int ext_id, count, reset_id_count, hdr_size; + uint32_t reset_id; + + if (!array_is_created(&t->ext_reset_ids)) { + reset_ids = NULL; + reset_id_count = 0; + } else { + reset_ids = array_get_modifiable(&t->ext_reset_ids, + &reset_id_count); + } + + updates = array_get(arr, &count); + for (ext_id = 0; ext_id < count; ext_id++) { + if (!array_is_created(&updates[ext_id])) + continue; + + reset_id = ext_id < reset_id_count ? reset_ids[ext_id] : 0; + log_append_ext_intro(ctx, ext_id, reset_id, &hdr_size); + + log_append_buffer(ctx, updates[ext_id].arr.buffer, type); + } +} + +static void +log_append_keyword_update(struct mail_index_export_context *ctx, + buffer_t *tmp_buf, enum modify_type modify_type, + const char *keyword, const buffer_t *uid_buffer) +{ + struct mail_transaction_keyword_update kt_hdr; + + i_assert(uid_buffer->used > 0); + + i_zero(&kt_hdr); + kt_hdr.modify_type = modify_type; + kt_hdr.name_size = strlen(keyword); + + buffer_set_used_size(tmp_buf, 0); + buffer_append(tmp_buf, &kt_hdr, sizeof(kt_hdr)); + buffer_append(tmp_buf, keyword, kt_hdr.name_size); + if ((tmp_buf->used % 4) != 0) + buffer_append_zero(tmp_buf, 4 - (tmp_buf->used % 4)); + buffer_append(tmp_buf, uid_buffer->data, uid_buffer->used); + + log_append_buffer(ctx, tmp_buf, MAIL_TRANSACTION_KEYWORD_UPDATE); +} + +static bool +log_append_keyword_updates(struct mail_index_export_context *ctx) +{ + const struct mail_index_transaction_keyword_update *updates; + const char *const *keywords; + buffer_t *tmp_buf; + unsigned int i, count, keywords_count; + bool changed = FALSE; + + tmp_buf = t_buffer_create(64); + + keywords = array_get_modifiable(&ctx->trans->view->index->keywords, + &keywords_count); + updates = array_get_modifiable(&ctx->trans->keyword_updates, &count); + i_assert(count <= keywords_count); + + for (i = 0; i < count; i++) { + if (array_is_created(&updates[i].add_seq) && + array_count(&updates[i].add_seq) > 0) { + changed = TRUE; + log_append_keyword_update(ctx, tmp_buf, + MODIFY_ADD, keywords[i], + updates[i].add_seq.arr.buffer); + } + if (array_is_created(&updates[i].remove_seq) && + array_count(&updates[i].remove_seq) > 0) { + changed = TRUE; + log_append_keyword_update(ctx, tmp_buf, + MODIFY_REMOVE, keywords[i], + updates[i].remove_seq.arr.buffer); + } + } + return changed; +} + +void mail_index_transaction_export(struct mail_index_transaction *t, + struct mail_transaction_log_append_ctx *append_ctx, + enum mail_index_transaction_change *changes_r) +{ + static uint8_t null4[4] = { 0, 0, 0, 0 }; + enum mail_index_fsync_mask change_mask = 0; + struct mail_index_export_context ctx; + + *changes_r = 0; + + i_zero(&ctx); + ctx.trans = t; + ctx.append_ctx = append_ctx; + + if (t->index_undeleted) { + i_assert(!t->index_deleted); + mail_transaction_log_append_add(ctx.append_ctx, + MAIL_TRANSACTION_INDEX_UNDELETED, &null4, 4); + } + + /* send all extension introductions and resizes before appends + to avoid resize overhead as much as possible */ + mail_transaction_log_append_ext_intros(&ctx); + + if (t->pre_hdr_changed) { + log_append_buffer(&ctx, log_get_hdr_update_buffer(t, TRUE), + MAIL_TRANSACTION_HEADER_UPDATE); + } + + if (append_ctx->output->used > 0) + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + + if (t->attribute_updates != NULL) { + buffer_append_c(t->attribute_updates, '\0'); + /* need to have 32bit alignment */ + if (t->attribute_updates->used % 4 != 0) { + buffer_append_zero(t->attribute_updates, + 4 - t->attribute_updates->used % 4); + } + /* append the timestamp and value lengths */ + buffer_append(t->attribute_updates, + t->attribute_updates_suffix->data, + t->attribute_updates_suffix->used); + i_assert(t->attribute_updates->used % 4 == 0); + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_ATTRIBUTE; + log_append_buffer(&ctx, t->attribute_updates, + MAIL_TRANSACTION_ATTRIBUTE_UPDATE); + } + if (array_is_created(&t->appends)) { + change_mask |= MAIL_INDEX_FSYNC_MASK_APPENDS; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_APPEND; + log_append_buffer(&ctx, t->appends.arr.buffer, + MAIL_TRANSACTION_APPEND); + } + + if (array_is_created(&t->updates)) { + change_mask |= MAIL_INDEX_FSYNC_MASK_FLAGS; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_FLAGS; + log_append_flag_updates(&ctx, t); + } + + if (array_is_created(&t->ext_rec_updates)) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + log_append_ext_recs(&ctx, &t->ext_rec_updates, + MAIL_TRANSACTION_EXT_REC_UPDATE); + } + if (array_is_created(&t->ext_rec_atomics)) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + log_append_ext_recs(&ctx, &t->ext_rec_atomics, + MAIL_TRANSACTION_EXT_ATOMIC_INC); + } + + if (array_is_created(&t->keyword_updates)) { + if (log_append_keyword_updates(&ctx)) { + change_mask |= MAIL_INDEX_FSYNC_MASK_KEYWORDS; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_KEYWORDS; + } + } + /* keep modseq updates almost last */ + if (array_is_created(&t->modseq_updates)) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_MODSEQ; + log_append_buffer(&ctx, t->modseq_updates.arr.buffer, + MAIL_TRANSACTION_MODSEQ_UPDATE); + } + + if (array_is_created(&t->expunges)) { + /* non-external expunges are only requests, ignore them when + checking fsync_mask */ + if ((t->flags & MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL) != 0) { + change_mask |= MAIL_INDEX_FSYNC_MASK_EXPUNGES; + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_EXPUNGE; + } else { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + } + log_append_buffer(&ctx, t->expunges.arr.buffer, + MAIL_TRANSACTION_EXPUNGE_GUID); + } + + if (t->post_hdr_changed) { + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + log_append_buffer(&ctx, log_get_hdr_update_buffer(t, FALSE), + MAIL_TRANSACTION_HEADER_UPDATE); + } + + if (t->index_deleted) { + i_assert(!t->index_undeleted); + *changes_r |= MAIL_INDEX_TRANSACTION_CHANGE_OTHERS; + mail_transaction_log_append_add(ctx.append_ctx, + MAIL_TRANSACTION_INDEX_DELETED, + &null4, 4); + } + + i_assert((append_ctx->output->used > 0) == (*changes_r != 0)); + + append_ctx->index_sync_transaction = t->sync_transaction; + append_ctx->tail_offset_changed = t->tail_offset_changed; + append_ctx->want_fsync = + (t->view->index->set.fsync_mask & change_mask) != 0 || + (t->flags & MAIL_INDEX_TRANSACTION_FLAG_FSYNC) != 0; +} |