diff options
Diffstat (limited to 'src/lib-storage/index/mbox/mbox-sync.c')
| -rw-r--r-- | src/lib-storage/index/mbox/mbox-sync.c | 2066 |
1 files changed, 2066 insertions, 0 deletions
diff --git a/src/lib-storage/index/mbox/mbox-sync.c b/src/lib-storage/index/mbox/mbox-sync.c new file mode 100644 index 0000000..0d2aa7f --- /dev/null +++ b/src/lib-storage/index/mbox/mbox-sync.c @@ -0,0 +1,2066 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +/* + Modifying mbox can be slow, so we try to do it all at once minimizing the + required disk I/O. We may need to: + + - Update message flags in Status, X-Status and X-Keywords headers + - Write missing X-UID and X-IMAPbase headers + - Write missing or broken Content-Length header if there's space + - Expunge specified messages + + Here's how we do it: + + - Start reading the mails from the beginning + - X-Keywords, X-UID and X-IMAPbase headers may contain padding at the end + of them, remember how much each message has and offset to beginning of the + padding + - If header needs to be rewritten and there's enough space, do it + - If we didn't have enough space, remember how much was missing + - Continue reading and counting the padding in each message. If available + padding is enough to rewrite all the previous messages needing it, do it + - When we encounter expunged message, treat all of it as padding and + rewrite previous messages if needed (and there's enough space). + Afterwards keep moving messages backwards to fill the expunged space. + Moving is done by rewriting each message's headers, with possibly adding + missing Content-Length header and padding. Message bodies are moved + without modifications. + - If we encounter end of file, grow the file and rewrite needed messages + - Rewriting is done by moving message body forward, rewriting message's + header and doing the same for previous message, until all of them are + rewritten. +*/ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "buffer.h" +#include "hostpid.h" +#include "istream.h" +#include "file-set-size.h" +#include "str.h" +#include "read-full.h" +#include "write-full.h" +#include "sleep.h" +#include "message-date.h" +#include "istream-raw-mbox.h" +#include "mbox-storage.h" +#include "index-sync-changes.h" +#include "mailbox-uidvalidity.h" +#include "mailbox-recent-flags.h" +#include "mbox-from.h" +#include "mbox-file.h" +#include "mbox-lock.h" +#include "mbox-sync-private.h" + +#include <stddef.h> +#include <utime.h> +#include <sys/stat.h> + +/* The text below was taken exactly as c-client wrote it to my mailbox, + so it's probably copyrighted by University of Washington. */ +#define PSEUDO_MESSAGE_BODY \ +"This text is part of the internal format of your mail folder, and is not\n" \ +"a real message. It is created automatically by the mail system software.\n" \ +"If deleted, important folder data will be lost, and it will be re-created\n" \ +"with the data reset to initial values.\n" + +void mbox_sync_set_critical(struct mbox_sync_context *sync_ctx, + const char *fmt, ...) +{ + va_list va; + + sync_ctx->errors = TRUE; + if (sync_ctx->ext_modified) { + mailbox_set_critical(&sync_ctx->mbox->box, + "mbox was modified while we were syncing, " + "check your locking settings"); + } + + va_start(va, fmt); + mailbox_set_critical(&sync_ctx->mbox->box, + "Sync failed for mbox: %s", + t_strdup_vprintf(fmt, va)); + va_end(va); +} + +int mbox_sync_seek(struct mbox_sync_context *sync_ctx, uoff_t from_offset) +{ + if (istream_raw_mbox_seek(sync_ctx->input, from_offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Unexpectedly lost From-line at offset %"PRIuUOFF_T, + from_offset); + return -1; + } + return 0; +} + +void mbox_sync_file_update_ext_modified(struct mbox_sync_context *sync_ctx) +{ + struct stat st; + + /* Do this even if ext_modified is already set. Expunging code relies + on last_stat being updated. */ + if (fstat(sync_ctx->write_fd, &st) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "fstat()"); + return; + } + + if (st.st_size != sync_ctx->last_stat.st_size || + (sync_ctx->last_stat.st_mtime != 0 && + !CMP_ST_MTIME(&st, &sync_ctx->last_stat))) + sync_ctx->ext_modified = TRUE; + + sync_ctx->last_stat = st; +} + +void mbox_sync_file_updated(struct mbox_sync_context *sync_ctx, bool dirty) +{ + if (dirty) { + /* just mark the stat as dirty. */ + sync_ctx->last_stat.st_mtime = 0; + return; + } + if (fstat(sync_ctx->write_fd, &sync_ctx->last_stat) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "fstat()"); + i_stream_sync(sync_ctx->input); +} + +static int +mbox_sync_read_next_mail(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + uoff_t offset; + + /* get EOF */ + (void)istream_raw_mbox_get_header_offset(sync_ctx->input, &offset); + if (istream_raw_mbox_is_eof(sync_ctx->input)) + return 0; + + p_clear(sync_ctx->mail_keyword_pool); + i_zero(mail_ctx); + mail_ctx->sync_ctx = sync_ctx; + mail_ctx->seq = ++sync_ctx->seq; + mail_ctx->header = sync_ctx->header; + + mail_ctx->mail.from_offset = + istream_raw_mbox_get_start_offset(sync_ctx->input); + if (istream_raw_mbox_get_header_offset(sync_ctx->input, &mail_ctx->mail.offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get header offset for seq=%u", mail_ctx->seq); + return -1; + } + + if (mbox_sync_parse_next_mail(sync_ctx->input, mail_ctx) < 0) + return -1; + if (istream_raw_mbox_is_corrupted(sync_ctx->input)) + return -1; + + i_assert(sync_ctx->input->v_offset != mail_ctx->mail.from_offset || + sync_ctx->input->eof); + + if (istream_raw_mbox_get_body_size(sync_ctx->input, + mail_ctx->content_length, + &mail_ctx->mail.body_size) < 0) { + mbox_sync_set_critical(sync_ctx, + "Couldn't get body size for seq=%u", mail_ctx->seq); + return -1; + } + i_assert(mail_ctx->mail.body_size < OFF_T_MAX); + + if ((mail_ctx->mail.flags & MAIL_RECENT) != 0 && + !mail_ctx->mail.pseudo) { + if (!sync_ctx->keep_recent) { + /* need to add 'O' flag to Status-header */ + mail_ctx->need_rewrite = TRUE; + } + mail_ctx->recent = TRUE; + } + return 1; +} + +static void mbox_sync_read_index_syncs(struct mbox_sync_context *sync_ctx, + uint32_t uid, bool *sync_expunge_r) +{ + guid_128_t expunged_guid_128; + + if (uid == 0 || sync_ctx->index_reset) { + /* nothing for this or the future ones */ + uid = (uint32_t)-1; + } + + index_sync_changes_read(sync_ctx->sync_changes, uid, sync_expunge_r, + expunged_guid_128); + if (sync_ctx->readonly) { + /* we can't expunge anything from read-only mboxes */ + *sync_expunge_r = FALSE; + } +} + +static bool +mbox_sync_read_index_rec(struct mbox_sync_context *sync_ctx, + uint32_t uid, const struct mail_index_record **rec_r) +{ + const struct mail_index_record *rec = NULL; + uint32_t messages_count; + bool ret = FALSE; + + if (sync_ctx->index_reset) { + *rec_r = NULL; + return TRUE; + } + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + while (sync_ctx->idx_seq <= messages_count) { + rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq); + if (uid <= rec->uid) + break; + + /* externally expunged message, remove from index */ + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq); + sync_ctx->idx_seq++; + rec = NULL; + } + + if (rec == NULL && uid < sync_ctx->idx_next_uid) { + /* this UID was already in index and it was expunged */ + mbox_sync_set_critical(sync_ctx, + "Expunged message reappeared to mailbox " + "(UID %u < %u, seq=%u, idx_msgs=%u)", + uid, sync_ctx->idx_next_uid, + sync_ctx->seq, messages_count); + ret = FALSE; rec = NULL; + } else if (rec != NULL && rec->uid != uid) { + /* new UID in the middle of the mailbox - shouldn't happen */ + mbox_sync_set_critical(sync_ctx, + "UID inserted in the middle of mailbox " + "(%u > %u, seq=%u, idx_msgs=%u)", + rec->uid, uid, sync_ctx->seq, messages_count); + ret = FALSE; rec = NULL; + } else { + ret = TRUE; + } + + *rec_r = rec; + return ret; +} + +static void mbox_sync_find_index_md5(struct mbox_sync_context *sync_ctx, + unsigned char hdr_md5_sum[], + const struct mail_index_record **rec_r) +{ + const struct mail_index_record *rec = NULL; + uint32_t messages_count; + const void *data; + + if (sync_ctx->index_reset) { + *rec_r = NULL; + return; + } + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + while (sync_ctx->idx_seq <= messages_count) { + rec = mail_index_lookup(sync_ctx->sync_view, sync_ctx->idx_seq); + mail_index_lookup_ext(sync_ctx->sync_view, + sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + &data, NULL); + if (data != NULL && memcmp(data, hdr_md5_sum, 16) == 0) + break; + + /* externally expunged message, remove from index */ + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq); + sync_ctx->idx_seq++; + rec = NULL; + } + + *rec_r = rec; +} + +static void +mbox_sync_update_from_offset(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail *mail, + bool nocheck) +{ + const void *data; + uint64_t offset; + + if (!nocheck) { + /* see if from_offset needs updating */ + mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq, + sync_ctx->mbox->mbox_ext_idx, + &data, NULL); + if (data != NULL && + *((const uint64_t *)data) == mail->from_offset) + return; + } + + offset = mail->from_offset; + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->mbox_ext_idx, &offset, NULL); +} + +static void +mbox_sync_update_index_keywords(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mail_index *index = sync_ctx->mbox->box.index; + struct mail_keywords *keywords; + + keywords = !array_is_created(&mail_ctx->mail.keywords) ? + mail_index_keywords_create(index, NULL) : + mail_index_keywords_create_from_indexes(index, + &mail_ctx->mail.keywords); + mail_index_update_keywords(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, keywords); + mail_index_keywords_unref(&keywords); +} + +static void +mbox_sync_update_md5_if_changed(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + const void *ext_data; + + mail_index_lookup_ext(sync_ctx->sync_view, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, &ext_data, NULL); + if (ext_data == NULL || + memcmp(mail_ctx->hdr_md5_sum, ext_data, 16) != 0) { + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + mail_ctx->hdr_md5_sum, NULL); + } +} + +static void mbox_sync_get_dirty_flags(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint8_t idx_flags, mbox_flags; + + /* default to undirtying the message. it gets added back if + flags/keywords don't match what is in the index. */ + mail_ctx->mail.flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY); + + /* replace flags */ + idx_flags = rec->flags & MAIL_FLAGS_NONRECENT; + mbox_flags = mail_ctx->mail.flags & MAIL_FLAGS_NONRECENT; + if (idx_flags != mbox_flags) { + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.flags = (mail_ctx->mail.flags & MAIL_RECENT) | + idx_flags | MAIL_INDEX_MAIL_FLAG_DIRTY; + } + + /* replace keywords */ + t_array_init(&idx_keywords, 32); + mail_index_lookup_keywords(sync_ctx->sync_view, sync_ctx->idx_seq, + &idx_keywords); + if (!index_keyword_array_cmp(&idx_keywords, &mail_ctx->mail.keywords)) { + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + + if (!array_is_created(&mail_ctx->mail.keywords)) { + p_array_init(&mail_ctx->mail.keywords, + sync_ctx->mail_keyword_pool, + array_count(&idx_keywords)); + } + array_clear(&mail_ctx->mail.keywords); + array_append_array(&mail_ctx->mail.keywords, &idx_keywords); + } +} + +static void mbox_sync_update_flags(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mailbox *box = &sync_ctx->mbox->box; + struct mbox_sync_mail *mail = &mail_ctx->mail; + enum mail_index_sync_type sync_type; + ARRAY_TYPE(keyword_indexes) orig_keywords = ARRAY_INIT; + uint8_t flags, orig_flags; + + if (rec != NULL) { + if ((rec->flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* flags and keywords are dirty. replace the current + ones from the flags in index file. */ + mbox_sync_get_dirty_flags(mail_ctx, rec); + } + } + + flags = orig_flags = mail->flags & MAIL_FLAGS_NONRECENT; + if (array_is_created(&mail->keywords)) { + t_array_init(&orig_keywords, 32); + array_append_array(&orig_keywords, &mail->keywords); + } + + /* apply new changes */ + index_sync_changes_apply(sync_ctx->sync_changes, + sync_ctx->mail_keyword_pool, + &flags, &mail->keywords, &sync_type); + if (flags != orig_flags || + !index_keyword_array_cmp(&mail->keywords, &orig_keywords)) { + mail_ctx->need_rewrite = TRUE; + mail->flags = flags | (mail->flags & MAIL_RECENT) | + MAIL_INDEX_MAIL_FLAG_DIRTY; + } + if (sync_type != 0) { + mailbox_sync_notify(box, mail_ctx->mail.uid, + index_sync_type_convert(sync_type)); + } +} + +static void mbox_sync_update_index(struct mbox_sync_mail_context *mail_ctx, + const struct mail_index_record *rec) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mbox_sync_mail *mail = &mail_ctx->mail; + ARRAY_TYPE(keyword_indexes) idx_keywords; + uint8_t mbox_flags; + + mbox_flags = mail->flags & ENUM_NEGATE(MAIL_RECENT); + if (!sync_ctx->delay_writes) { + /* changes are written to the mbox file */ + mbox_flags &= ENUM_NEGATE(MAIL_INDEX_MAIL_FLAG_DIRTY); + } else if (mail_ctx->need_rewrite) { + /* make sure this message gets written later */ + mbox_flags |= MAIL_INDEX_MAIL_FLAG_DIRTY; + } + + if (rec == NULL) { + /* new message */ + mail_index_append(sync_ctx->t, mail->uid, &sync_ctx->idx_seq); + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, mbox_flags); + mbox_sync_update_index_keywords(mail_ctx); + + if (sync_ctx->mbox->mbox_save_md5) { + mail_index_update_ext(sync_ctx->t, sync_ctx->idx_seq, + sync_ctx->mbox->md5hdr_ext_idx, + mail_ctx->hdr_md5_sum, NULL); + } + } else { + if ((rec->flags & MAIL_FLAGS_NONRECENT) != + (mbox_flags & MAIL_FLAGS_NONRECENT)) { + /* flags other than recent/dirty have changed */ + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + MODIFY_REPLACE, mbox_flags); + } else if (((rec->flags ^ mbox_flags) & + MAIL_INDEX_MAIL_FLAG_DIRTY) != 0) { + /* only dirty flag state changed */ + bool dirty; + + dirty = (mbox_flags & MAIL_INDEX_MAIL_FLAG_DIRTY) != 0; + mail_index_update_flags(sync_ctx->t, sync_ctx->idx_seq, + dirty ? MODIFY_ADD : MODIFY_REMOVE, + (enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY); + } + + /* see if keywords changed */ + t_array_init(&idx_keywords, 32); + mail_index_lookup_keywords(sync_ctx->sync_view, + sync_ctx->idx_seq, &idx_keywords); + if (!index_keyword_array_cmp(&idx_keywords, &mail->keywords)) + mbox_sync_update_index_keywords(mail_ctx); + + /* see if we need to update md5 sum. */ + if (sync_ctx->mbox->mbox_save_md5) + mbox_sync_update_md5_if_changed(mail_ctx); + } + + if (!mail_ctx->recent) { + /* Mail has "Status: O" header. No messages before this + can be recent. */ + sync_ctx->last_nonrecent_uid = mail->uid; + } + + /* update from_offsets, but not if we're going to rewrite this message. + rewriting would just move it anyway. */ + if (sync_ctx->need_space_seq == 0) { + bool nocheck = rec == NULL || sync_ctx->expunged_space > 0; + mbox_sync_update_from_offset(sync_ctx, mail, nocheck); + } +} + +static int mbox_read_from_line(struct mbox_sync_mail_context *ctx) +{ + struct istream *input = ctx->sync_ctx->file_input; + const unsigned char *data; + size_t size, from_line_size; + + buffer_set_used_size(ctx->sync_ctx->from_line, 0); + from_line_size = ctx->hdr_offset - ctx->mail.from_offset; + + i_stream_seek(input, ctx->mail.from_offset); + for (;;) { + data = i_stream_get_data(input, &size); + if (size >= from_line_size) + size = from_line_size; + + buffer_append(ctx->sync_ctx->from_line, data, size); + i_stream_skip(input, size); + from_line_size -= size; + + if (from_line_size == 0) + break; + + if (i_stream_read(input) < 0) + return -1; + } + + return 0; +} + +static int mbox_rewrite_base_uid_last(struct mbox_sync_context *sync_ctx) +{ + unsigned char buf[10]; + const char *str; + uint32_t uid_last; + unsigned int i; + int ret; + + i_assert(sync_ctx->base_uid_last_offset != 0); + + /* first check that the 10 bytes are there and they're exactly as + expected. just an extra safety check to make sure we never write + to wrong location in the mbox file. */ + ret = pread_full(sync_ctx->write_fd, buf, sizeof(buf), + sync_ctx->base_uid_last_offset); + if (ret < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pread_full()"); + return -1; + } + if (ret == 0) { + mbox_sync_set_critical(sync_ctx, + "X-IMAPbase uid-last offset unexpectedly outside mbox"); + return -1; + } + + for (i = 0, uid_last = 0; i < sizeof(buf); i++) { + if (buf[i] < '0' || buf[i] > '9') { + uid_last = (uint32_t)-1; + break; + } + uid_last = uid_last * 10 + (buf[i] - '0'); + } + + if (uid_last != sync_ctx->base_uid_last) { + mbox_sync_set_critical(sync_ctx, + "X-IMAPbase uid-last unexpectedly lost"); + return -1; + } + + /* and write it */ + str = t_strdup_printf("%010u", sync_ctx->next_uid - 1); + if (pwrite_full(sync_ctx->write_fd, str, 10, + sync_ctx->base_uid_last_offset) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite_full()"); + return -1; + } + mbox_sync_file_updated(sync_ctx, FALSE); + + sync_ctx->base_uid_last = sync_ctx->next_uid - 1; + return 0; +} + +static int +mbox_write_from_line(struct mbox_sync_mail_context *ctx) +{ + string_t *str = ctx->sync_ctx->from_line; + + if (pwrite_full(ctx->sync_ctx->write_fd, str_data(str), str_len(str), + ctx->mail.from_offset) < 0) { + mbox_set_syscall_error(ctx->sync_ctx->mbox, "pwrite_full()"); + return -1; + } + + mbox_sync_file_updated(ctx->sync_ctx, FALSE); + return 0; +} + +static void update_from_offsets(struct mbox_sync_context *sync_ctx) +{ + const struct mbox_sync_mail *mails; + unsigned int i, count; + uint32_t ext_idx; + uint64_t offset; + + ext_idx = sync_ctx->mbox->mbox_ext_idx; + + mails = array_get(&sync_ctx->mails, &count); + for (i = 0; i < count; i++) { + if (mails[i].idx_seq == 0 || mails[i].expunged) + continue; + + sync_ctx->moved_offsets = TRUE; + offset = mails[i].from_offset; + mail_index_update_ext(sync_ctx->t, mails[i].idx_seq, + ext_idx, &offset, NULL); + } +} + +static void mbox_sync_handle_expunge(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + struct mailbox *box = &sync_ctx->mbox->box; + + mailbox_sync_notify(box, mail_ctx->mail.uid, + MAILBOX_SYNC_TYPE_EXPUNGE); + mail_index_expunge(sync_ctx->t, mail_ctx->mail.idx_seq); + + mail_ctx->mail.expunged = TRUE; + mail_ctx->mail.offset = mail_ctx->mail.from_offset; + mail_ctx->mail.space = + mail_ctx->body_offset - mail_ctx->mail.from_offset + + mail_ctx->mail.body_size; + mail_ctx->mail.body_size = 0; + mail_ctx->mail.uid = 0; + + if (sync_ctx->seq == 1) { + /* expunging first message, fix space to contain next + message's \n header too since it will be removed. */ + mail_ctx->mail.space++; + if (istream_raw_mbox_has_crlf_ending(sync_ctx->input)) { + mail_ctx->mail.space++; + sync_ctx->first_mail_crlf_expunged = TRUE; + } + + /* uid-last offset is invalid now */ + sync_ctx->base_uid_last_offset = 0; + } + + sync_ctx->expunged_space += mail_ctx->mail.space; +} + +static int mbox_sync_handle_header(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + uoff_t orig_from_offset, postlf_from_offset = UOFF_T_MAX; + off_t move_diff; + int ret; + + if (sync_ctx->expunged_space > 0 && sync_ctx->need_space_seq == 0) { + /* move the header backwards to fill expunged space */ + move_diff = -sync_ctx->expunged_space; + + orig_from_offset = mail_ctx->mail.from_offset; + if (sync_ctx->dest_first_mail) { + /* we're moving this mail to beginning of file. + skip the initial \n (it's already counted in + expunged_space) */ + mail_ctx->mail.from_offset++; + if (sync_ctx->first_mail_crlf_expunged) + mail_ctx->mail.from_offset++; + } + postlf_from_offset = mail_ctx->mail.from_offset; + + /* read the From-line before rewriting overwrites it */ + if (mbox_read_from_line(mail_ctx) < 0) + return -1; + i_assert((off_t)mail_ctx->mail.from_offset + move_diff != 1 && + (off_t)mail_ctx->mail.from_offset + move_diff != 2); + + mbox_sync_update_header(mail_ctx); + ret = mbox_sync_try_rewrite(mail_ctx, move_diff); + if (ret < 0) + return -1; + + if (ret > 0) { + /* rewrite successful, write From-line to + new location */ + i_assert((off_t)mail_ctx->mail.from_offset >= + -move_diff); + mail_ctx->mail.from_offset = (off_t)mail_ctx->mail.from_offset + move_diff; + mail_ctx->mail.offset = (off_t)mail_ctx->mail.offset + move_diff; + if (mbox_write_from_line(mail_ctx) < 0) + return -1; + } else { + if (sync_ctx->dest_first_mail) { + /* didn't have enough space, move the offset + back so seeking into it doesn't fail */ + mail_ctx->mail.from_offset = orig_from_offset; + } + } + } else if (mail_ctx->need_rewrite) { + mbox_sync_update_header(mail_ctx); + if (sync_ctx->delay_writes && sync_ctx->need_space_seq == 0) { + /* mark it dirty and do it later. we can't do this + if we're in the middle of rewriting acquiring more + space. */ + mail_ctx->dirty = TRUE; + return 0; + } + + if ((ret = mbox_sync_try_rewrite(mail_ctx, 0)) < 0) + return -1; + } else { + /* nothing to do */ + return 0; + } + + if (ret == 0 && sync_ctx->need_space_seq == 0) { + /* first mail with no space to write it */ + sync_ctx->need_space_seq = sync_ctx->seq; + sync_ctx->space_diff = 0; + + if (sync_ctx->expunged_space > 0) { + /* create dummy message to describe the expunged data */ + struct mbox_sync_mail mail; + + /* if this is going to be the first mail, increase the + from_offset to point to the beginning of the + From-line, because the previous [CR]LF is already + covered by expunged_space. */ + i_assert(postlf_from_offset != UOFF_T_MAX); + mail_ctx->mail.from_offset = postlf_from_offset; + + i_zero(&mail); + mail.expunged = TRUE; + mail.offset = mail.from_offset = + mail_ctx->mail.from_offset - + sync_ctx->expunged_space; + mail.space = sync_ctx->expunged_space; + + sync_ctx->space_diff = sync_ctx->expunged_space; + sync_ctx->expunged_space = 0; + i_assert(sync_ctx->space_diff < -mail_ctx->mail.space); + + sync_ctx->need_space_seq--; + array_push_back(&sync_ctx->mails, &mail); + } + } + return 0; +} + +static int +mbox_sync_handle_missing_space(struct mbox_sync_mail_context *mail_ctx) +{ + struct mbox_sync_context *sync_ctx = mail_ctx->sync_ctx; + uoff_t end_offset, move_diff, extra_space, needed_space; + uint32_t last_seq; + ARRAY_TYPE(keyword_indexes) keywords_copy; + + i_assert(mail_ctx->mail.uid == 0 || mail_ctx->mail.space > 0 || + mail_ctx->mail.offset == mail_ctx->hdr_offset); + + if (array_is_created(&mail_ctx->mail.keywords)) { + /* mail's keywords are allocated from a pool that's cleared + for each mail. we'll need to copy it to something more + permanent. */ + p_array_init(&keywords_copy, sync_ctx->saved_keywords_pool, + array_count(&mail_ctx->mail.keywords)); + array_append_array(&keywords_copy, &mail_ctx->mail.keywords); + mail_ctx->mail.keywords = keywords_copy; + } + array_push_back(&sync_ctx->mails, &mail_ctx->mail); + + sync_ctx->space_diff += mail_ctx->mail.space; + if (sync_ctx->space_diff < 0) { + if (sync_ctx->expunged_space > 0) { + i_assert(sync_ctx->expunged_space == + mail_ctx->mail.space); + sync_ctx->expunged_space = 0; + } + return 0; + } + + /* we have enough space now */ + if (mail_ctx->mail.uid == 0) { + /* this message was expunged. fill more or less of the space. + space_diff now consists of a negative "bytes needed" sum, + plus the expunged space of this message. so it contains how + many bytes of _extra_ space we have. */ + i_assert(mail_ctx->mail.space >= sync_ctx->space_diff); + extra_space = MBOX_HEADER_PADDING * + (sync_ctx->seq - sync_ctx->need_space_seq + 1); + needed_space = mail_ctx->mail.space - sync_ctx->space_diff; + if ((uoff_t)sync_ctx->space_diff > needed_space + extra_space) { + /* don't waste too much on padding */ + move_diff = needed_space + extra_space; + sync_ctx->expunged_space = + mail_ctx->mail.space - move_diff; + } else { + move_diff = mail_ctx->mail.space; + extra_space = sync_ctx->space_diff; + sync_ctx->expunged_space = 0; + } + last_seq = sync_ctx->seq - 1; + array_pop_back(&sync_ctx->mails); + end_offset = mail_ctx->mail.from_offset; + } else { + /* this message gave enough space from headers. rewriting stops + at the end of this message's headers. */ + sync_ctx->expunged_space = 0; + last_seq = sync_ctx->seq; + end_offset = mail_ctx->body_offset; + + move_diff = 0; + extra_space = sync_ctx->space_diff; + } + + mbox_sync_file_update_ext_modified(sync_ctx); + if (mbox_sync_rewrite(sync_ctx, + last_seq == sync_ctx->seq ? mail_ctx : NULL, + end_offset, move_diff, extra_space, + sync_ctx->need_space_seq, last_seq) < 0) + return -1; + + update_from_offsets(sync_ctx); + + /* mail_ctx may contain wrong data after rewrite, so make sure we + don't try to access it */ + i_zero(mail_ctx); + + sync_ctx->need_space_seq = 0; + sync_ctx->space_diff = 0; + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + return 0; +} + +static int +mbox_sync_seek_to_seq(struct mbox_sync_context *sync_ctx, uint32_t seq) +{ + struct mbox_mailbox *mbox = sync_ctx->mbox; + uoff_t old_offset, offset; + uint32_t uid; + int ret; + bool deleted; + + if (seq == 0) { + if (istream_raw_mbox_seek(mbox->mbox_stream, 0) < 0) { + mbox->invalid_mbox_file = TRUE; + mail_storage_set_error(&mbox->storage->storage, + MAIL_ERROR_NOTPOSSIBLE, + "Mailbox isn't a valid mbox file"); + return -1; + } + seq++; + } else { + old_offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + + ret = mbox_file_seek(mbox, sync_ctx->sync_view, seq, &deleted); + if (ret < 0) { + if (deleted) { + mbox_sync_set_critical(sync_ctx, + "Message was expunged unexpectedly"); + } + return -1; + } + if (ret == 0) { + if (istream_raw_mbox_seek(mbox->mbox_stream, + old_offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Error seeking back to original " + "offset %s", dec2str(old_offset)); + return -1; + } + return 0; + } + } + + if (seq <= 1) + uid = 0; + else + mail_index_lookup_uid(sync_ctx->sync_view, seq-1, &uid); + + sync_ctx->prev_msg_uid = uid; + + /* set to -1, since it's always increased later */ + sync_ctx->seq = seq-1; + if (sync_ctx->seq == 0 && + istream_raw_mbox_get_start_offset(sync_ctx->input) != 0) { + /* this mbox has pseudo mail which contains the X-IMAP header */ + sync_ctx->seq++; + } + + sync_ctx->idx_seq = seq; + sync_ctx->dest_first_mail = sync_ctx->seq == 0; + if (istream_raw_mbox_get_body_offset(sync_ctx->input, &offset) < 0) { + mbox_sync_set_critical(sync_ctx, + "Message body offset lookup failed"); + return -1; + } + return 1; +} + +static int +mbox_sync_seek_to_uid(struct mbox_sync_context *sync_ctx, uint32_t uid) +{ + struct mail_index_view *sync_view = sync_ctx->sync_view; + uint32_t seq1, seq2; + uoff_t size; + int ret; + + i_assert(!sync_ctx->index_reset); + + if (!mail_index_lookup_seq_range(sync_view, uid, (uint32_t)-1, + &seq1, &seq2)) { + /* doesn't exist anymore, seek to end of file */ + ret = i_stream_get_size(sync_ctx->file_input, TRUE, &size); + if (ret < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_get_size()"); + return -1; + } + i_assert(ret != 0); + + if (istream_raw_mbox_seek(sync_ctx->mbox->mbox_stream, + size) < 0) { + mbox_sync_set_critical(sync_ctx, + "Error seeking to end of mbox"); + return -1; + } + sync_ctx->idx_seq = + mail_index_view_get_messages_count(sync_view) + 1; + return 1; + } + + return mbox_sync_seek_to_seq(sync_ctx, seq1); +} + +static int mbox_sync_partial_seek_next(struct mbox_sync_context *sync_ctx, + uint32_t next_uid, bool *partial, + bool *skipped_mails) +{ + uint32_t messages_count, uid; + int ret; + + i_assert(!sync_ctx->index_reset); + + /* delete sync records up to next message. so if there's still + something left in array, it means the next message needs modifying */ + index_sync_changes_delete_to(sync_ctx->sync_changes, next_uid); + if (index_sync_changes_have(sync_ctx->sync_changes)) + return 1; + + if (sync_ctx->hdr->first_recent_uid <= next_uid && + !sync_ctx->keep_recent) { + /* we'll need to rewrite Status: O headers */ + return 1; + } + + uid = index_sync_changes_get_next_uid(sync_ctx->sync_changes); + + if (sync_ctx->hdr->first_recent_uid < sync_ctx->hdr->next_uid && + (uid > sync_ctx->hdr->first_recent_uid || uid == 0) && + !sync_ctx->keep_recent) { + /* we'll need to rewrite Status: O headers */ + uid = sync_ctx->hdr->first_recent_uid; + } + + if (uid != 0) { + /* we can skip forward to next record which needs updating. */ + if (uid != next_uid) { + *skipped_mails = TRUE; + next_uid = uid; + } + ret = mbox_sync_seek_to_uid(sync_ctx, next_uid); + } else { + /* if there's no sync records left, we can stop. except if + this is a dirty sync, check if there are new messages. */ + if (sync_ctx->mbox->mbox_hdr.dirty_flag == 0) + return 0; + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + if (sync_ctx->seq + 1 != messages_count) { + ret = mbox_sync_seek_to_seq(sync_ctx, messages_count); + *skipped_mails = TRUE; + } else { + ret = 1; + } + *partial = FALSE; + } + + if (ret == 0) { + /* seek failed because the offset is dirty. just ignore and + continue from where we are now. */ + *partial = FALSE; + ret = 1; + } + return ret; +} + +static void mbox_sync_hdr_update(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + const struct mailbox_update *update = sync_ctx->mbox->sync_hdr_update; + + if (update->uid_validity != 0) { + sync_ctx->base_uid_validity = update->uid_validity; + mail_ctx->imapbase_rewrite = TRUE; + mail_ctx->need_rewrite = TRUE; + } + if (update->min_next_uid != 0 && + sync_ctx->base_uid_last+1 < update->min_next_uid) { + i_assert(sync_ctx->next_uid <= update->min_next_uid); + sync_ctx->base_uid_last = update->min_next_uid-1; + sync_ctx->next_uid = update->min_next_uid; + mail_ctx->imapbase_rewrite = TRUE; + mail_ctx->need_rewrite = TRUE; + } +} + +static bool mbox_sync_imapbase(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + if (sync_ctx->base_uid_validity != 0 && + sync_ctx->hdr->uid_validity != 0 && + sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { + i_warning("UIDVALIDITY changed (%u -> %u) in mbox file %s", + sync_ctx->hdr->uid_validity, + sync_ctx->base_uid_validity, + mailbox_get_path(&sync_ctx->mbox->box)); + sync_ctx->index_reset = TRUE; + return TRUE; + } + if (sync_ctx->mbox->sync_hdr_update != NULL) + mbox_sync_hdr_update(sync_ctx, mail_ctx); + return FALSE; +} + +static int mbox_sync_loop(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx, + bool partial) +{ + const struct mail_index_record *rec; + uint32_t uid, messages_count; + uoff_t offset; + int ret; + bool expunged, skipped_mails, uids_broken; + + messages_count = + mail_index_view_get_messages_count(sync_ctx->sync_view); + + /* always start from first message so we can read X-IMAP or + X-IMAPbase header */ + ret = mbox_sync_seek_to_seq(sync_ctx, 0); + if (ret <= 0) + return ret; + + if (sync_ctx->renumber_uids) { + /* expunge everything */ + while (sync_ctx->idx_seq <= messages_count) { + mail_index_expunge(sync_ctx->t, + sync_ctx->idx_seq++); + } + } + + skipped_mails = uids_broken = FALSE; + while ((ret = mbox_sync_read_next_mail(sync_ctx, mail_ctx)) > 0) { + uid = mail_ctx->mail.uid; + + if (mail_ctx->seq == 1) { + if (mbox_sync_imapbase(sync_ctx, mail_ctx)) { + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + return 0; + } + } + + if (mail_ctx->mail.uid_broken && partial) { + /* UID ordering problems, resync everything to make + sure we get everything right */ + if (sync_ctx->mbox->mbox_hdr.dirty_flag != 0) + return 0; + + mbox_sync_set_critical(sync_ctx, + "UIDs broken with partial sync"); + + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + return 0; + } + if (mail_ctx->mail.uid_broken) + uids_broken = TRUE; + + if (mail_ctx->mail.pseudo) + uid = 0; + + rec = NULL; ret = 1; + if (uid != 0) { + if (!mbox_sync_read_index_rec(sync_ctx, uid, &rec)) + ret = 0; + } + + if (ret == 0) { + /* UID found but it's broken */ + uid = 0; + } else if (uid == 0 && + !mail_ctx->mail.pseudo && + (sync_ctx->delay_writes || + sync_ctx->idx_seq <= messages_count)) { + /* If we can't use/store X-UID header, use MD5 sum. + Also check for existing MD5 sums when we're actually + able to write X-UIDs. */ + sync_ctx->mbox->mbox_save_md5 = TRUE; + + mbox_sync_find_index_md5(sync_ctx, + mail_ctx->hdr_md5_sum, &rec); + if (rec != NULL) + uid = mail_ctx->mail.uid = rec->uid; + } + + /* get all sync records related to this message. with pseudo + message just get the first sync record so we can jump to + it with partial seeking. */ + mbox_sync_read_index_syncs(sync_ctx, + mail_ctx->mail.pseudo ? 1 : uid, + &expunged); + + if (mail_ctx->mail.pseudo) { + /* if it was set, it was for the next message */ + expunged = FALSE; + } else { + if (rec == NULL) { + /* message wasn't found from index. we have to + read everything from now on, no skipping */ + partial = FALSE; + } + } + + if (uid == 0 && !mail_ctx->mail.pseudo) { + /* missing/broken X-UID. all the rest of the mails + need new UIDs. */ + while (sync_ctx->idx_seq <= messages_count) { + mail_index_expunge(sync_ctx->t, + sync_ctx->idx_seq++); + } + + if (sync_ctx->next_uid == (uint32_t)-1) { + /* oh no, we're out of UIDs. this shouldn't + happen normally, so just try to get it fixed + without crashing. */ + mailbox_set_critical(&sync_ctx->mbox->box, + "Out of UIDs, renumbering them in mbox"); + sync_ctx->renumber_uids = TRUE; + return 0; + } + + mail_ctx->need_rewrite = TRUE; + mail_ctx->mail.uid = sync_ctx->next_uid++; + } + sync_ctx->prev_msg_uid = mail_ctx->mail.uid; + + if (!mail_ctx->mail.pseudo) + mail_ctx->mail.idx_seq = sync_ctx->idx_seq; + + if (!expunged) { + if (!mail_ctx->mail.pseudo) T_BEGIN { + mbox_sync_update_flags(mail_ctx, rec); + } T_END; + if (mbox_sync_handle_header(mail_ctx) < 0) + return -1; + sync_ctx->dest_first_mail = FALSE; + } else { + mbox_sync_handle_expunge(mail_ctx); + } + + if (!mail_ctx->mail.pseudo) { + if (!expunged) T_BEGIN { + mbox_sync_update_index(mail_ctx, rec); + } T_END; + sync_ctx->idx_seq++; + } + + if (istream_raw_mbox_next(sync_ctx->input, + mail_ctx->mail.body_size) < 0) + return -1; + offset = istream_raw_mbox_get_start_offset(sync_ctx->input); + + if (sync_ctx->need_space_seq != 0) { + if (mbox_sync_handle_missing_space(mail_ctx) < 0) + return -1; + if (mbox_sync_seek(sync_ctx, offset) < 0) + return -1; + } else if (sync_ctx->expunged_space > 0) { + if (!expunged) { + /* move the body */ + mbox_sync_file_update_ext_modified(sync_ctx); + if (mbox_move(sync_ctx, + mail_ctx->body_offset - + sync_ctx->expunged_space, + mail_ctx->body_offset, + mail_ctx->mail.body_size) < 0) + return -1; + if (mbox_sync_seek(sync_ctx, offset) < 0) + return -1; + } + } else if (partial) { + ret = mbox_sync_partial_seek_next(sync_ctx, uid + 1, + &partial, + &skipped_mails); + if (ret <= 0) + break; + } + } + if (ret < 0) + return -1; + + if (istream_raw_mbox_is_eof(sync_ctx->input)) { + /* rest of the messages in index don't exist -> expunge them */ + while (sync_ctx->idx_seq <= messages_count) + mail_index_expunge(sync_ctx->t, sync_ctx->idx_seq++); + } + + if (!skipped_mails) + sync_ctx->mbox->mbox_hdr.dirty_flag = 0; + sync_ctx->mbox->mbox_broken_offsets = FALSE; + + if (uids_broken && sync_ctx->delay_writes) { + /* once we get around to writing the changes, we'll need to do + a full sync to avoid the "UIDs broken in partial sync" + error */ + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } + return 1; +} + +static int mbox_write_pseudo(struct mbox_sync_context *sync_ctx, bool force) +{ + string_t *str; + unsigned int uid_validity; + + i_assert(sync_ctx->write_fd != -1); + + if (sync_ctx->mbox->sync_hdr_update != NULL) { + const struct mailbox_update *update = + sync_ctx->mbox->sync_hdr_update; + bool change = FALSE; + + if (update->uid_validity != 0) { + sync_ctx->base_uid_validity = update->uid_validity; + change = TRUE; + } + if (update->min_next_uid != 0) { + sync_ctx->base_uid_last = update->min_next_uid-1; + change = TRUE; + } + if (!change && !force) + return 0; + } + + uid_validity = sync_ctx->base_uid_validity != 0 ? + sync_ctx->base_uid_validity : sync_ctx->hdr->uid_validity; + i_assert(uid_validity != 0); + + str = t_str_new(1024); + str_printfa(str, "%sDate: %s\n" + "From: Mail System Internal Data <MAILER-DAEMON@%s>\n" + "Subject: DON'T DELETE THIS MESSAGE -- FOLDER INTERNAL DATA" + "\nMessage-ID: <%s@%s>\n" + "X-IMAP: %u %010u\n" + "Status: RO\n" + "\n" + PSEUDO_MESSAGE_BODY + "\n", + mbox_from_create("MAILER_DAEMON", ioloop_time), + message_date_create(ioloop_time), + my_hostname, dec2str(ioloop_time), my_hostname, + uid_validity, sync_ctx->next_uid-1); + + if (pwrite_full(sync_ctx->write_fd, + str_data(str), str_len(str), 0) < 0) { + if (!ENOSPACE(errno)) { + mbox_set_syscall_error(sync_ctx->mbox, + "pwrite_full()"); + return -1; + } + + /* out of disk space, truncate to empty */ + if (ftruncate(sync_ctx->write_fd, 0) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + } + + sync_ctx->base_uid_validity = uid_validity; + sync_ctx->base_uid_last_offset = 0; /* don't bother calculating */ + sync_ctx->base_uid_last = sync_ctx->next_uid-1; + return 0; +} + +static int mbox_append_zero(struct mbox_sync_context *sync_ctx, + uoff_t orig_file_size, uoff_t count) +{ + char block[IO_BLOCK_SIZE]; + uoff_t offset = orig_file_size; + ssize_t ret = 0; + + memset(block, 0, I_MIN(sizeof(block), count)); + while (count > 0) { + ret = pwrite(sync_ctx->write_fd, block, + I_MIN(sizeof(block), count), offset); + if (ret < 0) + break; + offset += ret; + count -= ret; + } + + if (ret < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "pwrite()"); + if (ftruncate(sync_ctx->write_fd, orig_file_size) < 0) + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + return -1; + } + return 0; +} + +static int mbox_sync_handle_eof_updates(struct mbox_sync_context *sync_ctx, + struct mbox_sync_mail_context *mail_ctx) +{ + uoff_t file_size, offset, padding, trailer_size; + int ret; + + if (!istream_raw_mbox_is_eof(sync_ctx->input)) { + i_assert(sync_ctx->need_space_seq == 0); + i_assert(sync_ctx->expunged_space == 0); + return 0; + } + + ret = i_stream_get_size(sync_ctx->file_input, TRUE, &file_size); + if (ret < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_get_size()"); + return -1; + } + if (ret == 0) { + /* Not a file - allow anyway */ + return 0; + } + + if (file_size < sync_ctx->file_input->v_offset) { + mbox_sync_set_critical(sync_ctx, + "file size unexpectedly shrank " + "(%"PRIuUOFF_T" vs %"PRIuUOFF_T")", file_size, + sync_ctx->file_input->v_offset); + return -1; + } + trailer_size = file_size - sync_ctx->file_input->v_offset; + i_assert(trailer_size <= 2); + + if (sync_ctx->need_space_seq != 0) { + i_assert(sync_ctx->write_fd != -1); + + i_assert(sync_ctx->space_diff < 0); + padding = MBOX_HEADER_PADDING * + (sync_ctx->seq - sync_ctx->need_space_seq + 1); + sync_ctx->space_diff -= padding; + + i_assert(sync_ctx->expunged_space <= -sync_ctx->space_diff); + sync_ctx->space_diff += sync_ctx->expunged_space; + sync_ctx->expunged_space = 0; + + if (mail_ctx->have_eoh && !mail_ctx->updated) + str_append_c(mail_ctx->header, '\n'); + + i_assert(sync_ctx->space_diff < 0); + + if (mbox_append_zero(sync_ctx, file_size, + -sync_ctx->space_diff) < 0) + return -1; + mbox_sync_file_updated(sync_ctx, FALSE); + + if (mbox_sync_rewrite(sync_ctx, mail_ctx, file_size, + -sync_ctx->space_diff, padding, + sync_ctx->need_space_seq, + sync_ctx->seq) < 0) + return -1; + + update_from_offsets(sync_ctx); + + sync_ctx->need_space_seq = 0; + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + } + + if (sync_ctx->expunged_space > 0) { + i_assert(sync_ctx->write_fd != -1); + + mbox_sync_file_update_ext_modified(sync_ctx); + + /* copy trailer, then truncate the file */ + file_size = sync_ctx->last_stat.st_size; + if (file_size == (uoff_t)sync_ctx->expunged_space) { + /* everything deleted, the trailer_size still contains + the \n trailer though */ + trailer_size = 0; + } else if (sync_ctx->expunged_space == (off_t)file_size + 1 || + sync_ctx->expunged_space == (off_t)file_size + 2) { + /* everything deleted and we didn't have a proper + trailer. */ + trailer_size = 0; + sync_ctx->expunged_space = file_size; + } + + i_assert(file_size >= sync_ctx->expunged_space + trailer_size); + offset = file_size - sync_ctx->expunged_space - trailer_size; + i_assert(offset == 0 || offset > 31); + + if (mbox_move(sync_ctx, offset, + offset + sync_ctx->expunged_space, + trailer_size) < 0) + return -1; + if (ftruncate(sync_ctx->write_fd, + offset + trailer_size) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, "ftruncate()"); + return -1; + } + + if (offset == 0) { + if (mbox_write_pseudo(sync_ctx, TRUE) < 0) + return -1; + } + + sync_ctx->expunged_space = 0; + mbox_sync_file_updated(sync_ctx, FALSE); + } else { + if (file_size == 0 && sync_ctx->mbox->sync_hdr_update != NULL) { + if (mbox_write_pseudo(sync_ctx, FALSE) < 0) + return -1; + } + } + return 0; +} + +static void +mbox_sync_index_update_ext_header(struct mbox_mailbox *mbox, + struct mail_index_transaction *trans) +{ + const struct mailbox_update *update = mbox->sync_hdr_update; + const void *data; + size_t data_size; + + if (update != NULL && !guid_128_is_empty(update->mailbox_guid)) { + memcpy(mbox->mbox_hdr.mailbox_guid, update->mailbox_guid, + sizeof(mbox->mbox_hdr.mailbox_guid)); + } else if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + guid_128_generate(mbox->mbox_hdr.mailbox_guid); + } + + mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx, + &data, &data_size); + if (data_size != sizeof(mbox->mbox_hdr) || + memcmp(data, &mbox->mbox_hdr, data_size) != 0) { + if (data_size != sizeof(mbox->mbox_hdr)) { + /* upgrading from v1.x */ + mail_index_ext_resize(trans, mbox->mbox_ext_idx, + sizeof(mbox->mbox_hdr), + sizeof(uint64_t), + sizeof(uint64_t)); + } + mail_index_update_header_ext(trans, mbox->mbox_ext_idx, + 0, &mbox->mbox_hdr, + sizeof(mbox->mbox_hdr)); + } +} + +static uint32_t mbox_get_uidvalidity_next(struct mailbox_list *list) +{ + const char *path; + + path = mailbox_list_get_root_forced(list, MAILBOX_LIST_PATH_TYPE_CONTROL); + path = t_strconcat(path, "/"MBOX_UIDVALIDITY_FNAME, NULL); + return mailbox_uidvalidity_next(list, path); +} + +static int mbox_sync_update_index_header(struct mbox_sync_context *sync_ctx) +{ + struct mail_index_view *view; + const struct stat *st; + uint32_t first_recent_uid, seq, seq2; + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + + if (sync_ctx->moved_offsets && + ((uint64_t)st->st_size == sync_ctx->mbox->mbox_hdr.sync_size || + (uint64_t)st->st_size == sync_ctx->orig_size)) { + /* We moved messages inside the mbox file without changing + the file's size. If mtime doesn't change, another process + not using the same index file as us can't know that the file + was changed. So make sure the mtime changes. This should + happen rarely enough that the sleeping doesn't become a + performance problem. + + Note that to do this perfectly safe we should do this wait + whenever mails are moved or expunged, regardless of whether + the file's size changed. That however could become a + performance problem and the consequences of being wrong are + quite minimal (an extra logged error message). */ + while (sync_ctx->orig_mtime == st->st_mtime) { + i_sleep_msecs(500); + if (utime(mailbox_get_path(&sync_ctx->mbox->box), NULL) < 0) { + mbox_set_syscall_error(sync_ctx->mbox, + "utime()"); + return -1; + } + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + } + } + + sync_ctx->mbox->mbox_hdr.sync_mtime = st->st_mtime; + sync_ctx->mbox->mbox_hdr.sync_size = st->st_size; + mbox_sync_index_update_ext_header(sync_ctx->mbox, sync_ctx->t); + + /* only reason not to have UID validity at this point is if the file + is entirely empty. In that case just make up a new one if needed. */ + i_assert(sync_ctx->base_uid_validity != 0 || st->st_size <= 0); + + if (sync_ctx->base_uid_validity == 0) { + sync_ctx->base_uid_validity = sync_ctx->hdr->uid_validity != 0 ? + sync_ctx->hdr->uid_validity : + mbox_get_uidvalidity_next(sync_ctx->mbox->box.list); + } + if (sync_ctx->base_uid_validity != sync_ctx->hdr->uid_validity) { + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, uid_validity), + &sync_ctx->base_uid_validity, + sizeof(sync_ctx->base_uid_validity), TRUE); + } + + if (istream_raw_mbox_is_eof(sync_ctx->input) && + sync_ctx->next_uid != sync_ctx->hdr->next_uid) { + i_assert(sync_ctx->next_uid != 0); + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, next_uid), + &sync_ctx->next_uid, sizeof(sync_ctx->next_uid), FALSE); + } + + if (sync_ctx->last_nonrecent_uid < sync_ctx->hdr->first_recent_uid) { + /* other sessions have already marked more messages as + recent. */ + sync_ctx->last_nonrecent_uid = + sync_ctx->hdr->first_recent_uid - 1; + } + + /* mark recent messages */ + view = mail_index_transaction_open_updated_view(sync_ctx->t); + if (mail_index_lookup_seq_range(view, sync_ctx->last_nonrecent_uid + 1, + (uint32_t)-1, &seq, &seq2)) { + mailbox_recent_flags_set_seqs(&sync_ctx->mbox->box, + view, seq, seq2); + } + mail_index_view_close(&view); + + first_recent_uid = !sync_ctx->keep_recent ? + sync_ctx->next_uid : sync_ctx->last_nonrecent_uid + 1; + if (sync_ctx->hdr->first_recent_uid < first_recent_uid) { + mail_index_update_header(sync_ctx->t, + offsetof(struct mail_index_header, first_recent_uid), + &first_recent_uid, sizeof(first_recent_uid), FALSE); + } + return 0; +} + +static void mbox_sync_restart(struct mbox_sync_context *sync_ctx) +{ + sync_ctx->base_uid_validity = 0; + sync_ctx->base_uid_last = 0; + sync_ctx->base_uid_last_offset = 0; + + array_clear(&sync_ctx->mails); + p_clear(sync_ctx->saved_keywords_pool); + + index_sync_changes_reset(sync_ctx->sync_changes); + mail_index_sync_reset(sync_ctx->index_sync_ctx); + mail_index_transaction_reset(sync_ctx->t); + + if (sync_ctx->index_reset) { + mail_index_reset(sync_ctx->t); + sync_ctx->reset_hdr.next_uid = 1; + sync_ctx->hdr = &sync_ctx->reset_hdr; + mailbox_recent_flags_reset(&sync_ctx->mbox->box); + } + + sync_ctx->prev_msg_uid = 0; + sync_ctx->next_uid = sync_ctx->hdr->next_uid; + sync_ctx->idx_next_uid = sync_ctx->hdr->next_uid; + sync_ctx->seq = 0; + sync_ctx->idx_seq = 1; + sync_ctx->need_space_seq = 0; + sync_ctx->expunged_space = 0; + sync_ctx->space_diff = 0; + + sync_ctx->dest_first_mail = TRUE; + sync_ctx->ext_modified = FALSE; + sync_ctx->errors = FALSE; +} + +static int mbox_sync_do(struct mbox_sync_context *sync_ctx, + enum mbox_sync_flags flags) +{ + struct mbox_index_header *mbox_hdr = &sync_ctx->mbox->mbox_hdr; + struct mbox_sync_mail_context mail_ctx; + const struct stat *st; + unsigned int i; + bool partial; + int ret; + + if (i_stream_stat(sync_ctx->file_input, FALSE, &st) < 0) { + mbox_istream_set_syscall_error(sync_ctx->mbox, + sync_ctx->file_input, "i_stream_stat()"); + return -1; + } + sync_ctx->last_stat = *st; + sync_ctx->orig_size = st->st_size; + sync_ctx->orig_atime = st->st_atime; + sync_ctx->orig_mtime = st->st_mtime; + + if ((flags & MBOX_SYNC_FORCE_SYNC) != 0) { + /* forcing a full sync. assume file has changed. */ + partial = FALSE; + mbox_hdr->dirty_flag = 1; + } else if ((uint32_t)st->st_mtime == mbox_hdr->sync_mtime && + (uint64_t)st->st_size == mbox_hdr->sync_size) { + /* file is fully synced */ + if (mbox_hdr->dirty_flag != 0 && (flags & MBOX_SYNC_UNDIRTY) != 0) + partial = FALSE; + else + partial = TRUE; + } else if ((flags & MBOX_SYNC_UNDIRTY) != 0 || + (uint64_t)st->st_size == mbox_hdr->sync_size) { + /* we want to do full syncing. always do this if + file size hasn't changed but timestamp has. it most + likely means that someone had modified some header + and we probably want to know about it */ + partial = FALSE; + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } else { + /* see if we can delay syncing the whole file. + normally we only notice expunges and appends + in partial syncing. */ + partial = TRUE; + sync_ctx->mbox->mbox_hdr.dirty_flag = 1; + } + + mbox_sync_restart(sync_ctx); + for (i = 0;;) { + ret = mbox_sync_loop(sync_ctx, &mail_ctx, partial); + if (ret > 0 && !sync_ctx->errors) + break; + if (ret < 0) + return -1; + + /* a) partial sync didn't work + b) we ran out of UIDs + c) syncing had errors */ + if (sync_ctx->delay_writes && + (sync_ctx->errors || sync_ctx->renumber_uids)) { + /* fixing a broken mbox state, be sure to write + the changes (except if we're readonly). */ + if (!sync_ctx->readonly) + sync_ctx->delay_writes = FALSE; + } + if (++i == 3) + break; + + mbox_sync_restart(sync_ctx); + partial = FALSE; + } + + if (mbox_sync_handle_eof_updates(sync_ctx, &mail_ctx) < 0) + return -1; + + /* only syncs left should be just appends (and their updates) + which weren't synced yet for some reason (crash). we'll just + ignore them, as we've overwritten them above. */ + index_sync_changes_reset(sync_ctx->sync_changes); + + if (sync_ctx->base_uid_last != sync_ctx->next_uid-1 && + ret > 0 && !sync_ctx->delay_writes && + sync_ctx->base_uid_last_offset != 0) { + /* Rewrite uid_last in X-IMAPbase header if we've seen it + (ie. the file isn't empty) */ + ret = mbox_rewrite_base_uid_last(sync_ctx); + } else { + ret = 0; + } + + if (mbox_sync_update_index_header(sync_ctx) < 0) + return -1; + return ret; +} + +int mbox_sync_header_refresh(struct mbox_mailbox *mbox) +{ + const void *data; + size_t data_size; + + if (mail_index_refresh(mbox->box.index) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + + mail_index_get_header_ext(mbox->box.view, mbox->mbox_ext_idx, + &data, &data_size); + if (data_size == 0) { + /* doesn't exist yet. */ + i_zero(&mbox->mbox_hdr); + return 0; + } + + memcpy(&mbox->mbox_hdr, data, I_MIN(sizeof(mbox->mbox_hdr), data_size)); + if (mbox->mbox_broken_offsets) + mbox->mbox_hdr.dirty_flag = 1; + return 0; +} + +int mbox_sync_get_guid(struct mbox_mailbox *mbox) +{ + struct mail_index_transaction *trans; + unsigned int lock_id; + int ret; + + if (mbox_lock(mbox, F_WRLCK, &lock_id) <= 0) + return -1; + + ret = mbox_sync_header_refresh(mbox); + if (ret == 0) { + trans = mail_index_transaction_begin(mbox->box.view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + mbox_sync_index_update_ext_header(mbox, trans); + ret = mail_index_transaction_commit(&trans); + } + mbox_unlock(mbox, lock_id); + return ret; +} + +int mbox_sync_has_changed(struct mbox_mailbox *mbox, bool leave_dirty) +{ + const struct stat *st; + struct stat statbuf; + + if (mbox->mbox_file_stream != NULL && mbox->mbox_fd == -1) { + /* read-only stream */ + if (i_stream_stat(mbox->mbox_file_stream, FALSE, &st) < 0) { + if (errno == ENOENT) { + mailbox_set_deleted(&mbox->box); + return 0; + } + mbox_istream_set_syscall_error(mbox, + mbox->mbox_file_stream, "i_stream_stat()"); + return -1; + } + } else { + if (stat(mailbox_get_path(&mbox->box), &statbuf) < 0) { + if (errno == ENOENT) { + mailbox_set_deleted(&mbox->box); + return 0; + } + mbox_set_syscall_error(mbox, "stat()"); + return -1; + } + st = &statbuf; + } + + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + + if (guid_128_is_empty(mbox->mbox_hdr.mailbox_guid)) { + /* need to assign mailbox GUID */ + return 1; + } + + if ((uint32_t)st->st_mtime == mbox->mbox_hdr.sync_mtime && + (uint64_t)st->st_size == mbox->mbox_hdr.sync_size) { + /* fully synced */ + if (mbox->mbox_hdr.dirty_flag != 0 || leave_dirty) + return 0; + /* flushing dirtiness */ + } + + /* file changed */ + return 1; +} + +static void mbox_sync_context_free(struct mbox_sync_context *sync_ctx) +{ + index_sync_changes_deinit(&sync_ctx->sync_changes); + index_storage_expunging_deinit(&sync_ctx->mbox->box); + if (sync_ctx->index_sync_ctx != NULL) + mail_index_sync_rollback(&sync_ctx->index_sync_ctx); + pool_unref(&sync_ctx->mail_keyword_pool); + pool_unref(&sync_ctx->saved_keywords_pool); + str_free(&sync_ctx->header); + str_free(&sync_ctx->from_line); + array_free(&sync_ctx->mails); +} + +static int mbox_sync_int(struct mbox_mailbox *mbox, enum mbox_sync_flags flags, + unsigned int *lock_id) +{ + struct mail_index_sync_ctx *index_sync_ctx; + struct mail_index_view *sync_view; + struct mail_index_transaction *trans; + struct mbox_sync_context sync_ctx; + enum mail_index_sync_flags sync_flags; + int ret; + bool changed, delay_writes, readonly; + + readonly = mbox_is_backend_readonly(mbox) || + (flags & MBOX_SYNC_READONLY) != 0; + delay_writes = readonly || + ((flags & MBOX_SYNC_REWRITE) == 0 && + mbox->storage->set->mbox_lazy_writes); + + if (!mbox->storage->set->mbox_dirty_syncs && + !mbox->storage->set->mbox_very_dirty_syncs) + flags |= MBOX_SYNC_UNDIRTY; + + if ((flags & MBOX_SYNC_LOCK_READING) != 0) { + if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0) + return -1; + } + + if ((flags & MBOX_SYNC_HEADER) != 0 || + (flags & MBOX_SYNC_FORCE_SYNC) != 0) { + if (mbox_sync_header_refresh(mbox) < 0) + return -1; + changed = TRUE; + } else { + bool leave_dirty = (flags & MBOX_SYNC_UNDIRTY) == 0; + if ((ret = mbox_sync_has_changed(mbox, leave_dirty)) < 0) + return -1; + changed = ret > 0; + } + + if ((flags & MBOX_SYNC_LOCK_READING) != 0) { + /* we just want to lock it for reading. if mbox hasn't been + modified don't do any syncing. */ + if (!changed) + return 0; + + /* have to sync to make sure offsets have stayed the same */ + mbox_unlock(mbox, *lock_id); + *lock_id = 0; + } + + /* flush input streams' buffers */ + if (mbox->mbox_stream != NULL) + i_stream_sync(mbox->mbox_stream); + if (mbox->mbox_file_stream != NULL) + i_stream_sync(mbox->mbox_file_stream); + +again: + if (changed) { + /* we're most likely modifying the mbox while syncing, just + lock it for writing immediately. the mbox must be locked + before index syncing is started to avoid deadlocks, so we + don't have much choice either (well, easy ones anyway). */ + int lock_type = readonly ? F_RDLCK : F_WRLCK; + + if ((ret = mbox_lock(mbox, lock_type, lock_id)) <= 0) { + if (ret == 0 || lock_type == F_RDLCK) + return -1; + + /* try as read-only */ + if (mbox_lock(mbox, F_RDLCK, lock_id) <= 0) + return -1; + mbox->backend_readonly = readonly = TRUE; + mbox->backend_readonly_set = TRUE; + delay_writes = TRUE; + } + } + + sync_flags = index_storage_get_sync_flags(&mbox->box); + if ((flags & MBOX_SYNC_REWRITE) != 0) + sync_flags |= MAIL_INDEX_SYNC_FLAG_FLUSH_DIRTY; + + ret = index_storage_expunged_sync_begin(&mbox->box, &index_sync_ctx, + &sync_view, &trans, sync_flags); + if (ret <= 0) + return ret; + + if ((mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) != 0) { + /* see if we need to drop recent flags */ + sync_ctx.hdr = mail_index_get_header(sync_view); + if (sync_ctx.hdr->first_recent_uid < sync_ctx.hdr->next_uid) + changed = TRUE; + } + + if (!changed && !mail_index_sync_have_more(index_sync_ctx)) { + /* nothing to do */ + nothing_to_do: + /* index may need to do internal syncing though, so commit + instead of rolling back. */ + index_storage_expunging_deinit(&mbox->box); + if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + return -1; + } + return 0; + } + + i_zero(&sync_ctx); + sync_ctx.mbox = mbox; + sync_ctx.keep_recent = + (mbox->box.flags & MAILBOX_FLAG_DROP_RECENT) == 0; + + sync_ctx.hdr = mail_index_get_header(sync_view); + sync_ctx.from_line = str_new(default_pool, 256); + sync_ctx.header = str_new(default_pool, 4096); + + sync_ctx.index_sync_ctx = index_sync_ctx; + sync_ctx.sync_view = sync_view; + sync_ctx.t = trans; + sync_ctx.mail_keyword_pool = + pool_alloconly_create("mbox keywords", 512); + sync_ctx.saved_keywords_pool = + pool_alloconly_create("mbox saved keywords", 4096); + + /* make sure we've read the latest keywords in index */ + (void)mail_index_get_keywords(mbox->box.index); + + i_array_init(&sync_ctx.mails, 64); + + sync_ctx.flags = flags; + sync_ctx.readonly = readonly; + sync_ctx.delay_writes = delay_writes; + + sync_ctx.sync_changes = + index_sync_changes_init(index_sync_ctx, sync_view, trans, + sync_ctx.delay_writes); + + if (!changed && delay_writes) { + /* if we have only flag changes, we don't need to open the + mbox file */ + bool expunged; + uint32_t uid; + + mbox_sync_read_index_syncs(&sync_ctx, 1, &expunged); + uid = expunged ? 1 : + index_sync_changes_get_next_uid(sync_ctx.sync_changes); + if (uid == 0) { + sync_ctx.index_sync_ctx = NULL; + mbox_sync_context_free(&sync_ctx); + goto nothing_to_do; + } + } + + if (*lock_id == 0) { + /* ok, we have something to do but no locks. we'll have to + restart syncing to avoid deadlocking. */ + mbox_sync_context_free(&sync_ctx); + changed = TRUE; + goto again; + } + + if (mbox_file_open_stream(mbox) < 0) { + mbox_sync_context_free(&sync_ctx); + return -1; + } + + sync_ctx.file_input = sync_ctx.mbox->mbox_file_stream; + sync_ctx.input = sync_ctx.mbox->mbox_stream; + sync_ctx.write_fd = sync_ctx.mbox->mbox_lock_type != F_WRLCK ? -1 : + sync_ctx.mbox->mbox_fd; + + ret = mbox_sync_do(&sync_ctx, flags); + + if (ret < 0) + mail_index_sync_rollback(&index_sync_ctx); + else if (mail_index_sync_commit(&index_sync_ctx) < 0) { + mailbox_set_index_error(&mbox->box); + ret = -1; + } + sync_ctx.t = NULL; + sync_ctx.index_sync_ctx = NULL; + + if (ret == 0 && mbox->mbox_fd != -1 && sync_ctx.keep_recent && + !readonly) { + /* try to set atime back to its original value. + (it'll fail with EPERM for shared mailboxes where we aren't + the file's owner) */ + struct utimbuf buf; + struct stat st; + + if (fstat(mbox->mbox_fd, &st) < 0) + mbox_set_syscall_error(mbox, "fstat()"); + else { + buf.modtime = st.st_mtime; + buf.actime = sync_ctx.orig_atime; + if (utime(mailbox_get_path(&mbox->box), &buf) < 0 && + errno != EPERM) + mbox_set_syscall_error(mbox, "utime()"); + } + } + + i_assert(*lock_id != 0); + + if (mbox->storage->storage.set->mail_nfs_storage && + mbox->mbox_fd != -1) { + if (fdatasync(mbox->mbox_fd) < 0) { + mbox_set_syscall_error(mbox, "fdatasync()"); + ret = -1; + } + } + + mbox_sync_context_free(&sync_ctx); + return ret; +} + +int mbox_sync(struct mbox_mailbox *mbox, enum mbox_sync_flags flags) +{ + unsigned int lock_id = 0; + int ret; + + i_assert(mbox->mbox_lock_type != F_RDLCK || + (flags & MBOX_SYNC_READONLY) != 0); + + mbox->syncing = TRUE; + ret = mbox_sync_int(mbox, flags, &lock_id); + mbox->syncing = FALSE; + + if (lock_id != 0) { + if (ret < 0) { + /* syncing failed, don't leave it locked */ + mbox_unlock(mbox, lock_id); + } else if ((flags & MBOX_SYNC_LOCK_READING) == 0) { + if (mbox_unlock(mbox, lock_id) < 0) + ret = -1; + } else if (mbox->mbox_lock_type != F_RDLCK) { + /* drop to read lock */ + unsigned int read_lock_id = 0; + + if (mbox_lock(mbox, F_RDLCK, &read_lock_id) <= 0) + ret = -1; + if (mbox_unlock(mbox, lock_id) < 0) + ret = -1; + } + } + + mailbox_sync_notify(&mbox->box, 0, 0); + return ret; +} + +struct mailbox_sync_context * +mbox_storage_sync_init(struct mailbox *box, enum mailbox_sync_flags flags) +{ + struct mbox_mailbox *mbox = MBOX_MAILBOX(box); + enum mbox_sync_flags mbox_sync_flags = 0; + int ret = 0; + + if (index_mailbox_want_full_sync(&mbox->box, flags)) { + if ((flags & MAILBOX_SYNC_FLAG_FULL_READ) != 0 && + !mbox->storage->set->mbox_very_dirty_syncs) + mbox_sync_flags |= MBOX_SYNC_UNDIRTY; + if ((flags & MAILBOX_SYNC_FLAG_FULL_WRITE) != 0) + mbox_sync_flags |= MBOX_SYNC_REWRITE; + if ((flags & MAILBOX_SYNC_FLAG_FORCE_RESYNC) != 0) { + mbox_sync_flags |= MBOX_SYNC_UNDIRTY | + MBOX_SYNC_REWRITE | MBOX_SYNC_FORCE_SYNC; + } + + ret = mbox_sync(mbox, mbox_sync_flags); + } + + return index_mailbox_sync_init(box, flags, ret < 0); +} |