1 files changed, 466 insertions, 0 deletions

diff --git a/src/lib-storage/index/mbox/mbox-sync-update.c b/src/lib-storage/index/mbox/mbox-sync-update.c
new file mode 100644
index 0000000..8442013
--- /dev/null
+++ b/src/lib-storage/index/mbox/mbox-sync-update.c
@@ -0,0 +1,466 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "str.h"
+#include "message-parser.h"
+#include "index-storage.h"
+#include "index-sync-changes.h"
+#include "mbox-storage.h"
+#include "mbox-sync-private.h"
+
+/* Line length when to wrap X-IMAP, X-IMAPbase and X-Keywords headers to next
+   line. Keep this pretty long, as after we wrap we lose compatibility with
+   UW-IMAP */
+#define KEYWORD_WRAP_LINE_LENGTH 1024
+
+static void status_flags_append(struct mbox_sync_mail_context *ctx,
+				const struct mbox_flag_type *flags_list)
+{
+	int i;
+
+	for (i = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			str_append_c(ctx->header, flags_list[i].chr);
+	}
+}
+
+void mbox_sync_move_buffer(struct mbox_sync_mail_context *ctx,
+			   size_t pos, size_t need, size_t have)
+{
+	ssize_t diff = (ssize_t)need - (ssize_t)have;
+	int i;
+
+	i_assert(have < SSIZE_T_MAX);
+
+	if (diff == 0) {
+		if (ctx->header_last_change < pos + have ||
+		    ctx->header_last_change == SIZE_MAX)
+			ctx->header_last_change = pos + have;
+	} else {
+		/* FIXME: if (diff < ctx->space && pos < ctx->offset) then
+		   move the data only up to space offset and give/take the
+		   space from there. update header_last_change accordingly.
+		   (except pos and offset can't be compared directly) */
+		ctx->header_last_change = SIZE_MAX;
+		for (i = 0; i < MBOX_HDR_COUNT; i++) {
+			if (ctx->hdr_pos[i] > pos &&
+			    ctx->hdr_pos[i] != SIZE_MAX)
+				ctx->hdr_pos[i] = (ssize_t)ctx->hdr_pos[i] + diff;
+		}
+
+		if (ctx->mail.space > 0) {
+			i_assert(ctx->mail.offset + ctx->mail.space <=
+				 ctx->hdr_offset + pos ||
+				 ctx->mail.offset > ctx->hdr_offset + pos + have);
+			if (ctx->mail.offset > ctx->hdr_offset + pos) {
+				/* free space offset moves */
+				ctx->mail.offset = (ssize_t)ctx->mail.offset + diff;
+			}
+		}
+
+		if (diff < 0)
+			str_delete(ctx->header, pos, -diff);
+		else {
+			ctx->header_last_change = SIZE_MAX;
+			buffer_copy(ctx->header, pos + diff,
+				    ctx->header, pos, SIZE_MAX);
+		}
+	}
+}
+
+static void status_flags_replace(struct mbox_sync_mail_context *ctx, size_t pos,
+				 const struct mbox_flag_type *flags_list)
+{
+	unsigned char *data;
+	size_t size;
+	int i, need, have;
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+
+	if (ctx->header_first_change > pos)
+		ctx->header_first_change = pos;
+
+	/* how many bytes do we need? */
+	for (i = 0, need = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			need++;
+	}
+
+	/* how many bytes do we have now? */
+	data = buffer_get_modifiable_data(ctx->header, &size);
+	for (have = 0; pos < size; pos++) {
+		if (data[pos] == '\n' || data[pos] == '\r')
+			break;
+
+		/* see if this is unknown flag for us */
+		for (i = 0; flags_list[i].chr != 0; i++) {
+			if (flags_list[i].chr == (char)data[pos])
+				break;
+		}
+
+		if (flags_list[i].chr != 0)
+			have++;
+		else {
+			/* save this one */
+			data[pos-have] = data[pos];
+		}
+	}
+	pos -= have;
+        mbox_sync_move_buffer(ctx, pos, need, have);
+
+	/* @UNSAFE */
+	data = buffer_get_space_unsafe(ctx->header, pos, need);
+	for (i = 0; flags_list[i].chr != 0; i++) {
+		if ((ctx->mail.flags & flags_list[i].flag) != 0)
+			*data++ = flags_list[i].chr;
+	}
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+}
+
+static void
+keywords_append(struct mbox_sync_context *sync_ctx, string_t *dest,
+		const ARRAY_TYPE(keyword_indexes) *keyword_indexes_arr)
+{
+	struct index_mailbox_context *ibox =
+		INDEX_STORAGE_CONTEXT(&sync_ctx->mbox->box);
+	const char *const *keyword_names;
+	const unsigned int *keyword_indexes;
+	unsigned int i, idx_count, keywords_count;
+	size_t last_break;
+
+	keyword_names = array_get(ibox->keyword_names,
+				  &keywords_count);
+	keyword_indexes = array_get(keyword_indexes_arr, &idx_count);
+
+	for (i = 0, last_break = str_len(dest); i < idx_count; i++) {
+		i_assert(keyword_indexes[i] < keywords_count);
+
+		/* wrap the line whenever it gets too long */
+		if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH) {
+			if (i > 0)
+				str_append_c(dest, ' ');
+		} else {
+			str_append(dest, "\n\t");
+			last_break = str_len(dest);
+		}
+		str_append(dest, keyword_names[keyword_indexes[i]]);
+	}
+}
+
+static void
+keywords_append_all(struct mbox_sync_mail_context *ctx, string_t *dest,
+		    size_t startpos)
+{
+	struct index_mailbox_context *ibox =
+		INDEX_STORAGE_CONTEXT(&ctx->sync_ctx->mbox->box);
+	const char *const *names;
+	const unsigned char *p;
+	unsigned int i, count;
+	size_t last_break;
+
+	p = str_data(dest);
+	if (str_len(dest) - startpos < KEYWORD_WRAP_LINE_LENGTH)
+		last_break = startpos;
+	else {
+		/* set last_break to beginning of line */
+		for (last_break = str_len(dest); last_break > 0; last_break--) {
+			if (p[last_break-1] == '\n')
+				break;
+		}
+	}
+
+	names = array_get(ibox->keyword_names, &count);
+	for (i = 0; i < count; i++) {
+		/* wrap the line whenever it gets too long */
+		if (str_len(dest) - last_break < KEYWORD_WRAP_LINE_LENGTH)
+			str_append_c(dest, ' ');
+		else {
+			str_append(dest, "\n\t");
+			last_break = str_len(dest);
+		}
+		str_append(dest, names[i]);
+	}
+}
+
+static void mbox_sync_add_missing_headers(struct mbox_sync_mail_context *ctx)
+{
+	size_t new_hdr_size, startpos;
+
+	new_hdr_size = str_len(ctx->header);
+	if (new_hdr_size > 0 &&
+	    str_data(ctx->header)[new_hdr_size-1] != '\n') {
+		/* broken header - doesn't end with \n. fix it. */
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->sync_ctx->dest_first_mail &&
+	    ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX) {
+		i_assert(ctx->sync_ctx->base_uid_validity != 0);
+
+		str_append(ctx->header, "X-IMAPbase: ");
+		ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] = str_len(ctx->header);
+		/* startpos must start from identical position as when
+		   updating */
+		startpos = str_len(ctx->header);
+		str_printfa(ctx->header, "%u ",
+			    ctx->sync_ctx->base_uid_validity);
+
+		ctx->last_uid_updated_value = ctx->sync_ctx->next_uid-1;
+		ctx->last_uid_value_start_pos = str_len(ctx->header) -
+			ctx->hdr_pos[MBOX_HDR_X_IMAPBASE];
+		ctx->imapbase_updated = TRUE;
+		str_printfa(ctx->header, "%010u", ctx->last_uid_updated_value);
+
+		keywords_append_all(ctx, ctx->header, startpos);
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX && !ctx->mail.pseudo) {
+		str_append(ctx->header, "X-UID: ");
+		ctx->hdr_pos[MBOX_HDR_X_UID] = str_len(ctx->header);
+		str_printfa(ctx->header, "%u\n", ctx->mail.uid);
+	}
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+
+	if (ctx->hdr_pos[MBOX_HDR_STATUS] == SIZE_MAX &&
+	    (ctx->mail.flags & STATUS_FLAGS_MASK) != 0) {
+		str_append(ctx->header, "Status: ");
+		ctx->hdr_pos[MBOX_HDR_STATUS] = str_len(ctx->header);
+		status_flags_append(ctx, mbox_status_flags);
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] == SIZE_MAX &&
+	    (ctx->mail.flags & XSTATUS_FLAGS_MASK) != 0) {
+		str_append(ctx->header, "X-Status: ");
+		ctx->hdr_pos[MBOX_HDR_X_STATUS] = str_len(ctx->header);
+		status_flags_append(ctx, mbox_xstatus_flags);
+		str_append_c(ctx->header, '\n');
+	}
+
+	ctx->mail.flags ^= MBOX_NONRECENT_KLUDGE;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX &&
+	    array_is_created(&ctx->mail.keywords) &&
+	    array_count(&ctx->mail.keywords) > 0) {
+		str_append(ctx->header, "X-Keywords: ");
+		ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] = str_len(ctx->header);
+		keywords_append(ctx->sync_ctx, ctx->header,
+				&ctx->mail.keywords);
+		str_append_c(ctx->header, '\n');
+	}
+
+	if (ctx->content_length == UOFF_T_MAX &&
+	    ctx->mail.body_size >= MBOX_MIN_CONTENT_LENGTH_SIZE) {
+		str_printfa(ctx->header, "Content-Length: %"PRIuUOFF_T"\n",
+			    ctx->mail.body_size);
+	}
+
+	if (str_len(ctx->header) != new_hdr_size) {
+		if (ctx->header_first_change == SIZE_MAX)
+			ctx->header_first_change = new_hdr_size;
+		ctx->header_last_change = SIZE_MAX;
+	}
+
+	if (ctx->have_eoh)
+		str_append_c(ctx->header, '\n');
+}
+
+static void mbox_sync_update_status(struct mbox_sync_mail_context *ctx)
+{
+	if (ctx->hdr_pos[MBOX_HDR_STATUS] != SIZE_MAX) {
+		status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_STATUS],
+				     mbox_status_flags);
+	}
+}
+
+static void mbox_sync_update_xstatus(struct mbox_sync_mail_context *ctx)
+{
+	if (ctx->hdr_pos[MBOX_HDR_X_STATUS] != SIZE_MAX) {
+		status_flags_replace(ctx, ctx->hdr_pos[MBOX_HDR_X_STATUS],
+				     mbox_xstatus_flags);
+	}
+}
+
+static void mbox_sync_update_line(struct mbox_sync_mail_context *ctx,
+				  size_t pos, string_t *new_line)
+{
+	const char *hdr, *p;
+	uoff_t file_pos;
+
+	if (ctx->header_first_change > pos)
+		ctx->header_first_change = pos;
+
+	/* set p = end of header, handle also wrapped headers */
+	hdr = p = str_c(ctx->header) + pos;
+	for (;;) {
+		p = strchr(p, '\n');
+		if (p == NULL) {
+			/* shouldn't really happen, but allow anyway.. */
+			p = hdr + strlen(hdr);
+			break;
+		}
+		if (p[1] != '\t' && p[1] != ' ')
+			break;
+		p += 2;
+	}
+
+	file_pos = pos + ctx->hdr_offset;
+	if (ctx->mail.space > 0 && ctx->mail.offset >= file_pos &&
+	    ctx->mail.offset < file_pos + (p - hdr)) {
+		/* extra space points to this line. remove it. */
+		ctx->mail.offset = ctx->hdr_offset;
+		ctx->mail.space = 0;
+	}
+
+	mbox_sync_move_buffer(ctx, pos, str_len(new_line), p - hdr + 1);
+	buffer_copy(ctx->header, pos, new_line, 0, SIZE_MAX);
+}
+
+static void mbox_sync_update_xkeywords(struct mbox_sync_mail_context *ctx)
+{
+	string_t *str;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_KEYWORDS] == SIZE_MAX)
+		return;
+
+	str = t_str_new(256);
+	if (array_is_created(&ctx->mail.keywords))
+		keywords_append(ctx->sync_ctx, str, &ctx->mail.keywords);
+	str_append_c(str, '\n');
+	mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_KEYWORDS], str);
+}
+
+static void mbox_sync_update_x_imap_base(struct mbox_sync_mail_context *ctx)
+{
+	struct mbox_sync_context *sync_ctx = ctx->sync_ctx;
+	string_t *str;
+
+	i_assert(sync_ctx->base_uid_validity != 0);
+
+	if (!sync_ctx->dest_first_mail ||
+	    ctx->hdr_pos[MBOX_HDR_X_IMAPBASE] == SIZE_MAX)
+		return;
+
+	if (!ctx->imapbase_rewrite) {
+		/* uid-last might need updating, but we'll do it later by
+		   writing it directly where needed. */
+		return;
+	}
+
+	/* a) keyword list changed, b) uid-last didn't use 10 digits */
+	str = t_str_new(200);
+	str_printfa(str, "%u ", sync_ctx->base_uid_validity);
+
+	ctx->last_uid_updated_value = sync_ctx->next_uid-1;
+	ctx->last_uid_value_start_pos = str_len(str);
+	ctx->imapbase_updated = TRUE;
+	str_printfa(str, "%010u", ctx->last_uid_updated_value);
+
+	keywords_append_all(ctx, str, 0);
+	str_append_c(str, '\n');
+
+        mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_IMAPBASE], str);
+}
+
+static void mbox_sync_update_x_uid(struct mbox_sync_mail_context *ctx)
+{
+	string_t *str;
+
+	if (ctx->hdr_pos[MBOX_HDR_X_UID] == SIZE_MAX ||
+	    ctx->mail.uid == ctx->parsed_uid)
+		return;
+
+	str = t_str_new(64);
+	str_printfa(str, "%u\n", ctx->mail.uid);
+	mbox_sync_update_line(ctx, ctx->hdr_pos[MBOX_HDR_X_UID], str);
+}
+
+static void mbox_sync_update_header_real(struct mbox_sync_mail_context *ctx)
+{
+	i_assert(ctx->mail.uid != 0 || ctx->mail.pseudo);
+
+	if (!ctx->sync_ctx->keep_recent)
+		ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT);
+
+	mbox_sync_update_status(ctx);
+	mbox_sync_update_xstatus(ctx);
+	mbox_sync_update_xkeywords(ctx);
+
+	mbox_sync_update_x_imap_base(ctx);
+	mbox_sync_update_x_uid(ctx);
+
+	mbox_sync_add_missing_headers(ctx);
+	ctx->updated = TRUE;
+}
+
+void mbox_sync_update_header(struct mbox_sync_mail_context *ctx)
+{
+	T_BEGIN {
+		mbox_sync_update_header_real(ctx);
+	} T_END;
+}
+
+static void
+mbox_sync_update_header_from_real(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail)
+{
+	if (mail->status_broken ||
+	    (ctx->mail.flags & STATUS_FLAGS_MASK) !=
+	    (mail->flags & STATUS_FLAGS_MASK) ||
+	    (ctx->mail.flags & MAIL_RECENT) != 0) {
+		ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(STATUS_FLAGS_MASK)) |
+			(mail->flags & STATUS_FLAGS_MASK);
+		if (!ctx->sync_ctx->keep_recent)
+                        ctx->mail.flags &= ENUM_NEGATE(MAIL_RECENT);
+		mbox_sync_update_status(ctx);
+	}
+	if (mail->xstatus_broken ||
+	    (ctx->mail.flags & XSTATUS_FLAGS_MASK) !=
+	    (mail->flags & XSTATUS_FLAGS_MASK)) {
+		ctx->mail.flags = (ctx->mail.flags & ENUM_NEGATE(XSTATUS_FLAGS_MASK)) |
+			(mail->flags & XSTATUS_FLAGS_MASK);
+		mbox_sync_update_xstatus(ctx);
+	}
+	if (!array_is_created(&mail->keywords) ||
+	    array_count(&mail->keywords) == 0) {
+		/* no keywords for this mail */
+		if (array_is_created(&ctx->mail.keywords)) {
+			array_clear(&ctx->mail.keywords);
+			mbox_sync_update_xkeywords(ctx);
+		}
+	} else if (!array_is_created(&ctx->mail.keywords)) {
+		/* adding first keywords */
+		p_array_init(&ctx->mail.keywords,
+			     ctx->sync_ctx->mail_keyword_pool,
+			     array_count(&mail->keywords));
+		array_append_array(&ctx->mail.keywords,
+				   &mail->keywords);
+		mbox_sync_update_xkeywords(ctx);
+	} else if (!array_cmp(&ctx->mail.keywords, &mail->keywords)) {
+		/* keywords changed. */
+		array_clear(&ctx->mail.keywords);
+		array_append_array(&ctx->mail.keywords,
+				   &mail->keywords);
+		mbox_sync_update_xkeywords(ctx);
+	}
+
+	i_assert(ctx->mail.uid == 0 || ctx->mail.uid == mail->uid);
+	ctx->mail.uid = mail->uid;
+
+	mbox_sync_update_x_imap_base(ctx);
+	mbox_sync_update_x_uid(ctx);
+	mbox_sync_add_missing_headers(ctx);
+}
+
+void mbox_sync_update_header_from(struct mbox_sync_mail_context *ctx,
+				  const struct mbox_sync_mail *mail)
+{
+	T_BEGIN {
+		mbox_sync_update_header_from_real(ctx, mail);
+	} T_END;
+}