path: root/src/lib-storage/index/imapc/imapc-mailbox.c

/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "ioloop.h"
#include "str.h"
#include "mail-index-modseq.h"
#include "imap-arg.h"
#include "imap-seqset.h"
#include "imap-util.h"
#include "imapc-mail.h"
#include "imapc-msgmap.h"
#include "imapc-list.h"
#include "imapc-search.h"
#include "imapc-sync.h"
#include "imapc-storage.h"

#define NOTIFY_DELAY_MSECS 500

void imapc_mailbox_set_corrupted(struct imapc_mailbox *mbox,
				 const char *reason, ...)
{
	const char *errmsg;
	va_list va;

	va_start(va, reason);
	errmsg = t_strdup_printf("Mailbox '%s' state corrupted: %s",
		mbox->box.name, t_strdup_vprintf(reason, va));
	va_end(va);

	mail_storage_set_internal_error(&mbox->storage->storage);

	if (!mbox->initial_sync_done) {
		/* we failed during initial sync. need to rebuild indexes if
		   we want to get this fixed */
		mail_index_mark_corrupted(mbox->box.index);
	} else {
		/* maybe the remote server is buggy and has become confused.
		   try reconnecting. */
	}
	imapc_client_mailbox_reconnect(mbox->client_box, errmsg);
}

struct mail_index_view *
imapc_mailbox_get_sync_view(struct imapc_mailbox *mbox)
{
	if (mbox->sync_view == NULL)
		mbox->sync_view = mail_index_view_open(mbox->box.index);
	return mbox->sync_view;
}

static void imapc_mailbox_init_delayed_trans(struct imapc_mailbox *mbox)
{
	if (mbox->delayed_sync_trans != NULL)
		return;

	i_assert(mbox->delayed_sync_cache_view == NULL);
	i_assert(mbox->delayed_sync_cache_trans == NULL);

	mbox->delayed_sync_trans =
		mail_index_transaction_begin(imapc_mailbox_get_sync_view(mbox),
					MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
	mbox->delayed_sync_view =
		mail_index_transaction_open_updated_view(mbox->delayed_sync_trans);

	mbox->delayed_sync_cache_view =
		mail_cache_view_open(mbox->box.cache, mbox->delayed_sync_view);
	mbox->delayed_sync_cache_trans =
		mail_cache_get_transaction(mbox->delayed_sync_cache_view,
					   mbox->delayed_sync_trans);
}

static int imapc_mailbox_commit_delayed_expunges(struct imapc_mailbox *mbox)
{
	struct mail_index_view *view = imapc_mailbox_get_sync_view(mbox);
	struct mail_index_transaction *trans;
	struct seq_range_iter iter;
	unsigned int n;
	uint32_t lseq, uid;
	int ret;

	trans = mail_index_transaction_begin(view,
			MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);

	seq_range_array_iter_init(&iter, &mbox->delayed_expunged_uids); n = 0;
	while (seq_range_array_iter_nth(&iter, n++, &uid)) {
		if (mail_index_lookup_seq(view, uid, &lseq))
			mail_index_expunge(trans, lseq);
	}
	array_clear(&mbox->delayed_expunged_uids);
	ret = mail_index_transaction_commit(&trans);
	if (ret < 0)
		mailbox_set_index_error(&mbox->box);
	return ret;
}

int imapc_mailbox_commit_delayed_trans(struct imapc_mailbox *mbox,
				       bool force, bool *changes_r)
{
	int ret = 0;

	*changes_r = FALSE;

	if (mbox->delayed_sync_view != NULL)
		mail_index_view_close(&mbox->delayed_sync_view);
	if (mbox->delayed_sync_trans == NULL)
		;
	else if (!mbox->selected && !force) {
		/* ignore any changes done during SELECT */
		mail_index_transaction_rollback(&mbox->delayed_sync_trans);
	} else {
		if (mail_index_transaction_commit(&mbox->delayed_sync_trans) < 0) {
			mailbox_set_index_error(&mbox->box);
			ret = -1;
		}
		*changes_r = TRUE;
	}
	mbox->delayed_sync_cache_trans = NULL;
	if (mbox->delayed_sync_cache_view != NULL)
		mail_cache_view_close(&mbox->delayed_sync_cache_view);

	if (array_count(&mbox->delayed_expunged_uids) > 0) {
		/* delayed expunges - commit them now in a separate
		   transaction. Reopen mbox->sync_view to see changes
		   committed in delayed_sync_trans. */
		if (mbox->sync_view != NULL)
			mail_index_view_close(&mbox->sync_view);
		if (imapc_mailbox_commit_delayed_expunges(mbox) < 0)
			ret = -1;
	}

	if (mbox->sync_view != NULL)
		mail_index_view_close(&mbox->sync_view);
	i_assert(mbox->delayed_sync_trans == NULL);
	i_assert(mbox->delayed_sync_view == NULL);
	i_assert(mbox->delayed_sync_cache_trans == NULL);
	return ret;
}

static void imapc_mailbox_idle_timeout(struct imapc_mailbox *mbox)
{
	timeout_remove(&mbox->to_idle_delay);
	if (mbox->box.notify_callback != NULL)
		mbox->box.notify_callback(&mbox->box, mbox->box.notify_context);
}

static void imapc_mailbox_idle_notify(struct imapc_mailbox *mbox)
{
	struct ioloop *old_ioloop = current_ioloop;

	if (mbox->box.notify_callback != NULL &&
	    mbox->to_idle_delay == NULL) {
		io_loop_set_current(mbox->storage->root_ioloop);
		mbox->to_idle_delay =
			timeout_add_short(NOTIFY_DELAY_MSECS,
					  imapc_mailbox_idle_timeout, mbox);
		io_loop_set_current(old_ioloop);
	}
}

static void
imapc_mailbox_index_expunge(struct imapc_mailbox *mbox, uint32_t uid)
{
	uint32_t lseq;

	if (mail_index_lookup_seq(mbox->sync_view, uid, &lseq))
		mail_index_expunge(mbox->delayed_sync_trans, lseq);
	else if (mail_index_lookup_seq(mbox->delayed_sync_view, uid, &lseq)) {
		/* this message exists only in this transaction. lib-index
		   can't currently handle expunging anything except the last
		   appended message in a transaction, and fixing it would be
		   quite a lot of trouble. so instead we'll just delay doing
		   this expunge until after the current transaction has been
		   committed. */
		seq_range_array_add(&mbox->delayed_expunged_uids, uid);
	} else {
		/* already expunged by another session */
	}
}

static void
imapc_mailbox_fetch_state_finish(struct imapc_mailbox *mbox)
{
	uint32_t lseq, uid, msg_count;

	if (mbox->sync_next_lseq == 0) {
		/* FETCH n:*, not 1:* */
		i_assert(mbox->state_fetched_success ||
			 (mbox->box.flags & MAILBOX_FLAG_SAVEONLY) != 0);
		return;
	}

	/* if we haven't seen FETCH reply for some messages at the end of
	   mailbox they've been externally expunged. */
	msg_count = mail_index_view_get_messages_count(mbox->delayed_sync_view);
	for (lseq = mbox->sync_next_lseq; lseq <= msg_count; lseq++) {
		mail_index_lookup_uid(mbox->delayed_sync_view, lseq, &uid);
		if (uid >= mbox->sync_uid_next) {
			/* another process already added new messages to index
			   that our IMAP connection hasn't seen yet */
			break;
		}
		imapc_mailbox_index_expunge(mbox, uid);
	}

	mbox->sync_next_lseq = 0;
	mbox->sync_next_rseq = 0;
	mbox->state_fetched_success = TRUE;
}

static void
imapc_mailbox_fetch_state_callback(const struct imapc_command_reply *reply,
				   void *context)
{
	struct imapc_mailbox *mbox = context;

	mbox->state_fetching_uid1 = FALSE;
	mbox->delayed_untagged_exists = FALSE;
	imapc_client_stop(mbox->storage->client->client);

	switch (reply->state) {
	case IMAPC_COMMAND_STATE_OK:
		imapc_mailbox_fetch_state_finish(mbox);
		break;
	case IMAPC_COMMAND_STATE_NO:
		imapc_copy_error_from_reply(mbox->storage, MAIL_ERROR_PARAMS, reply);
		break;
	case IMAPC_COMMAND_STATE_DISCONNECTED:
		mail_storage_set_internal_error(mbox->box.storage);

		break;
	default:
		mail_storage_set_critical(mbox->box.storage,
			"imapc: state FETCH failed: %s", reply->text_full);
		break;
	}
}

void imap_mailbox_select_finish(struct imapc_mailbox *mbox)
{
	if (mbox->exists_count == 0) {
		/* no mails. expunge everything. */
		mbox->sync_next_lseq = 1;
		imapc_mailbox_init_delayed_trans(mbox);
		imapc_mailbox_fetch_state_finish(mbox);
	}
	mbox->selected = TRUE;
}

bool
imapc_mailbox_fetch_state(struct imapc_mailbox *mbox, uint32_t first_uid)
{
	struct imapc_command *cmd;

	if (mbox->exists_count == 0) {
		/* empty mailbox - no point in fetching anything.
		   just make sure everything is expunged in local index.
		   Delay calling imapc_mailbox_fetch_state_finish() until
		   SELECT finishes, so we see the updated UIDNEXT. */
		return FALSE;
	}
	if (mbox->state_fetching_uid1) {
		/* retrying after reconnection - don't send duplicate */
		return FALSE;
	}

	string_t *str = t_str_new(64);
	str_printfa(str, "UID FETCH %u:* (FLAGS", first_uid);
	if (imapc_mailbox_has_modseqs(mbox)) {
		str_append(str, " MODSEQ");
		mail_index_modseq_enable(mbox->box.index);
	}
	if (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
		enum mailbox_info_flags flags;

		if (!mail_index_is_in_memory(mbox->box.index)) {
			/* these can be efficiently fetched among flags and
			   stored into cache */
			str_append(str, " X-GM-MSGID");
		}
		/* do this only for the \All mailbox */
		if (imapc_list_get_mailbox_flags(mbox->box.list,
						 mbox->box.name, &flags) == 0 &&
		    (flags & MAILBOX_SPECIALUSE_ALL) != 0)
			str_append(str, " X-GM-LABELS");

	}
	str_append_c(str, ')');

	cmd = imapc_client_mailbox_cmd(mbox->client_box,
		imapc_mailbox_fetch_state_callback, mbox);
	if (first_uid == 1) {
		mbox->sync_next_lseq = 1;
		mbox->sync_next_rseq = 1;
		mbox->state_fetched_success = FALSE;
		/* only the FETCH 1:* is retriable - others will be retried
		   by the 1:* after the reconnection */
		imapc_command_set_flags(cmd, IMAPC_COMMAND_FLAG_RETRIABLE);
	}
	mbox->state_fetching_uid1 = first_uid == 1;
	imapc_command_send(cmd, str_c(str));
	return TRUE;
}

static void
imapc_untagged_exists(const struct imapc_untagged_reply *reply,
		      struct imapc_mailbox *mbox)
{
	struct mail_index_view *view;
	uint32_t exists_count = reply->num;

	if (mbox == NULL)
		return;
	if (mbox->exists_received &&
	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) {
		/* ignore all except the first EXISTS reply (returned by
		   SELECT) */
		return;
	}

	mbox->exists_count = exists_count;
	mbox->exists_received = TRUE;

	view = mbox->delayed_sync_view;
	if (view == NULL)
		view = imapc_mailbox_get_sync_view(mbox);

	if (mbox->selecting) {
		/* We don't know the latest flags, refresh them. */
		(void)imapc_mailbox_fetch_state(mbox, 1);
	} else if (mbox->sync_fetch_first_uid != 1) {
		const struct mail_index_header *hdr;
		hdr = mail_index_get_header(view);
		mbox->sync_fetch_first_uid = hdr->next_uid;
		mbox->delayed_untagged_exists = TRUE;
	}
	imapc_mailbox_idle_notify(mbox);
}

static bool keywords_are_equal(struct mail_keywords *kw,
			       const ARRAY_TYPE(keyword_indexes) *kw_arr)
{
	const unsigned int *kw_idx;
	unsigned int i, j, count;

	kw_idx = array_get(kw_arr, &count);
	if (count != kw->count)
		return FALSE;

	/* there are normally only a few keywords, so O(n^2) is fine */
	for (i = 0; i < count; i++) {
		for (j = 0; j < count; j++) {
			if (kw->idx[i] == kw_idx[j])
				break;
		}
		if (j == count)
			return FALSE;
	}
	return TRUE;
}

static int
imapc_mailbox_msgmap_update(struct imapc_mailbox *mbox,
			    uint32_t rseq, uint32_t fetch_uid,
			    uint32_t *lseq_r, uint32_t *uid_r,
			    bool *new_message_r)
{
	struct imapc_msgmap *msgmap;
	uint32_t uid, msg_count, rseq2;

	*lseq_r = 0;
	*uid_r = uid = fetch_uid;
	*new_message_r = FALSE;

	if (rseq > mbox->exists_count) {
		/* Receiving a FETCH for a message that EXISTS hasn't
		   announced yet. MS Exchange has a bug where our UID FETCH
		   request sometimes sends replies where sequences are above
		   EXISTS value, but their UIDs are for existing messages.
		   We'll just ignore these replies. */
		return 0;
	}
	if (rseq < mbox->prev_skipped_rseq &&
	    fetch_uid > mbox->prev_skipped_uid) {
		/* This was the initial attempt at catching the above
		   MS Exchange bug, but the above one appears to catch all
		   these cases. But keep it here just in case. */
		imapc_mailbox_set_corrupted(mbox,
			"FETCH sequence/UID order is mixed "
			"(seq=%u,%u vs uid=%u,%u)",
			mbox->prev_skipped_rseq, rseq,
			mbox->prev_skipped_uid, fetch_uid);
		return -1;
	}

	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
	msg_count = imapc_msgmap_count(msgmap);
	if (fetch_uid != 0 && mbox->state_fetched_success &&
	    (IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_FETCH_MSN_WORKAROUNDS) ||
	     IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES))) {
		/* if we know the UID, use own own generated rseq instead of
		   the potentially broken rseq that the server sent.
		   Skip this during the initial FETCH 1:* (UID ..) handling,
		   or we can't detect duplicate UIDs and will instead
		   assert-crash later on. */
		uint32_t fixed_rseq;

		if (imapc_msgmap_uid_to_rseq(msgmap, fetch_uid, &fixed_rseq))
			rseq = fixed_rseq;
		else if (fetch_uid >= imapc_msgmap_uidnext(msgmap) &&
			 rseq <= msg_count) {
			/* The current rseq is wrong. Lets hope that the
			   correct rseq is the next new one. This happens
			   especially with no-msn-updates when mails have been
			   expunged and new mails arrive in the same session. */
			rseq = msg_count+1;
		}
	}

	if (rseq <= msg_count) {
		uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
		if (uid != fetch_uid && fetch_uid != 0) {
			imapc_mailbox_set_corrupted(mbox,
				"FETCH UID mismatch (%u != %u)",
				fetch_uid, uid);
			return -1;
		}
		*uid_r = uid;
	} else if (fetch_uid == 0 || rseq != msg_count+1) {
		/* probably a flag update for a message we haven't yet
		   received our initial UID FETCH for. we should get
		   another one. */
		if (fetch_uid == 0)
			return 0;

		if (imapc_msgmap_uid_to_rseq(msgmap, fetch_uid, &rseq2)) {
			imapc_mailbox_set_corrupted(mbox,
				"FETCH returned wrong sequence for UID %u "
				"(%u != %u)", fetch_uid, rseq, rseq2);
			return -1;
		}
		mbox->prev_skipped_rseq = rseq;
		mbox->prev_skipped_uid = fetch_uid;
		/* Check if this uid must be added later when syncing. */
		*new_message_r = TRUE;
	} else if (fetch_uid < imapc_msgmap_uidnext(msgmap)) {
		imapc_mailbox_set_corrupted(mbox,
			"Expunged message reappeared in session "
			"(uid=%u < next_uid=%u)",
			fetch_uid, imapc_msgmap_uidnext(msgmap));
		return -1;
	} else {
		/* newly seen message */
		imapc_msgmap_append(msgmap, rseq, uid);
		if (uid < mbox->min_append_uid ||
		    uid < mail_index_get_header(mbox->delayed_sync_view)->next_uid) {
			/* message is already added to index */
		} else if (mbox->state_fetching_uid1) {
			/* Initial fetching, allow messages to be appened to
			   index directly */
			mail_index_append(mbox->delayed_sync_trans,
					  uid, lseq_r);
			mbox->min_append_uid = uid + 1;
		} else {
			/* message is not yet added to index, in order to
			   prevent log synchronization errors add this
			   message later, when the mailbox is synced. */
			*new_message_r = TRUE;
		}
	}
	return 0;
}

bool imapc_mailbox_name_equals(struct imapc_mailbox *mbox,
			       const char *remote_name)
{
	const char *imapc_remote_name =
		imapc_mailbox_get_remote_name(mbox);

	if (strcmp(imapc_remote_name, remote_name) == 0) {
		/* match */
		return TRUE;
	} else if (strcasecmp(mbox->box.name, "INBOX") == 0 &&
		   strcasecmp(remote_name, "INBOX") == 0) {
		/* case-insensitive INBOX */
		return TRUE;
	}
	return FALSE;
}

static struct imapc_untagged_fetch_ctx *
imapc_untagged_fetch_ctx_create(void)
{
	pool_t pool = pool_alloconly_create("imapc untagged fetch ctx", 128);
	struct imapc_untagged_fetch_ctx *ctx =
		p_new(pool, struct imapc_untagged_fetch_ctx, 1);
	ctx->pool = pool;
	return ctx;
}

void imapc_untagged_fetch_ctx_free(struct imapc_untagged_fetch_ctx **_ctx)
{
	struct imapc_untagged_fetch_ctx *ctx = *_ctx;

	*_ctx = NULL;
	i_assert(ctx != NULL);

	pool_unref(&ctx->pool);
}

void imapc_untagged_fetch_update_flags(struct imapc_mailbox *mbox,
				       struct imapc_untagged_fetch_ctx *ctx,
				       struct mail_index_view *view,
				       uint32_t lseq)
{
	ARRAY_TYPE(keyword_indexes) old_kws;
	struct mail_keywords *kw;
	const struct mail_index_record *rec = NULL;
	const char *atom;

	if (!ctx->have_flags)
		return;

	rec = mail_index_lookup(view, lseq);
	if (rec->flags != ctx->flags) {
		mail_index_update_flags(mbox->delayed_sync_trans, lseq,
					MODIFY_REPLACE, ctx->flags);
	}

	t_array_init(&old_kws, 8);
	mail_index_lookup_keywords(view, lseq, &old_kws);

	if (ctx->have_gmail_labels) {
		/* add keyword for mails that have GMail labels.
		   this can be used for "All Mail" mailbox migrations
		   with dsync */
		atom = "$GMailHaveLabels";
		array_push_back(&ctx->keywords, &atom);
	}

	array_append_zero(&ctx->keywords);
	kw = mail_index_keywords_create(mbox->box.index,
					array_front(&ctx->keywords));
	if (!keywords_are_equal(kw, &old_kws)) {
		mail_index_update_keywords(mbox->delayed_sync_trans,
					   lseq, MODIFY_REPLACE, kw);
	}
	mail_index_keywords_unref(&kw);
}

static bool imapc_untagged_fetch_handle(struct imapc_mailbox *mbox,
					struct imapc_untagged_fetch_ctx *ctx,
					uint32_t rseq)
{
	uint32_t lseq;
	bool new_message;

	imapc_mailbox_init_delayed_trans(mbox);
	if (imapc_mailbox_msgmap_update(mbox, rseq, ctx->fetch_uid,
					&lseq, &ctx->uid,
					&new_message) < 0 || ctx->uid == 0)
		return FALSE;

	if ((ctx->flags & MAIL_RECENT) == 0 && mbox->highest_nonrecent_uid < ctx->uid) {
		/* remember for STATUS_FIRST_RECENT_UID */
		mbox->highest_nonrecent_uid = ctx->uid;
	}
	/* FIXME: we should ideally also pass these through so they show up
	   to clients. */
	ctx->flags &= ENUM_NEGATE(MAIL_RECENT);

	if (lseq == 0) {
		if (!mail_index_lookup_seq(mbox->delayed_sync_view,
					   ctx->uid, &lseq)) {
			/* already expunged by another session */
			if (rseq == mbox->sync_next_rseq)
				mbox->sync_next_rseq++;
			return new_message;
		}
	}

	if (rseq == mbox->sync_next_rseq) {
		/* we're doing the initial full sync of mails. expunge any
		   mails that no longer exist. */
		while (mbox->sync_next_lseq < lseq) {
			mail_index_lookup_uid(mbox->delayed_sync_view,
					      mbox->sync_next_lseq, &ctx->uid);
			imapc_mailbox_index_expunge(mbox, ctx->uid);
			mbox->sync_next_lseq++;
		}
		i_assert(lseq == mbox->sync_next_lseq);
		mbox->sync_next_rseq++;
		mbox->sync_next_lseq++;
	}

	if (!new_message) {
		/* Only update flags immediately for existing messages */
		imapc_untagged_fetch_update_flags(mbox, ctx,
						  mbox->delayed_sync_view, lseq);
	}

	if (ctx->modseq != 0) {
		if (mail_index_modseq_lookup(mbox->delayed_sync_view, lseq) < ctx->modseq)
			mail_index_update_modseq(mbox->delayed_sync_trans, lseq, ctx->modseq);
		array_idx_set(&mbox->rseq_modseqs, rseq-1, &ctx->modseq);
	}
	if (ctx->guid != NULL) {
		struct index_mailbox_context *ibox = INDEX_STORAGE_CONTEXT(&mbox->box);
		const enum index_cache_field guid_cache_idx =
			ibox->cache_fields[MAIL_CACHE_GUID].idx;

		if (mail_cache_field_can_add(mbox->delayed_sync_cache_trans,
					     lseq, guid_cache_idx)) {
			mail_cache_add(mbox->delayed_sync_cache_trans, lseq,
				       guid_cache_idx, ctx->guid, strlen(ctx->guid));
		}
	}
	return new_message;
}

static bool imapc_untagged_fetch_parse(struct imapc_mailbox *mbox,
				       struct imapc_untagged_fetch_ctx *ctx,
				       const struct imap_arg *list)
{
	const struct imap_arg *flags_list, *modseq_list;
	const char *atom, *patom;
	unsigned int i, j;

	ctx->fetch_uid = 0; ctx->flags = 0;
	for (i = 0; list[i].type != IMAP_ARG_EOL; i += 2) {
		if (!imap_arg_get_atom(&list[i], &atom) ||
		    list[i+1].type == IMAP_ARG_EOL)
			return FALSE;

		if (strcasecmp(atom, "UID") == 0) {
			if (!imap_arg_get_atom(&list[i+1], &atom) ||
			    str_to_uint32(atom, &ctx->fetch_uid) < 0)
				return FALSE;
		} else if (strcasecmp(atom, "FLAGS") == 0) {
			if (!imap_arg_get_list(&list[i+1], &flags_list))
				return FALSE;

			p_array_init(&ctx->keywords, ctx->pool, 8);
			ctx->have_flags = TRUE;
			for (j = 0; flags_list[j].type != IMAP_ARG_EOL; j++) {
				if (!imap_arg_get_atom(&flags_list[j], &atom))
					return FALSE;
				if (atom[0] == '\\')
					ctx->flags |= imap_parse_system_flag(atom);
				else {
					patom = p_strdup(ctx->pool, atom);
					/* keyword */
					array_push_back(&ctx->keywords, &patom);
				}
			}
		} else if (strcasecmp(atom, "MODSEQ") == 0 &&
			   imapc_mailbox_has_modseqs(mbox)) {
			/* (modseq-number) */
			if (!imap_arg_get_list(&list[i+1], &modseq_list))
				return FALSE;
			if (!imap_arg_get_atom(&modseq_list[0], &atom) ||
			    str_to_uint64(atom, &ctx->modseq) < 0 ||
			    modseq_list[1].type != IMAP_ARG_EOL)
				return FALSE;
		} else if (strcasecmp(atom, "X-GM-MSGID") == 0 &&
			   !mbox->initial_sync_done) {
			if (imap_arg_get_atom(&list[i+1], &atom))
				ctx->guid = atom;
		} else if (strcasecmp(atom, "X-GM-LABELS") == 0 &&
			   IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_GMAIL_MIGRATION)) {
			if (!imap_arg_get_list(&list[i+1], &flags_list))
				return FALSE;
			for (j = 0; flags_list[j].type != IMAP_ARG_EOL; j++) {
				if (!imap_arg_get_astring(&flags_list[j], &atom))
					return FALSE;
				if (strcasecmp(atom, "\\Muted") != 0)
					ctx->have_gmail_labels = TRUE;
			}
		}
	}
	if (ctx->fetch_uid == 0 &&
	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES)) {
		/* UID missing and we're not tracking MSNs */
		return FALSE;
	}

	return TRUE;
}

static void imapc_untagged_fetch(const struct imapc_untagged_reply *reply,
				 struct imapc_mailbox *mbox)
{
	const struct imap_arg *list;
	struct imapc_fetch_request *fetch_request;
	struct imapc_mail *mail;
	bool new_message = FALSE;

	if (mbox == NULL || reply->num == 0 || !imap_arg_get_list(reply->args, &list))
		return;

	struct imapc_untagged_fetch_ctx *ctx =
		imapc_untagged_fetch_ctx_create();
	if (!imapc_untagged_fetch_parse(mbox, ctx, list)) {
		imapc_untagged_fetch_ctx_free(&ctx);
		return;
	}

	new_message = imapc_untagged_fetch_handle(mbox, ctx, reply->num);

	/* if this is a reply to some FETCH request, update the mail's fields */
	array_foreach_elem(&mbox->fetch_requests, fetch_request) {
		array_foreach_elem(&fetch_request->mails, mail) {
			if (mail->imail.mail.mail.uid == ctx->uid)
				imapc_mail_fetch_update(mail, reply, list);
		}
	}

	if (!new_message) {
		/* Handling this context is finished if the mail was not new
		   to the local index. It has not been added to
		   mbox->untagged_fetch_contexts so no need to delete it from
		   the array. The context itself can be freed here. */
		imapc_untagged_fetch_ctx_free(&ctx);
	} else {
		/* If this is a new message store this context to be handled
		   when syncing */
		array_push_back(&mbox->untagged_fetch_contexts, &ctx);
	}
	imapc_mailbox_idle_notify(mbox);
}

static void imapc_untagged_expunge(const struct imapc_untagged_reply *reply,
				   struct imapc_mailbox *mbox)
{
	struct imapc_msgmap *msgmap;
	uint32_t uid, rseq = reply->num;

	if (mbox == NULL || rseq == 0 ||
	    IMAPC_BOX_HAS_FEATURE(mbox, IMAPC_FEATURE_NO_MSN_UPDATES))
		return;

	mbox->prev_skipped_rseq = 0;
	mbox->prev_skipped_uid = 0;

	if (mbox->exists_count == 0) {
		imapc_mailbox_set_corrupted(mbox,
			"EXPUNGE received for empty mailbox");
		return;
	}
	mbox->exists_count--;

	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
	if (rseq > imapc_msgmap_count(msgmap)) {
		/* we haven't even seen this message yet */
		return;
	}
	uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
	imapc_msgmap_expunge(msgmap, rseq);
	if (array_is_created(&mbox->rseq_modseqs))
		array_delete(&mbox->rseq_modseqs, rseq-1, 1);

	imapc_mailbox_init_delayed_trans(mbox);
	imapc_mailbox_index_expunge(mbox, uid);
	imapc_mailbox_idle_notify(mbox);
}

static void
imapc_untagged_esearch_gmail_pop3(const struct imap_arg *args,
				  struct imapc_mailbox *mbox)
{
	struct imapc_msgmap *msgmap;
	const char *atom;
	struct seq_range_iter iter;
	ARRAY_TYPE(seq_range) rseqs;
	unsigned int n;
	uint32_t rseq, lseq, uid;
	ARRAY_TYPE(keyword_indexes) keywords;
	struct mail_keywords *kw;
	unsigned int pop3_deleted_kw_idx;

	i_free_and_null(mbox->sync_gmail_pop3_search_tag);

	/* It should contain ALL <seqset> or nonexistent if nothing matched */
	if (args[0].type == IMAP_ARG_EOL)
		return;
	t_array_init(&rseqs, 64);
	if (!imap_arg_atom_equals(&args[0], "ALL") ||
	    !imap_arg_get_atom(&args[1], &atom) ||
	    imap_seq_set_nostar_parse(atom, &rseqs) < 0) {
		i_error("Invalid gmail-pop3 ESEARCH reply");
		return;
	}

	mail_index_keyword_lookup_or_create(mbox->box.index,
		mbox->storage->set->pop3_deleted_flag, &pop3_deleted_kw_idx);

	t_array_init(&keywords, 1);
	array_push_back(&keywords, &pop3_deleted_kw_idx);
	kw = mail_index_keywords_create_from_indexes(mbox->box.index, &keywords);

	msgmap = imapc_client_mailbox_get_msgmap(mbox->client_box);
	seq_range_array_iter_init(&iter, &rseqs); n = 0;
	while (seq_range_array_iter_nth(&iter, n++, &rseq)) {
		if (rseq > imapc_msgmap_count(msgmap)) {
			/* we haven't even seen this message yet */
			break;
		}
		uid = imapc_msgmap_rseq_to_uid(msgmap, rseq);
		if (!mail_index_lookup_seq(mbox->delayed_sync_view,
					   uid, &lseq))
			continue;

		/* add the pop3_deleted_flag */
		mail_index_update_keywords(mbox->delayed_sync_trans,
					   lseq, MODIFY_ADD, kw);
	}
	mail_index_keywords_unref(&kw);
}

static void imapc_untagged_search(const struct imapc_untagged_reply *reply,
				  struct imapc_mailbox *mbox)
{
	if (mbox != NULL)
		imapc_search_reply_search(reply->args, mbox);
}

static void imapc_untagged_esearch(const struct imapc_untagged_reply *reply,
				   struct imapc_mailbox *mbox)
{
	const struct imap_arg *tag_list;
	const char *str;

	if (mbox == NULL || !imap_arg_get_list(reply->args, &tag_list))
		return;

	/* ESEARCH begins with (TAG <tag>) */
	if (!imap_arg_atom_equals(&tag_list[0], "TAG") ||
	    !imap_arg_get_string(&tag_list[1], &str) ||
	    tag_list[2].type != IMAP_ARG_EOL)
		return;

	/* for now the only ESEARCH reply that we have is for getting GMail's
	   list of hidden POP3 messages. */
	if (mbox->sync_gmail_pop3_search_tag != NULL &&
	    strcmp(mbox->sync_gmail_pop3_search_tag, str) == 0)
		imapc_untagged_esearch_gmail_pop3(reply->args+1, mbox);
	else
		imapc_search_reply_esearch(reply->args+1, mbox);
}

static void imapc_sync_uid_validity(struct imapc_mailbox *mbox)
{
	const struct mail_index_header *hdr;

	imapc_mailbox_init_delayed_trans(mbox);
	hdr = mail_index_get_header(mbox->delayed_sync_view);
	if (hdr->uid_validity != mbox->sync_uid_validity &&
	    mbox->sync_uid_validity != 0) {
		if (hdr->uid_validity != 0) {
			/* uidvalidity changed, reset the entire mailbox */
			mail_index_reset(mbox->delayed_sync_trans);
			mbox->sync_fetch_first_uid = 1;
			/* The reset needs to be committed before FETCH 1:*
			   results are received. */
			bool changes;
			if (imapc_mailbox_commit_delayed_trans(mbox, TRUE, &changes) < 0)
				mail_index_mark_corrupted(mbox->box.index);
			imapc_mailbox_init_delayed_trans(mbox);
		}
		mail_index_update_header(mbox->delayed_sync_trans,
			offsetof(struct mail_index_header, uid_validity),
			&mbox->sync_uid_validity,
			sizeof(mbox->sync_uid_validity), TRUE);
	}
}

static void
imapc_resp_text_uidvalidity(const struct imapc_untagged_reply *reply,
			    struct imapc_mailbox *mbox)
{
	uint32_t uid_validity;

	if (mbox == NULL ||
	    str_to_uint32(reply->resp_text_value, &uid_validity) < 0 ||
	    uid_validity == 0)
		return;

	if (mbox->sync_uid_validity != uid_validity) {
		mbox->sync_uid_validity = uid_validity;
		imapc_mail_cache_free(&mbox->prev_mail_cache);
		imapc_sync_uid_validity(mbox);
	}
}

static void
imapc_resp_text_uidnext(const struct imapc_untagged_reply *reply,
			struct imapc_mailbox *mbox)
{
	uint32_t uid_next;

	if (mbox == NULL ||
	    str_to_uint32(reply->resp_text_value, &uid_next) < 0)
		return;

	mbox->sync_uid_next = uid_next;
}

static void
imapc_resp_text_highestmodseq(const struct imapc_untagged_reply *reply,
			      struct imapc_mailbox *mbox)
{
	uint64_t highestmodseq;

	if (mbox == NULL ||
	    str_to_uint64(reply->resp_text_value, &highestmodseq) < 0)
		return;

	mbox->sync_highestmodseq = highestmodseq;
}

static void
imapc_resp_text_permanentflags(const struct imapc_untagged_reply *reply,
			       struct imapc_mailbox *mbox)
{
	const struct imap_arg *flags_args, *arg;
	const char *flag;
	unsigned int idx;

	i_assert(reply->args[0].type == IMAP_ARG_ATOM);

	if (mbox == NULL || !imap_arg_get_list(&reply->args[1], &flags_args))
		return;

	mbox->permanent_flags = 0;
	mbox->box.disallow_new_keywords = TRUE;

	for (arg = flags_args; arg->type != IMAP_ARG_EOL; arg++) {
		if (!imap_arg_get_atom(arg, &flag))
			continue;

		if (strcmp(flag, "\\*") == 0)
			mbox->box.disallow_new_keywords = FALSE;
		else if (*flag == '\\')
			mbox->permanent_flags |= imap_parse_system_flag(flag);
		else {
			/* we'll simply make sure that it exists in the index */
			mail_index_keyword_lookup_or_create(mbox->box.index,
							    flag, &idx);
		}
	}
}

void imapc_mailbox_register_untagged(struct imapc_mailbox *mbox,
				     const char *key,
				     imapc_mailbox_callback_t *callback)
{
	struct imapc_mailbox_event_callback *cb;

	cb = array_append_space(&mbox->untagged_callbacks);
	cb->name = p_strdup(mbox->box.pool, key);
	cb->callback = callback;
}

void imapc_mailbox_register_resp_text(struct imapc_mailbox *mbox,
				      const char *key,
				      imapc_mailbox_callback_t *callback)
{
	struct imapc_mailbox_event_callback *cb;

	cb = array_append_space(&mbox->resp_text_callbacks);
	cb->name = p_strdup(mbox->box.pool, key);
	cb->callback = callback;
}

void imapc_mailbox_register_callbacks(struct imapc_mailbox *mbox)
{
	imapc_mailbox_register_untagged(mbox, "EXISTS",
					imapc_untagged_exists);
	imapc_mailbox_register_untagged(mbox, "FETCH",
					imapc_untagged_fetch);
	imapc_mailbox_register_untagged(mbox, "EXPUNGE",
					imapc_untagged_expunge);
	imapc_mailbox_register_untagged(mbox, "SEARCH",
					imapc_untagged_search);
	imapc_mailbox_register_untagged(mbox, "ESEARCH",
					imapc_untagged_esearch);
	imapc_mailbox_register_resp_text(mbox, "UIDVALIDITY",
					 imapc_resp_text_uidvalidity);
	imapc_mailbox_register_resp_text(mbox, "UIDNEXT",
					 imapc_resp_text_uidnext);
	imapc_mailbox_register_resp_text(mbox, "HIGHESTMODSEQ",
					 imapc_resp_text_highestmodseq);
	imapc_mailbox_register_resp_text(mbox, "PERMANENTFLAGS",
					 imapc_resp_text_permanentflags);
}