path: root/src/lib-index/test-mail-transaction-log-file.c

/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "ioloop.h"
#include "test-common.h"
#include "mail-index-private.h"
#include "mail-transaction-log-private.h"

#define TEST_LOG_VERSION MAIL_TRANSACTION_LOG_VERSION_FULL(1, 3)

#define INITIAL_MODSEQ 100

struct update_modseq_test {
	enum mail_transaction_type type;
	unsigned int version;
#define NOUPDATE (INITIAL_MODSEQ)
#define UPDATE (INITIAL_MODSEQ+1)
	uint64_t expected_modseq;
	unsigned int count;
	union {
		const struct mail_transaction_flag_update *flag_update;
		const struct mail_transaction_modseq_update *modseq_update;
	} v;
} update_modseq_tests[] = {
	/* expunges: increase modseq */
	{ MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, UPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, UPDATE, 1, { } },
	/* expunges: don't increase modseq */
	{ MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXTERNAL, TEST_LOG_VERSION, NOUPDATE, 1, { } },

	/* flag changes: don't increase modseq */
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = 0 }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_BACKEND }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_INDEX_MAIL_FLAG_BACKEND }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_INDEX_MAIL_FLAG_DIRTY }
		}
	} },
	/* flag changes: increase modseq */
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_SEEN }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_SEEN }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_BACKEND }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_DIRTY }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_BACKEND }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .remove_flags = MAIL_SEEN | MAIL_INDEX_MAIL_FLAG_DIRTY }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 2, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY },
			{ .uid1 = 3, .uid2 = 4, .add_flags = MAIL_SEEN }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = 0, .modseq_inc_flag = 1 }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, TEST_LOG_VERSION, UPDATE, 2, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY },
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY, .modseq_inc_flag = 1 }
		}
	} },
	/* flag changes: increase modseq with old version */
	{ MAIL_TRANSACTION_FLAG_UPDATE, MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2), UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_BACKEND }
		}
	} },
	{ MAIL_TRANSACTION_FLAG_UPDATE, MAIL_TRANSACTION_LOG_VERSION_FULL(1, 2), UPDATE, 1, {
		.flag_update = (const struct mail_transaction_flag_update[]) {
			{ .uid1 = 1, .uid2 = 2, .add_flags = MAIL_INDEX_MAIL_FLAG_DIRTY }
		}
	} },
	/* modseq updates: don't increase modseq */
	{ MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.modseq_update = (const struct mail_transaction_modseq_update[]) {
			{ .uid = 1, .modseq_low32 = 50, .modseq_high32 = 0 }
		}
	} },
	{ MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, {
		.modseq_update = (const struct mail_transaction_modseq_update[]) {
			{ .uid = 1, .modseq_low32 = 100, .modseq_high32 = 0 }
		}
	} },
	/* modseq updates: increase modseq */
	{ MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, 500, 1, {
		.modseq_update = (const struct mail_transaction_modseq_update[]) {
			{ .uid = 1, .modseq_low32 = 500, .modseq_high32 = 0 }
		}
	} },
	{ MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, 500, 2, {
		.modseq_update = (const struct mail_transaction_modseq_update[]) {
			{ .uid = 1, .modseq_low32 = 50, .modseq_high32 = 0 },
			{ .uid = 1, .modseq_low32 = 500, .modseq_high32 = 0 }
		}
	} },
	{ MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, 500, 1, {
		.modseq_update = (const struct mail_transaction_modseq_update[]) {
			{ .uid = 1, .modseq_low32 = 500, .modseq_high32 = 0 },
			{ .uid = 1, .modseq_low32 = 200, .modseq_high32 = 0 }
		}
	} },
	{ MAIL_TRANSACTION_MODSEQ_UPDATE, TEST_LOG_VERSION, (uint64_t)4294967346, 1, {
		.modseq_update = (const struct mail_transaction_modseq_update[]) {
			{ .uid = 1, .modseq_low32 = 50, .modseq_high32 = 1 }
		}
	} },

	/* appends, keyword changes, attribute changes: increase modseq */
	{ MAIL_TRANSACTION_APPEND, TEST_LOG_VERSION, UPDATE, 1, { } },
	{ MAIL_TRANSACTION_KEYWORD_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { } },
	{ MAIL_TRANSACTION_KEYWORD_RESET, TEST_LOG_VERSION, UPDATE, 1, { } },
	{ MAIL_TRANSACTION_ATTRIBUTE_UPDATE, TEST_LOG_VERSION, UPDATE, 1, { } },

	/* others: don't increase modseq */
	{ MAIL_TRANSACTION_HEADER_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_HEADER_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXT_INTRO, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXT_RESET, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXT_HDR_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXT_REC_UPDATE, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXT_ATOMIC_INC, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_EXT_HDR_UPDATE32, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_INDEX_DELETED, TEST_LOG_VERSION, NOUPDATE, 1, { } },
	{ MAIL_TRANSACTION_INDEX_UNDELETED, TEST_LOG_VERSION, NOUPDATE, 1, { } },
};

static size_t update_modseq_test_get_size(const struct update_modseq_test *test)
{
	enum mail_transaction_type type =
		test->type & MAIL_TRANSACTION_TYPE_MASK;

	if (type == (MAIL_TRANSACTION_EXPUNGE | MAIL_TRANSACTION_EXPUNGE_PROT))
		type = MAIL_TRANSACTION_EXPUNGE;
	if (type == (MAIL_TRANSACTION_EXPUNGE_GUID | MAIL_TRANSACTION_EXPUNGE_PROT))
		type = MAIL_TRANSACTION_EXPUNGE_GUID;

	switch (type) {
	case MAIL_TRANSACTION_EXPUNGE:
		return sizeof(struct mail_transaction_expunge);
	case MAIL_TRANSACTION_EXPUNGE_GUID:
		return sizeof(struct mail_transaction_expunge_guid);
	case MAIL_TRANSACTION_APPEND:
		return sizeof(struct mail_index_record);
	case MAIL_TRANSACTION_KEYWORD_UPDATE:
		return sizeof(struct mail_transaction_keyword_update);
	case MAIL_TRANSACTION_KEYWORD_RESET:
		return sizeof(struct mail_transaction_keyword_reset);
	case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
		return 4;
	case MAIL_TRANSACTION_FLAG_UPDATE:
		return sizeof(struct mail_transaction_flag_update);
	case MAIL_TRANSACTION_MODSEQ_UPDATE:
		return sizeof(struct mail_transaction_modseq_update);
	case MAIL_TRANSACTION_HEADER_UPDATE:
	case MAIL_TRANSACTION_EXT_INTRO:
	case MAIL_TRANSACTION_EXT_RESET:
	case MAIL_TRANSACTION_EXT_HDR_UPDATE:
	case MAIL_TRANSACTION_EXT_REC_UPDATE:
	case MAIL_TRANSACTION_EXT_ATOMIC_INC:
	case MAIL_TRANSACTION_EXT_HDR_UPDATE32:
	case MAIL_TRANSACTION_INDEX_DELETED:
	case MAIL_TRANSACTION_INDEX_UNDELETED:
		return 4;
	case MAIL_TRANSACTION_TYPE_MASK:
	case MAIL_TRANSACTION_BOUNDARY:
	case MAIL_TRANSACTION_EXPUNGE_PROT:
	case MAIL_TRANSACTION_EXTERNAL:
	case MAIL_TRANSACTION_SYNC:
		break;
	}
	i_unreached();
}

static void test_mail_transaction_update_modseq(void)
{
	struct mail_transaction_header hdr;
	unsigned char tempbuf[1024] = { 0 };

	test_begin("mail_transaction_update_modseq()");
	for (unsigned int i = 0; i < N_ELEMENTS(update_modseq_tests); i++) {
		const struct update_modseq_test *test = &update_modseq_tests[i];
		const void *data = test->v.flag_update;
		uint64_t cur_modseq = INITIAL_MODSEQ;

		if (data == NULL)
			data = tempbuf;

		hdr.type = test->type;
		hdr.size = sizeof(hdr) + update_modseq_test_get_size(test) * test->count;
		hdr.size = mail_index_uint32_to_offset(hdr.size);
		mail_transaction_update_modseq(&hdr, data, &cur_modseq, test->version);
		test_assert_idx(cur_modseq >= INITIAL_MODSEQ, i);
		test_assert_idx(test->expected_modseq == cur_modseq, i);
	}
	test_end();
}

static struct mail_index *test_mail_index_open(void)
{
	struct mail_index *index = mail_index_alloc(NULL, NULL, "test.dovecot.index");
	test_assert(mail_index_open_or_create(index, MAIL_INDEX_OPEN_FLAG_CREATE) == 0);
	struct mail_index_view *view = mail_index_view_open(index);

	struct mail_index_transaction *trans =
		mail_index_transaction_begin(view, 0);
	uint32_t uid_validity = 1234;
	mail_index_update_header(trans,
		offsetof(struct mail_index_header, uid_validity),
		&uid_validity, sizeof(uid_validity), TRUE);
	test_assert(mail_index_transaction_commit(&trans) == 0);
	mail_index_view_close(&view);
	return index;
}

static void test_mail_transaction_log_file_modseq_offsets(void)
{
	test_begin("mail_transaction_log_file_get_modseq_next_offset() and _get_highest_modseq_at()");

	struct mail_index *index = test_mail_index_open();
	struct mail_transaction_log_file *file = index->log->head;

	const unsigned int max_modseq = LOG_FILE_MODSEQ_CACHE_SIZE+2;
	uoff_t modseq_next_offset[max_modseq+1];
	uoff_t modseq_alt_next_offset[max_modseq+1];

	/* start with modseq=2, because modseq=1 is the initial state */
	modseq_next_offset[1] = sizeof(struct mail_transaction_log_header);
	modseq_alt_next_offset[1] = sizeof(struct mail_transaction_log_header);
	for (uint64_t modseq = 2; modseq <= max_modseq; modseq++) {
		uint32_t seq;

		struct mail_index_view *view = mail_index_view_open(index);
		struct mail_index_transaction *trans =
			mail_index_transaction_begin(view, 0);
		mail_index_append(trans, modseq, &seq);
		test_assert(mail_index_transaction_commit(&trans) == 0);
		modseq_next_offset[modseq] = file->sync_offset;
		mail_index_view_close(&view);

		/* add a non-modseq updating change */
		view = mail_index_view_open(index);
		trans = mail_index_transaction_begin(view, 0);
		mail_index_update_flags(trans, seq, MODIFY_ADD,
			(enum mail_flags)MAIL_INDEX_MAIL_FLAG_DIRTY);
		test_assert(mail_index_transaction_commit(&trans) == 0);
		mail_index_view_close(&view);
		modseq_alt_next_offset[modseq] = file->sync_offset;
	}

	/* mail_transaction_log_file_get_highest_modseq_at() is simultaneously
	   tested and it can also add offsets to cache. The difference is that
	   it adds the highest possible offset, while
	   mail_transaction_log_file_get_modseq_next_offset() adds the lowest
	   possible offset. So we'll need to allow both. */
#define MODSEQ_MATCH(modseq, next_offset) \
	((next_offset) == modseq_next_offset[modseq] || \
	 (next_offset) == modseq_alt_next_offset[modseq])

	/* 1) mail_transaction_log_file_get_modseq_next_offset() tests */
	uint64_t modseq;
	uoff_t next_offset;
	/* initial_modseq fast path */
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 1, &next_offset) == 0);
	test_assert(next_offset == modseq_next_offset[1]);
	/* sync_highest_modseq fast path - it skips to sync_offset instead of
	   using exactly the same max_modseq */
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, max_modseq, &next_offset) == 0);
	test_assert(next_offset == file->sync_offset);
	test_assert(next_offset != modseq_next_offset[max_modseq]);
	/* update the offset for the random tests */
	modseq_next_offset[max_modseq] = file->sync_offset;
	/* add to cache */
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0);
	test_assert(MODSEQ_MATCH(2, next_offset));
	/* get it from cache */
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0);
	test_assert(MODSEQ_MATCH(2, next_offset));
	/* get next value from cache */
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 3, &next_offset) == 0);
	test_assert(MODSEQ_MATCH(3, next_offset));
	/* get previous value from cache again */
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0);
	test_assert(MODSEQ_MATCH(2, next_offset));
	/* do some random testing with cache */
	for (unsigned int i = 0; i < LOG_FILE_MODSEQ_CACHE_SIZE*10; i++) {
		modseq = i_rand_minmax(1, max_modseq);
		test_assert(mail_transaction_log_file_get_modseq_next_offset(file, modseq, &next_offset) == 0);
		test_assert(MODSEQ_MATCH(modseq, next_offset));
	}
	/* go through all modseqs - do this after randomness testing or
	   modseq_alt_next_offset[] matching isn't triggered */
	for (modseq = 1; modseq <= max_modseq; modseq++) {
		test_assert(mail_transaction_log_file_get_modseq_next_offset(file, modseq, &next_offset) == 0);
		test_assert(MODSEQ_MATCH(modseq, next_offset));
	}

	/* 2) mail_transaction_log_file_get_highest_modseq_at() tests */
	uint64_t modseq_at;
	const char *error;
	/* initial_offset */
	test_assert(mail_transaction_log_file_get_highest_modseq_at(
		file, modseq_next_offset[1], &modseq, &error) == 1);
	test_assert(modseq == 1);
	/* sync_offset fast path */
	test_assert(mail_transaction_log_file_get_highest_modseq_at(
		file, file->sync_offset, &modseq, &error) == 1);
	test_assert(modseq == max_modseq);
	/* do some random testing with cache */
	for (unsigned int i = 0; i < LOG_FILE_MODSEQ_CACHE_SIZE*10; i++) {
		modseq = i_rand_minmax(1, max_modseq);
		test_assert(mail_transaction_log_file_get_highest_modseq_at(
			file, modseq_next_offset[modseq], &modseq_at, &error) == 1);
		test_assert(modseq_at == modseq);
		test_assert(mail_transaction_log_file_get_highest_modseq_at(
			file, modseq_alt_next_offset[modseq], &modseq_at, &error) == 1);
		test_assert(modseq_at == modseq);
	}
	/* go through all modseqs - do this after randomness testing or
	   modseq_alt_next_offset[] matching isn't triggered */
	for (modseq = 1; modseq <= max_modseq; modseq++) {
		test_assert(mail_transaction_log_file_get_highest_modseq_at(
			file, modseq_next_offset[modseq], &modseq_at, &error) == 1);
		test_assert(modseq_at == modseq);
	}

	mail_index_close(index);
	mail_index_free(&index);
	test_end();
}

static void
test_mail_transaction_log_file_get_modseq_next_offset_inconsistency(void)
{
	test_begin("mail_transaction_log_file_get_modseq_next_offset() inconsistency");

	struct mail_index *index = test_mail_index_open();
	struct mail_transaction_log_file *file = index->log->head;
	uint32_t seq;

	/* add modseq=2 */
	struct mail_index_view *view = mail_index_view_open(index);
	struct mail_index_transaction *trans =
		mail_index_transaction_begin(view, 0);
	mail_index_append(trans, 1, &seq);
	test_assert(mail_index_transaction_commit(&trans) == 0);
	mail_index_view_close(&view);

	/* emulate a broken mail_index_modseq_header header */
	file->sync_highest_modseq = 3;

	uoff_t next_offset;
	test_expect_error_string("Transaction log modseq tracking is corrupted");
	test_assert(mail_transaction_log_file_get_modseq_next_offset(file, 2, &next_offset) == 0);
	test_expect_no_more_errors();
	test_assert(next_offset == file->sync_offset);

	mail_index_close(index);
	mail_index_free(&index);
	test_end();
}

int main(void)
{
	static void (*const test_functions[])(void) = {
		test_mail_transaction_update_modseq,
		test_mail_transaction_log_file_modseq_offsets,
		test_mail_transaction_log_file_get_modseq_next_offset_inconsistency,
		NULL
	};
	ioloop_time = 1;
	return test_run(test_functions);
}