/* Copyright (c) 2020 Dovecot authors, see the included COPYING file */

#include "lib.h"
#include "str.h"
#include "array.h"
#include "test-common.h"
#include "test-mail-cache.h"

#include <stdio.h>
#include <sys/wait.h>

static void test_mail_cache_read_during_purge2(void)
{
	struct test_mail_cache_ctx ctx;
	struct mail_cache_view *cache_view;
	string_t *str = t_str_new(16);

	i_set_failure_prefix("index2: ");

	/* read from cache via 2nd index */
	test_mail_cache_init(test_mail_index_open(), &ctx);

	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	test_assert(mail_cache_lookup_field(cache_view, str, 1,
					    ctx.cache_field.idx) == 1);
	test_assert(strcmp(str_c(str), "foo1") == 0);
	mail_cache_view_close(&cache_view);

	test_mail_cache_deinit(&ctx);
}

static void test_mail_cache_read_during_purge(void)
{
	struct test_mail_cache_ctx ctx;
	struct mail_index_transaction *trans;
	int status;

	test_begin("mail cache read during purge");
	test_mail_cache_init(test_mail_index_init(), &ctx);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");

	/* lock the index for cache purge */
	uint32_t log_seq;
	uoff_t log_offset;
	test_assert(mail_transaction_log_sync_lock(ctx.index->log, "purge", &log_seq, &log_offset) == 0);

	/* start purging cache using the 1st index, but don't commit yet */
	trans = mail_index_transaction_begin(ctx.view, 0);
	test_assert(mail_cache_purge_with_trans(ctx.cache, trans, (uint32_t)-1, "test") == 0);

	switch (fork()) {
	case (pid_t)-1:
		i_fatal("fork() failed: %m");
	case 0:
		test_mail_cache_read_during_purge2();
		/* cleanup so valgrind doesn't complain about memory leaks */
		mail_index_transaction_rollback(&trans);
		mail_transaction_log_sync_unlock(ctx.index->log, "purge");
		test_mail_cache_deinit(&ctx);
		test_exit(test_has_failed() ? 10 : 0);
	default:
		break;
	}

	/* Wait a bit to make sure the child function has had a chance to run.
	   It's supposed to be waiting on the locked .log file. */
	usleep(100000);
	/* finish cache purging */
	test_assert(mail_index_transaction_commit(&trans) == 0);
	mail_transaction_log_sync_unlock(ctx.index->log, "purge");
	mail_index_view_close(&ctx.view);

	/* wait for child to finish execution */
	if (wait(&status) == -1)
		i_error("wait() failed: %m");
	test_assert(status == 0);

	test_assert(test_mail_cache_get_purge_count(&ctx) == 1);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
	test_end();
}

static void test_mail_cache_write_during_purge2(void)
{
	struct test_mail_cache_ctx ctx;

	i_set_failure_prefix("index2: ");

	/* add to cache via 2nd index */
	test_mail_cache_init(test_mail_index_open(), &ctx);
	test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar2");
	test_mail_cache_deinit(&ctx);
}

static void test_mail_cache_write_during_purge(void)
{
	struct test_mail_cache_ctx ctx;
	struct mail_index_view *view;
	struct mail_cache_view *cache_view;
	struct mail_index_transaction *trans;
	string_t *str = t_str_new(16);
	int status;

	test_begin("mail cache write during purge");
	test_mail_cache_init(test_mail_index_init(), &ctx);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");

	/* lock the index for cache purge */
	uint32_t log_seq;
	uoff_t log_offset;
	test_assert(mail_transaction_log_sync_lock(ctx.index->log, "purge", &log_seq, &log_offset) == 0);

	/* start purging cache using the 1st index, but don't commit yet */
	trans = mail_index_transaction_begin(ctx.view, 0);
	test_assert(mail_cache_purge_with_trans(ctx.cache, trans, (uint32_t)-1, "test") == 0);

	switch (fork()) {
	case (pid_t)-1:
		i_fatal("fork() failed: %m");
	case 0:
		test_mail_cache_write_during_purge2();
		/* cleanup so valgrind doesn't complain about memory leaks */
		mail_index_transaction_rollback(&trans);
		mail_transaction_log_sync_unlock(ctx.index->log, "purge");
		test_mail_cache_deinit(&ctx);
		test_exit(test_has_failed() ? 10 : 0);
	default:
		break;
	}

	/* Wait a bit to make sure the child function has had a chance to run.
	   It's supposed to be waiting on the locked .log file. */
	usleep(100000);
	/* finish cache purge */
	test_assert(mail_index_transaction_commit(&trans) == 0);
	mail_transaction_log_sync_unlock(ctx.index->log, "purge");
	mail_index_view_close(&ctx.view);

	/* wait for child to finish execution */
	if (wait(&status) == -1)
		i_error("wait() failed: %m");
	test_assert(status == 0);

	/* make sure both cache fields are visible */
	test_assert(mail_index_refresh(ctx.index) == 0);

	view = mail_index_view_open(ctx.index);
	cache_view = mail_cache_view_open(ctx.cache, view);
	test_assert(mail_cache_lookup_field(cache_view, str, 1,
					    ctx.cache_field.idx) == 1);
	test_assert(strcmp(str_c(str), "foo1") == 0);
	str_truncate(str, 0);
	test_assert(mail_cache_lookup_field(cache_view, str, 1,
					    ctx.cache_field2.idx) == 1);
	test_assert(strcmp(str_c(str), "bar2") == 0);
	mail_cache_view_close(&cache_view);
	mail_index_view_close(&view);

	test_assert(test_mail_cache_get_purge_count(&ctx) == 1);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
	test_end();
}

static void test_mail_cache_purge_while_cache_locked(void)
{
	struct test_mail_cache_ctx ctx;
	struct mail_cache_view *cache_view;
	string_t *str = t_str_new(16);
	int status;

	test_begin("mail cache purge while cache locked");
	test_mail_cache_init(test_mail_index_init(), &ctx);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");

	/* lock the cache */
	test_assert(mail_cache_lock(ctx.cache) == 1);

	/* purge the cache in another process */
	switch (fork()) {
	case (pid_t)-1:
		i_fatal("fork() failed: %m");
	case 0:
		test_mail_cache_purge();
		test_mail_cache_deinit(&ctx);
		test_exit(test_has_failed() ? 10 : 0);
	default:
		break;
	}

	/* Wait a bit to make sure the child function has had a chance to run.
	   It should start purging, which would wait for our cache lock. */
	usleep(100000);

	mail_cache_unlock(ctx.cache);

	/* wait for child to finish execution */
	if (wait(&status) == -1)
		i_error("wait() failed: %m");
	test_assert(status == 0);

	/* make sure the cache is still usable */
	test_assert(mail_index_refresh(ctx.index) == 0);
	test_mail_cache_view_sync(&ctx);

	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	test_assert(mail_cache_lookup_field(cache_view, str, 1,
					    ctx.cache_field.idx) == 1);
	test_assert(strcmp(str_c(str), "foo1") == 0);
	mail_cache_view_close(&cache_view);

	test_assert(test_mail_cache_get_purge_count(&ctx) == 1);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
	test_end();
}

static bool cache_equals(struct mail_cache_view *cache_view, uint32_t seq,
			 unsigned int field_idx, const char *value)
{
	string_t *str = str_new(default_pool, 128);
	int ret = mail_cache_lookup_field(cache_view, str, seq, field_idx);
	bool match;

	if (value != NULL) {
		test_assert_idx(ret == 1, seq);
		match = strcmp(str_c(str), value) == 0;
		test_assert_idx(match, seq);
	} else {
		test_assert_idx(ret == 0, seq);
		match = ret == 0;
	}
	str_free(&str);
	return match;
}

static void test_mail_cache_purge_during_write_n(unsigned int num_mails,
						 bool commit_saves)
{
	const struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.record_max_size = 1024*1024,
		},
	};
	struct test_mail_cache_ctx ctx;
	struct mail_index_view *updated_view;
	struct mail_cache_view *cache_view;
	struct mail_cache_transaction_ctx *cache_trans;
	struct mail_index_transaction *trans;
	uint32_t seq;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);

	/* Add mails */
	test_mail_cache_add_mail(&ctx, UINT_MAX, "");
	trans = mail_index_transaction_begin(ctx.view, 0);
	for (seq = 2; seq <= num_mails; seq++)
		mail_index_append(trans, seq, &seq);

	if (commit_saves) {
		test_assert(mail_index_transaction_commit(&trans) == 0);
		test_mail_cache_view_sync(&ctx);
		trans = mail_index_transaction_begin(ctx.view, 0);
	}
	/* start adding a small cached field to mail1 */
	updated_view = mail_index_transaction_open_updated_view(trans);
	cache_view = mail_cache_view_open(ctx.cache, updated_view);
	cache_trans = mail_cache_get_transaction(cache_view, trans);
	mail_cache_add(cache_trans, 1, ctx.cache_field.idx, "foo1", 4);

	/* add a huge field to mail2, which triggers flushing */
	size_t huge_field_size = MAIL_CACHE_MAX_WRITE_BUFFER + 1024;
	char *huge_field = i_malloc(huge_field_size + 1);
	memset(huge_field, 'x', huge_field_size);
	mail_cache_add(cache_trans, 2, ctx.cache_field.idx,
		       huge_field, huge_field_size);

	/* verify that cached fields are still accessible */
	test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1"));
	test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field));

	/* purge using a 2nd index */
	test_mail_cache_purge();

	if (num_mails == 2) {
		/* the mails are still accessible after purge */
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1"));
		test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field));
	} else if (!commit_saves) {
		/* add 3rd mail, which attempts to flush 2nd mail and finds
		   that the first mail is already lost */
		test_expect_error_string("Purging lost 1 written cache records");
		mail_cache_add(cache_trans, 3, ctx.cache_field.idx, "foo3", 4);
		test_expect_no_more_errors();

		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL));
		test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field));
		test_assert(cache_equals(cache_view, 3, ctx.cache_field.idx, "foo3"));
	} else {
		/* add 3rd mail, which commits the first two mails */
		mail_cache_add(cache_trans, 3, ctx.cache_field.idx, "foo3", 4);
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1"));
		test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field));
		test_assert(cache_equals(cache_view, 3, ctx.cache_field.idx, "foo3"));
	}

	/* finish committing cached fields */
	if (num_mails == 2 && !commit_saves)
		test_expect_error_string("Purging lost 1 written cache records");
	test_assert(mail_index_transaction_commit(&trans) == 0);
	test_expect_no_more_errors();
	mail_index_view_close(&updated_view);
	mail_cache_view_close(&cache_view);

	/* see that we lost the first flush without commit_saves, but not the others */
	test_mail_cache_view_sync(&ctx);
	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	if (commit_saves)
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1"));
	else
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL));
	test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, huge_field));
	if (num_mails >= 3)
		test_assert(cache_equals(cache_view, 3, ctx.cache_field.idx, "foo3"));
	mail_cache_view_close(&cache_view);

	mail_index_view_close(&ctx.view);
	test_assert(test_mail_cache_get_purge_count(&ctx) == 1);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
	i_free(huge_field);
}

static void test_mail_cache_write_lost_during_purge(void)
{
	test_begin("mail cache write lost during purge");
	test_mail_cache_purge_during_write_n(2, FALSE);
	test_end();
}

static void test_mail_cache_write_lost_during_purge2(void)
{
	test_begin("mail cache write lost during purge (2)");
	test_mail_cache_purge_during_write_n(3, FALSE);
	test_end();
}

static void test_mail_cache_write_autocommit(void)
{
	test_begin("mail cache write autocommit");
	test_mail_cache_purge_during_write_n(2, TRUE);
	test_end();
}

static void test_mail_cache_write_autocommit2(void)
{
	test_begin("mail cache write autocommit");
	test_mail_cache_purge_during_write_n(3, TRUE);
	test_end();
}

static size_t max_field_size(size_t max_size, size_t current_size)
{
	return max_size - current_size
		- sizeof(struct mail_cache_record)
		- sizeof(uint32_t) /* field_idx */
		- sizeof(uint32_t); /* data_size */
}

static void test_mail_cache_delete_too_large_int(bool exceed_on_first_write)
{
	const struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.max_size = 1024,
		},
	};
	struct test_mail_cache_ctx ctx;
	struct stat st;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo2");

	test_assert(stat(ctx.index->cache->filepath, &st) == 0);

	/* create cache file that is exactly max_size */
	size_t field_size =
		max_field_size(optimization_set.cache.max_size, st.st_size);
	if (exceed_on_first_write) {
		test_expect_error_string("Cache file too large");
		field_size++;
	}
	char *field = i_malloc(field_size + 1);
	memset(field, 'x', field_size);
	test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, field);
	test_expect_no_more_errors();
	i_free(field);

	if (!exceed_on_first_write) {
		test_assert(stat(ctx.index->cache->filepath, &st) == 0);
		test_assert(st.st_size == 1024);

		/* adding anything more will delete the cache. */
		test_expect_error_string("Cache file too large");
		test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar1");
		test_expect_no_more_errors();
	}
	test_assert(stat(ctx.index->cache->filepath, &st) < 0 && errno == ENOENT);

	mail_index_view_close(&ctx.view);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_delete_too_large(void)
{
	test_begin("mail cache delete too large");
	test_mail_cache_delete_too_large_int(FALSE);
	test_end();
}

static void test_mail_cache_delete_too_large2(void)
{
	test_begin("mail cache delete too large (2)");
	test_mail_cache_delete_too_large_int(TRUE);
	test_end();
}

static void test_mail_cache_purge_too_large_int(bool exceed_size)
{
	const struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.max_size = 1024,
		},
	};
	struct mail_index_transaction *trans;
	struct mail_cache_view *cache_view;
	struct test_mail_cache_ctx ctx;
	struct stat st;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);

	/* add two mails with some cache field and expunge the first mail */
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "bar2");
	trans = mail_index_transaction_begin(ctx.view, 0);
	mail_index_expunge(trans, 1);
	test_assert(mail_index_transaction_commit(&trans) == 0);
	test_mail_cache_index_sync(&ctx);

	/* Add a second mail whose cache field size is exactly the
	   max_size [+1 if exceed_size] */
	test_assert(stat(ctx.index->cache->filepath, &st) == 0);
	size_t field_size = (exceed_size ? 1 : 0) +
		max_field_size(optimization_set.cache.max_size, st.st_size);
	char *field = i_malloc(field_size + 1);
	memset(field, 'x', field_size);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, field);

	test_assert(stat(ctx.index->cache->filepath, &st) == 0);
	if (exceed_size)
		test_assert((uoff_t)st.st_size < optimization_set.cache.max_size);
	else
		test_assert((uoff_t)st.st_size == optimization_set.cache.max_size);

	/* make sure we still find the cache fields */
	test_mail_cache_view_sync(&ctx);
	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "bar2"));
	test_assert(cache_equals(cache_view, 2, ctx.cache_field.idx, field));
	mail_cache_view_close(&cache_view);

	i_free(field);
	if (exceed_size)
		test_assert(test_mail_cache_get_purge_count(&ctx) == 1);
	else
		test_assert(test_mail_cache_get_purge_count(&ctx) == 0);
	mail_index_view_close(&ctx.view);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_purge_too_large(void)
{
	test_begin("mail cache purge too large");
	test_mail_cache_purge_too_large_int(FALSE);
	test_end();
}

static void test_mail_cache_purge_too_large2(void)
{
	test_begin("mail cache purge too large (2)");
	test_mail_cache_purge_too_large_int(TRUE);
	test_end();
}

static void test_mail_cache_unexpectedly_lost_int(bool read_first)
{
	struct test_mail_cache_ctx ctx;
	struct mail_cache_view *cache_view;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");

	test_mail_cache_purge();

	/* Unexpectedly delete the cache file under us */
	i_unlink(ctx.cache->filepath);

	if (read_first) {
		/* the cache file is already open, so initial reading should
		   work without errors */
		cache_view = mail_cache_view_open(ctx.cache, ctx.view);
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1"));
		mail_cache_view_close(&cache_view);

		/* if we refresh the index we get new reset_id, which requires
		   reopening the cache and that fails */
		test_assert(mail_index_refresh(ctx.index) == 0);
		test_mail_cache_view_sync(&ctx);
		cache_view = mail_cache_view_open(ctx.cache, ctx.view);
		test_expect_error_string("test.dovecot.index.cache: No such file or directory");
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL));
		test_expect_no_more_errors();
		mail_cache_view_close(&cache_view);
	} else {
		test_expect_error_string("test.dovecot.index.cache: No such file or directory");
	}

	/* writing after losing the cache should still work */
	test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar1");
	test_expect_no_more_errors();

	/* verify that the second cache field is found, but first is lost */
	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL));
	test_assert(cache_equals(cache_view, 1, ctx.cache_field2.idx, "bar1"));
	mail_cache_view_close(&cache_view);

	test_assert(test_mail_cache_get_purge_count(&ctx) == 2);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_unexpectedly_lost(void)
{
	test_begin("mail cache unexpectedly lost");
	test_mail_cache_unexpectedly_lost_int(FALSE);
	test_end();
}

static void test_mail_cache_unexpectedly_lost2(void)
{
	test_begin("mail cache unexpectedly lost (2)");
	test_mail_cache_unexpectedly_lost_int(TRUE);
	test_end();
}

static void test_mail_cache_resetid_mismatch_int(bool read_first)
{
	struct test_mail_cache_ctx ctx;
	struct mail_cache_view *cache_view;
	const char *temp_cache_path;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");

	/* make a copy of the first cache file */
	temp_cache_path = t_strdup_printf("%s.test", ctx.cache->filepath);
	test_assert(link(ctx.cache->filepath, temp_cache_path) == 0);

	if (read_first) {
		/* use a secondary index to purge the cache */
		test_mail_cache_purge();

		/* Replace the new cache file with an old one */
		test_assert(rename(temp_cache_path, ctx.cache->filepath) == 0);

		/* the cache file is already open, so initial reading should
		   work without errors */
		cache_view = mail_cache_view_open(ctx.cache, ctx.view);
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, "foo1"));
		mail_cache_view_close(&cache_view);

		/* if we refresh the index we get new reset_id, which requires
		   reopening the cache and that fails */
		test_assert(mail_index_refresh(ctx.index) == 0);
		test_mail_cache_view_sync(&ctx);
		cache_view = mail_cache_view_open(ctx.cache, ctx.view);

		test_expect_error_string("reset_id mismatch even after locking");
		test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL));
		test_expect_no_more_errors();
		mail_cache_view_close(&cache_view);
	} else {
		/* purge cache to update reset_id in index */
		test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);

		/* Replace the new cache file with an old one */
		test_assert(rename(temp_cache_path, ctx.cache->filepath) == 0);

		test_expect_error_string("reset_id mismatch even after locking");
	}

	/* writing should automatically fix the reset_id mismatch */
	test_mail_cache_add_field(&ctx, 1, ctx.cache_field2.idx, "bar1");
	test_expect_no_more_errors();

	/* verify that the second cache field is found, but first is lost */
	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	test_assert(cache_equals(cache_view, 1, ctx.cache_field.idx, NULL));
	test_assert(cache_equals(cache_view, 1, ctx.cache_field2.idx, "bar1"));
	mail_cache_view_close(&cache_view);

	test_assert(test_mail_cache_get_purge_count(&ctx) == 2);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_resetid_mismatch(void)
{
	test_begin("mail cache resetid mismatch");
	test_mail_cache_resetid_mismatch_int(FALSE);
	test_end();
}

static void test_mail_cache_resetid_mismatch2(void)
{
	test_begin("mail cache resetid mismatch (2)");
	test_mail_cache_resetid_mismatch_int(TRUE);
	test_end();
}

enum test_drop {
	TEST_DROP_NOTHING,
	TEST_DROP_YES_TO_TEMP_FIRST,
	TEST_DROP_YES_TO_TEMP_LAST,
	TEST_DROP_TEMP_TO_NO,
};

static void test_mail_cache_purge_field_changes_int(enum test_drop drop)
{
	enum {
		TEST_FIELD_NO,
		TEST_FIELD_NO_FORCED,
		TEST_FIELD_TEMP,
		TEST_FIELD_TEMP_FORCED,
		TEST_FIELD_YES,
		TEST_FIELD_YES_FORCED,
	};
	struct mail_cache_field cache_fields[] = {
		{
			.name = "no",
			.type = MAIL_CACHE_FIELD_STRING,
			.decision = MAIL_CACHE_DECISION_NO,
		},
		{
			.name = "no-forced",
			.type = MAIL_CACHE_FIELD_STRING,
			.decision = MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED,
		},
		{
			.name = "temp",
			.type = MAIL_CACHE_FIELD_STRING,
			.decision = MAIL_CACHE_DECISION_TEMP,
		},
		{
			.name = "temp-forced",
			.type = MAIL_CACHE_FIELD_STRING,
			.decision = MAIL_CACHE_DECISION_TEMP | MAIL_CACHE_DECISION_FORCED,
		},
		{
			.name = "yes",
			.type = MAIL_CACHE_FIELD_STRING,
			.decision = MAIL_CACHE_DECISION_YES,
		},
		{
			.name = "yes-forced",
			.type = MAIL_CACHE_FIELD_STRING,
			.decision = MAIL_CACHE_DECISION_YES | MAIL_CACHE_DECISION_FORCED,
		},
	};
	const struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.unaccessed_field_drop_secs = 61,
		},
	};
	struct test_mail_cache_ctx ctx;
	struct mail_cache_view *cache_view;
	struct mail_cache_transaction_ctx *cache_trans;
	struct mail_index_transaction *trans;
	unsigned int i;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);

	/* add two mails with all of the cache fields */
	test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
	test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);

	/* Create the cache file before registering any of the cache_fields
	   that we're testing. Otherwise our caching decisions are messed up
	   by purging (which is called to auto-create the cache). */
	test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
	mail_cache_register_fields(ctx.cache, cache_fields,
				   N_ELEMENTS(cache_fields));

	trans = mail_index_transaction_begin(ctx.view, 0);
	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	cache_trans = mail_cache_get_transaction(cache_view, trans);
	for (i = 0; i < N_ELEMENTS(cache_fields); i++) {
		const char *value = t_strdup_printf("%s-value",
						    cache_fields[i].name);
		if ((cache_fields[i].decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) !=
		    MAIL_CACHE_DECISION_NO) {
			mail_cache_add(cache_trans, 1, cache_fields[i].idx,
				       value, strlen(value));
			mail_cache_add(cache_trans, 2, cache_fields[i].idx,
				       value, strlen(value));
		}
	}

	/* day_stamp in index is used for deciding when a cache field needs to
	   be dropped. */
	uint32_t day_stamp = 123456789;
	mail_index_update_header(trans,
		offsetof(struct mail_index_header, day_stamp),
		&day_stamp, sizeof(day_stamp), FALSE);
	/* day_first_uid[7] is used to determine which mails are "old" and
	   which mails are "new". [7] is the first "new" mail. */
	uint32_t first_new_uid = 2;
	mail_index_update_header(trans,
		offsetof(struct mail_index_header, day_first_uid[7]),
		&first_new_uid, sizeof(first_new_uid), FALSE);
	test_assert(mail_index_transaction_commit(&trans) == 0);

	/* set the last_used time just at the boundary of being dropped or
	   being kept */
	for (i = 0; i < ctx.cache->fields_count; i++) {
		unsigned int secs = optimization_set.cache.unaccessed_field_drop_secs;
		switch (drop) {
		case TEST_DROP_NOTHING:
			break;
		case TEST_DROP_YES_TO_TEMP_FIRST:
			secs++;
			break;
		case TEST_DROP_YES_TO_TEMP_LAST:
			secs *= 2;
			break;
		case TEST_DROP_TEMP_TO_NO:
			secs *= 2;
			secs++;
			break;
		}
		ctx.cache->fields[i].field.last_used = day_stamp - secs;
	}
	test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
	test_mail_cache_view_sync(&ctx);

	/* verify that caching decisions are as expected after purging */
	test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO].idx].field.decision ==
		    MAIL_CACHE_DECISION_NO);
	test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_NO_FORCED].idx].field.decision ==
		    (MAIL_CACHE_DECISION_NO | MAIL_CACHE_DECISION_FORCED));
	test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP_FORCED].idx].field.decision ==
		    (MAIL_CACHE_DECISION_TEMP | MAIL_CACHE_DECISION_FORCED));
	test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES_FORCED].idx].field.decision ==
		    (MAIL_CACHE_DECISION_YES | MAIL_CACHE_DECISION_FORCED));

	switch (drop) {
	case TEST_DROP_NOTHING:
		test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision ==
			    MAIL_CACHE_DECISION_TEMP);
		test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES].idx].field.decision ==
			    MAIL_CACHE_DECISION_YES);
		break;
	case TEST_DROP_YES_TO_TEMP_FIRST:
	case TEST_DROP_YES_TO_TEMP_LAST:
		test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision ==
			    MAIL_CACHE_DECISION_TEMP);
		test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES].idx].field.decision ==
			    MAIL_CACHE_DECISION_TEMP);
		break;
	case TEST_DROP_TEMP_TO_NO:
		test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_TEMP].idx].field.decision ==
			    MAIL_CACHE_DECISION_NO);
		test_assert(ctx.cache->fields[cache_fields[TEST_FIELD_YES].idx].field.decision ==
			    MAIL_CACHE_DECISION_NO);
		break;
	}

	/* verify that cache fields exist as expected after purging */
	test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_NO].idx, NULL));
	test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_NO].idx, NULL));
	test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_NO_FORCED].idx, NULL));
	test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_NO_FORCED].idx, NULL));
	test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_TEMP].idx, NULL));
	if (drop == TEST_DROP_TEMP_TO_NO)
		test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_TEMP].idx, NULL));
	else
		test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_TEMP].idx, "temp-value"));
	test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_TEMP_FORCED].idx, NULL));
	test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_TEMP_FORCED].idx, "temp-forced-value"));
	if (drop != TEST_DROP_NOTHING)
		test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_YES].idx, NULL));
	else
		test_assert(cache_equals(cache_view, 1, cache_fields[TEST_FIELD_YES].idx, "yes-value"));
	test_assert(cache_equals(cache_view, 2, cache_fields[TEST_FIELD_YES_FORCED].idx, "yes-forced-value"));

	test_assert(test_mail_cache_get_purge_count(&ctx) == 1);
	mail_cache_view_close(&cache_view);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_purge_field_changes(void)
{
	test_begin("mail cache purge field changes (nothing)");
	test_mail_cache_purge_field_changes_int(TEST_DROP_NOTHING);
	test_end();
}

static void test_mail_cache_purge_field_changes2(void)
{
	test_begin("mail cache purge field changes (yes -> temp, first)");
	test_mail_cache_purge_field_changes_int(TEST_DROP_YES_TO_TEMP_FIRST);
	test_end();
}

static void test_mail_cache_purge_field_changes3(void)
{
	test_begin("mail cache purge field changes (yes -> temp, last)");
	test_mail_cache_purge_field_changes_int(TEST_DROP_YES_TO_TEMP_LAST);
	test_end();
}

static void test_mail_cache_purge_field_changes4(void)
{
	test_begin("mail cache purge field changes (temp -> no)");
	test_mail_cache_purge_field_changes_int(TEST_DROP_TEMP_TO_NO);
	test_end();
}

static void test_mail_cache_purge_already_done(void)
{
	struct test_mail_cache_ctx ctx;

	test_begin("mail cache purge already done");
	test_mail_cache_init(test_mail_index_init(), &ctx);
	test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, "foo1");

	test_mail_cache_purge();
	test_assert(mail_cache_purge(ctx.cache, 1, "test") == 0);
	test_assert(test_mail_cache_get_purge_count(&ctx) == 1);

	test_assert(mail_cache_purge(ctx.cache, 2, "test") == 0);
	test_assert(test_mail_cache_get_purge_count(&ctx) == 2);

	test_assert(mail_cache_purge(ctx.cache, 2, "test") == 0);
	test_assert(test_mail_cache_get_purge_count(&ctx) == 2);

	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
	test_end();
}

static void test_mail_cache_purge_bitmask(void)
{
	struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.unaccessed_field_drop_secs = 60,
		},
	};
	struct mail_cache_field bitmask_field = {
		.name = "bitmask",
		.type = MAIL_CACHE_FIELD_BITMASK,
		.field_size = 1,
		.decision = MAIL_CACHE_DECISION_TEMP,
	};
	struct test_mail_cache_ctx ctx;
	struct mail_cache_view *cache_view;

	test_begin("mail cache purge bitmask");
	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);
	ioloop_time = 1000000;
	test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
	test_mail_cache_add_mail(&ctx, UINT_MAX, NULL);
	test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);
	mail_cache_register_fields(ctx.cache, &bitmask_field, 1);

	test_mail_cache_update_day_first_uid7(&ctx, 3);

	test_mail_cache_add_field(&ctx, 1, bitmask_field.idx, "\x01");
	test_mail_cache_add_field(&ctx, 1, bitmask_field.idx, "\x02");
	test_mail_cache_add_field(&ctx, 1, bitmask_field.idx, "\x04");
	test_mail_cache_add_field(&ctx, 2, bitmask_field.idx, "\x01");
	test_mail_cache_add_field(&ctx, 2, bitmask_field.idx, "\x02");
	test_mail_cache_add_field(&ctx, 2, bitmask_field.idx, "\x04");

	/* avoid dropping the field */
	ctx.cache->fields[bitmask_field.idx].field.last_used = ioloop_time;

	/* purge with TEMP decision, which causes the bitmask to be dropped */
	test_assert(mail_cache_purge(ctx.cache, (uint32_t)-1, "test") == 0);

	cache_view = mail_cache_view_open(ctx.cache, ctx.view);
	test_assert(cache_equals(cache_view, 1, bitmask_field.idx, NULL));
	test_assert(cache_equals(cache_view, 2, bitmask_field.idx, NULL));
	mail_cache_view_close(&cache_view);

	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
	test_end();
}


static void
test_mail_cache_update_need_purge_continued_records_int(bool big_min_size)
{
	struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.purge_min_size = big_min_size ? 1024*1024 : 1,
			.purge_continued_percentage = 30,
		},
	};
	char value[30];
	struct test_mail_cache_ctx ctx;
	uint32_t seq;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);

	for (seq = 1; seq <= 100; seq++) {
		i_snprintf(value, sizeof(value), "foo%d", seq);
		test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, value);
	}

	/* up to 29% no need to purge */
	for (seq = 1; seq <= 29; seq++) {
		i_snprintf(value, sizeof(value), "bar%d", seq);
		test_mail_cache_add_field(&ctx, seq, ctx.cache_field2.idx, value);
	}
	test_assert(ctx.cache->need_purge_file_seq == 0);

	/* at 30% need to purge */
	test_mail_cache_add_field(&ctx, 30, ctx.cache_field2.idx, "bar30");
	if (big_min_size)
		test_assert(ctx.cache->need_purge_file_seq == 0);
	else
		test_assert(ctx.cache->need_purge_file_seq == ctx.cache->hdr->file_seq);

	test_assert(test_mail_cache_get_purge_count(&ctx) == 0);
	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_update_need_purge_continued_records(void)
{
	test_begin("mail cache update need purge continued records");
	test_mail_cache_update_need_purge_continued_records_int(FALSE);
	test_end();
}

static void test_mail_cache_update_need_purge_continued_records2(void)
{
	test_begin("mail cache update need purge continued records (2)");
	test_mail_cache_update_need_purge_continued_records_int(TRUE);
	test_end();
}

static void
test_mail_cache_update_need_purge_deleted_records_int(bool big_min_size)
{
	struct mail_index_optimization_settings optimization_set = {
		.cache = {
			.purge_min_size = big_min_size ? 1024*1024 : 1,
			.purge_delete_percentage = 30,
		},
	};
	char value[30];
	struct mail_index_transaction *trans;
	struct test_mail_cache_ctx ctx;
	uint32_t seq;

	test_mail_cache_init(test_mail_index_init(), &ctx);
	mail_index_set_optimization_settings(ctx.index, &optimization_set);

	for (seq = 1; seq <= 100; seq++) {
		i_snprintf(value, sizeof(value), "foo%d", seq);
		test_mail_cache_add_mail(&ctx, ctx.cache_field.idx, value);
	}

	/* up to 29% no need to purge */
	trans = mail_index_transaction_begin(ctx.view, 0);
	for (seq = 1; seq <= 29; seq++) {
		i_snprintf(value, sizeof(value), "bar%d", seq);
		mail_index_expunge(trans, seq);
	}
	test_assert(mail_index_transaction_commit(&trans) == 0);
	test_mail_cache_index_sync(&ctx);

	test_assert(ctx.cache->need_purge_file_seq == 0);
	test_assert(mail_cache_reopen(ctx.cache) == 1);
	test_assert(ctx.cache->need_purge_file_seq == 0);
	test_assert(test_mail_cache_get_purge_count(&ctx) == 0);

	/* at 30% need to purge */
	trans = mail_index_transaction_begin(ctx.view, 0);
	mail_index_expunge(trans, 1);
	test_assert(mail_index_transaction_commit(&trans) == 0);
	/* syncing will internally purge if !big_min_size */
	test_mail_cache_index_sync(&ctx);

	test_assert(ctx.cache->need_purge_file_seq == 0);
	test_assert(mail_cache_reopen(ctx.cache) == 1);
	test_assert(ctx.cache->need_purge_file_seq == 0);
	if (big_min_size)
		test_assert(test_mail_cache_get_purge_count(&ctx) == 0);
	else
		test_assert(test_mail_cache_get_purge_count(&ctx) == 1);

	test_mail_cache_deinit(&ctx);
	test_mail_index_delete();
}

static void test_mail_cache_update_need_purge_deleted_records(void)
{
	test_begin("mail cache update need purge deleted records");
	test_mail_cache_update_need_purge_deleted_records_int(FALSE);
	test_end();
}

static void test_mail_cache_update_need_purge_deleted_records2(void)
{
	test_begin("mail cache update need purge deleted records (2)");
	test_mail_cache_update_need_purge_deleted_records_int(TRUE);
	test_end();
}

int main(void)
{
	static void (*const test_functions[])(void) = {
		test_mail_cache_read_during_purge,
		test_mail_cache_write_during_purge,
		test_mail_cache_purge_while_cache_locked,
		test_mail_cache_write_lost_during_purge,
		test_mail_cache_write_lost_during_purge2,
		test_mail_cache_write_autocommit,
		test_mail_cache_write_autocommit2,
		test_mail_cache_delete_too_large,
		test_mail_cache_delete_too_large2,
		test_mail_cache_purge_too_large,
		test_mail_cache_purge_too_large2,
		test_mail_cache_unexpectedly_lost,
		test_mail_cache_unexpectedly_lost2,
		test_mail_cache_resetid_mismatch,
		test_mail_cache_resetid_mismatch2,
		test_mail_cache_purge_field_changes,
		test_mail_cache_purge_field_changes2,
		test_mail_cache_purge_field_changes3,
		test_mail_cache_purge_field_changes4,
		test_mail_cache_purge_already_done,
		test_mail_cache_purge_bitmask,
		test_mail_cache_update_need_purge_continued_records,
		test_mail_cache_update_need_purge_continued_records2,
		test_mail_cache_update_need_purge_deleted_records,
		test_mail_cache_update_need_purge_deleted_records2,
		NULL
	};
	return test_run(test_functions);
}