1 files changed, 724 insertions, 0 deletions

diff --git a/src/lib-index/mail-cache-purge.c b/src/lib-index/mail-cache-purge.c
new file mode 100644
index 0000000..fa5874d
--- /dev/null
+++ b/src/lib-index/mail-cache-purge.c
@@ -0,0 +1,724 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ostream.h"
+#include "nfs-workarounds.h"
+#include "read-full.h"
+#include "file-dotlock.h"
+#include "file-cache.h"
+#include "file-set-size.h"
+#include "mail-cache-private.h"
+
+#include <stdio.h>
+#include <sys/stat.h>
+
+struct mail_cache_copy_context {
+	struct mail_cache *cache;
+	struct event *event;
+	struct mail_cache_purge_drop_ctx drop_ctx;
+
+	buffer_t *buffer, *field_seen;
+	ARRAY(unsigned int) bitmask_pos;
+	uint32_t *field_file_map;
+
+	uint8_t field_seen_value;
+	bool new_msg;
+};
+
+static void
+mail_cache_merge_bitmask(struct mail_cache_copy_context *ctx,
+			 const struct mail_cache_iterate_field *field)
+{
+	unsigned char *dest;
+	unsigned int i, *pos;
+
+	pos = array_idx_get_space(&ctx->bitmask_pos, field->field_idx);
+	if (*pos == 0) {
+		/* we decided to drop this field */
+		return;
+	}
+
+	dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size);
+	for (i = 0; i < field->size; i++)
+		dest[i] |= ((const unsigned char*)field->data)[i];
+}
+
+static void
+mail_cache_purge_field(struct mail_cache_copy_context *ctx,
+		       const struct mail_cache_iterate_field *field)
+{
+        struct mail_cache_field *cache_field;
+	enum mail_cache_decision_type dec;
+	uint32_t file_field_idx, size32;
+	uint8_t *field_seen;
+
+	file_field_idx = ctx->field_file_map[field->field_idx];
+	if (file_field_idx == (uint32_t)-1)
+		return;
+
+	cache_field = &ctx->cache->fields[field->field_idx].field;
+
+	field_seen = buffer_get_space_unsafe(ctx->field_seen,
+					     field->field_idx, 1);
+	if (*field_seen == ctx->field_seen_value) {
+		/* duplicate */
+		if (cache_field->type == MAIL_CACHE_FIELD_BITMASK)
+			mail_cache_merge_bitmask(ctx, field);
+		return;
+	}
+	*field_seen = ctx->field_seen_value;
+
+	dec = cache_field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED);
+	if (ctx->new_msg) {
+		if (dec == MAIL_CACHE_DECISION_NO)
+			return;
+	} else {
+		if (dec != MAIL_CACHE_DECISION_YES)
+			return;
+	}
+
+	buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx));
+
+	if (cache_field->field_size == UINT_MAX) {
+		size32 = (uint32_t)field->size;
+		buffer_append(ctx->buffer, &size32, sizeof(size32));
+	}
+
+	if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) {
+		/* remember the position in case we need to update it */
+		unsigned int pos = ctx->buffer->used;
+
+		array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos);
+	}
+	buffer_append(ctx->buffer, field->data, field->size);
+	if ((field->size & 3) != 0)
+		buffer_append_zero(ctx->buffer, 4 - (field->size & 3));
+}
+
+static uint32_t get_next_file_seq(struct mail_cache *cache)
+{
+	const struct mail_index_ext *ext;
+	struct mail_index_view *view;
+	uint32_t file_seq;
+
+	/* make sure we look up the latest reset_id */
+	if (mail_index_refresh(cache->index) < 0)
+		return -1;
+
+	view = mail_index_view_open(cache->index);
+	ext = mail_index_view_get_ext(view, cache->ext_id);
+	file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time;
+
+	if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq)
+		file_seq = cache->hdr->file_seq + 1;
+	mail_index_view_close(&view);
+
+	return file_seq != 0 ? file_seq : 1;
+}
+
+static void
+mail_cache_purge_get_fields(struct mail_cache_copy_context *ctx,
+			    unsigned int used_fields_count)
+{
+	struct mail_cache *cache = ctx->cache;
+	unsigned int i, j, idx;
+
+	/* Make mail_cache_header_fields_get() return the fields in
+	   the same order as we saved them. */
+	memcpy(cache->field_file_map, ctx->field_file_map,
+	       sizeof(uint32_t) * cache->fields_count);
+
+	/* reverse mapping */
+	cache->file_fields_count = used_fields_count;
+	i_free(cache->file_field_map);
+	cache->file_field_map = used_fields_count == 0 ? NULL :
+		i_new(unsigned int, used_fields_count);
+	for (i = j = 0; i < cache->fields_count; i++) {
+		idx = cache->field_file_map[i];
+		if (idx != (uint32_t)-1) {
+			i_assert(idx < used_fields_count &&
+				 cache->file_field_map != NULL &&
+				 cache->file_field_map[idx] == 0);
+			cache->file_field_map[idx] = i;
+			j++;
+		}
+	}
+	i_assert(j == used_fields_count);
+
+	buffer_set_used_size(ctx->buffer, 0);
+	mail_cache_header_fields_get(cache, ctx->buffer);
+}
+
+static bool
+mail_cache_purge_check_field(struct mail_cache_copy_context *ctx,
+			     unsigned int field)
+{
+	struct mail_cache_field_private *priv = &ctx->cache->fields[field];
+	enum mail_cache_decision_type dec = priv->field.decision;
+
+	switch (mail_cache_purge_drop_test(&ctx->drop_ctx, field)) {
+	case MAIL_CACHE_PURGE_DROP_DECISION_NONE:
+		break;
+	case MAIL_CACHE_PURGE_DROP_DECISION_DROP: {
+		const char *dec_str = mail_cache_decision_to_string(dec);
+		struct event_passthrough *e =
+			event_create_passthrough(ctx->event)->
+			set_name("mail_cache_purge_drop_field")->
+			add_str("field", priv->field.name)->
+			add_str("decision", dec_str)->
+			add_int("last_used", priv->field.last_used);
+		e_debug(e->event(), "Purge dropped field %s "
+			"(decision=%s, last_used=%"PRIdTIME_T")",
+			priv->field.name, dec_str, priv->field.last_used);
+		dec = MAIL_CACHE_DECISION_NO;
+		break;
+	}
+	case MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP: {
+		struct event_passthrough *e =
+			mail_cache_decision_changed_event(
+				ctx->cache, ctx->event, field)->
+			add_str("old_decision", "yes")->
+			add_str("new_decision", "temp");
+		e_debug(e->event(), "Purge changes field %s "
+			"cache decision yes -> temp "
+			"(last_used=%"PRIdTIME_T")",
+			priv->field.name, priv->field.last_used);
+		dec = MAIL_CACHE_DECISION_TEMP;
+		break;
+	}
+	}
+	priv->field.decision = dec;
+
+	/* drop all fields we don't want */
+	if ((dec & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) == MAIL_CACHE_DECISION_NO) {
+		priv->used = FALSE;
+		priv->field.last_used = 0;
+	}
+	return priv->used;
+}
+
+static int
+mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans,
+		struct event *event, int fd, const char *reason,
+		uint32_t *file_seq_r, uoff_t *file_size_r, uint32_t *max_uid_r,
+		uint32_t *ext_first_seq_r, ARRAY_TYPE(uint32_t) *ext_offsets)
+{
+        struct mail_cache_copy_context ctx;
+	struct mail_cache_lookup_iterate_ctx iter;
+	struct mail_cache_iterate_field field;
+	struct mail_index_view *view;
+	struct mail_cache_view *cache_view;
+	const struct mail_index_header *idx_hdr;
+	struct mail_cache_header hdr;
+	struct mail_cache_record cache_rec;
+	struct ostream *output;
+	uint32_t message_count, seq, first_new_seq, ext_offset;
+	unsigned int i, used_fields_count, orig_fields_count, record_count;
+
+	i_assert(reason != NULL);
+
+	*max_uid_r = 0;
+	*ext_first_seq_r = 0;
+
+	/* get the latest info on fields */
+	if (mail_cache_header_fields_read(cache) < 0)
+		return -1;
+
+	view = mail_index_transaction_open_updated_view(trans);
+	cache_view = mail_cache_view_open(cache, view);
+	output = o_stream_create_fd_file(fd, 0, FALSE);
+
+	i_zero(&hdr);
+	hdr.major_version = MAIL_CACHE_MAJOR_VERSION;
+	hdr.minor_version = MAIL_CACHE_MINOR_VERSION;
+	hdr.compat_sizeof_uoff_t = sizeof(uoff_t);
+	hdr.indexid = cache->index->indexid;
+	hdr.file_seq = get_next_file_seq(cache);
+	o_stream_nsend(output, &hdr, sizeof(hdr));
+
+	event_add_str(event, "reason", reason);
+	event_add_int(event, "file_seq", hdr.file_seq);
+	event_set_name(event, "mail_cache_purge_started");
+	e_debug(event, "Purging (new file_seq=%u): %s", hdr.file_seq, reason);
+
+	i_zero(&ctx);
+	ctx.cache = cache;
+	ctx.event = event;
+	ctx.buffer = buffer_create_dynamic(default_pool, 4096);
+	ctx.field_seen = buffer_create_dynamic(default_pool, 64);
+	ctx.field_seen_value = 0;
+	ctx.field_file_map = t_new(uint32_t, cache->fields_count + 1);
+	t_array_init(&ctx.bitmask_pos, 32);
+
+	/* @UNSAFE: drop unused fields and create a field mapping for
+	   used fields */
+	idx_hdr = mail_index_get_header(view);
+	mail_cache_purge_drop_init(cache, idx_hdr, &ctx.drop_ctx);
+
+	orig_fields_count = cache->fields_count;
+	if (cache->file_fields_count == 0) {
+		/* creating the initial cache file. add all fields. */
+		for (i = 0; i < orig_fields_count; i++)
+			ctx.field_file_map[i] = i;
+		used_fields_count = i;
+	} else {
+		for (i = used_fields_count = 0; i < orig_fields_count; i++) {
+			if (!mail_cache_purge_check_field(&ctx, i))
+				ctx.field_file_map[i] = (uint32_t)-1;
+			else
+				ctx.field_file_map[i] = used_fields_count++;
+		}
+	}
+
+	/* get sequence of first message which doesn't need its temp fields
+	   removed. */
+	first_new_seq = mail_cache_get_first_new_seq(view);
+	message_count = mail_index_view_get_messages_count(view);
+	if (!trans->reset)
+		seq = 1;
+	else {
+		/* Index is being rebuilt. Ignore old messages. */
+		seq = trans->first_new_seq;
+	}
+
+	*ext_first_seq_r = seq;
+	i_array_init(ext_offsets, message_count); record_count = 0;
+	for (; seq <= message_count; seq++) {
+		if (mail_index_transaction_is_expunged(trans, seq)) {
+			array_append_zero(ext_offsets);
+			continue;
+		}
+
+		ctx.new_msg = seq >= first_new_seq;
+		buffer_set_used_size(ctx.buffer, 0);
+
+		ctx.field_seen_value = (ctx.field_seen_value + 1) & UINT8_MAX;
+		if (ctx.field_seen_value == 0) {
+			memset(buffer_get_modifiable_data(ctx.field_seen, NULL),
+			       0, buffer_get_size(ctx.field_seen));
+			ctx.field_seen_value++;
+		}
+		array_clear(&ctx.bitmask_pos);
+
+		i_zero(&cache_rec);
+		buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec));
+
+		mail_cache_lookup_iter_init(cache_view, seq, &iter);
+		while (mail_cache_lookup_iter_next(&iter, &field) > 0)
+			mail_cache_purge_field(&ctx, &field);
+
+		if (ctx.buffer->used == sizeof(cache_rec) ||
+		    ctx.buffer->used > cache->index->optimization_set.cache.record_max_size) {
+			/* nothing cached */
+			ext_offset = 0;
+		} else {
+			mail_index_lookup_uid(view, seq, max_uid_r);
+			cache_rec.size = ctx.buffer->used;
+			ext_offset = output->offset;
+			buffer_write(ctx.buffer, 0, &cache_rec,
+				     sizeof(cache_rec));
+			o_stream_nsend(output, ctx.buffer->data, cache_rec.size);
+			record_count++;
+		}
+
+		array_push_back(ext_offsets, &ext_offset);
+	}
+	i_assert(orig_fields_count == cache->fields_count);
+
+	bool file_too_large =
+		output->offset > cache->index->optimization_set.cache.max_size;
+	if (!file_too_large) {
+		hdr.record_count = record_count;
+		hdr.field_header_offset = mail_index_uint32_to_offset(output->offset);
+		mail_cache_purge_get_fields(&ctx, used_fields_count);
+		o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used);
+	}
+
+	hdr.backwards_compat_used_file_size = output->offset;
+	buffer_free(&ctx.buffer);
+	buffer_free(&ctx.field_seen);
+
+	*file_size_r = output->offset;
+	(void)o_stream_seek(output, 0);
+	o_stream_nsend(output, &hdr, sizeof(hdr));
+
+	mail_cache_view_close(&cache_view);
+	mail_index_view_close(&view);
+
+	if (file_too_large || o_stream_finish(output) < 0) {
+		if (!file_too_large) {
+			errno = output->stream_errno;
+			mail_cache_set_syscall_error(cache, "write()");
+		} else {
+			/* start from a new empty cache file */
+			mail_index_set_error(cache->index,
+				"Cache file %s: File is too large - deleting",
+				cache->filepath);
+			i_unlink(cache->filepath);
+		}
+		o_stream_destroy(&output);
+		array_free(ext_offsets);
+		return -1;
+	}
+	o_stream_destroy(&output);
+
+	if (cache->index->set.fsync_mode == FSYNC_MODE_ALWAYS) {
+		if (fdatasync(fd) < 0) {
+			mail_cache_set_syscall_error(cache, "fdatasync()");
+			array_free(ext_offsets);
+			return -1;
+		}
+	}
+
+	*file_seq_r = hdr.file_seq;
+	return 0;
+}
+
+static int
+mail_cache_purge_write(struct mail_cache *cache,
+		       struct mail_index_transaction *trans,
+		       int fd, const char *temp_path, const char *
+		       reason, bool *unlock)
+{
+	struct event *event;
+	struct stat st;
+	uint32_t prev_file_seq, file_seq, old_offset, max_uid, ext_first_seq;
+	ARRAY_TYPE(uint32_t) ext_offsets;
+	const uint32_t *offsets;
+	uoff_t prev_file_size, file_size;
+	unsigned int i, count, prev_deleted_records;
+
+	if (cache->hdr == NULL) {
+		prev_file_seq = 0;
+		prev_file_size = 0;
+		prev_deleted_records = 0;
+	} else {
+		prev_file_seq = cache->hdr->file_seq;
+		prev_file_size = cache->last_stat_size;
+		prev_deleted_records = cache->hdr->deleted_record_count;
+	}
+	event = event_create(cache->event);
+	event_add_int(event, "prev_file_seq", prev_file_seq);
+	event_add_int(event, "prev_file_size", prev_file_size);
+	event_add_int(event, "prev_deleted_records", prev_deleted_records);
+
+	if (mail_cache_copy(cache, trans, event, fd, reason,
+			    &file_seq, &file_size, &max_uid,
+			    &ext_first_seq, &ext_offsets) < 0) {
+		event_unref(&event);
+		return -1;
+	}
+
+	if (fstat(fd, &st) < 0) {
+		mail_cache_set_syscall_error(cache, "fstat()");
+		array_free(&ext_offsets);
+		event_unref(&event);
+		return -1;
+	}
+	if (rename(temp_path, cache->filepath) < 0) {
+		mail_cache_set_syscall_error(cache, "rename()");
+		array_free(&ext_offsets);
+		event_unref(&event);
+		return -1;
+	}
+
+	event_add_int(event, "file_size", file_size);
+	event_add_int(event, "max_uid", max_uid);
+	event_set_name(event, "mail_cache_purge_finished");
+	e_debug(event, "Purging finished, file_seq changed %u -> %u, "
+		"size=%"PRIuUOFF_T" -> %"PRIuUOFF_T", max_uid=%u",
+		prev_file_seq, file_seq, prev_file_size, file_size, max_uid);
+	event_unref(&event);
+
+	/* once we're sure that the purging was successful,
+	   update the offsets */
+	mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE);
+	offsets = array_get(&ext_offsets, &count);
+	for (i = 0; i < count; i++) {
+		if (offsets[i] != 0) {
+			mail_index_update_ext(trans, ext_first_seq + i,
+					      cache->ext_id,
+					      &offsets[i], &old_offset);
+		}
+	}
+	array_free(&ext_offsets);
+
+	if (*unlock) {
+		mail_cache_unlock(cache);
+		*unlock = FALSE;
+	}
+
+	mail_cache_file_close(cache);
+	cache->opened = TRUE;
+	cache->fd = fd;
+	cache->st_ino = st.st_ino;
+	cache->st_dev = st.st_dev;
+	cache->field_header_write_pending = FALSE;
+	return 0;
+}
+
+static int
+mail_cache_purge_has_file_changed(struct mail_cache *cache,
+				  uint32_t purge_file_seq)
+{
+	struct mail_cache_header hdr;
+	unsigned int i;
+	int fd, ret;
+
+	for (i = 0;; i++) {
+		fd = nfs_safe_open(cache->filepath, O_RDONLY);
+		if (fd == -1) {
+			if (errno == ENOENT)
+				return 0;
+
+			mail_cache_set_syscall_error(cache, "open()");
+			return -1;
+		}
+
+		ret = read_full(fd, &hdr, sizeof(hdr));
+		i_close_fd(&fd);
+
+		if (ret >= 0) {
+			if (ret == 0)
+				return 0;
+			if (purge_file_seq == 0) {
+				/* previously it didn't exist or it
+				   was unusable and was just unlinked */
+				return 1;
+			}
+			return hdr.file_seq != purge_file_seq ? 1 : 0;
+		} else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) {
+			mail_cache_set_syscall_error(cache, "read()");
+			return -1;
+		}
+	}
+}
+
+static int mail_cache_purge_locked(struct mail_cache *cache,
+				   uint32_t purge_file_seq,
+				   struct mail_index_transaction *trans,
+				   const char *reason, bool *unlock)
+{
+	const char *temp_path;
+	int fd, ret;
+
+	/* we've locked the cache purging now. if somebody else had just
+	   recreated the cache, reopen the cache and return success. */
+	if (purge_file_seq != (uint32_t)-1 &&
+	    (ret = mail_cache_purge_has_file_changed(cache, purge_file_seq)) != 0) {
+		if (ret < 0)
+			return -1;
+
+		/* was just purged, forget this */
+		mail_cache_purge_later_reset(cache);
+
+		if (*unlock) {
+			(void)mail_cache_unlock(cache);
+			*unlock = FALSE;
+		}
+
+		return mail_cache_reopen(cache) < 0 ? -1 : 0;
+	}
+	if (cache->fd != -1) {
+		/* make sure we have mapped it before reading. */
+		if (mail_cache_map_all(cache) <= 0)
+			return -1;
+	}
+
+	/* we want to recreate the cache. write it first to a temporary file */
+	fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path);
+	if (fd == -1)
+		return -1;
+	if (mail_cache_purge_write(cache, trans, fd, temp_path, reason, unlock) < 0) {
+		i_close_fd(&fd);
+		i_unlink(temp_path);
+		return -1;
+	}
+	if (cache->file_cache != NULL)
+		file_cache_set_fd(cache->file_cache, cache->fd);
+
+	if (mail_cache_map_all(cache) <= 0)
+		return -1;
+	if (mail_cache_header_fields_read(cache) < 0)
+		return -1;
+
+	mail_cache_purge_later_reset(cache);
+	return 0;
+}
+
+static int
+mail_cache_purge_full(struct mail_cache *cache,
+		      struct mail_index_transaction *trans,
+		      uint32_t purge_file_seq, const char *reason)
+{
+	bool unlock = FALSE;
+	int ret;
+
+	i_assert(!cache->purging);
+	i_assert(cache->index->log_sync_locked);
+
+	if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly)
+		return 0;
+
+	/* purging isn't very efficient with small read()s */
+	if (cache->map_with_read) {
+		cache->map_with_read = FALSE;
+		if (cache->read_buf != NULL)
+			buffer_set_used_size(cache->read_buf, 0);
+		cache->hdr = NULL;
+		cache->mmap_length = 0;
+	}
+
+	/* .log lock already prevents other processes from purging cache at
+	   the same time, but locking the cache file itself prevents other
+	   processes from doing other changes to it (header changes, adding
+	   more cached data). */
+	switch (mail_cache_lock(cache)) {
+	case -1:
+		/* lock timeout or some other error */
+		return -1;
+	case 0:
+		/* cache is broken or doesn't exist.
+		   just start creating it. */
+		break;
+	default:
+		/* locking succeeded. */
+		unlock = TRUE;
+	}
+	cache->purging = TRUE;
+	ret = mail_cache_purge_locked(cache, purge_file_seq, trans, reason, &unlock);
+	cache->purging = FALSE;
+	if (unlock)
+		mail_cache_unlock(cache);
+	i_assert(!cache->hdr_modified);
+	if (ret < 0) {
+		/* the fields may have been updated in memory already.
+		   reverse those changes by re-reading them from file. */
+		(void)mail_cache_header_fields_read(cache);
+	}
+	return ret;
+}
+
+int mail_cache_purge_with_trans(struct mail_cache *cache,
+				struct mail_index_transaction *trans,
+				uint32_t purge_file_seq, const char *reason)
+{
+	return mail_cache_purge_full(cache, trans, purge_file_seq, reason);
+}
+
+int mail_cache_purge(struct mail_cache *cache, uint32_t purge_file_seq,
+		     const char *reason)
+{
+	struct mail_index_view *view;
+	struct mail_index_transaction *trans;
+	bool lock_log;
+	int ret;
+
+	lock_log = !cache->index->log_sync_locked;
+	if (lock_log) {
+		uint32_t file_seq;
+		uoff_t file_offset;
+
+		if (mail_transaction_log_sync_lock(cache->index->log,
+						   "mail cache purge",
+						   &file_seq, &file_offset) < 0)
+			return -1;
+	}
+	/* make sure we see the latest changes in index */
+	ret = mail_index_refresh(cache->index);
+
+	view = mail_index_view_open(cache->index);
+	trans = mail_index_transaction_begin(view,
+		MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+	if (ret < 0)
+		;
+	else if ((ret = mail_cache_purge_full(cache, trans, purge_file_seq,
+					      reason)) < 0)
+		mail_index_transaction_rollback(&trans);
+	else {
+		if (mail_index_transaction_commit(&trans) < 0)
+			ret = -1;
+	}
+	mail_index_view_close(&view);
+	if (lock_log) {
+		mail_transaction_log_sync_unlock(cache->index->log,
+						 "mail cache purge");
+	}
+	return ret;
+}
+
+bool mail_cache_need_purge(struct mail_cache *cache, const char **reason_r)
+{
+	if (cache->need_purge_file_seq == 0)
+		return FALSE; /* delayed purging not requested */
+	if (cache->index->readonly)
+		return FALSE; /* no purging when opened as read-only */
+	if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0) {
+		/* Mail deliveries don't really need to purge, even if there
+		   could be some work to do. Just delay until the next regular
+		   mail access comes before doing any extra work. */
+		return FALSE;
+	}
+
+	i_assert(cache->need_purge_reason != NULL);
+	/* t_strdup() the reason in case it gets freed (or replaced)
+	   before it's used */
+	*reason_r = t_strdup(cache->need_purge_reason);
+	return TRUE;
+}
+
+void mail_cache_purge_later(struct mail_cache *cache, const char *reason)
+{
+	i_assert(cache->hdr != NULL);
+
+	cache->need_purge_file_seq = cache->hdr->file_seq;
+	i_free(cache->need_purge_reason);
+	cache->need_purge_reason = i_strdup(reason);
+}
+
+void mail_cache_purge_later_reset(struct mail_cache *cache)
+{
+	cache->need_purge_file_seq = 0;
+	i_free(cache->need_purge_reason);
+}
+
+void mail_cache_purge_drop_init(struct mail_cache *cache,
+				const struct mail_index_header *hdr,
+				struct mail_cache_purge_drop_ctx *ctx_r)
+{
+	i_zero(ctx_r);
+	ctx_r->cache = cache;
+	if (hdr->day_stamp != 0) {
+		const struct mail_index_cache_optimization_settings *opt =
+			&cache->index->optimization_set.cache;
+		ctx_r->max_yes_downgrade_time = hdr->day_stamp -
+			opt->unaccessed_field_drop_secs;
+		ctx_r->max_temp_drop_time = hdr->day_stamp -
+			2 * opt->unaccessed_field_drop_secs;
+	}
+}
+
+enum mail_cache_purge_drop_decision
+mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx,
+			   unsigned int field)
+{
+	struct mail_cache_field_private *priv = &ctx->cache->fields[field];
+	enum mail_cache_decision_type dec = priv->field.decision;
+
+	if ((dec & MAIL_CACHE_DECISION_FORCED) != 0)
+		return MAIL_CACHE_PURGE_DROP_DECISION_NONE;
+	if (dec != MAIL_CACHE_DECISION_NO &&
+	    priv->field.last_used < ctx->max_temp_drop_time) {
+		/* YES or TEMP decision field hasn't been accessed for a long
+		   time now. Drop it. */
+		return MAIL_CACHE_PURGE_DROP_DECISION_DROP;
+	}
+	if (dec == MAIL_CACHE_DECISION_YES &&
+	    priv->field.last_used < ctx->max_yes_downgrade_time) {
+		/* YES decision field hasn't been accessed for a while
+		   now. Change its decision to TEMP. */
+		return MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP;
+	}
+	return MAIL_CACHE_PURGE_DROP_DECISION_NONE;
+}