diff options
Diffstat (limited to '')
| -rw-r--r-- | src/lib-index/mail-cache-purge.c | 707 |
1 files changed, 707 insertions, 0 deletions
diff --git a/src/lib-index/mail-cache-purge.c b/src/lib-index/mail-cache-purge.c new file mode 100644 index 0000000..bc86bf1 --- /dev/null +++ b/src/lib-index/mail-cache-purge.c @@ -0,0 +1,707 @@ +/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ostream.h" +#include "nfs-workarounds.h" +#include "read-full.h" +#include "file-dotlock.h" +#include "file-cache.h" +#include "file-set-size.h" +#include "mail-cache-private.h" + +#include <stdio.h> +#include <sys/stat.h> + +struct mail_cache_copy_context { + struct mail_cache *cache; + struct event *event; + struct mail_cache_purge_drop_ctx drop_ctx; + + buffer_t *buffer, *field_seen; + ARRAY(unsigned int) bitmask_pos; + uint32_t *field_file_map; + + uint8_t field_seen_value; + bool new_msg; +}; + +static void +mail_cache_merge_bitmask(struct mail_cache_copy_context *ctx, + const struct mail_cache_iterate_field *field) +{ + unsigned char *dest; + unsigned int i, *pos; + + pos = array_idx_get_space(&ctx->bitmask_pos, field->field_idx); + if (*pos == 0) { + /* we decided to drop this field */ + return; + } + + dest = buffer_get_space_unsafe(ctx->buffer, *pos, field->size); + for (i = 0; i < field->size; i++) + dest[i] |= ((const unsigned char*)field->data)[i]; +} + +static void +mail_cache_purge_field(struct mail_cache_copy_context *ctx, + const struct mail_cache_iterate_field *field) +{ + struct mail_cache_field *cache_field; + enum mail_cache_decision_type dec; + uint32_t file_field_idx, size32; + uint8_t *field_seen; + + file_field_idx = ctx->field_file_map[field->field_idx]; + if (file_field_idx == (uint32_t)-1) + return; + + cache_field = &ctx->cache->fields[field->field_idx].field; + + field_seen = buffer_get_space_unsafe(ctx->field_seen, + field->field_idx, 1); + if (*field_seen == ctx->field_seen_value) { + /* duplicate */ + if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) + mail_cache_merge_bitmask(ctx, field); + return; + } + *field_seen = ctx->field_seen_value; + + dec = cache_field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED); + if (ctx->new_msg) { + if (dec == MAIL_CACHE_DECISION_NO) + return; + } else { + if (dec != MAIL_CACHE_DECISION_YES) + return; + } + + buffer_append(ctx->buffer, &file_field_idx, sizeof(file_field_idx)); + + if (cache_field->field_size == UINT_MAX) { + size32 = (uint32_t)field->size; + buffer_append(ctx->buffer, &size32, sizeof(size32)); + } + + if (cache_field->type == MAIL_CACHE_FIELD_BITMASK) { + /* remember the position in case we need to update it */ + unsigned int pos = ctx->buffer->used; + + array_idx_set(&ctx->bitmask_pos, field->field_idx, &pos); + } + buffer_append(ctx->buffer, field->data, field->size); + if ((field->size & 3) != 0) + buffer_append_zero(ctx->buffer, 4 - (field->size & 3)); +} + +static uint32_t get_next_file_seq(struct mail_cache *cache) +{ + const struct mail_index_ext *ext; + struct mail_index_view *view; + uint32_t file_seq; + + /* make sure we look up the latest reset_id */ + if (mail_index_refresh(cache->index) < 0) + return -1; + + view = mail_index_view_open(cache->index); + ext = mail_index_view_get_ext(view, cache->ext_id); + file_seq = ext != NULL ? ext->reset_id + 1 : (uint32_t)ioloop_time; + + if (cache->hdr != NULL && file_seq <= cache->hdr->file_seq) + file_seq = cache->hdr->file_seq + 1; + mail_index_view_close(&view); + + return file_seq != 0 ? file_seq : 1; +} + +static void +mail_cache_purge_get_fields(struct mail_cache_copy_context *ctx, + unsigned int used_fields_count) +{ + struct mail_cache *cache = ctx->cache; + unsigned int i, j, idx; + + /* Make mail_cache_header_fields_get() return the fields in + the same order as we saved them. */ + memcpy(cache->field_file_map, ctx->field_file_map, + sizeof(uint32_t) * cache->fields_count); + + /* reverse mapping */ + cache->file_fields_count = used_fields_count; + i_free(cache->file_field_map); + cache->file_field_map = used_fields_count == 0 ? NULL : + i_new(unsigned int, used_fields_count); + for (i = j = 0; i < cache->fields_count; i++) { + idx = cache->field_file_map[i]; + if (idx != (uint32_t)-1) { + i_assert(idx < used_fields_count && + cache->file_field_map != NULL && + cache->file_field_map[idx] == 0); + cache->file_field_map[idx] = i; + j++; + } + } + i_assert(j == used_fields_count); + + buffer_set_used_size(ctx->buffer, 0); + mail_cache_header_fields_get(cache, ctx->buffer); +} + +static bool +mail_cache_purge_check_field(struct mail_cache_copy_context *ctx, + unsigned int field) +{ + struct mail_cache_field_private *priv = &ctx->cache->fields[field]; + enum mail_cache_decision_type dec = priv->field.decision; + + switch (mail_cache_purge_drop_test(&ctx->drop_ctx, field)) { + case MAIL_CACHE_PURGE_DROP_DECISION_NONE: + break; + case MAIL_CACHE_PURGE_DROP_DECISION_DROP: { + const char *dec_str = mail_cache_decision_to_string(dec); + struct event_passthrough *e = + event_create_passthrough(ctx->event)-> + set_name("mail_cache_purge_drop_field")-> + add_str("field", priv->field.name)-> + add_str("decision", dec_str)-> + add_int("last_used", priv->field.last_used); + e_debug(e->event(), "Purge dropped field %s " + "(decision=%s, last_used=%"PRIdTIME_T")", + priv->field.name, dec_str, priv->field.last_used); + dec = MAIL_CACHE_DECISION_NO; + break; + } + case MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP: { + struct event_passthrough *e = + mail_cache_decision_changed_event( + ctx->cache, ctx->event, field)-> + add_str("old_decision", "yes")-> + add_str("new_decision", "temp"); + e_debug(e->event(), "Purge changes field %s " + "cache decision yes -> temp " + "(last_used=%"PRIdTIME_T")", + priv->field.name, priv->field.last_used); + dec = MAIL_CACHE_DECISION_TEMP; + break; + } + } + priv->field.decision = dec; + + /* drop all fields we don't want */ + if ((dec & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) == MAIL_CACHE_DECISION_NO) { + priv->used = FALSE; + priv->field.last_used = 0; + } + return priv->used; +} + +static int +mail_cache_copy(struct mail_cache *cache, struct mail_index_transaction *trans, + struct event *event, int fd, const char *reason, + uint32_t *file_seq_r, uoff_t *file_size_r, uint32_t *max_uid_r, + uint32_t *ext_first_seq_r, ARRAY_TYPE(uint32_t) *ext_offsets) +{ + struct mail_cache_copy_context ctx; + struct mail_cache_lookup_iterate_ctx iter; + struct mail_cache_iterate_field field; + struct mail_index_view *view; + struct mail_cache_view *cache_view; + const struct mail_index_header *idx_hdr; + struct mail_cache_header hdr; + struct mail_cache_record cache_rec; + struct ostream *output; + uint32_t message_count, seq, first_new_seq, ext_offset; + unsigned int i, used_fields_count, orig_fields_count, record_count; + + i_assert(reason != NULL); + + *max_uid_r = 0; + *ext_first_seq_r = 0; + + /* get the latest info on fields */ + if (mail_cache_header_fields_read(cache) < 0) + return -1; + + view = mail_index_transaction_open_updated_view(trans); + cache_view = mail_cache_view_open(cache, view); + output = o_stream_create_fd_file(fd, 0, FALSE); + + i_zero(&hdr); + hdr.major_version = MAIL_CACHE_MAJOR_VERSION; + hdr.minor_version = MAIL_CACHE_MINOR_VERSION; + hdr.compat_sizeof_uoff_t = sizeof(uoff_t); + hdr.indexid = cache->index->indexid; + hdr.file_seq = get_next_file_seq(cache); + o_stream_nsend(output, &hdr, sizeof(hdr)); + + event_add_str(event, "reason", reason); + event_add_int(event, "file_seq", hdr.file_seq); + event_set_name(event, "mail_cache_purge_started"); + e_debug(event, "Purging (new file_seq=%u): %s", hdr.file_seq, reason); + + i_zero(&ctx); + ctx.cache = cache; + ctx.event = event; + ctx.buffer = buffer_create_dynamic(default_pool, 4096); + ctx.field_seen = buffer_create_dynamic(default_pool, 64); + ctx.field_seen_value = 0; + ctx.field_file_map = t_new(uint32_t, cache->fields_count + 1); + t_array_init(&ctx.bitmask_pos, 32); + + /* @UNSAFE: drop unused fields and create a field mapping for + used fields */ + idx_hdr = mail_index_get_header(view); + mail_cache_purge_drop_init(cache, idx_hdr, &ctx.drop_ctx); + + orig_fields_count = cache->fields_count; + if (cache->file_fields_count == 0) { + /* creating the initial cache file. add all fields. */ + for (i = 0; i < orig_fields_count; i++) + ctx.field_file_map[i] = i; + used_fields_count = i; + } else { + for (i = used_fields_count = 0; i < orig_fields_count; i++) { + if (!mail_cache_purge_check_field(&ctx, i)) + ctx.field_file_map[i] = (uint32_t)-1; + else + ctx.field_file_map[i] = used_fields_count++; + } + } + + /* get sequence of first message which doesn't need its temp fields + removed. */ + first_new_seq = mail_cache_get_first_new_seq(view); + message_count = mail_index_view_get_messages_count(view); + if (!trans->reset) + seq = 1; + else { + /* Index is being rebuilt. Ignore old messages. */ + seq = trans->first_new_seq; + } + + *ext_first_seq_r = seq; + i_array_init(ext_offsets, message_count); record_count = 0; + for (; seq <= message_count; seq++) { + if (mail_index_transaction_is_expunged(trans, seq)) { + array_append_zero(ext_offsets); + continue; + } + + ctx.new_msg = seq >= first_new_seq; + buffer_set_used_size(ctx.buffer, 0); + + ctx.field_seen_value = (ctx.field_seen_value + 1) & UINT8_MAX; + if (ctx.field_seen_value == 0) { + memset(buffer_get_modifiable_data(ctx.field_seen, NULL), + 0, buffer_get_size(ctx.field_seen)); + ctx.field_seen_value++; + } + array_clear(&ctx.bitmask_pos); + + i_zero(&cache_rec); + buffer_append(ctx.buffer, &cache_rec, sizeof(cache_rec)); + + mail_cache_lookup_iter_init(cache_view, seq, &iter); + while (mail_cache_lookup_iter_next(&iter, &field) > 0) + mail_cache_purge_field(&ctx, &field); + + if (ctx.buffer->used == sizeof(cache_rec) || + ctx.buffer->used > cache->index->optimization_set.cache.record_max_size) { + /* nothing cached */ + ext_offset = 0; + } else { + mail_index_lookup_uid(view, seq, max_uid_r); + cache_rec.size = ctx.buffer->used; + ext_offset = output->offset; + buffer_write(ctx.buffer, 0, &cache_rec, + sizeof(cache_rec)); + o_stream_nsend(output, ctx.buffer->data, cache_rec.size); + record_count++; + } + + array_push_back(ext_offsets, &ext_offset); + } + i_assert(orig_fields_count == cache->fields_count); + + hdr.record_count = record_count; + hdr.field_header_offset = mail_index_uint32_to_offset(output->offset); + mail_cache_purge_get_fields(&ctx, used_fields_count); + o_stream_nsend(output, ctx.buffer->data, ctx.buffer->used); + + hdr.backwards_compat_used_file_size = output->offset; + buffer_free(&ctx.buffer); + buffer_free(&ctx.field_seen); + + *file_size_r = output->offset; + (void)o_stream_seek(output, 0); + o_stream_nsend(output, &hdr, sizeof(hdr)); + + mail_cache_view_close(&cache_view); + mail_index_view_close(&view); + + if (o_stream_finish(output) < 0) { + mail_cache_set_syscall_error(cache, "write()"); + o_stream_destroy(&output); + array_free(ext_offsets); + return -1; + } + o_stream_destroy(&output); + + if (cache->index->set.fsync_mode == FSYNC_MODE_ALWAYS) { + if (fdatasync(fd) < 0) { + mail_cache_set_syscall_error(cache, "fdatasync()"); + array_free(ext_offsets); + return -1; + } + } + + *file_seq_r = hdr.file_seq; + return 0; +} + +static int +mail_cache_purge_write(struct mail_cache *cache, + struct mail_index_transaction *trans, + int fd, const char *temp_path, const char * + reason, bool *unlock) +{ + struct event *event; + struct stat st; + uint32_t prev_file_seq, file_seq, old_offset, max_uid, ext_first_seq; + ARRAY_TYPE(uint32_t) ext_offsets; + const uint32_t *offsets; + uoff_t prev_file_size, file_size; + unsigned int i, count, prev_deleted_records; + + if (cache->hdr == NULL) { + prev_file_seq = 0; + prev_file_size = 0; + prev_deleted_records = 0; + } else { + prev_file_seq = cache->hdr->file_seq; + prev_file_size = cache->last_stat_size; + prev_deleted_records = cache->hdr->deleted_record_count; + } + event = event_create(cache->event); + event_add_int(event, "prev_file_seq", prev_file_seq); + event_add_int(event, "prev_file_size", prev_file_size); + event_add_int(event, "prev_deleted_records", prev_deleted_records); + + if (mail_cache_copy(cache, trans, event, fd, reason, + &file_seq, &file_size, &max_uid, + &ext_first_seq, &ext_offsets) < 0) + return -1; + + if (fstat(fd, &st) < 0) { + mail_cache_set_syscall_error(cache, "fstat()"); + array_free(&ext_offsets); + return -1; + } + if (rename(temp_path, cache->filepath) < 0) { + mail_cache_set_syscall_error(cache, "rename()"); + array_free(&ext_offsets); + return -1; + } + + event_add_int(event, "file_size", file_size); + event_add_int(event, "max_uid", max_uid); + event_set_name(event, "mail_cache_purge_finished"); + e_debug(event, "Purging finished, file_seq changed %u -> %u, " + "size=%"PRIuUOFF_T" -> %"PRIuUOFF_T", max_uid=%u", + prev_file_seq, file_seq, prev_file_size, file_size, max_uid); + event_unref(&event); + + /* once we're sure that the purging was successful, + update the offsets */ + mail_index_ext_reset(trans, cache->ext_id, file_seq, TRUE); + offsets = array_get(&ext_offsets, &count); + for (i = 0; i < count; i++) { + if (offsets[i] != 0) { + mail_index_update_ext(trans, ext_first_seq + i, + cache->ext_id, + &offsets[i], &old_offset); + } + } + array_free(&ext_offsets); + + if (*unlock) { + mail_cache_unlock(cache); + *unlock = FALSE; + } + + mail_cache_file_close(cache); + cache->opened = TRUE; + cache->fd = fd; + cache->st_ino = st.st_ino; + cache->st_dev = st.st_dev; + cache->field_header_write_pending = FALSE; + return 0; +} + +static int +mail_cache_purge_has_file_changed(struct mail_cache *cache, + uint32_t purge_file_seq) +{ + struct mail_cache_header hdr; + unsigned int i; + int fd, ret; + + for (i = 0;; i++) { + fd = nfs_safe_open(cache->filepath, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) + return 0; + + mail_cache_set_syscall_error(cache, "open()"); + return -1; + } + + ret = read_full(fd, &hdr, sizeof(hdr)); + i_close_fd(&fd); + + if (ret >= 0) { + if (ret == 0) + return 0; + if (purge_file_seq == 0) { + /* previously it didn't exist or it + was unusable and was just unlinked */ + return 1; + } + return hdr.file_seq != purge_file_seq ? 1 : 0; + } else if (errno != ESTALE || i >= NFS_ESTALE_RETRY_COUNT) { + mail_cache_set_syscall_error(cache, "read()"); + return -1; + } + } +} + +static int mail_cache_purge_locked(struct mail_cache *cache, + uint32_t purge_file_seq, + struct mail_index_transaction *trans, + const char *reason, bool *unlock) +{ + const char *temp_path; + int fd, ret; + + /* we've locked the cache purging now. if somebody else had just + recreated the cache, reopen the cache and return success. */ + if (purge_file_seq != (uint32_t)-1 && + (ret = mail_cache_purge_has_file_changed(cache, purge_file_seq)) != 0) { + if (ret < 0) + return -1; + + /* was just purged, forget this */ + mail_cache_purge_later_reset(cache); + + if (*unlock) { + (void)mail_cache_unlock(cache); + *unlock = FALSE; + } + + return mail_cache_reopen(cache) < 0 ? -1 : 0; + } + if (cache->fd != -1) { + /* make sure we have mapped it before reading. */ + if (mail_cache_map_all(cache) <= 0) + return -1; + } + + /* we want to recreate the cache. write it first to a temporary file */ + fd = mail_index_create_tmp_file(cache->index, cache->filepath, &temp_path); + if (fd == -1) + return -1; + if (mail_cache_purge_write(cache, trans, fd, temp_path, reason, unlock) < 0) { + i_close_fd(&fd); + i_unlink(temp_path); + return -1; + } + if (cache->file_cache != NULL) + file_cache_set_fd(cache->file_cache, cache->fd); + + if (mail_cache_map_all(cache) <= 0) + return -1; + if (mail_cache_header_fields_read(cache) < 0) + return -1; + + mail_cache_purge_later_reset(cache); + return 0; +} + +static int +mail_cache_purge_full(struct mail_cache *cache, + struct mail_index_transaction *trans, + uint32_t purge_file_seq, const char *reason) +{ + bool unlock = FALSE; + int ret; + + i_assert(!cache->purging); + i_assert(cache->index->log_sync_locked); + + if (MAIL_INDEX_IS_IN_MEMORY(cache->index) || cache->index->readonly) + return 0; + + /* purging isn't very efficient with small read()s */ + if (cache->map_with_read) { + cache->map_with_read = FALSE; + if (cache->read_buf != NULL) + buffer_set_used_size(cache->read_buf, 0); + cache->hdr = NULL; + cache->mmap_length = 0; + } + + /* .log lock already prevents other processes from purging cache at + the same time, but locking the cache file itself prevents other + processes from doing other changes to it (header changes, adding + more cached data). */ + switch (mail_cache_lock(cache)) { + case -1: + /* lock timeout or some other error */ + return -1; + case 0: + /* cache is broken or doesn't exist. + just start creating it. */ + break; + default: + /* locking succeeded. */ + unlock = TRUE; + } + cache->purging = TRUE; + ret = mail_cache_purge_locked(cache, purge_file_seq, trans, reason, &unlock); + cache->purging = FALSE; + if (unlock) + mail_cache_unlock(cache); + i_assert(!cache->hdr_modified); + if (ret < 0) { + /* the fields may have been updated in memory already. + reverse those changes by re-reading them from file. */ + (void)mail_cache_header_fields_read(cache); + } + return ret; +} + +int mail_cache_purge_with_trans(struct mail_cache *cache, + struct mail_index_transaction *trans, + uint32_t purge_file_seq, const char *reason) +{ + return mail_cache_purge_full(cache, trans, purge_file_seq, reason); +} + +int mail_cache_purge(struct mail_cache *cache, uint32_t purge_file_seq, + const char *reason) +{ + struct mail_index_view *view; + struct mail_index_transaction *trans; + bool lock_log; + int ret; + + lock_log = !cache->index->log_sync_locked; + if (lock_log) { + uint32_t file_seq; + uoff_t file_offset; + + if (mail_transaction_log_sync_lock(cache->index->log, + "mail cache purge", + &file_seq, &file_offset) < 0) + return -1; + } + /* make sure we see the latest changes in index */ + ret = mail_index_refresh(cache->index); + + view = mail_index_view_open(cache->index); + trans = mail_index_transaction_begin(view, + MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + if (ret < 0) + ; + else if ((ret = mail_cache_purge_full(cache, trans, purge_file_seq, + reason)) < 0) + mail_index_transaction_rollback(&trans); + else { + if (mail_index_transaction_commit(&trans) < 0) + ret = -1; + } + mail_index_view_close(&view); + if (lock_log) { + mail_transaction_log_sync_unlock(cache->index->log, + "mail cache purge"); + } + return ret; +} + +bool mail_cache_need_purge(struct mail_cache *cache, const char **reason_r) +{ + if (cache->need_purge_file_seq == 0) + return FALSE; /* delayed purging not requested */ + if (cache->index->readonly) + return FALSE; /* no purging when opened as read-only */ + if ((cache->index->flags & MAIL_INDEX_OPEN_FLAG_SAVEONLY) != 0) { + /* Mail deliveries don't really need to purge, even if there + could be some work to do. Just delay until the next regular + mail access comes before doing any extra work. */ + return FALSE; + } + + i_assert(cache->need_purge_reason != NULL); + /* t_strdup() the reason in case it gets freed (or replaced) + before it's used */ + *reason_r = t_strdup(cache->need_purge_reason); + return TRUE; +} + +void mail_cache_purge_later(struct mail_cache *cache, const char *reason) +{ + i_assert(cache->hdr != NULL); + + cache->need_purge_file_seq = cache->hdr->file_seq; + i_free(cache->need_purge_reason); + cache->need_purge_reason = i_strdup(reason); +} + +void mail_cache_purge_later_reset(struct mail_cache *cache) +{ + cache->need_purge_file_seq = 0; + i_free(cache->need_purge_reason); +} + +void mail_cache_purge_drop_init(struct mail_cache *cache, + const struct mail_index_header *hdr, + struct mail_cache_purge_drop_ctx *ctx_r) +{ + i_zero(ctx_r); + ctx_r->cache = cache; + if (hdr->day_stamp != 0) { + const struct mail_index_cache_optimization_settings *opt = + &cache->index->optimization_set.cache; + ctx_r->max_yes_downgrade_time = hdr->day_stamp - + opt->unaccessed_field_drop_secs; + ctx_r->max_temp_drop_time = hdr->day_stamp - + 2 * opt->unaccessed_field_drop_secs; + } +} + +enum mail_cache_purge_drop_decision +mail_cache_purge_drop_test(struct mail_cache_purge_drop_ctx *ctx, + unsigned int field) +{ + struct mail_cache_field_private *priv = &ctx->cache->fields[field]; + enum mail_cache_decision_type dec = priv->field.decision; + + if ((dec & MAIL_CACHE_DECISION_FORCED) != 0) + return MAIL_CACHE_PURGE_DROP_DECISION_NONE; + if (dec != MAIL_CACHE_DECISION_NO && + priv->field.last_used < ctx->max_temp_drop_time) { + /* YES or TEMP decision field hasn't been accessed for a long + time now. Drop it. */ + return MAIL_CACHE_PURGE_DROP_DECISION_DROP; + } + if (dec == MAIL_CACHE_DECISION_YES && + priv->field.last_used < ctx->max_yes_downgrade_time) { + /* YES decision field hasn't been accessed for a while + now. Change its decision to TEMP. */ + return MAIL_CACHE_PURGE_DROP_DECISION_TO_TEMP; + } + return MAIL_CACHE_PURGE_DROP_DECISION_NONE; +} |