summaryrefslogtreecommitdiffstats
path: root/src/lib-index/mail-index-map-read.c
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--src/lib-index/mail-index-map-read.c519
1 files changed, 519 insertions, 0 deletions
diff --git a/src/lib-index/mail-index-map-read.c b/src/lib-index/mail-index-map-read.c
new file mode 100644
index 0000000..6999bf1
--- /dev/null
+++ b/src/lib-index/mail-index-map-read.c
@@ -0,0 +1,519 @@
+/* Copyright (c) 2003-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "nfs-workarounds.h"
+#include "mmap-util.h"
+#include "read-full.h"
+#include "mail-index-private.h"
+#include "mail-index-sync-private.h"
+#include "mail-transaction-log-private.h"
+#include "mail-index-modseq.h"
+#include "ioloop.h"
+
+static void mail_index_map_copy_hdr(struct mail_index_map *map,
+ const struct mail_index_header *hdr)
+{
+ if (hdr->base_header_size < sizeof(map->hdr)) {
+ /* header smaller than ours, make a copy so our newer headers
+ won't have garbage in them */
+ i_zero(&map->hdr);
+ memcpy(&map->hdr, hdr, hdr->base_header_size);
+ } else {
+ map->hdr = *hdr;
+ }
+
+ /* FIXME: backwards compatibility, remove later. In case this index is
+ accessed with Dovecot v1.0, avoid recent message counter errors. */
+ map->hdr.unused_old_recent_messages_count = 0;
+}
+
+static int mail_index_mmap(struct mail_index_map *map, uoff_t file_size)
+{
+ struct mail_index *index = map->index;
+ struct mail_index_record_map *rec_map = map->rec_map;
+ const struct mail_index_header *hdr;
+ const char *error;
+
+ i_assert(rec_map->mmap_base == NULL);
+
+ buffer_free(&rec_map->buffer);
+ if (file_size > SSIZE_T_MAX) {
+ /* too large file to map into memory */
+ mail_index_set_error(index, "Index file too large: %s",
+ index->filepath);
+ return -1;
+ }
+
+ rec_map->mmap_base = mmap(NULL, file_size, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE, index->fd, 0);
+ if (rec_map->mmap_base == MAP_FAILED) {
+ rec_map->mmap_base = NULL;
+ if (ioloop_time != index->last_mmap_error_time) {
+ index->last_mmap_error_time = ioloop_time;
+ mail_index_set_syscall_error(index, t_strdup_printf(
+ "mmap(size=%"PRIuUOFF_T")", file_size));
+ }
+ return -1;
+ }
+ rec_map->mmap_size = file_size;
+
+ hdr = rec_map->mmap_base;
+ if (rec_map->mmap_size >
+ offsetof(struct mail_index_header, major_version) &&
+ hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
+ /* major version change - handle silently */
+ return 0;
+ }
+
+ if (rec_map->mmap_size < MAIL_INDEX_HEADER_MIN_SIZE) {
+ mail_index_set_error(index, "Corrupted index file %s: "
+ "File too small (%zu)",
+ index->filepath, rec_map->mmap_size);
+ return 0;
+ }
+
+ if (!mail_index_check_header_compat(index, hdr, rec_map->mmap_size, &error)) {
+ /* Can't use this file */
+ mail_index_set_error(index, "Corrupted index file %s: %s",
+ index->filepath, error);
+ return 0;
+ }
+
+ rec_map->mmap_used_size = hdr->header_size +
+ hdr->messages_count * hdr->record_size;
+
+ if (rec_map->mmap_used_size <= rec_map->mmap_size)
+ rec_map->records_count = hdr->messages_count;
+ else {
+ rec_map->records_count =
+ (rec_map->mmap_size - hdr->header_size) /
+ hdr->record_size;
+ rec_map->mmap_used_size = hdr->header_size +
+ rec_map->records_count * hdr->record_size;
+ mail_index_set_error(index, "Corrupted index file %s: "
+ "messages_count too large (%u > %u)",
+ index->filepath, hdr->messages_count,
+ rec_map->records_count);
+ }
+
+ mail_index_map_copy_hdr(map, hdr);
+ buffer_set_used_size(map->hdr_copy_buf, 0);
+ buffer_append(map->hdr_copy_buf, rec_map->mmap_base, hdr->header_size);
+
+ rec_map->records = PTR_OFFSET(rec_map->mmap_base, map->hdr.header_size);
+ return 1;
+}
+
+static int mail_index_read_header(struct mail_index *index,
+ void *buf, size_t buf_size, size_t *pos_r)
+{
+ size_t pos;
+ int ret;
+
+ memset(buf, 0, sizeof(struct mail_index_header));
+
+ /* try to read the whole header, but it's not necessarily an error to
+ read less since the older versions of the index format could be
+ smaller. Request reading up to buf_size, but accept if we only got
+ the header. */
+ pos = 0;
+ do {
+ ret = pread(index->fd, PTR_OFFSET(buf, pos),
+ buf_size - pos, pos);
+ if (ret > 0)
+ pos += ret;
+ } while (ret > 0 && pos < sizeof(struct mail_index_header));
+
+ *pos_r = pos;
+ return ret;
+}
+
+static int
+mail_index_try_read_map(struct mail_index_map *map,
+ uoff_t file_size, bool *retry_r, bool try_retry)
+{
+ struct mail_index *index = map->index;
+ const struct mail_index_header *hdr;
+ unsigned char read_buf[IO_BLOCK_SIZE];
+ const char *error;
+ const void *buf;
+ void *data = NULL;
+ ssize_t ret;
+ size_t pos, records_size, initial_buf_pos = 0;
+ unsigned int records_count = 0, extra;
+
+ i_assert(map->rec_map->mmap_base == NULL);
+
+ *retry_r = FALSE;
+ ret = mail_index_read_header(index, read_buf, sizeof(read_buf), &pos);
+ buf = read_buf; hdr = buf;
+
+ if (pos > (ssize_t)offsetof(struct mail_index_header, major_version) &&
+ hdr->major_version != MAIL_INDEX_MAJOR_VERSION) {
+ /* major version change - handle silently */
+ return 0;
+ }
+
+ if (ret >= 0 && pos >= MAIL_INDEX_HEADER_MIN_SIZE &&
+ (ret > 0 || pos >= hdr->base_header_size)) {
+ if (!mail_index_check_header_compat(index, hdr, file_size, &error)) {
+ /* Can't use this file */
+ mail_index_set_error(index, "Corrupted index file %s: %s",
+ index->filepath, error);
+ return 0;
+ }
+
+ initial_buf_pos = pos;
+ if (pos > hdr->header_size)
+ pos = hdr->header_size;
+
+ /* place the base header into memory. */
+ buffer_set_used_size(map->hdr_copy_buf, 0);
+ buffer_append(map->hdr_copy_buf, buf, pos);
+
+ if (pos != hdr->header_size) {
+ /* @UNSAFE: read the rest of the header into memory */
+ data = buffer_append_space_unsafe(map->hdr_copy_buf,
+ hdr->header_size -
+ pos);
+ ret = pread_full(index->fd, data,
+ hdr->header_size - pos, pos);
+ }
+ }
+
+ if (ret > 0) {
+ /* header read, read the records now. */
+ records_size = (size_t)hdr->messages_count * hdr->record_size;
+ records_count = hdr->messages_count;
+
+ if (file_size - hdr->header_size < records_size ||
+ (hdr->record_size != 0 &&
+ records_size / hdr->record_size != hdr->messages_count)) {
+ records_count = (file_size - hdr->header_size) /
+ hdr->record_size;
+ records_size = (size_t)records_count * hdr->record_size;
+ mail_index_set_error(index, "Corrupted index file %s: "
+ "messages_count too large (%u > %u)",
+ index->filepath, hdr->messages_count,
+ records_count);
+ }
+
+ if (map->rec_map->buffer == NULL) {
+ map->rec_map->buffer =
+ buffer_create_dynamic(default_pool,
+ records_size);
+ }
+
+ /* @UNSAFE */
+ buffer_set_used_size(map->rec_map->buffer, 0);
+ if (initial_buf_pos <= hdr->header_size)
+ extra = 0;
+ else {
+ extra = initial_buf_pos - hdr->header_size;
+ buffer_append(map->rec_map->buffer,
+ CONST_PTR_OFFSET(buf, hdr->header_size),
+ extra);
+ }
+ if (records_size > extra) {
+ data = buffer_append_space_unsafe(map->rec_map->buffer,
+ records_size - extra);
+ ret = pread_full(index->fd, data, records_size - extra,
+ hdr->header_size + extra);
+ }
+ }
+
+ if (ret < 0) {
+ if (errno == ESTALE && try_retry) {
+ /* a new index file was renamed over this one. */
+ *retry_r = TRUE;
+ return 0;
+ }
+ mail_index_set_syscall_error(index, "pread_full()");
+ return -1;
+ }
+ if (ret == 0) {
+ mail_index_set_error(index,
+ "Corrupted index file %s: File too small",
+ index->filepath);
+ return 0;
+ }
+
+ map->rec_map->records =
+ buffer_get_modifiable_data(map->rec_map->buffer, NULL);
+ map->rec_map->records_count = records_count;
+
+ mail_index_map_copy_hdr(map, hdr);
+ i_assert(map->hdr_copy_buf->used == map->hdr.header_size);
+ return 1;
+}
+
+static int mail_index_read_map(struct mail_index_map *map, uoff_t file_size)
+{
+ struct mail_index *index = map->index;
+ struct stat st;
+ unsigned int i;
+ int ret;
+ bool try_retry, retry;
+
+ for (i = 0;; i++) {
+ try_retry = i < MAIL_INDEX_ESTALE_RETRY_COUNT;
+ if (file_size == UOFF_T_MAX) {
+ /* fstat() below failed */
+ ret = 0;
+ retry = try_retry;
+ } else {
+ ret = mail_index_try_read_map(map, file_size,
+ &retry, try_retry);
+ }
+ if (ret != 0 || !retry)
+ break;
+
+ /* ESTALE - reopen index file */
+ mail_index_close_file(index);
+
+ ret = mail_index_try_open_only(index);
+ if (ret <= 0) {
+ if (ret == 0) {
+ /* the file was lost */
+ errno = ENOENT;
+ mail_index_set_syscall_error(index, "open()");
+ }
+ return -1;
+ }
+
+ if (fstat(index->fd, &st) == 0)
+ file_size = st.st_size;
+ else {
+ if (!ESTALE_FSTAT(errno)) {
+ mail_index_set_syscall_error(index, "fstat()");
+ return -1;
+ }
+ file_size = UOFF_T_MAX;
+ }
+ }
+ return ret;
+}
+
+/* returns -1 = error, 0 = index files are unusable,
+ 1 = index files are usable or at least repairable */
+static int
+mail_index_map_latest_file(struct mail_index *index, const char **reason_r)
+{
+ struct mail_index_map *old_map, *new_map;
+ struct stat st;
+ uoff_t file_size;
+ bool use_mmap, reopened, unusable = FALSE;
+ const char *error;
+ int ret, try;
+
+ *reason_r = NULL;
+
+ index->reopen_main_index = FALSE;
+ ret = mail_index_reopen_if_changed(index, &reopened, reason_r);
+ if (ret <= 0) {
+ if (ret < 0)
+ return -1;
+
+ /* the index file is lost/broken. let's hope that we can
+ build it from the transaction log. */
+ return 1;
+ }
+ i_assert(index->fd != -1);
+
+ if ((index->flags & MAIL_INDEX_OPEN_FLAG_NFS_FLUSH) != 0)
+ nfs_flush_attr_cache_fd_locked(index->filepath, index->fd);
+
+ if (fstat(index->fd, &st) == 0)
+ file_size = st.st_size;
+ else {
+ if (!ESTALE_FSTAT(errno)) {
+ mail_index_set_syscall_error(index, "fstat()");
+ return -1;
+ }
+ file_size = UOFF_T_MAX;
+ }
+
+ /* mmaping seems to be slower than just reading the file, so even if
+ mmap isn't disabled don't use it unless the file is large enough */
+ use_mmap = (index->flags & MAIL_INDEX_OPEN_FLAG_MMAP_DISABLE) == 0 &&
+ file_size != UOFF_T_MAX && file_size > MAIL_INDEX_MMAP_MIN_SIZE;
+
+ new_map = mail_index_map_alloc(index);
+ if (use_mmap) {
+ ret = mail_index_mmap(new_map, file_size);
+ } else {
+ ret = mail_index_read_map(new_map, file_size);
+ }
+ if (ret == 0) {
+ /* the index files are unusable */
+ unusable = TRUE;
+ }
+
+ for (try = 0; ret > 0; try++) {
+ /* make sure the header is ok before using this mapping */
+ ret = mail_index_map_check_header(new_map, &error);
+ if (ret < 0) {
+ mail_index_set_error(index,
+ "Corrupted index file %s: %s",
+ index->filepath, error);
+ }
+ if (ret > 0) T_BEGIN {
+ if (mail_index_map_parse_extensions(new_map) < 0)
+ ret = 0;
+ else if (mail_index_map_parse_keywords(new_map) < 0)
+ ret = 0;
+ } T_END;
+ if (ret != 0 || try == 2) {
+ if (ret < 0) {
+ *reason_r = "Corrupted index file";
+ unusable = TRUE;
+ ret = 0;
+ }
+ break;
+ }
+
+ /* fsck and try again */
+ old_map = index->map;
+ index->map = new_map;
+ if (mail_index_fsck(index) < 0) {
+ ret = -1;
+ break;
+ }
+
+ /* fsck replaced the map */
+ new_map = index->map;
+ index->map = old_map;
+ }
+ if (ret <= 0) {
+ mail_index_unmap(&new_map);
+ return ret < 0 ? -1 : (unusable ? 0 : 1);
+ }
+ i_assert(new_map->rec_map->records != NULL);
+
+ index->main_index_hdr_log_file_seq = new_map->hdr.log_file_seq;
+ index->main_index_hdr_log_file_tail_offset =
+ new_map->hdr.log_file_tail_offset;
+ mail_index_modseq_hdr_snapshot_update(new_map);
+
+ mail_index_unmap(&index->map);
+ index->map = new_map;
+ *reason_r = t_strdup_printf("Index mapped (file_seq=%u)",
+ index->map->hdr.log_file_seq);
+ return 1;
+}
+
+static int
+mail_index_map_latest_sync(struct mail_index *index,
+ enum mail_index_sync_handler_type type,
+ const char *reason)
+{
+ const char *map_reason, *reopen_reason;
+ bool reopened;
+ int ret;
+
+ if (index->log->head == NULL || index->indexid == 0) {
+ /* we're creating the index file, we don't have any
+ logs yet */
+ return 1;
+ }
+
+ /* and update the map with the latest changes from transaction log */
+ ret = mail_index_sync_map(&index->map, type, &map_reason);
+ if (ret != 0)
+ return ret;
+
+ if (index->fd == -1) {
+ reopen_reason = "Index not open";
+ reopened = FALSE;
+ } else {
+ /* Check if the index was recreated while we were opening it.
+ This is unlikely, but could happen if
+ mail_index_log_optimization_settings.max_size is tiny. */
+ ret = mail_index_reopen_if_changed(index, &reopened, &reopen_reason);
+ if (ret < 0)
+ return -1;
+ if (ret == 0) {
+ /* Index was unexpectedly lost. The mailbox was
+ probably deleted while we were opening it. Handle
+ this as an error. */
+ index->index_deleted = TRUE;
+ return -1;
+ }
+ }
+ if (!reopened) {
+ /* fsck the index and try to reopen */
+ mail_index_set_error(index, "Index %s: %s: %s - fscking "
+ "(reopen_reason: %s)",
+ index->filepath, reason, map_reason,
+ reopen_reason);
+ if (!index->readonly) {
+ if (mail_index_fsck(index) < 0)
+ return -1;
+ }
+ }
+
+ ret = mail_index_map_latest_file(index, &reason);
+ if (ret > 0 && index->indexid != 0) {
+ ret = mail_index_sync_map(&index->map, type, &map_reason);
+ if (ret == 0) {
+ mail_index_set_error(index, "Index %s: %s: %s",
+ index->filepath, reason, map_reason);
+ }
+ }
+ return ret;
+}
+
+int mail_index_map(struct mail_index *index,
+ enum mail_index_sync_handler_type type)
+{
+ const char *reason;
+ int ret;
+
+ i_assert(!index->mapping);
+
+ index->mapping = TRUE;
+
+ if (index->map == NULL)
+ index->map = mail_index_map_alloc(index);
+
+ /* first try updating the existing mapping from transaction log. */
+ if (!index->initial_mapped || index->reopen_main_index) {
+ /* index is being created/opened for the first time */
+ ret = 0;
+ } else if (mail_index_sync_map_want_index_reopen(index->map, type)) {
+ /* it's likely more efficient to reopen the index file than
+ sync from the transaction log. */
+ ret = 0;
+ } else {
+ /* sync the map from the transaction log. */
+ ret = mail_index_sync_map(&index->map, type, &reason);
+ if (ret == 0) {
+ e_debug(index->event,
+ "Couldn't sync map from transaction log: %s - "
+ "reopening index instead",
+ reason);
+ }
+ }
+
+ if (ret == 0) {
+ /* try to open and read the latest index. if it fails, we'll
+ fallback to updating the existing mapping from transaction
+ logs (which we'll also do even if the reopening succeeds).
+ if index files are unusable (e.g. major version change)
+ don't even try to use the transaction log. */
+ ret = mail_index_map_latest_file(index, &reason);
+ if (ret > 0) {
+ ret = mail_index_map_latest_sync(index, type, reason);
+ } else if (ret == 0 && !index->readonly) {
+ /* make sure we don't try to open the file again */
+ if (unlink(index->filepath) < 0 && errno != ENOENT)
+ mail_index_set_syscall_error(index, "unlink()");
+ }
+ }
+
+ if (ret >= 0)
+ index->initial_mapped = TRUE;
+ index->mapping = FALSE;
+ return ret;
+}