diff options
Diffstat (limited to 'src/lib-storage/index/dbox-common/dbox-file.c')
| -rw-r--r-- | src/lib-storage/index/dbox-common/dbox-file.c | 796 |
1 files changed, 796 insertions, 0 deletions
diff --git a/src/lib-storage/index/dbox-common/dbox-file.c b/src/lib-storage/index/dbox-common/dbox-file.c new file mode 100644 index 0000000..16810b0 --- /dev/null +++ b/src/lib-storage/index/dbox-common/dbox-file.c @@ -0,0 +1,796 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "hex-dec.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "istream.h" +#include "ostream.h" +#include "file-lock.h" +#include "file-dotlock.h" +#include "mkdir-parents.h" +#include "eacces-error.h" +#include "str.h" +#include "dbox-storage.h" +#include "dbox-file.h" + +#include <stdio.h> +#include <unistd.h> +#include <ctype.h> +#include <fcntl.h> + +#define DBOX_READ_BLOCK_SIZE IO_BLOCK_SIZE + +#ifndef DBOX_FILE_LOCK_METHOD_FLOCK +static const struct dotlock_settings dotlock_set = { + .stale_timeout = 60*10, + .use_excl_lock = TRUE +}; +#endif + +const char *dbox_generate_tmp_filename(void) +{ + static unsigned int create_count = 0; + + return t_strdup_printf(DBOX_TEMP_FILE_PREFIX"%"PRIdTIME_T".P%sQ%uM%u.%s", + ioloop_timeval.tv_sec, my_pid, + create_count++, + (unsigned int)ioloop_timeval.tv_usec, + my_hostname); +} + +void dbox_file_set_syscall_error(struct dbox_file *file, const char *function) +{ + mail_storage_set_critical(&file->storage->storage, + "%s failed for file %s: %m", + function, file->cur_path); +} + +void dbox_file_set_corrupted(struct dbox_file *file, const char *reason, ...) +{ + va_list args; + + va_start(args, reason); + mail_storage_set_critical(&file->storage->storage, + "Corrupted dbox file %s (around offset=%"PRIuUOFF_T"): %s", + file->cur_path, file->input == NULL ? 0 : file->input->v_offset, + t_strdup_vprintf(reason, args)); + va_end(args); + + file->storage->v.set_file_corrupted(file); +} + +void dbox_file_init(struct dbox_file *file) +{ + file->refcount = 1; + file->fd = -1; + file->cur_offset = UOFF_T_MAX; + file->cur_path = file->primary_path; +} + +void dbox_file_free(struct dbox_file *file) +{ + i_assert(file->refcount == 0); + + pool_unref(&file->metadata_pool); + dbox_file_close(file); + i_free(file->primary_path); + i_free(file->alt_path); + i_free(file); +} + +void dbox_file_unref(struct dbox_file **_file) +{ + struct dbox_file *file = *_file; + + *_file = NULL; + + i_assert(file->refcount > 0); + if (--file->refcount == 0) + file->storage->v.file_unrefed(file); +} + +static int dbox_file_parse_header(struct dbox_file *file, const char *line) +{ + const char *const *tmp, *value; + enum dbox_header_key key; + + file->file_version = *line - '0'; + if (!i_isdigit(line[0]) || line[1] != ' ' || + (file->file_version != 1 && file->file_version != DBOX_VERSION)) { + dbox_file_set_corrupted(file, "Invalid dbox version"); + return -1; + } + line += 2; + + file->msg_header_size = 0; + + for (tmp = t_strsplit(line, " "); *tmp != NULL; tmp++) { + uintmax_t time; + key = **tmp; + value = *tmp + 1; + + switch (key) { + case DBOX_HEADER_OLDV1_APPEND_OFFSET: + break; + case DBOX_HEADER_MSG_HEADER_SIZE: + if (str_to_uint_hex(value, &file->msg_header_size) < 0) { + dbox_file_set_corrupted(file, "Invalid message header size"); + return -1; + } + break; + case DBOX_HEADER_CREATE_STAMP: + if (str_to_uintmax_hex(value, &time) < 0) { + dbox_file_set_corrupted(file, "Invalid create time stamp"); + return -1; + } + file->create_time = (time_t)time; + break; + } + } + + if (file->msg_header_size == 0) { + dbox_file_set_corrupted(file, "Missing message header size"); + return -1; + } + return 0; +} + +static int dbox_file_read_header(struct dbox_file *file) +{ + const char *line; + unsigned int hdr_size; + int ret; + + i_stream_seek(file->input, 0); + line = i_stream_read_next_line(file->input); + if (line == NULL) { + if (file->input->stream_errno == 0) { + dbox_file_set_corrupted(file, + "EOF while reading file header"); + return 0; + } + + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + hdr_size = file->input->v_offset; + T_BEGIN { + ret = dbox_file_parse_header(file, line) < 0 ? 0 : 1; + } T_END; + if (ret > 0) + file->file_header_size = hdr_size; + return ret; +} + +static int dbox_file_open_fd(struct dbox_file *file, bool try_altpath) +{ + const char *path; + int flags = O_RDWR; + bool alt = FALSE; + + /* try the primary path first */ + path = file->primary_path; + while ((file->fd = open(path, flags)) == -1) { + if (errno == EACCES && flags == O_RDWR) { + flags = O_RDONLY; + continue; + } + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "open(%s) failed: %m", path); + return -1; + } + + if (file->alt_path == NULL || alt || !try_altpath) { + /* not found */ + return 0; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + file->cur_path = path; + return 1; +} + +static int dbox_file_open_full(struct dbox_file *file, bool try_altpath, + bool *notfound_r) +{ + int ret, fd; + + *notfound_r = FALSE; + if (file->input != NULL) + return 1; + + if (file->fd == -1) { + T_BEGIN { + ret = dbox_file_open_fd(file, try_altpath); + } T_END; + if (ret <= 0) { + if (ret < 0) + return -1; + *notfound_r = TRUE; + return 1; + } + } + + /* we're manually checking at dbox_file_close() if we need to close the + fd or not. */ + fd = file->fd; + file->input = i_stream_create_fd_autoclose(&fd, DBOX_READ_BLOCK_SIZE); + i_stream_set_name(file->input, file->cur_path); + i_stream_set_init_buffer_size(file->input, DBOX_READ_BLOCK_SIZE); + return dbox_file_read_header(file); +} + +int dbox_file_open(struct dbox_file *file, bool *deleted_r) +{ + return dbox_file_open_full(file, TRUE, deleted_r); +} + +int dbox_file_open_primary(struct dbox_file *file, bool *notfound_r) +{ + return dbox_file_open_full(file, FALSE, notfound_r); +} + +int dbox_file_stat(struct dbox_file *file, struct stat *st_r) +{ + const char *path; + bool alt = FALSE; + + if (dbox_file_is_open(file)) { + if (fstat(file->fd, st_r) < 0) { + mail_storage_set_critical(&file->storage->storage, + "fstat(%s) failed: %m", file->cur_path); + return -1; + } + return 0; + } + + /* try the primary path first */ + path = file->primary_path; + while (stat(path, st_r) < 0) { + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "stat(%s) failed: %m", path); + return -1; + } + + if (file->alt_path == NULL || alt) { + /* not found */ + return -1; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + file->cur_path = path; + return 0; +} + +int dbox_file_header_write(struct dbox_file *file, struct ostream *output) +{ + string_t *hdr; + + hdr = t_str_new(128); + str_printfa(hdr, "%u %c%x %c%x\n", DBOX_VERSION, + DBOX_HEADER_MSG_HEADER_SIZE, + (unsigned int)sizeof(struct dbox_message_header), + DBOX_HEADER_CREATE_STAMP, (unsigned int)ioloop_time); + + file->file_version = DBOX_VERSION; + file->file_header_size = str_len(hdr); + file->msg_header_size = sizeof(struct dbox_message_header); + return o_stream_send(output, str_data(hdr), str_len(hdr)); +} + +void dbox_file_close(struct dbox_file *file) +{ + dbox_file_unlock(file); + if (file->input != NULL) { + /* stream autocloses the fd when it gets destroyed. note that + the stream may outlive the struct dbox_file. */ + i_stream_unref(&file->input); + file->fd = -1; + } else if (file->fd != -1) { + if (close(file->fd) < 0) + dbox_file_set_syscall_error(file, "close()"); + file->fd = -1; + } + file->cur_offset = UOFF_T_MAX; +} + +int dbox_file_try_lock(struct dbox_file *file) +{ + const char *error; + int ret; + + i_assert(file->fd != -1); + +#ifdef DBOX_FILE_LOCK_METHOD_FLOCK + struct file_lock_settings lock_set = { + .lock_method = FILE_LOCK_METHOD_FLOCK, + }; + ret = file_try_lock(file->fd, file->cur_path, F_WRLCK, + &lock_set, &file->lock, &error); + if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "file_try_lock(%s) failed: %s", file->cur_path, error); + } +#else + ret = file_dotlock_create(&dotlock_set, file->cur_path, + DOTLOCK_CREATE_FLAG_NONBLOCK, &file->lock); + if (ret < 0) { + mail_storage_set_critical(&file->storage->storage, + "file_dotlock_create(%s) failed: %m", file->cur_path); + } +#endif + return ret; +} + +void dbox_file_unlock(struct dbox_file *file) +{ + i_assert(!file->appending || file->lock == NULL); + + if (file->lock != NULL) { +#ifdef DBOX_FILE_LOCK_METHOD_FLOCK + file_unlock(&file->lock); +#else + file_dotlock_delete(&file->lock); +#endif + } + if (file->input != NULL) + i_stream_sync(file->input); +} + +int dbox_file_read_mail_header(struct dbox_file *file, uoff_t *physical_size_r) +{ + struct dbox_message_header hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(file->input, &data, &size, + file->msg_header_size); + if (ret <= 0) { + if (file->input->stream_errno == 0) { + /* EOF, broken offset or file truncated */ + dbox_file_set_corrupted(file, "EOF reading msg header " + "(got %zu/%u bytes)", + size, file->msg_header_size); + return 0; + } + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + memcpy(&hdr, data, I_MIN(sizeof(hdr), file->msg_header_size)); + if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) { + /* probably broken offset */ + dbox_file_set_corrupted(file, "msg header has bad magic value"); + return 0; + } + + if (data[file->msg_header_size-1] != '\n') { + dbox_file_set_corrupted(file, "msg header doesn't end with LF"); + return 0; + } + + *physical_size_r = hex2dec(hdr.message_size_hex, + sizeof(hdr.message_size_hex)); + return 1; +} + +int dbox_file_seek(struct dbox_file *file, uoff_t offset) +{ + uoff_t size; + int ret; + + i_assert(file->input != NULL); + + if (offset == 0) + offset = file->file_header_size; + + if (offset != file->cur_offset) { + i_stream_seek(file->input, offset); + ret = dbox_file_read_mail_header(file, &size); + if (ret <= 0) + return ret; + file->cur_offset = offset; + file->cur_physical_size = size; + } + i_stream_seek(file->input, offset + file->msg_header_size); + return 1; +} + +static int +dbox_file_seek_next_at_metadata(struct dbox_file *file, uoff_t *offset) +{ + const char *line; + size_t buf_size; + int ret; + + i_stream_seek(file->input, *offset); + if ((ret = dbox_file_metadata_skip_header(file)) <= 0) + return ret; + + /* skip over the actual metadata */ + buf_size = i_stream_get_max_buffer_size(file->input); + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + break; + } + } + i_stream_set_max_buffer_size(file->input, buf_size); + *offset = file->input->v_offset; + return 1; +} + +void dbox_file_seek_rewind(struct dbox_file *file) +{ + file->cur_offset = UOFF_T_MAX; +} + +int dbox_file_seek_next(struct dbox_file *file, uoff_t *offset_r, bool *last_r) +{ + uoff_t offset; + int ret; + + i_assert(file->input != NULL); + + if (file->cur_offset == UOFF_T_MAX) { + /* first mail. we may not have read the file at all yet, + so set the offset afterwards. */ + offset = 0; + } else { + offset = file->cur_offset + file->msg_header_size + + file->cur_physical_size; + if ((ret = dbox_file_seek_next_at_metadata(file, &offset)) <= 0) { + *offset_r = file->cur_offset; + return ret; + } + if (i_stream_read_eof(file->input)) { + *last_r = TRUE; + return 0; + } + } + *offset_r = offset; + + *last_r = FALSE; + + ret = dbox_file_seek(file, offset); + if (*offset_r == 0) + *offset_r = file->file_header_size; + return ret; +} + +struct dbox_file_append_context *dbox_file_append_init(struct dbox_file *file) +{ + struct dbox_file_append_context *ctx; + + i_assert(!file->appending); + + file->appending = TRUE; + + ctx = i_new(struct dbox_file_append_context, 1); + ctx->file = file; + if (file->fd != -1) { + ctx->output = o_stream_create_fd_file(file->fd, 0, FALSE); + o_stream_set_name(ctx->output, file->cur_path); + o_stream_set_finish_via_child(ctx->output, FALSE); + o_stream_cork(ctx->output); + } + return ctx; +} + +int dbox_file_append_commit(struct dbox_file_append_context **_ctx) +{ + struct dbox_file_append_context *ctx = *_ctx; + int ret; + + i_assert(ctx->file->appending); + + *_ctx = NULL; + + ret = dbox_file_append_flush(ctx); + if (ctx->last_checkpoint_offset != ctx->output->offset) { + o_stream_close(ctx->output); + if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "ftruncate()"); + return -1; + } + } + o_stream_unref(&ctx->output); + ctx->file->appending = FALSE; + i_free(ctx); + return ret; +} + +void dbox_file_append_rollback(struct dbox_file_append_context **_ctx) +{ + struct dbox_file_append_context *ctx = *_ctx; + struct dbox_file *file = ctx->file; + bool close_file = FALSE; + + i_assert(ctx->file->appending); + + *_ctx = NULL; + if (ctx->first_append_offset == 0) { + /* nothing changed */ + } else if (ctx->first_append_offset == file->file_header_size) { + /* rolling back everything */ + if (unlink(file->cur_path) < 0) + dbox_file_set_syscall_error(file, "unlink()"); + close_file = TRUE; + } else { + /* truncating only some mails */ + o_stream_close(ctx->output); + if (ftruncate(file->fd, ctx->first_append_offset) < 0) + dbox_file_set_syscall_error(file, "ftruncate()"); + } + if (ctx->output != NULL) { + o_stream_abort(ctx->output); + o_stream_unref(&ctx->output); + } + i_free(ctx); + + if (close_file) + dbox_file_close(file); + file->appending = FALSE; +} + +int dbox_file_append_flush(struct dbox_file_append_context *ctx) +{ + struct mail_storage *storage = &ctx->file->storage->storage; + + if (ctx->last_flush_offset == ctx->output->offset && + ctx->last_checkpoint_offset == ctx->output->offset) + return 0; + + if (o_stream_flush(ctx->output) < 0) { + dbox_file_set_syscall_error(ctx->file, "write()"); + return -1; + } + + if (ctx->last_checkpoint_offset != ctx->output->offset) { + if (ftruncate(ctx->file->fd, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "ftruncate()"); + return -1; + } + if (o_stream_seek(ctx->output, ctx->last_checkpoint_offset) < 0) { + dbox_file_set_syscall_error(ctx->file, "lseek()"); + return -1; + } + } + + if (storage->set->parsed_fsync_mode != FSYNC_MODE_NEVER) { + if (fdatasync(ctx->file->fd) < 0) { + dbox_file_set_syscall_error(ctx->file, "fdatasync()"); + return -1; + } + } + ctx->last_flush_offset = ctx->output->offset; + return 0; +} + +void dbox_file_append_checkpoint(struct dbox_file_append_context *ctx) +{ + ctx->last_checkpoint_offset = ctx->output->offset; +} + +int dbox_file_get_append_stream(struct dbox_file_append_context *ctx, + struct ostream **output_r) +{ + struct dbox_file *file = ctx->file; + struct stat st; + + if (ctx->output == NULL) { + /* file creation had failed */ + return -1; + } + if (ctx->last_checkpoint_offset != ctx->output->offset) { + /* a message was aborted. don't try appending to this + file anymore. */ + return -1; + } + + if (file->file_version == 0) { + /* newly created file, write the file header */ + if (dbox_file_header_write(file, ctx->output) < 0) { + dbox_file_set_syscall_error(file, "write()"); + return -1; + } + *output_r = ctx->output; + return 1; + } + + /* file has existing mails */ + if (file->file_version != DBOX_VERSION || + file->msg_header_size != sizeof(struct dbox_message_header)) { + /* created by an incompatible version, can't append */ + return 0; + } + + if (ctx->output->offset == 0) { + /* first append to existing file. seek to eof first. */ + if (fstat(file->fd, &st) < 0) { + dbox_file_set_syscall_error(file, "fstat()"); + return -1; + } + if (st.st_size < file->msg_header_size) { + dbox_file_set_corrupted(file, + "dbox file size too small"); + return 0; + } + if (o_stream_seek(ctx->output, st.st_size) < 0) { + dbox_file_set_syscall_error(file, "lseek()"); + return -1; + } + } + *output_r = ctx->output; + return 1; +} + +int dbox_file_metadata_skip_header(struct dbox_file *file) +{ + struct dbox_metadata_header metadata_hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(file->input, &data, &size, + sizeof(metadata_hdr)); + if (ret <= 0) { + if (file->input->stream_errno == 0) { + /* EOF, broken offset */ + dbox_file_set_corrupted(file, + "Unexpected EOF while reading metadata header"); + return 0; + } + dbox_file_set_syscall_error(file, "read()"); + return -1; + } + memcpy(&metadata_hdr, data, sizeof(metadata_hdr)); + if (memcmp(metadata_hdr.magic_post, DBOX_MAGIC_POST, + sizeof(metadata_hdr.magic_post)) != 0) { + /* probably broken offset */ + dbox_file_set_corrupted(file, + "metadata header has bad magic value"); + return 0; + } + i_stream_skip(file->input, sizeof(metadata_hdr)); + return 1; +} + +static int +dbox_file_metadata_read_at(struct dbox_file *file, uoff_t metadata_offset) +{ + const char *line; + size_t buf_size; + int ret; + + if (file->metadata_pool != NULL) + p_clear(file->metadata_pool); + else { + file->metadata_pool = + pool_alloconly_create("dbox metadata", 1024); + } + p_array_init(&file->metadata, file->metadata_pool, 16); + + i_stream_seek(file->input, metadata_offset); + if ((ret = dbox_file_metadata_skip_header(file)) <= 0) + return ret; + + ret = 0; + buf_size = i_stream_get_max_buffer_size(file->input); + /* use unlimited line length for metadata */ + i_stream_set_max_buffer_size(file->input, SIZE_MAX); + while ((line = i_stream_read_next_line(file->input)) != NULL) { + if (*line == DBOX_METADATA_OLDV1_SPACE || *line == '\0') { + /* end of metadata */ + ret = 1; + break; + } + line = p_strdup(file->metadata_pool, line); + array_push_back(&file->metadata, &line); + } + i_stream_set_max_buffer_size(file->input, buf_size); + if (ret == 0) + dbox_file_set_corrupted(file, "missing end-of-metadata line"); + return ret; +} + +int dbox_file_metadata_read(struct dbox_file *file) +{ + uoff_t metadata_offset; + int ret; + + i_assert(file->cur_offset != UOFF_T_MAX); + + if (file->metadata_read_offset == file->cur_offset) + return 1; + + metadata_offset = file->cur_offset + file->msg_header_size + + file->cur_physical_size; + ret = dbox_file_metadata_read_at(file, metadata_offset); + if (ret <= 0) + return ret; + + file->metadata_read_offset = file->cur_offset; + return 1; +} + +const char *dbox_file_metadata_get(struct dbox_file *file, + enum dbox_metadata_key key) +{ + const char *const *metadata; + unsigned int i, count; + + metadata = array_get(&file->metadata, &count); + for (i = 0; i < count; i++) { + if (*metadata[i] == (char)key) + return metadata[i] + 1; + } + return NULL; +} + +uoff_t dbox_file_get_plaintext_size(struct dbox_file *file) +{ + const char *value; + uintmax_t size; + + i_assert(file->metadata_read_offset == file->cur_offset); + + /* see if we have it in metadata */ + value = dbox_file_metadata_get(file, DBOX_METADATA_PHYSICAL_SIZE); + if (value == NULL || + str_to_uintmax_hex(value, &size) < 0 || + size > UOFF_T_MAX) { + /* no. that means we can use the size in the header */ + return file->cur_physical_size; + } + + return (uoff_t)size; +} + +void dbox_msg_header_fill(struct dbox_message_header *dbox_msg_hdr, + uoff_t message_size) +{ + memset(dbox_msg_hdr, ' ', sizeof(*dbox_msg_hdr)); + memcpy(dbox_msg_hdr->magic_pre, DBOX_MAGIC_PRE, + sizeof(dbox_msg_hdr->magic_pre)); + dbox_msg_hdr->type = DBOX_MESSAGE_TYPE_NORMAL; + dec2hex(dbox_msg_hdr->message_size_hex, message_size, + sizeof(dbox_msg_hdr->message_size_hex)); + dbox_msg_hdr->save_lf = '\n'; +} + +int dbox_file_unlink(struct dbox_file *file) +{ + const char *path; + bool alt = FALSE; + + path = file->primary_path; + while (unlink(path) < 0) { + if (errno != ENOENT) { + mail_storage_set_critical(&file->storage->storage, + "unlink(%s) failed: %m", path); + return -1; + } + if (file->alt_path == NULL || alt) { + /* not found */ + return 0; + } + + /* try the alternative path */ + path = file->alt_path; + alt = TRUE; + } + return 1; +} |