diff options
Diffstat (limited to 'src/doveadm/dsync/dsync-brain.c')
-rw-r--r-- | src/doveadm/dsync/dsync-brain.c | 901 |
1 files changed, 901 insertions, 0 deletions
diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c new file mode 100644 index 0000000..8ff7247 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain.c @@ -0,0 +1,901 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "hostpid.h" +#include "str.h" +#include "file-create-locked.h" +#include "process-title.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "mail-namespace.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" + +#include <sys/stat.h> + +enum dsync_brain_title { + DSYNC_BRAIN_TITLE_NONE = 0, + DSYNC_BRAIN_TITLE_LOCKING, +}; + +static const char *dsync_state_names[] = { + "master_recv_handshake", + "slave_recv_handshake", + "master_send_last_common", + "slave_recv_last_common", + "send_mailbox_tree", + "send_mailbox_tree_deletes", + "recv_mailbox_tree", + "recv_mailbox_tree_deletes", + "master_send_mailbox", + "slave_recv_mailbox", + "sync_mails", + "finish", + "done" +}; + +struct dsync_mailbox_list_module dsync_mailbox_list_module = + MODULE_CONTEXT_INIT(&mailbox_list_module_register); + +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain); + +static const char * +dsync_brain_get_proctitle_full(struct dsync_brain *brain, + enum dsync_brain_title title) +{ + string_t *str = t_str_new(128); + const char *import_title, *export_title; + + str_append_c(str, '['); + if (brain->process_title_prefix != NULL) + str_append(str, brain->process_title_prefix); + str_append(str, brain->user->username); + if (brain->box == NULL) { + str_append_c(str, ' '); + str_append(str, dsync_state_names[brain->state]); + } else { + str_append_c(str, ' '); + str_append(str, mailbox_get_vname(brain->box)); + import_title = brain->box_importer == NULL ? "" : + dsync_mailbox_import_get_proctitle(brain->box_importer); + export_title = brain->box_exporter == NULL ? "" : + dsync_mailbox_export_get_proctitle(brain->box_exporter); + if (import_title[0] == '\0' && export_title[0] == '\0') { + str_printfa(str, " send:%s recv:%s", + dsync_box_state_names[brain->box_send_state], + dsync_box_state_names[brain->box_recv_state]); + } else { + if (import_title[0] != '\0') { + str_append(str, " import:"); + str_append(str, import_title); + } + if (export_title[0] != '\0') { + str_append(str, " export:"); + str_append(str, export_title); + } + } + } + switch (title) { + case DSYNC_BRAIN_TITLE_NONE: + break; + case DSYNC_BRAIN_TITLE_LOCKING: + str_append(str, " locking "DSYNC_LOCK_FILENAME); + break; + } + str_append_c(str, ']'); + return str_c(str); +} + +static const char *dsync_brain_get_proctitle(struct dsync_brain *brain) +{ + return dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_NONE); +} + +static void dsync_brain_run_io(void *context) +{ + struct dsync_brain *brain = context; + bool changed, try_pending; + + if (dsync_ibc_has_failed(brain->ibc)) { + io_loop_stop(current_ioloop); + brain->failed = TRUE; + return; + } + + try_pending = TRUE; + do { + if (!dsync_brain_run(brain, &changed)) { + io_loop_stop(current_ioloop); + break; + } + if (changed) + try_pending = TRUE; + else if (try_pending) { + if (dsync_ibc_has_pending_data(brain->ibc)) + changed = TRUE; + try_pending = FALSE; + } + } while (changed); +} + +static struct dsync_brain * +dsync_brain_common_init(struct mail_user *user, struct dsync_ibc *ibc) +{ + struct dsync_brain *brain; + const struct master_service_settings *service_set; + pool_t pool; + + service_set = master_service_settings_get(master_service); + mail_user_ref(user); + + pool = pool_alloconly_create("dsync brain", 10240); + brain = p_new(pool, struct dsync_brain, 1); + brain->pool = pool; + brain->user = user; + brain->ibc = ibc; + brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN; + brain->lock_fd = -1; + brain->verbose_proctitle = service_set->verbose_proctitle; + hash_table_create(&brain->mailbox_states, pool, 0, + guid_128_hash, guid_128_cmp); + p_array_init(&brain->remote_mailbox_states, pool, 64); + return brain; +} + +static void +dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags) +{ + brain->mail_requests = + (flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0; + brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0; + brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0; + brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0; + brain->sync_visible_namespaces = + (flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0; + brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0; + brain->no_backup_overwrite = + (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0; + brain->no_mail_prefetch = + (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; + brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0; + brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0; +} + +static void +dsync_brain_open_virtual_all_box(struct dsync_brain *brain, + const char *vname) +{ + struct mail_namespace *ns; + + ns = mail_namespace_find(brain->user->namespaces, vname); + brain->virtual_all_box = + mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); +} + +struct dsync_brain * +dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const struct dsync_brain_settings *set) +{ + struct dsync_ibc_settings ibc_set; + struct dsync_brain *brain; + struct mail_namespace *ns; + string_t *sync_ns_str = NULL; + const char *error; + + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE || + (set->state != NULL && *set->state != '\0')); + i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1); + + brain = dsync_brain_common_init(user, ibc); + brain->process_title_prefix = + p_strdup(brain->pool, set->process_title_prefix); + brain->sync_type = sync_type; + if (array_count(&set->sync_namespaces) > 0) { + sync_ns_str = t_str_new(128); + p_array_init(&brain->sync_namespaces, brain->pool, + array_count(&set->sync_namespaces)); + array_foreach_elem(&set->sync_namespaces, ns) { + str_append(sync_ns_str, ns->prefix); + str_append_c(sync_ns_str, '\n'); + array_push_back(&brain->sync_namespaces, &ns); + } + str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1); + } + brain->alt_char = set->mailbox_alt_char == '\0' ? '_' : + set->mailbox_alt_char; + brain->sync_since_timestamp = set->sync_since_timestamp; + brain->sync_until_timestamp = set->sync_until_timestamp; + brain->sync_max_size = set->sync_max_size; + brain->sync_flag = p_strdup(brain->pool, set->sync_flag); + brain->sync_box = p_strdup(brain->pool, set->sync_box); + brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(brain->pool, set->exclude_mailboxes); + memcpy(brain->sync_box_guid, set->sync_box_guid, + sizeof(brain->sync_box_guid)); + brain->lock_timeout = set->lock_timeout_secs; + if (brain->lock_timeout != 0) + brain->mailbox_lock_timeout_secs = brain->lock_timeout; + else + brain->mailbox_lock_timeout_secs = + DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS; + brain->import_commit_msgs_interval = set->import_commit_msgs_interval; + brain->master_brain = TRUE; + brain->hashed_headers = + (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers); + dsync_brain_set_flags(brain, flags); + + if (set->virtual_all_box != NULL) + dsync_brain_open_virtual_all_box(brain, set->virtual_all_box); + + if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + ; + else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool, + set->state, &error) < 0) { + hash_table_clear(brain->mailbox_states, FALSE); + i_error("Saved sync state is invalid, " + "falling back to full sync: %s", error); + brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + } else { + if (brain->debug) { + i_debug("brain %c: Imported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } + } + dsync_brain_mailbox_trees_init(brain); + + i_zero(&ibc_set); + ibc_set.hostname = my_hostdomain(); + ibc_set.sync_ns_prefixes = sync_ns_str == NULL ? + NULL : str_c(sync_ns_str); + ibc_set.sync_box = set->sync_box; + ibc_set.virtual_all_box = set->virtual_all_box; + ibc_set.exclude_mailboxes = set->exclude_mailboxes; + ibc_set.sync_since_timestamp = set->sync_since_timestamp; + ibc_set.sync_until_timestamp = set->sync_until_timestamp; + ibc_set.sync_max_size = set->sync_max_size; + ibc_set.sync_flags = set->sync_flag; + memcpy(ibc_set.sync_box_guid, set->sync_box_guid, + sizeof(ibc_set.sync_box_guid)); + ibc_set.alt_char = brain->alt_char; + ibc_set.sync_type = sync_type; + ibc_set.hdr_hash_v2 = TRUE; + ibc_set.lock_timeout = set->lock_timeout_secs; + ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval; + ibc_set.hashed_headers = set->hashed_headers; + /* reverse the backup direction for the slave */ + ibc_set.brain_flags = flags & ENUM_NEGATE(DSYNC_BRAIN_FLAG_BACKUP_SEND | + DSYNC_BRAIN_FLAG_BACKUP_RECV); + if ((flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0) + ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + else if ((flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0) + ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + dsync_ibc_send_handshake(ibc, &ibc_set); + + dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain); + brain->state = DSYNC_STATE_MASTER_RECV_HANDSHAKE; + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + return brain; +} + +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, + bool local, const char *process_title_prefix, + char default_alt_char) +{ + struct dsync_ibc_settings ibc_set; + struct dsync_brain *brain; + + i_assert(default_alt_char != '\0'); + + brain = dsync_brain_common_init(user, ibc); + brain->alt_char = default_alt_char; + brain->process_title_prefix = + p_strdup(brain->pool, process_title_prefix); + brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE; + + if (local) { + /* both master and slave are running within the same process, + update the proctitle only for master. */ + brain->verbose_proctitle = FALSE; + } + + i_zero(&ibc_set); + ibc_set.hdr_hash_v2 = TRUE; + ibc_set.hostname = my_hostdomain(); + dsync_ibc_send_handshake(ibc, &ibc_set); + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain); + return brain; +} + +static void dsync_brain_purge(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + struct mail_storage *storage; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + storage = mail_namespace_get_default_storage(ns); + if (mail_storage_purge(storage) < 0) { + i_error("Purging namespace '%s' failed: %s", ns->prefix, + mail_storage_get_last_internal_error(storage, NULL)); + } + } +} + +int dsync_brain_deinit(struct dsync_brain **_brain, enum mail_error *error_r) +{ + struct dsync_brain *brain = *_brain; + int ret; + + *_brain = NULL; + + if (dsync_ibc_has_timed_out(brain->ibc)) { + i_error("Timeout during state=%s%s", + dsync_state_names[brain->state], + brain->state != DSYNC_STATE_SYNC_MAILS ? "" : + t_strdup_printf(" (send=%s recv=%s)", + dsync_box_state_names[brain->box_send_state], + dsync_box_state_names[brain->box_recv_state])); + } + if (dsync_ibc_has_failed(brain->ibc) || + brain->state != DSYNC_STATE_DONE) + brain->failed = TRUE; + dsync_ibc_close_mail_streams(brain->ibc); + + if (brain->purge && !brain->failed) + dsync_brain_purge(brain); + + if (brain->box != NULL) + dsync_brain_sync_mailbox_deinit(brain); + if (brain->virtual_all_box != NULL) + mailbox_free(&brain->virtual_all_box); + if (brain->local_tree_iter != NULL) + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + if (brain->local_mailbox_tree != NULL) + dsync_mailbox_tree_deinit(&brain->local_mailbox_tree); + if (brain->remote_mailbox_tree != NULL) + dsync_mailbox_tree_deinit(&brain->remote_mailbox_tree); + hash_table_iterate_deinit(&brain->mailbox_states_iter); + hash_table_destroy(&brain->mailbox_states); + + pool_unref(&brain->dsync_box_pool); + + if (brain->lock_fd != -1) { + /* unlink the lock file before it gets unlocked */ + i_unlink(brain->lock_path); + if (brain->debug) { + i_debug("brain %c: Unlocked %s", + brain->master_brain ? 'M' : 'S', + brain->lock_path); + } + file_lock_free(&brain->lock); + i_close_fd(&brain->lock_fd); + } + + ret = brain->failed ? -1 : 0; + mail_user_unref(&brain->user); + + *error_r = !brain->failed ? 0 : + (brain->mail_error == 0 ? MAIL_ERROR_TEMP : brain->mail_error); + pool_unref(&brain->pool); + return ret; +} + +static int +dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname) +{ + const struct file_create_settings lock_set = { + .lock_timeout_secs = brain->lock_timeout, + .lock_settings = { + .lock_method = FILE_LOCK_METHOD_FCNTL, + }, + }; + const char *home, *error, *local_hostname = my_hostdomain(); + bool created; + int ret; + + if ((ret = strcmp(remote_hostname, local_hostname)) < 0) { + /* locking done by remote */ + if (brain->debug) { + i_debug("brain %c: Locking done by remote " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + local_hostname, remote_hostname); + } + return 0; + } + if (ret == 0 && !brain->master_brain) { + /* running dsync within the same server. + locking done by master brain. */ + if (brain->debug) { + i_debug("brain %c: Locking done by local master-brain " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + local_hostname, remote_hostname); + } + return 0; + } + + if ((ret = mail_user_get_home(brain->user, &home)) < 0) { + i_error("Couldn't look up user's home dir"); + return -1; + } + if (ret == 0) { + i_error("User has no home directory"); + return -1; + } + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING)); + brain->lock_path = p_strconcat(brain->pool, home, + "/"DSYNC_LOCK_FILENAME, NULL); + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + if (brain->lock_fd == -1 && errno == ENOENT) { + /* home directory not created */ + if (mail_user_home_mkdir(brain->user) < 0) + return -1; + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + } + if (brain->lock_fd == -1) + i_error("Couldn't lock %s: %s", brain->lock_path, error); + else if (brain->debug) { + i_debug("brain %c: Locking done locally in %s " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + brain->lock_path, local_hostname, remote_hostname); + } + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + return brain->lock_fd == -1 ? -1 : 0; +} + +static void +dsync_brain_set_hdr_hash_version(struct dsync_brain *brain, + const struct dsync_ibc_settings *ibc_set) +{ + if (ibc_set->hdr_hash_v3) + brain->hdr_hash_version = 3; + else if (ibc_set->hdr_hash_v2) + brain->hdr_hash_version = 3; + else + brain->hdr_hash_version = 1; +} + +static bool dsync_brain_master_recv_handshake(struct dsync_brain *brain) +{ + const struct dsync_ibc_settings *ibc_set; + + i_assert(brain->master_brain); + + if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0) + return FALSE; + + if (brain->lock_timeout > 0) { + if (dsync_brain_lock(brain, ibc_set->hostname) < 0) { + brain->failed = TRUE; + return FALSE; + } + } + dsync_brain_set_hdr_hash_version(brain, ibc_set); + + brain->state = brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE ? + DSYNC_STATE_MASTER_SEND_LAST_COMMON : + DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; +} + +static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) +{ + const struct dsync_ibc_settings *ibc_set; + struct mail_namespace *ns; + const char *const *prefixes; + + i_assert(!brain->master_brain); + + if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0) + return FALSE; + dsync_brain_set_hdr_hash_version(brain, ibc_set); + + if (ibc_set->lock_timeout > 0) { + brain->lock_timeout = ibc_set->lock_timeout; + brain->mailbox_lock_timeout_secs = brain->lock_timeout; + if (dsync_brain_lock(brain, ibc_set->hostname) < 0) { + brain->failed = TRUE; + return FALSE; + } + } else { + brain->mailbox_lock_timeout_secs = + DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS; + } + + if (ibc_set->sync_ns_prefixes != NULL) { + p_array_init(&brain->sync_namespaces, brain->pool, 4); + prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n"); + if (prefixes[0] == NULL) { + /* ugly workaround for strsplit API: there was one + prefix="" entry */ + static const char *empty_prefix[] = { "", NULL }; + prefixes = empty_prefix; + } + for (; *prefixes != NULL; prefixes++) { + ns = mail_namespace_find(brain->user->namespaces, + *prefixes); + if (ns == NULL) { + i_error("Namespace not found: '%s'", *prefixes); + brain->failed = TRUE; + return FALSE; + } + array_push_back(&brain->sync_namespaces, &ns); + } + } + brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box); + brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes); + brain->sync_since_timestamp = ibc_set->sync_since_timestamp; + brain->sync_until_timestamp = ibc_set->sync_until_timestamp; + brain->sync_max_size = ibc_set->sync_max_size; + brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags); + memcpy(brain->sync_box_guid, ibc_set->sync_box_guid, + sizeof(brain->sync_box_guid)); + if (ibc_set->alt_char != '\0') + brain->alt_char = ibc_set->alt_char; + i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + brain->sync_type = ibc_set->sync_type; + + dsync_brain_set_flags(brain, ibc_set->brain_flags); + if (ibc_set->hashed_headers != NULL) + brain->hashed_headers = + p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers); + /* this flag is only set on the remote slave brain */ + brain->purge = (ibc_set->brain_flags & + DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0; + + if (ibc_set->virtual_all_box != NULL) + dsync_brain_open_virtual_all_box(brain, ibc_set->virtual_all_box); + dsync_brain_mailbox_trees_init(brain); + + if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) + brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON; + else + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; +} + +static void dsync_brain_master_send_last_common(struct dsync_brain *brain) +{ + struct dsync_mailbox_state *state; + uint8_t *guid; + enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK; + + i_assert(brain->master_brain); + + if (brain->mailbox_states_iter == NULL) { + brain->mailbox_states_iter = + hash_table_iterate_init(brain->mailbox_states); + } + + for (;;) { + if (ret == DSYNC_IBC_SEND_RET_FULL) + return; + if (!hash_table_iterate(brain->mailbox_states_iter, + brain->mailbox_states, &guid, &state)) + break; + ret = dsync_ibc_send_mailbox_state(brain->ibc, state); + } + hash_table_iterate_deinit(&brain->mailbox_states_iter); + + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_STATE); + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; +} + +static void dsync_mailbox_state_add(struct dsync_brain *brain, + const struct dsync_mailbox_state *state) +{ + struct dsync_mailbox_state *dupstate; + uint8_t *guid_p; + + dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1); + *dupstate = *state; + guid_p = dupstate->mailbox_guid; + hash_table_insert(brain->mailbox_states, guid_p, dupstate); +} + +static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain) +{ + struct dsync_mailbox_state state; + enum dsync_ibc_recv_ret ret; + bool changed = FALSE; + + i_assert(!brain->master_brain); + + while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) { + dsync_mailbox_state_add(brain, &state); + changed = TRUE; + } + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + changed = TRUE; + } + return changed; +} + +static bool dsync_brain_finish(struct dsync_brain *brain) +{ + const char *error; + enum mail_error mail_error; + bool require_full_resync; + enum dsync_ibc_recv_ret ret; + + if (!brain->master_brain) { + dsync_ibc_send_finish(brain->ibc, + brain->failed ? "dsync failed" : NULL, + brain->mail_error, + brain->require_full_resync); + brain->state = DSYNC_STATE_DONE; + return TRUE; + } + ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error, + &require_full_resync); + if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN) + return FALSE; + if (error != NULL) { + i_error("Remote dsync failed: %s", error); + brain->failed = TRUE; + if (mail_error != 0 && + (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP)) + brain->mail_error = mail_error; + } + if (require_full_resync) + brain->require_full_resync = TRUE; + brain->state = DSYNC_STATE_DONE; + return TRUE; +} + +static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r) +{ + enum dsync_state orig_state = brain->state; + enum dsync_box_state orig_box_recv_state = brain->box_recv_state; + enum dsync_box_state orig_box_send_state = brain->box_send_state; + bool changed = FALSE, ret = TRUE; + + if (brain->failed) + return FALSE; + + switch (brain->state) { + case DSYNC_STATE_MASTER_RECV_HANDSHAKE: + changed = dsync_brain_master_recv_handshake(brain); + break; + case DSYNC_STATE_SLAVE_RECV_HANDSHAKE: + changed = dsync_brain_slave_recv_handshake(brain); + break; + case DSYNC_STATE_MASTER_SEND_LAST_COMMON: + dsync_brain_master_send_last_common(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_LAST_COMMON: + changed = dsync_brain_slave_recv_last_common(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE: + dsync_brain_send_mailbox_tree(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE: + changed = dsync_brain_recv_mailbox_tree(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES: + dsync_brain_send_mailbox_tree_deletes(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES: + changed = dsync_brain_recv_mailbox_tree_deletes(brain); + break; + case DSYNC_STATE_MASTER_SEND_MAILBOX: + dsync_brain_master_send_mailbox(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_MAILBOX: + changed = dsync_brain_slave_recv_mailbox(brain); + break; + case DSYNC_STATE_SYNC_MAILS: + changed = dsync_brain_sync_mails(brain); + break; + case DSYNC_STATE_FINISH: + changed = dsync_brain_finish(brain); + break; + case DSYNC_STATE_DONE: + changed = TRUE; + ret = FALSE; + break; + } + if (brain->verbose_proctitle) { + if (orig_state != brain->state || + orig_box_recv_state != brain->box_recv_state || + orig_box_send_state != brain->box_send_state || + ++brain->proctitle_update_counter % 100 == 0) + process_title_set(dsync_brain_get_proctitle(brain)); + } + *changed_r = changed; + return brain->failed ? FALSE : ret; +} + +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r) +{ + bool ret; + + *changed_r = FALSE; + + if (dsync_ibc_has_failed(brain->ibc)) { + brain->failed = TRUE; + return FALSE; + } + + T_BEGIN { + ret = dsync_brain_run_real(brain, changed_r); + } T_END; + return ret; +} + +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_state *state; + uint8_t *guid; + + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%"PRIu64" pvt_modseq=%"PRIu64" messages=%u changes_during_sync=%d", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid), + state->last_uidvalidity, + state->last_common_uid, + state->last_common_modseq, + state->last_common_pvt_modseq, + state->last_messages_count, + state->changes_during_sync ? 1 : 0); + } + hash_table_iterate_deinit(&iter); +} + +void dsync_brain_get_state(struct dsync_brain *brain, string_t *output) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_node *node; + const struct dsync_mailbox_state *new_state; + struct dsync_mailbox_state *state; + const uint8_t *guid_p; + uint8_t *guid; + + if (brain->require_full_resync) + return; + + /* update mailbox states */ + array_foreach(&brain->remote_mailbox_states, new_state) { + guid_p = new_state->mailbox_guid; + state = hash_table_lookup(brain->mailbox_states, guid_p); + if (state != NULL) + *state = *new_state; + else + dsync_mailbox_state_add(brain, new_state); + } + + /* remove nonexistent mailboxes */ + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree, + guid); + if (node == NULL || + node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + if (brain->debug) { + i_debug("brain %c: Removed state for deleted mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid)); + } + hash_table_remove(brain->mailbox_states, guid); + } + } + hash_table_iterate_deinit(&iter); + + if (brain->debug) { + i_debug("brain %c: Exported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } + dsync_mailbox_states_export(brain->mailbox_states, output); +} + +enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain) +{ + return brain->sync_type; +} + +bool dsync_brain_has_failed(struct dsync_brain *brain) +{ + return brain->failed; +} + +const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain, + bool *remote_only_r) +{ + if (brain->changes_during_sync == NULL && + brain->changes_during_remote_sync) { + *remote_only_r = TRUE; + return "Remote notified that changes happened during sync"; + } + *remote_only_r = FALSE; + return brain->changes_during_sync; +} + +static bool dsync_brain_want_shared_namespace(const struct mail_namespace *ns, + const struct mail_namespace *sync_ns) +{ + /* Include shared namespaces and all its + children in the sync (e.g. "Shared/example.com" + will be synced to "Shared/"). + This also allows "dsync -n Shared/example.com/" + with "Shared/example.com/username/" style + shared namespace config. */ + return (ns->type == MAIL_NAMESPACE_TYPE_SHARED) && + (sync_ns->type == MAIL_NAMESPACE_TYPE_SHARED) && + str_begins(ns->prefix, sync_ns->prefix); +} + +bool dsync_brain_want_namespace(struct dsync_brain *brain, + struct mail_namespace *ns) +{ + struct mail_namespace *sync_ns; + + if (array_is_created(&brain->sync_namespaces)) { + array_foreach_elem(&brain->sync_namespaces, sync_ns) { + if (ns == sync_ns) + return TRUE; + if (dsync_brain_want_shared_namespace(ns, sync_ns)) + return TRUE; + } + return FALSE; + } + if (ns->alias_for != NULL) { + /* always skip aliases */ + return FALSE; + } + if (brain->sync_visible_namespaces) { + if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0) + return TRUE; + if ((ns->flags & (NAMESPACE_FLAG_LIST_PREFIX | + NAMESPACE_FLAG_LIST_CHILDREN)) != 0) + return TRUE; + return FALSE; + } else { + return strcmp(ns->unexpanded_set->location, + SETTING_STRVAR_UNEXPANDED) == 0; + } +} + +void dsync_brain_set_changes_during_sync(struct dsync_brain *brain, + const char *reason) +{ + if (brain->debug) { + i_debug("brain %c: Change during sync: %s", + brain->master_brain ? 'M' : 'S', reason); + } + if (brain->changes_during_sync == NULL) + brain->changes_during_sync = p_strdup(brain->pool, reason); +} |