Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

36 files changed, 16782 insertions, 0 deletions

diff --git a/src/doveadm/dsync/Makefile.am b/src/doveadm/dsync/Makefile.am
new file mode 100644
index 0000000..20cbfc7
--- /dev/null
+++ b/src/doveadm/dsync/Makefile.am
@@ -0,0 +1,76 @@
+pkglib_LTLIBRARIES = libdovecot-dsync.la
+noinst_LTLIBRARIES = libdsync.la
+
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage
+
+libdsync_la_SOURCES = \
+	dsync-brain.c \
+	dsync-brain-mailbox.c \
+	dsync-brain-mailbox-tree.c \
+	dsync-brain-mailbox-tree-sync.c \
+	dsync-brain-mails.c \
+	dsync-deserializer.c \
+	dsync-mail.c \
+	dsync-mailbox.c \
+	dsync-mailbox-import.c \
+	dsync-mailbox-export.c \
+	dsync-mailbox-state.c \
+	dsync-mailbox-tree.c \
+	dsync-mailbox-tree-fill.c \
+	dsync-mailbox-tree-sync.c \
+	dsync-serializer.c \
+	dsync-ibc.c \
+	dsync-ibc-stream.c \
+	dsync-ibc-pipe.c \
+	dsync-transaction-log-scan.c
+
+libdovecot_dsync_la_SOURCES =
+libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la
+libdovecot_dsync_la_DEPENDENCIES = libdsync.la
+libdovecot_dsync_la_LDFLAGS = -export-dynamic
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+	dsync-brain.h \
+	dsync-ibc.h
+
+noinst_HEADERS = \
+	dsync-brain-private.h \
+	dsync-mail.h \
+	dsync-mailbox.h \
+	dsync-mailbox-import.h \
+	dsync-mailbox-export.h \
+	dsync-mailbox-state.h \
+	dsync-mailbox-tree.h \
+	dsync-mailbox-tree-private.h \
+	dsync-serializer.h \
+	dsync-deserializer.h \
+	dsync-ibc-private.h \
+	dsync-transaction-log-scan.h
+
+test_programs = \
+	test-dsync-mailbox-tree-sync
+
+noinst_PROGRAMS = $(test_programs)
+
+test_libs = \
+	../../lib-test/libtest.la \
+	../../lib/liblib.la
+
+test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c
+test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs)
+test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs)
+
+check-local:
+	for bin in $(test_programs); do \
+	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+	done
diff --git a/src/doveadm/dsync/Makefile.in b/src/doveadm/dsync/Makefile.in
new file mode 100644
index 0000000..ba39895
--- /dev/null
+++ b/src/doveadm/dsync/Makefile.in
@@ -0,0 +1,1019 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+  if test -z '$(MAKELEVEL)'; then \
+    false; \
+  elif test -n '$(MAKE_HOST)'; then \
+    true; \
+  elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+    true; \
+  else \
+    false; \
+  fi; \
+}
+am__make_running_with_option = \
+  case $${target_option-} in \
+      ?) ;; \
+      *) echo "am__make_running_with_option: internal error: invalid" \
+              "target option '$${target_option-}' specified" >&2; \
+         exit 1;; \
+  esac; \
+  has_opt=no; \
+  sane_makeflags=$$MAKEFLAGS; \
+  if $(am__is_gnu_make); then \
+    sane_makeflags=$$MFLAGS; \
+  else \
+    case $$MAKEFLAGS in \
+      *\\[\ \	]*) \
+        bs=\\; \
+        sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+          | sed "s/$$bs$$bs[$$bs $$bs	]*//g"`;; \
+    esac; \
+  fi; \
+  skip_next=no; \
+  strip_trailopt () \
+  { \
+    flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+  }; \
+  for flg in $$sane_makeflags; do \
+    test $$skip_next = yes && { skip_next=no; continue; }; \
+    case $$flg in \
+      *=*|--*) continue;; \
+        -*I) strip_trailopt 'I'; skip_next=yes;; \
+      -*I?*) strip_trailopt 'I';; \
+        -*O) strip_trailopt 'O'; skip_next=yes;; \
+      -*O?*) strip_trailopt 'O';; \
+        -*l) strip_trailopt 'l'; skip_next=yes;; \
+      -*l?*) strip_trailopt 'l';; \
+      -[dEDm]) skip_next=yes;; \
+      -[JT]) skip_next=yes;; \
+    esac; \
+    case $$flg in \
+      *$$target_option*) has_opt=yes; break;; \
+    esac; \
+  done; \
+  test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+noinst_PROGRAMS = $(am__EXEEXT_1)
+subdir = src/doveadm/dsync
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+	$(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+	$(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+	$(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+	$(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+	$(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+	$(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+	$(top_srcdir)/m4/flexible_array_member.m4 \
+	$(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+	$(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+	$(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+	$(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+	$(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+	$(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+	$(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+	$(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+	$(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+	$(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+	$(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+	$(top_srcdir)/m4/pr_set_dumpable.m4 \
+	$(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+	$(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+	$(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+	$(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+	$(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+	$(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+	$(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+	$(top_srcdir)/m4/typeof_dev_t.m4 \
+	$(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+	$(top_srcdir)/m4/want_apparmor.m4 \
+	$(top_srcdir)/m4/want_bsdauth.m4 \
+	$(top_srcdir)/m4/want_bzlib.m4 \
+	$(top_srcdir)/m4/want_cassandra.m4 \
+	$(top_srcdir)/m4/want_cdb.m4 \
+	$(top_srcdir)/m4/want_checkpassword.m4 \
+	$(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+	$(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+	$(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+	$(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+	$(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+	$(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+	$(top_srcdir)/m4/want_prefetch.m4 \
+	$(top_srcdir)/m4/want_shadow.m4 \
+	$(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+	$(top_srcdir)/m4/want_sqlite.m4 \
+	$(top_srcdir)/m4/want_stemmer.m4 \
+	$(top_srcdir)/m4/want_systemd.m4 \
+	$(top_srcdir)/m4/want_textcat.m4 \
+	$(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+	$(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+	$(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+	$(pkginc_lib_HEADERS) $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__EXEEXT_1 = test-dsync-mailbox-tree-sync$(EXEEXT)
+PROGRAMS = $(noinst_PROGRAMS)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+    $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+    *) f=$$p;; \
+  esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+  srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+  for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+  for p in $$list; do echo "$$p $$p"; done | \
+  sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+  $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+    if (++n[$$2] == $(am__install_max)) \
+      { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+    END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+  sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+  sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+  test -z "$$files" \
+    || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+    || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+         $(am__cd) "$$dir" && rm -f $$files; }; \
+  }
+am__installdirs = "$(DESTDIR)$(pkglibdir)" \
+	"$(DESTDIR)$(pkginc_libdir)"
+LTLIBRARIES = $(noinst_LTLIBRARIES) $(pkglib_LTLIBRARIES)
+am_libdovecot_dsync_la_OBJECTS =
+libdovecot_dsync_la_OBJECTS = $(am_libdovecot_dsync_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 = 
+libdovecot_dsync_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+	$(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+	$(AM_CFLAGS) $(CFLAGS) $(libdovecot_dsync_la_LDFLAGS) \
+	$(LDFLAGS) -o $@
+libdsync_la_LIBADD =
+am_libdsync_la_OBJECTS = dsync-brain.lo dsync-brain-mailbox.lo \
+	dsync-brain-mailbox-tree.lo dsync-brain-mailbox-tree-sync.lo \
+	dsync-brain-mails.lo dsync-deserializer.lo dsync-mail.lo \
+	dsync-mailbox.lo dsync-mailbox-import.lo \
+	dsync-mailbox-export.lo dsync-mailbox-state.lo \
+	dsync-mailbox-tree.lo dsync-mailbox-tree-fill.lo \
+	dsync-mailbox-tree-sync.lo dsync-serializer.lo dsync-ibc.lo \
+	dsync-ibc-stream.lo dsync-ibc-pipe.lo \
+	dsync-transaction-log-scan.lo
+libdsync_la_OBJECTS = $(am_libdsync_la_OBJECTS)
+am_test_dsync_mailbox_tree_sync_OBJECTS =  \
+	test-dsync-mailbox-tree-sync.$(OBJEXT)
+test_dsync_mailbox_tree_sync_OBJECTS =  \
+	$(am_test_dsync_mailbox_tree_sync_OBJECTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo "  GEN     " $@;
+am__v_GEN_1 = 
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 = 
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo \
+	./$(DEPDIR)/dsync-brain-mailbox-tree.Plo \
+	./$(DEPDIR)/dsync-brain-mailbox.Plo \
+	./$(DEPDIR)/dsync-brain-mails.Plo ./$(DEPDIR)/dsync-brain.Plo \
+	./$(DEPDIR)/dsync-deserializer.Plo \
+	./$(DEPDIR)/dsync-ibc-pipe.Plo \
+	./$(DEPDIR)/dsync-ibc-stream.Plo ./$(DEPDIR)/dsync-ibc.Plo \
+	./$(DEPDIR)/dsync-mail.Plo \
+	./$(DEPDIR)/dsync-mailbox-export.Plo \
+	./$(DEPDIR)/dsync-mailbox-import.Plo \
+	./$(DEPDIR)/dsync-mailbox-state.Plo \
+	./$(DEPDIR)/dsync-mailbox-tree-fill.Plo \
+	./$(DEPDIR)/dsync-mailbox-tree-sync.Plo \
+	./$(DEPDIR)/dsync-mailbox-tree.Plo \
+	./$(DEPDIR)/dsync-mailbox.Plo ./$(DEPDIR)/dsync-serializer.Plo \
+	./$(DEPDIR)/dsync-transaction-log-scan.Plo \
+	./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+	$(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+	$(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo "  CC      " $@;
+am__v_CC_1 = 
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+	$(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+	$(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo "  CCLD    " $@;
+am__v_CCLD_1 = 
+SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \
+	$(test_dsync_mailbox_tree_sync_SOURCES)
+DIST_SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \
+	$(test_dsync_mailbox_tree_sync_SOURCES)
+am__can_run_installinfo = \
+  case $$AM_UPDATE_INFO_DIR in \
+    n|no|NO) false;; \
+    *) (install-info --version) >/dev/null 2>&1;; \
+  esac
+HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates.  Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+  BEGIN { nonempty = 0; } \
+  { items[$$0] = 1; nonempty = 1; } \
+  END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique.  This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+  list='$(am__tagged_files)'; \
+  unique=`for i in $$list; do \
+    if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+  done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+pkglib_LTLIBRARIES = libdovecot-dsync.la
+noinst_LTLIBRARIES = libdsync.la
+AM_CPPFLAGS = \
+	-I$(top_srcdir)/src/lib \
+	-I$(top_srcdir)/src/lib-test \
+	-I$(top_srcdir)/src/lib-settings \
+	-I$(top_srcdir)/src/lib-ssl-iostream \
+	-I$(top_srcdir)/src/lib-master \
+	-I$(top_srcdir)/src/lib-mail \
+	-I$(top_srcdir)/src/lib-imap \
+	-I$(top_srcdir)/src/lib-index \
+	-I$(top_srcdir)/src/lib-storage
+
+libdsync_la_SOURCES = \
+	dsync-brain.c \
+	dsync-brain-mailbox.c \
+	dsync-brain-mailbox-tree.c \
+	dsync-brain-mailbox-tree-sync.c \
+	dsync-brain-mails.c \
+	dsync-deserializer.c \
+	dsync-mail.c \
+	dsync-mailbox.c \
+	dsync-mailbox-import.c \
+	dsync-mailbox-export.c \
+	dsync-mailbox-state.c \
+	dsync-mailbox-tree.c \
+	dsync-mailbox-tree-fill.c \
+	dsync-mailbox-tree-sync.c \
+	dsync-serializer.c \
+	dsync-ibc.c \
+	dsync-ibc-stream.c \
+	dsync-ibc-pipe.c \
+	dsync-transaction-log-scan.c
+
+libdovecot_dsync_la_SOURCES = 
+libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la
+libdovecot_dsync_la_DEPENDENCIES = libdsync.la
+libdovecot_dsync_la_LDFLAGS = -export-dynamic
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+	dsync-brain.h \
+	dsync-ibc.h
+
+noinst_HEADERS = \
+	dsync-brain-private.h \
+	dsync-mail.h \
+	dsync-mailbox.h \
+	dsync-mailbox-import.h \
+	dsync-mailbox-export.h \
+	dsync-mailbox-state.h \
+	dsync-mailbox-tree.h \
+	dsync-mailbox-tree-private.h \
+	dsync-serializer.h \
+	dsync-deserializer.h \
+	dsync-ibc-private.h \
+	dsync-transaction-log-scan.h
+
+test_programs = \
+	test-dsync-mailbox-tree-sync
+
+test_libs = \
+	../../lib-test/libtest.la \
+	../../lib/liblib.la
+
+test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c
+test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs)
+test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am  $(am__configure_deps)
+	@for dep in $?; do \
+	  case '$(am__configure_deps)' in \
+	    *$$dep*) \
+	      ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+	        && { if test -f $@; then exit 0; else break; fi; }; \
+	      exit 1;; \
+	  esac; \
+	done; \
+	echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile'; \
+	$(am__cd) $(top_srcdir) && \
+	  $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+	@case '$?' in \
+	  *config.status*) \
+	    cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+	  *) \
+	    echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+	    cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+	esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+	cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstPROGRAMS:
+	@list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+	echo " rm -f" $$list; \
+	rm -f $$list || exit $$?; \
+	test -n "$(EXEEXT)" || exit 0; \
+	list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+	echo " rm -f" $$list; \
+	rm -f $$list
+
+clean-noinstLTLIBRARIES:
+	-test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+	@list='$(noinst_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES)
+	@$(NORMAL_INSTALL)
+	@list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+	list2=; for p in $$list; do \
+	  if test -f $$p; then \
+	    list2="$$list2 $$p"; \
+	  else :; fi; \
+	done; \
+	test -z "$$list2" || { \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkglibdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkglibdir)" || exit 1; \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(pkglibdir)'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(pkglibdir)"; \
+	}
+
+uninstall-pkglibLTLIBRARIES:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+	for p in $$list; do \
+	  $(am__strip_dir) \
+	  echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(pkglibdir)/$$f'"; \
+	  $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(pkglibdir)/$$f"; \
+	done
+
+clean-pkglibLTLIBRARIES:
+	-test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES)
+	@list='$(pkglib_LTLIBRARIES)'; \
+	locs=`for p in $$list; do echo $$p; done | \
+	      sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+	      sort -u`; \
+	test -z "$$locs" || { \
+	  echo rm -f $${locs}; \
+	  rm -f $${locs}; \
+	}
+
+libdovecot-dsync.la: $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_DEPENDENCIES) $(EXTRA_libdovecot_dsync_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(libdovecot_dsync_la_LINK) -rpath $(pkglibdir) $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_LIBADD) $(LIBS)
+
+libdsync.la: $(libdsync_la_OBJECTS) $(libdsync_la_DEPENDENCIES) $(EXTRA_libdsync_la_DEPENDENCIES) 
+	$(AM_V_CCLD)$(LINK)  $(libdsync_la_OBJECTS) $(libdsync_la_LIBADD) $(LIBS)
+
+test-dsync-mailbox-tree-sync$(EXEEXT): $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_DEPENDENCIES) $(EXTRA_test_dsync_mailbox_tree_sync_DEPENDENCIES) 
+	@rm -f test-dsync-mailbox-tree-sync$(EXEEXT)
+	$(AM_V_CCLD)$(LINK) $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_LDADD) $(LIBS)
+
+mostlyclean-compile:
+	-rm -f *.$(OBJEXT)
+
+distclean-compile:
+	-rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mails.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-deserializer.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-pipe.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-stream.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-export.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-import.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-state.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-fill.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-sync.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-serializer.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-transaction-log-scan.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+	@$(MKDIR_P) $(@D)
+	@echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@	$(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@	$(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	$(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@	DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@	$(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+	-rm -f *.lo
+
+clean-libtool:
+	-rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+	@$(NORMAL_INSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	if test -n "$$list"; then \
+	  echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+	fi; \
+	for p in $$list; do \
+	  if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+	  echo "$$d$$p"; \
+	done | $(am__base_list) | \
+	while read files; do \
+	  echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+	  $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+	done
+
+uninstall-pkginc_libHEADERS:
+	@$(NORMAL_UNINSTALL)
+	@list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+	files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+	dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+	$(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	set x; \
+	here=`pwd`; \
+	$(am__define_uniq_tagged_files); \
+	shift; \
+	if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+	  test -n "$$unique" || unique=$$empty_fix; \
+	  if test $$# -gt 0; then \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      "$$@" $$unique; \
+	  else \
+	    $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+	      $$unique; \
+	  fi; \
+	fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+	$(am__define_uniq_tagged_files); \
+	test -z "$(CTAGS_ARGS)$$unique" \
+	  || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+	     $$unique
+
+GTAGS:
+	here=`$(am__cd) $(top_builddir) && pwd` \
+	  && $(am__cd) $(top_srcdir) \
+	  && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+	list='$(am__tagged_files)'; \
+	case "$(srcdir)" in \
+	  [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+	  *) sdir=$(subdir)/$(srcdir) ;; \
+	esac; \
+	for i in $$list; do \
+	  if test -f "$$i"; then \
+	    echo "$(subdir)/$$i"; \
+	  else \
+	    echo "$$sdir/$$i"; \
+	  fi; \
+	done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+	$(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+	@srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+	list='$(DISTFILES)'; \
+	  dist_files=`for file in $$list; do echo $$file; done | \
+	  sed -e "s|^$$srcdirstrip/||;t" \
+	      -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+	case $$dist_files in \
+	  */*) $(MKDIR_P) `echo "$$dist_files" | \
+			   sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+			   sort -u` ;; \
+	esac; \
+	for file in $$dist_files; do \
+	  if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+	  if test -d $$d/$$file; then \
+	    dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+	    if test -d "$(distdir)/$$file"; then \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+	      cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+	      find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+	    fi; \
+	    cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+	  else \
+	    test -f "$(distdir)/$$file" \
+	    || cp -p $$d/$$file "$(distdir)/$$file" \
+	    || exit 1; \
+	  fi; \
+	done
+check-am: all-am
+	$(MAKE) $(AM_MAKEFLAGS) check-local
+check: check-am
+all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS)
+installdirs:
+	for dir in "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(pkginc_libdir)"; do \
+	  test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+	done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+	@$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+	if test -z '$(STRIP)'; then \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	      install; \
+	else \
+	  $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+	    install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+	    "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+	fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+	-test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+	-test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+	@echo "This command is intended for maintainers to use"
+	@echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+	clean-noinstPROGRAMS clean-pkglibLTLIBRARIES mostlyclean-am
+
+distclean: distclean-am
+		-rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain-mails.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain.Plo
+	-rm -f ./$(DEPDIR)/dsync-deserializer.Plo
+	-rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo
+	-rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo
+	-rm -f ./$(DEPDIR)/dsync-ibc.Plo
+	-rm -f ./$(DEPDIR)/dsync-mail.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox.Plo
+	-rm -f ./$(DEPDIR)/dsync-serializer.Plo
+	-rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo
+	-rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po
+	-rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+	distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am: install-pkglibLTLIBRARIES
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+		-rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain-mails.Plo
+	-rm -f ./$(DEPDIR)/dsync-brain.Plo
+	-rm -f ./$(DEPDIR)/dsync-deserializer.Plo
+	-rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo
+	-rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo
+	-rm -f ./$(DEPDIR)/dsync-ibc.Plo
+	-rm -f ./$(DEPDIR)/dsync-mail.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo
+	-rm -f ./$(DEPDIR)/dsync-mailbox.Plo
+	-rm -f ./$(DEPDIR)/dsync-serializer.Plo
+	-rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo
+	-rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po
+	-rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+	mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES
+
+.MAKE: check-am install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \
+	check-local clean clean-generic clean-libtool \
+	clean-noinstLTLIBRARIES clean-noinstPROGRAMS \
+	clean-pkglibLTLIBRARIES cscopelist-am ctags ctags-am distclean \
+	distclean-compile distclean-generic distclean-libtool \
+	distclean-tags distdir dvi dvi-am html html-am info info-am \
+	install install-am install-data install-data-am install-dvi \
+	install-dvi-am install-exec install-exec-am install-html \
+	install-html-am install-info install-info-am install-man \
+	install-pdf install-pdf-am install-pkginc_libHEADERS \
+	install-pkglibLTLIBRARIES install-ps install-ps-am \
+	install-strip installcheck installcheck-am installdirs \
+	maintainer-clean maintainer-clean-generic mostlyclean \
+	mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
+	pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \
+	uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES
+
+.PRECIOUS: Makefile
+
+
+check-local:
+	for bin in $(test_programs); do \
+	  if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+	done
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c
new file mode 100644
index 0000000..2030e04
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c
@@ -0,0 +1,229 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-brain-private.h"
+
+static int
+sync_create_box(struct dsync_brain *brain, struct mailbox *box,
+		const guid_128_t mailbox_guid, uint32_t uid_validity,
+		enum mail_error *error_r)
+{
+	struct mailbox_metadata metadata;
+	struct mailbox_update update;
+	enum mail_error error;
+	const char *errstr;
+	int ret;
+
+	i_zero(&update);
+	memcpy(update.mailbox_guid, mailbox_guid, sizeof(update.mailbox_guid));
+	update.uid_validity = uid_validity;
+
+	if (mailbox_create(box, &update, FALSE) < 0) {
+		errstr = mailbox_get_last_internal_error(box, &error);
+		if (error != MAIL_ERROR_EXISTS) {
+			i_error("Can't create mailbox %s: %s",
+				mailbox_get_vname(box), errstr);
+			*error_r = error;
+			return -1;
+		}
+	}
+	if (brain->no_mail_sync) {
+		/* trust that create worked, we can't actually open it
+		   and verify. */
+		return 0;
+	}
+	/* sync the mailbox so we can look up its latest status */
+	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+		i_error("Can't sync mailbox %s: %s",
+			mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, error_r));
+		return -1;
+	}
+
+	/* verify that the GUID is what we wanted. if it's not, it probably
+	   means that the mailbox had already been created. then we'll use the
+	   GUID that is higher.
+
+	   mismatching UIDVALIDITY is handled later, because we choose it by
+	   checking which mailbox has more messages */
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) {
+		i_error("Can't get mailbox GUID %s: %s",
+			mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, error_r));
+		return -1;
+	}
+
+	ret = memcmp(mailbox_guid, metadata.guid, sizeof(metadata.guid));
+
+	/* if THEIR guid is bigger than OUR guid, and we are not doing
+	   backup in either direction, OR GUID did not match and we are
+	   receiving backup, try change the mailbox GUID.
+	*/
+
+	if ((ret > 0 && !brain->backup_recv &&
+	     !brain->backup_send) || (ret != 0 && brain->backup_recv)) {
+		if (brain->debug) {
+			i_debug("brain %c: Changing mailbox %s GUID %s -> %s",
+				brain->master_brain ? 'M' : 'S',
+				mailbox_get_vname(box),
+				guid_128_to_string(metadata.guid),
+				guid_128_to_string(mailbox_guid));
+		}
+		i_zero(&update);
+		memcpy(update.mailbox_guid, mailbox_guid,
+		       sizeof(update.mailbox_guid));
+		if (mailbox_update(box, &update) < 0) {
+			i_error("Can't update mailbox GUID %s: %s",
+				mailbox_get_vname(box),
+				mailbox_get_last_internal_error(box, error_r));
+			return -1;
+		}
+		/* verify that the update worked */
+		if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
+					 &metadata) < 0) {
+			i_error("Can't get mailbox GUID %s: %s",
+				mailbox_get_vname(box),
+				mailbox_get_last_internal_error(box, error_r));
+			return -1;
+		}
+		if (memcmp(mailbox_guid, metadata.guid,
+			   sizeof(metadata.guid)) != 0) {
+			i_error("Backend didn't update mailbox %s GUID",
+				mailbox_get_vname(box));
+			*error_r = MAIL_ERROR_TEMP;
+			return -1;
+		}
+	} else if (ret < 0) {
+		if (brain->debug) {
+			i_debug("brain %c: Other brain should change mailbox "
+				"%s GUID %s -> %s",
+				brain->master_brain ? 'M' : 'S',
+				mailbox_get_vname(box),
+				guid_128_to_string(mailbox_guid),
+				guid_128_to_string(metadata.guid));
+		}
+	}
+	return 0;
+}
+
+int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain,
+			const struct dsync_mailbox_tree_sync_change *change,
+			enum mail_error *error_r)
+{
+	struct mailbox *box = NULL, *destbox;
+	const char *errstr, *func_name = NULL, *storage_name;
+	enum mail_error error;
+	int ret = -1;
+
+	if (brain->backup_send) {
+		i_assert(brain->no_backup_overwrite);
+		return 0;
+	}
+
+	switch (change->type) {
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX:
+		/* make sure we're deleting the correct mailbox */
+		ret = dsync_brain_mailbox_alloc(brain, change->mailbox_guid,
+						&box, &errstr, error_r);
+		if (ret < 0) {
+			i_error("Mailbox sync: Couldn't allocate mailbox %s GUID %s: %s",
+				change->full_name,
+				guid_128_to_string(change->mailbox_guid), errstr);
+			return -1;
+		}
+		if (ret == 0) {
+			dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+				"Mailbox %s GUID %s deletion conflict: %s",
+				change->full_name,
+				guid_128_to_string(change->mailbox_guid), errstr));
+			return 0;
+		}
+		break;
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR:
+		storage_name = mailbox_list_get_storage_name(change->ns->list,
+							     change->full_name);
+		if (mailbox_list_delete_dir(change->ns->list, storage_name) == 0)
+			return 0;
+
+		errstr = mailbox_list_get_last_internal_error(change->ns->list, &error);
+		if (error == MAIL_ERROR_NOTFOUND ||
+		    error == MAIL_ERROR_EXISTS) {
+			dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+				"Mailbox %s mailbox_list_delete_dir conflict: %s",
+				change->full_name, errstr));
+			return 0;
+		} else {
+			i_error("Mailbox sync: mailbox_list_delete_dir failed: %s",
+				errstr);
+			*error_r = error;
+			return -1;
+		}
+	default:
+		box = mailbox_alloc(change->ns->list, change->full_name, 0);
+		break;
+	}
+	mailbox_skip_create_name_restrictions(box, TRUE);
+	switch (change->type) {
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX:
+		ret = sync_create_box(brain, box, change->mailbox_guid,
+				      change->uid_validity, error_r);
+		mailbox_free(&box);
+		return ret;
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR:
+		ret = mailbox_create(box, NULL, TRUE);
+		if (ret < 0 &&
+		    mailbox_get_last_mail_error(box) == MAIL_ERROR_EXISTS) {
+			/* it doesn't matter if somebody else created this
+			   directory or we automatically did while creating its
+			   child mailbox. it's there now anyway and we don't
+			   gain anything by treating this failure any
+			   differently from success. */
+			ret = 0;
+		}
+		func_name = "mailbox_create";
+		break;
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX:
+		ret = mailbox_delete(box);
+		func_name = "mailbox_delete";
+		break;
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR:
+		i_unreached();
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME:
+		destbox = mailbox_alloc(change->ns->list,
+					change->rename_dest_name, 0);
+		mailbox_skip_create_name_restrictions(destbox, TRUE);
+		ret = mailbox_rename(box, destbox);
+		func_name = "mailbox_rename";
+		mailbox_free(&destbox);
+		break;
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE:
+		ret = mailbox_set_subscribed(box, TRUE);
+		func_name = "mailbox_set_subscribed";
+		break;
+	case DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE:
+		ret = mailbox_set_subscribed(box, FALSE);
+		func_name = "mailbox_set_subscribed";
+		break;
+	}
+	if (ret < 0) {
+		errstr = mailbox_get_last_internal_error(box, &error);
+		if (error == MAIL_ERROR_EXISTS ||
+		    error == MAIL_ERROR_NOTFOUND) {
+			/* mailbox was already created or was already deleted.
+			   let the next sync figure out what to do */
+			dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+				"Mailbox %s %s conflict: %s",
+				mailbox_get_vname(box), func_name, errstr));
+			ret = 0;
+		} else {
+			i_error("Mailbox %s sync: %s failed: %s",
+				mailbox_get_vname(box), func_name, errstr);
+			*error_r = error;
+		}
+	}
+	mailbox_free(&box);
+	return ret;
+}
diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree.c b/src/doveadm/dsync/dsync-brain-mailbox-tree.c
new file mode 100644
index 0000000..44b76bf
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c
@@ -0,0 +1,624 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "mail-namespace.h"
+#include "mailbox-list-private.h"
+#include "dsync-ibc.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-brain-private.h"
+
+#include <ctype.h>
+
+static void dsync_brain_check_namespaces(struct dsync_brain *brain)
+{
+	struct mail_namespace *ns, *first_ns = NULL;
+	char sep, escape_char;
+
+	i_assert(brain->hierarchy_sep == '\0');
+	i_assert(brain->escape_char == '\0');
+
+	for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+		if (!dsync_brain_want_namespace(brain, ns))
+			continue;
+
+		sep = mail_namespace_get_sep(ns);
+		escape_char = mailbox_list_get_settings(ns->list)->vname_escape_char;
+		if (first_ns == NULL) {
+			brain->hierarchy_sep = sep;
+			brain->escape_char = escape_char;
+			first_ns = ns;
+		} else if (brain->hierarchy_sep != sep) {
+			i_fatal("Synced namespaces have conflicting separators "
+				"('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")",
+				brain->hierarchy_sep, first_ns->prefix,
+				sep, ns->prefix);
+		} else if (brain->escape_char != escape_char) {
+			i_fatal("Synced namespaces have conflicting escape chars "
+				"('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")",
+				brain->escape_char, first_ns->prefix,
+				escape_char, ns->prefix);
+		}
+	}
+	if (brain->hierarchy_sep != '\0')
+		return;
+
+	i_fatal("All your namespaces have a location setting. "
+		"Only namespaces with empty location settings are converted. "
+		"(One namespace should default to mail_location setting)");
+}
+
+void dsync_brain_mailbox_trees_init(struct dsync_brain *brain)
+{
+	struct mail_namespace *ns;
+
+	dsync_brain_check_namespaces(brain);
+
+	brain->local_mailbox_tree =
+		dsync_mailbox_tree_init(brain->hierarchy_sep,
+					brain->escape_char, brain->alt_char);
+	/* we'll convert remote mailbox names to use our own separator */
+	brain->remote_mailbox_tree =
+		dsync_mailbox_tree_init(brain->hierarchy_sep,
+					brain->escape_char, brain->alt_char);
+
+	/* fill the local mailbox tree */
+	for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+		if (!dsync_brain_want_namespace(brain, ns))
+			continue;
+		if (brain->debug)
+			i_debug("brain %c: Namespace %s has location %s",
+				brain->master_brain ? 'M' : 'S',
+				ns->prefix, ns->set->location);
+		if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, ns,
+					    brain->sync_box,
+					    brain->sync_box_guid,
+					    brain->exclude_mailboxes,
+					    &brain->mail_error) < 0) {
+			brain->failed = TRUE;
+			break;
+		}
+	}
+
+	brain->local_tree_iter =
+		dsync_mailbox_tree_iter_init(brain->local_mailbox_tree);
+}
+
+static const char *const *
+dsync_brain_mailbox_to_parts(struct dsync_brain *brain, const char *name)
+{
+	char sep[] = { brain->hierarchy_sep, '\0' };
+	char **parts = p_strsplit(unsafe_data_stack_pool, name, sep);
+	for (unsigned int i = 0; parts[i] != NULL; i++) {
+		mailbox_list_name_unescape((const char **)&parts[i],
+					   brain->escape_char);
+	}
+	return (const char *const *)parts;
+}
+
+void dsync_brain_send_mailbox_tree(struct dsync_brain *brain)
+{
+	struct dsync_mailbox_node *node;
+	enum dsync_ibc_send_ret ret;
+	const char *full_name;
+
+	while (dsync_mailbox_tree_iter_next(brain->local_tree_iter,
+					    &full_name, &node)) {
+		if (node->ns == NULL) {
+			/* This node was created when adding a namespace prefix
+			   to the tree that has multiple hierarchical names,
+			   but the parent names don't belong to any synced
+			   namespace. For example when syncing "-n Shared/user/"
+			   so "Shared/" is skipped. Or if there is e.g.
+			   "Public/files/" namespace prefix, but no "Public/"
+			   namespace at all. */
+			continue;
+		}
+
+		T_BEGIN {
+			const char *const *parts;
+
+			if (brain->debug) {
+				i_debug("brain %c: Local mailbox tree: %s %s",
+					brain->master_brain ? 'M' : 'S', full_name,
+					dsync_mailbox_node_to_string(node));
+			}
+
+			parts = dsync_brain_mailbox_to_parts(brain, full_name);
+			ret = dsync_ibc_send_mailbox_tree_node(brain->ibc,
+							       parts, node);
+		} T_END;
+		if (ret == DSYNC_IBC_SEND_RET_FULL)
+			return;
+	}
+	dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
+	dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_TREE);
+
+	brain->state = DSYNC_STATE_SEND_MAILBOX_TREE_DELETES;
+}
+
+void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox_delete *deletes;
+	unsigned int count;
+
+	deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree,
+						 &count);
+	dsync_ibc_send_mailbox_deletes(brain->ibc, deletes, count,
+				       brain->hierarchy_sep,
+				       brain->escape_char);
+
+	brain->state = DSYNC_STATE_RECV_MAILBOX_TREE;
+}
+
+static bool
+dsync_namespace_match_parts(struct mail_namespace *ns,
+			    const char *const *name_parts)
+{
+	const char *part, *prefix = ns->prefix;
+	size_t part_len;
+	char ns_sep = mail_namespace_get_sep(ns);
+
+	if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 &&
+	    strcmp(name_parts[0], "INBOX") == 0 && name_parts[1] == NULL)
+		return TRUE;
+
+	for (; *name_parts != NULL && *prefix != '\0'; name_parts++) {
+		part = *name_parts;
+		part_len = strlen(part);
+
+		if (!str_begins(prefix, part))
+			return FALSE;
+		if (prefix[part_len] != ns_sep)
+			return FALSE;
+		prefix += part_len + 1;
+	}
+	if (*name_parts != NULL) {
+		/* namespace prefix found with a mailbox */
+		return TRUE;
+	}
+	if (*prefix == '\0') {
+		/* namespace prefix itself matched */
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static struct mail_namespace *
+dsync_find_namespace(struct dsync_brain *brain, const char *const *name_parts)
+{
+	struct mail_namespace *ns, *best_ns = NULL;
+
+	for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+		if (!dsync_brain_want_namespace(brain, ns))
+			continue;
+
+		if (ns->prefix_len == 0) {
+			/* prefix="" is the fallback namespace */
+			if (best_ns == NULL)
+				best_ns = ns;
+		} else if (dsync_namespace_match_parts(ns, name_parts)) {
+			if (best_ns == NULL ||
+			    best_ns->prefix_len < ns->prefix_len)
+				best_ns = ns;
+		}
+	}
+	return best_ns;
+}
+
+static bool
+dsync_is_valid_name(struct mail_namespace *ns, const char *vname)
+{
+	struct mailbox *box;
+	bool ret;
+
+	box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
+	ret = mailbox_verify_create_name(box) == 0;
+	mailbox_free(&box);
+	return ret;
+}
+
+static bool
+dsync_is_valid_name_until(struct mail_namespace *ns, string_t *vname_full,
+			  unsigned int end_pos)
+{
+	const char *vname;
+	if (end_pos == str_len(vname_full))
+		vname = str_c(vname_full);
+	else
+		vname = t_strndup(str_c(vname_full), end_pos);
+	return dsync_is_valid_name(ns, vname);
+}
+
+static bool
+dsync_fix_mailbox_name_until(struct mail_namespace *ns, string_t *vname_full,
+			     char alt_char, unsigned int start_pos,
+			     unsigned int *_end_pos)
+{
+	unsigned int end_pos = *_end_pos;
+	unsigned int i;
+
+	if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+		return TRUE;
+
+	/* 1) change any real separators to alt separators (this
+	   wouldn't be necessary with listescape, but don't bother
+	   detecting it) */
+	char list_sep = mailbox_list_get_hierarchy_sep(ns->list);
+	char ns_sep = mail_namespace_get_sep(ns);
+	if (list_sep != ns_sep) {
+		char *v = str_c_modifiable(vname_full);
+		for (i = start_pos; i < end_pos; i++) {
+			if (v[i] == list_sep)
+				v[i] = alt_char;
+		}
+		if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+			return TRUE;
+	}
+
+	/* 2) '/' characters aren't valid without listescape */
+	if (ns_sep != '/' && list_sep != '/') {
+		char *v = str_c_modifiable(vname_full);
+		for (i = start_pos; i < end_pos; i++) {
+			if (v[i] == '/')
+				v[i] = alt_char;
+		}
+		if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+			return TRUE;
+	}
+
+	/* 3) probably some reserved name (e.g. dbox-Mails or ..) */
+	str_insert(vname_full, start_pos, "_"); end_pos++; *_end_pos += 1;
+	if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+		return TRUE;
+
+	return FALSE;
+}
+
+static void
+dsync_fix_mailbox_name(struct mail_namespace *ns, string_t *vname_str,
+		       char alt_char)
+{
+	const char *old_vname;
+	char *vname;
+	char ns_sep = mail_namespace_get_sep(ns);
+	guid_128_t guid;
+	unsigned int i, start_pos;
+
+	vname = str_c_modifiable(vname_str);
+	if (strncmp(vname, ns->prefix, ns->prefix_len) == 0)
+		start_pos = ns->prefix_len;
+	else
+		start_pos = 0;
+
+	/* replace control chars */
+	for (i = start_pos; vname[i] != '\0'; i++) {
+		if ((unsigned char)vname[i] < ' ')
+			vname[i] = alt_char;
+	}
+	/* make it valid UTF8 */
+	if (!uni_utf8_str_is_valid(vname)) {
+		old_vname = t_strdup(vname + start_pos);
+		str_truncate(vname_str, start_pos);
+		if (uni_utf8_get_valid_data((const void *)old_vname,
+					    strlen(old_vname), vname_str))
+			i_unreached();
+		vname = str_c_modifiable(vname_str);
+	}
+	if (dsync_is_valid_name(ns, vname))
+		return;
+
+	/* Check/fix each hierarchical name separately */
+	const char *p;
+	do {
+		i_assert(start_pos <= str_len(vname_str));
+		p = strchr(str_c(vname_str) + start_pos, ns_sep);
+		unsigned int end_pos;
+		if (p == NULL)
+			end_pos = str_len(vname_str);
+		else
+			end_pos = p - str_c(vname_str);
+
+		if (!dsync_fix_mailbox_name_until(ns, vname_str, alt_char,
+						  start_pos, &end_pos)) {
+			/* Couldn't fix it. Name is too long? Just give up and
+			   generate a unique name. */
+			guid_128_generate(guid);
+			str_truncate(vname_str, 0);
+			str_append(vname_str, ns->prefix);
+			str_append(vname_str, guid_128_to_string(guid));
+			i_assert(dsync_is_valid_name(ns, str_c(vname_str)));
+			break;
+		}
+		start_pos = end_pos + 1;
+	} while (p != NULL);
+}
+
+static int
+dsync_get_mailbox_name(struct dsync_brain *brain, const char *const *name_parts,
+		       const char **name_r, struct mail_namespace **ns_r)
+{
+	struct mail_namespace *ns;
+	const char *p;
+	string_t *vname;
+	char ns_sep;
+
+	i_assert(*name_parts != NULL);
+
+	ns = dsync_find_namespace(brain, name_parts);
+	if (ns == NULL)
+		return -1;
+	ns_sep = mail_namespace_get_sep(ns);
+
+	/* build the mailbox name */
+	char escape_chars[] = {
+		brain->escape_char,
+		ns_sep,
+		'\0'
+	};
+	struct dsync_mailbox_list *dlist = DSYNC_LIST_CONTEXT(ns->list);
+	if (dlist != NULL && !dlist->have_orig_escape_char) {
+		/* The escape character was added only for dsync internally.
+		   Normally there is no escape character configured. Change
+		   the mailbox names so that it doesn't rely on it. */
+		escape_chars[0] = '\0';
+	}
+	vname = t_str_new(128);
+	for (; *name_parts != NULL; name_parts++) {
+		if (escape_chars[0] != '\0') {
+			mailbox_list_name_escape(*name_parts, escape_chars,
+						 vname);
+		} else {
+			for (p = *name_parts; *p != '\0'; p++) {
+				if (*p != ns_sep)
+					str_append_c(vname, *p);
+				else
+					str_append_c(vname, brain->alt_char);
+			}
+		}
+		str_append_c(vname, ns_sep);
+	}
+	str_truncate(vname, str_len(vname)-1);
+
+	dsync_fix_mailbox_name(ns, vname, brain->alt_char);
+	*name_r = str_c(vname);
+	*ns_r = ns;
+	return 0;
+}
+
+static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain)
+{
+	struct dsync_mailbox_tree_sync_ctx *ctx;
+	const struct dsync_mailbox_tree_sync_change *change;
+	enum dsync_mailbox_trees_sync_type sync_type;
+	enum dsync_mailbox_trees_sync_flags sync_flags =
+		(brain->debug ? DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG : 0) |
+		(brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0);
+	int ret;
+
+	if (brain->no_backup_overwrite)
+		sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY;
+	else if (brain->backup_send)
+		sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL;
+	else if (brain->backup_recv)
+		sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE;
+	else
+		sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY;
+
+	ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree,
+					    brain->remote_mailbox_tree,
+					    sync_type, sync_flags);
+	while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) {
+		T_BEGIN {
+			ret = dsync_brain_mailbox_tree_sync_change(
+				brain, change, &brain->mail_error);
+		} T_END;
+		if (ret < 0) {
+			brain->failed = TRUE;
+			break;
+		}
+	}
+	if (dsync_mailbox_trees_sync_deinit(&ctx) < 0)
+		brain->failed = TRUE;
+}
+
+static int
+dsync_brain_recv_mailbox_tree_add(struct dsync_brain *brain,
+				  const char *const *parts,
+				  const struct dsync_mailbox_node *remote_node,
+				  const char *sep)
+{
+	struct dsync_mailbox_node *node;
+	struct mail_namespace *ns;
+	const char *name;
+
+	if (dsync_get_mailbox_name(brain, parts, &name, &ns) < 0)
+		return -1;
+	if (brain->debug) {
+		i_debug("brain %c: Remote mailbox tree: %s %s",
+			brain->master_brain ? 'M' : 'S',
+			t_strarray_join(parts, sep),
+			dsync_mailbox_node_to_string(remote_node));
+	}
+	node = dsync_mailbox_tree_get(brain->remote_mailbox_tree, name);
+	node->ns = ns;
+	dsync_mailbox_node_copy_data(node, remote_node);
+	return 0;
+}
+
+bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox_node *remote_node;
+	struct dsync_mailbox_node *dup_node1, *dup_node2;
+	const char *const *parts;
+	enum dsync_ibc_recv_ret ret;
+	int ret2;
+	char sep[2];
+	bool changed = FALSE;
+
+	sep[0] = brain->hierarchy_sep; sep[1] = '\0';
+	while ((ret = dsync_ibc_recv_mailbox_tree_node(brain->ibc, &parts,
+						       &remote_node)) > 0) {
+		T_BEGIN {
+			ret2 = dsync_brain_recv_mailbox_tree_add(
+					brain, parts, remote_node, sep);
+		} T_END;
+		if (ret2 < 0) {
+			i_error("Couldn't find namespace for mailbox %s",
+				t_strarray_join(parts, sep));
+			brain->failed = TRUE;
+			return TRUE;
+		}
+	}
+	if (ret != DSYNC_IBC_RECV_RET_FINISHED)
+		return changed;
+
+	if (dsync_mailbox_tree_build_guid_hash(brain->remote_mailbox_tree,
+					       &dup_node1, &dup_node2) < 0) {
+		i_error("Remote sent duplicate mailbox GUID %s for mailboxes %s and %s",
+			guid_128_to_string(dup_node1->mailbox_guid),
+			dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree,
+							 dup_node1),
+			dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree,
+							 dup_node2));
+		brain->failed = TRUE;
+	}
+
+	brain->state = DSYNC_STATE_RECV_MAILBOX_TREE_DELETES;
+	return TRUE;
+}
+
+static void
+dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree,
+				    struct dsync_mailbox_tree *other_tree,
+				    const struct dsync_mailbox_delete *other_del,
+				    const struct dsync_mailbox_node **node_r,
+				    const char **status_r)
+{
+	const struct dsync_mailbox_node *node;
+	struct dsync_mailbox_node *other_node, *old_node;
+	const char *name;
+
+	/* see if we can find the deletion based on mailbox tree that should
+	   still have the mailbox */
+	node = *node_r = dsync_mailbox_tree_find_delete(tree, other_del);
+	if (node == NULL) {
+		*status_r = "not found";
+		return;
+	}
+
+	switch (other_del->type) {
+	case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX:
+		/* mailbox is always deleted */
+		break;
+	case DSYNC_MAILBOX_DELETE_TYPE_DIR:
+		if (other_del->timestamp <= node->last_renamed_or_created) {
+			/* we don't want to delete this directory, we already
+			   have a newer timestamp for it */
+			*status_r = "keep directory, we have a newer timestamp";
+			return;
+		}
+		break;
+	case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE:
+		if (other_del->timestamp <= node->last_subscription_change) {
+			/* we don't want to unsubscribe, since we already have
+			   a newer subscription timestamp */
+			*status_r = "keep subscription, we have a newer timestamp";
+			return;
+		}
+		break;
+	}
+
+	/* make a node for it in the other mailbox tree */
+	name = dsync_mailbox_node_get_full_name(tree, node);
+	other_node = dsync_mailbox_tree_get(other_tree, name);
+
+	if (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+	    (!guid_128_is_empty(other_node->mailbox_guid) ||
+	     other_del->type != DSYNC_MAILBOX_DELETE_TYPE_MAILBOX)) {
+		/* other side has already created a new mailbox or
+		   directory with this name, we can't delete it */
+		*status_r = "name has already been recreated";
+		return;
+	}
+
+	/* ok, mark the other node deleted */
+	if (other_del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) {
+		memcpy(other_node->mailbox_guid, node->mailbox_guid,
+		       sizeof(other_node->mailbox_guid));
+	}
+	if (other_node->ns != node->ns && other_node->ns != NULL) {
+		/* namespace mismatch for this node. this shouldn't happen
+		   normally, but especially during some misconfigurations it's
+		   possible that one side has created mailboxes that conflict
+		   with another namespace's prefix. since we're here because
+		   one of the mailboxes was deleted, we'll just ignore this. */
+		*status_r = "namespace mismatch";
+		return;
+	}
+	other_node->ns = node->ns;
+	if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) {
+		other_node->existence = DSYNC_MAILBOX_NODE_DELETED;
+		*status_r = "marked as deleted";
+	} else {
+		other_node->last_subscription_change = other_del->timestamp;
+		other_node->subscribed = FALSE;
+		*status_r = "marked as unsubscribed";
+	}
+
+	if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node,
+					     &old_node) < 0)
+		i_unreached();
+}
+
+bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox_node *node;
+	const char *status;
+	const struct dsync_mailbox_delete *deletes;
+	unsigned int i, count;
+	char sep, escape_char;
+
+	if (dsync_ibc_recv_mailbox_deletes(brain->ibc, &deletes, &count,
+					   &sep, &escape_char) == 0)
+		return FALSE;
+
+	/* apply remote's mailbox deletions based on our local tree */
+	dsync_mailbox_tree_set_remote_chars(brain->local_mailbox_tree, sep,
+					    escape_char);
+	for (i = 0; i < count; i++) {
+		dsync_brain_mailbox_tree_add_delete(brain->local_mailbox_tree,
+						    brain->remote_mailbox_tree,
+						    &deletes[i], &node, &status);
+		if (brain->debug) {
+			const char *node_name = node == NULL ? "" :
+				dsync_mailbox_node_get_full_name(brain->local_mailbox_tree, node);
+			i_debug("brain %c: Remote mailbox tree deletion: guid=%s type=%s timestamp=%ld name=%s local update=%s",
+				brain->master_brain ? 'M' : 'S',
+				guid_128_to_string(deletes[i].guid),
+				dsync_mailbox_delete_type_to_string(deletes[i].type),
+				deletes[i].timestamp, node_name, status);
+		}
+	}
+
+	/* apply local mailbox deletions based on remote tree */
+	deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree,
+						 &count);
+	dsync_mailbox_tree_set_remote_chars(brain->remote_mailbox_tree,
+					    brain->hierarchy_sep,
+					    brain->escape_char);
+	for (i = 0; i < count; i++) {
+		dsync_brain_mailbox_tree_add_delete(brain->remote_mailbox_tree,
+						    brain->local_mailbox_tree,
+						    &deletes[i], &node, &status);
+	}
+
+	dsync_brain_mailbox_trees_sync(brain);
+	brain->state = brain->master_brain ?
+		DSYNC_STATE_MASTER_SEND_MAILBOX :
+		DSYNC_STATE_SLAVE_RECV_MAILBOX;
+	i_assert(brain->local_tree_iter == NULL);
+	brain->local_tree_iter =
+		dsync_mailbox_tree_iter_init(brain->local_mailbox_tree);
+	return TRUE;
+}
diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
new file mode 100644
index 0000000..759018f
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -0,0 +1,923 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "mail-cache-private.h"
+#include "mail-namespace.h"
+#include "mail-storage-private.h"
+#include "dsync-ibc.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-mailbox-import.h"
+#include "dsync-mailbox-export.h"
+#include "dsync-transaction-log-scan.h"
+#include "dsync-brain-private.h"
+
+static int
+ns_mailbox_try_alloc(struct dsync_brain *brain, struct mail_namespace *ns,
+		     const guid_128_t guid, struct mailbox **box_r,
+		     const char **errstr_r, enum mail_error *error_r)
+{
+	enum mailbox_flags flags = 0;
+	struct mailbox *box;
+	enum mailbox_existence existence;
+	int ret;
+
+	if (brain->backup_send) {
+		/* make sure mailbox isn't modified */
+		flags |= MAILBOX_FLAG_READONLY;
+	}
+
+	box = mailbox_alloc_guid(ns->list, guid, flags);
+	ret = mailbox_exists(box, FALSE, &existence);
+	if (ret < 0) {
+		*errstr_r = mailbox_get_last_internal_error(box, error_r);
+		mailbox_free(&box);
+		return -1;
+	}
+	if (existence != MAILBOX_EXISTENCE_SELECT) {
+		mailbox_free(&box);
+		*errstr_r = existence == MAILBOX_EXISTENCE_NONE ?
+			"Mailbox was already deleted" :
+			"Mailbox is no longer selectable";
+		return 0;
+	}
+	*box_r = box;
+	return 1;
+}
+
+int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
+			      struct mailbox **box_r, const char **errstr_r,
+			      enum mail_error *error_r)
+{
+	struct mail_namespace *ns;
+	int ret;
+
+	*box_r = NULL;
+
+	for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+		if (!dsync_brain_want_namespace(brain, ns))
+			continue;
+		if ((ret = ns_mailbox_try_alloc(brain, ns, guid, box_r,
+						errstr_r, error_r)) != 0)
+			return ret;
+	}
+	return 0;
+}
+
+static void
+dsync_mailbox_cache_field_dup(ARRAY_TYPE(mailbox_cache_field) *dest,
+			      const ARRAY_TYPE(mailbox_cache_field) *src,
+			      pool_t pool)
+{
+	const struct mailbox_cache_field *src_field;
+	struct mailbox_cache_field *dest_field;
+
+	p_array_init(dest, pool, array_count(src));
+	array_foreach(src, src_field) {
+		dest_field = array_append_space(dest);
+		dest_field->name = p_strdup(pool, src_field->name);
+		dest_field->decision = src_field->decision;
+		dest_field->last_used = src_field->last_used;
+	}
+}
+
+
+static const struct dsync_mailbox_state *
+dsync_mailbox_state_find(struct dsync_brain *brain,
+			 const guid_128_t mailbox_guid)
+{
+	const uint8_t *guid_p;
+
+	guid_p = mailbox_guid;
+	return hash_table_lookup(brain->mailbox_states, guid_p);
+}
+
+static void
+dsync_mailbox_state_remove(struct dsync_brain *brain,
+			   const guid_128_t mailbox_guid)
+{
+	const uint8_t *guid_p;
+
+	guid_p = mailbox_guid;
+	if (hash_table_lookup(brain->mailbox_states, guid_p) != NULL)
+		hash_table_remove(brain->mailbox_states, guid_p);
+}
+
+void dsync_brain_sync_init_box_states(struct dsync_brain *brain)
+{
+	if (brain->backup_send) {
+		/* we have an exporter, but no importer. */
+		brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES;
+		brain->box_recv_state = brain->mail_requests ?
+			DSYNC_BOX_STATE_MAIL_REQUESTS :
+			DSYNC_BOX_STATE_RECV_LAST_COMMON;
+	} else if (brain->backup_recv) {
+		/* we have an importer, but no exporter */
+		brain->box_send_state = brain->mail_requests ?
+			DSYNC_BOX_STATE_MAIL_REQUESTS :
+			DSYNC_BOX_STATE_DONE;
+		brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES;
+	} else {
+		brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES;
+		brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES;
+	}
+}
+
+static void
+dsync_brain_sync_mailbox_init(struct dsync_brain *brain,
+			      struct mailbox *box,
+			      struct file_lock *lock,
+			      const struct dsync_mailbox *local_dsync_box,
+			      bool wait_for_remote_box)
+{
+	const struct dsync_mailbox_state *state;
+
+	i_assert(brain->box_importer == NULL);
+	i_assert(brain->box_exporter == NULL);
+	i_assert(box->synced);
+
+	brain->box = box;
+	brain->box_lock = lock;
+	brain->pre_box_state = brain->state;
+	if (wait_for_remote_box) {
+		brain->box_send_state = DSYNC_BOX_STATE_MAILBOX;
+		brain->box_recv_state = DSYNC_BOX_STATE_MAILBOX;
+	} else {
+		dsync_brain_sync_init_box_states(brain);
+	}
+	brain->local_dsync_box = *local_dsync_box;
+	if (brain->dsync_box_pool != NULL)
+		p_clear(brain->dsync_box_pool);
+	else {
+		brain->dsync_box_pool =
+			pool_alloconly_create(MEMPOOL_GROWING"dsync brain box pool", 2048);
+	}
+	dsync_mailbox_cache_field_dup(&brain->local_dsync_box.cache_fields,
+				      &local_dsync_box->cache_fields,
+				      brain->dsync_box_pool);
+	i_zero(&brain->remote_dsync_box);
+
+	state = dsync_mailbox_state_find(brain, local_dsync_box->mailbox_guid);
+	if (state != NULL)
+		brain->mailbox_state = *state;
+	else {
+		i_zero(&brain->mailbox_state);
+		memcpy(brain->mailbox_state.mailbox_guid,
+		       local_dsync_box->mailbox_guid,
+		       sizeof(brain->mailbox_state.mailbox_guid));
+		brain->mailbox_state.last_uidvalidity =
+			local_dsync_box->uid_validity;
+	}
+}
+
+static void
+dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
+				     const struct dsync_mailbox *remote_dsync_box)
+{
+	enum dsync_mailbox_import_flags import_flags = 0;
+	const struct dsync_mailbox_state *state;
+	uint32_t last_common_uid;
+	uint64_t last_common_modseq, last_common_pvt_modseq;
+
+	i_assert(brain->box_importer == NULL);
+	i_assert(brain->log_scan != NULL);
+
+	i_assert(memcmp(brain->local_dsync_box.mailbox_guid,
+			remote_dsync_box->mailbox_guid,
+			sizeof(remote_dsync_box->mailbox_guid)) == 0);
+
+	brain->remote_dsync_box = *remote_dsync_box;
+	dsync_mailbox_cache_field_dup(&brain->remote_dsync_box.cache_fields,
+				      &remote_dsync_box->cache_fields,
+				      brain->dsync_box_pool);
+
+	state = dsync_mailbox_state_find(brain, remote_dsync_box->mailbox_guid);
+	if (state != NULL) {
+		last_common_uid = state->last_common_uid;
+		last_common_modseq = state->last_common_modseq;
+		last_common_pvt_modseq = state->last_common_pvt_modseq;
+	} else {
+		last_common_uid = 0;
+		last_common_modseq = 0;
+		last_common_pvt_modseq = 0;
+	}
+
+	if (brain->mail_requests)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS;
+	if (brain->master_brain)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN;
+	if (brain->backup_recv && !brain->no_backup_overwrite)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES;
+	if (brain->debug)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_DEBUG;
+	if (brain->local_dsync_box.have_save_guids &&
+	    (remote_dsync_box->have_save_guids ||
+	     (brain->backup_recv && remote_dsync_box->have_guids)))
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS;
+	if (brain->local_dsync_box.have_only_guid128 ||
+	    remote_dsync_box->have_only_guid128)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128;
+	if (brain->no_notify)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY;
+	if (brain->empty_hdr_workaround)
+		import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND;
+
+	brain->box_importer = brain->backup_send ? NULL :
+		dsync_mailbox_import_init(brain->box, brain->virtual_all_box,
+					  brain->log_scan,
+					  last_common_uid, last_common_modseq,
+					  last_common_pvt_modseq,
+					  remote_dsync_box->uid_next,
+					  remote_dsync_box->first_recent_uid,
+					  remote_dsync_box->highest_modseq,
+					  remote_dsync_box->highest_pvt_modseq,
+					  brain->sync_since_timestamp,
+					  brain->sync_until_timestamp,
+					  brain->sync_max_size,
+					  brain->sync_flag,
+					  brain->import_commit_msgs_interval,
+					  import_flags, brain->hdr_hash_version,
+					  brain->hashed_headers);
+}
+
+int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
+				  const struct dsync_mailbox *remote_dsync_box)
+{
+	struct mailbox_status status;
+	enum dsync_mailbox_exporter_flags exporter_flags = 0;
+	uint32_t last_common_uid, highest_wanted_uid;
+	uint64_t last_common_modseq, last_common_pvt_modseq;
+	const char *desync_reason = "";
+	bool pvt_too_old;
+	int ret;
+
+	i_assert(brain->log_scan == NULL);
+	i_assert(brain->box_exporter == NULL);
+
+	last_common_uid = brain->mailbox_state.last_common_uid;
+	last_common_modseq = brain->mailbox_state.last_common_modseq;
+	last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq;
+	highest_wanted_uid = last_common_uid == 0 ?
+		(uint32_t)-1 : last_common_uid;
+	ret = dsync_transaction_log_scan_init(brain->box->view,
+					      brain->box->view_pvt,
+					      highest_wanted_uid,
+					      last_common_modseq,
+					      last_common_pvt_modseq,
+					      &brain->log_scan, &pvt_too_old);
+	if (ret < 0) {
+		i_error("Failed to read transaction log for mailbox %s",
+			mailbox_get_vname(brain->box));
+		brain->failed = TRUE;
+		return -1;
+	}
+
+	mailbox_get_open_status(brain->box, STATUS_UIDNEXT |
+				STATUS_HIGHESTMODSEQ |
+				STATUS_HIGHESTPVTMODSEQ, &status);
+	if (status.nonpermanent_modseqs)
+		status.highest_modseq = 0;
+	if (ret == 0) {
+		if (pvt_too_old) {
+			desync_reason = t_strdup_printf(
+				"Private modseq %"PRIu64" no longer in transaction log "
+				"(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)",
+				last_common_pvt_modseq,
+				status.highest_pvt_modseq, last_common_uid,
+				status.uidnext);
+		} else {
+			desync_reason = t_strdup_printf(
+				"Modseq %"PRIu64" no longer in transaction log "
+				"(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)",
+				last_common_modseq,
+				status.highest_modseq, last_common_uid,
+				status.uidnext);
+		}
+	}
+
+	if (last_common_uid != 0) {
+		/* if last_common_* is higher than our current ones it means
+		   that the incremental sync state is stale and we need to do
+		   a full resync */
+		if (status.uidnext < last_common_uid) {
+			desync_reason = t_strdup_printf("uidnext %u < %u",
+				status.uidnext, last_common_uid);
+			ret = 0;
+		} else if (status.highest_modseq < last_common_modseq) {
+			desync_reason = t_strdup_printf("highest_modseq %"PRIu64" < %"PRIu64,
+				status.highest_modseq, last_common_modseq);
+			ret = 0;
+		} else if (status.highest_pvt_modseq < last_common_pvt_modseq) {
+			desync_reason = t_strdup_printf("highest_pvt_modseq %"PRIu64" < %"PRIu64,
+				status.highest_pvt_modseq, last_common_pvt_modseq);
+			ret = 0;
+		}
+	}
+	if (ret == 0) {
+		i_warning("Failed to do incremental sync for mailbox %s, "
+			  "retry with a full sync (%s)",
+			  mailbox_get_vname(brain->box), desync_reason);
+		dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+			"Incremental sync failed: %s", desync_reason));
+		brain->require_full_resync = TRUE;
+		return 0;
+	}
+
+	if (!brain->mail_requests)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS;
+	if (remote_dsync_box->have_save_guids &&
+	    (brain->local_dsync_box.have_save_guids ||
+	     (brain->backup_send && brain->local_dsync_box.have_guids)))
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
+	if (brain->no_mail_prefetch)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
+	if (brain->sync_since_timestamp > 0 ||
+	    brain->sync_until_timestamp > 0)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
+	if (brain->sync_max_size > 0)
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES;
+	if (remote_dsync_box->messages_count == 0) {
+		/* remote mailbox is empty - we don't really need to export
+		   header hashes since they're not going to match anything
+		   anyway. */
+		exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES;
+	}
+
+	brain->box_exporter = brain->backup_recv ? NULL :
+		dsync_mailbox_export_init(brain->box, brain->log_scan,
+					  last_common_uid,
+					  exporter_flags,
+					  brain->hdr_hash_version,
+					  brain->hashed_headers);
+	dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
+	return 1;
+}
+
+void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
+{
+	enum mail_error error;
+
+	i_assert(brain->box != NULL);
+
+	array_push_back(&brain->remote_mailbox_states, &brain->mailbox_state);
+	if (brain->box_exporter != NULL) {
+		const char *errstr;
+
+		i_assert(brain->failed || brain->require_full_resync ||
+			 brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED);
+		if (dsync_mailbox_export_deinit(&brain->box_exporter,
+						&errstr, &error) < 0)
+			i_error("Mailbox export failed: %s", errstr);
+	}
+	if (brain->box_importer != NULL) {
+		uint32_t last_common_uid, last_messages_count;
+		uint64_t last_common_modseq, last_common_pvt_modseq;
+		const char *changes_during_sync;
+		bool require_full_resync;
+
+		i_assert(brain->failed);
+		(void)dsync_mailbox_import_deinit(&brain->box_importer,
+						  FALSE,
+						  &last_common_uid,
+						  &last_common_modseq,
+						  &last_common_pvt_modseq,
+						  &last_messages_count,
+						  &changes_during_sync,
+						  &require_full_resync,
+						  &brain->mail_error);
+		if (require_full_resync)
+			brain->require_full_resync = TRUE;
+	}
+	if (brain->log_scan != NULL)
+		dsync_transaction_log_scan_deinit(&brain->log_scan);
+	file_lock_free(&brain->box_lock);
+	mailbox_free(&brain->box);
+
+	brain->state = brain->pre_box_state;
+}
+
+static int dsync_box_get(struct mailbox *box, struct dsync_mailbox *dsync_box_r,
+			 enum mail_error *error_r)
+{
+	const enum mailbox_status_items status_items =
+		STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES |
+		STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ |
+		STATUS_HIGHESTPVTMODSEQ;
+	const enum mailbox_metadata_items metadata_items =
+		MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID;
+	struct mailbox_status status;
+	struct mailbox_metadata metadata;
+	const char *errstr;
+	enum mail_error error;
+
+	/* get metadata first, since it may autocreate the mailbox */
+	if (mailbox_get_metadata(box, metadata_items, &metadata) < 0 ||
+	    mailbox_get_status(box, status_items, &status) < 0) {
+		errstr = mailbox_get_last_internal_error(box, &error);
+		if (error == MAIL_ERROR_NOTFOUND ||
+		    error == MAIL_ERROR_NOTPOSSIBLE) {
+			/* Mailbox isn't selectable, try the next one. We
+			   should have already caught \Noselect mailboxes, but
+			   check them anyway here. The NOTPOSSIBLE check is
+			   mainly for invalid mbox files. */
+			return 0;
+		}
+		i_error("Failed to access mailbox %s: %s",
+			mailbox_get_vname(box), errstr);
+		*error_r = error;
+		return -1;
+	}
+	if (status.nonpermanent_modseqs)
+		status.highest_modseq = 0;
+
+	i_assert(status.uidvalidity != 0 || status.messages == 0);
+
+	i_zero(dsync_box_r);
+	memcpy(dsync_box_r->mailbox_guid, metadata.guid,
+	       sizeof(dsync_box_r->mailbox_guid));
+	dsync_box_r->uid_validity = status.uidvalidity;
+	dsync_box_r->uid_next = status.uidnext;
+	dsync_box_r->messages_count = status.messages;
+	dsync_box_r->first_recent_uid = status.first_recent_uid;
+	dsync_box_r->highest_modseq = status.highest_modseq;
+	dsync_box_r->highest_pvt_modseq = status.highest_pvt_modseq;
+	dsync_mailbox_cache_field_dup(&dsync_box_r->cache_fields,
+				      metadata.cache_fields,
+				      pool_datastack_create());
+	dsync_box_r->have_guids = status.have_guids;
+	dsync_box_r->have_save_guids = status.have_save_guids;
+	dsync_box_r->have_only_guid128 = status.have_only_guid128;
+	return 1;
+}
+
+static bool
+dsync_brain_has_mailbox_state_changed(struct dsync_brain *brain,
+				      const struct dsync_mailbox *dsync_box)
+{
+	const struct dsync_mailbox_state *state;
+
+	if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE)
+		return TRUE;
+
+	state = dsync_mailbox_state_find(brain, dsync_box->mailbox_guid);
+	return state == NULL ||
+		state->last_uidvalidity != dsync_box->uid_validity ||
+		state->last_common_uid+1 != dsync_box->uid_next ||
+		state->last_common_modseq != dsync_box->highest_modseq ||
+		state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq ||
+		state->last_messages_count != dsync_box->messages_count;
+}
+
+static int
+dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r,
+			     struct file_lock **lock_r,
+			     struct dsync_mailbox *dsync_box_r)
+{
+	enum mailbox_flags flags = 0;
+	struct dsync_mailbox dsync_box;
+	struct mailbox *box;
+	struct file_lock *lock = NULL;
+	struct dsync_mailbox_node *node;
+	const char *vname = NULL;
+	enum mail_error error;
+	bool synced = FALSE;
+	int ret;
+
+	*box_r = NULL;
+
+	while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) {
+		if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    !guid_128_is_empty(node->mailbox_guid))
+			break;
+		vname = NULL;
+	}
+	if (vname == NULL) {
+		/* no more mailboxes */
+		dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
+		return -1;
+	}
+
+	if (brain->backup_send) {
+		/* make sure mailbox isn't modified */
+		flags |= MAILBOX_FLAG_READONLY;
+	}
+	box = mailbox_alloc(node->ns->list, vname, flags);
+	for (;;) {
+		if ((ret = dsync_box_get(box, &dsync_box, &error)) <= 0) {
+			if (ret < 0) {
+				brain->mail_error = error;
+				brain->failed = TRUE;
+			}
+			mailbox_free(&box);
+			file_lock_free(&lock);
+			return ret;
+		}
+
+		/* if mailbox's last_common_* state equals the current state,
+		   we can skip the mailbox */
+		if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) {
+			if (brain->debug) {
+				i_debug("brain %c: Skipping mailbox %s with unchanged state "
+					"uidvalidity=%u uidnext=%u highestmodseq=%"PRIu64" highestpvtmodseq=%"PRIu64" messages=%u",
+					brain->master_brain ? 'M' : 'S',
+					guid_128_to_string(dsync_box.mailbox_guid),
+					dsync_box.uid_validity,
+					dsync_box.uid_next,
+					dsync_box.highest_modseq,
+					dsync_box.highest_pvt_modseq,
+					dsync_box.messages_count);
+			}
+			mailbox_free(&box);
+			file_lock_free(&lock);
+			return 0;
+		}
+		if (synced) {
+			/* ok, the mailbox really changed */
+			break;
+		}
+
+		/* mailbox appears to have changed. do a full sync here and get the
+		   state again. Lock before syncing. */
+		if (dsync_mailbox_lock(brain, box, &lock) < 0) {
+			brain->failed = TRUE;
+			mailbox_free(&box);
+			return -1;
+		}
+		if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+			i_error("Can't sync mailbox %s: %s",
+				mailbox_get_vname(box),
+				mailbox_get_last_internal_error(box, &brain->mail_error));
+			brain->failed = TRUE;
+			mailbox_free(&box);
+			file_lock_free(&lock);
+			return -1;
+		}
+		synced = TRUE;
+	}
+
+	*box_r = box;
+	*lock_r = lock;
+	*dsync_box_r = dsync_box;
+	return 1;
+}
+
+static bool
+dsync_brain_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r,
+			 struct file_lock **lock_r,
+			 struct dsync_mailbox *dsync_box_r)
+{
+	int ret;
+
+	if (brain->no_mail_sync)
+		return FALSE;
+
+	while ((ret = dsync_brain_try_next_mailbox(brain, box_r, lock_r, dsync_box_r)) == 0)
+		;
+	return ret > 0;
+}
+
+void dsync_brain_master_send_mailbox(struct dsync_brain *brain)
+{
+	struct dsync_mailbox dsync_box;
+	struct mailbox *box;
+	struct file_lock *lock;
+
+	i_assert(brain->master_brain);
+	i_assert(brain->box == NULL);
+
+	if (!dsync_brain_next_mailbox(brain, &box, &lock, &dsync_box)) {
+		brain->state = DSYNC_STATE_FINISH;
+		dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX);
+		return;
+	}
+
+	/* start exporting this mailbox (wait for remote to start importing) */
+	dsync_ibc_send_mailbox(brain->ibc, &dsync_box);
+	dsync_brain_sync_mailbox_init(brain, box, lock, &dsync_box, TRUE);
+	brain->state = DSYNC_STATE_SYNC_MAILS;
+}
+
+bool dsync_boxes_need_sync(struct dsync_brain *brain,
+			   const struct dsync_mailbox *box1,
+			   const struct dsync_mailbox *box2,
+			   const char **reason_r)
+{
+	if (brain->no_mail_sync)
+		return FALSE;
+	if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_CHANGED) {
+		*reason_r = "Full sync";
+		return TRUE;
+	}
+	if (box1->uid_validity != box2->uid_validity)
+		*reason_r = t_strdup_printf("UIDVALIDITY changed: %u -> %u",
+			box1->uid_validity, box2->uid_validity);
+	else if (box1->uid_next != box2->uid_next)
+		*reason_r = t_strdup_printf("UIDNEXT changed: %u -> %u",
+			box1->uid_next, box2->uid_next);
+	else if (box1->messages_count != box2->messages_count)
+		*reason_r = t_strdup_printf("Message count changed: %u -> %u",
+			box1->messages_count, box2->messages_count);
+	else if (box1->highest_modseq != box2->highest_modseq) {
+		*reason_r = t_strdup_printf("HIGHESTMODSEQ changed %"
+					    PRIu64" -> %"PRIu64,
+					    box1->highest_modseq,
+					    box2->highest_modseq);
+		if (box1->highest_modseq == 0 ||
+		    box2->highest_modseq == 0) {
+			*reason_r = t_strdup_printf(
+				"%s (Permanent MODSEQs aren't supported)",
+				*reason_r);
+		}
+	} else if (box1->highest_pvt_modseq != box2->highest_pvt_modseq)
+		*reason_r = t_strdup_printf("Private HIGHESTMODSEQ changed %"
+					    PRIu64" -> %"PRIu64,
+					    box1->highest_pvt_modseq,
+					    box2->highest_pvt_modseq);
+	else if (box1->first_recent_uid != box2->first_recent_uid)
+		*reason_r = t_strdup_printf("First RECENT UID changed: %u -> %u",
+			box1->first_recent_uid, box2->first_recent_uid);
+	else
+		return FALSE;
+	return TRUE;
+}
+
+static int
+mailbox_cache_field_name_cmp(const struct mailbox_cache_field *f1,
+			     const struct mailbox_cache_field *f2)
+{
+	return strcmp(f1->name, f2->name);
+}
+
+static void
+dsync_cache_fields_update(const struct dsync_mailbox *local_box,
+			  const struct dsync_mailbox *remote_box,
+			  struct mailbox *box,
+			  struct mailbox_update *update)
+{
+	ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes;
+	const struct mailbox_cache_field *local_fields, *remote_fields;
+	unsigned int li, ri, local_count, remote_count;
+	time_t drop_older_timestamp;
+	int ret;
+
+	if (array_count(&remote_box->cache_fields) == 0) {
+		/* remote has no cached fields. there's nothing to update. */
+		return;
+	}
+
+	t_array_init(&local_sorted, array_count(&local_box->cache_fields));
+	t_array_init(&remote_sorted, array_count(&remote_box->cache_fields));
+	array_append_array(&local_sorted, &local_box->cache_fields);
+	array_append_array(&remote_sorted, &remote_box->cache_fields);
+	array_sort(&local_sorted, mailbox_cache_field_name_cmp);
+	array_sort(&remote_sorted, mailbox_cache_field_name_cmp);
+
+	if (array_count(&local_sorted) == 0) {
+		/* local has no cached fields. set them to same as remote. */
+		array_append_zero(&remote_sorted);
+		update->cache_updates = array_front(&remote_sorted);
+		return;
+	}
+
+	/* figure out what to change */
+	local_fields = array_get(&local_sorted, &local_count);
+	remote_fields = array_get(&remote_sorted, &remote_count);
+	t_array_init(&changes, local_count + remote_count);
+	drop_older_timestamp = ioloop_time -
+		box->index->optimization_set.cache.unaccessed_field_drop_secs;
+
+	for (li = ri = 0; li < local_count || ri < remote_count; ) {
+		ret = li == local_count ? 1 :
+			ri == remote_count ? -1 :
+			strcmp(local_fields[li].name, remote_fields[ri].name);
+		if (ret == 0) {
+			/* field exists in both local and remote */
+			const struct mailbox_cache_field *lf = &local_fields[li];
+			const struct mailbox_cache_field *rf = &remote_fields[ri];
+
+			if (lf->last_used > rf->last_used ||
+			    (lf->last_used == rf->last_used &&
+			     lf->decision > rf->decision)) {
+				/* use local decision and timestamp */
+			} else {
+				array_push_back(&changes, rf);
+			}
+			li++; ri++;
+		} else if (ret < 0) {
+			/* remote field doesn't exist */
+			li++;
+		} else {
+			/* local field doesn't exist */
+			if (remote_fields[ri].last_used < drop_older_timestamp) {
+				/* field hasn't be used for a long time, remote
+				   will probably drop this soon as well */
+			} else {
+				array_push_back(&changes, &remote_fields[ri]);
+			}
+			ri++;
+		}
+	}
+	i_assert(li == local_count && ri == remote_count);
+	if (array_count(&changes) > 0) {
+		array_append_zero(&changes);
+		update->cache_updates = array_front(&changes);
+	}
+}
+
+bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
+				    struct mailbox *box,
+				    const struct dsync_mailbox *local_box,
+				    const struct dsync_mailbox *remote_box,
+				    const char **reason_r)
+{
+	struct mailbox_update update;
+	const struct dsync_mailbox_state *state;
+	bool ret = TRUE;
+
+	*reason_r = NULL;
+	i_zero(&update);
+
+	if (local_box->uid_validity != remote_box->uid_validity) {
+		/* Keep the UIDVALIDITY for the mailbox that has more
+		   messages. If they equal, use the higher UIDVALIDITY. */
+		if (remote_box->messages_count > local_box->messages_count ||
+		    (remote_box->messages_count == local_box->messages_count &&
+		     remote_box->uid_validity > local_box->uid_validity))
+			update.uid_validity = remote_box->uid_validity;
+
+		state = dsync_mailbox_state_find(brain, local_box->mailbox_guid);
+		if (state != NULL && state->last_common_uid > 0) {
+			/* we can't continue syncing this mailbox in this
+			   session, because the other side already started
+			   sending mailbox changes, but not for all mails. */
+			dsync_mailbox_state_remove(brain, local_box->mailbox_guid);
+			*reason_r = "UIDVALIDITY changed during a stateful sync, need to restart";
+			brain->failed = TRUE;
+			ret = FALSE;
+		}
+	}
+
+	dsync_cache_fields_update(local_box, remote_box, box, &update);
+
+	if (update.uid_validity == 0 &&
+	    update.cache_updates == NULL) {
+		/* no changes */
+		return ret;
+	}
+
+	if (mailbox_update(box, &update) < 0) {
+		i_error("Couldn't update mailbox %s metadata: %s",
+			mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, &brain->mail_error));
+		brain->failed = TRUE;
+	}
+	return ret;
+}
+
+static void
+dsync_brain_slave_send_mailbox_lost(struct dsync_brain *brain,
+				    const struct dsync_mailbox *dsync_box,
+				    bool ignore)
+{
+	struct dsync_mailbox delete_box;
+
+	if (brain->debug) {
+		i_debug("brain %c: We don't have mailbox %s",
+			brain->master_brain ? 'M' : 'S',
+			guid_128_to_string(dsync_box->mailbox_guid));
+	}
+	i_zero(&delete_box);
+	memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid,
+	       sizeof(delete_box.mailbox_guid));
+	t_array_init(&delete_box.cache_fields, 0);
+	if (ignore)
+		delete_box.mailbox_ignore = TRUE;
+	else
+		delete_box.mailbox_lost = TRUE;
+	dsync_ibc_send_mailbox(brain->ibc, &delete_box);
+}
+
+bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox *dsync_box;
+	struct dsync_mailbox local_dsync_box;
+	struct mailbox *box;
+	struct file_lock *lock;
+	const char *errstr, *resync_reason, *reason;
+	enum mail_error error;
+	int ret;
+	bool resync;
+
+	i_assert(!brain->master_brain);
+	i_assert(brain->box == NULL);
+
+	if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0)
+		return FALSE;
+	if (ret < 0) {
+		brain->state = DSYNC_STATE_FINISH;
+		return TRUE;
+	}
+
+	if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid,
+				      &box, &errstr, &error) < 0) {
+		i_error("Couldn't allocate mailbox GUID %s: %s",
+			guid_128_to_string(dsync_box->mailbox_guid), errstr);
+		brain->mail_error = error;
+		brain->failed = TRUE;
+		return TRUE;
+	}
+	if (box == NULL) {
+		/* mailbox was probably deleted/renamed during sync */
+		if (brain->backup_send && brain->no_backup_overwrite) {
+			if (brain->debug) {
+				i_debug("brain %c: Ignore nonexistent "
+					"mailbox GUID %s with -1 sync",
+					brain->master_brain ? 'M' : 'S',
+					guid_128_to_string(dsync_box->mailbox_guid));
+			}
+			dsync_brain_slave_send_mailbox_lost(brain, dsync_box, TRUE);
+			return TRUE;
+		}
+		//FIXME: verify this from log, and if not log an error.
+		dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+			"Mailbox GUID %s was lost",
+			guid_128_to_string(dsync_box->mailbox_guid)));
+		dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE);
+		return TRUE;
+	}
+	/* Lock before syncing */
+	if (dsync_mailbox_lock(brain, box, &lock) < 0) {
+		mailbox_free(&box);
+		brain->failed = TRUE;
+		return TRUE;
+	}
+	if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+		i_error("Can't sync mailbox %s: %s",
+			mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, &brain->mail_error));
+		file_lock_free(&lock);
+		mailbox_free(&box);
+		brain->failed = TRUE;
+		return TRUE;
+	}
+
+	if ((ret = dsync_box_get(box, &local_dsync_box, &error)) <= 0) {
+		file_lock_free(&lock);
+		mailbox_free(&box);
+		if (ret < 0) {
+			brain->mail_error = error;
+			brain->failed = TRUE;
+			return TRUE;
+		}
+		/* another process just deleted this mailbox? */
+		if (brain->debug) {
+			i_debug("brain %c: Skipping lost mailbox %s",
+				brain->master_brain ? 'M' : 'S',
+				guid_128_to_string(dsync_box->mailbox_guid));
+		}
+		dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE);
+		return TRUE;
+	}
+	i_assert(local_dsync_box.uid_validity != 0);
+	i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid,
+			sizeof(dsync_box->mailbox_guid)) == 0);
+
+	resync = !dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box,
+						 dsync_box, &resync_reason);
+
+	if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box, &reason)) {
+		/* no fields appear to have changed, skip this mailbox */
+		if (brain->debug) {
+			i_debug("brain %c: Skipping unchanged mailbox %s",
+				brain->master_brain ? 'M' : 'S',
+				guid_128_to_string(dsync_box->mailbox_guid));
+		}
+		dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
+		file_lock_free(&lock);
+		mailbox_free(&box);
+		return TRUE;
+	}
+	if (brain->debug) {
+		i_debug("brain %c: Syncing mailbox %s: %s",
+			brain->master_brain ? 'M' : 'S',
+			guid_128_to_string(dsync_box->mailbox_guid), reason);
+	}
+
+	/* start export/import */
+	dsync_brain_sync_mailbox_init(brain, box, lock, &local_dsync_box, FALSE);
+	if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0)
+		return TRUE;
+	if (resync)
+		dsync_brain_set_changes_during_sync(brain, resync_reason);
+	if (ret == 0 || resync) {
+		brain->require_full_resync = TRUE;
+		dsync_brain_sync_mailbox_deinit(brain);
+		dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE);
+		return TRUE;
+	}
+
+	dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
+	brain->state = DSYNC_STATE_SYNC_MAILS;
+	return TRUE;
+}
diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c
new file mode 100644
index 0000000..3feea20
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mails.c
@@ -0,0 +1,438 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "dsync-ibc.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox-import.h"
+#include "dsync-mailbox-export.h"
+#include "dsync-brain-private.h"
+
+const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1] = {
+	"mailbox",
+	"changes",
+	"attributes",
+	"mail_requests",
+	"mails",
+	"recv_last_common",
+	"done"
+};
+
+static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox *dsync_box;
+	const char *resync_reason, *reason;
+	enum dsync_ibc_recv_ret ret;
+	bool resync;
+
+	i_assert(brain->master_brain);
+
+	if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0)
+		return FALSE;
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		i_error("Remote sent end-of-list instead of a mailbox");
+		brain->failed = TRUE;
+		return TRUE;
+	}
+	if (memcmp(dsync_box->mailbox_guid, brain->local_dsync_box.mailbox_guid,
+		   sizeof(dsync_box->mailbox_guid)) != 0) {
+		i_error("Remote sent mailbox with a wrong GUID");
+		brain->failed = TRUE;
+		return TRUE;
+	}
+
+	if (dsync_box->mailbox_ignore) {
+		/* ignore this box */
+		if (brain->debug)
+			i_debug("brain %c: Ignoring missing remote box GUID %s",
+				brain->master_brain ? 'M' : 'S',
+			        guid_128_to_string(dsync_box->mailbox_guid));
+		dsync_brain_sync_mailbox_deinit(brain);
+		return TRUE;
+	}
+	if (dsync_box->mailbox_lost) {
+		/* remote lost the mailbox. it's probably already deleted, but
+		   verify it on next sync just to be sure */
+		dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+			"Remote lost mailbox GUID %s (maybe it was just deleted?)",
+			guid_128_to_string(dsync_box->mailbox_guid)));
+		brain->require_full_resync = TRUE;
+		dsync_brain_sync_mailbox_deinit(brain);
+		return TRUE;
+	}
+	resync = !dsync_brain_mailbox_update_pre(brain, brain->box,
+						 &brain->local_dsync_box,
+						 dsync_box, &resync_reason);
+
+	if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box,
+				   &reason)) {
+		/* no fields appear to have changed, skip this mailbox */
+		dsync_brain_sync_mailbox_deinit(brain);
+		return TRUE;
+	}
+	if (brain->debug) {
+		i_debug("brain %c: Syncing mailbox %s: %s",
+			brain->master_brain ? 'M' : 'S',
+			guid_128_to_string(dsync_box->mailbox_guid), reason);
+	}
+	if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0)
+		return TRUE;
+	if (resync)
+		dsync_brain_set_changes_during_sync(brain, resync_reason);
+	if (ret == 0 || resync) {
+		brain->require_full_resync = TRUE;
+		brain->failed = TRUE;
+		dsync_brain_sync_mailbox_deinit(brain);
+		return TRUE;
+	}
+	dsync_brain_sync_init_box_states(brain);
+	return TRUE;
+}
+
+static bool dsync_brain_recv_mailbox_attribute(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox_attribute *attr;
+	struct istream *input;
+	enum dsync_ibc_recv_ret ret;
+
+	if ((ret = dsync_ibc_recv_mailbox_attribute(brain->ibc, &attr)) == 0)
+		return FALSE;
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		brain->box_recv_state = DSYNC_BOX_STATE_CHANGES;
+		return TRUE;
+	}
+	if (dsync_mailbox_import_attribute(brain->box_importer, attr) < 0)
+		brain->failed = TRUE;
+	input = attr->value_stream;
+	i_stream_unref(&input);
+	return TRUE;
+}
+
+static void dsync_brain_send_end_of_list(struct dsync_brain *brain,
+					 enum dsync_ibc_eol_type type)
+{
+	i_assert(!brain->failed);
+	dsync_ibc_send_end_of_list(brain->ibc, type);
+}
+
+static int dsync_brain_export_deinit(struct dsync_brain *brain)
+{
+	const char *errstr;
+	enum mail_error error;
+
+	if (dsync_mailbox_export_deinit(&brain->box_exporter,
+					&errstr, &error) < 0) {
+		i_error("Exporting mailbox %s failed: %s",
+			mailbox_get_vname(brain->box), errstr);
+		brain->mail_error = error;
+		brain->failed = TRUE;
+		return -1;
+	}
+	return 0;
+}
+
+static void dsync_brain_send_mailbox_attribute(struct dsync_brain *brain)
+{
+	const struct dsync_mailbox_attribute *attr;
+	int ret;
+
+	while ((ret = dsync_mailbox_export_next_attr(brain->box_exporter, &attr)) > 0) {
+		if (dsync_ibc_send_mailbox_attribute(brain->ibc, attr) == 0)
+			return;
+	}
+	if (ret < 0) {
+		if (dsync_brain_export_deinit(brain) == 0)
+			i_unreached();
+		return;
+	}
+	dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE);
+	brain->box_send_state = DSYNC_BOX_STATE_CHANGES;
+}
+
+static bool dsync_brain_recv_mail_change(struct dsync_brain *brain)
+{
+	const struct dsync_mail_change *change;
+	enum dsync_ibc_recv_ret ret;
+
+	if ((ret = dsync_ibc_recv_change(brain->ibc, &change)) == 0)
+		return FALSE;
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		if (dsync_mailbox_import_changes_finish(brain->box_importer) < 0)
+			brain->failed = TRUE;
+		if (brain->mail_requests && brain->box_exporter != NULL)
+			brain->box_recv_state = DSYNC_BOX_STATE_MAIL_REQUESTS;
+		else
+			brain->box_recv_state = DSYNC_BOX_STATE_MAILS;
+		return TRUE;
+	}
+	if (dsync_mailbox_import_change(brain->box_importer, change) < 0)
+		brain->failed = TRUE;
+	return TRUE;
+}
+
+static void dsync_brain_send_mail_change(struct dsync_brain *brain)
+{
+	const struct dsync_mail_change *change;
+	int ret;
+
+	while ((ret = dsync_mailbox_export_next(brain->box_exporter, &change)) > 0) {
+		if (dsync_ibc_send_change(brain->ibc, change) == 0)
+			return;
+	}
+	if (ret < 0) {
+		if (dsync_brain_export_deinit(brain) == 0)
+			i_unreached();
+		return;
+	}
+	dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_CHANGES);
+	if (brain->mail_requests && brain->box_importer != NULL)
+		brain->box_send_state = DSYNC_BOX_STATE_MAIL_REQUESTS;
+	else
+		brain->box_send_state = DSYNC_BOX_STATE_MAILS;
+}
+
+static bool dsync_brain_recv_mail_request(struct dsync_brain *brain)
+{
+	const struct dsync_mail_request *request;
+	enum dsync_ibc_recv_ret ret;
+
+	i_assert(brain->mail_requests);
+	i_assert(brain->box_exporter != NULL);
+
+	if ((ret = dsync_ibc_recv_mail_request(brain->ibc, &request)) == 0)
+		return FALSE;
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		brain->box_recv_state = brain->box_importer != NULL ?
+			DSYNC_BOX_STATE_MAILS :
+			DSYNC_BOX_STATE_RECV_LAST_COMMON;
+		return TRUE;
+	}
+	dsync_mailbox_export_want_mail(brain->box_exporter, request);
+	return TRUE;
+}
+
+static bool dsync_brain_send_mail_request(struct dsync_brain *brain)
+{
+	const struct dsync_mail_request *request;
+
+	i_assert(brain->mail_requests);
+
+	while ((request = dsync_mailbox_import_next_request(brain->box_importer)) != NULL) {
+		if (dsync_ibc_send_mail_request(brain->ibc, request) == 0)
+			return TRUE;
+	}
+	if (brain->box_recv_state < DSYNC_BOX_STATE_MAIL_REQUESTS)
+		return FALSE;
+
+	dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_REQUESTS);
+	if (brain->box_exporter != NULL)
+		brain->box_send_state = DSYNC_BOX_STATE_MAILS;
+	else {
+		i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE);
+		brain->box_send_state = DSYNC_BOX_STATE_DONE;
+	}
+	return TRUE;
+}
+
+static void dsync_brain_sync_half_finished(struct dsync_brain *brain)
+{
+	struct dsync_mailbox_state state;
+	const char *changes_during_sync;
+	bool require_full_resync;
+
+	if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON ||
+	    brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON)
+		return;
+
+	/* finished with this mailbox */
+	i_zero(&state);
+	memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid,
+	       sizeof(state.mailbox_guid));
+	state.last_uidvalidity = brain->local_dsync_box.uid_validity;
+	if (brain->box_importer == NULL) {
+		/* this mailbox didn't exist on remote */
+		state.last_common_uid = brain->local_dsync_box.uid_next-1;
+		state.last_common_modseq =
+			brain->local_dsync_box.highest_modseq;
+		state.last_common_pvt_modseq =
+			brain->local_dsync_box.highest_pvt_modseq;
+		state.last_messages_count =
+			brain->local_dsync_box.messages_count;
+	} else {
+		if (dsync_mailbox_import_deinit(&brain->box_importer,
+						!brain->failed,
+						&state.last_common_uid,
+						&state.last_common_modseq,
+						&state.last_common_pvt_modseq,
+						&state.last_messages_count,
+						&changes_during_sync,
+						&require_full_resync,
+						&brain->mail_error) < 0) {
+			if (require_full_resync) {
+				/* don't treat this as brain failure or the
+				   state won't be sent to the other brain.
+				   this also means we'll continue syncing the
+				   following mailboxes. */
+				brain->require_full_resync = TRUE;
+			} else {
+				brain->failed = TRUE;
+			}
+		}
+		if (changes_during_sync != NULL) {
+			state.changes_during_sync = TRUE;
+			dsync_brain_set_changes_during_sync(brain, changes_during_sync);
+		}
+	}
+	if (brain->require_full_resync) {
+		state.last_uidvalidity = 0;
+		state.changes_during_sync = TRUE;
+	}
+	brain->mailbox_state = state;
+	dsync_ibc_send_mailbox_state(brain->ibc, &state);
+}
+
+static bool dsync_brain_recv_mail(struct dsync_brain *brain)
+{
+	struct dsync_mail *mail;
+	enum dsync_ibc_recv_ret ret;
+
+	if ((ret = dsync_ibc_recv_mail(brain->ibc, &mail)) == 0)
+		return FALSE;
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON;
+		if (brain->box_exporter != NULL &&
+		    brain->box_send_state >= DSYNC_BOX_STATE_RECV_LAST_COMMON) {
+			if (dsync_brain_export_deinit(brain) < 0)
+				return TRUE;
+		}
+		dsync_brain_sync_half_finished(brain);
+		return TRUE;
+	}
+	if (brain->debug) {
+		i_debug("brain %c: import mail uid %u guid %s",
+			brain->master_brain ? 'M' : 'S', mail->uid, mail->guid);
+	}
+	if (dsync_mailbox_import_mail(brain->box_importer, mail) < 0)
+		brain->failed = TRUE;
+	i_stream_unref(&mail->input);
+	return TRUE;
+}
+
+static bool dsync_brain_send_mail(struct dsync_brain *brain)
+{
+	const struct dsync_mail *mail;
+
+	if (brain->mail_requests &&
+	    brain->box_recv_state < DSYNC_BOX_STATE_MAILS) {
+		/* wait for mail requests to finish. we could already start
+		   exporting, but then we're going to do quite a lot of
+		   separate searches. especially with pipe backend we'd do
+		   a separate search for each mail. */
+		return FALSE;
+	}
+
+	while (dsync_mailbox_export_next_mail(brain->box_exporter, &mail) > 0) {
+		if (dsync_ibc_send_mail(brain->ibc, mail) == 0)
+			return TRUE;
+	}
+
+	if (dsync_brain_export_deinit(brain) < 0)
+		return TRUE;
+
+	brain->box_send_state = DSYNC_BOX_STATE_DONE;
+	dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS);
+
+	dsync_brain_sync_half_finished(brain);
+	return TRUE;
+}
+
+static bool dsync_brain_recv_last_common(struct dsync_brain *brain)
+{
+	enum dsync_ibc_recv_ret ret;
+	struct dsync_mailbox_state state;
+
+	if ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) == 0)
+		return FALSE;
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		i_error("Remote sent end-of-list instead of a mailbox state");
+		brain->failed = TRUE;
+		return TRUE;
+	}
+	i_assert(brain->box_send_state == DSYNC_BOX_STATE_DONE);
+	i_assert(memcmp(state.mailbox_guid, brain->local_dsync_box.mailbox_guid,
+			sizeof(state.mailbox_guid)) == 0);
+
+	/* normally the last_common_* values should be the same in local and
+	   remote, but during unexpected changes they may differ. use the
+	   values that are lower as the final state. */
+	if (brain->mailbox_state.last_common_uid > state.last_common_uid)
+		brain->mailbox_state.last_common_uid = state.last_common_uid;
+	if (brain->mailbox_state.last_common_modseq > state.last_common_modseq)
+		brain->mailbox_state.last_common_modseq = state.last_common_modseq;
+	if (brain->mailbox_state.last_common_pvt_modseq > state.last_common_pvt_modseq)
+		brain->mailbox_state.last_common_pvt_modseq = state.last_common_pvt_modseq;
+	if (state.changes_during_sync)
+		brain->changes_during_remote_sync = TRUE;
+
+	dsync_brain_sync_mailbox_deinit(brain);
+	return TRUE;
+}
+
+bool dsync_brain_sync_mails(struct dsync_brain *brain)
+{
+	bool changed = FALSE;
+
+	i_assert(brain->box != NULL);
+
+	switch (brain->box_recv_state) {
+	case DSYNC_BOX_STATE_MAILBOX:
+		changed = dsync_brain_master_sync_recv_mailbox(brain);
+		break;
+	case DSYNC_BOX_STATE_ATTRIBUTES:
+		changed = dsync_brain_recv_mailbox_attribute(brain);
+		break;
+	case DSYNC_BOX_STATE_CHANGES:
+		changed = dsync_brain_recv_mail_change(brain);
+		break;
+	case DSYNC_BOX_STATE_MAIL_REQUESTS:
+		changed = dsync_brain_recv_mail_request(brain);
+		break;
+	case DSYNC_BOX_STATE_MAILS:
+		changed = dsync_brain_recv_mail(brain);
+		break;
+	case DSYNC_BOX_STATE_RECV_LAST_COMMON:
+		changed = dsync_brain_recv_last_common(brain);
+		break;
+	case DSYNC_BOX_STATE_DONE:
+		break;
+	}
+
+	if (!dsync_ibc_is_send_queue_full(brain->ibc) && !brain->failed) {
+		switch (brain->box_send_state) {
+		case DSYNC_BOX_STATE_MAILBOX:
+			/* wait for mailbox to be received first */
+			break;
+		case DSYNC_BOX_STATE_ATTRIBUTES:
+			dsync_brain_send_mailbox_attribute(brain);
+			changed = TRUE;
+			break;
+		case DSYNC_BOX_STATE_CHANGES:
+			dsync_brain_send_mail_change(brain);
+			changed = TRUE;
+			break;
+		case DSYNC_BOX_STATE_MAIL_REQUESTS:
+			if (dsync_brain_send_mail_request(brain))
+				changed = TRUE;
+			break;
+		case DSYNC_BOX_STATE_MAILS:
+			if (dsync_brain_send_mail(brain))
+				changed = TRUE;
+			break;
+		case DSYNC_BOX_STATE_RECV_LAST_COMMON:
+			i_unreached();
+		case DSYNC_BOX_STATE_DONE:
+			break;
+		}
+	}
+	return changed;
+}
diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
new file mode 100644
index 0000000..9849703
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -0,0 +1,163 @@
+#ifndef DSYNC_BRAIN_PRIVATE_H
+#define DSYNC_BRAIN_PRIVATE_H
+
+#include "hash.h"
+#include "dsync-brain.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-state.h"
+
+#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock"
+#define DSYNC_MAILBOX_LOCK_FILENAME ".dovecot-box-sync.lock"
+#define DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS 30
+
+struct dsync_mailbox_tree_sync_change;
+
+enum dsync_state {
+	DSYNC_STATE_MASTER_RECV_HANDSHAKE,
+	DSYNC_STATE_SLAVE_RECV_HANDSHAKE,
+	/* if sync_type=STATE, the master brain knows the saved "last common
+	   mailbox state". this state is sent to the slave. */
+	DSYNC_STATE_MASTER_SEND_LAST_COMMON,
+	DSYNC_STATE_SLAVE_RECV_LAST_COMMON,
+
+	/* both sides send their mailbox trees */
+	DSYNC_STATE_SEND_MAILBOX_TREE,
+	DSYNC_STATE_SEND_MAILBOX_TREE_DELETES,
+	DSYNC_STATE_RECV_MAILBOX_TREE,
+	DSYNC_STATE_RECV_MAILBOX_TREE_DELETES,
+
+	/* master decides in which order mailboxes are synced (it knows the
+	   slave's mailboxes by looking at the received mailbox tree) */
+	DSYNC_STATE_MASTER_SEND_MAILBOX,
+	DSYNC_STATE_SLAVE_RECV_MAILBOX,
+	/* once mailbox is selected, the mails inside it are synced.
+	   after the mails are synced, another mailbox is synced. */
+	DSYNC_STATE_SYNC_MAILS,
+
+	DSYNC_STATE_FINISH,
+	DSYNC_STATE_DONE
+};
+
+enum dsync_box_state {
+	DSYNC_BOX_STATE_MAILBOX,
+	DSYNC_BOX_STATE_CHANGES,
+	DSYNC_BOX_STATE_ATTRIBUTES,
+	DSYNC_BOX_STATE_MAIL_REQUESTS,
+	DSYNC_BOX_STATE_MAILS,
+	DSYNC_BOX_STATE_RECV_LAST_COMMON,
+	DSYNC_BOX_STATE_DONE
+};
+
+struct dsync_brain {
+	pool_t pool;
+	struct mail_user *user;
+	struct dsync_ibc *ibc;
+	const char *process_title_prefix;
+	ARRAY(struct mail_namespace *) sync_namespaces;
+	const char *sync_box;
+	struct mailbox *virtual_all_box;
+	guid_128_t sync_box_guid;
+	const char *const *exclude_mailboxes;
+	enum dsync_brain_sync_type sync_type;
+	time_t sync_since_timestamp;
+	time_t sync_until_timestamp;
+	uoff_t sync_max_size;
+	const char *sync_flag;
+	char alt_char;
+	unsigned int import_commit_msgs_interval;
+	unsigned int hdr_hash_version;
+
+	unsigned int lock_timeout;
+	int lock_fd;
+	const char *lock_path;
+	struct file_lock *lock;
+
+	char hierarchy_sep, escape_char;
+	struct dsync_mailbox_tree *local_mailbox_tree;
+	struct dsync_mailbox_tree *remote_mailbox_tree;
+	struct dsync_mailbox_tree_iter *local_tree_iter;
+
+	enum dsync_state state, pre_box_state;
+	enum dsync_box_state box_recv_state;
+	enum dsync_box_state box_send_state;
+	unsigned int proctitle_update_counter;
+
+	struct dsync_transaction_log_scan *log_scan;
+	struct dsync_mailbox_importer *box_importer;
+	struct dsync_mailbox_exporter *box_exporter;
+
+	struct mailbox *box;
+	struct file_lock *box_lock;
+	unsigned int mailbox_lock_timeout_secs;
+	struct dsync_mailbox local_dsync_box, remote_dsync_box;
+	pool_t dsync_box_pool;
+	/* list of mailbox states
+	   for master brain: given to brain at init and
+	   for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */
+	HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states;
+	/* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */
+	struct hash_iterate_context *mailbox_states_iter;
+	/* state of the mailbox we're currently syncing, changed at
+	   init and deinit */
+	struct dsync_mailbox_state mailbox_state;
+	/* new states for synced mailboxes */
+	ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states;
+
+	const char *changes_during_sync;
+	enum mail_error mail_error;
+
+	const char *const *hashed_headers;
+
+	bool master_brain:1;
+	bool mail_requests:1;
+	bool backup_send:1;
+	bool backup_recv:1;
+	bool purge:1;
+	bool debug:1;
+	bool sync_visible_namespaces:1;
+	bool no_mail_sync:1;
+	bool no_backup_overwrite:1;
+	bool no_mail_prefetch:1;
+	bool changes_during_remote_sync:1;
+	bool require_full_resync:1;
+	bool verbose_proctitle:1;
+	bool no_notify:1;
+	bool failed:1;
+	bool empty_hdr_workaround:1;
+};
+
+extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];
+
+void dsync_brain_mailbox_trees_init(struct dsync_brain *brain);
+void dsync_brain_send_mailbox_tree(struct dsync_brain *brain);
+void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain);
+bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain);
+bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain);
+int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain,
+			const struct dsync_mailbox_tree_sync_change *change,
+			enum mail_error *error_r);
+
+void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain);
+int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
+			      struct mailbox **box_r, const char **errstr_r,
+			      enum mail_error *error_r);
+bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
+				    struct mailbox *box,
+				    const struct dsync_mailbox *local_box,
+				    const struct dsync_mailbox *remote_box,
+				    const char **reason_r);
+bool dsync_boxes_need_sync(struct dsync_brain *brain,
+			   const struct dsync_mailbox *box1,
+			   const struct dsync_mailbox *box2,
+			   const char **reason_r);
+void dsync_brain_sync_init_box_states(struct dsync_brain *brain);
+void dsync_brain_set_changes_during_sync(struct dsync_brain *brain,
+					 const char *reason);
+
+void dsync_brain_master_send_mailbox(struct dsync_brain *brain);
+bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain);
+int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
+				  const struct dsync_mailbox *remote_dsync_box);
+bool dsync_brain_sync_mails(struct dsync_brain *brain);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
new file mode 100644
index 0000000..8ff7247
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -0,0 +1,901 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "hostpid.h"
+#include "str.h"
+#include "file-create-locked.h"
+#include "process-title.h"
+#include "settings-parser.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "mail-namespace.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-ibc.h"
+#include "dsync-brain-private.h"
+#include "dsync-mailbox-import.h"
+#include "dsync-mailbox-export.h"
+
+#include <sys/stat.h>
+
+enum dsync_brain_title {
+	DSYNC_BRAIN_TITLE_NONE = 0,
+	DSYNC_BRAIN_TITLE_LOCKING,
+};
+
+static const char *dsync_state_names[] = {
+	"master_recv_handshake",
+	"slave_recv_handshake",
+	"master_send_last_common",
+	"slave_recv_last_common",
+	"send_mailbox_tree",
+	"send_mailbox_tree_deletes",
+	"recv_mailbox_tree",
+	"recv_mailbox_tree_deletes",
+	"master_send_mailbox",
+	"slave_recv_mailbox",
+	"sync_mails",
+	"finish",
+	"done"
+};
+
+struct dsync_mailbox_list_module dsync_mailbox_list_module =
+	MODULE_CONTEXT_INIT(&mailbox_list_module_register);
+
+static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain);
+
+static const char *
+dsync_brain_get_proctitle_full(struct dsync_brain *brain,
+			       enum dsync_brain_title title)
+{
+	string_t *str = t_str_new(128);
+	const char *import_title, *export_title;
+
+	str_append_c(str, '[');
+	if (brain->process_title_prefix != NULL)
+		str_append(str, brain->process_title_prefix);
+	str_append(str, brain->user->username);
+	if (brain->box == NULL) {
+		str_append_c(str, ' ');
+		str_append(str, dsync_state_names[brain->state]);
+	} else {
+		str_append_c(str, ' ');
+		str_append(str, mailbox_get_vname(brain->box));
+		import_title = brain->box_importer == NULL ? "" :
+			dsync_mailbox_import_get_proctitle(brain->box_importer);
+		export_title = brain->box_exporter == NULL ? "" :
+			dsync_mailbox_export_get_proctitle(brain->box_exporter);
+		if (import_title[0] == '\0' && export_title[0] == '\0') {
+			str_printfa(str, " send:%s recv:%s",
+				    dsync_box_state_names[brain->box_send_state],
+				    dsync_box_state_names[brain->box_recv_state]);
+		} else {
+			if (import_title[0] != '\0') {
+				str_append(str, " import:");
+				str_append(str, import_title);
+			}
+			if (export_title[0] != '\0') {
+				str_append(str, " export:");
+				str_append(str, export_title);
+			}
+		}
+	}
+	switch (title) {
+	case DSYNC_BRAIN_TITLE_NONE:
+		break;
+	case DSYNC_BRAIN_TITLE_LOCKING:
+		str_append(str, " locking "DSYNC_LOCK_FILENAME);
+		break;
+	}
+	str_append_c(str, ']');
+	return str_c(str);
+}
+
+static const char *dsync_brain_get_proctitle(struct dsync_brain *brain)
+{
+	return dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_NONE);
+}
+
+static void dsync_brain_run_io(void *context)
+{
+	struct dsync_brain *brain = context;
+	bool changed, try_pending;
+
+	if (dsync_ibc_has_failed(brain->ibc)) {
+		io_loop_stop(current_ioloop);
+		brain->failed = TRUE;
+		return;
+	}
+
+	try_pending = TRUE;
+	do {
+		if (!dsync_brain_run(brain, &changed)) {
+			io_loop_stop(current_ioloop);
+			break;
+		}
+		if (changed)
+			try_pending = TRUE;
+		else if (try_pending) {
+			if (dsync_ibc_has_pending_data(brain->ibc))
+				changed = TRUE;
+			try_pending = FALSE;
+		}
+	} while (changed);
+}
+
+static struct dsync_brain *
+dsync_brain_common_init(struct mail_user *user, struct dsync_ibc *ibc)
+{
+	struct dsync_brain *brain;
+	const struct master_service_settings *service_set;
+	pool_t pool;
+
+	service_set = master_service_settings_get(master_service);
+	mail_user_ref(user);
+
+	pool = pool_alloconly_create("dsync brain", 10240);
+	brain = p_new(pool, struct dsync_brain, 1);
+	brain->pool = pool;
+	brain->user = user;
+	brain->ibc = ibc;
+	brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN;
+	brain->lock_fd = -1;
+	brain->verbose_proctitle = service_set->verbose_proctitle;
+	hash_table_create(&brain->mailbox_states, pool, 0,
+			  guid_128_hash, guid_128_cmp);
+	p_array_init(&brain->remote_mailbox_states, pool, 64);
+	return brain;
+}
+
+static void
+dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
+{
+	brain->mail_requests =
+		(flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0;
+	brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0;
+	brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0;
+	brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0;
+	brain->sync_visible_namespaces =
+		(flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0;
+	brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0;
+	brain->no_backup_overwrite =
+		(flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0;
+	brain->no_mail_prefetch =
+		(flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
+	brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0;
+	brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0;
+}
+
+static void
+dsync_brain_open_virtual_all_box(struct dsync_brain *brain,
+				 const char *vname)
+{
+	struct mail_namespace *ns;
+
+	ns = mail_namespace_find(brain->user->namespaces, vname);
+	brain->virtual_all_box =
+		mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
+}
+
+struct dsync_brain *
+dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
+			enum dsync_brain_sync_type sync_type,
+			enum dsync_brain_flags flags,
+			const struct dsync_brain_settings *set)
+{
+	struct dsync_ibc_settings ibc_set;
+	struct dsync_brain *brain;
+	struct mail_namespace *ns;
+	string_t *sync_ns_str = NULL;
+	const char *error;
+
+	i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
+	i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE ||
+		 (set->state != NULL && *set->state != '\0'));
+	i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1);
+
+	brain = dsync_brain_common_init(user, ibc);
+	brain->process_title_prefix =
+		p_strdup(brain->pool, set->process_title_prefix);
+	brain->sync_type = sync_type;
+	if (array_count(&set->sync_namespaces) > 0) {
+		sync_ns_str = t_str_new(128);
+		p_array_init(&brain->sync_namespaces, brain->pool,
+			     array_count(&set->sync_namespaces));
+		array_foreach_elem(&set->sync_namespaces, ns) {
+			str_append(sync_ns_str, ns->prefix);
+			str_append_c(sync_ns_str, '\n');
+			array_push_back(&brain->sync_namespaces, &ns);
+		}
+		str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1);
+	}
+	brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
+		set->mailbox_alt_char;
+	brain->sync_since_timestamp = set->sync_since_timestamp;
+	brain->sync_until_timestamp = set->sync_until_timestamp;
+	brain->sync_max_size = set->sync_max_size;
+	brain->sync_flag = p_strdup(brain->pool, set->sync_flag);
+	brain->sync_box = p_strdup(brain->pool, set->sync_box);
+	brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
+		p_strarray_dup(brain->pool, set->exclude_mailboxes);
+	memcpy(brain->sync_box_guid, set->sync_box_guid,
+	       sizeof(brain->sync_box_guid));
+	brain->lock_timeout = set->lock_timeout_secs;
+	if (brain->lock_timeout != 0)
+		brain->mailbox_lock_timeout_secs = brain->lock_timeout;
+	else
+		brain->mailbox_lock_timeout_secs =
+			DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS;
+	brain->import_commit_msgs_interval = set->import_commit_msgs_interval;
+	brain->master_brain = TRUE;
+	brain->hashed_headers =
+		(const char*const*)p_strarray_dup(brain->pool, set->hashed_headers);
+	dsync_brain_set_flags(brain, flags);
+
+	if (set->virtual_all_box != NULL)
+		dsync_brain_open_virtual_all_box(brain, set->virtual_all_box);
+
+	if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE)
+		;
+	else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool,
+					     set->state, &error) < 0) {
+		hash_table_clear(brain->mailbox_states, FALSE);
+		i_error("Saved sync state is invalid, "
+			"falling back to full sync: %s", error);
+		brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
+	} else {
+		if (brain->debug) {
+			i_debug("brain %c: Imported mailbox states:",
+				brain->master_brain ? 'M' : 'S');
+			dsync_brain_mailbox_states_dump(brain);
+		}
+	}
+	dsync_brain_mailbox_trees_init(brain);
+
+	i_zero(&ibc_set);
+	ibc_set.hostname = my_hostdomain();
+	ibc_set.sync_ns_prefixes = sync_ns_str == NULL ?
+		NULL : str_c(sync_ns_str);
+	ibc_set.sync_box = set->sync_box;
+	ibc_set.virtual_all_box = set->virtual_all_box;
+	ibc_set.exclude_mailboxes = set->exclude_mailboxes;
+	ibc_set.sync_since_timestamp = set->sync_since_timestamp;
+	ibc_set.sync_until_timestamp = set->sync_until_timestamp;
+	ibc_set.sync_max_size = set->sync_max_size;
+	ibc_set.sync_flags = set->sync_flag;
+	memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
+	       sizeof(ibc_set.sync_box_guid));
+	ibc_set.alt_char = brain->alt_char;
+	ibc_set.sync_type = sync_type;
+	ibc_set.hdr_hash_v2 = TRUE;
+	ibc_set.lock_timeout = set->lock_timeout_secs;
+	ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval;
+	ibc_set.hashed_headers = set->hashed_headers;
+	/* reverse the backup direction for the slave */
+	ibc_set.brain_flags = flags & ENUM_NEGATE(DSYNC_BRAIN_FLAG_BACKUP_SEND |
+						  DSYNC_BRAIN_FLAG_BACKUP_RECV);
+	if ((flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
+		ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV;
+	else if ((flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0)
+		ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
+	dsync_ibc_send_handshake(ibc, &ibc_set);
+
+	dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain);
+	brain->state = DSYNC_STATE_MASTER_RECV_HANDSHAKE;
+
+	if (brain->verbose_proctitle)
+		process_title_set(dsync_brain_get_proctitle(brain));
+	return brain;
+}
+
+struct dsync_brain *
+dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc,
+		       bool local, const char *process_title_prefix,
+		       char default_alt_char)
+{
+	struct dsync_ibc_settings ibc_set;
+	struct dsync_brain *brain;
+
+	i_assert(default_alt_char != '\0');
+
+	brain = dsync_brain_common_init(user, ibc);
+	brain->alt_char = default_alt_char;
+	brain->process_title_prefix =
+		p_strdup(brain->pool, process_title_prefix);
+	brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE;
+
+	if (local) {
+		/* both master and slave are running within the same process,
+		   update the proctitle only for master. */
+		brain->verbose_proctitle = FALSE;
+	}
+
+	i_zero(&ibc_set);
+	ibc_set.hdr_hash_v2 = TRUE;
+	ibc_set.hostname = my_hostdomain();
+	dsync_ibc_send_handshake(ibc, &ibc_set);
+
+	if (brain->verbose_proctitle)
+		process_title_set(dsync_brain_get_proctitle(brain));
+	dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain);
+	return brain;
+}
+
+static void dsync_brain_purge(struct dsync_brain *brain)
+{
+	struct mail_namespace *ns;
+	struct mail_storage *storage;
+
+	for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+		if (!dsync_brain_want_namespace(brain, ns))
+			continue;
+
+		storage = mail_namespace_get_default_storage(ns);
+		if (mail_storage_purge(storage) < 0) {
+			i_error("Purging namespace '%s' failed: %s", ns->prefix,
+				mail_storage_get_last_internal_error(storage, NULL));
+		}
+	}
+}
+
+int dsync_brain_deinit(struct dsync_brain **_brain, enum mail_error *error_r)
+{
+	struct dsync_brain *brain = *_brain;
+	int ret;
+
+	*_brain = NULL;
+
+	if (dsync_ibc_has_timed_out(brain->ibc)) {
+		i_error("Timeout during state=%s%s",
+			dsync_state_names[brain->state],
+			brain->state != DSYNC_STATE_SYNC_MAILS ? "" :
+			t_strdup_printf(" (send=%s recv=%s)",
+				dsync_box_state_names[brain->box_send_state],
+				dsync_box_state_names[brain->box_recv_state]));
+	}
+	if (dsync_ibc_has_failed(brain->ibc) ||
+	    brain->state != DSYNC_STATE_DONE)
+		brain->failed = TRUE;
+	dsync_ibc_close_mail_streams(brain->ibc);
+
+	if (brain->purge && !brain->failed)
+		dsync_brain_purge(brain);
+
+	if (brain->box != NULL)
+		dsync_brain_sync_mailbox_deinit(brain);
+	if (brain->virtual_all_box != NULL)
+		mailbox_free(&brain->virtual_all_box);
+	if (brain->local_tree_iter != NULL)
+		dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
+	if (brain->local_mailbox_tree != NULL)
+		dsync_mailbox_tree_deinit(&brain->local_mailbox_tree);
+	if (brain->remote_mailbox_tree != NULL)
+		dsync_mailbox_tree_deinit(&brain->remote_mailbox_tree);
+	hash_table_iterate_deinit(&brain->mailbox_states_iter);
+	hash_table_destroy(&brain->mailbox_states);
+
+	pool_unref(&brain->dsync_box_pool);
+
+	if (brain->lock_fd != -1) {
+		/* unlink the lock file before it gets unlocked */
+		i_unlink(brain->lock_path);
+		if (brain->debug) {
+			i_debug("brain %c: Unlocked %s",
+				brain->master_brain ? 'M' : 'S',
+				brain->lock_path);
+		}
+		file_lock_free(&brain->lock);
+		i_close_fd(&brain->lock_fd);
+	}
+
+	ret = brain->failed ? -1 : 0;
+	mail_user_unref(&brain->user);
+
+	*error_r = !brain->failed ? 0 :
+		(brain->mail_error == 0 ? MAIL_ERROR_TEMP : brain->mail_error);
+	pool_unref(&brain->pool);
+	return ret;
+}
+
+static int
+dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname)
+{
+	const struct file_create_settings lock_set = {
+		.lock_timeout_secs = brain->lock_timeout,
+		.lock_settings = {
+			.lock_method = FILE_LOCK_METHOD_FCNTL,
+		},
+	};
+	const char *home, *error, *local_hostname = my_hostdomain();
+	bool created;
+	int ret;
+
+	if ((ret = strcmp(remote_hostname, local_hostname)) < 0) {
+		/* locking done by remote */
+		if (brain->debug) {
+			i_debug("brain %c: Locking done by remote "
+				"(local hostname=%s, remote hostname=%s)",
+				brain->master_brain ? 'M' : 'S',
+				local_hostname, remote_hostname);
+		}
+		return 0;
+	}
+	if (ret == 0 && !brain->master_brain) {
+		/* running dsync within the same server.
+		   locking done by master brain. */
+		if (brain->debug) {
+			i_debug("brain %c: Locking done by local master-brain "
+				"(local hostname=%s, remote hostname=%s)",
+				brain->master_brain ? 'M' : 'S',
+				local_hostname, remote_hostname);
+		}
+		return 0;
+	}
+
+	if ((ret = mail_user_get_home(brain->user, &home)) < 0) {
+		i_error("Couldn't look up user's home dir");
+		return -1;
+	}
+	if (ret == 0) {
+		i_error("User has no home directory");
+		return -1;
+	}
+
+	if (brain->verbose_proctitle)
+		process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING));
+	brain->lock_path = p_strconcat(brain->pool, home,
+				       "/"DSYNC_LOCK_FILENAME, NULL);
+	brain->lock_fd = file_create_locked(brain->lock_path, &lock_set,
+					    &brain->lock, &created, &error);
+	if (brain->lock_fd == -1 && errno == ENOENT) {
+		/* home directory not created */
+		if (mail_user_home_mkdir(brain->user) < 0)
+			return -1;
+		brain->lock_fd = file_create_locked(brain->lock_path, &lock_set,
+			&brain->lock, &created, &error);
+	}
+	if (brain->lock_fd == -1)
+		i_error("Couldn't lock %s: %s", brain->lock_path, error);
+	else if (brain->debug) {
+		i_debug("brain %c: Locking done locally in %s "
+			"(local hostname=%s, remote hostname=%s)",
+			brain->master_brain ? 'M' : 'S',
+			brain->lock_path, local_hostname, remote_hostname);
+	}
+	if (brain->verbose_proctitle)
+		process_title_set(dsync_brain_get_proctitle(brain));
+	return brain->lock_fd == -1 ? -1 : 0;
+}
+
+static void
+dsync_brain_set_hdr_hash_version(struct dsync_brain *brain,
+				 const struct dsync_ibc_settings *ibc_set)
+{
+	if (ibc_set->hdr_hash_v3)
+		brain->hdr_hash_version = 3;
+	else if (ibc_set->hdr_hash_v2)
+		brain->hdr_hash_version = 3;
+	else
+		brain->hdr_hash_version = 1;
+}
+
+static bool dsync_brain_master_recv_handshake(struct dsync_brain *brain)
+{
+	const struct dsync_ibc_settings *ibc_set;
+
+	i_assert(brain->master_brain);
+
+	if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0)
+		return FALSE;
+
+	if (brain->lock_timeout > 0) {
+		if (dsync_brain_lock(brain, ibc_set->hostname) < 0) {
+			brain->failed = TRUE;
+			return FALSE;
+		}
+	}
+	dsync_brain_set_hdr_hash_version(brain, ibc_set);
+
+	brain->state = brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE ?
+		DSYNC_STATE_MASTER_SEND_LAST_COMMON :
+		DSYNC_STATE_SEND_MAILBOX_TREE;
+	return TRUE;
+}
+
+static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
+{
+	const struct dsync_ibc_settings *ibc_set;
+	struct mail_namespace *ns;
+	const char *const *prefixes;
+
+	i_assert(!brain->master_brain);
+
+	if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0)
+		return FALSE;
+	dsync_brain_set_hdr_hash_version(brain, ibc_set);
+
+	if (ibc_set->lock_timeout > 0) {
+		brain->lock_timeout = ibc_set->lock_timeout;
+		brain->mailbox_lock_timeout_secs = brain->lock_timeout;
+		if (dsync_brain_lock(brain, ibc_set->hostname) < 0) {
+			brain->failed = TRUE;
+			return FALSE;
+		}
+	} else {
+		brain->mailbox_lock_timeout_secs =
+			DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS;
+	}
+
+	if (ibc_set->sync_ns_prefixes != NULL) {
+		p_array_init(&brain->sync_namespaces, brain->pool, 4);
+		prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n");
+		if (prefixes[0] == NULL) {
+			/* ugly workaround for strsplit API: there was one
+			   prefix="" entry */
+			static const char *empty_prefix[] = { "", NULL };
+			prefixes = empty_prefix;
+		}
+		for (; *prefixes != NULL; prefixes++) {
+			ns = mail_namespace_find(brain->user->namespaces,
+						 *prefixes);
+			if (ns == NULL) {
+				i_error("Namespace not found: '%s'", *prefixes);
+				brain->failed = TRUE;
+				return FALSE;
+			}
+			array_push_back(&brain->sync_namespaces, &ns);
+		}
+	}
+	brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
+	brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
+		p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
+	brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
+	brain->sync_until_timestamp = ibc_set->sync_until_timestamp;
+	brain->sync_max_size = ibc_set->sync_max_size;
+	brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags);
+	memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
+	       sizeof(brain->sync_box_guid));
+	if (ibc_set->alt_char != '\0')
+		brain->alt_char = ibc_set->alt_char;
+	i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
+	brain->sync_type = ibc_set->sync_type;
+
+	dsync_brain_set_flags(brain, ibc_set->brain_flags);
+	if (ibc_set->hashed_headers != NULL)
+		brain->hashed_headers =
+			p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers);
+	/* this flag is only set on the remote slave brain */
+	brain->purge = (ibc_set->brain_flags &
+			DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0;
+
+	if (ibc_set->virtual_all_box != NULL)
+		dsync_brain_open_virtual_all_box(brain, ibc_set->virtual_all_box);
+	dsync_brain_mailbox_trees_init(brain);
+
+	if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE)
+		brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON;
+	else
+		brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
+	return TRUE;
+}
+
+static void dsync_brain_master_send_last_common(struct dsync_brain *brain)
+{
+	struct dsync_mailbox_state *state;
+	uint8_t *guid;
+	enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK;
+
+	i_assert(brain->master_brain);
+
+	if (brain->mailbox_states_iter == NULL) {
+		brain->mailbox_states_iter =
+			hash_table_iterate_init(brain->mailbox_states);
+	}
+
+	for (;;) {
+		if (ret == DSYNC_IBC_SEND_RET_FULL)
+			return;
+		if (!hash_table_iterate(brain->mailbox_states_iter,
+					brain->mailbox_states, &guid, &state))
+			break;
+		ret = dsync_ibc_send_mailbox_state(brain->ibc, state);
+	}
+	hash_table_iterate_deinit(&brain->mailbox_states_iter);
+
+	dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_STATE);
+	brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
+}
+
+static void dsync_mailbox_state_add(struct dsync_brain *brain,
+				    const struct dsync_mailbox_state *state)
+{
+	struct dsync_mailbox_state *dupstate;
+	uint8_t *guid_p;
+
+	dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1);
+	*dupstate = *state;
+	guid_p = dupstate->mailbox_guid;
+	hash_table_insert(brain->mailbox_states, guid_p, dupstate);
+}
+
+static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain)
+{
+	struct dsync_mailbox_state state;
+	enum dsync_ibc_recv_ret ret;
+	bool changed = FALSE;
+
+	i_assert(!brain->master_brain);
+
+	while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) {
+		dsync_mailbox_state_add(brain, &state);
+		changed = TRUE;
+	}
+	if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+		brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
+		changed = TRUE;
+	}
+	return changed;
+}
+
+static bool dsync_brain_finish(struct dsync_brain *brain)
+{
+	const char *error;
+	enum mail_error mail_error;
+	bool require_full_resync;
+	enum dsync_ibc_recv_ret ret;
+
+	if (!brain->master_brain) {
+		dsync_ibc_send_finish(brain->ibc,
+				      brain->failed ? "dsync failed" : NULL,
+				      brain->mail_error,
+				      brain->require_full_resync);
+		brain->state = DSYNC_STATE_DONE;
+		return TRUE;
+	} 
+	ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error,
+				    &require_full_resync);
+	if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN)
+		return FALSE;
+	if (error != NULL) {
+		i_error("Remote dsync failed: %s", error);
+		brain->failed = TRUE;
+		if (mail_error != 0 &&
+		    (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP))
+			brain->mail_error = mail_error;
+	}
+	if (require_full_resync)
+		brain->require_full_resync = TRUE;
+	brain->state = DSYNC_STATE_DONE;
+	return TRUE;
+}
+
+static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r)
+{
+	enum dsync_state orig_state = brain->state;
+	enum dsync_box_state orig_box_recv_state = brain->box_recv_state;
+	enum dsync_box_state orig_box_send_state = brain->box_send_state;
+	bool changed = FALSE, ret = TRUE;
+
+	if (brain->failed)
+		return FALSE;
+
+	switch (brain->state) {
+	case DSYNC_STATE_MASTER_RECV_HANDSHAKE:
+		changed = dsync_brain_master_recv_handshake(brain);
+		break;
+	case DSYNC_STATE_SLAVE_RECV_HANDSHAKE:
+		changed = dsync_brain_slave_recv_handshake(brain);
+		break;
+	case DSYNC_STATE_MASTER_SEND_LAST_COMMON:
+		dsync_brain_master_send_last_common(brain);
+		changed = TRUE;
+		break;
+	case DSYNC_STATE_SLAVE_RECV_LAST_COMMON:
+		changed = dsync_brain_slave_recv_last_common(brain);
+		break;
+	case DSYNC_STATE_SEND_MAILBOX_TREE:
+		dsync_brain_send_mailbox_tree(brain);
+		changed = TRUE;
+		break;
+	case DSYNC_STATE_RECV_MAILBOX_TREE:
+		changed = dsync_brain_recv_mailbox_tree(brain);
+		break;
+	case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES:
+		dsync_brain_send_mailbox_tree_deletes(brain);
+		changed = TRUE;
+		break;
+	case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES:
+		changed = dsync_brain_recv_mailbox_tree_deletes(brain);
+		break;
+	case DSYNC_STATE_MASTER_SEND_MAILBOX:
+		dsync_brain_master_send_mailbox(brain);
+		changed = TRUE;
+		break;
+	case DSYNC_STATE_SLAVE_RECV_MAILBOX:
+		changed = dsync_brain_slave_recv_mailbox(brain);
+		break;
+	case DSYNC_STATE_SYNC_MAILS:
+		changed = dsync_brain_sync_mails(brain);
+		break;
+	case DSYNC_STATE_FINISH:
+		changed = dsync_brain_finish(brain);
+		break;
+	case DSYNC_STATE_DONE:
+		changed = TRUE;
+		ret = FALSE;
+		break;
+	}
+	if (brain->verbose_proctitle) {
+		if (orig_state != brain->state ||
+		    orig_box_recv_state != brain->box_recv_state ||
+		    orig_box_send_state != brain->box_send_state ||
+		    ++brain->proctitle_update_counter % 100 == 0)
+			process_title_set(dsync_brain_get_proctitle(brain));
+	}
+	*changed_r = changed;
+	return brain->failed ? FALSE : ret;
+}
+
+bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r)
+{
+	bool ret;
+
+	*changed_r = FALSE;
+
+	if (dsync_ibc_has_failed(brain->ibc)) {
+		brain->failed = TRUE;
+		return FALSE;
+	}
+
+	T_BEGIN {
+		ret = dsync_brain_run_real(brain, changed_r);
+	} T_END;
+	return ret;
+}
+
+static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain)
+{
+	struct hash_iterate_context *iter;
+	struct dsync_mailbox_state *state;
+	uint8_t *guid;
+
+	iter = hash_table_iterate_init(brain->mailbox_states);
+	while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
+		i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%"PRIu64" pvt_modseq=%"PRIu64" messages=%u changes_during_sync=%d",
+			brain->master_brain ? 'M' : 'S',
+			guid_128_to_string(guid),
+			state->last_uidvalidity,
+			state->last_common_uid,
+			state->last_common_modseq,
+			state->last_common_pvt_modseq,
+			state->last_messages_count,
+			state->changes_during_sync ? 1 : 0);
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+void dsync_brain_get_state(struct dsync_brain *brain, string_t *output)
+{
+	struct hash_iterate_context *iter;
+	struct dsync_mailbox_node *node;
+	const struct dsync_mailbox_state *new_state;
+	struct dsync_mailbox_state *state;
+	const uint8_t *guid_p;
+	uint8_t *guid;
+
+	if (brain->require_full_resync)
+		return;
+
+	/* update mailbox states */
+	array_foreach(&brain->remote_mailbox_states, new_state) {
+		guid_p = new_state->mailbox_guid;
+		state = hash_table_lookup(brain->mailbox_states, guid_p);
+		if (state != NULL)
+			*state = *new_state;
+		else
+			dsync_mailbox_state_add(brain, new_state);
+	}
+
+	/* remove nonexistent mailboxes */
+	iter = hash_table_iterate_init(brain->mailbox_states);
+	while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
+		node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree,
+						      guid);
+		if (node == NULL ||
+		    node->existence != DSYNC_MAILBOX_NODE_EXISTS) {
+			if (brain->debug) {
+				i_debug("brain %c: Removed state for deleted mailbox %s",
+					brain->master_brain ? 'M' : 'S',
+					guid_128_to_string(guid));
+			}
+			hash_table_remove(brain->mailbox_states, guid);
+		}
+	}
+	hash_table_iterate_deinit(&iter);
+
+	if (brain->debug) {
+		i_debug("brain %c: Exported mailbox states:",
+			brain->master_brain ? 'M' : 'S');
+		dsync_brain_mailbox_states_dump(brain);
+	}
+	dsync_mailbox_states_export(brain->mailbox_states, output);
+}
+
+enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain)
+{
+	return brain->sync_type;
+}
+
+bool dsync_brain_has_failed(struct dsync_brain *brain)
+{
+	return brain->failed;
+}
+
+const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain,
+						      bool *remote_only_r)
+{
+	if (brain->changes_during_sync == NULL &&
+	    brain->changes_during_remote_sync) {
+		*remote_only_r = TRUE;
+		return "Remote notified that changes happened during sync";
+	}
+	*remote_only_r = FALSE;
+	return brain->changes_during_sync;
+}
+
+static bool dsync_brain_want_shared_namespace(const struct mail_namespace *ns,
+					      const struct mail_namespace *sync_ns)
+{
+	/* Include shared namespaces and all its
+	   children in the sync (e.g. "Shared/example.com"
+	   will be synced to "Shared/").
+	   This also allows "dsync -n Shared/example.com/"
+	   with "Shared/example.com/username/" style
+	   shared namespace config. */
+	return (ns->type == MAIL_NAMESPACE_TYPE_SHARED) &&
+	       (sync_ns->type == MAIL_NAMESPACE_TYPE_SHARED) &&
+	       str_begins(ns->prefix, sync_ns->prefix);
+}
+
+bool dsync_brain_want_namespace(struct dsync_brain *brain,
+				struct mail_namespace *ns)
+{
+	struct mail_namespace *sync_ns;
+
+	if (array_is_created(&brain->sync_namespaces)) {
+		array_foreach_elem(&brain->sync_namespaces, sync_ns) {
+			if (ns == sync_ns)
+				return TRUE;
+			if (dsync_brain_want_shared_namespace(ns, sync_ns))
+				return TRUE;
+		}
+		return FALSE;
+	}
+	if (ns->alias_for != NULL) {
+		/* always skip aliases */
+		return FALSE;
+	}
+	if (brain->sync_visible_namespaces) {
+		if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0)
+			return TRUE;
+		if ((ns->flags & (NAMESPACE_FLAG_LIST_PREFIX |
+				  NAMESPACE_FLAG_LIST_CHILDREN)) != 0)
+			return TRUE;
+		return FALSE;
+	} else {
+		return strcmp(ns->unexpanded_set->location,
+			      SETTING_STRVAR_UNEXPANDED) == 0;
+	}
+}
+
+void dsync_brain_set_changes_during_sync(struct dsync_brain *brain,
+					 const char *reason)
+{
+	if (brain->debug) {
+		i_debug("brain %c: Change during sync: %s",
+			brain->master_brain ? 'M' : 'S', reason);
+	}
+	if (brain->changes_during_sync == NULL)
+		brain->changes_during_sync = p_strdup(brain->pool, reason);
+}
diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
new file mode 100644
index 0000000..5813148
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -0,0 +1,130 @@
+#ifndef DSYNC_BRAIN_H
+#define DSYNC_BRAIN_H
+
+#include "module-context.h"
+#include "guid.h"
+#include "mail-error.h"
+#include "mailbox-list-private.h"
+
+struct mail_namespace;
+struct mail_user;
+struct dsync_ibc;
+
+enum dsync_brain_flags {
+	DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS	= 0x01,
+	DSYNC_BRAIN_FLAG_BACKUP_SEND		= 0x02,
+	DSYNC_BRAIN_FLAG_BACKUP_RECV		= 0x04,
+	DSYNC_BRAIN_FLAG_DEBUG			= 0x08,
+	DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES= 0x10,
+	/* Sync everything but the actual mails (e.g. mailbox creates,
+	   deletes) */
+	DSYNC_BRAIN_FLAG_NO_MAIL_SYNC		= 0x20,
+	/* Used with BACKUP_SEND/RECV: Don't force the
+	   Use the two-way syncing algorithm, but don't actually modify
+	   anything locally. (Useful during migration.) */
+	DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE	= 0x40,
+	/* Run storage purge on the remote after syncing.
+	   Useful with e.g. a nightly doveadm backup. */
+	DSYNC_BRAIN_FLAG_PURGE_REMOTE		= 0x80,
+	/* Don't prefetch mail bodies until they're actually needed. This works
+	   only with pipe ibc. It's useful if most of the mails can be copied
+	   directly within filesystem without having to read them. */
+	DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH	= 0x100,
+	/* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */
+	DSYNC_BRAIN_FLAG_NO_NOTIFY		= 0x400,
+	/* Workaround missing Date/Message-ID headers */
+	DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND	= 0x800,
+};
+
+enum dsync_brain_sync_type {
+	DSYNC_BRAIN_SYNC_TYPE_UNKNOWN,
+	/* Go through all mailboxes to make sure everything is synced */
+	DSYNC_BRAIN_SYNC_TYPE_FULL,
+	/* Go through all mailboxes that have changed (based on UIDVALIDITY,
+	   UIDNEXT, HIGHESTMODSEQ). If both sides have had equal amount of
+	   changes in some mailbox, it may get incorrectly skipped. */
+	DSYNC_BRAIN_SYNC_TYPE_CHANGED,
+	/* Use saved state to find out what has changed. */
+	DSYNC_BRAIN_SYNC_TYPE_STATE
+};
+
+struct dsync_brain_settings {
+	const char *process_title_prefix;
+	/* Sync only these namespaces */
+	ARRAY(struct mail_namespace *) sync_namespaces;
+	/* Sync only this mailbox name */
+	const char *sync_box;
+	/* Use this virtual \All mailbox to be able to copy mails with the same
+	   GUID instead of saving them twice. With most storages this results
+	   in less disk space usage. */
+	const char *virtual_all_box;
+	/* Sync only this mailbox GUID */
+	guid_128_t sync_box_guid;
+	/* Exclude these mailboxes from the sync. They can contain '*'
+	   wildcards and be \special-use flags. */
+	const char *const *exclude_mailboxes;
+	/* Alternative character to use in mailbox names where the original
+	   character cannot be used. */
+	char mailbox_alt_char;
+	/* Sync only mails with received timestamp at least this high. */
+	time_t sync_since_timestamp;
+	/* Sync only mails with received timestamp less or equal than this */
+	time_t sync_until_timestamp;
+	/* Don't sync mails larger than this. */
+	uoff_t sync_max_size;
+	/* Sync only mails which contains / doesn't contain this flag.
+	   '-' at the beginning means this flag must not exist. */
+	const char *sync_flag;
+	/* Headers to hash (defaults to Date, Message-ID) */
+	const char *const *hashed_headers;
+
+	/* If non-zero, use dsync lock file for this user */
+	unsigned int lock_timeout_secs;
+	/* If non-zero, importing will attempt to commit transaction after
+	   saving this many messages. */
+	unsigned int import_commit_msgs_interval;
+	/* Input state for DSYNC_BRAIN_SYNC_TYPE_STATE */
+	const char *state;
+};
+
+#define DSYNC_LIST_CONTEXT(obj) \
+	MODULE_CONTEXT(obj, dsync_mailbox_list_module)
+struct dsync_mailbox_list {
+	union mailbox_list_module_context module_ctx;
+	bool have_orig_escape_char;
+};
+extern MODULE_CONTEXT_DEFINE(dsync_mailbox_list_module,
+			     &mailbox_list_module_register);
+
+struct dsync_brain *
+dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
+			enum dsync_brain_sync_type sync_type,
+			enum dsync_brain_flags flags,
+			const struct dsync_brain_settings *set);
+struct dsync_brain *
+dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc,
+		       bool local, const char *process_title_prefix,
+		       char default_alt_char);
+/* Returns 0 if everything was successful, -1 if syncing failed in some way */
+int dsync_brain_deinit(struct dsync_brain **brain, enum mail_error *error_r);
+
+/* Returns TRUE if brain needs to run more, FALSE if it's finished.
+   changed_r is TRUE if anything happened during this run. */
+bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r);
+/* Returns TRUE if brain has failed, and there's no point in continuing. */
+bool dsync_brain_has_failed(struct dsync_brain *brain);
+/* Returns the current sync state string, which can be given as parameter to
+   dsync_brain_master_init() to quickly sync only the new changes. */
+void dsync_brain_get_state(struct dsync_brain *brain, string_t *output);
+/* Returns the sync type that was used. Mainly useful with slave brain. */
+enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain);
+/* If there were any unexpected changes during the sync, return the reason
+   for them. Otherwise return NULL. If remote_only_r=TRUE, this brain itself
+   didn't see any changes, but the remote brain did. */
+const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain,
+						      bool *remote_only_r);
+/* Returns TRUE if we want to sync this namespace. */
+bool dsync_brain_want_namespace(struct dsync_brain *brain,
+				struct mail_namespace *ns);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-deserializer.c b/src/doveadm/dsync/dsync-deserializer.c
new file mode 100644
index 0000000..2a90c47
--- /dev/null
+++ b/src/doveadm/dsync/dsync-deserializer.c
@@ -0,0 +1,193 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "dsync-serializer.h"
+#include "dsync-deserializer.h"
+
+struct dsync_deserializer {
+	pool_t pool;
+	const char *name;
+	const char *const *required_fields;
+	const char *const *keys;
+	unsigned int *required_field_indexes;
+	unsigned int required_field_count;
+};
+
+struct dsync_deserializer_decoder {
+	pool_t pool;
+	struct dsync_deserializer *deserializer;
+	const char *const *values;
+	unsigned int values_count;
+};
+
+static bool field_find(const char *const *names, const char *name,
+		       unsigned int *idx_r)
+{
+	unsigned int i;
+
+	for (i = 0; names[i] != NULL; i++) {
+		if (strcmp(names[i], name) == 0) {
+			*idx_r = i;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+int dsync_deserializer_init(const char *name, const char *const *required_fields,
+			    const char *header_line,
+			    struct dsync_deserializer **deserializer_r,
+			    const char **error_r)
+{
+	struct dsync_deserializer *deserializer;
+	const char **dup_required_fields;
+	unsigned int i, required_count;
+	pool_t pool;
+
+	*deserializer_r = NULL;
+
+	pool = pool_alloconly_create("dsync deserializer", 1024);
+	deserializer = p_new(pool, struct dsync_deserializer, 1);
+	deserializer->pool = pool;
+	deserializer->name = p_strdup(pool, name);
+	deserializer->keys = (void *)p_strsplit_tabescaped(pool, header_line);
+
+	deserializer->required_field_count = required_count =
+		required_fields == NULL ? 0 :
+		str_array_length(required_fields);
+	dup_required_fields = p_new(pool, const char *, required_count + 1);
+	deserializer->required_field_indexes =
+		p_new(pool, unsigned int, required_count + 1);
+	for (i = 0; i < required_count; i++) {
+		dup_required_fields[i] =
+			p_strdup(pool, required_fields[i]);
+		if (!field_find(deserializer->keys, required_fields[i],
+				&deserializer->required_field_indexes[i])) {
+			*error_r = t_strdup_printf(
+				"Header missing required field %s",
+				required_fields[i]);
+			pool_unref(&pool);
+			return -1;
+		}
+	}
+	deserializer->required_fields = dup_required_fields;
+
+	*deserializer_r = deserializer;
+	return 0;
+}
+
+void dsync_deserializer_deinit(struct dsync_deserializer **_deserializer)
+{
+	struct dsync_deserializer *deserializer = *_deserializer;
+
+	*_deserializer = NULL;
+
+	pool_unref(&deserializer->pool);
+}
+
+int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer,
+				    const char *input,
+				    struct dsync_deserializer_decoder **decoder_r,
+				    const char **error_r)
+{
+	struct dsync_deserializer_decoder *decoder;
+	unsigned int i;
+	char **values;
+	pool_t pool;
+
+	*decoder_r = NULL;
+
+	pool = pool_alloconly_create("dsync deserializer decode", 1024);
+	decoder = p_new(pool, struct dsync_deserializer_decoder, 1);
+	decoder->pool = pool;
+	decoder->deserializer = deserializer;
+	values = p_strsplit_tabescaped(pool, input);
+
+	/* fix NULLs */
+	for (i = 0; values[i] != NULL; i++) {
+		if (values[i][0] == NULL_CHR) {
+			/* NULL? */
+			if (values[i][1] == '\0')
+				values[i] = NULL;
+			else
+				values[i] += 1;
+		}
+	}
+	decoder->values_count = i;
+
+	/* see if all required fields exist */
+	for (i = 0; i < deserializer->required_field_count; i++) {
+		unsigned int ridx = deserializer->required_field_indexes[i];
+
+		if (ridx >= decoder->values_count || values[ridx] == NULL) {
+			*error_r = t_strdup_printf("Missing required field %s",
+				deserializer->required_fields[i]);
+			pool_unref(&pool);
+			return -1;
+		}
+	}
+	decoder->values = (void *)values;
+
+	*decoder_r = decoder;
+	return 0;
+}
+
+static bool
+dsync_deserializer_find_field(struct dsync_deserializer *deserializer,
+			      const char *key, unsigned int *idx_r)
+{
+	unsigned int i;
+
+	for (i = 0; deserializer->keys[i] != NULL; i++) {
+		if (strcmp(deserializer->keys[i], key) == 0) {
+			*idx_r = i;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder,
+				   const char *key, const char **value_r)
+{
+	unsigned int idx;
+
+	if (!dsync_deserializer_find_field(decoder->deserializer, key, &idx) ||
+	    idx >= decoder->values_count) {
+		*value_r = NULL;
+		return FALSE;
+	} else {
+		*value_r = decoder->values[idx];
+		return *value_r != NULL;
+	}
+}
+
+const char *
+dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder,
+			      const char *key)
+{
+	const char *value;
+
+	if (!dsync_deserializer_decode_try(decoder, key, &value)) {
+		i_panic("dsync_deserializer_decode_get() "
+			"used for non-required key %s", key);
+	}
+	return value;
+}
+
+const char *
+dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder)
+{
+	return decoder->deserializer->name;
+}
+
+void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **_decoder)
+{
+	struct dsync_deserializer_decoder *decoder = *_decoder;
+
+	*_decoder = NULL;
+
+	pool_unref(&decoder->pool);
+}
diff --git a/src/doveadm/dsync/dsync-deserializer.h b/src/doveadm/dsync/dsync-deserializer.h
new file mode 100644
index 0000000..45b19cc
--- /dev/null
+++ b/src/doveadm/dsync/dsync-deserializer.h
@@ -0,0 +1,27 @@
+#ifndef DSYNC_DESERIALIZER_H
+#define DSYNC_DESERIALIZER_H
+
+struct dsync_deserializer;
+struct dsync_deserializer_decoder;
+
+int dsync_deserializer_init(const char *name, const char *const *required_fields,
+			    const char *header_line,
+			    struct dsync_deserializer **deserializer_r,
+			    const char **error_r);
+void dsync_deserializer_deinit(struct dsync_deserializer **deserializer);
+
+int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer,
+				    const char *input,
+				    struct dsync_deserializer_decoder **decoder_r,
+				    const char **error_r);
+bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder,
+				   const char *key, const char **value_r);
+/* key must be in required fields. The return value is never NULL. */
+const char *
+dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder,
+			      const char *key);
+const char *
+dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder);
+void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **decoder);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c
new file mode 100644
index 0000000..1b8886e
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc-pipe.c
@@ -0,0 +1,599 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-state.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-ibc-private.h"
+
+enum item_type {
+	ITEM_END_OF_LIST,
+	ITEM_HANDSHAKE,
+	ITEM_MAILBOX_STATE,
+	ITEM_MAILBOX_TREE_NODE,
+	ITEM_MAILBOX_DELETE,
+	ITEM_MAILBOX,
+	ITEM_MAILBOX_ATTRIBUTE,
+	ITEM_MAIL_CHANGE,
+	ITEM_MAIL_REQUEST,
+	ITEM_MAIL,
+	ITEM_FINISH
+};
+
+struct item {
+	enum item_type type;
+	pool_t pool;
+
+	union {
+		struct dsync_ibc_settings set;
+		struct dsync_mailbox_state state;
+		struct dsync_mailbox_node node;
+		guid_128_t mailbox_guid;
+		struct dsync_mailbox dsync_box;
+		struct dsync_mailbox_attribute attr;
+		struct dsync_mail_change change;
+		struct dsync_mail_request request;
+		struct dsync_mail mail;
+		struct {
+			const struct dsync_mailbox_delete *deletes;
+			unsigned int count;
+			char hierarchy_sep;
+			char escape_char;
+		} mailbox_delete;
+		struct {
+			const char *error;
+			enum mail_error mail_error;
+			bool require_full_resync;
+		} finish;
+	} u;
+};
+
+struct dsync_ibc_pipe {
+	struct dsync_ibc ibc;
+
+	ARRAY(pool_t) pools;
+	ARRAY(struct item) item_queue;
+	struct dsync_ibc_pipe *remote;
+
+	pool_t pop_pool;
+	struct item pop_item;
+};
+
+static pool_t dsync_ibc_pipe_get_pool(struct dsync_ibc_pipe *pipe)
+{
+	pool_t *pools, ret;
+	unsigned int count;
+
+	pools = array_get_modifiable(&pipe->pools, &count);
+	if (count == 0)
+		return pool_alloconly_create(MEMPOOL_GROWING"pipe item pool", 1024);
+
+	ret = pools[count-1];
+	array_delete(&pipe->pools, count-1, 1);
+	p_clear(ret);
+	return ret;
+}
+
+static struct item * ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_pipe_push_item(struct dsync_ibc_pipe *pipe, enum item_type type)
+{
+	struct item *item;
+
+	item = array_append_space(&pipe->item_queue);
+	item->type = type;
+
+	switch (type) {
+	case ITEM_END_OF_LIST:
+	case ITEM_MAILBOX_STATE:
+	case ITEM_MAILBOX_DELETE:
+		break;
+	case ITEM_HANDSHAKE:
+	case ITEM_MAILBOX:
+	case ITEM_MAILBOX_TREE_NODE:
+	case ITEM_MAILBOX_ATTRIBUTE:
+	case ITEM_MAIL_CHANGE:
+	case ITEM_MAIL_REQUEST:
+	case ITEM_MAIL:
+	case ITEM_FINISH:
+		item->pool = dsync_ibc_pipe_get_pool(pipe);
+		break;
+	}
+	return item;
+}
+
+static struct item *
+dsync_ibc_pipe_pop_item(struct dsync_ibc_pipe *pipe, enum item_type type)
+{
+	struct item *item;
+
+	if (array_count(&pipe->item_queue) == 0)
+		return NULL;
+
+	item = array_front_modifiable(&pipe->item_queue);
+	i_assert(item->type == type);
+	pipe->pop_item = *item;
+	array_pop_front(&pipe->item_queue);
+	item = NULL;
+
+	pool_unref(&pipe->pop_pool);
+	pipe->pop_pool = pipe->pop_item.pool;
+	return &pipe->pop_item;
+}
+
+static bool dsync_ibc_pipe_try_pop_eol(struct dsync_ibc_pipe *pipe)
+{
+	const struct item *item;
+
+	if (array_count(&pipe->item_queue) == 0)
+		return FALSE;
+
+	item = array_front(&pipe->item_queue);
+	if (item->type != ITEM_END_OF_LIST)
+		return FALSE;
+
+	array_pop_front(&pipe->item_queue);
+	return TRUE;
+}
+
+static void dsync_ibc_pipe_deinit(struct dsync_ibc *ibc)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+	pool_t pool;
+
+	if (pipe->remote != NULL) {
+		i_assert(pipe->remote->remote == pipe);
+		pipe->remote->remote = NULL;
+	}
+
+	pool_unref(&pipe->pop_pool);
+	array_foreach_modifiable(&pipe->item_queue, item) {
+		pool_unref(&item->pool);
+	}
+	array_foreach_elem(&pipe->pools, pool)
+		pool_unref(&pool);
+	array_free(&pipe->pools);
+	array_free(&pipe->item_queue);
+	i_free(pipe);
+}
+
+static void
+dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc,
+			      const struct dsync_ibc_settings *set)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_HANDSHAKE);
+	item->u.set = *set;
+	item->u.set.sync_ns_prefixes =
+		p_strdup(item->pool, set->sync_ns_prefixes);
+	item->u.set.sync_box = p_strdup(item->pool, set->sync_box);
+	item->u.set.virtual_all_box = p_strdup(item->pool, set->virtual_all_box);
+	item->u.set.exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
+		p_strarray_dup(item->pool, set->exclude_mailboxes);
+	memcpy(item->u.set.sync_box_guid, set->sync_box_guid,
+	       sizeof(item->u.set.sync_box_guid));
+	item->u.set.sync_since_timestamp = set->sync_since_timestamp;
+	item->u.set.sync_until_timestamp = set->sync_until_timestamp;
+	item->u.set.sync_max_size = set->sync_max_size;
+	item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_handshake(struct dsync_ibc *ibc,
+			      const struct dsync_ibc_settings **set_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_HANDSHAKE);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*set_r = &item->u.set;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static bool dsync_ibc_pipe_is_send_queue_full(struct dsync_ibc *ibc)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+	return array_count(&pipe->remote->item_queue) > 0;
+}
+
+static bool dsync_ibc_pipe_has_pending_data(struct dsync_ibc *ibc)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+	return array_count(&pipe->item_queue) > 0;
+}
+
+static void
+dsync_ibc_pipe_send_end_of_list(struct dsync_ibc *ibc,
+				enum dsync_ibc_eol_type type ATTR_UNUSED)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+	dsync_ibc_pipe_push_item(pipe->remote, ITEM_END_OF_LIST);
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_state(struct dsync_ibc *ibc,
+				  const struct dsync_mailbox_state *state)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_STATE);
+	item->u.state = *state;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_state(struct dsync_ibc *ibc,
+				  struct dsync_mailbox_state *state_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_STATE);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*state_r = item->u.state;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_tree_node(struct dsync_ibc *ibc,
+				      const char *const *name,
+				      const struct dsync_mailbox_node *node)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_TREE_NODE);
+
+	/* a little bit kludgy way to send it */
+	item->u.node.name = (void *)p_strarray_dup(item->pool, name);
+	dsync_mailbox_node_copy_data(&item->u.node, node);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_tree_node(struct dsync_ibc *ibc,
+				      const char *const **name_r,
+				      const struct dsync_mailbox_node **node_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_TREE_NODE);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*name_r = (const void *)item->u.node.name;
+	item->u.node.name = NULL;
+
+	*node_r = &item->u.node;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_deletes(struct dsync_ibc *ibc,
+				    const struct dsync_mailbox_delete *deletes,
+				    unsigned int count, char hierarchy_sep,
+				    char escape_char)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_DELETE);
+
+	/* we'll assume that the deletes are permanent. this works for now.. */
+	/* a little bit kludgy way to send it */
+	item->u.mailbox_delete.deletes = deletes;
+	item->u.mailbox_delete.count = count;
+	item->u.mailbox_delete.hierarchy_sep = hierarchy_sep;
+	item->u.mailbox_delete.escape_char = escape_char;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_deletes(struct dsync_ibc *ibc,
+				    const struct dsync_mailbox_delete **deletes_r,
+				    unsigned int *count_r,
+				    char *hierarchy_sep_r,
+				    char *escape_char_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_DELETE);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*deletes_r = item->u.mailbox_delete.deletes;
+	*count_r = item->u.mailbox_delete.count;
+	*hierarchy_sep_r = item->u.mailbox_delete.hierarchy_sep;
+	*escape_char_r = item->u.mailbox_delete.escape_char;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox(struct dsync_ibc *ibc,
+			    const struct dsync_mailbox *dsync_box)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+	const struct mailbox_cache_field *cf;
+	struct mailbox_cache_field *ncf;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX);
+	item->u.dsync_box = *dsync_box;
+	p_array_init(&item->u.dsync_box.cache_fields, item->pool,
+		     array_count(&dsync_box->cache_fields));
+	array_foreach(&dsync_box->cache_fields, cf) {
+		ncf = array_append_space(&item->u.dsync_box.cache_fields);
+		ncf->name = p_strdup(item->pool, cf->name);
+		ncf->decision = cf->decision;
+		ncf->last_used = cf->last_used;
+	}
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox(struct dsync_ibc *ibc,
+			    const struct dsync_mailbox **dsync_box_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*dsync_box_r = &item->u.dsync_box;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_attribute(struct dsync_ibc *ibc,
+				      const struct dsync_mailbox_attribute *attr)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_ATTRIBUTE);
+	dsync_mailbox_attribute_dup(item->pool, attr, &item->u.attr);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_attribute(struct dsync_ibc *ibc,
+				      const struct dsync_mailbox_attribute **attr_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_ATTRIBUTE);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*attr_r = &item->u.attr;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_change(struct dsync_ibc *ibc,
+			   const struct dsync_mail_change *change)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_CHANGE);
+	dsync_mail_change_dup(item->pool, change, &item->u.change);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_change(struct dsync_ibc *ibc,
+			   const struct dsync_mail_change **change_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_CHANGE);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*change_r = &item->u.change;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mail_request(struct dsync_ibc *ibc,
+				 const struct dsync_mail_request *request)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_REQUEST);
+	item->u.request.guid = p_strdup(item->pool, request->guid);
+	item->u.request.uid = request->uid;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mail_request(struct dsync_ibc *ibc,
+				 const struct dsync_mail_request **request_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_REQUEST);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*request_r = &item->u.request;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL);
+	item->u.mail.guid = p_strdup(item->pool, mail->guid);
+	item->u.mail.uid = mail->uid;
+	item->u.mail.pop3_uidl = p_strdup(item->pool, mail->pop3_uidl);
+	item->u.mail.pop3_order = mail->pop3_order;
+	item->u.mail.received_date = mail->received_date;
+	if (mail->input != NULL) {
+		item->u.mail.input = mail->input;
+		i_stream_ref(mail->input);
+	}
+	item->u.mail.input_mail = mail->input_mail;
+	item->u.mail.input_mail_uid = mail->input_mail_uid;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	if (dsync_ibc_pipe_try_pop_eol(pipe))
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*mail_r = &item->u.mail;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error,
+			   enum mail_error mail_error,
+			   bool require_full_resync)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_FINISH);
+	item->u.finish.error = p_strdup(item->pool, error);
+	item->u.finish.mail_error = mail_error;
+	item->u.finish.require_full_resync = require_full_resync;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r,
+			   enum mail_error *mail_error_r,
+			   bool *require_full_resync_r)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+	struct item *item;
+
+	item = dsync_ibc_pipe_pop_item(pipe, ITEM_FINISH);
+	if (item == NULL)
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	*error_r = item->u.finish.error;
+	*mail_error_r = item->u.finish.mail_error;
+	*require_full_resync_r = item->u.finish.require_full_resync;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void pipe_close_mail_streams(struct dsync_ibc_pipe *pipe)
+{
+	struct item *item;
+
+	if (array_count(&pipe->item_queue) > 0) {
+		item = array_front_modifiable(&pipe->item_queue);
+		if (item->type == ITEM_MAIL &&
+		    item->u.mail.input != NULL)
+			i_stream_unref(&item->u.mail.input);
+	}
+}
+
+static void dsync_ibc_pipe_close_mail_streams(struct dsync_ibc *ibc)
+{
+	struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+	pipe_close_mail_streams(pipe);
+	pipe_close_mail_streams(pipe->remote);
+}
+
+static const struct dsync_ibc_vfuncs dsync_ibc_pipe_vfuncs = {
+	dsync_ibc_pipe_deinit,
+	dsync_ibc_pipe_send_handshake,
+	dsync_ibc_pipe_recv_handshake,
+	dsync_ibc_pipe_send_end_of_list,
+	dsync_ibc_pipe_send_mailbox_state,
+	dsync_ibc_pipe_recv_mailbox_state,
+	dsync_ibc_pipe_send_mailbox_tree_node,
+	dsync_ibc_pipe_recv_mailbox_tree_node,
+	dsync_ibc_pipe_send_mailbox_deletes,
+	dsync_ibc_pipe_recv_mailbox_deletes,
+	dsync_ibc_pipe_send_mailbox,
+	dsync_ibc_pipe_recv_mailbox,
+	dsync_ibc_pipe_send_mailbox_attribute,
+	dsync_ibc_pipe_recv_mailbox_attribute,
+	dsync_ibc_pipe_send_change,
+	dsync_ibc_pipe_recv_change,
+	dsync_ibc_pipe_send_mail_request,
+	dsync_ibc_pipe_recv_mail_request,
+	dsync_ibc_pipe_send_mail,
+	dsync_ibc_pipe_recv_mail,
+	dsync_ibc_pipe_send_finish,
+	dsync_ibc_pipe_recv_finish,
+	dsync_ibc_pipe_close_mail_streams,
+	dsync_ibc_pipe_is_send_queue_full,
+	dsync_ibc_pipe_has_pending_data
+};
+
+static struct dsync_ibc_pipe *
+dsync_ibc_pipe_alloc(void)
+{
+	struct dsync_ibc_pipe *pipe;
+
+	pipe = i_new(struct dsync_ibc_pipe, 1);
+	pipe->ibc.v = dsync_ibc_pipe_vfuncs;
+	i_array_init(&pipe->pools, 4);
+	i_array_init(&pipe->item_queue, 4);
+	return pipe;
+}
+
+void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r, struct dsync_ibc **ibc2_r)
+{
+	struct dsync_ibc_pipe *pipe1, *pipe2;
+
+	pipe1 = dsync_ibc_pipe_alloc();
+	pipe2 = dsync_ibc_pipe_alloc();
+	pipe1->remote = pipe2;
+	pipe2->remote = pipe1;
+	*ibc1_r = &pipe1->ibc;
+	*ibc2_r = &pipe2->ibc;
+}
diff --git a/src/doveadm/dsync/dsync-ibc-private.h b/src/doveadm/dsync/dsync-ibc-private.h
new file mode 100644
index 0000000..c055aca
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc-private.h
@@ -0,0 +1,96 @@
+#ifndef DSYNC_IBC_PRIVATE_H
+#define DSYNC_IBC_PRIVATE_H
+
+#include "dsync-ibc.h"
+
+struct dsync_ibc_vfuncs {
+	void (*deinit)(struct dsync_ibc *ibc);
+
+	void (*send_handshake)(struct dsync_ibc *ibc,
+			       const struct dsync_ibc_settings *set);
+	enum dsync_ibc_recv_ret
+		(*recv_handshake)(struct dsync_ibc *ibc,
+				  const struct dsync_ibc_settings **set_r);
+
+	void (*send_end_of_list)(struct dsync_ibc *ibc,
+				 enum dsync_ibc_eol_type type);
+
+	void (*send_mailbox_state)(struct dsync_ibc *ibc,
+				   const struct dsync_mailbox_state *state);
+	enum dsync_ibc_recv_ret
+		(*recv_mailbox_state)(struct dsync_ibc *ibc,
+				      struct dsync_mailbox_state *state_r);
+
+	void (*send_mailbox_tree_node)(struct dsync_ibc *ibc,
+				       const char *const *name,
+				       const struct dsync_mailbox_node *node);
+	enum dsync_ibc_recv_ret
+		(*recv_mailbox_tree_node)(struct dsync_ibc *ibc,
+					  const char *const **name_r,
+					  const struct dsync_mailbox_node **node_r);
+
+	void (*send_mailbox_deletes)(struct dsync_ibc *ibc,
+				     const struct dsync_mailbox_delete *deletes,
+				     unsigned int count, char hierarchy_sep,
+				     char escape_char);
+	enum dsync_ibc_recv_ret
+		(*recv_mailbox_deletes)(struct dsync_ibc *ibc,
+					const struct dsync_mailbox_delete **deletes_r,
+					unsigned int *count_r,
+					char *hierarchy_sep_r,
+					char *escape_char_r);
+
+	void (*send_mailbox)(struct dsync_ibc *ibc,
+			     const struct dsync_mailbox *dsync_box);
+	enum dsync_ibc_recv_ret
+		(*recv_mailbox)(struct dsync_ibc *ibc,
+				const struct dsync_mailbox **dsync_box_r);
+
+	void (*send_mailbox_attribute)(struct dsync_ibc *ibc,
+				       const struct dsync_mailbox_attribute *attr);
+	enum dsync_ibc_recv_ret
+		(*recv_mailbox_attribute)(struct dsync_ibc *ibc,
+					  const struct dsync_mailbox_attribute **attr_r);
+
+	void (*send_change)(struct dsync_ibc *ibc,
+			    const struct dsync_mail_change *change);
+	enum dsync_ibc_recv_ret
+		(*recv_change)(struct dsync_ibc *ibc,
+			       const struct dsync_mail_change **change_r);
+
+	void (*send_mail_request)(struct dsync_ibc *ibc,
+				  const struct dsync_mail_request *request);
+	enum dsync_ibc_recv_ret
+		(*recv_mail_request)(struct dsync_ibc *ibc,
+				     const struct dsync_mail_request **request_r);
+
+	void (*send_mail)(struct dsync_ibc *ibc,
+			  const struct dsync_mail *mail);
+	enum dsync_ibc_recv_ret
+		(*recv_mail)(struct dsync_ibc *ibc,
+			     struct dsync_mail **mail_r);
+
+	void (*send_finish)(struct dsync_ibc *ibc, const char *error,
+			    enum mail_error mail_error,
+			    bool require_full_resync);
+	enum dsync_ibc_recv_ret
+		(*recv_finish)(struct dsync_ibc *ibc, const char **error_r,
+			       enum mail_error *mail_error_r,
+			       bool *require_full_resync_r);
+
+	void (*close_mail_streams)(struct dsync_ibc *ibc);
+	bool (*is_send_queue_full)(struct dsync_ibc *ibc);
+	bool (*has_pending_data)(struct dsync_ibc *ibc);
+};
+
+struct dsync_ibc {
+	struct dsync_ibc_vfuncs v;
+
+	io_callback_t *io_callback;
+	void *io_context;
+
+	bool failed:1;
+	bool timeout:1;
+};
+
+#endif
diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
new file mode 100644
index 0000000..17115b0
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -0,0 +1,2138 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "safe-mkstemp.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "istream-seekable.h"
+#include "istream-dot.h"
+#include "ostream.h"
+#include "str.h"
+#include "strescape.h"
+#include "master-service.h"
+#include "mail-cache.h"
+#include "mail-storage-private.h"
+#include "dsync-serializer.h"
+#include "dsync-deserializer.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-state.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-ibc-private.h"
+
+
+#define DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE (1024*128)
+
+#define DSYNC_PROTOCOL_VERSION_MAJOR 3
+#define DSYNC_PROTOCOL_VERSION_MINOR 5
+#define DSYNC_HANDSHAKE_VERSION "VERSION\tdsync\t3\t5\n"
+
+#define DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES 1
+#define DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID 2
+#define DSYNC_PROTOCOL_MINOR_HAVE_FINISH 3
+#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2 4
+#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3 5
+
+enum item_type {
+	ITEM_NONE,
+	ITEM_DONE,
+
+	ITEM_HANDSHAKE,
+	ITEM_MAILBOX_STATE,
+	ITEM_MAILBOX_TREE_NODE,
+	ITEM_MAILBOX_DELETE,
+	ITEM_MAILBOX,
+
+	ITEM_MAILBOX_ATTRIBUTE,
+	ITEM_MAIL_CHANGE,
+	ITEM_MAIL_REQUEST,
+	ITEM_MAIL,
+	ITEM_FINISH,
+
+	ITEM_MAILBOX_CACHE_FIELD,
+
+	ITEM_END_OF_LIST
+};
+
+#define END_OF_LIST_LINE "."
+static const struct {
+	/* full human readable name of the item */
+	const char *name;
+	/* unique character identifying the item */
+	char chr;
+	const char *required_keys;
+	const char *optional_keys;
+	unsigned int min_minor_version;
+} items[ITEM_END_OF_LIST+1] = {
+	{ NULL, '\0', NULL, NULL, 0 },
+	{ .name = "done",
+	  .chr = 'X',
+	  .optional_keys = ""
+	},
+	{ .name = "handshake",
+	  .chr = 'H',
+	  .required_keys = "hostname",
+	  .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type "
+	  	"debug sync_visible_namespaces exclude_mailboxes  "
+	  	"send_mail_requests backup_send backup_recv lock_timeout "
+	  	"no_mail_sync no_backup_overwrite purge_remote "
+		"no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp "
+		"virtual_all_box empty_hdr_workaround import_commit_msgs_interval "
+		"hashed_headers alt_char"
+	},
+	{ .name = "mailbox_state",
+	  .chr = 'S',
+	  .required_keys = "mailbox_guid last_uidvalidity last_common_uid "
+	  	"last_common_modseq last_common_pvt_modseq",
+	  .optional_keys = "last_messages_count changes_during_sync"
+	},
+	{ .name = "mailbox_tree_node",
+	  .chr = 'N',
+	  .required_keys = "name existence",
+	  .optional_keys = "mailbox_guid uid_validity uid_next "
+	  	"last_renamed_or_created subscribed last_subscription_change"
+	},
+	{ .name = "mailbox_delete",
+	  .chr = 'D',
+	  .required_keys = "hierarchy_sep",
+	  .optional_keys = "escape_char mailboxes dirs unsubscribes"
+	},
+	{ .name = "mailbox",
+	  .chr = 'B',
+	  .required_keys = "mailbox_guid uid_validity uid_next messages_count "
+		"first_recent_uid highest_modseq highest_pvt_modseq",
+	  .optional_keys = "mailbox_lost mailbox_ignore "
+			   "cache_fields have_guids have_save_guids have_only_guid128"
+	},
+	{ .name = "mailbox_attribute",
+	  .chr = 'A',
+	  .required_keys = "type key",
+	  .optional_keys = "value stream deleted last_change modseq",
+	  .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES
+	},
+	{ .name = "mail_change",
+	  .chr = 'C',
+	  .required_keys = "type uid",
+	  .optional_keys = "guid hdr_hash modseq pvt_modseq "
+	  	"add_flags remove_flags final_flags "
+		"keywords_reset keyword_changes received_timestamp virtual_size"
+	},
+	{ .name = "mail_request",
+	  .chr = 'R',
+	  .optional_keys = "guid uid"
+	},
+	{ .name = "mail",
+	  .chr = 'M',
+	  .optional_keys = "guid uid pop3_uidl pop3_order received_date saved_date stream"
+	},
+	{ .name = "finish",
+	  .chr = 'F',
+	  .optional_keys = "error mail_error require_full_resync",
+	  .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_FINISH
+	},
+	{ .name = "mailbox_cache_field",
+	  .chr = 'c',
+	  .required_keys = "name decision",
+	  .optional_keys = "last_used"
+	},
+
+	{ "end_of_list", '\0', NULL, NULL, 0 }
+};
+
+struct dsync_ibc_stream {
+	struct dsync_ibc ibc;
+
+	char *name, *temp_path_prefix;
+	unsigned int timeout_secs;
+	struct istream *input;
+	struct ostream *output;
+	struct io *io;
+	struct timeout *to;
+
+	unsigned int minor_version;
+	struct dsync_serializer *serializers[ITEM_END_OF_LIST];
+	struct dsync_deserializer *deserializers[ITEM_END_OF_LIST];
+
+	pool_t ret_pool;
+	struct dsync_deserializer_decoder *cur_decoder;
+
+	struct istream *value_output, *value_input;
+	struct dsync_mail *cur_mail;
+	struct dsync_mailbox_attribute *cur_attr;
+	char value_output_last;
+
+	enum item_type last_recv_item, last_sent_item;
+	bool last_recv_item_eol:1;
+	bool last_sent_item_eol:1;
+
+	bool version_received:1;
+	bool handshake_received:1;
+	bool has_pending_data:1;
+	bool finish_received:1;
+	bool done_received:1;
+	bool stopped:1;
+};
+
+static const char *dsync_ibc_stream_get_state(struct dsync_ibc_stream *ibc)
+{
+	if (!ibc->version_received)
+		return "version not received";
+	else if (!ibc->handshake_received)
+		return "handshake not received";
+
+	return t_strdup_printf("last sent=%s%s, last recv=%s%s",
+			       items[ibc->last_sent_item].name,
+			       ibc->last_sent_item_eol ? " (EOL)" : "",
+			       items[ibc->last_recv_item].name,
+			       ibc->last_recv_item_eol ? " (EOL)" : "");
+}
+
+static void dsync_ibc_stream_stop(struct dsync_ibc_stream *ibc)
+{
+	ibc->stopped = TRUE;
+	i_stream_close(ibc->input);
+	o_stream_close(ibc->output);
+	io_loop_stop(current_ioloop);
+}
+
+static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc)
+{
+	do {
+		i_stream_skip(ibc->value_input,
+			      i_stream_get_data_size(ibc->value_input));
+	} while (i_stream_read(ibc->value_input) > 0);
+	if (ibc->value_input->eof) {
+		if (ibc->value_input->stream_errno != 0) {
+			i_error("dsync(%s): read(%s) failed: %s (%s)", ibc->name,
+				i_stream_get_name(ibc->value_input),
+				i_stream_get_error(ibc->value_input),
+				dsync_ibc_stream_get_state(ibc));
+			dsync_ibc_stream_stop(ibc);
+			return -1;
+		}
+		/* finished reading the mail stream */
+		i_assert(ibc->value_input->eof);
+		i_stream_seek(ibc->value_input, 0);
+		ibc->has_pending_data = TRUE;
+		ibc->value_input = NULL;
+		return 1;
+	}
+	return 0;
+}
+
+static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc)
+{
+	timeout_reset(ibc->to);
+	if (ibc->value_input != NULL) {
+		if (dsync_ibc_stream_read_mail_stream(ibc) == 0)
+			return;
+	}
+	o_stream_cork(ibc->output);
+	ibc->ibc.io_callback(ibc->ibc.io_context);
+	o_stream_uncork(ibc->output);
+}
+
+static int dsync_ibc_stream_send_value_stream(struct dsync_ibc_stream *ibc)
+{
+	const unsigned char *data;
+	unsigned char add;
+	size_t i, size;
+	int ret;
+
+	while ((ret = i_stream_read_more(ibc->value_output, &data, &size)) > 0) {
+		add = '\0';
+		for (i = 0; i < size; i++) {
+			if (data[i] == '.' &&
+			    ((i == 0 && ibc->value_output_last == '\n') ||
+			     (i > 0 && data[i-1] == '\n'))) {
+				/* escape the dot */
+				add = '.';
+				break;
+			}
+		}
+
+		if (i > 0) {
+			o_stream_nsend(ibc->output, data, i);
+			ibc->value_output_last = data[i-1];
+			i_stream_skip(ibc->value_output, i);
+		}
+
+		if (o_stream_get_buffer_used_size(ibc->output) >= 4096) {
+			if ((ret = o_stream_flush(ibc->output)) < 0) {
+				dsync_ibc_stream_stop(ibc);
+				return -1;
+			}
+			if (ret == 0) {
+				/* continue later */
+				o_stream_set_flush_pending(ibc->output, TRUE);
+				return 0;
+			}
+		}
+
+		if (add != '\0') {
+			o_stream_nsend(ibc->output, &add, 1);
+			ibc->value_output_last = add;
+		}
+	}
+	i_assert(ret == -1);
+
+	if (ibc->value_output->stream_errno != 0) {
+		i_error("dsync(%s): read(%s) failed: %s (%s)",
+			ibc->name, i_stream_get_name(ibc->value_output),
+			i_stream_get_error(ibc->value_output),
+			dsync_ibc_stream_get_state(ibc));
+		dsync_ibc_stream_stop(ibc);
+		return -1;
+	}
+
+	/* finished sending the stream. use "CRLF." instead of "LF." just in
+	   case we're sending binary data that ends with CR. */
+	o_stream_nsend_str(ibc->output, "\r\n.\r\n");
+	i_stream_unref(&ibc->value_output);
+	return 1;
+}
+
+static int dsync_ibc_stream_output(struct dsync_ibc_stream *ibc)
+{
+	struct ostream *output = ibc->output;
+	int ret;
+
+	if ((ret = o_stream_flush(output)) < 0)
+		ret = 1;
+	else if (ibc->value_output != NULL) {
+		if (dsync_ibc_stream_send_value_stream(ibc) < 0)
+			ret = 1;
+	}
+	timeout_reset(ibc->to);
+
+	if (!dsync_ibc_is_send_queue_full(&ibc->ibc))
+		ibc->ibc.io_callback(ibc->ibc.io_context);
+	return ret;
+}
+
+static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc)
+{
+	i_error("dsync(%s): I/O has stalled, no activity for %u seconds (%s)",
+		ibc->name, ibc->timeout_secs, dsync_ibc_stream_get_state(ibc));
+	ibc->ibc.timeout = TRUE;
+	dsync_ibc_stream_stop(ibc);
+}
+
+static void dsync_ibc_stream_init(struct dsync_ibc_stream *ibc)
+{
+	unsigned int i;
+
+	ibc->io = io_add_istream(ibc->input, dsync_ibc_stream_input, ibc);
+	io_set_pending(ibc->io);
+	o_stream_set_no_error_handling(ibc->output, TRUE);
+	o_stream_set_flush_callback(ibc->output, dsync_ibc_stream_output, ibc);
+	ibc->to = timeout_add(ibc->timeout_secs * 1000,
+			      dsync_ibc_stream_timeout, ibc);
+	o_stream_cork(ibc->output);
+	o_stream_nsend_str(ibc->output, DSYNC_HANDSHAKE_VERSION);
+
+	/* initialize serializers and send their headers to remote */
+	for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) T_BEGIN {
+		const char *keys;
+
+		keys = items[i].required_keys == NULL ? items[i].optional_keys :
+			t_strconcat(items[i].required_keys, " ",
+				    items[i].optional_keys, NULL);
+		if (keys != NULL) {
+			i_assert(items[i].chr != '\0');
+
+			ibc->serializers[i] =
+				dsync_serializer_init(t_strsplit_spaces(keys, " "));
+			o_stream_nsend(ibc->output, &items[i].chr, 1);
+			o_stream_nsend_str(ibc->output,
+				dsync_serializer_encode_header_line(ibc->serializers[i]));
+		}
+	} T_END;
+	o_stream_nsend_str(ibc->output, ".\n");
+	o_stream_uncork(ibc->output);
+}
+
+static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	unsigned int i;
+
+	for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) {
+		if (ibc->serializers[i] != NULL)
+			dsync_serializer_deinit(&ibc->serializers[i]);
+		if (ibc->deserializers[i] != NULL)
+			dsync_deserializer_deinit(&ibc->deserializers[i]);
+	}
+	if (ibc->cur_decoder != NULL)
+		dsync_deserializer_decode_finish(&ibc->cur_decoder);
+	if (ibc->value_output != NULL)
+		i_stream_unref(&ibc->value_output);
+	else {
+		/* If the remote has not told us that they are closing we
+		   notify remote that we're closing. this is mainly to avoid
+		   "read() failed: EOF" errors on failing dsyncs.
+
+		   Avoid a race condition:
+		   We do not tell the remote we are closing if they have
+		   already told us because they close the
+		   connection after sending ITEM_DONE and will
+		   not be ever receive anything else from us unless
+		   it just happens to get combined into the same packet
+		   as a previous response and is already in the buffer.
+		*/
+		if (!ibc->done_received && !ibc->finish_received) {
+			o_stream_nsend_str(ibc->output,
+				t_strdup_printf("%c\n", items[ITEM_DONE].chr));
+		}
+		(void)o_stream_finish(ibc->output);
+	}
+
+	timeout_remove(&ibc->to);
+	io_remove(&ibc->io);
+	i_stream_destroy(&ibc->input);
+	o_stream_destroy(&ibc->output);
+	pool_unref(&ibc->ret_pool);
+	i_free(ibc->temp_path_prefix);
+	i_free(ibc->name);
+	i_free(ibc);
+}
+
+static int dsync_ibc_stream_next_line(struct dsync_ibc_stream *ibc,
+				      const char **line_r)
+{
+	string_t *error;
+	const char *line;
+	ssize_t ret;
+
+	line = i_stream_next_line(ibc->input);
+	if (line != NULL) {
+		*line_r = line;
+		return 1;
+	}
+	/* try reading some */
+	if ((ret = i_stream_read(ibc->input)) == -1) {
+		if (ibc->stopped)
+			return -1;
+		error = t_str_new(128);
+		if (ibc->input->stream_errno != 0) {
+			str_printfa(error, "read(%s) failed: %s", ibc->name,
+				    i_stream_get_error(ibc->input));
+		} else {
+			i_assert(ibc->input->eof);
+			str_printfa(error, "read(%s) failed: EOF", ibc->name);
+		}
+		str_printfa(error, " (%s)", dsync_ibc_stream_get_state(ibc));
+		i_error("%s", str_c(error));
+		dsync_ibc_stream_stop(ibc);
+		return -1;
+	}
+	i_assert(ret >= 0);
+	*line_r = i_stream_next_line(ibc->input);
+	if (*line_r == NULL) {
+		ibc->has_pending_data = FALSE;
+		return 0;
+	}
+	ibc->has_pending_data = TRUE;
+	return 1;
+}
+
+static void ATTR_FORMAT(3, 4) ATTR_NULL(2)
+dsync_ibc_input_error(struct dsync_ibc_stream *ibc,
+		      struct dsync_deserializer_decoder *decoder,
+		      const char *fmt, ...)
+{
+	va_list args;
+	const char *error;
+
+	va_start(args, fmt);
+	error = t_strdup_vprintf(fmt, args);
+	if (decoder == NULL)
+		i_error("dsync(%s): %s", ibc->name, error);
+	else {
+		i_error("dsync(%s): %s: %s", ibc->name,
+			dsync_deserializer_decoder_get_name(decoder), error);
+	}
+	va_end(args);
+
+	dsync_ibc_stream_stop(ibc);
+}
+
+static void
+dsync_ibc_stream_send_string(struct dsync_ibc_stream *ibc,
+			     const string_t *str)
+{
+	i_assert(ibc->value_output == NULL);
+	o_stream_nsend(ibc->output, str_data(str), str_len(str));
+}
+
+static int seekable_fd_callback(const char **path_r, void *context)
+{
+	struct dsync_ibc_stream *ibc = context;
+	string_t *path;
+	int fd;
+
+	path = t_str_new(128);
+	str_append(path, ibc->temp_path_prefix);
+	fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1);
+	if (fd == -1) {
+		i_error("safe_mkstemp(%s) failed: %m", str_c(path));
+		return -1;
+	}
+
+	/* we just want the fd, unlink it */
+	if (i_unlink(str_c(path)) < 0) {
+		/* shouldn't happen.. */
+		i_close_fd(&fd);
+		return -1;
+	}
+
+	*path_r = str_c(path);
+	return fd;
+}
+
+static struct istream *
+dsync_ibc_stream_input_stream(struct dsync_ibc_stream *ibc)
+{
+	struct istream *inputs[2];
+
+	inputs[0] = i_stream_create_dot(ibc->input, FALSE);
+	inputs[1] = NULL;
+	ibc->value_input = i_stream_create_seekable(inputs, MAIL_READ_FULL_BLOCK_SIZE,
+						    seekable_fd_callback, ibc);
+	i_stream_unref(&inputs[0]);
+
+	return ibc->value_input;
+}
+
+static int
+dsync_ibc_check_missing_deserializers(struct dsync_ibc_stream *ibc)
+{
+	unsigned int i;
+	int ret = 0;
+
+	for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) {
+		if (ibc->deserializers[i] == NULL &&
+		    ibc->minor_version >= items[i].min_minor_version &&
+		    (items[i].required_keys != NULL ||
+		     items[i].optional_keys != NULL)) {
+			dsync_ibc_input_error(ibc, NULL,
+				"Remote didn't handshake deserializer for %s",
+				items[i].name);
+			ret = -1;
+		}
+	}
+	return ret;
+}
+
+static bool
+dsync_ibc_stream_handshake(struct dsync_ibc_stream *ibc, const char *line)
+{
+	enum item_type item = ITEM_NONE;
+	const char *const *required_keys, *error;
+	unsigned int i;
+
+	if (ibc->handshake_received)
+		return TRUE;
+
+	if (!ibc->version_received) {
+		if (!version_string_verify_full(line, "dsync",
+						DSYNC_PROTOCOL_VERSION_MAJOR,
+						&ibc->minor_version)) {
+			dsync_ibc_input_error(ibc, NULL,
+				"Remote dsync doesn't use compatible protocol");
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		ibc->version_received = TRUE;
+		return FALSE;
+	}
+
+	if (strcmp(line, END_OF_LIST_LINE) == 0) {
+		/* finished handshaking */
+		if (dsync_ibc_check_missing_deserializers(ibc) < 0)
+			return FALSE;
+		ibc->handshake_received = TRUE;
+		ibc->last_recv_item = ITEM_HANDSHAKE;
+		return FALSE;
+	}
+
+	for (i = 1; i < ITEM_END_OF_LIST; i++) {
+		if (items[i].chr == line[0]) {
+			item = i;
+			break;
+		}
+	}
+	if (item == ITEM_NONE) {
+		/* unknown deserializer, ignore */
+		return FALSE;
+	}
+
+	required_keys = items[item].required_keys == NULL ? NULL :
+		t_strsplit(items[item].required_keys, " ");
+	if (dsync_deserializer_init(items[item].name,
+				    required_keys, line + 1,
+				    &ibc->deserializers[item], &error) < 0) {
+		dsync_ibc_input_error(ibc, NULL,
+			"Remote sent invalid handshake for %s: %s",
+			items[item].name, error);
+	}
+	return FALSE;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_input_next(struct dsync_ibc_stream *ibc, enum item_type item,
+			    struct dsync_deserializer_decoder **decoder_r)
+{
+	enum item_type line_item = ITEM_NONE;
+	const char *line, *error;
+	unsigned int i;
+
+	i_assert(ibc->value_input == NULL);
+
+	timeout_reset(ibc->to);
+
+	do {
+		if (dsync_ibc_stream_next_line(ibc, &line) <= 0)
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	} while (!dsync_ibc_stream_handshake(ibc, line));
+
+	ibc->last_recv_item = item;
+	ibc->last_recv_item_eol = FALSE;
+
+	if (strcmp(line, END_OF_LIST_LINE) == 0) {
+		/* end of this list */
+		ibc->last_recv_item_eol = TRUE;
+		return DSYNC_IBC_RECV_RET_FINISHED;
+	}
+	if (line[0] == items[ITEM_DONE].chr) {
+		/* remote cleanly closed the connection, possibly because of
+		   some failure (which it should have logged). we don't want to
+		   log any stream errors anyway after this. */
+		ibc->done_received = TRUE;
+		dsync_ibc_stream_stop(ibc);
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+	}
+	for (i = 1; i < ITEM_END_OF_LIST; i++) {
+		if (*line == items[i].chr) {
+			line_item = i;
+			break;
+		}
+	}
+	if (line_item != item) {
+		dsync_ibc_input_error(ibc, NULL,
+			"Received unexpected input %c != %c",
+			*line, items[item].chr);
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	if (ibc->cur_decoder != NULL)
+		dsync_deserializer_decode_finish(&ibc->cur_decoder);
+	if (dsync_deserializer_decode_begin(ibc->deserializers[item],
+					    line+1, &ibc->cur_decoder,
+					    &error) < 0) {
+		dsync_ibc_input_error(ibc, NULL, "Invalid input to %s: %s",
+				      items[item].name, error);
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	*decoder_r = ibc->cur_decoder;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static struct dsync_serializer_encoder *
+dsync_ibc_send_encode_begin(struct dsync_ibc_stream *ibc, enum item_type item)
+{
+	ibc->last_sent_item = item;
+	ibc->last_sent_item_eol = FALSE;
+	return dsync_serializer_encode_begin(ibc->serializers[item]);
+}
+
+static void
+dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
+				const struct dsync_ibc_settings *set)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+	char sync_type[2];
+
+	str_append_c(str, items[ITEM_HANDSHAKE].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_HANDSHAKE);
+	dsync_serializer_encode_add(encoder, "hostname", set->hostname);
+	if (set->sync_ns_prefixes != NULL) {
+		dsync_serializer_encode_add(encoder, "sync_ns_prefix",
+					    set->sync_ns_prefixes);
+	}
+	if (set->sync_box != NULL)
+		dsync_serializer_encode_add(encoder, "sync_box", set->sync_box);
+	if (set->virtual_all_box != NULL) {
+		dsync_serializer_encode_add(encoder, "virtual_all_box",
+					    set->virtual_all_box);
+	}
+	if (set->exclude_mailboxes != NULL) {
+		string_t *substr = t_str_new(64);
+		unsigned int i;
+
+		for (i = 0; set->exclude_mailboxes[i] != NULL; i++) {
+			if (i != 0)
+				str_append_c(substr, '\t');
+			str_append_tabescaped(substr, set->exclude_mailboxes[i]);
+		}
+		dsync_serializer_encode_add(encoder, "exclude_mailboxes",
+					    str_c(substr));
+	}
+	if (!guid_128_is_empty(set->sync_box_guid)) {
+		dsync_serializer_encode_add(encoder, "sync_box_guid",
+			guid_128_to_string(set->sync_box_guid));
+	}
+
+	sync_type[0] = sync_type[1] = '\0';
+	switch (set->sync_type) {
+	case DSYNC_BRAIN_SYNC_TYPE_UNKNOWN:
+		break;
+	case DSYNC_BRAIN_SYNC_TYPE_FULL:
+		sync_type[0] = 'f';
+		break;
+	case DSYNC_BRAIN_SYNC_TYPE_CHANGED:
+		sync_type[0] = 'c';
+		break;
+	case DSYNC_BRAIN_SYNC_TYPE_STATE:
+		sync_type[0] = 's';
+		break;
+	}
+	if (sync_type[0] != '\0')
+		dsync_serializer_encode_add(encoder, "sync_type", sync_type);
+	if (set->lock_timeout > 0) {
+		dsync_serializer_encode_add(encoder, "lock_timeout",
+			t_strdup_printf("%u", set->lock_timeout));
+	}
+	if (set->import_commit_msgs_interval > 0) {
+		dsync_serializer_encode_add(encoder, "import_commit_msgs_interval",
+			t_strdup_printf("%u", set->import_commit_msgs_interval));
+	}
+	if (set->sync_since_timestamp > 0) {
+		dsync_serializer_encode_add(encoder, "sync_since_timestamp",
+			t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+	}
+	if (set->sync_until_timestamp > 0) {
+		dsync_serializer_encode_add(encoder, "sync_until_timestamp",
+			t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+	}
+	if (set->sync_max_size > 0) {
+		dsync_serializer_encode_add(encoder, "sync_max_size",
+			t_strdup_printf("%"PRIu64, set->sync_max_size));
+	}
+	if (set->sync_flags != NULL) {
+		dsync_serializer_encode_add(encoder, "sync_flags",
+					    set->sync_flags);
+	}
+	if (set->alt_char != '\0') {
+		dsync_serializer_encode_add(encoder, "alt_char",
+			t_strdup_printf("%c", set->alt_char));
+	}
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0)
+		dsync_serializer_encode_add(encoder, "send_mail_requests", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
+		dsync_serializer_encode_add(encoder, "backup_send", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0)
+		dsync_serializer_encode_add(encoder, "backup_recv", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_DEBUG) != 0)
+		dsync_serializer_encode_add(encoder, "debug", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0)
+		dsync_serializer_encode_add(encoder, "sync_visible_namespaces", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0)
+		dsync_serializer_encode_add(encoder, "no_mail_sync", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0)
+		dsync_serializer_encode_add(encoder, "no_backup_overwrite", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0)
+		dsync_serializer_encode_add(encoder, "purge_remote", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0)
+		dsync_serializer_encode_add(encoder, "no_notify", "");
+	if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0)
+		dsync_serializer_encode_add(encoder, "empty_hdr_workaround", "");
+	/* this can be NULL in slave */
+	string_t *str2 = t_str_new(32);
+	if (set->hashed_headers != NULL) {
+		for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) {
+			str_append_tabescaped(str2, *ptr);
+			str_append_c(str2, '\t');
+		}
+	}
+	dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2));
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
+				const struct dsync_ibc_settings **set_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_ibc_settings *set;
+	const char *value;
+	pool_t pool = ibc->ret_pool;
+	enum dsync_ibc_recv_ret ret;
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_HANDSHAKE, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK) {
+		if (ret != DSYNC_IBC_RECV_RET_TRYAGAIN) {
+			i_error("dsync(%s): Unexpected input in handshake",
+				ibc->name);
+			dsync_ibc_stream_stop(ibc);
+		}
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	p_clear(pool);
+	set = p_new(pool, struct dsync_ibc_settings, 1);
+
+	value = dsync_deserializer_decode_get(decoder, "hostname");
+	set->hostname = p_strdup(pool, value);
+	/* now that we know the remote's hostname, use it for the
+	   stream's name */
+	i_free(ibc->name);
+	ibc->name = i_strdup(set->hostname);
+
+	if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value))
+		set->sync_ns_prefixes = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "sync_box", &value))
+		set->sync_box = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "virtual_all_box", &value))
+		set->virtual_all_box = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "sync_box_guid", &value) &&
+	    guid_128_from_string(value, set->sync_box_guid) < 0) {
+		dsync_ibc_input_error(ibc, decoder,
+				      "Invalid sync_box_guid: %s", value);
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "exclude_mailboxes", &value) &&
+	    *value != '\0') {
+		char **boxes = p_strsplit_tabescaped(pool, value);
+		set->exclude_mailboxes = (const void *)boxes;
+	}
+	if (dsync_deserializer_decode_try(decoder, "sync_type", &value)) {
+		switch (value[0]) {
+		case 'f':
+			set->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
+			break;
+		case 'c':
+			set->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED;
+			break;
+		case 's':
+			set->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
+			break;
+		default:
+			dsync_ibc_input_error(ibc, decoder,
+				"Unknown sync_type: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "lock_timeout", &value)) {
+		if (str_to_uint(value, &set->lock_timeout) < 0 ||
+		    set->lock_timeout == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid lock_timeout: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "import_commit_msgs_interval", &value)) {
+		if (str_to_uint(value, &set->import_commit_msgs_interval) < 0 ||
+		    set->import_commit_msgs_interval == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid import_commit_msgs_interval: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) {
+		if (str_to_time(value, &set->sync_since_timestamp) < 0 ||
+		    set->sync_since_timestamp == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid sync_since_timestamp: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "sync_until_timestamp", &value)) {
+		if (str_to_time(value, &set->sync_until_timestamp) < 0 ||
+		    set->sync_until_timestamp == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid sync_until_timestamp: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) {
+		if (str_to_uoff(value, &set->sync_max_size) < 0 ||
+		    set->sync_max_size == 0) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid sync_max_size: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "sync_flags", &value))
+		set->sync_flags = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "alt_char", &value))
+		set->alt_char = value[0];
+	if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
+	if (dsync_deserializer_decode_try(decoder, "backup_send", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
+	if (dsync_deserializer_decode_try(decoder, "backup_recv", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV;
+	if (dsync_deserializer_decode_try(decoder, "debug", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_DEBUG;
+	if (dsync_deserializer_decode_try(decoder, "sync_visible_namespaces", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES;
+	if (dsync_deserializer_decode_try(decoder, "no_mail_sync", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC;
+	if (dsync_deserializer_decode_try(decoder, "no_backup_overwrite", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE;
+	if (dsync_deserializer_decode_try(decoder, "purge_remote", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE;
+	if (dsync_deserializer_decode_try(decoder, "no_notify", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY;
+	if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value))
+		set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
+	if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value))
+		set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value);
+	set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2;
+	set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3;
+
+	*set_r = set;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_end_of_list(struct dsync_ibc *_ibc,
+				  enum dsync_ibc_eol_type type)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+
+	i_assert(ibc->value_output == NULL);
+
+	switch (type) {
+	case DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE:
+		if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
+			return;
+		break;
+	default:
+		break;
+	}
+
+	ibc->last_sent_item_eol = TRUE;
+	o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n");
+}
+
+static void
+dsync_ibc_stream_send_mailbox_state(struct dsync_ibc *_ibc,
+				    const struct dsync_mailbox_state *state)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+
+	str_append_c(str, items[ITEM_MAILBOX_STATE].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_STATE);
+	dsync_serializer_encode_add(encoder, "mailbox_guid",
+				    guid_128_to_string(state->mailbox_guid));
+	dsync_serializer_encode_add(encoder, "last_uidvalidity",
+				    dec2str(state->last_uidvalidity));
+	dsync_serializer_encode_add(encoder, "last_common_uid",
+				    dec2str(state->last_common_uid));
+	dsync_serializer_encode_add(encoder, "last_common_modseq",
+				    dec2str(state->last_common_modseq));
+	dsync_serializer_encode_add(encoder, "last_common_pvt_modseq",
+				    dec2str(state->last_common_pvt_modseq));
+	dsync_serializer_encode_add(encoder, "last_messages_count",
+				    dec2str(state->last_messages_count));
+	if (state->changes_during_sync)
+		dsync_serializer_encode_add(encoder, "changes_during_sync", "");
+
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_state(struct dsync_ibc *_ibc,
+				    struct dsync_mailbox_state *state_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_deserializer_decoder *decoder;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	i_zero(state_r);
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_STATE, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	value = dsync_deserializer_decode_get(decoder, "mailbox_guid");
+	if (guid_128_from_string(value, state_r->mailbox_guid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "last_uidvalidity");
+	if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_uidvalidity");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "last_common_uid");
+	if (str_to_uint32(value, &state_r->last_common_uid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_common_uid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "last_common_modseq");
+	if (str_to_uint64(value, &state_r->last_common_modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_common_modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "last_common_pvt_modseq");
+	if (str_to_uint64(value, &state_r->last_common_pvt_modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "last_messages_count", &value) &&
+	    str_to_uint32(value, &state_r->last_messages_count) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "changes_during_sync", &value))
+		state_r->changes_during_sync = TRUE;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mailbox_tree_node(struct dsync_ibc *_ibc,
+					const char *const *name,
+					const struct dsync_mailbox_node *node)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str, *namestr;
+
+	i_assert(*name != NULL);
+
+	str = t_str_new(128);
+	str_append_c(str, items[ITEM_MAILBOX_TREE_NODE].chr);
+
+	/* convert all hierarchy separators to tabs. mailbox names really
+	   aren't supposed to have any tabs, but escape them anyway if there
+	   are. */
+	namestr = t_str_new(128);
+	for (; *name != NULL; name++) {
+		str_append_tabescaped(namestr, *name);
+		str_append_c(namestr, '\t');
+	}
+	str_truncate(namestr, str_len(namestr)-1);
+
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_TREE_NODE);
+	dsync_serializer_encode_add(encoder, "name", str_c(namestr));
+	switch (node->existence) {
+	case DSYNC_MAILBOX_NODE_NONEXISTENT:
+		dsync_serializer_encode_add(encoder, "existence", "n");
+		break;
+	case DSYNC_MAILBOX_NODE_EXISTS:
+		dsync_serializer_encode_add(encoder, "existence", "y");
+		break;
+	case DSYNC_MAILBOX_NODE_DELETED:
+		dsync_serializer_encode_add(encoder, "existence", "d");
+		break;
+	}
+
+	if (!guid_128_is_empty(node->mailbox_guid)) {
+		dsync_serializer_encode_add(encoder, "mailbox_guid",
+			guid_128_to_string(node->mailbox_guid));
+	}
+	if (node->uid_validity != 0) {
+		dsync_serializer_encode_add(encoder, "uid_validity",
+					    dec2str(node->uid_validity));
+	}
+	if (node->uid_next != 0) {
+		dsync_serializer_encode_add(encoder, "uid_next",
+					    dec2str(node->uid_next));
+	}
+	if (node->last_renamed_or_created != 0) {
+		dsync_serializer_encode_add(encoder, "last_renamed_or_created",
+					    dec2str(node->last_renamed_or_created));
+	}
+	if (node->last_subscription_change != 0) {
+		dsync_serializer_encode_add(encoder, "last_subscription_change",
+			dec2str(node->last_subscription_change));
+	}
+	if (node->subscribed)
+		dsync_serializer_encode_add(encoder, "subscribed", "");
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_tree_node(struct dsync_ibc *_ibc,
+					const char *const **name_r,
+					const struct dsync_mailbox_node **node_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_mailbox_node *node;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_TREE_NODE, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	p_clear(ibc->ret_pool);
+	node = p_new(ibc->ret_pool, struct dsync_mailbox_node, 1);
+
+	value = dsync_deserializer_decode_get(decoder, "name");
+	if (*value == '\0') {
+		dsync_ibc_input_error(ibc, decoder, "Empty name");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	*name_r = (void *)p_strsplit_tabescaped(ibc->ret_pool, value);
+
+	value = dsync_deserializer_decode_get(decoder, "existence");
+	switch (*value) {
+	case 'n':
+		node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+		break;
+	case 'y':
+		node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+		break;
+	case 'd':
+		node->existence = DSYNC_MAILBOX_NODE_DELETED;
+		break;
+	}
+
+	if (dsync_deserializer_decode_try(decoder, "mailbox_guid", &value) &&
+	    guid_128_from_string(value, node->mailbox_guid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "uid_validity", &value) &&
+	    str_to_uint32(value, &node->uid_validity) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "uid_next", &value) &&
+	    str_to_uint32(value, &node->uid_next) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid_next");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "last_renamed_or_created", &value) &&
+	    str_to_time(value, &node->last_renamed_or_created) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_renamed_or_created");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "last_subscription_change", &value) &&
+	    str_to_time(value, &node->last_subscription_change) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_subscription_change");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "subscribed", &value))
+		node->subscribed = TRUE;
+
+	*node_r = node;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_encode_delete(string_t *str,
+			       struct dsync_serializer_encoder *encoder,
+			       const struct dsync_mailbox_delete *deletes,
+			       unsigned int count, const char *key,
+			       enum dsync_mailbox_delete_type type)
+{
+	unsigned int i;
+
+	str_truncate(str, 0);
+	for (i = 0; i < count; i++) {
+		if (deletes[i].type == type) {
+			str_append(str, guid_128_to_string(deletes[i].guid));
+			str_printfa(str, " %ld ", (long)deletes[i].timestamp);
+		}
+	}
+	if (str_len(str) > 0) {
+		str_truncate(str, str_len(str)-1);
+		dsync_serializer_encode_add(encoder, key, str_c(str));
+	}
+}
+
+static void
+dsync_ibc_stream_send_mailbox_deletes(struct dsync_ibc *_ibc,
+				      const struct dsync_mailbox_delete *deletes,
+				      unsigned int count, char hierarchy_sep,
+				      char escape_char)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str, *substr;
+	char sep[2];
+
+	str = t_str_new(128);
+	str_append_c(str, items[ITEM_MAILBOX_DELETE].chr);
+
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_DELETE);
+	sep[0] = hierarchy_sep; sep[1] = '\0';
+	dsync_serializer_encode_add(encoder, "hierarchy_sep", sep);
+	sep[0] = escape_char; sep[1] = '\0';
+	dsync_serializer_encode_add(encoder, "escape_char", sep);
+
+	substr = t_str_new(128);
+	dsync_ibc_stream_encode_delete(substr, encoder, deletes, count,
+				       "mailboxes",
+				       DSYNC_MAILBOX_DELETE_TYPE_MAILBOX);
+	dsync_ibc_stream_encode_delete(substr, encoder, deletes, count,
+				       "dirs",
+				       DSYNC_MAILBOX_DELETE_TYPE_DIR);
+	dsync_ibc_stream_encode_delete(substr, encoder, deletes, count,
+				       "unsubscribes",
+				       DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE);
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+ARRAY_DEFINE_TYPE(dsync_mailbox_delete, struct dsync_mailbox_delete);
+static int
+decode_mailbox_deletes(ARRAY_TYPE(dsync_mailbox_delete) *deletes,
+		       const char *value, enum dsync_mailbox_delete_type type)
+{
+	struct dsync_mailbox_delete *del;
+	const char *const *tmp;
+	unsigned int i;
+
+	tmp = t_strsplit(value, " ");
+	for (i = 0; tmp[i] != NULL; i += 2) {
+		del = array_append_space(deletes);
+		del->type = type;
+		if (guid_128_from_string(tmp[i], del->guid) < 0)
+			return -1;
+		if (tmp[i+1] == NULL ||
+		    str_to_time(tmp[i+1], &del->timestamp) < 0)
+			return -1;
+	}
+	return 0;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_deletes(struct dsync_ibc *_ibc,
+				      const struct dsync_mailbox_delete **deletes_r,
+				      unsigned int *count_r, char *hierarchy_sep_r,
+				      char *escape_char_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_deserializer_decoder *decoder;
+	ARRAY_TYPE(dsync_mailbox_delete) deletes;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_DELETE, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	p_clear(ibc->ret_pool);
+	p_array_init(&deletes, ibc->ret_pool, 16);
+
+	value = dsync_deserializer_decode_get(decoder, "hierarchy_sep");
+	if (strlen(value) != 1) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid hierarchy_sep");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	*hierarchy_sep_r = value[0];
+
+	if (!dsync_deserializer_decode_try(decoder, "escape_char", &value))
+		*escape_char_r = '\0';
+	else {
+		if (strlen(value) > 1) {
+			dsync_ibc_input_error(ibc, decoder, "Invalid escape_char '%s'", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		*escape_char_r = value[0];
+	}
+
+	if (dsync_deserializer_decode_try(decoder, "mailboxes", &value) &&
+	    decode_mailbox_deletes(&deletes, value,
+				   DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid mailboxes");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "dirs", &value) &&
+	    decode_mailbox_deletes(&deletes, value,
+				   DSYNC_MAILBOX_DELETE_TYPE_DIR) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid dirs");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "unsubscribes", &value) &&
+	    decode_mailbox_deletes(&deletes, value,
+				   DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid dirs");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	*deletes_r = array_get(&deletes, count_r);
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static const char *
+get_cache_fields(struct dsync_ibc_stream *ibc,
+		 const struct dsync_mailbox *dsync_box)
+{
+	struct dsync_serializer_encoder *encoder;
+	string_t *str;
+	const struct mailbox_cache_field *cache_fields;
+	unsigned int i, count;
+	char decision[3];
+
+	cache_fields = array_get(&dsync_box->cache_fields, &count);
+	if (count == 0)
+		return "";
+
+	str = t_str_new(128);
+	for (i = 0; i < count; i++) {
+		const struct mailbox_cache_field *field = &cache_fields[i];
+
+		encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_CACHE_FIELD]);
+		dsync_serializer_encode_add(encoder, "name", field->name);
+
+		memset(decision, 0, sizeof(decision));
+		switch (field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) {
+		case MAIL_CACHE_DECISION_NO:
+			decision[0] = 'n';
+			break;
+		case MAIL_CACHE_DECISION_TEMP:
+			decision[0] = 't';
+			break;
+		case MAIL_CACHE_DECISION_YES:
+			decision[0] = 'y';
+			break;
+		}
+		i_assert(decision[0] != '\0');
+		if ((field->decision & MAIL_CACHE_DECISION_FORCED) != 0)
+			decision[1] = 'F';
+		dsync_serializer_encode_add(encoder, "decision", decision);
+		if (field->last_used != 0) {
+			dsync_serializer_encode_add(encoder, "last_used",
+						    dec2str(field->last_used));
+		}
+		dsync_serializer_encode_finish(&encoder, str);
+	}
+	if (i > 0) {
+		/* remove the trailing LF */
+		str_truncate(str, str_len(str)-1);
+	}
+	return str_c(str);
+}
+
+static void
+dsync_ibc_stream_send_mailbox(struct dsync_ibc *_ibc,
+			      const struct dsync_mailbox *dsync_box)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+	const char *value;
+
+	str_append_c(str, items[ITEM_MAILBOX].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX);
+	dsync_serializer_encode_add(encoder, "mailbox_guid",
+				    guid_128_to_string(dsync_box->mailbox_guid));
+
+	if (dsync_box->mailbox_lost)
+		dsync_serializer_encode_add(encoder, "mailbox_lost", "");
+	if (dsync_box->mailbox_ignore)
+		dsync_serializer_encode_add(encoder, "mailbox_ignore", "");
+	if (dsync_box->have_guids)
+		dsync_serializer_encode_add(encoder, "have_guids", "");
+	if (dsync_box->have_save_guids)
+		dsync_serializer_encode_add(encoder, "have_save_guids", "");
+	if (dsync_box->have_only_guid128)
+		dsync_serializer_encode_add(encoder, "have_only_guid128", "");
+	dsync_serializer_encode_add(encoder, "uid_validity",
+				    dec2str(dsync_box->uid_validity));
+	dsync_serializer_encode_add(encoder, "uid_next",
+				    dec2str(dsync_box->uid_next));
+	dsync_serializer_encode_add(encoder, "messages_count",
+				    dec2str(dsync_box->messages_count));
+	dsync_serializer_encode_add(encoder, "first_recent_uid",
+				    dec2str(dsync_box->first_recent_uid));
+	dsync_serializer_encode_add(encoder, "highest_modseq",
+				    dec2str(dsync_box->highest_modseq));
+	dsync_serializer_encode_add(encoder, "highest_pvt_modseq",
+				    dec2str(dsync_box->highest_pvt_modseq));
+
+	value = get_cache_fields(ibc, dsync_box);
+	if (value != NULL)
+		dsync_serializer_encode_add(encoder, "cache_fields", value);
+
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static int
+parse_cache_field(struct dsync_ibc_stream *ibc, struct dsync_mailbox *box,
+		  const char *value)
+{
+	struct dsync_deserializer_decoder *decoder;
+	struct mailbox_cache_field field;
+	const char *error;
+	int ret = 0;
+
+	if (dsync_deserializer_decode_begin(ibc->deserializers[ITEM_MAILBOX_CACHE_FIELD],
+					    value, &decoder, &error) < 0) {
+		dsync_ibc_input_error(ibc, NULL,
+			"cache_field: Invalid input: %s", error);
+		return -1;
+	}
+
+	i_zero(&field);
+	value = dsync_deserializer_decode_get(decoder, "name");
+	field.name = p_strdup(ibc->ret_pool, value);
+
+	value = dsync_deserializer_decode_get(decoder, "decision");
+	switch (*value) {
+	case 'n':
+		field.decision = MAIL_CACHE_DECISION_NO;
+		break;
+	case 't':
+		field.decision = MAIL_CACHE_DECISION_TEMP;
+		break;
+	case 'y':
+		field.decision = MAIL_CACHE_DECISION_YES;
+		break;
+	default:
+		dsync_ibc_input_error(ibc, decoder, "Invalid decision: %s",
+				      value);
+		ret = -1;
+		break;
+	}
+	if (value[1] == 'F')
+		field.decision |= MAIL_CACHE_DECISION_FORCED;
+
+	if (dsync_deserializer_decode_try(decoder, "last_used", &value) &&
+	    str_to_time(value, &field.last_used) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_used");
+		ret = -1;
+	}
+	array_push_back(&box->cache_fields, &field);
+
+	dsync_deserializer_decode_finish(&decoder);
+	return ret;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox(struct dsync_ibc *_ibc,
+			      const struct dsync_mailbox **dsync_box_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	pool_t pool = ibc->ret_pool;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_mailbox *box;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	p_clear(pool);
+	box = p_new(pool, struct dsync_mailbox, 1);
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	value = dsync_deserializer_decode_get(decoder, "mailbox_guid");
+	if (guid_128_from_string(value, box->mailbox_guid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	if (dsync_deserializer_decode_try(decoder, "mailbox_lost", &value))
+		box->mailbox_lost = TRUE;
+	if (dsync_deserializer_decode_try(decoder, "mailbox_ignore", &value))
+		box->mailbox_ignore = TRUE;
+	if (dsync_deserializer_decode_try(decoder, "have_guids", &value))
+		box->have_guids = TRUE;
+	if (dsync_deserializer_decode_try(decoder, "have_save_guids", &value) ||
+	    (box->have_guids && ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID))
+		box->have_save_guids = TRUE;
+	if (dsync_deserializer_decode_try(decoder, "have_only_guid128", &value))
+		box->have_only_guid128 = TRUE;
+	value = dsync_deserializer_decode_get(decoder, "uid_validity");
+	if (str_to_uint32(value, &box->uid_validity) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "uid_next");
+	if (str_to_uint32(value, &box->uid_next) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid_next");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "messages_count");
+	if (str_to_uint32(value, &box->messages_count) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid messages_count");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "first_recent_uid");
+	if (str_to_uint32(value, &box->first_recent_uid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid first_recent_uid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "highest_modseq");
+	if (str_to_uint64(value, &box->highest_modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid highest_modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	value = dsync_deserializer_decode_get(decoder, "highest_pvt_modseq");
+	if (str_to_uint64(value, &box->highest_pvt_modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid highest_pvt_modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	p_array_init(&box->cache_fields, pool, 32);
+	if (dsync_deserializer_decode_try(decoder, "cache_fields", &value)) {
+		const char *const *fields = t_strsplit(value, "\n");
+		for (; *fields != NULL; fields++) {
+			if (parse_cache_field(ibc, box, *fields) < 0)
+				return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+	}
+
+	*dsync_box_r = box;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mailbox_attribute(struct dsync_ibc *_ibc,
+					const struct dsync_mailbox_attribute *attr)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+	char type[2];
+
+	if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
+		return;
+
+	str_append_c(str, items[ITEM_MAILBOX_ATTRIBUTE].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_ATTRIBUTE);
+
+	type[0] = type[1] = '\0';
+	switch (attr->type) {
+	case MAIL_ATTRIBUTE_TYPE_PRIVATE:
+		type[0] = 'p';
+		break;
+	case MAIL_ATTRIBUTE_TYPE_SHARED:
+		type[0] = 's';
+		break;
+	}
+	i_assert(type[0] != '\0');
+	dsync_serializer_encode_add(encoder, "type", type);
+	dsync_serializer_encode_add(encoder, "key", attr->key);
+	if (attr->value != NULL)
+		dsync_serializer_encode_add(encoder, "value", attr->value);
+	else if (attr->value_stream != NULL)
+		dsync_serializer_encode_add(encoder, "stream", "");
+
+	if (attr->deleted)
+		dsync_serializer_encode_add(encoder, "deleted", "");
+	if (attr->last_change != 0) {
+		dsync_serializer_encode_add(encoder, "last_change",
+					    dec2str(attr->last_change));
+	}
+	if (attr->modseq != 0) {
+		dsync_serializer_encode_add(encoder, "modseq",
+					    dec2str(attr->modseq));
+	}
+
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+
+	if (attr->value_stream != NULL) {
+		ibc->value_output_last = '\0';
+		ibc->value_output = attr->value_stream;
+		i_stream_ref(ibc->value_output);
+		(void)dsync_ibc_stream_send_value_stream(ibc);
+	}
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_attribute(struct dsync_ibc *_ibc,
+					const struct dsync_mailbox_attribute **attr_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	pool_t pool = ibc->ret_pool;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_mailbox_attribute *attr;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
+		return DSYNC_IBC_RECV_RET_FINISHED;
+
+	if (ibc->value_input != NULL) {
+		/* wait until the mail's stream has been read */
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	if (ibc->cur_attr != NULL) {
+		/* finished reading the stream, return the mail now */
+		*attr_r = ibc->cur_attr;
+		ibc->cur_attr = NULL;
+		return DSYNC_IBC_RECV_RET_OK;
+	}
+
+	p_clear(pool);
+	attr = p_new(pool, struct dsync_mailbox_attribute, 1);
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_ATTRIBUTE, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	value = dsync_deserializer_decode_get(decoder, "type");
+	switch (*value) {
+	case 'p':
+		attr->type = MAIL_ATTRIBUTE_TYPE_PRIVATE;
+		break;
+	case 's':
+		attr->type = MAIL_ATTRIBUTE_TYPE_SHARED;
+		break;
+	default:
+		dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value);
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	value = dsync_deserializer_decode_get(decoder, "key");
+	attr->key = p_strdup(pool, value);
+
+	if (dsync_deserializer_decode_try(decoder, "deleted", &value))
+		attr->deleted = TRUE;
+	if (dsync_deserializer_decode_try(decoder, "last_change", &value) &&
+	    str_to_time(value, &attr->last_change) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid last_change");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "modseq", &value) &&
+	    str_to_uint64(value, &attr->modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	/* NOTE: stream reading must be the last here, because reading a large
+	   stream will be finished later by return TRYAGAIN. We need to
+	   deserialize all the other fields before that or they'll get lost. */
+	if (dsync_deserializer_decode_try(decoder, "stream", &value)) {
+		attr->value_stream = dsync_ibc_stream_input_stream(ibc);
+		if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) {
+			ibc->cur_attr = attr;
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		/* already finished reading the stream */
+		i_assert(ibc->value_input == NULL);
+	} else if (dsync_deserializer_decode_try(decoder, "value", &value))
+		attr->value = p_strdup(pool, value);
+
+	*attr_r = attr;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_change(struct dsync_ibc *_ibc,
+			     const struct dsync_mail_change *change)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+	char type[2];
+
+	str_append_c(str, items[ITEM_MAIL_CHANGE].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_CHANGE);
+
+	type[0] = type[1] = '\0';
+	switch (change->type) {
+	case DSYNC_MAIL_CHANGE_TYPE_SAVE:
+		type[0] = 's';
+		break;
+	case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:
+		type[0] = 'e';
+		break;
+	case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:
+		type[0] = 'f';
+		break;
+	}
+	i_assert(type[0] != '\0');
+	dsync_serializer_encode_add(encoder, "type", type);
+	dsync_serializer_encode_add(encoder, "uid", dec2str(change->uid));
+	if (change->guid != NULL)
+		dsync_serializer_encode_add(encoder, "guid", change->guid);
+	if (change->hdr_hash != NULL) {
+		dsync_serializer_encode_add(encoder, "hdr_hash",
+					    change->hdr_hash);
+	}
+	if (change->modseq != 0) {
+		dsync_serializer_encode_add(encoder, "modseq",
+					    dec2str(change->modseq));
+	}
+	if (change->pvt_modseq != 0) {
+		dsync_serializer_encode_add(encoder, "pvt_modseq",
+					    dec2str(change->pvt_modseq));
+	}
+	if (change->add_flags != 0) {
+		dsync_serializer_encode_add(encoder, "add_flags",
+			t_strdup_printf("%x", change->add_flags));
+	}
+	if (change->remove_flags != 0) {
+		dsync_serializer_encode_add(encoder, "remove_flags",
+			t_strdup_printf("%x", change->remove_flags));
+	}
+	if (change->final_flags != 0) {
+		dsync_serializer_encode_add(encoder, "final_flags",
+			t_strdup_printf("%x", change->final_flags));
+	}
+	if (change->keywords_reset)
+		dsync_serializer_encode_add(encoder, "keywords_reset", "");
+
+	if (array_is_created(&change->keyword_changes) &&
+	    array_count(&change->keyword_changes) > 0) {
+		string_t *kw_str = t_str_new(128);
+		const char *const *changes;
+		unsigned int i, count;
+
+		changes = array_get(&change->keyword_changes, &count);
+		str_append_tabescaped(kw_str, changes[0]);
+		for (i = 1; i < count; i++) {
+			str_append_c(kw_str, '\t');
+			str_append_tabescaped(kw_str, changes[i]);
+		}
+		dsync_serializer_encode_add(encoder, "keyword_changes",
+					    str_c(kw_str));
+	}
+	if (change->received_timestamp > 0) {
+		dsync_serializer_encode_add(encoder, "received_timestamp",
+			t_strdup_printf("%"PRIxTIME_T, change->received_timestamp));
+	}
+	if (change->virtual_size > 0) {
+		dsync_serializer_encode_add(encoder, "virtual_size",
+			t_strdup_printf("%llx", (unsigned long long)change->virtual_size));
+	}
+
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc,
+			     const struct dsync_mail_change **change_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	pool_t pool = ibc->ret_pool;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_mail_change *change;
+	const char *value;
+	unsigned int uintval;
+	unsigned long long ullongval;
+	enum dsync_ibc_recv_ret ret;
+
+	p_clear(pool);
+	change = p_new(pool, struct dsync_mail_change, 1);
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_CHANGE, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	value = dsync_deserializer_decode_get(decoder, "type");
+	switch (*value) {
+	case 's':
+		change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE;
+		break;
+	case 'e':
+		change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
+		break;
+	case 'f':
+		change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
+		break;
+	default:
+		dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value);
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	value = dsync_deserializer_decode_get(decoder, "uid");
+	if (str_to_uint32(value, &change->uid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	if (dsync_deserializer_decode_try(decoder, "guid", &value))
+		change->guid = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "hdr_hash", &value))
+		change->hdr_hash = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "modseq", &value) &&
+	    str_to_uint64(value, &change->modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "pvt_modseq", &value) &&
+	    str_to_uint64(value, &change->pvt_modseq) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid pvt_modseq");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) {
+		if (str_to_uint_hex(value, &uintval) < 0 ||
+		    uintval > (uint8_t)-1) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid add_flags: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		change->add_flags = uintval;
+	}
+	if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) {
+		if (str_to_uint_hex(value, &uintval) < 0 ||
+		    uintval > (uint8_t)-1) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid remove_flags: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		change->remove_flags = uintval;
+	}
+	if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) {
+		if (str_to_uint_hex(value, &uintval) < 0 ||
+		    uintval > (uint8_t)-1) {
+			dsync_ibc_input_error(ibc, decoder,
+				"Invalid final_flags: %s", value);
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		change->final_flags = uintval;
+	}
+	if (dsync_deserializer_decode_try(decoder, "keywords_reset", &value))
+		change->keywords_reset = TRUE;
+
+	if (dsync_deserializer_decode_try(decoder, "keyword_changes", &value) &&
+	    *value != '\0') {
+		const char *const *changes = t_strsplit_tabescaped(value);
+		unsigned int i, count = str_array_length(changes);
+
+		p_array_init(&change->keyword_changes, pool, count);
+		for (i = 0; i < count; i++) {
+			value = p_strdup(pool, changes[i]);
+			array_push_back(&change->keyword_changes, &value);
+		}
+	}
+	if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value)) {
+		if (str_to_ullong_hex(value, &ullongval) < 0) {
+			dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		change->received_timestamp = ullongval;
+	}
+	if (dsync_deserializer_decode_try(decoder, "virtual_size", &value)) {
+		if (str_to_ullong_hex(value, &ullongval) < 0) {
+			dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size");
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		change->virtual_size = ullongval;
+	}
+
+	*change_r = change;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mail_request(struct dsync_ibc *_ibc,
+				   const struct dsync_mail_request *request)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+
+	str_append_c(str, items[ITEM_MAIL_REQUEST].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_REQUEST);
+	if (request->guid != NULL)
+		dsync_serializer_encode_add(encoder, "guid", request->guid);
+	if (request->uid != 0) {
+		dsync_serializer_encode_add(encoder, "uid",
+					    dec2str(request->uid));
+	}
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mail_request(struct dsync_ibc *_ibc,
+				   const struct dsync_mail_request **request_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_mail_request *request;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	p_clear(ibc->ret_pool);
+	request = p_new(ibc->ret_pool, struct dsync_mail_request, 1);
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_REQUEST, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	if (dsync_deserializer_decode_try(decoder, "guid", &value))
+		request->guid = p_strdup(ibc->ret_pool, value);
+	if (dsync_deserializer_decode_try(decoder, "uid", &value) &&
+	    str_to_uint32(value, &request->uid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+
+	*request_r = request;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mail(struct dsync_ibc *_ibc,
+			   const struct dsync_mail *mail)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+
+	i_assert(!mail->minimal_fields);
+	i_assert(ibc->value_output == NULL);
+
+	str_append_c(str, items[ITEM_MAIL].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL);
+	if (mail->guid != NULL)
+		dsync_serializer_encode_add(encoder, "guid", mail->guid);
+	if (mail->uid != 0)
+		dsync_serializer_encode_add(encoder, "uid", dec2str(mail->uid));
+	if (mail->pop3_uidl != NULL) {
+		dsync_serializer_encode_add(encoder, "pop3_uidl",
+					    mail->pop3_uidl);
+	}
+	if (mail->pop3_order > 0) {
+		dsync_serializer_encode_add(encoder, "pop3_order",
+					    dec2str(mail->pop3_order));
+	}
+	if (mail->received_date > 0) {
+		dsync_serializer_encode_add(encoder, "received_date",
+					    dec2str(mail->received_date));
+	}
+	if (mail->saved_date != 0) {
+		dsync_serializer_encode_add(encoder, "saved_date",
+					    dec2str(mail->saved_date));
+	}
+	if (mail->input != NULL)
+		dsync_serializer_encode_add(encoder, "stream", "");
+
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+
+	if (mail->input != NULL) {
+		ibc->value_output_last = '\0';
+		ibc->value_output = mail->input;
+		i_stream_ref(ibc->value_output);
+		(void)dsync_ibc_stream_send_value_stream(ibc);
+	}
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	pool_t pool = ibc->ret_pool;
+	struct dsync_deserializer_decoder *decoder;
+	struct dsync_mail *mail;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+
+	if (ibc->value_input != NULL) {
+		/* wait until the mail's stream has been read */
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (ibc->cur_mail != NULL) {
+		/* finished reading the stream, return the mail now */
+		*mail_r = ibc->cur_mail;
+		ibc->cur_mail = NULL;
+		return DSYNC_IBC_RECV_RET_OK;
+	}
+
+	p_clear(pool);
+	mail = p_new(pool, struct dsync_mail, 1);
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	if (dsync_deserializer_decode_try(decoder, "guid", &value))
+		mail->guid = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "uid", &value) &&
+	    str_to_uint32(value, &mail->uid) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid uid");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "pop3_uidl", &value))
+		mail->pop3_uidl = p_strdup(pool, value);
+	if (dsync_deserializer_decode_try(decoder, "pop3_order", &value) &&
+	    str_to_uint32(value, &mail->pop3_order) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid pop3_order");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "received_date", &value) &&
+	    str_to_time(value, &mail->received_date) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid received_date");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "saved_date", &value) &&
+	    str_to_time(value, &mail->saved_date) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid saved_date");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "stream", &value)) {
+		mail->input = dsync_ibc_stream_input_stream(ibc);
+		if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) {
+			ibc->cur_mail = mail;
+			return DSYNC_IBC_RECV_RET_TRYAGAIN;
+		}
+		/* already finished reading the stream */
+		i_assert(ibc->value_input == NULL);
+	}
+
+	*mail_r = mail;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error,
+			     enum mail_error mail_error,
+			     bool require_full_resync)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_serializer_encoder *encoder;
+	string_t *str = t_str_new(128);
+
+	str_append_c(str, items[ITEM_FINISH].chr);
+	encoder = dsync_ibc_send_encode_begin(ibc, ITEM_FINISH);
+	if (error != NULL)
+		dsync_serializer_encode_add(encoder, "error", error);
+	if (mail_error != 0) {
+		dsync_serializer_encode_add(encoder, "mail_error",
+					    dec2str(mail_error));
+	}
+	if (require_full_resync)
+		dsync_serializer_encode_add(encoder, "require_full_resync", "");
+	dsync_serializer_encode_finish(&encoder, str);
+	dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r,
+			     enum mail_error *mail_error_r,
+			     bool *require_full_resync_r)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	struct dsync_deserializer_decoder *decoder;
+	const char *value;
+	enum dsync_ibc_recv_ret ret;
+	int i = 0;
+
+	*error_r = NULL;
+	*mail_error_r = 0;
+	*require_full_resync_r = FALSE;
+
+	p_clear(ibc->ret_pool);
+
+	if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_FINISH)
+		return DSYNC_IBC_RECV_RET_OK;
+
+	ret = dsync_ibc_stream_input_next(ibc, ITEM_FINISH, &decoder);
+	if (ret != DSYNC_IBC_RECV_RET_OK)
+		return ret;
+
+	if (dsync_deserializer_decode_try(decoder, "error", &value))
+		*error_r = p_strdup(ibc->ret_pool, value);
+	if (dsync_deserializer_decode_try(decoder, "mail_error", &value) &&
+	    str_to_int(value, &i) < 0) {
+		dsync_ibc_input_error(ibc, decoder, "Invalid mail_error");
+		return DSYNC_IBC_RECV_RET_TRYAGAIN;
+	}
+	if (dsync_deserializer_decode_try(decoder, "require_full_resync", &value))
+		*require_full_resync_r = TRUE;
+	*mail_error_r = i;
+
+	ibc->finish_received = TRUE;
+	return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void dsync_ibc_stream_close_mail_streams(struct dsync_ibc *_ibc)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+
+	if (ibc->value_output != NULL) {
+		i_stream_unref(&ibc->value_output);
+		dsync_ibc_stream_stop(ibc);
+	}
+}
+
+static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+	size_t bytes;
+
+	if (ibc->value_output != NULL)
+		return TRUE;
+
+	bytes = o_stream_get_buffer_used_size(ibc->output);
+	if (bytes < DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE)
+		return FALSE;
+
+	o_stream_set_flush_pending(ibc->output, TRUE);
+	return TRUE;
+}
+
+static bool dsync_ibc_stream_has_pending_data(struct dsync_ibc *_ibc)
+{
+	struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+
+	return ibc->has_pending_data;
+}
+
+static const struct dsync_ibc_vfuncs dsync_ibc_stream_vfuncs = {
+	dsync_ibc_stream_deinit,
+	dsync_ibc_stream_send_handshake,
+	dsync_ibc_stream_recv_handshake,
+	dsync_ibc_stream_send_end_of_list,
+	dsync_ibc_stream_send_mailbox_state,
+	dsync_ibc_stream_recv_mailbox_state,
+	dsync_ibc_stream_send_mailbox_tree_node,
+	dsync_ibc_stream_recv_mailbox_tree_node,
+	dsync_ibc_stream_send_mailbox_deletes,
+	dsync_ibc_stream_recv_mailbox_deletes,
+	dsync_ibc_stream_send_mailbox,
+	dsync_ibc_stream_recv_mailbox,
+	dsync_ibc_stream_send_mailbox_attribute,
+	dsync_ibc_stream_recv_mailbox_attribute,
+	dsync_ibc_stream_send_change,
+	dsync_ibc_stream_recv_change,
+	dsync_ibc_stream_send_mail_request,
+	dsync_ibc_stream_recv_mail_request,
+	dsync_ibc_stream_send_mail,
+	dsync_ibc_stream_recv_mail,
+	dsync_ibc_stream_send_finish,
+	dsync_ibc_stream_recv_finish,
+	dsync_ibc_stream_close_mail_streams,
+	dsync_ibc_stream_is_send_queue_full,
+	dsync_ibc_stream_has_pending_data
+};
+
+struct dsync_ibc *
+dsync_ibc_init_stream(struct istream *input, struct ostream *output,
+		      const char *name, const char *temp_path_prefix,
+		      unsigned int timeout_secs)
+{
+	struct dsync_ibc_stream *ibc;
+
+	ibc = i_new(struct dsync_ibc_stream, 1);
+	ibc->ibc.v = dsync_ibc_stream_vfuncs;
+	ibc->input = input;
+	ibc->output = output;
+	i_stream_ref(ibc->input);
+	o_stream_ref(ibc->output);
+	ibc->name = i_strdup(name);
+	ibc->temp_path_prefix = i_strdup(temp_path_prefix);
+	ibc->timeout_secs = timeout_secs;
+	ibc->ret_pool = pool_alloconly_create("ibc stream data", 2048);
+	dsync_ibc_stream_init(ibc);
+	return &ibc->ibc;
+}
diff --git a/src/doveadm/dsync/dsync-ibc.c b/src/doveadm/dsync/dsync-ibc.c
new file mode 100644
index 0000000..ede7b83
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc.c
@@ -0,0 +1,239 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dsync-mail.h"
+#include "dsync-ibc-private.h"
+
+void dsync_ibc_deinit(struct dsync_ibc **_ibc)
+{
+	struct dsync_ibc *ibc = *_ibc;
+
+	*_ibc = NULL;
+	ibc->v.deinit(ibc);
+}
+
+void dsync_ibc_set_io_callback(struct dsync_ibc *ibc,
+			       io_callback_t *callback, void *context)
+{
+	ibc->io_callback = callback;
+	ibc->io_context = context;
+}
+
+void dsync_ibc_send_handshake(struct dsync_ibc *ibc,
+			      const struct dsync_ibc_settings *set)
+{
+	ibc->v.send_handshake(ibc, set);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_handshake(struct dsync_ibc *ibc,
+			 const struct dsync_ibc_settings **set_r)
+{
+	return ibc->v.recv_handshake(ibc, set_r);
+}
+
+static enum dsync_ibc_send_ret
+dsync_ibc_send_ret(struct dsync_ibc *ibc)
+{
+	return ibc->v.is_send_queue_full(ibc) ?
+		DSYNC_IBC_SEND_RET_FULL :
+		DSYNC_IBC_SEND_RET_OK;
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type)
+{
+	ibc->v.send_end_of_list(ibc, type);
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc,
+			     const struct dsync_mailbox_state *state)
+{
+	T_BEGIN {
+		ibc->v.send_mailbox_state(ibc, state);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc,
+			     struct dsync_mailbox_state *state_r)
+{
+	return ibc->v.recv_mailbox_state(ibc, state_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc,
+				 const char *const *name,
+				 const struct dsync_mailbox_node *node)
+{
+	i_assert(*name != NULL);
+
+	T_BEGIN {
+		ibc->v.send_mailbox_tree_node(ibc, name, node);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc,
+				 const char *const **name_r,
+				 const struct dsync_mailbox_node **node_r)
+{
+	return ibc->v.recv_mailbox_tree_node(ibc, name_r, node_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc,
+			       const struct dsync_mailbox_delete *deletes,
+			       unsigned int count, char hierarchy_sep,
+			       char escape_char)
+{
+	T_BEGIN {
+		ibc->v.send_mailbox_deletes(ibc, deletes, count,
+					    hierarchy_sep, escape_char);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc,
+			       const struct dsync_mailbox_delete **deletes_r,
+			       unsigned int *count_r, char *hierarchy_sep_r,
+			       char *escape_char_r)
+{
+	return ibc->v.recv_mailbox_deletes(ibc, deletes_r, count_r,
+					   hierarchy_sep_r, escape_char_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox(struct dsync_ibc *ibc,
+		       const struct dsync_mailbox *dsync_box)
+{
+	T_BEGIN {
+		ibc->v.send_mailbox(ibc, dsync_box);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox(struct dsync_ibc *ibc,
+		       const struct dsync_mailbox **dsync_box_r)
+{
+	return ibc->v.recv_mailbox(ibc, dsync_box_r);
+}
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc,
+				 const struct dsync_mailbox_attribute *attr)
+{
+	T_BEGIN {
+		ibc->v.send_mailbox_attribute(ibc, attr);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc,
+				 const struct dsync_mailbox_attribute **attr_r)
+{
+	return ibc->v.recv_mailbox_attribute(ibc, attr_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_change(struct dsync_ibc *ibc,
+		      const struct dsync_mail_change *change)
+{
+	i_assert(change->uid > 0);
+
+	T_BEGIN {
+		ibc->v.send_change(ibc, change);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_change(struct dsync_ibc *ibc,
+		      const struct dsync_mail_change **change_r)
+{
+	return ibc->v.recv_change(ibc, change_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mail_request(struct dsync_ibc *ibc,
+			    const struct dsync_mail_request *request)
+{
+	i_assert(request->guid != NULL || request->uid != 0);
+
+	T_BEGIN {
+		ibc->v.send_mail_request(ibc, request);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail_request(struct dsync_ibc *ibc,
+			    const struct dsync_mail_request **request_r)
+{
+	return ibc->v.recv_mail_request(ibc, request_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail)
+{
+	i_assert(*mail->guid != '\0' || mail->uid != 0);
+
+	T_BEGIN {
+		ibc->v.send_mail(ibc, mail);
+	} T_END;
+	return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r)
+{
+	return ibc->v.recv_mail(ibc, mail_r);
+}
+
+void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error,
+			   enum mail_error mail_error,
+			   bool require_full_resync)
+{
+	ibc->v.send_finish(ibc, error, mail_error, require_full_resync);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r,
+		      enum mail_error *mail_error_r,
+		      bool *require_full_resync_r)
+{
+	return ibc->v.recv_finish(ibc, error_r, mail_error_r,
+				  require_full_resync_r);
+}
+
+void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc)
+{
+	ibc->v.close_mail_streams(ibc);
+}
+
+bool dsync_ibc_has_failed(struct dsync_ibc *ibc)
+{
+	return ibc->failed;
+}
+
+bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc)
+{
+	return ibc->timeout;
+}
+
+bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc)
+{
+	return ibc->v.is_send_queue_full(ibc);
+}
+
+bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc)
+{
+	return ibc->v.has_pending_data(ibc);
+}
diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h
new file mode 100644
index 0000000..dc85560
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc.h
@@ -0,0 +1,178 @@
+#ifndef DSYNC_IBC_H
+#define DSYNC_IBC_H
+
+/* dsync inter-brain communicator */
+
+#include "ioloop.h"
+#include "guid.h"
+#include "mail-error.h"
+#include "dsync-brain.h"
+
+struct dsync_mailbox;
+struct dsync_mailbox_state;
+struct dsync_mailbox_node;
+struct dsync_mailbox_delete;
+struct dsync_mailbox_attribute;
+struct dsync_mail;
+struct dsync_mail_change;
+struct dsync_mail_request;
+
+enum dsync_ibc_send_ret {
+	DSYNC_IBC_SEND_RET_OK	= 1,
+	/* send queue is full, stop sending more */
+	DSYNC_IBC_SEND_RET_FULL	= 0
+};
+
+enum dsync_ibc_recv_ret {
+	DSYNC_IBC_RECV_RET_FINISHED	= -1,
+	/* try again / error (the error handling delayed until io callback) */
+	DSYNC_IBC_RECV_RET_TRYAGAIN	= 0,
+	DSYNC_IBC_RECV_RET_OK		= 1
+};
+
+enum dsync_ibc_eol_type {
+	DSYNC_IBC_EOL_MAILBOX_STATE,
+	DSYNC_IBC_EOL_MAILBOX_TREE,
+	DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE,
+	DSYNC_IBC_EOL_MAILBOX,
+	DSYNC_IBC_EOL_MAIL_CHANGES,
+	DSYNC_IBC_EOL_MAIL_REQUESTS,
+	DSYNC_IBC_EOL_MAILS
+};
+
+struct dsync_ibc_settings {
+	/* Server hostname. Used for determining which server does the
+	   locking. */
+	const char *hostname;
+	/* if non-NULL, sync only these namespaces (LF-separated) */
+	const char *sync_ns_prefixes;
+	/* if non-NULL, sync only this mailbox name */
+	const char *sync_box;
+	/* if non-NULL, use this mailbox for finding messages with GUIDs and
+	   copying them instead of saving them again. */
+	const char *virtual_all_box;
+	/* if non-empty, sync only this mailbox GUID */
+	guid_128_t sync_box_guid;
+	/* Exclude these mailboxes from the sync. They can contain '*'
+	   wildcards and be \special-use flags. */
+	const char *const *exclude_mailboxes;
+	/* Sync only mails with received timestamp at least this high. */
+	time_t sync_since_timestamp;
+	/* Sync only mails with received timestamp less or equal than this */
+	time_t sync_until_timestamp;
+	/* Don't sync mails larger than this. */
+	uoff_t sync_max_size;
+	/* Sync only mails with specified flags. */
+	const char *sync_flags;
+	/* Hashed headers */
+	const char *const *hashed_headers;
+
+	char alt_char;
+	enum dsync_brain_sync_type sync_type;
+	enum dsync_brain_flags brain_flags;
+	bool hdr_hash_v2;
+	bool hdr_hash_v3;
+	unsigned int lock_timeout;
+	unsigned int import_commit_msgs_interval;
+};
+
+void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r,
+			 struct dsync_ibc **ibc2_r);
+struct dsync_ibc *
+dsync_ibc_init_stream(struct istream *input, struct ostream *output,
+		      const char *name, const char *temp_path_prefix,
+		      unsigned int timeout_secs);
+void dsync_ibc_deinit(struct dsync_ibc **ibc);
+
+/* I/O callback is called whenever new data is available. It's also called on
+   errors, so check first the error status. */
+void dsync_ibc_set_io_callback(struct dsync_ibc *ibc,
+			       io_callback_t *callback, void *context);
+
+void dsync_ibc_send_handshake(struct dsync_ibc *ibc,
+			      const struct dsync_ibc_settings *set);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_handshake(struct dsync_ibc *ibc,
+			 const struct dsync_ibc_settings **set_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc,
+			     const struct dsync_mailbox_state *state);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc,
+			     struct dsync_mailbox_state *state_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc,
+				 const char *const *name,
+				 const struct dsync_mailbox_node *node);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc,
+				 const char *const **name_r,
+				 const struct dsync_mailbox_node **node_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc,
+			       const struct dsync_mailbox_delete *deletes,
+			       unsigned int count, char hierarchy_sep,
+			       char escape_char);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc,
+			       const struct dsync_mailbox_delete **deletes_r,
+			       unsigned int *count_r, char *hierarchy_sep_r,
+			       char *escape_char_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox(struct dsync_ibc *ibc,
+		       const struct dsync_mailbox *dsync_box);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox(struct dsync_ibc *ibc,
+		       const struct dsync_mailbox **dsync_box_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc,
+				 const struct dsync_mailbox_attribute *attr);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc,
+				 const struct dsync_mailbox_attribute **attr_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_change(struct dsync_ibc *ibc,
+		      const struct dsync_mail_change *change);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_change(struct dsync_ibc *ibc,
+		      const struct dsync_mail_change **change_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mail_request(struct dsync_ibc *ibc,
+			    const struct dsync_mail_request *request);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail_request(struct dsync_ibc *ibc,
+			    const struct dsync_mail_request **request_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r);
+
+void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error,
+			   enum mail_error mail_error,
+			   bool require_full_resync);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r,
+		      enum mail_error *mail_error_r,
+		      bool *require_full_resync_r);
+
+/* Close any mail input streams that are kept open. This needs to be called
+   before the mail is attempted to be freed (usually on error conditions). */
+void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc);
+
+bool dsync_ibc_has_failed(struct dsync_ibc *ibc);
+bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc);
+bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc);
+bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c
new file mode 100644
index 0000000..b82818f
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mail.c
@@ -0,0 +1,156 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hex-binary.h"
+#include "md5.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "message-header-hash.h"
+#include "message-size.h"
+#include "mail-storage.h"
+#include "dsync-mail.h"
+
+struct mailbox_header_lookup_ctx *
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers)
+{
+	return mailbox_header_lookup_init(box, hashed_headers);
+}
+
+int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
+			    const char *const *hashed_headers, const char **hdr_hash_r)
+{
+	struct istream *hdr_input, *input;
+	struct mailbox_header_lookup_ctx *hdr_ctx;
+	struct message_header_hash_context hash_ctx;
+	struct md5_context md5_ctx;
+	unsigned char md5_result[MD5_RESULTLEN];
+	const unsigned char *data;
+	size_t size;
+	ssize_t sret;
+	int ret = 0;
+
+	hdr_ctx = mailbox_header_lookup_init(mail->box, hashed_headers);
+	ret = mail_get_header_stream(mail, hdr_ctx, &hdr_input);
+	mailbox_header_lookup_unref(&hdr_ctx);
+	if (ret < 0)
+		return -1;
+
+	input = i_stream_create_lf(hdr_input);
+
+	md5_init(&md5_ctx);
+	i_zero(&hash_ctx);
+	while ((sret = i_stream_read_more(input, &data, &size)) > 0) {
+		message_header_hash_more(&hash_ctx, &hash_method_md5, &md5_ctx,
+					 version, data, size);
+		i_stream_skip(input, size);
+	}
+	i_assert(sret == -1);
+	if (input->stream_errno != 0)
+		ret = -1;
+	i_stream_unref(&input);
+
+	md5_final(&md5_ctx, md5_result);
+	*hdr_hash_r = binary_to_hex(md5_result, sizeof(md5_result));
+	return ret;
+}
+
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+		    struct dsync_mail *dmail_r, const char **error_field_r)
+{
+	const char *guid;
+
+	i_zero(dmail_r);
+
+	if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
+		*error_field_r = "GUID";
+		return -1;
+	}
+	dmail_r->guid = guid;
+	dmail_r->uid = mail->uid;
+
+	dmail_r->input_mail = mail;
+	dmail_r->input_mail_uid = mail->uid;
+
+	if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
+		*error_field_r = "saved-date";
+		return -1;
+	}
+	if (!minimal_fill)
+		return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r);
+	dmail_r->minimal_fields = TRUE;
+	return 0;
+}
+
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+			       const char **error_field_r)
+{
+	const char *str;
+
+	if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) {
+		*error_field_r = "body";
+		return -1;
+	}
+
+	if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail_r->pop3_uidl) < 0) {
+		*error_field_r = "pop3-uidl";
+		return -1;
+	}
+	if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) {
+		*error_field_r = "pop3-order";
+		return -1;
+	}
+	if (*str != '\0') {
+		if (str_to_uint32(str, &dmail_r->pop3_order) < 0)
+			i_unreached();
+	}
+	if (mail_get_received_date(mail, &dmail_r->received_date) < 0) {
+		*error_field_r = "received-date";
+		return -1;
+	}
+	return 0;
+}
+
+static void
+const_string_array_dup(pool_t pool, const ARRAY_TYPE(const_string) *src,
+		       ARRAY_TYPE(const_string) *dest)
+{
+	const char *const *strings, *str;
+	unsigned int i, count;
+
+	if (!array_is_created(src))
+		return;
+
+	strings = array_get(src, &count);
+	if (count == 0)
+		return;
+
+	p_array_init(dest, pool, count);
+	for (i = 0; i < count; i++) {
+		str = p_strdup(pool, strings[i]);
+		array_push_back(dest, &str);
+	}
+}
+
+void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
+			   struct dsync_mail_change *dest_r)
+{
+	dest_r->type = src->type;
+	dest_r->uid = src->uid;
+	if (src->guid != NULL) {
+		dest_r->guid = *src->guid == '\0' ? "" :
+			p_strdup(pool, src->guid);
+	}
+	dest_r->hdr_hash = p_strdup(pool, src->hdr_hash);
+	dest_r->modseq = src->modseq;
+	dest_r->pvt_modseq = src->pvt_modseq;
+
+	dest_r->add_flags = src->add_flags;
+	dest_r->remove_flags = src->remove_flags;
+	dest_r->final_flags = src->final_flags;
+	dest_r->keywords_reset = src->keywords_reset;
+	const_string_array_dup(pool, &src->keyword_changes,
+			       &dest_r->keyword_changes);
+	dest_r->received_timestamp = src->received_timestamp;
+	dest_r->virtual_size = src->virtual_size;
+}
diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
new file mode 100644
index 0000000..fc00059
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -0,0 +1,108 @@
+#ifndef DSYNC_MAIL_H
+#define DSYNC_MAIL_H
+
+#include "mail-types.h"
+
+struct md5_context;
+struct mail;
+struct mailbox;
+
+struct dsync_mail {
+	/* either GUID="" or uid=0 */
+	const char *guid;
+	uint32_t uid;
+	time_t saved_date;
+
+	/* If non-NULL, we're syncing within the dsync process using ibc-pipe.
+	   This mail can be used to mailbox_copy() the mail. */
+	struct mail *input_mail;
+	/* Verify that this equals to input_mail->uid */
+	uint32_t input_mail_uid;
+
+	/* TRUE if the following fields aren't set, because minimal_fill=TRUE
+	   parameter was used. */
+	bool minimal_fields;
+
+	const char *pop3_uidl;
+	uint32_t pop3_order;
+	time_t received_date;
+	/* Input stream containing the message text, or NULL if all instances
+	   of the message were already expunged from this mailbox. */
+	struct istream *input;
+};
+
+struct dsync_mail_request {
+	/* either GUID=NULL or uid=0 */
+	const char *guid;
+	uint32_t uid;
+};
+
+enum dsync_mail_change_type {
+	DSYNC_MAIL_CHANGE_TYPE_SAVE,
+	DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+	DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE
+};
+
+#define KEYWORD_CHANGE_ADD '+'
+#define KEYWORD_CHANGE_REMOVE '-'
+#define KEYWORD_CHANGE_FINAL '='
+#define KEYWORD_CHANGE_ADD_AND_FINAL '&'
+
+struct dsync_mail_change {
+	enum dsync_mail_change_type type;
+
+	uint32_t uid;
+	/* Message's GUID:
+	    - for expunges either 128bit hex or NULL if unknown
+	    - "" if backend doesn't support GUIDs */
+	const char *guid;
+	/* If GUID is "", this contains hash of the message header,
+	   otherwise NULL */
+	const char *hdr_hash;
+
+	/* Message's current modseq (saves, flag changes) */
+	uint64_t modseq;
+	/* Message's current private modseq (for private flags in
+	   shared mailboxes, otherwise 0) */
+	uint64_t pvt_modseq;
+
+	/* List of flag/keyword changes: (saves, flag changes) */
+
+	/* Flags added/removed since last sync, and final flags containing
+	   flags that exist now but haven't changed */
+	uint8_t add_flags, remove_flags, final_flags;
+	uint8_t add_pvt_flags, remove_pvt_flags;
+	/* Remove all keywords before applying changes. This is used only with
+	   old transaction logs, new ones never reset keywords (just explicitly
+	   remove unwanted keywords) */
+	bool keywords_reset;
+	/* +add, -remove, =final, &add_and_final. */
+	ARRAY_TYPE(const_string) keyword_changes;
+
+	/* Received timestamp for saves, if brain.sync_since/until_timestamp is set */
+	time_t received_timestamp;
+	/* Mail's size for saves if brain.sync_max_size is set,
+	   UOFF_T_MAX otherwise. */
+	uoff_t virtual_size;
+};
+
+struct mailbox_header_lookup_ctx *
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers);
+
+int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
+			    const char *const *hashed_headers, const char **hdr_hash_r);
+static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash)
+{
+	/* md5(\n) */
+	return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0;
+}
+
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+		    struct dsync_mail *dmail_r, const char **error_field_r);
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+			       const char **error_field_r);
+
+void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
+			   struct dsync_mail_change *dest_r);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c
new file mode 100644
index 0000000..a58fbec
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-export.c
@@ -0,0 +1,961 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "istream.h"
+#include "mail-index-modseq.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "dsync-transaction-log-scan.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-export.h"
+
+struct dsync_mail_guid_instances {
+	ARRAY_TYPE(seq_range) seqs;
+	bool requested;
+	bool searched;
+};
+
+struct dsync_mailbox_exporter {
+	pool_t pool;
+	struct mailbox *box;
+	struct dsync_transaction_log_scan *log_scan;
+	uint32_t last_common_uid;
+
+	struct mailbox_header_lookup_ctx *wanted_headers;
+	struct mailbox_transaction_context *trans;
+	struct mail_search_context *search_ctx;
+	unsigned int search_pos, search_count;
+	unsigned int hdr_hash_version;
+
+	const char *const *hashed_headers;
+
+	/* GUID => instances */
+	HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
+	ARRAY_TYPE(seq_range) requested_uids;
+	ARRAY_TYPE(seq_range) search_uids;
+
+	ARRAY_TYPE(seq_range) expunged_seqs;
+	ARRAY_TYPE(const_string) expunged_guids;
+	unsigned int expunged_guid_idx;
+
+	/* uint32_t UID => struct dsync_mail_change */
+	HASH_TABLE(void *, struct dsync_mail_change *) changes;
+	/* changes sorted by UID */
+	ARRAY(struct dsync_mail_change *) sorted_changes;
+	unsigned int change_idx;
+	uint32_t highest_changed_uid;
+
+	struct mailbox_attribute_iter *attr_iter;
+	struct hash_iterate_context *attr_change_iter;
+	enum mail_attribute_type attr_type;
+	struct dsync_mailbox_attribute attr;
+
+	struct dsync_mail_change change;
+	struct dsync_mail dsync_mail;
+
+	const char *error;
+	enum mail_error mail_error;
+
+	bool body_search_initialized:1;
+	bool auto_export_mails:1;
+	bool mails_have_guids:1;
+	bool minimal_dmail_fill:1;
+	bool return_all_mails:1;
+	bool export_received_timestamps:1;
+	bool export_virtual_sizes:1;
+	bool no_hdr_hashes:1;
+};
+
+static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
+			    struct mail *mail, const char *field)
+{
+	const char *errstr;
+	enum mail_error error;
+
+	errstr = mailbox_get_last_internal_error(exporter->box, &error);
+	if (error == MAIL_ERROR_EXPUNGED)
+		return 0;
+
+	exporter->mail_error = error;
+	exporter->error = p_strdup_printf(exporter->pool,
+		"Can't lookup %s for UID=%u: %s",
+		field, mail->uid, errstr);
+	return -1;
+}
+
+static bool
+final_keyword_check(struct dsync_mail_change *change, const char *name,
+		    char *type_r)
+{
+	const char *const *changes;
+	unsigned int i, count;
+
+	*type_r = KEYWORD_CHANGE_FINAL;
+
+	changes = array_get(&change->keyword_changes, &count);
+	for (i = 0; i < count; i++) {
+		if (strcmp(changes[i]+1, name) != 0)
+			continue;
+
+		switch (changes[i][0]) {
+		case KEYWORD_CHANGE_ADD:
+			/* replace with ADD_AND_FINAL */
+			array_delete(&change->keyword_changes, i, 1);
+			*type_r = KEYWORD_CHANGE_ADD_AND_FINAL;
+			return FALSE;
+		case KEYWORD_CHANGE_REMOVE:
+			/* a final keyword is marked as removed.
+			   this shouldn't normally happen. */
+			array_delete(&change->keyword_changes, i, 1);
+			return FALSE;
+		case KEYWORD_CHANGE_ADD_AND_FINAL:
+		case KEYWORD_CHANGE_FINAL:
+			/* no change */
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void
+search_update_flag_changes(struct dsync_mailbox_exporter *exporter,
+			   struct mail *mail, struct dsync_mail_change *change)
+{
+	const char *const *keywords;
+	unsigned int i;
+	char type;
+
+	i_assert((change->add_flags & change->remove_flags) == 0);
+
+	change->modseq = mail_get_modseq(mail);
+	change->pvt_modseq = mail_get_pvt_modseq(mail);
+	change->final_flags = mail_get_flags(mail);
+
+	keywords = mail_get_keywords(mail);
+	if (!array_is_created(&change->keyword_changes) &&
+	    keywords[0] != NULL) {
+		p_array_init(&change->keyword_changes, exporter->pool,
+			     str_array_length(keywords));
+	}
+	for (i = 0; keywords[i] != NULL; i++) {
+		/* add the final keyword if it's not already there
+		   as +keyword */
+		if (!final_keyword_check(change, keywords[i], &type)) {
+			const char *keyword_change =
+				p_strdup_printf(exporter->pool, "%c%s",
+						type, keywords[i]);
+			array_push_back(&change->keyword_changes,
+					&keyword_change);
+		}
+	}
+}
+
+static int
+exporter_get_guids(struct dsync_mailbox_exporter *exporter,
+		   struct mail *mail, const char **guid_r,
+		   const char **hdr_hash_r)
+{
+	*guid_r = "";
+	*hdr_hash_r = NULL;
+
+	/* always try to get GUID, even if we're also getting header hash */
+	if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0)
+		return dsync_mail_error(exporter, mail, "GUID");
+
+	if (!exporter->mails_have_guids) {
+		/* get header hash also */
+		if (exporter->no_hdr_hashes) {
+			*hdr_hash_r = "";
+			return 1;
+		}
+		if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version,
+					    exporter->hashed_headers, hdr_hash_r) < 0)
+			return dsync_mail_error(exporter, mail, "hdr-stream");
+		return 1;
+	} else if (**guid_r == '\0') {
+		exporter->mail_error = MAIL_ERROR_TEMP;
+		exporter->error = "Backend doesn't support GUIDs, "
+			"sync with header hashes instead";
+		return -1;
+	} else {
+		/* GUIDs are required, we don't need header hash */
+		return 1;
+	} 
+}
+
+static int
+search_update_flag_change_guid(struct dsync_mailbox_exporter *exporter,
+			       struct mail *mail)
+{
+	struct dsync_mail_change *change, *log_change;
+	const char *guid, *hdr_hash;
+	int ret;
+
+	change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid));
+	if (change != NULL) {
+		i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
+	} else {
+		i_assert(exporter->return_all_mails);
+
+		change = p_new(exporter->pool, struct dsync_mail_change, 1);
+		change->uid = mail->uid;
+		change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
+		hash_table_insert(exporter->changes,
+				  POINTER_CAST(mail->uid), change);
+	}
+
+	if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0)
+		return -1;
+	if (ret == 0) {
+		/* the message was expunged during export */
+		i_zero(change);
+		change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
+		change->uid = mail->uid;
+
+		/* find its GUID from log if possible */
+		log_change = dsync_transaction_log_scan_find_new_expunge(
+			exporter->log_scan, mail->uid);
+		if (log_change != NULL)
+			change->guid = log_change->guid;
+	} else {
+		change->guid = *guid == '\0' ? "" :
+			p_strdup(exporter->pool, guid);
+		change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+		search_update_flag_changes(exporter, mail, change);
+	}
+	return 0;
+}
+
+static struct dsync_mail_change *
+export_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid)
+{
+	struct dsync_mail_change *change;
+
+	change = hash_table_lookup(exporter->changes, POINTER_CAST(uid));
+	if (change == NULL) {
+		change = p_new(exporter->pool, struct dsync_mail_change, 1);
+		change->uid = uid;
+		hash_table_insert(exporter->changes, POINTER_CAST(uid), change);
+	} else {
+		/* move flag changes into a save. this happens only when
+		   last_common_uid isn't known */
+		i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
+		i_assert(exporter->last_common_uid == 0);
+	}
+
+	change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE;
+	return change;
+}
+
+static void
+export_add_mail_instance(struct dsync_mailbox_exporter *exporter,
+			 struct dsync_mail_change *change, uint32_t seq)
+{
+	struct dsync_mail_guid_instances *instances;
+
+	if (exporter->auto_export_mails && !exporter->mails_have_guids) {
+		/* GUIDs not supported, mail is requested by UIDs */
+		seq_range_array_add(&exporter->requested_uids, change->uid);
+		return;
+	}
+	if (*change->guid == '\0') {
+		/* mail UIDs are manually requested */
+		i_assert(!exporter->mails_have_guids);
+		return;
+	}
+
+	instances = hash_table_lookup(exporter->export_guids, change->guid);
+	if (instances == NULL) {
+		instances = p_new(exporter->pool,
+				  struct dsync_mail_guid_instances, 1);
+		p_array_init(&instances->seqs, exporter->pool, 2);
+		hash_table_insert(exporter->export_guids,
+				  p_strdup(exporter->pool, change->guid),
+				  instances);
+		if (exporter->auto_export_mails)
+			instances->requested = TRUE;
+	}
+	seq_range_array_add(&instances->seqs, seq);
+}
+
+static int
+search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
+{
+	struct dsync_mail_change *change;
+	const char *guid, *hdr_hash;
+	enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
+	time_t received_timestamp = 0;
+	uoff_t virtual_size = UOFF_T_MAX;
+	int ret;
+
+	/* update wanted fields in case we didn't already set them for the
+	   search */
+	if (exporter->export_received_timestamps)
+		wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+	if (exporter->export_virtual_sizes)
+		wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE;
+	mail_add_temp_wanted_fields(mail, wanted_fields,
+				    exporter->wanted_headers);
+
+	/* If message is already expunged here, just skip it */
+	if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
+		return ret;
+
+	if (exporter->export_received_timestamps) {
+		if (mail_get_received_date(mail, &received_timestamp) < 0)
+			return dsync_mail_error(exporter, mail, "received-time");
+		if (received_timestamp == 0) {
+			/* don't allow timestamps to be zero. we want to have
+			   asserts verify that the timestamp is set properly. */
+			received_timestamp = 1;
+		}
+	}
+	if (exporter->export_virtual_sizes) {
+		if (mail_get_virtual_size(mail, &virtual_size) < 0)
+			return dsync_mail_error(exporter, mail, "virtual-size");
+		i_assert(virtual_size != UOFF_T_MAX);
+	}
+
+	change = export_save_change_get(exporter, mail->uid);
+	change->guid = *guid == '\0' ? "" :
+		p_strdup(exporter->pool, guid);
+	change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+	change->received_timestamp = received_timestamp;
+	change->virtual_size = virtual_size;
+	search_update_flag_changes(exporter, mail, change);
+
+	export_add_mail_instance(exporter, change, mail->seq);
+	return 1;
+}
+
+static void
+dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter,
+					 ARRAY_TYPE(seq_range) *uids)
+{
+	struct hash_iterate_context *iter;
+	void *key;
+	struct dsync_mail_change *change;
+
+	iter = hash_table_iterate_init(exporter->changes);
+	while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
+		if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE)
+			seq_range_array_add(uids, change->uid);
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+static void
+dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter)
+{
+	struct hash_iterate_context *iter;
+	void *key;
+	struct dsync_mail_change *change;
+
+	/* any flag changes for UIDs above last_common_uid weren't found by
+	   mailbox search, which means they were already expunged. for some
+	   reason the log scanner found flag changes for the message, but not
+	   the expunge. just remove these. */
+	iter = hash_table_iterate_init(exporter->changes);
+	while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
+		if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE &&
+		    change->uid > exporter->last_common_uid)
+			hash_table_remove(exporter->changes, key);
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+static void
+dsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter)
+{
+	struct mail_search_context *search_ctx;
+	struct mail_search_args *search_args;
+	struct mail_search_arg *sarg;
+	struct mail *mail;
+	enum mail_fetch_field wanted_fields = 0;
+	struct mailbox_header_lookup_ctx *wanted_headers = NULL;
+	int ret = 0;
+
+	search_args = mail_search_build_init();
+	sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
+	p_array_init(&sarg->value.seqset, search_args->pool, 1);
+
+	if (exporter->return_all_mails || exporter->last_common_uid == 0) {
+		/* we want to know about all mails */
+		seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1);
+	} else {
+		/* lookup GUIDs for messages with flag changes */
+		dsync_mailbox_export_add_flagchange_uids(exporter,
+							 &sarg->value.seqset);
+		/* lookup new messages */
+		seq_range_array_add_range(&sarg->value.seqset,
+					  exporter->last_common_uid + 1,
+					  (uint32_t)-1);
+	}
+
+	if (exporter->last_common_uid == 0) {
+		/* we're syncing all mails, so we can request the wanted
+		   fields for all the mails */
+		wanted_fields = MAIL_FETCH_GUID;
+		wanted_headers = exporter->wanted_headers;
+	}
+
+	exporter->trans = mailbox_transaction_begin(exporter->box,
+						MAILBOX_TRANSACTION_FLAG_SYNC,
+						__func__);
+	search_ctx = mailbox_search_init(exporter->trans, search_args, NULL,
+					 wanted_fields, wanted_headers);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		T_BEGIN {
+			if (mail->uid <= exporter->last_common_uid)
+				ret = search_update_flag_change_guid(exporter, mail);
+			else
+				ret = search_add_save(exporter, mail);
+		} T_END;
+		if (ret < 0)
+			break;
+	}
+	i_assert(ret >= 0 || exporter->error != NULL);
+
+	dsync_mailbox_export_drop_expunged_flag_changes(exporter);
+
+	if (mailbox_search_deinit(&search_ctx) < 0 &&
+	    exporter->error == NULL) {
+		exporter->error = p_strdup_printf(exporter->pool,
+			"Mail search failed: %s",
+			mailbox_get_last_internal_error(exporter->box,
+							&exporter->mail_error));
+	}
+}
+
+static int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1,
+				       struct dsync_mail_change *const *c2)
+{
+	if ((*c1)->uid < (*c2)->uid)
+		return -1;
+	if ((*c1)->uid > (*c2)->uid)
+		return 1;
+	return 0;
+}
+
+static void
+dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter)
+{
+	struct hash_iterate_context *iter;
+	void *key;
+	struct dsync_mail_change *change;
+
+	p_array_init(&exporter->sorted_changes, exporter->pool,
+		     hash_table_count(exporter->changes));
+
+	iter = hash_table_iterate_init(exporter->changes);
+	while (hash_table_iterate(iter, exporter->changes, &key, &change))
+		array_push_back(&exporter->sorted_changes, &change);
+	hash_table_iterate_deinit(&iter);
+	array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp);
+}
+
+static void
+dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter,
+			       enum mail_attribute_type type)
+{
+	exporter->attr_iter =
+		mailbox_attribute_iter_init(exporter->box, type, "");
+	exporter->attr_type = type;
+}
+
+static void
+dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter,
+			      struct dsync_transaction_log_scan *log_scan)
+{
+	HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes;
+	struct hash_iterate_context *iter;
+	void *key;
+	struct dsync_mail_change *change, *dup_change;
+
+	log_changes = dsync_transaction_log_scan_get_hash(log_scan);
+	if (dsync_transaction_log_scan_has_all_changes(log_scan)) {
+		/* we tried to access too old/invalid modseqs. to make sure
+		   no changes get lost, we need to send all of the messages */
+		exporter->return_all_mails = TRUE;
+	}
+
+	/* clone the hash table, since we're changing it. */
+	hash_table_create_direct(&exporter->changes, exporter->pool,
+				 hash_table_count(log_changes));
+	iter = hash_table_iterate_init(log_changes);
+	while (hash_table_iterate(iter, log_changes, &key, &change)) {
+		dup_change = p_new(exporter->pool, struct dsync_mail_change, 1);
+		*dup_change = *change;
+		hash_table_insert(exporter->changes, key, dup_change);
+		if (exporter->highest_changed_uid < change->uid)
+			exporter->highest_changed_uid = change->uid;
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+struct dsync_mailbox_exporter *
+dsync_mailbox_export_init(struct mailbox *box,
+			  struct dsync_transaction_log_scan *log_scan,
+			  uint32_t last_common_uid,
+			  enum dsync_mailbox_exporter_flags flags,
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers)
+{
+	struct dsync_mailbox_exporter *exporter;
+	pool_t pool;
+
+	pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export",
+				     4096);
+	exporter = p_new(pool, struct dsync_mailbox_exporter, 1);
+	exporter->pool = pool;
+	exporter->box = box;
+	exporter->log_scan = log_scan;
+	exporter->last_common_uid = last_common_uid;
+	exporter->auto_export_mails =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
+	exporter->mails_have_guids =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
+	exporter->minimal_dmail_fill =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
+	exporter->export_received_timestamps =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
+	exporter->export_virtual_sizes =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0;
+	exporter->hdr_hash_version = hdr_hash_version;
+	exporter->no_hdr_hashes =
+		(flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0;
+	exporter->hashed_headers = hashed_headers;
+
+	p_array_init(&exporter->requested_uids, pool, 16);
+	p_array_init(&exporter->search_uids, pool, 16);
+	hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
+	p_array_init(&exporter->expunged_seqs, pool, 16);
+	p_array_init(&exporter->expunged_guids, pool, 16);
+
+	if (!exporter->mails_have_guids && !exporter->no_hdr_hashes)
+		exporter->wanted_headers =
+			dsync_mail_get_hash_headers(box, exporter->hashed_headers);
+
+	/* first scan transaction log and save any expunges and flag changes */
+	dsync_mailbox_export_log_scan(exporter, log_scan);
+	/* get saves and also find GUIDs for flag changes */
+	dsync_mailbox_export_search(exporter);
+	/* get the changes sorted by UID */
+	dsync_mailbox_export_sort_changes(exporter);
+
+	dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
+	return exporter;
+}
+
+static int
+dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter)
+{
+	struct dsync_mailbox_attribute *attr;
+	struct mail_attribute_value value;
+
+	while (hash_table_iterate(exporter->attr_change_iter,
+				  dsync_transaction_log_scan_get_attr_hash(exporter->log_scan),
+				  &attr, &attr)) {
+		if (attr->exported || !attr->deleted)
+			continue;
+
+		/* lookup the value mainly to get its last_change value. */
+		if (mailbox_attribute_get_stream(exporter->box, attr->type,
+						 attr->key, &value) < 0) {
+			exporter->error = p_strdup_printf(exporter->pool,
+				"Mailbox attribute %s lookup failed: %s", attr->key,
+				mailbox_get_last_internal_error(exporter->box,
+								&exporter->mail_error));
+			break;
+		}
+		if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
+			i_stream_unref(&value.value_stream);
+			continue;
+		}
+
+		attr->last_change = value.last_change;
+		if (value.value != NULL || value.value_stream != NULL) {
+			attr->value = p_strdup(exporter->pool, value.value);
+			attr->value_stream = value.value_stream;
+			attr->deleted = FALSE;
+		}
+		attr->exported = TRUE;
+		exporter->attr = *attr;
+		return 1;
+	}
+	hash_table_iterate_deinit(&exporter->attr_change_iter);
+	return 0;
+}
+
+static int
+dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter)
+{
+	HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
+	struct dsync_mailbox_attribute lookup_attr, *attr;
+	struct dsync_mailbox_attribute *attr_change;
+	const char *key;
+	struct mail_attribute_value value;
+	bool export_all_attrs;
+
+	export_all_attrs = exporter->return_all_mails ||
+		exporter->last_common_uid == 0;
+	attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
+	lookup_attr.type = exporter->attr_type;
+
+	/* note that the order of processing may be important for some
+	   attributes. for example sieve can't set a script active until it's
+	   first been created */
+	while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) {
+		lookup_attr.key = key;
+		attr_change = hash_table_lookup(attr_changes, &lookup_attr);
+		if (attr_change == NULL && !export_all_attrs)
+			continue;
+
+		if (mailbox_attribute_get_stream(exporter->box,
+						 exporter->attr_type, key,
+						 &value) < 0) {
+			exporter->error = p_strdup_printf(exporter->pool,
+				"Mailbox attribute %s lookup failed: %s", key,
+				mailbox_get_last_internal_error(exporter->box,
+								&exporter->mail_error));
+			return -1;
+		}
+		if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
+			/* readonly attributes can't be changed,
+			   no point in exporting them */
+			if (value.value_stream != NULL)
+				i_stream_unref(&value.value_stream);
+			continue;
+		}
+		if (value.value == NULL && value.value_stream == NULL &&
+		    (attr_change == NULL || !attr_change->deleted)) {
+			/* the attribute was just deleted?
+			   skip for this sync. */
+			continue;
+		}
+		if (attr_change != NULL && attr_change->exported) {
+			/* duplicate attribute returned.
+			   shouldn't normally happen, but don't crash. */
+			i_warning("Ignoring duplicate attributes '%s'", key);
+			continue;
+		}
+
+		attr = &exporter->attr;
+		i_zero(attr);
+		attr->type = exporter->attr_type;
+		attr->value = p_strdup(exporter->pool, value.value);
+		attr->value_stream = value.value_stream;
+		attr->last_change = value.last_change;
+		if (attr_change != NULL) {
+			attr_change->exported = TRUE;
+			attr->key = attr_change->key;
+			attr->deleted = attr_change->deleted &&
+				!DSYNC_ATTR_HAS_VALUE(attr);
+			attr->modseq = attr_change->modseq;
+		} else {
+			attr->key = p_strdup(exporter->pool, key);
+		}
+		return 1;
+	}
+	if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) {
+		exporter->error = p_strdup_printf(exporter->pool,
+			"Mailbox attribute iteration failed: %s",
+			mailbox_get_last_internal_error(exporter->box,
+							&exporter->mail_error));
+		return -1;
+	}
+	if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
+		/* export shared attributes */
+		dsync_mailbox_export_attr_init(exporter,
+					       MAIL_ATTRIBUTE_TYPE_SHARED);
+		return dsync_mailbox_export_iter_next_attr(exporter);
+	}
+	exporter->attr_change_iter = hash_table_iterate_init(attr_changes);
+	return dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
+}
+
+int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
+				   const struct dsync_mailbox_attribute **attr_r)
+{
+	int ret;
+
+	if (exporter->error != NULL)
+		return -1;
+
+	i_stream_unref(&exporter->attr.value_stream);
+
+	if (exporter->attr_iter != NULL) {
+		ret = dsync_mailbox_export_iter_next_attr(exporter);
+	} else {
+		ret = dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
+	}
+	if (ret > 0)
+		*attr_r = &exporter->attr;
+	return ret;
+}
+
+int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter,
+			      const struct dsync_mail_change **change_r)
+{
+	struct dsync_mail_change *const *changes;
+	unsigned int count;
+
+	if (exporter->error != NULL)
+		return -1;
+
+	changes = array_get(&exporter->sorted_changes, &count);
+	if (exporter->change_idx == count)
+		return 0;
+	*change_r = changes[exporter->change_idx++];
+	return 1;
+}
+
+static int
+dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
+{
+	struct mail_search_args *search_args;
+	struct mail_search_arg *sarg;
+	struct hash_iterate_context *iter;
+	const struct seq_range *uids;
+	char *guid;
+	const char *const_guid;
+	enum mail_fetch_field wanted_fields;
+	struct dsync_mail_guid_instances *instances;
+	const struct seq_range *range;
+	unsigned int i, count;
+	uint32_t seq, seq1, seq2;
+
+	i_assert(exporter->search_ctx == NULL);
+
+	search_args = mail_search_build_init();
+	sarg = mail_search_build_add(search_args, SEARCH_SEQSET);
+	p_array_init(&sarg->value.seqset, search_args->pool, 128);
+
+	/* get a list of messages we want to fetch. if there are more than one
+	   instance for a GUID, use the first one. */
+	iter = hash_table_iterate_init(exporter->export_guids);
+	while (hash_table_iterate(iter, exporter->export_guids,
+				  &guid, &instances)) {
+		if (!instances->requested ||
+		    array_count(&instances->seqs) == 0)
+			continue;
+
+		uids = array_front(&instances->seqs);
+		seq = uids[0].seq1;
+		if (!instances->searched) {
+			instances->searched = TRUE;
+			seq_range_array_add(&sarg->value.seqset, seq);
+		} else if (seq_range_exists(&exporter->expunged_seqs, seq)) {
+			/* we're on a second round, refetching expunged
+			   messages */
+			seq_range_array_remove(&instances->seqs, seq);
+			seq_range_array_remove(&exporter->expunged_seqs, seq);
+			if (array_count(&instances->seqs) == 0) {
+				/* no instances left */
+				const_guid = guid;
+				array_push_back(&exporter->expunged_guids,
+						&const_guid);
+				continue;
+			}
+			uids = array_front(&instances->seqs);
+			seq = uids[0].seq1;
+			seq_range_array_add(&sarg->value.seqset, seq);
+		}
+	}
+	hash_table_iterate_deinit(&iter);
+
+	/* add requested UIDs */
+	range = array_get(&exporter->requested_uids, &count);
+	for (i = 0; i < count; i++) {
+		mailbox_get_seq_range(exporter->box,
+				      range[i].seq1, range[i].seq2,
+				      &seq1, &seq2);
+		seq_range_array_add_range(&sarg->value.seqset,
+					  seq1, seq2);
+	}
+	array_clear(&exporter->search_uids);
+	array_append_array(&exporter->search_uids, &exporter->requested_uids);
+	array_clear(&exporter->requested_uids);
+
+	wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
+	if (!exporter->minimal_dmail_fill) {
+		wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
+			MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
+			MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
+	}
+	exporter->search_count += seq_range_count(&sarg->value.seqset);
+	exporter->search_ctx =
+		mailbox_search_init(exporter->trans, search_args, NULL,
+				    wanted_fields, NULL);
+	mail_search_args_unref(&search_args);
+	return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
+}
+
+static void
+dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter)
+{
+	if (exporter->search_ctx == NULL)
+		return;
+
+	if (mailbox_search_deinit(&exporter->search_ctx) < 0 &&
+	    exporter->error == NULL) {
+		exporter->error = p_strdup_printf(exporter->pool,
+			"Mail search failed: %s",
+			mailbox_get_last_internal_error(exporter->box,
+							&exporter->mail_error));
+	}
+}
+
+static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
+				     struct mail *mail)
+{
+	struct dsync_mail_guid_instances *instances;
+	const char *error_field;
+
+	if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
+			    &exporter->dsync_mail, &error_field) < 0)
+		return dsync_mail_error(exporter, mail, error_field);
+
+	instances = *exporter->dsync_mail.guid == '\0' ? NULL :
+		hash_table_lookup(exporter->export_guids,
+				  exporter->dsync_mail.guid);
+	if (instances != NULL) {
+		/* GUID found */
+	} else if (exporter->dsync_mail.uid != 0) {
+		/* mail requested by UID */
+	} else {
+		exporter->mail_error = MAIL_ERROR_TEMP;
+		exporter->error = p_strdup_printf(exporter->pool,
+			"GUID unexpectedly changed for UID=%u GUID=%s",
+			mail->uid, exporter->dsync_mail.guid);
+		return -1;
+	}
+
+	if (!seq_range_exists(&exporter->search_uids, mail->uid))
+		exporter->dsync_mail.uid = 0;
+	else
+		exporter->dsync_mail.guid = "";
+
+	/* this message was successfully returned, don't try retrying it */
+	if (instances != NULL)
+		array_clear(&instances->seqs);
+	return 1;
+}
+
+void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
+				    const struct dsync_mail_request *request)
+{
+	struct dsync_mail_guid_instances *instances;
+
+	i_assert(!exporter->auto_export_mails);
+
+	if (request->guid == NULL) {
+		i_assert(request->uid > 0);
+		seq_range_array_add(&exporter->requested_uids, request->uid);
+		return;
+	}
+
+	instances = hash_table_lookup(exporter->export_guids, request->guid);
+	if (instances == NULL) {
+		exporter->mail_error = MAIL_ERROR_TEMP;
+		exporter->error = p_strdup_printf(exporter->pool,
+			"Remote requested unexpected GUID %s", request->guid);
+		return;
+	}
+	instances->requested = TRUE;
+}
+
+int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter,
+				   const struct dsync_mail **mail_r)
+{
+	struct mail *mail;
+	const char *const *guids;
+	unsigned int count;
+	int ret;
+
+	if (exporter->error != NULL)
+		return -1;
+	if (!exporter->body_search_initialized) {
+		exporter->body_search_initialized = TRUE;
+		if (dsync_mailbox_export_body_search_init(exporter) < 0) {
+			i_assert(exporter->error != NULL);
+			return -1;
+		}
+	}
+
+	while (mailbox_search_next(exporter->search_ctx, &mail)) {
+		exporter->search_pos++;
+		if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) {
+			*mail_r = &exporter->dsync_mail;
+			return 1;
+		}
+		if (ret < 0) {
+			i_assert(exporter->error != NULL);
+			return -1;
+		}
+		/* the message was expunged. if the GUID has another instance,
+		   try sending it later. */
+		seq_range_array_add(&exporter->expunged_seqs, mail->seq);
+	}
+	/* if some instances of messages were expunged, retry fetching them
+	   with other instances */
+	dsync_mailbox_export_body_search_deinit(exporter);
+	if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) {
+		i_assert(exporter->error != NULL);
+		return -1;
+	}
+	if (ret > 0) {
+		/* not finished yet */
+		return dsync_mailbox_export_next_mail(exporter, mail_r);
+	}
+
+	/* finished with messages. if there are any expunged messages,
+	   return them */
+	guids = array_get(&exporter->expunged_guids, &count);
+	if (exporter->expunged_guid_idx < count) {
+		i_zero(&exporter->dsync_mail);
+		exporter->dsync_mail.guid =
+			guids[exporter->expunged_guid_idx++];
+		*mail_r = &exporter->dsync_mail;
+		return 1;
+	}
+	return 0;
+}
+
+int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter,
+				const char **errstr_r, enum mail_error *error_r)
+{
+	struct dsync_mailbox_exporter *exporter = *_exporter;
+
+	*_exporter = NULL;
+
+	if (exporter->attr_iter != NULL)
+		(void)mailbox_attribute_iter_deinit(&exporter->attr_iter);
+	dsync_mailbox_export_body_search_deinit(exporter);
+	(void)mailbox_transaction_commit(&exporter->trans);
+	mailbox_header_lookup_unref(&exporter->wanted_headers);
+
+	i_stream_unref(&exporter->attr.value_stream);
+	hash_table_destroy(&exporter->export_guids);
+	hash_table_destroy(&exporter->changes);
+
+	i_assert((exporter->error != NULL) == (exporter->mail_error != 0));
+
+	*error_r = exporter->mail_error;
+	*errstr_r = t_strdup(exporter->error);
+	pool_unref(&exporter->pool);
+	return *errstr_r != NULL ? -1 : 0;
+}
+
+const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter)
+{
+	if (exporter->search_ctx == NULL)
+		return "";
+	return t_strdup_printf("%u/%u", exporter->search_pos,
+			       exporter->search_count);
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h
new file mode 100644
index 0000000..67e3216
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-export.h
@@ -0,0 +1,37 @@
+#ifndef DSYNC_MAILBOX_EXPORT_H
+#define DSYNC_MAILBOX_EXPORT_H
+
+enum dsync_mailbox_exporter_flags {
+	DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS	= 0x01,
+	DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS	= 0x02,
+	DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL	= 0x04,
+	DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS		= 0x08,
+	DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES	= 0x20,
+	DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES		= 0x40,
+};
+
+struct dsync_mailbox_exporter *
+dsync_mailbox_export_init(struct mailbox *box,
+			  struct dsync_transaction_log_scan *log_scan,
+			  uint32_t last_common_uid,
+			  enum dsync_mailbox_exporter_flags flags,
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers);
+/* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */
+int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
+				   const struct dsync_mailbox_attribute **attr_r);
+/* Returns 1 if change was returned, 0 if no more changes, -1 on error */
+int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter,
+			      const struct dsync_mail_change **change_r);
+
+void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
+				    const struct dsync_mail_request *request);
+/* Returns 1 if mail was returned, 0 if no more mails, -1 on error */
+int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter,
+				   const struct dsync_mail **mail_r);
+int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **exporter,
+				const char **errstr_r, enum mail_error *error_r);
+
+const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
new file mode 100644
index 0000000..8a8fbe4
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -0,0 +1,2997 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "istream.h"
+#include "seq-range-array.h"
+#include "imap-util.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "dsync-transaction-log-scan.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-import.h"
+
+struct importer_mail {
+	const char *guid;
+	uint32_t uid;
+};
+
+struct importer_new_mail {
+	/* linked list of mails for this GUID */
+	struct importer_new_mail *next;
+	/* if non-NULL, this mail exists in both local and remote. this link
+	   points to the other side. */
+	struct importer_new_mail *link;
+
+	const char *guid;
+	struct dsync_mail_change *change;
+
+	/* the final UID for the message */
+	uint32_t final_uid;
+	/* the original local UID, or 0 if exists only remotely */
+	uint32_t local_uid;
+	/* the original remote UID, or 0 if exists only remotely */
+	uint32_t remote_uid;
+	/* UID for the mail in the virtual \All mailbox */
+	uint32_t virtual_all_uid;
+
+	bool uid_in_local:1;
+	bool uid_is_usable:1;
+	bool skip:1;
+	bool expunged:1;
+	bool copy_failed:1;
+	bool saved:1;
+};
+
+/* for quickly testing that two-way sync doesn't actually do any unexpected
+   modifications. */
+#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/
+
+HASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *);
+HASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *);
+
+struct dsync_mailbox_importer {
+	pool_t pool;
+	struct mailbox *box;
+	uint32_t last_common_uid;
+	uint64_t last_common_modseq, last_common_pvt_modseq;
+	uint32_t remote_uid_next;
+	uint32_t remote_first_recent_uid;
+	uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
+	time_t sync_since_timestamp;
+	time_t sync_until_timestamp;
+	uoff_t sync_max_size;
+	enum mailbox_transaction_flags transaction_flags;
+	unsigned int hdr_hash_version;
+	unsigned int commit_msgs_interval;
+
+	const char *const *hashed_headers;
+
+	enum mail_flags sync_flag;
+	const char *sync_keyword;
+	bool sync_flag_dontwant;
+
+	struct mailbox_transaction_context *trans, *ext_trans;
+	struct mail_search_context *search_ctx;
+	struct mail *mail, *ext_mail;
+
+	struct mailbox *virtual_all_box;
+	struct mailbox_transaction_context *virtual_trans;
+	struct mail *virtual_mail;
+
+	struct mail *cur_mail;
+	const char *cur_guid;
+	const char *cur_hdr_hash;
+
+	/* UID => struct dsync_mail_change */
+	HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;
+	HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes;
+
+	ARRAY_TYPE(seq_range) maybe_expunge_uids;
+	ARRAY(struct dsync_mail_change *) maybe_saves;
+
+	/* GUID => struct importer_new_mail */
+	HASH_TABLE_TYPE(guid_new_mail) import_guids;
+	/* UID => struct importer_new_mail */
+	HASH_TABLE_TYPE(uid_new_mail) import_uids;
+
+	ARRAY(struct importer_new_mail *) newmails;
+	ARRAY_TYPE(uint32_t) wanted_uids;
+	ARRAY_TYPE(uint32_t) saved_uids;
+	uint32_t highest_wanted_uid;
+
+	ARRAY(struct dsync_mail_request) mail_requests;
+	unsigned int mail_request_idx;
+
+	uint32_t prev_uid, next_local_seq, local_uid_next;
+	uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq;
+	unsigned int import_pos, import_count;
+	unsigned int first_unsaved_idx, saves_since_commit;
+
+	enum mail_error mail_error;
+
+	bool failed:1;
+	bool require_full_resync:1;
+	bool debug:1;
+	bool stateful_import:1;
+	bool last_common_uid_found:1;
+	bool cur_uid_has_change:1;
+	bool cur_mail_skip:1;
+	bool local_expunged_guids_set:1;
+	bool new_uids_assigned:1;
+	bool want_mail_requests:1;
+	bool master_brain:1;
+	bool revert_local_changes:1;
+	bool mails_have_guids:1;
+	bool mails_use_guid128:1;
+	bool delete_mailbox:1;
+	bool empty_hdr_workaround:1;
+};
+
+static const char *dsync_mail_change_type_names[] = {
+	"save", "expunge", "flag-change"
+};
+
+static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
+					const struct dsync_mail *mail,
+					struct importer_new_mail *all_newmails,
+					bool remote_mail);
+static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer,
+				       bool final);
+
+static void ATTR_FORMAT(2, 3)
+imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...)
+{
+	va_list args;
+
+	if (importer->debug) T_BEGIN {
+		va_start(args, fmt);
+		i_debug("brain %c: Import %s: %s",
+			importer->master_brain ? 'M' : 'S',
+			mailbox_get_vname(importer->box),
+			t_strdup_vprintf(fmt, args));
+		va_end(args);
+	} T_END;
+}
+
+static void
+dsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
+			      const char *error)
+{
+	if (!importer->stateful_import) {
+		i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
+			error);
+	} else {
+		i_warning("Mailbox %s doesn't match previous state: %s "
+			  "(dsync must be run again without the state)",
+			  mailbox_get_vname(importer->box), error);
+	}
+	importer->require_full_resync = TRUE;
+}
+
+static void
+dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
+{
+	struct mail_search_args *search_args;
+	struct mail_search_arg *sarg;
+
+	search_args = mail_search_build_init();
+	sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
+	p_array_init(&sarg->value.seqset, search_args->pool, 128);
+	seq_range_array_add_range(&sarg->value.seqset,
+				  importer->last_common_uid+1, (uint32_t)-1);
+
+	importer->search_ctx =
+		mailbox_search_init(importer->trans, search_args, NULL,
+				    0, NULL);
+	mail_search_args_unref(&search_args);
+
+	if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))
+		importer->next_local_seq = importer->cur_mail->seq;
+	/* this flag causes cur_guid to be looked up later */
+	importer->cur_mail_skip = TRUE;
+}
+
+static void
+dsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer)
+{
+	const enum mailbox_transaction_flags ext_trans_flags =
+		importer->transaction_flags |
+		MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+		MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
+
+	importer->trans = mailbox_transaction_begin(importer->box,
+						    importer->transaction_flags,
+						    "dsync import");
+	importer->ext_trans = mailbox_transaction_begin(importer->box,
+							ext_trans_flags,
+							"dsync ext import");
+	importer->mail = mail_alloc(importer->trans, 0, NULL);
+	importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);
+}
+
+struct dsync_mailbox_importer *
+dsync_mailbox_import_init(struct mailbox *box,
+			  struct mailbox *virtual_all_box,
+			  struct dsync_transaction_log_scan *log_scan,
+			  uint32_t last_common_uid,
+			  uint64_t last_common_modseq,
+			  uint64_t last_common_pvt_modseq,
+			  uint32_t remote_uid_next,
+			  uint32_t remote_first_recent_uid,
+			  uint64_t remote_highest_modseq,
+			  uint64_t remote_highest_pvt_modseq,
+			  time_t sync_since_timestamp,
+			  time_t sync_until_timestamp,
+			  uoff_t sync_max_size,
+			  const char *sync_flag,
+			  unsigned int commit_msgs_interval,
+			  enum dsync_mailbox_import_flags flags,
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers)
+{
+	struct dsync_mailbox_importer *importer;
+	struct mailbox_status status;
+	pool_t pool;
+
+	pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",
+				     10240);
+	importer = p_new(pool, struct dsync_mailbox_importer, 1);
+	importer->pool = pool;
+	importer->box = box;
+	importer->virtual_all_box = virtual_all_box;
+	importer->last_common_uid = last_common_uid;
+	importer->last_common_modseq = last_common_modseq;
+	importer->last_common_pvt_modseq = last_common_pvt_modseq;
+	importer->last_common_uid_found =
+		last_common_uid != 0 || last_common_modseq != 0;
+	importer->remote_uid_next = remote_uid_next;
+	importer->remote_first_recent_uid = remote_first_recent_uid;
+	importer->remote_highest_modseq = remote_highest_modseq;
+	importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
+	importer->sync_since_timestamp = sync_since_timestamp;
+	importer->sync_until_timestamp = sync_until_timestamp;
+	importer->sync_max_size = sync_max_size;
+	importer->stateful_import = importer->last_common_uid_found;
+	importer->hashed_headers = hashed_headers;
+
+	if (sync_flag != NULL) {
+		if (sync_flag[0] == '-') {
+			importer->sync_flag_dontwant = TRUE;
+			sync_flag++;
+		}
+		if (sync_flag[0] == '\\')
+			importer->sync_flag = imap_parse_system_flag(sync_flag);
+		else
+			importer->sync_keyword = p_strdup(pool, sync_flag);
+	}
+	importer->commit_msgs_interval = commit_msgs_interval;
+	importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC;
+	if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0)
+		importer->transaction_flags |= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY;
+
+	hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
+	hash_table_create_direct(&importer->import_uids, pool, 0);
+	i_array_init(&importer->maybe_expunge_uids, 16);
+	i_array_init(&importer->maybe_saves, 128);
+	i_array_init(&importer->newmails, 128);
+	i_array_init(&importer->wanted_uids, 128);
+	i_array_init(&importer->saved_uids, 128);
+
+	dsync_mailbox_import_transaction_begin(importer);
+
+	if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {
+		i_array_init(&importer->mail_requests, 128);
+		importer->want_mail_requests = TRUE;
+	}
+	importer->master_brain =
+		(flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;
+	importer->revert_local_changes =
+		(flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0;
+	importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0;
+	importer->mails_have_guids =
+		(flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;
+	importer->mails_use_guid128 =
+		(flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
+	importer->hdr_hash_version = hdr_hash_version;
+	importer->empty_hdr_workaround =
+		(flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
+
+	mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
+				STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
+				&status);
+	if (status.nonpermanent_modseqs)
+		status.highest_modseq = 0;
+	importer->local_uid_next = status.uidnext;
+	importer->local_initial_highestmodseq = status.highest_modseq;
+	importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
+	dsync_mailbox_import_search_init(importer);
+
+	if (!importer->stateful_import)
+		;
+	else if (importer->local_uid_next <= last_common_uid) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"local UIDNEXT %u <= last common UID %u",
+			importer->local_uid_next, last_common_uid));
+	} else if (importer->local_initial_highestmodseq < last_common_modseq) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
+			importer->local_initial_highestmodseq,
+			last_common_modseq));
+	} else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
+			importer->local_initial_highestpvtmodseq,
+			last_common_pvt_modseq));
+	}
+
+	importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
+	importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan);
+	return importer;
+}
+
+static int
+dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer,
+				 enum mail_attribute_type type, const char *key,
+				 struct dsync_mailbox_attribute **attr_r)
+{
+	struct dsync_mailbox_attribute lookup_attr, *attr;
+	const struct dsync_mailbox_attribute *attr_change;
+	struct mail_attribute_value value;
+
+	*attr_r = NULL;
+
+	if (mailbox_attribute_get_stream(importer->box, type, key, &value) < 0) {
+		i_error("Mailbox %s: Failed to get attribute %s: %s",
+			mailbox_get_vname(importer->box), key,
+			mailbox_get_last_internal_error(importer->box,
+							&importer->mail_error));
+		importer->failed = TRUE;
+		return -1;
+	}
+
+	lookup_attr.type = type;
+	lookup_attr.key = key;
+
+	attr_change = hash_table_lookup(importer->local_attr_changes,
+					&lookup_attr);
+	if (attr_change == NULL &&
+	    value.value == NULL && value.value_stream == NULL) {
+		/* we have no knowledge of this attribute */
+		return 0;
+	}
+	attr = t_new(struct dsync_mailbox_attribute, 1);
+	attr->type = type;
+	attr->key = key;
+	attr->value = value.value;
+	attr->value_stream = value.value_stream;
+	attr->last_change = value.last_change;
+	if (attr_change != NULL) {
+		attr->deleted = attr_change->deleted &&
+			!DSYNC_ATTR_HAS_VALUE(attr);
+		attr->modseq = attr_change->modseq;
+	}
+	*attr_r = attr;
+	return 0;
+}
+
+static int
+dsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r)
+{
+	const unsigned char *data1, *data2;
+	size_t size1, size2, size;
+
+	*cmp_r = -1; /* quiet gcc */
+
+	for (;;) {
+		(void)i_stream_read_more(input1, &data1, &size1);
+		(void)i_stream_read_more(input2, &data2, &size2);
+
+		if (size1 == 0 || size2 == 0)
+			break;
+		size = I_MIN(size1, size2);
+		*cmp_r = memcmp(data1, data2, size);
+		if (*cmp_r != 0)
+			return 0;
+		i_stream_skip(input1, size);
+		i_stream_skip(input2, size);
+	}
+	if (input1->stream_errno != 0) {
+		i_error("read(%s) failed: %s", i_stream_get_name(input1),
+			i_stream_get_error(input1));
+		return -1;
+	}
+	if (input2->stream_errno != 0) {
+		i_error("read(%s) failed: %s", i_stream_get_name(input2),
+			i_stream_get_error(input2));
+		return -1;
+	}
+	if (size1 == 0 && size2 == 0)
+		*cmp_r = 0;
+	else
+		*cmp_r = size1 == 0 ? -1 : 1;
+	return 0;
+}
+
+static int
+dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1,
+			    const struct dsync_mailbox_attribute *attr2,
+			    int *cmp_r)
+{
+	struct istream *input1, *input2;
+	int ret;
+
+	i_assert(attr1->value_stream != NULL || attr1->value != NULL);
+	i_assert(attr2->value_stream != NULL || attr2->value != NULL);
+
+	if (attr1->value != NULL && attr2->value != NULL) {
+		*cmp_r = strcmp(attr1->value, attr2->value);
+		return 0;
+	}
+	/* at least one of them is a stream. make both of them streams. */
+	input1 = attr1->value_stream != NULL ? attr1->value_stream :
+		i_stream_create_from_data(attr1->value, strlen(attr1->value));
+	input2 = attr2->value_stream != NULL ? attr2->value_stream :
+		i_stream_create_from_data(attr2->value, strlen(attr2->value));
+	i_stream_seek(input1, 0);
+	i_stream_seek(input2, 0);
+	ret = dsync_istreams_cmp(input1, input2, cmp_r);
+	if (attr1->value_stream == NULL)
+		i_stream_unref(&input1);
+	if (attr2->value_stream == NULL)
+		i_stream_unref(&input2);
+	return ret;
+}
+
+static int
+dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr,
+		     const struct dsync_mailbox_attribute *local_attr,
+		     int *cmp_r)
+{
+	if (DSYNC_ATTR_HAS_VALUE(attr) &&
+	    !DSYNC_ATTR_HAS_VALUE(local_attr)) {
+		/* remote has a value and local doesn't -> use it */
+		*cmp_r = 1;
+		return 0;
+	} else if (!DSYNC_ATTR_HAS_VALUE(attr) &&
+		   DSYNC_ATTR_HAS_VALUE(local_attr)) {
+		/* remote doesn't have a value, bt local does -> skip */
+		*cmp_r = -1;
+		return 0;
+	}
+
+	return dsync_attributes_cmp_values(attr, local_attr, cmp_r);
+}
+
+static int
+dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer,
+				    const struct dsync_mailbox_attribute *attr,
+				    const struct dsync_mailbox_attribute *local_attr,
+				    const char **result_r)
+{
+	struct mail_attribute_value value;
+	int cmp;
+	bool ignore = FALSE;
+
+	i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted);
+
+	if (attr->deleted &&
+	    (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) {
+		/* attribute doesn't exist on either side -> ignore */
+		*result_r = "Nonexistent in both sides";
+		return 0;
+	}
+	if (local_attr == NULL) {
+		/* we haven't seen this locally -> use whatever remote has */
+		*result_r = "Nonexistent locally";
+	} else if (local_attr->modseq <= importer->last_common_modseq &&
+		   attr->modseq > importer->last_common_modseq &&
+		   importer->last_common_modseq > 0) {
+		/* we're doing incremental syncing, and we can see that the
+		   attribute was changed remotely, but not locally -> use it */
+		*result_r = "Changed remotely";
+	} else if (local_attr->modseq > importer->last_common_modseq &&
+		   attr->modseq <= importer->last_common_modseq &&
+		   importer->last_common_modseq > 0) {
+		/* we're doing incremental syncing, and we can see that the
+		   attribute was changed locally, but not remotely -> ignore */
+		*result_r = "Changed locally";
+		ignore = TRUE;
+	} else if (attr->last_change > local_attr->last_change) {
+		/* remote has a newer timestamp -> use it */
+		*result_r = "Remote has newer timestamp";
+	} else if (attr->last_change < local_attr->last_change) {
+		/* remote has an older timestamp -> ignore */
+		*result_r = "Local has newer timestamp";
+		ignore = TRUE;
+	} else {
+		/* the timestamps are the same. now we're down to guessing
+		   the right answer, unless the values are actually equal,
+		   so check that first. next try to use modseqs, but if even
+		   they are the same, fallback to just picking one based on the
+		   value. */
+		if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) {
+			importer->mail_error = MAIL_ERROR_TEMP;
+			importer->failed = TRUE;
+			return -1;
+		}
+		if (cmp == 0) {
+			/* identical scripts */
+			*result_r = "Unchanged value";
+			return 0;
+		}
+
+		if (attr->modseq > local_attr->modseq) {
+			/* remote has a higher modseq -> use it */
+			*result_r = "Remote has newer modseq";
+		} else if (attr->modseq < local_attr->modseq) {
+			/* remote has an older modseq -> ignore */
+			*result_r = "Local has newer modseq";
+			ignore = TRUE;
+		} else if (cmp < 0) {
+			ignore = TRUE;
+			*result_r = "Value changed, but unknown which is newer - picking local";
+		} else {
+			*result_r = "Value changed, but unknown which is newer - picking remote";
+		}
+	}
+	if (ignore)
+		return 0;
+
+	i_zero(&value);
+	value.value = attr->value;
+	value.value_stream = attr->value_stream;
+	value.last_change = attr->last_change;
+	if (mailbox_attribute_set(importer->trans, attr->type,
+				  attr->key, &value) < 0) {
+		i_error("Mailbox %s: Failed to set attribute %s: %s",
+			mailbox_get_vname(importer->box), attr->key,
+			mailbox_get_last_internal_error(importer->box, NULL));
+		/* the attributes aren't vital, don't fail everything just
+		   because of them. */
+	}
+	return 0;
+}
+
+int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
+				   const struct dsync_mailbox_attribute *attr)
+{
+	struct dsync_mailbox_attribute *local_attr;
+	const char *result = "";
+	int ret;
+
+	if (dsync_mailbox_import_lookup_attr(importer, attr->type,
+					     attr->key, &local_attr) < 0)
+		ret = -1;
+	else {
+		ret = dsync_mailbox_import_attribute_real(importer, attr,
+							  local_attr, &result);
+		if (local_attr != NULL && local_attr->value_stream != NULL)
+			i_stream_unref(&local_attr->value_stream);
+	}
+	imp_debug(importer, "Import attribute %s: %s", attr->key,
+		  ret < 0 ? "failed" : result);
+	return ret;
+}
+
+static void dsync_mail_error(struct dsync_mailbox_importer *importer,
+			     struct mail *mail, const char *field)
+{
+	const char *errstr;
+	enum mail_error error;
+
+	errstr = mailbox_get_last_internal_error(importer->box, &error);
+	if (error == MAIL_ERROR_EXPUNGED)
+		return;
+
+	i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
+		mailbox_get_vname(mail->box), field, mail->uid, errstr);
+	importer->mail_error = error;
+	importer->failed = TRUE;
+}
+
+static bool
+dsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer,
+			      const struct dsync_mail_change *change,
+			      const char *guid, const char **cmp_guid_r)
+{
+	guid_128_t guid_128, change_guid_128;
+
+	if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+		if (guid_128_from_string(change->guid, change_guid_128) < 0)
+			i_unreached();
+	} else if (importer->mails_use_guid128) {
+		mail_generate_guid_128_hash(change->guid, change_guid_128);
+	} else {
+		if (cmp_guid_r != NULL)
+			*cmp_guid_r = change->guid;
+		return strcmp(change->guid, guid) == 0;
+	}
+
+	mail_generate_guid_128_hash(guid, guid_128);
+	if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {
+		if (cmp_guid_r != NULL) {
+			*cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)",
+				binary_to_hex(change_guid_128, sizeof(change_guid_128)),
+				change->guid);
+		}
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static int
+importer_try_next_mail(struct dsync_mailbox_importer *importer,
+		       uint32_t wanted_uid)
+{
+	struct mail_private *pmail;
+	const char *hdr_hash;
+
+	if (importer->cur_mail == NULL) {
+		/* end of search */
+		return -1;
+	}
+	while (importer->cur_mail->seq < importer->next_local_seq ||
+	       importer->cur_mail->uid < wanted_uid) {
+		if (!importer->cur_uid_has_change &&
+		    !importer->last_common_uid_found) {
+			/* this message exists locally, but remote didn't send
+			   expunge-change for it. if the message's
+			   uid <= last-common-uid, it should be deleted */
+			seq_range_array_add(&importer->maybe_expunge_uids, 
+					    importer->cur_mail->uid);
+		}
+
+		importer->cur_mail_skip = FALSE;
+		if (!mailbox_search_next(importer->search_ctx,
+					 &importer->cur_mail)) {
+			importer->cur_mail = NULL;
+			importer->cur_guid = NULL;
+			importer->cur_hdr_hash = NULL;
+			return -1;
+		}
+		importer->cur_uid_has_change = FALSE;
+	}
+	importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid;
+	if (importer->mails_have_guids) {
+		if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,
+				     &importer->cur_guid) < 0) {
+			dsync_mail_error(importer, importer->cur_mail, "GUID");
+			return 0;
+		}
+	} else {
+		if (dsync_mail_get_hdr_hash(importer->cur_mail,
+					    importer->hdr_hash_version,
+					    importer->hashed_headers,
+					    &hdr_hash) < 0) {
+			dsync_mail_error(importer, importer->cur_mail,
+					 "header hash");
+			return 0;
+		} 
+		pmail = (struct mail_private *)importer->cur_mail;
+		importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash);
+		importer->cur_guid = "";
+	}
+	/* make sure next_local_seq gets updated in case we came here
+	   because of min_uid */
+	importer->next_local_seq = importer->cur_mail->seq;
+	return 1;
+}
+
+static bool
+importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)
+{
+	int ret;
+
+	for (;;) {
+		T_BEGIN {
+			ret = importer_try_next_mail(importer, wanted_uid);
+		} T_END;
+		if (ret != 0 || importer->failed)
+			break;
+		importer->next_local_seq = importer->cur_mail->seq + 1;
+	}
+	return ret > 0;
+}
+
+static int
+importer_mail_cmp(const struct importer_mail *m1,
+		  const struct importer_mail *m2)
+{
+	int ret;
+
+	if (m1->guid == NULL)
+		return 1;
+	if (m2->guid == NULL)
+		return -1;
+
+	ret = strcmp(m1->guid, m2->guid);
+	if (ret != 0)
+		return ret;
+
+	if (m1->uid < m2->uid)
+		return -1;
+	if (m1->uid > m2->uid)
+		return 1;
+	return 0;
+}
+
+static void newmail_link(struct dsync_mailbox_importer *importer,
+			 struct importer_new_mail *newmail, uint32_t remote_uid)
+{
+	struct importer_new_mail *first_mail, **last, *mail, *link = NULL;
+
+	if (*newmail->guid != '\0') {
+		first_mail = hash_table_lookup(importer->import_guids,
+					       newmail->guid);
+		if (first_mail == NULL) {
+			/* first mail for this GUID */
+			hash_table_insert(importer->import_guids,
+					  newmail->guid, newmail);
+			return;
+		}
+	} else {
+		if (remote_uid == 0) {
+			/* mail exists locally. we don't want to request
+			   it, and we'll assume it has no duplicate
+			   instances. */
+			return;
+		}
+		first_mail = hash_table_lookup(importer->import_uids,
+					       POINTER_CAST(remote_uid));
+		if (first_mail == NULL) {
+			/* first mail for this UID */
+			hash_table_insert(importer->import_uids,
+					  POINTER_CAST(remote_uid), newmail);
+			return;
+		}
+	}
+	/* 1) add the newmail to the end of the linked list
+	   2) find our link
+
+	   FIXME: this loop is slow if the same GUID has a ton of instances.
+	   Could it be improved in some way? */
+	last = &first_mail->next;
+	for (mail = first_mail; mail != NULL; mail = mail->next) {
+		if (mail->final_uid == newmail->final_uid)
+			mail->uid_is_usable = TRUE;
+		if (link == NULL && mail->link == NULL &&
+		    mail->uid_in_local != newmail->uid_in_local)
+			link = mail;
+		last = &mail->next;
+	}
+	*last = newmail;
+	if (link != NULL && newmail->link == NULL) {
+		link->link = newmail;
+		newmail->link = link;
+	}
+}
+
+static void
+dsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer,
+				  uint32_t uid, const char *reason)
+{
+	i_assert(importer->revert_local_changes);
+
+	/* UID either already exists or UIDNEXT is too high. we can't set the
+	   wanted UID, so we'll need to delete the whole mailbox and resync */
+	i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s",
+		  mailbox_get_vname(importer->box), uid, reason);
+	importer->delete_mailbox = TRUE;
+	importer->mail_error = MAIL_ERROR_TEMP;
+	importer->failed = TRUE;
+}
+
+static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
+				       struct dsync_mail_change *save_change)
+{
+	struct importer_mail m1, m2;
+	struct importer_new_mail *newmail;
+	int diff;
+	bool remote_saved;
+
+	i_zero(&m1);
+	if (importer->cur_mail != NULL) {
+		m1.guid = importer->mails_have_guids ?
+			importer->cur_guid : importer->cur_hdr_hash;
+		m1.uid = importer->cur_mail->uid;
+	}
+	i_zero(&m2);
+	if (save_change != NULL) {
+		m2.guid = importer->mails_have_guids ?
+			save_change->guid : save_change->hdr_hash;
+		m2.uid = save_change->uid;
+		i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+	}
+
+	if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
+	    importer->cur_mail != NULL && save_change != NULL &&
+	    (dsync_mail_hdr_hash_is_empty(m1.guid) ||
+	     dsync_mail_hdr_hash_is_empty(m2.guid))) {
+		/* one of the headers is empty. assume it's broken and that
+		   the header matches what we have currently. */
+		diff = 0;
+	} else {
+		diff = importer_mail_cmp(&m1, &m2);
+	}
+	if (diff < 0) {
+		/* add a record for local mail */
+		i_assert(importer->cur_mail != NULL);
+		if (importer->revert_local_changes) {
+			if (save_change == NULL &&
+			    importer->cur_mail->uid >= importer->remote_uid_next) {
+				dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid,
+					t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
+				return TRUE;
+			}
+			mail_expunge(importer->cur_mail);
+			importer->cur_mail_skip = TRUE;
+			importer->next_local_seq++;
+			return FALSE;
+		}
+		newmail = p_new(importer->pool, struct importer_new_mail, 1);
+		newmail->guid = p_strdup(importer->pool, importer->cur_guid);
+		newmail->final_uid = importer->cur_mail->uid;
+		newmail->local_uid = importer->cur_mail->uid;
+		newmail->uid_in_local = TRUE;
+		newmail->uid_is_usable =
+			newmail->final_uid >= importer->remote_uid_next;
+		remote_saved = FALSE;
+	} else if (diff > 0) {
+		i_assert(save_change != NULL);
+		newmail = p_new(importer->pool, struct importer_new_mail, 1);
+		newmail->guid = save_change->guid;
+		newmail->final_uid = save_change->uid;
+		newmail->remote_uid = save_change->uid;
+		newmail->uid_in_local = FALSE;
+		newmail->uid_is_usable =
+			newmail->final_uid >= importer->local_uid_next;
+		if (!newmail->uid_is_usable && importer->revert_local_changes) {
+			dsync_mailbox_revert_existing_uid(importer, newmail->final_uid,
+				t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next));
+			return TRUE;
+		}
+		remote_saved = TRUE;
+	} else {
+		/* identical */
+		i_assert(importer->cur_mail != NULL);
+		i_assert(save_change != NULL);
+		newmail = p_new(importer->pool, struct importer_new_mail, 1);
+		newmail->guid = save_change->guid;
+		newmail->final_uid = importer->cur_mail->uid;
+		newmail->local_uid = importer->cur_mail->uid;
+		newmail->remote_uid = save_change->uid;
+		newmail->uid_in_local = TRUE;
+		newmail->uid_is_usable = TRUE;
+		newmail->link = newmail;
+		remote_saved = TRUE;
+	}
+
+	if (newmail->uid_in_local) {
+		importer->cur_mail_skip = TRUE;
+		importer->next_local_seq++;
+	}
+	/* NOTE: assumes save_change is allocated from importer pool */
+	newmail->change = save_change;
+
+	array_push_back(&importer->newmails, &newmail);
+	if (newmail->uid_in_local)
+		newmail_link(importer, newmail, 0);
+	else {
+		i_assert(save_change != NULL);
+		newmail_link(importer, newmail, save_change->uid);
+	}
+	return remote_saved;
+}
+
+static bool ATTR_NULL(2)
+dsync_mailbox_try_save(struct dsync_mailbox_importer *importer,
+		       struct dsync_mail_change *save_change)
+{
+	if (importer->cur_mail_skip) {
+		if (!importer_next_mail(importer, 0) && save_change == NULL)
+			return FALSE;
+	}
+	return dsync_mailbox_try_save_cur(importer, save_change);
+}
+
+static void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
+			       struct dsync_mail_change *save_change)
+{
+	while (!dsync_mailbox_try_save(importer, save_change)) ;
+}
+
+static bool
+dsync_import_set_mail(struct dsync_mailbox_importer *importer,
+		      const struct dsync_mail_change *change)
+{
+	const char *guid, *cmp_guid;
+
+	if (!mail_set_uid(importer->mail, change->uid))
+		return FALSE;
+	if (change->guid == NULL) {
+		/* GUID is unknown */
+		return TRUE;
+	}
+	if (*change->guid == '\0') {
+		/* backend doesn't support GUIDs. if hdr_hash is set, we could
+		   verify it, but since this message really is supposed to
+		   match, it's probably too much trouble. */
+		return TRUE;
+	}
+
+	/* verify that GUID matches, just in case */
+	if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {
+		dsync_mail_error(importer, importer->mail, "GUID");
+		return FALSE;
+	}
+	if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"Unexpected GUID mismatch for UID=%u: %s != %s",
+			change->uid, guid, cmp_guid));
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
+				 const struct dsync_mail_change *change)
+{
+	const char *cmp_guid;
+
+	if (change->guid == NULL || change->guid[0] == '\0' ||
+	    importer->cur_guid[0] == '\0')
+		return TRUE;
+	if (!dsync_mail_change_guid_equals(importer, change,
+					   importer->cur_guid, &cmp_guid)) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"Unexpected GUID mismatch (2) for UID=%u: %s != %s",
+			change->uid, importer->cur_guid, cmp_guid));
+		return FALSE;
+	}
+	return TRUE;
+}
+
+static void
+merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,
+	    uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,
+	    uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote,
+	    uint32_t *change_add_r, uint32_t *change_remove_r,
+	    bool *remote_changed, bool *remote_pvt_changed)
+{
+	uint32_t combined_add, combined_remove, conflict_flags;
+	uint32_t local_wanted, remote_wanted, conflict_pvt_flags;
+
+	/* resolve conflicts */
+	conflict_flags = local_add & remote_remove;
+	if (conflict_flags != 0) {
+		conflict_pvt_flags = conflict_flags & pvt_mask;
+		conflict_flags &= ~pvt_mask;
+		if (prefer_remote)
+			local_add &= ~conflict_flags;
+		else
+			remote_remove &= ~conflict_flags;
+		if (prefer_pvt_remote)
+			local_add &= ~conflict_pvt_flags;
+		else
+			remote_remove &= ~conflict_pvt_flags;
+	}
+	conflict_flags = local_remove & remote_add;
+	if (conflict_flags != 0) {
+		conflict_pvt_flags = conflict_flags & pvt_mask;
+		conflict_flags &= ~pvt_mask;
+		if (prefer_remote)
+			local_remove &= ~conflict_flags;
+		else
+			remote_add &= ~conflict_flags;
+		if (prefer_pvt_remote)
+			local_remove &= ~conflict_pvt_flags;
+		else
+			remote_add &= ~conflict_pvt_flags;
+	}
+	
+	combined_add = local_add|remote_add;
+	combined_remove = local_remove|remote_remove;
+	i_assert((combined_add & combined_remove) == 0);
+
+	/* don't change flags that are currently identical in both sides */
+	conflict_flags = local_final ^ remote_final;
+	combined_add &= conflict_flags;
+	combined_remove &= conflict_flags;
+
+	/* see if there are conflicting final flags */
+	local_wanted = (local_final|combined_add) & ~combined_remove;
+	remote_wanted = (remote_final|combined_add) & ~combined_remove;
+
+	conflict_flags = local_wanted ^ remote_wanted;
+	if (conflict_flags != 0) {
+		if (prefer_remote && prefer_pvt_remote)
+			local_wanted = remote_wanted;
+		else if (prefer_remote && !prefer_pvt_remote) {
+			local_wanted = (local_wanted & pvt_mask) |
+				(remote_wanted & ~pvt_mask);
+		} else if (!prefer_remote && prefer_pvt_remote) {
+			local_wanted = (local_wanted & ~pvt_mask) |
+				(remote_wanted & pvt_mask);
+		}
+	}
+
+	*change_add_r = local_wanted & ~local_final;
+	*change_remove_r = local_final & ~local_wanted;
+	if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask))
+		*remote_changed = TRUE;
+	if ((local_wanted & pvt_mask) != (remote_final & pvt_mask))
+		*remote_pvt_changed = TRUE;
+}
+
+static bool
+keyword_find(ARRAY_TYPE(const_string) *keywords, const char *name,
+	     unsigned int *idx_r)
+{
+	const char *const *names;
+	unsigned int i, count;
+
+	names = array_get(keywords, &count);
+	for (i = 0; i < count; i++) {
+		if (strcmp(names[i], name) == 0) {
+			*idx_r = i;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void keywords_append(ARRAY_TYPE(const_string) *dest,
+			    const ARRAY_TYPE(const_string) *keywords,
+			    uint32_t bits, unsigned int start_idx)
+{
+	const char *name;
+	unsigned int i;
+
+	for (i = 0; i < 32; i++) {
+		if ((bits & (1U << i)) == 0)
+			continue;
+
+		name = array_idx_elem(keywords, start_idx+i);
+		array_push_back(dest, &name);
+	}
+}
+
+static void
+merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes,
+	       const ARRAY_TYPE(const_string) *remote_changes,
+	       bool prefer_remote,
+	       bool *remote_changed, bool *remote_pvt_changed)
+{
+	/* local_changes and remote_changes are assumed to have no
+	   duplicates names */
+	uint32_t *local_add, *local_remove, *local_final;
+	uint32_t *remote_add, *remote_remove, *remote_final;
+	uint32_t *change_add, *change_remove;
+	ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;
+	const char *const *changes, *name, *const *local_keywords;
+	struct mail_keywords *kw;
+	unsigned int i, count, name_idx, array_size;
+
+	local_keywords = mail_get_keywords(mail);
+
+	/* we'll assign a common index for each keyword name and place
+	   the changes to separate bit arrays. */
+	if (array_is_created(remote_changes))
+		changes = array_get(remote_changes, &count);
+	else {
+		changes = NULL;
+		count = 0;
+	}
+
+	array_size = str_array_length(local_keywords) + count;
+	if (array_is_created(local_changes))
+		array_size += array_count(local_changes);
+	if (array_size == 0) {
+		/* this message has no keywords */
+		return;
+	}
+	t_array_init(&all_keywords, array_size);
+	t_array_init(&add_keywords, array_size);
+	t_array_init(&remove_keywords, array_size);
+
+	/* @UNSAFE: create large enough arrays to fit all keyword indexes. */
+	array_size = (array_size+31)/32;
+	local_add = t_new(uint32_t, array_size);
+	local_remove = t_new(uint32_t, array_size);
+	local_final = t_new(uint32_t, array_size);
+	remote_add = t_new(uint32_t, array_size);
+	remote_remove = t_new(uint32_t, array_size);
+	remote_final = t_new(uint32_t, array_size);
+	change_add = t_new(uint32_t, array_size);
+	change_remove = t_new(uint32_t, array_size);
+
+	/* get remote changes */
+	for (i = 0; i < count; i++) {
+		name = changes[i]+1;
+		name_idx = array_count(&all_keywords);
+		array_push_back(&all_keywords, &name);
+
+		switch (changes[i][0]) {
+		case KEYWORD_CHANGE_ADD:
+			remote_add[name_idx/32] |= 1U << (name_idx%32);
+			break;
+		case KEYWORD_CHANGE_REMOVE:
+			remote_remove[name_idx/32] |= 1U << (name_idx%32);
+			break;
+		case KEYWORD_CHANGE_FINAL:
+			remote_final[name_idx/32] |= 1U << (name_idx%32);
+			break;
+		case KEYWORD_CHANGE_ADD_AND_FINAL:
+			remote_add[name_idx/32] |= 1U << (name_idx%32);
+			remote_final[name_idx/32] |= 1U << (name_idx%32);
+			break;
+		}
+	}
+
+	/* get local changes. use existing indexes for names when they exist. */
+	if (array_is_created(local_changes))
+		changes = array_get(local_changes, &count);
+	else {
+		changes = NULL;
+		count = 0;
+	}
+	for (i = 0; i < count; i++) {
+		name = changes[i]+1;
+		if (!keyword_find(&all_keywords, name, &name_idx)) {
+			name_idx = array_count(&all_keywords);
+			array_push_back(&all_keywords, &name);
+		}
+
+		switch (changes[i][0]) {
+		case KEYWORD_CHANGE_ADD:
+		case KEYWORD_CHANGE_ADD_AND_FINAL:
+			local_add[name_idx/32] |= 1U << (name_idx%32);
+			break;
+		case KEYWORD_CHANGE_REMOVE:
+			local_remove[name_idx/32] |= 1U << (name_idx%32);
+			break;
+		case KEYWORD_CHANGE_FINAL:
+			break;
+		}
+	}
+	for (i = 0; local_keywords[i] != NULL; i++) {
+		name = local_keywords[i];
+		if (!keyword_find(&all_keywords, name, &name_idx)) {
+			name_idx = array_count(&all_keywords);
+			array_push_back(&all_keywords, &name);
+		}
+		local_final[name_idx/32] |= 1U << (name_idx%32);
+	}
+	i_assert(array_count(&all_keywords) <= array_size*32);
+	array_size = (array_count(&all_keywords)+31) / 32;
+
+	/* merge keywords */
+	for (i = 0; i < array_size; i++) {
+		merge_flags(local_final[i], local_add[i], local_remove[i],
+			    remote_final[i], remote_add[i], remote_remove[i],
+			    0, prefer_remote, prefer_remote,
+			    &change_add[i], &change_remove[i],
+			    remote_changed, remote_pvt_changed);
+		if (change_add[i] != 0) {
+			keywords_append(&add_keywords, &all_keywords,
+					change_add[i], i*32);
+		}
+		if (change_remove[i] != 0) {
+			keywords_append(&remove_keywords, &all_keywords,
+					change_remove[i], i*32);
+		}
+	}
+
+	/* apply changes */
+	if (array_count(&add_keywords) > 0) {
+		array_append_zero(&add_keywords);
+		kw = mailbox_keywords_create_valid(mail->box,
+			array_front(&add_keywords));
+		mail_update_keywords(mail, MODIFY_ADD, kw);
+		mailbox_keywords_unref(&kw);
+	}
+	if (array_count(&remove_keywords) > 0) {
+		array_append_zero(&remove_keywords);
+		kw = mailbox_keywords_create_valid(mail->box,
+			array_front(&remove_keywords));
+		mail_update_keywords(mail, MODIFY_REMOVE, kw);
+		mailbox_keywords_unref(&kw);
+	}
+}
+
+static void
+dsync_mailbox_import_replace_flags(struct mail *mail,
+				   const struct dsync_mail_change *change)
+{
+	ARRAY_TYPE(const_string) keywords;
+	struct mail_keywords *kw;
+	const char *const *changes, *name;
+	unsigned int i, count;
+
+	if (array_is_created(&change->keyword_changes))
+		changes = array_get(&change->keyword_changes, &count);
+	else {
+		changes = NULL;
+		count = 0;
+	}
+	t_array_init(&keywords, count+1);
+	for (i = 0; i < count; i++) {
+		switch (changes[i][0]) {
+		case KEYWORD_CHANGE_ADD:
+		case KEYWORD_CHANGE_FINAL:
+		case KEYWORD_CHANGE_ADD_AND_FINAL:
+			name = changes[i]+1;
+			array_push_back(&keywords, &name);
+			break;
+		case KEYWORD_CHANGE_REMOVE:
+			break;
+		}
+	}
+	array_append_zero(&keywords);
+
+	kw = mailbox_keywords_create_valid(mail->box, array_front(&keywords));
+	mail_update_keywords(mail, MODIFY_REPLACE, kw);
+	mailbox_keywords_unref(&kw);
+
+	mail_update_flags(mail, MODIFY_REPLACE,
+			  change->add_flags | change->final_flags);
+	if (mail_get_modseq(mail) < change->modseq)
+		mail_update_modseq(mail, change->modseq);
+	if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
+		mail_update_pvt_modseq(mail, change->pvt_modseq);
+}
+
+static void
+dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,
+				 const struct dsync_mail_change *change)
+{
+	const struct dsync_mail_change *local_change;
+	enum mail_flags local_add, local_remove;
+	uint32_t change_add, change_remove;
+	uint64_t new_modseq;
+	ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;
+	struct mail *mail;
+	bool prefer_remote, prefer_pvt_remote;
+	bool remote_changed = FALSE, remote_pvt_changed = FALSE;
+
+	i_assert((change->add_flags & change->remove_flags) == 0);
+
+	if (importer->cur_mail != NULL &&
+	    importer->cur_mail->uid == change->uid) {
+		if (!dsync_check_cur_guid(importer, change))
+			return;
+		mail = importer->cur_mail;
+	} else {
+		if (!dsync_import_set_mail(importer, change))
+			return;
+		mail = importer->mail;
+	}
+
+	if (importer->revert_local_changes) {
+		/* dsync backup: just make the local look like remote. */
+		dsync_mailbox_import_replace_flags(mail, change);
+		return;
+	}
+
+	local_change = hash_table_lookup(importer->local_changes,
+					 POINTER_CAST(change->uid));
+	if (local_change == NULL) {
+		local_add = local_remove = 0;
+	} else {
+		local_add = local_change->add_flags;
+		local_remove = local_change->remove_flags;
+		local_keyword_changes = local_change->keyword_changes;
+	}
+
+	if (mail_get_modseq(mail) < change->modseq)
+		prefer_remote = TRUE;
+	else if (mail_get_modseq(mail) > change->modseq)
+		prefer_remote = FALSE;
+	else {
+		/* identical modseq, we'll just have to pick one.
+		   Note that both brains need to pick the same one, otherwise
+		   they become unsynced. */
+		prefer_remote = !importer->master_brain;
+	}
+	if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
+		prefer_pvt_remote = TRUE;
+	else if (mail_get_pvt_modseq(mail) > change->pvt_modseq)
+		prefer_pvt_remote = FALSE;
+	else
+		prefer_pvt_remote = !importer->master_brain;
+
+	/* merge flags */
+	merge_flags(mail_get_flags(mail), local_add, local_remove,
+		    change->final_flags, change->add_flags, change->remove_flags,
+		    mailbox_get_private_flags_mask(mail->box),
+		    prefer_remote, prefer_pvt_remote,
+		    &change_add, &change_remove,
+		    &remote_changed, &remote_pvt_changed);
+
+	if (change_add != 0)
+		mail_update_flags(mail, MODIFY_ADD, change_add);
+	if (change_remove != 0)
+		mail_update_flags(mail, MODIFY_REMOVE, change_remove);
+
+	/* merge keywords */
+	merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,
+		       prefer_remote, &remote_changed, &remote_pvt_changed);
+
+	/* update modseqs. try to anticipate when we have to increase modseq
+	   to get it closer to what remote has (although we can't guess it
+	   exactly correctly) */
+	new_modseq = change->modseq;
+	if (remote_changed && new_modseq <= importer->remote_highest_modseq)
+		new_modseq = importer->remote_highest_modseq+1;
+	if (mail_get_modseq(mail) < new_modseq)
+		mail_update_modseq(mail, new_modseq);
+
+	new_modseq = change->pvt_modseq;
+	if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq)
+		new_modseq = importer->remote_highest_pvt_modseq+1;
+	if (mail_get_pvt_modseq(mail) < new_modseq)
+		mail_update_pvt_modseq(mail, new_modseq);
+}
+
+static bool
+dsync_mail_change_have_keyword(const struct dsync_mail_change *change,
+			       const char *keyword)
+{
+	const char *str;
+
+	if (!array_is_created(&change->keyword_changes))
+		return FALSE;
+
+	array_foreach_elem(&change->keyword_changes, str) {
+		switch (str[0]) {
+		case KEYWORD_CHANGE_FINAL:
+		case KEYWORD_CHANGE_ADD_AND_FINAL:
+			if (strcasecmp(str+1, keyword) == 0)
+				return TRUE;
+			break;
+		default:
+			break;
+		}
+	}
+	return FALSE;
+}
+
+static bool
+dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
+				 const struct dsync_mail_change *change,
+				 const char **result_r)
+{
+	if (importer->sync_since_timestamp > 0) {
+		i_assert(change->received_timestamp > 0);
+		if (change->received_timestamp < importer->sync_since_timestamp) {
+			/* mail has too old timestamp - skip it */
+			*result_r = "Ignoring missing local mail with too old timestamp";
+			return FALSE;
+		}
+	}
+	if (importer->sync_until_timestamp > 0) {
+		i_assert(change->received_timestamp > 0);
+		if (change->received_timestamp > importer->sync_until_timestamp) {
+			/* mail has too new timestamp - skip it */
+			*result_r = "Ignoring missing local mail with too new timestamp";
+			return FALSE;
+		}
+	}
+	if (importer->sync_max_size > 0) {
+		i_assert(change->virtual_size != UOFF_T_MAX);
+		if (change->virtual_size > importer->sync_max_size) {
+			/* mail is too large - skip it */
+			*result_r = "Ignoring missing local mail with too large size";
+			return FALSE;
+		}
+	}
+	if (importer->sync_flag != 0) {
+		bool have_flag = (change->final_flags & importer->sync_flag) != 0;
+
+		if (have_flag && importer->sync_flag_dontwant) {
+			*result_r = "Ignoring missing local mail that doesn't have wanted flags";
+			return FALSE;
+		}
+		if (!have_flag && !importer->sync_flag_dontwant) {
+			*result_r = "Ignoring missing local mail that has unwanted flags";
+			return FALSE;
+		}
+	}
+	if (importer->sync_keyword != NULL) {
+		bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword);
+
+		if (have_kw && importer->sync_flag_dontwant) {
+			*result_r = "Ignoring missing local mail that doesn't have wanted keywords";
+			return FALSE;
+		}
+		if (!have_kw && !importer->sync_flag_dontwant) {
+			*result_r = "Ignoring missing local mail that has unwanted keywords";
+			return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+static void
+dsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
+			  const struct dsync_mail_change *change)
+{
+	struct dsync_mail_change *save;
+	const char *result;
+
+	i_assert(change->guid != NULL);
+
+	if (change->uid == importer->last_common_uid) {
+		/* we've already verified that the GUID matches.
+		   apply flag changes if there are any. */
+		i_assert(!importer->last_common_uid_found);
+		dsync_mailbox_import_flag_change(importer, change);
+		return;
+	}
+	if (!dsync_mailbox_import_want_change(importer, change, &result))
+		return;
+
+	save = p_new(importer->pool, struct dsync_mail_change, 1);
+	dsync_mail_change_dup(importer->pool, change, save);
+
+	if (importer->last_common_uid_found) {
+		/* this is a new mail. its UID may or may not conflict with
+		   an existing local mail, we'll figure it out later. */
+		i_assert(change->uid > importer->last_common_uid);
+		dsync_mailbox_save(importer, save);
+	} else {
+		/* the local mail is expunged. we'll decide later if we want
+		   to save this mail locally or expunge it form remote. */
+		i_assert(change->uid > importer->last_common_uid);
+		i_assert(importer->cur_mail == NULL ||
+			 change->uid < importer->cur_mail->uid);
+		array_push_back(&importer->maybe_saves, &save);
+	}
+}
+
+static void
+dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,
+			     const struct dsync_mail_change *change)
+{
+
+	if (importer->last_common_uid_found) {
+		/* expunge the message, unless its GUID unexpectedly doesn't
+		   match */
+		i_assert(change->uid <= importer->last_common_uid);
+		if (dsync_import_set_mail(importer, change))
+			mail_expunge(importer->mail);
+	} else if (importer->cur_mail == NULL ||
+		   change->uid < importer->cur_mail->uid) {
+		/* already expunged locally, we can ignore this.
+		   uid=last_common_uid if we managed to verify from
+		   transaction log that the GUIDs match */
+		i_assert(change->uid >= importer->last_common_uid);
+	} else if (change->uid == importer->last_common_uid) {
+		/* already verified that the GUID matches */
+		i_assert(importer->cur_mail->uid == change->uid);
+		if (dsync_check_cur_guid(importer, change))
+			mail_expunge(importer->cur_mail);
+	} else {
+		/* we don't know yet if we should expunge this
+		   message or not. queue it until we do. */
+		i_assert(change->uid > importer->last_common_uid);
+		seq_range_array_add(&importer->maybe_expunge_uids, change->uid);
+	}
+}
+
+static void
+dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)
+{
+	/* If there are local mails after last_common_uid which we skipped
+	   while trying to match the next message, we need to now go back */
+	if (importer->cur_mail != NULL &&
+	    importer->cur_mail->uid <= importer->last_common_uid+1)
+		return;
+
+	importer->cur_mail = NULL;
+	importer->cur_guid = NULL;
+	importer->cur_hdr_hash = NULL;
+	importer->next_local_seq = 0;
+
+	(void)mailbox_search_deinit(&importer->search_ctx);
+	dsync_mailbox_import_search_init(importer);
+}
+
+static void
+dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)
+{
+	struct dsync_mail_change *const *saves;
+	struct seq_range_iter iter;
+	unsigned int n, i, count;
+	uint32_t uid;
+
+	if (importer->debug) T_BEGIN {
+		string_t *expunges = t_str_new(64);
+
+		imap_write_seq_range(expunges, &importer->maybe_expunge_uids);
+		imp_debug(importer, "Last common UID=%u. Delayed expunges=%s",
+			  importer->last_common_uid, str_c(expunges));
+	} T_END;
+
+	importer->last_common_uid_found = TRUE;
+	dsync_mailbox_rewind_search(importer);
+
+	/* expunge the messages whose expunge-decision we delayed previously */
+	seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;
+	while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+		if (uid > importer->last_common_uid) {
+			/* we expunge messages only up to last_common_uid,
+			   ignore the rest */
+			break;
+		}
+
+		if (mail_set_uid(importer->mail, uid))
+			mail_expunge(importer->mail);
+	}
+
+	/* handle pending saves */
+	saves = array_get(&importer->maybe_saves, &count);
+	for (i = 0; i < count; i++) {
+		if (saves[i]->uid > importer->last_common_uid) {
+			imp_debug(importer, "Delayed save UID=%u: Save",
+				  saves[i]->uid);
+			dsync_mailbox_save(importer, saves[i]);
+		} else {
+			imp_debug(importer, "Delayed save UID=%u: Ignore",
+				  saves[i]->uid);
+		}
+	}
+}
+
+static int
+dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
+			       const struct dsync_mail_change *change,
+			       const char **result_r)
+{
+	const char *hdr_hash, *cmp_guid;
+
+	if (*change->guid != '\0' && *importer->cur_guid != '\0') {
+		/* we have GUIDs, verify them */
+		if (dsync_mail_change_guid_equals(importer, change,
+						  importer->cur_guid, &cmp_guid)) {
+			*result_r = "GUIDs match";
+			return 1;
+		} else {
+			*result_r = t_strdup_printf("GUIDs don't match (%s vs %s)",
+						    importer->cur_guid, cmp_guid);
+			return 0;
+		}
+	}
+
+	/* verify hdr_hash if it exists */
+	if (change->hdr_hash == NULL) {
+		i_assert(*importer->cur_guid == '\0');
+		if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+			/* the message was already expunged, so we don't know
+			   its header. return "unknown". */
+			*result_r = "Unknown match for expunge";
+			return -1;
+		}
+		i_error("Mailbox %s: GUIDs not supported, "
+			"sync with header hashes instead",
+			mailbox_get_vname(importer->box));
+		importer->mail_error = MAIL_ERROR_TEMP;
+		importer->failed = TRUE;
+		*result_r = "Error, invalid parameters";
+		return -1;
+	}
+
+	if (dsync_mail_get_hdr_hash(importer->cur_mail,
+				    importer->hdr_hash_version,
+				    importer->hashed_headers, &hdr_hash) < 0) {
+		dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
+		*result_r = "Error fetching header stream";
+		return -1;
+	}
+	if (importer->empty_hdr_workaround &&
+	    (dsync_mail_hdr_hash_is_empty(change->hdr_hash) ||
+	     dsync_mail_hdr_hash_is_empty(hdr_hash))) {
+		*result_r = "Empty headers found with workaround enabled - assuming a match";
+		return 1;
+	} else if (strcmp(change->hdr_hash, hdr_hash) == 0) {
+		*result_r = "Headers hashes match";
+		return 1;
+	} else {
+		*result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)",
+					    change->hdr_hash, hdr_hash);
+		return 0;
+	}
+}
+
+static bool
+dsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer,
+				       const struct dsync_mail_change *change,
+				       const char **result_r)
+{
+	const struct dsync_mail_change *local_change;
+
+	if (*change->guid == '\0') {
+		/* remote doesn't support GUIDs, can't verify expunge */
+		*result_r = "GUIDs not supported, can't verify expunge";
+		return FALSE;
+	}
+
+	/* local message is expunged. see if we can find its GUID from
+	   transaction log and check if the GUIDs match. The GUID in
+	   log is a 128bit GUID, so we may need to convert the remote's
+	   GUID string to 128bit GUID first. */
+	local_change = hash_table_lookup(importer->local_changes,
+					 POINTER_CAST(change->uid));
+	if (local_change == NULL || local_change->guid == NULL) {
+		*result_r = "Expunged local mail's GUID not found";
+		return FALSE;
+	}
+
+	i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+	if (dsync_mail_change_guid_equals(importer, local_change,
+					  change->guid, NULL)) {
+		importer->last_common_uid = change->uid;
+		*result_r = "Expunged local mail's GUID matches remote";
+	} else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+		dsync_mailbox_common_uid_found(importer);
+		*result_r = "Expunged local mail's GUID doesn't match remote GUID";
+	} else {
+		/* GUID mismatch for two expunged mails. dsync can't update
+		   GUIDs for already expunged messages, so we can't immediately
+		   determine that the rest of the messages are a mismatch. so
+		   for now we'll just skip over this pair. */
+		*result_r = "Expunged mails' GUIDs don't match - delaying decision";
+		/* NOTE: the return value here doesn't matter, because the only
+		   caller that checks for it never reaches this code path */
+	}
+	return TRUE;
+}
+
+static void
+dsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer,
+			     const struct dsync_mail_change *change)
+{
+	i_assert(importer->revert_local_changes);
+
+	/* mail exists on remote, but not locally. we'll need to
+	   insert this mail back, which means deleting the whole
+	   mailbox and resyncing. */
+	i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally",
+		  mailbox_get_vname(importer->box),
+		  change->uid, change->guid);
+	importer->delete_mailbox = TRUE;
+	importer->mail_error = MAIL_ERROR_TEMP;
+	importer->failed = TRUE;
+}
+
+static void
+dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
+			      const struct dsync_mail_change *change,
+			      const char **result_r)
+{
+	int ret;
+
+	i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+		 ((change->received_timestamp > 0 ||
+		   (importer->sync_since_timestamp == 0 &&
+		    importer->sync_until_timestamp == 0)) &&
+		  (change->virtual_size != UOFF_T_MAX || importer->sync_max_size == 0)));
+
+	/* try to find the matching local mail */
+	if (!importer_next_mail(importer, change->uid)) {
+		/* no more local mails. we can still try to match
+		   expunged mails though. */
+		if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+			/* mail doesn't exist remotely either, don't bother
+			   looking it up locally. */
+			*result_r = "Expunged mail not found locally";
+			return;
+		}
+		i_assert(change->guid != NULL);
+		if (!dsync_mailbox_import_want_change(importer, change, result_r))
+			;
+		else if (importer->local_uid_next <= change->uid) {
+			dsync_mailbox_common_uid_found(importer);
+			*result_r = "Mail's UID is above local UIDNEXT";
+		} else if (importer->revert_local_changes) {
+			dsync_mailbox_revert_missing(importer, change);
+			*result_r = "Reverting local change by deleting mailbox";
+		} else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) {
+			/* it's unknown if this mail existed locally and was
+			   expunged. since we don't want to lose any mails,
+			   assume that we need to preserve the mail. use the
+			   last message with a matching GUID as the last common
+			   UID. */
+			dsync_mailbox_common_uid_found(importer);
+		}
+		*result_r = t_strdup_printf("%s - No more local mails found", *result_r);
+		return;
+	}
+
+	if (change->guid == NULL) {
+		/* we can't know if this UID matches */
+		i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+		*result_r = "Expunged mail has no GUID, can't verify it";
+		return;
+	}
+	if (importer->cur_mail->uid == change->uid) {
+		/* we have a matching local UID. check GUID to see if it's
+		   really the same mail or not */
+		if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) {
+			/* unknown */
+			return;
+		}
+		if (ret > 0) {
+			importer->last_common_uid = change->uid;
+		} else if (!importer->revert_local_changes) {
+			/* mismatch - found the first non-common UID */
+			dsync_mailbox_common_uid_found(importer);
+		} else {
+			/* mismatch and we want to revert local changes -
+			   need to delete the mailbox. */
+			dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r);
+		}
+		return;
+	}
+	/* mail exists remotely, but doesn't exist locally. */
+	if (!dsync_mailbox_import_want_change(importer, change, result_r))
+		return;
+	if (importer->revert_local_changes &&
+	    change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+		dsync_mailbox_revert_missing(importer, change);
+		*result_r = "Reverting local change by deleting mailbox";
+	} else {
+		(void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r);
+	}
+	*result_r = t_strdup_printf("%s (next local mail UID=%u)",
+		*result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid);
+}
+
+int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
+				const struct dsync_mail_change *change)
+{
+	const char *result;
+
+	i_assert(!importer->new_uids_assigned);
+	i_assert(importer->prev_uid < change->uid);
+
+	importer->prev_uid = change->uid;
+
+	if (importer->failed)
+		return -1;
+	if (importer->require_full_resync)
+		return 0;
+
+	if (!importer->last_common_uid_found) {
+		result = NULL;
+		dsync_mailbox_find_common_uid(importer, change, &result);
+		i_assert(result != NULL);
+	} else {
+		result = "New mail";
+	}
+
+	imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s",
+		  dsync_mail_change_type_names[change->type],
+		  change->guid != NULL ? change->guid : "<unknown>", change->uid,
+		  change->hdr_hash != NULL ? change->hdr_hash : "", result);
+
+	if (importer->failed)
+		return -1;
+	if (importer->require_full_resync)
+		return 0;
+
+	if (importer->last_common_uid_found) {
+		/* a) uid <= last_common_uid for flag changes and expunges.
+		   this happens only when last_common_uid was originally given
+		   as parameter to importer.
+
+		   when we're finding the last_common_uid ourself,
+		   uid>last_common_uid always in here, because
+		   last_common_uid_found=TRUE only after we find the first
+		   mismatch.
+
+		   b) uid > last_common_uid for i) new messages, ii) expunges
+		   that were sent "just in case" */
+		if (change->uid <= importer->last_common_uid) {
+			i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
+		} else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+			/* ignore */
+			return 0;
+		} else {
+			i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
+		}
+	} else {
+		/* a) uid < last_common_uid can never happen */
+		i_assert(change->uid >= importer->last_common_uid);
+		/* b) uid = last_common_uid if we've verified that the
+		   messages' GUIDs match so far.
+
+		   c) uid > last_common_uid: i) TYPE_EXPUNGE change has
+		   GUID=NULL, so we couldn't verify yet if it matches our
+		   local message, ii) local message is expunged and we couldn't
+		   find its GUID */
+		if (change->uid > importer->last_common_uid) {
+			i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+				 importer->cur_mail == NULL ||
+				 change->uid < importer->cur_mail->uid);
+		}
+	}
+
+	switch (change->type) {
+	case DSYNC_MAIL_CHANGE_TYPE_SAVE:
+		dsync_mailbox_import_save(importer, change);
+		break;
+	case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:
+		dsync_mailbox_import_expunge(importer, change);
+		break;
+	case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:
+		i_assert(importer->last_common_uid_found);
+		dsync_mailbox_import_flag_change(importer, change);
+		break;
+	}
+	return importer->failed ? -1 : 0;
+}
+
+static int
+importer_new_mail_final_uid_cmp(struct importer_new_mail *const *newmail1,
+				struct importer_new_mail *const *newmail2)
+{
+	if ((*newmail1)->final_uid < (*newmail2)->final_uid)
+		return -1;
+	if ((*newmail1)->final_uid > (*newmail2)->final_uid)
+		return 1;
+	return 0;
+}
+
+static void
+dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)
+{
+	struct importer_new_mail *newmail;
+	uint32_t common_uid_next, new_uid;
+
+	common_uid_next = I_MAX(importer->local_uid_next,
+				importer->remote_uid_next);
+	array_foreach_elem(&importer->newmails, newmail) {
+		if (newmail->skip) {
+			/* already assigned */
+			i_assert(newmail->final_uid != 0);
+			continue;
+		}
+
+		/* figure out what UID to use for the mail */
+		if (newmail->uid_is_usable) {
+			/* keep the UID */
+			new_uid = newmail->final_uid;
+		} else if (newmail->link != NULL &&
+			   newmail->link->uid_is_usable) {
+			/* we can use the linked message's UID and expunge
+			   this mail */
+			new_uid = newmail->link->final_uid;
+		} else {
+			i_assert(!importer->revert_local_changes);
+			new_uid = common_uid_next++;
+			imp_debug(importer, "UID %u isn't usable, assigning new UID %u",
+				  newmail->final_uid, new_uid);
+		}
+
+		newmail->final_uid = new_uid;
+		if (newmail->link != NULL && newmail->link != newmail) {
+			/* skip processing the linked mail */
+			newmail->link->skip = TRUE;
+		}
+	}
+	importer->last_common_uid = common_uid_next-1;
+	importer->new_uids_assigned = TRUE;
+	/* Sort the newmails by their final_uid. This is used for tracking
+	   whether an intermediate commit is allowed. */
+	array_sort(&importer->newmails, importer_new_mail_final_uid_cmp);
+}
+
+static int
+dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
+			       struct mail *mail, uint32_t uid, const char *guid,
+			       struct dsync_mail *dmail_r)
+{
+	const char *error_field, *errstr;
+	enum mail_error error;
+
+	if (!mail_set_uid(mail, uid))
+		return 0;
+
+	/* NOTE: Errors are logged, but they don't cause the entire import
+	   to fail. */
+	if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) {
+		errstr = mailbox_get_last_internal_error(mail->box, &error);
+		if (error == MAIL_ERROR_EXPUNGED)
+			return 0;
+
+		i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
+			mailbox_get_vname(importer->box),
+			error_field, uid, errstr);
+		return -1;
+	}
+	if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) {
+		dsync_import_unexpected_state(importer, t_strdup_printf(
+			"Unexpected GUID mismatch (3) for UID=%u: %s != %s",
+			uid, dmail_r->guid, guid));
+		return -1;
+	}
+	return 1;
+}
+
+static void
+dsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer,
+			       uint32_t uid)
+{
+	i_assert(importer->search_ctx == NULL);
+
+	if (importer->highest_wanted_uid < uid)
+		importer->highest_wanted_uid = uid;
+	array_push_back(&importer->wanted_uids, &uid);
+}
+
+static void
+dsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer)
+{
+	struct importer_new_mail *const *newmails;
+	unsigned int count;
+
+	newmails = array_get(&importer->newmails, &count);
+	while (importer->first_unsaved_idx < count) {
+		if (!newmails[importer->first_unsaved_idx]->saved)
+			break;
+		importer->first_unsaved_idx++;
+	}
+}
+
+static void
+dsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer,
+				   struct importer_new_mail *newmail)
+{
+	dsync_mailbox_import_saved_uid(importer, newmail->final_uid);
+	newmail->saved = TRUE;
+
+	dsync_mailbox_import_update_first_saved(importer);
+	importer->saves_since_commit++;
+	/* we can commit only if all the upcoming mails will have UIDs that
+	   are larger than we're committing.
+
+	   Note that if any existing UIDs have been changed, the new UID is
+	   usually higher than anything that is being saved so we can't do
+	   an intermediate commit. It's too much extra work to try to handle
+	   that situation. So here this never happens, because then
+	   array_count(wanted_uids) is always higher than first_unsaved_idx. */
+	if (importer->saves_since_commit >= importer->commit_msgs_interval &&
+	    importer->first_unsaved_idx == array_count(&importer->wanted_uids)) {
+		if (dsync_mailbox_import_commit(importer, FALSE) < 0)
+			importer->failed = TRUE;
+		importer->saves_since_commit = 0;
+	}
+}
+
+static bool
+dsync_msg_change_uid(struct dsync_mailbox_importer *importer,
+		     uint32_t old_uid, uint32_t new_uid)
+{
+	struct mail_save_context *save_ctx;
+
+	IMPORTER_DEBUG_CHANGE(importer);
+
+	if (!mail_set_uid(importer->mail, old_uid))
+		return FALSE;
+
+	save_ctx = mailbox_save_alloc(importer->ext_trans);
+	mailbox_save_copy_flags(save_ctx, importer->mail);
+	mailbox_save_set_uid(save_ctx, new_uid);
+	if (mailbox_move(&save_ctx, importer->mail) < 0)
+		return FALSE;
+	dsync_mailbox_import_saved_uid(importer, new_uid);
+	return TRUE;
+}
+
+static bool
+dsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer,
+				ARRAY_TYPE(seq_range) *unwanted_uids,
+				uint32_t wanted_uid)
+{
+	const struct seq_range *range;
+	unsigned int count, n;
+	struct seq_range_iter iter;
+	uint32_t uid;
+
+	/* optimize by first trying to use the latest UID */
+	range = array_get(unwanted_uids, &count);
+	if (count == 0)
+		return FALSE;
+	if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) {
+		seq_range_array_remove(unwanted_uids, range[count-1].seq2);
+		return TRUE;
+	}
+	if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
+		seq_range_array_remove(unwanted_uids, range[count-1].seq2);
+
+	/* now try to use any of them by iterating through them. (would be
+	   easier&faster to just iterate backwards, but probably too much
+	   trouble to add such API) */
+	n = 0; seq_range_array_iter_init(&iter, unwanted_uids);
+	while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+		if (dsync_msg_change_uid(importer, uid, wanted_uid)) {
+			seq_range_array_remove(unwanted_uids, uid);
+			return TRUE;
+		}
+		if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
+			seq_range_array_remove(unwanted_uids, uid);
+	}
+	return FALSE;
+}
+
+static bool
+dsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer,
+			       struct importer_new_mail *all_newmails,
+			       ARRAY_TYPE(seq_range) *local_uids,
+			       ARRAY_TYPE(seq_range) *wanted_uids)
+{
+	ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids;
+	struct seq_range_iter local_iter, wanted_iter;
+	unsigned int local_n, wanted_n;
+	uint32_t local_uid, wanted_uid;
+	struct importer_new_mail *mail;
+	struct dsync_mail dmail;
+
+	if (array_count(local_uids) == 0)
+		return FALSE;
+
+	local_n = wanted_n = 0;
+	seq_range_array_iter_init(&local_iter, local_uids);
+	seq_range_array_iter_init(&wanted_iter, wanted_uids);
+
+	/* wanted_uids contains UIDs that need to exist at the end. those that
+	   don't already exist in local_uids have a higher UID than any
+	   existing local UID */
+	t_array_init(&assigned_uids, array_count(wanted_uids));
+	t_array_init(&unwanted_uids, 8);
+	while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+		if (seq_range_array_iter_nth(&wanted_iter, wanted_n,
+					     &wanted_uid)) {
+			if (local_uid == wanted_uid) {
+				/* we have exactly the UID we want. keep it. */
+				seq_range_array_add(&assigned_uids, wanted_uid);
+				wanted_n++;
+				continue;
+			}
+			i_assert(local_uid < wanted_uid);
+		}
+		/* we no longer want this local UID. */
+		seq_range_array_add(&unwanted_uids, local_uid);
+	}
+
+	/* reuse as many existing messages as possible by changing their UIDs */
+	while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
+		if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids,
+						     wanted_uid))
+			break;
+		seq_range_array_add(&assigned_uids, wanted_uid);
+		wanted_n++;
+	}
+
+	/* expunge all unwanted messages */
+	local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids);
+	while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+		IMPORTER_DEBUG_CHANGE(importer);
+		if (mail_set_uid(importer->mail, local_uid))
+			mail_expunge(importer->mail);
+	}
+
+	/* mark mails whose UIDs we got to be skipped over later */
+	for (mail = all_newmails; mail != NULL; mail = mail->next) {
+		if (!mail->skip &&
+		    seq_range_exists(&assigned_uids, mail->final_uid))
+			mail->skip = TRUE;
+	}
+
+	if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
+		/* we've assigned all wanted UIDs */
+		return TRUE;
+	}
+
+	/* try to find one existing message that we can use to copy to the
+	   other instances */
+	local_n = 0; seq_range_array_iter_init(&local_iter, local_uids);
+	while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+		if (dsync_mailbox_import_local_uid(importer, importer->mail,
+						   local_uid, all_newmails->guid,
+						   &dmail) > 0) {
+			if (dsync_mailbox_save_newmails(importer, &dmail,
+							all_newmails, FALSE))
+				return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static bool
+dsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer,
+				     struct importer_new_mail *all_newmails)
+{
+	struct dsync_mail dmail;
+
+	if (all_newmails->virtual_all_uid == 0)
+		return FALSE;
+
+	if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail,
+					   all_newmails->virtual_all_uid,
+					   all_newmails->guid, &dmail) > 0) {
+		if (dsync_mailbox_save_newmails(importer, &dmail,
+						all_newmails, FALSE))
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static bool
+dsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer,
+				 struct importer_new_mail *all_newmails)
+{
+	ARRAY_TYPE(seq_range) local_uids, wanted_uids;
+	struct dsync_mail_request *request;
+	struct importer_new_mail *mail;
+	const char *request_guid = NULL;
+	uint32_t request_uid = 0;
+
+	i_assert(all_newmails != NULL);
+
+	/* get the list of the current local UIDs and the wanted UIDs.
+	   find the first remote instance that we can request in case there are
+	   no local instances */
+	t_array_init(&local_uids, 8);
+	t_array_init(&wanted_uids, 8);
+	for (mail = all_newmails; mail != NULL; mail = mail->next) {
+		if (mail->uid_in_local)
+			seq_range_array_add(&local_uids, mail->local_uid);
+		else if (request_guid == NULL) {
+			if (*mail->guid != '\0')
+				request_guid = mail->guid;
+			request_uid = mail->remote_uid;
+			i_assert(request_uid != 0);
+		}
+		if (!mail->skip)
+			seq_range_array_add(&wanted_uids, mail->final_uid);
+	}
+	i_assert(array_count(&wanted_uids) > 0);
+
+	if (!dsync_mailbox_import_try_local(importer, all_newmails,
+					    &local_uids, &wanted_uids) &&
+	    !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) {
+		/* no local instance. request from remote */
+		IMPORTER_DEBUG_CHANGE(importer);
+		if (importer->want_mail_requests) {
+			request = array_append_space(&importer->mail_requests);
+			request->guid = request_guid;
+			request->uid = request_uid;
+		}
+		return FALSE;
+	}
+	/* successfully handled all the mails locally */
+	importer->import_pos++;
+	return TRUE;
+}
+
+static void
+dsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer)
+{
+	struct mail_search_context *search_ctx;
+	struct mail_search_args *search_args;
+	struct importer_new_mail *newmail;
+	struct mail *mail;
+	const char *guid;
+
+	if (mailbox_sync(importer->virtual_all_box, 0) < 0) {
+		i_error("Couldn't sync \\All mailbox '%s': %s",
+			mailbox_get_vname(importer->virtual_all_box),
+			mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
+		return;
+	}
+
+	search_args = mail_search_build_init();
+	mail_search_build_add_all(search_args);
+
+	importer->virtual_trans =
+		mailbox_transaction_begin(importer->virtual_all_box,
+					  importer->transaction_flags,
+					  __func__);
+	search_ctx = mailbox_search_init(importer->virtual_trans, search_args,
+					 NULL, MAIL_FETCH_GUID, NULL);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
+			/* ignore errors */
+			continue;
+		}
+		newmail = hash_table_lookup(importer->import_guids, guid);
+		if (newmail != NULL && newmail->virtual_all_uid == 0)
+			newmail->virtual_all_uid = mail->uid;
+	}
+	if (mailbox_search_deinit(&search_ctx) < 0) {
+		i_error("Couldn't search \\All mailbox '%s': %s",
+			mailbox_get_vname(importer->virtual_all_box),
+			mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
+	}
+
+	importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL);
+}
+
+static void
+dsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer)
+{
+	struct hash_iterate_context *iter;
+	const char *key;
+	void *key2;
+	struct importer_new_mail *mail;
+
+	if (importer->virtual_all_box != NULL &&
+	    hash_table_count(importer->import_guids) > 0) {
+		/* find UIDs in \All mailbox for all wanted GUIDs. */
+		dsync_mailbox_import_find_virtual_uids(importer);
+	}
+
+	iter = hash_table_iterate_init(importer->import_guids);
+	while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
+		T_BEGIN {
+			if (dsync_mailbox_import_handle_mail(importer, mail))
+				hash_table_remove(importer->import_guids, key);
+		} T_END;
+	}
+	hash_table_iterate_deinit(&iter);
+
+	iter = hash_table_iterate_init(importer->import_uids);
+	while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) {
+		T_BEGIN {
+			if (dsync_mailbox_import_handle_mail(importer, mail))
+				hash_table_remove(importer->import_uids, key2);
+		} T_END;
+	}
+	hash_table_iterate_deinit(&iter);
+	if (!importer->mails_have_guids) {
+		array_foreach_elem(&importer->newmails, mail) {
+			if (mail->uid_in_local)
+				(void)dsync_mailbox_import_handle_mail(importer, mail);
+		}
+	}
+}
+
+int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)
+{
+	i_assert(!importer->new_uids_assigned);
+
+	if (!importer->last_common_uid_found) {
+		/* handle pending expunges and flag updates */
+		dsync_mailbox_common_uid_found(importer);
+	}
+	/* skip common local mails */
+	(void)importer_next_mail(importer, importer->last_common_uid+1);
+	/* if there are any local mails left, add them to newmails list */
+	while (importer->cur_mail != NULL && !importer->failed)
+		(void)dsync_mailbox_try_save(importer, NULL);
+
+	if (importer->search_ctx != NULL) {
+		if (mailbox_search_deinit(&importer->search_ctx) < 0) {
+			i_error("Mailbox %s: Search failed: %s",
+				mailbox_get_vname(importer->box),
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			importer->failed = TRUE;
+		}
+	}
+	importer->import_count = hash_table_count(importer->import_guids) +
+		hash_table_count(importer->import_uids);
+
+	dsync_mailbox_import_assign_new_uids(importer);
+	/* save mails from local sources where possible,
+	   request the rest from remote */
+	if (!importer->failed)
+		dsync_mailbox_import_handle_local_mails(importer);
+	return importer->failed ? -1 : 0;
+}
+
+const struct dsync_mail_request *
+dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)
+{
+	const struct dsync_mail_request *requests;
+	unsigned int count;
+
+	requests = array_get(&importer->mail_requests, &count);
+	if (importer->mail_request_idx == count)
+		return NULL;
+	return &requests[importer->mail_request_idx++];
+}
+
+static const char *const *
+dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)
+{
+	ARRAY_TYPE(const_string) keywords;
+	const char *const *changes;
+	unsigned int i, count;
+
+	if (!array_is_created(&change->keyword_changes))
+		return NULL;
+
+	changes = array_get(&change->keyword_changes, &count);
+	t_array_init(&keywords, count);
+	for (i = 0; i < count; i++) {
+		if (changes[i][0] == KEYWORD_CHANGE_ADD ||
+		    changes[i][0] == KEYWORD_CHANGE_FINAL ||
+		    changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) {
+			const char *name = changes[i]+1;
+
+			array_push_back(&keywords, &name);
+		}
+	}
+	if (array_count(&keywords) == 0)
+		return NULL;
+
+	array_append_zero(&keywords);
+	return array_front(&keywords);
+}
+
+static void
+dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,
+				struct mail_save_context *save_ctx,
+				const struct dsync_mail_change *change)
+{
+	const char *const *keyword_names;
+	struct mail_keywords *keywords;
+
+	keyword_names = dsync_mailbox_get_final_keywords(change);
+	keywords = keyword_names == NULL ? NULL :
+		mailbox_keywords_create_valid(importer->box,
+					      keyword_names);
+	mailbox_save_set_flags(save_ctx, change->final_flags, keywords);
+	if (keywords != NULL)
+		mailbox_keywords_unref(&keywords);
+
+	if (change->modseq > 1) {
+		(void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);
+		mailbox_save_set_min_modseq(save_ctx, change->modseq);
+	}
+	/* FIXME: if there already are private flags, they get lost because
+	   saving can't handle updating private index. they get added on the
+	   next sync though. if this is fixed here, set min_pvt_modseq also. */
+}
+
+static int
+dsync_msg_try_copy(struct dsync_mailbox_importer *importer,
+		   struct mail_save_context **save_ctx_p,
+		   struct importer_new_mail **all_newmails_forcopy)
+{
+	struct importer_new_mail *inst;
+
+	for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) {
+		if (inst->uid_in_local && !inst->copy_failed &&
+		    mail_set_uid(importer->mail, inst->local_uid)) {
+			if (mailbox_copy(save_ctx_p, importer->mail) < 0) {
+				enum mail_error error;
+				const char *errstr;
+
+				errstr = mailbox_get_last_internal_error(importer->box, &error);
+				if (error != MAIL_ERROR_EXPUNGED) {
+					i_warning("Failed to copy mail from UID=%u: "
+						  "%s - falling back to other means",
+						 inst->local_uid, errstr);
+				}
+				inst->copy_failed = TRUE;
+				return -1;
+			}
+			*all_newmails_forcopy = inst;
+			return 1;
+		}
+	}
+	*all_newmails_forcopy = NULL;
+	return 0;
+}
+
+static void
+dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
+				  const struct dsync_mail *mail)
+{
+	if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
+		mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
+	if (mail->pop3_order > 0)
+		mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
+	mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+}
+
+static struct mail_save_context *
+dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
+			const struct dsync_mail *mail,
+			struct importer_new_mail *newmail)
+{
+	struct mail_save_context *save_ctx;
+
+	save_ctx = mailbox_save_alloc(importer->ext_trans);
+	mailbox_save_set_uid(save_ctx, newmail->final_uid);
+	if (*mail->guid != '\0')
+		mailbox_save_set_guid(save_ctx, mail->guid);
+	if (mail->saved_date != 0)
+		mailbox_save_set_save_date(save_ctx, mail->saved_date);
+	dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
+
+	if (!mail->minimal_fields)
+		dsync_mailbox_save_set_nonminimal(save_ctx, mail);
+	return save_ctx;
+}
+
+static bool
+dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
+			const struct dsync_mail *mail,
+			struct importer_new_mail *newmail,
+			struct importer_new_mail **all_newmails_forcopy,
+			bool remote_mail)
+{
+	struct mail_save_context *save_ctx;
+	struct istream *input;
+	ssize_t ret;
+	bool save_failed = FALSE;
+
+	/* try to save the mail by copying an existing mail */
+	save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
+	if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) {
+		if (save_ctx == NULL)
+			save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
+	}
+	if (ret <= 0 && mail->input_mail != NULL) {
+		/* copy using the source mail */
+		i_assert(mail->input_mail->uid == mail->input_mail_uid);
+		if (mailbox_copy(&save_ctx, mail->input_mail) == 0)
+			ret = 1;
+		else {
+			enum mail_error error;
+			const char *errstr;
+
+			errstr = mailbox_get_last_internal_error(importer->box, &error);
+			if (error != MAIL_ERROR_EXPUNGED) {
+				i_warning("Failed to copy source UID=%u mail: "
+					  "%s - falling back to regular saving",
+					  mail->input_mail->uid, errstr);
+			}
+			ret = -1;
+			save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
+		}
+
+	}
+	if (ret > 0) {
+		i_assert(save_ctx == NULL);
+		dsync_mailbox_import_saved_newmail(importer, newmail);
+		return TRUE;
+	}
+	/* fallback to saving from remote stream */
+	if (!remote_mail) {
+		/* the mail isn't remote yet. we were just trying to copy a
+		   local mail to avoid downloading the remote mail. */
+		mailbox_save_cancel(&save_ctx);
+		return FALSE;
+	}
+	if (mail->minimal_fields) {
+		struct dsync_mail mail2;
+		const char *error_field;
+
+		i_assert(mail->input_mail != NULL);
+
+		if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
+					       &error_field) < 0) {
+			i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
+				mailbox_get_vname(importer->box),
+				error_field, mail->uid,
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			importer->failed = TRUE;
+			mailbox_save_cancel(&save_ctx);
+			return TRUE;
+		}
+		dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
+		input = mail2.input;
+	} else {
+		input = mail->input;
+	}
+
+	if (input == NULL) {
+		/* it was just expunged in remote, skip it */
+		mailbox_save_cancel(&save_ctx);
+		return TRUE;
+	}
+
+	i_stream_seek(input, 0);
+	if (mailbox_save_begin(&save_ctx, input) < 0) {
+		i_error("Mailbox %s: Saving failed: %s",
+			mailbox_get_vname(importer->box),
+			mailbox_get_last_internal_error(importer->box,
+							&importer->mail_error));
+		importer->failed = TRUE;
+		return TRUE;
+	}
+	while ((ret = i_stream_read(input)) > 0 || ret == -2) {
+		if (mailbox_save_continue(save_ctx) < 0) {
+			save_failed = TRUE;
+			ret = -1;
+			break;
+		}
+	}
+	i_assert(ret == -1);
+
+	if (input->stream_errno != 0) {
+		i_error("Mailbox %s: read(msg input) failed: %s",
+			mailbox_get_vname(importer->box),
+			i_stream_get_error(input));
+		mailbox_save_cancel(&save_ctx);
+		importer->mail_error = MAIL_ERROR_TEMP;
+		importer->failed = TRUE;
+	} else if (save_failed) {
+		i_error("Mailbox %s: Saving failed: %s",
+			mailbox_get_vname(importer->box),
+			mailbox_get_last_internal_error(importer->box,
+							&importer->mail_error));
+		mailbox_save_cancel(&save_ctx);
+		importer->failed = TRUE;
+	} else {
+		i_assert(input->eof);
+		if (mailbox_save_finish(&save_ctx) < 0) {
+			i_error("Mailbox %s: Saving failed: %s",
+				mailbox_get_vname(importer->box),
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			importer->failed = TRUE;
+		} else {
+			dsync_mailbox_import_saved_newmail(importer, newmail);
+		}
+	}
+	return TRUE;
+}
+
+static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
+					const struct dsync_mail *mail,
+					struct importer_new_mail *all_newmails,
+					bool remote_mail)
+{
+	struct importer_new_mail *newmail, *all_newmails_forcopy;
+	bool ret = TRUE;
+
+	/* if all_newmails list is large, avoid scanning through the
+	   uninteresting ones for each newmail */
+	all_newmails_forcopy = all_newmails;
+
+	/* save all instances of the message */
+	for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) {
+		if (!newmail->skip) T_BEGIN {
+			if (!dsync_mailbox_save_body(importer, mail, newmail,
+						     &all_newmails_forcopy,
+						     remote_mail))
+				ret = FALSE;
+		} T_END;
+	}
+	return ret;
+}
+
+int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
+			      const struct dsync_mail *mail)
+{
+	struct importer_new_mail *all_newmails;
+
+	i_assert(mail->input == NULL || mail->input->seekable);
+	i_assert(importer->new_uids_assigned);
+
+	if (importer->failed)
+		return -1;
+	if (importer->require_full_resync)
+		return 0;
+
+	imp_debug(importer, "Import mail body for GUID=%s UID=%u",
+		  mail->guid, mail->uid);
+
+	all_newmails = *mail->guid != '\0' ?
+		hash_table_lookup(importer->import_guids, mail->guid) :
+		hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));
+	if (all_newmails == NULL) {
+		if (importer->want_mail_requests) {
+			i_error("Mailbox %s: Remote sent unwanted message body for "
+				"GUID=%s UID=%u",
+				mailbox_get_vname(importer->box),
+				mail->guid, mail->uid);
+		} else {
+			imp_debug(importer, "Skip unwanted mail body for "
+				  "GUID=%s UID=%u", mail->guid, mail->uid);
+		}
+		return 0;
+	}
+	if (*mail->guid != '\0')
+		hash_table_remove(importer->import_guids, mail->guid);
+	else {
+		hash_table_remove(importer->import_uids,
+				  POINTER_CAST(mail->uid));
+	}
+	importer->import_pos++;
+	if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE))
+		i_unreached();
+	return importer->failed ? -1 : 0;
+}
+
+static int
+reassign_uids_in_seq_range(struct dsync_mailbox_importer *importer,
+			   const ARRAY_TYPE(seq_range) *unwanted_uids)
+{
+	struct mailbox *box = importer->box;
+	const enum mailbox_transaction_flags trans_flags =
+		importer->transaction_flags |
+		MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+		MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
+	struct mailbox_transaction_context *trans;
+	struct mail_search_args *search_args;
+	struct mail_search_arg *arg;
+	struct mail_search_context *search_ctx;
+	struct mail_save_context *save_ctx;
+	struct mail *mail;
+	unsigned int renumber_count = 0;
+	int ret = 1;
+
+	if (array_count(unwanted_uids) == 0)
+		return 1;
+
+	if (importer->debug) T_BEGIN {
+		string_t *str = t_str_new(256);
+		imap_write_seq_range(str, unwanted_uids);
+		imp_debug(importer, "Reassign UIDs: %s", str_c(str));
+	} T_END;
+
+	search_args = mail_search_build_init();
+	arg = mail_search_build_add(search_args, SEARCH_UIDSET);
+	p_array_init(&arg->value.seqset, search_args->pool,
+		     array_count(unwanted_uids));
+	array_append_array(&arg->value.seqset, unwanted_uids);
+
+	trans = mailbox_transaction_begin(box, trans_flags, __func__);
+	search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
+	mail_search_args_unref(&search_args);
+
+	while (mailbox_search_next(search_ctx, &mail)) {
+		save_ctx = mailbox_save_alloc(trans);
+		mailbox_save_copy_flags(save_ctx, mail);
+		if (mailbox_move(&save_ctx, mail) < 0) {
+			i_error("Mailbox %s: Couldn't move mail within mailbox: %s",
+				mailbox_get_vname(box),
+				mailbox_get_last_internal_error(box, &importer->mail_error));
+			ret = -1;
+		} else if (ret > 0) {
+			ret = 0;
+		}
+		renumber_count++;
+	}
+	if (mailbox_search_deinit(&search_ctx) < 0) {
+		i_error("Mailbox %s: mail search failed: %s",
+			mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, &importer->mail_error));
+		ret = -1;
+	}
+
+	if (mailbox_transaction_commit(&trans) < 0) {
+		i_error("Mailbox %s: UID reassign commit failed: %s",
+			mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, &importer->mail_error));
+		ret = -1;
+	}
+	if (ret == 0) {
+		imp_debug(importer, "Mailbox %s: Change during sync: "
+			  "Renumbered %u of %u unwanted UIDs",
+			  mailbox_get_vname(box),
+			  renumber_count, array_count(unwanted_uids));
+	}
+	return ret;
+}
+
+static int
+reassign_unwanted_uids(struct dsync_mailbox_importer *importer,
+		       const char **changes_during_sync_r)
+{
+	ARRAY_TYPE(seq_range) unwanted_uids;
+	const uint32_t *wanted_uids, *saved_uids;
+	uint32_t highest_seen_uid;
+	unsigned int i, wanted_count, saved_count;
+	int ret = 0;
+
+	wanted_uids = array_get(&importer->wanted_uids, &wanted_count);
+	saved_uids = array_get(&importer->saved_uids, &saved_count);
+	i_assert(wanted_count == saved_count);
+	if (wanted_count == 0)
+		return 0;
+	/* wanted_uids contains the UIDs we tried to save mails with.
+	   if nothing changed during dsync, we should have the expected UIDs
+	   (saved_uids) and all is well.
+
+	   if any new messages got inserted during dsync, we'll need to fix up
+	   the UIDs and let the next dsync fix up the other side. for example:
+
+	   remote uids = 5,7,9 = wanted_uids
+	   remote uidnext = 12
+	   locally added new uid=5 ->
+	   saved_uids = 10,7,9
+
+	   we'll now need to reassign UIDs 5 and 10. to be fully future-proof
+	   we'll reassign all UIDs between [original local uidnext .. highest
+	   UID we think we know] that aren't in saved_uids. */
+
+	/* create uidset for the list of UIDs we don't want to exist */
+	t_array_init(&unwanted_uids, 8);
+	highest_seen_uid = I_MAX(importer->remote_uid_next-1,
+				 importer->highest_wanted_uid);
+	i_assert(importer->local_uid_next <= highest_seen_uid);
+	seq_range_array_add_range(&unwanted_uids,
+				  importer->local_uid_next, highest_seen_uid);
+	for (i = 0; i < wanted_count; i++) {
+		i_assert(i < wanted_count);
+		if (saved_uids[i] == wanted_uids[i])
+			seq_range_array_remove(&unwanted_uids, saved_uids[i]);
+	}
+
+	ret = reassign_uids_in_seq_range(importer, &unwanted_uids);
+	if (ret == 0) {
+		*changes_during_sync_r = t_strdup_printf(
+			"%u UIDs changed due to UID conflicts",
+			seq_range_count(&unwanted_uids));
+		/* conflicting changes during sync, revert our last-common-uid
+		   back to a safe value. */
+		importer->last_common_uid = importer->local_uid_next - 1;
+	}
+	return ret < 0 ? -1 : 0;
+}
+
+static int
+dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final)
+{
+	struct mail_transaction_commit_changes changes;
+	struct seq_range_iter iter;
+	uint32_t uid;
+	unsigned int n;
+	int ret = importer->failed ? -1 : 0;
+
+	mail_free(&importer->mail);
+	mail_free(&importer->ext_mail);
+
+	/* commit saves */
+	if (mailbox_transaction_commit_get_changes(&importer->ext_trans,
+						   &changes) < 0) {
+		i_error("Mailbox %s: Save commit failed: %s",
+			mailbox_get_vname(importer->box),
+			mailbox_get_last_internal_error(importer->box, &importer->mail_error));
+		/* removed wanted_uids that weren't actually saved */
+		array_delete(&importer->wanted_uids,
+			     array_count(&importer->saved_uids),
+			     array_count(&importer->wanted_uids) -
+			     array_count(&importer->saved_uids));
+		mailbox_transaction_rollback(&importer->trans);
+		ret = -1;
+	} else {
+		/* remember the UIDs that were successfully saved */
+		if (importer->debug) T_BEGIN {
+			string_t *str = t_str_new(256);
+			imap_write_seq_range(str, &changes.saved_uids);
+			imp_debug(importer, "Saved UIDs: %s", str_c(str));
+		} T_END;
+		seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0;
+		while (seq_range_array_iter_nth(&iter, n++, &uid))
+			array_push_back(&importer->saved_uids, &uid);
+		pool_unref(&changes.pool);
+
+		/* commit flag changes and expunges */
+		if (mailbox_transaction_commit(&importer->trans) < 0) {
+			i_error("Mailbox %s: Commit failed: %s",
+				mailbox_get_vname(importer->box),
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			ret = -1;
+		}
+	}
+
+	if (!final)
+		dsync_mailbox_import_transaction_begin(importer);
+	return ret;
+}
+
+static int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer,
+				       const char **changes_during_sync_r)
+{
+	struct mailbox_update update;
+	int ret;
+
+	ret = dsync_mailbox_import_commit(importer, TRUE);
+
+	if (ret == 0) {
+		/* update mailbox metadata if we successfully saved
+		   everything. */
+		i_zero(&update);
+		update.min_next_uid = importer->remote_uid_next;
+		update.min_first_recent_uid =
+			I_MIN(importer->last_common_uid+1,
+			      importer->remote_first_recent_uid);
+		update.min_highest_modseq = importer->remote_highest_modseq;
+		update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq;
+
+		imp_debug(importer, "Finish update: min_next_uid=%u "
+			  "min_first_recent_uid=%u min_highest_modseq=%"PRIu64" "
+			  "min_highest_pvt_modseq=%"PRIu64,
+			  update.min_next_uid, update.min_first_recent_uid,
+			  update.min_highest_modseq,
+			  update.min_highest_pvt_modseq);
+
+		if (mailbox_update(importer->box, &update) < 0) {
+			i_error("Mailbox %s: Update failed: %s",
+				mailbox_get_vname(importer->box),
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			ret = -1;
+		}
+	}
+
+	/* sync mailbox to finish flag changes and expunges. */
+	if (mailbox_sync(importer->box, 0) < 0) {
+		i_error("Mailbox %s: Sync failed: %s",
+			mailbox_get_vname(importer->box),
+			mailbox_get_last_internal_error(importer->box,
+							&importer->mail_error));
+		ret = -1;
+	}
+	if (ret == 0) {
+		/* give new UIDs to messages that got saved with unwanted UIDs.
+		   do it only if the whole transaction succeeded. */
+		if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0)
+			ret = -1;
+	}
+	return ret;
+}
+
+static void
+dsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer)
+{
+	struct hash_iterate_context *iter;
+	const char *key;
+	struct importer_new_mail *mail;
+
+	iter = hash_table_iterate_init(importer->import_guids);
+	while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
+		for (; mail != NULL; mail = mail->next) {
+			if (mail->skip)
+				continue;
+
+			i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)",
+				mailbox_get_vname(importer->box),
+				mail->guid, mail->remote_uid);
+			importer->mail_error = MAIL_ERROR_TEMP;
+			importer->failed = TRUE;
+		}
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+static void
+dsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer)
+{
+	struct hash_iterate_context *iter;
+	void *key;
+	struct importer_new_mail *mail;
+
+	iter = hash_table_iterate_init(importer->import_uids);
+	while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) {
+		for (; mail != NULL; mail = mail->next) {
+			if (mail->skip)
+				continue;
+
+			i_error("Mailbox %s: Remote didn't send mail UID=%u",
+				mailbox_get_vname(importer->box),
+				mail->remote_uid);
+			importer->mail_error = MAIL_ERROR_TEMP;
+			importer->failed = TRUE;
+		}
+	}
+	hash_table_iterate_deinit(&iter);
+}
+
+int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
+				bool success,
+				uint32_t *last_common_uid_r,
+				uint64_t *last_common_modseq_r,
+				uint64_t *last_common_pvt_modseq_r,
+				uint32_t *last_messages_count_r,
+				const char **changes_during_sync_r,
+				bool *require_full_resync_r,
+				enum mail_error *error_r)
+{
+	struct dsync_mailbox_importer *importer = *_importer;
+	struct mailbox_status status;
+	int ret;
+
+	*_importer = NULL;
+	*changes_during_sync_r = NULL;
+	*require_full_resync_r = importer->require_full_resync;
+
+	if ((!success || importer->require_full_resync) && !importer->failed) {
+		importer->mail_error = MAIL_ERROR_TEMP;
+		importer->failed = TRUE;
+	}
+
+	if (!importer->new_uids_assigned && !importer->failed)
+		dsync_mailbox_import_assign_new_uids(importer);
+
+	if (!importer->failed) {
+		dsync_mailbox_import_check_missing_guid_imports(importer);
+		dsync_mailbox_import_check_missing_uid_imports(importer);
+	}
+
+	if (importer->search_ctx != NULL) {
+		if (mailbox_search_deinit(&importer->search_ctx) < 0) {
+			i_error("Mailbox %s: Search failed: %s",
+				mailbox_get_vname(importer->box),
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			importer->failed = TRUE;
+		}
+	}
+	if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0)
+		importer->failed = TRUE;
+
+	if (importer->virtual_mail != NULL)
+		mail_free(&importer->virtual_mail);
+	if (importer->virtual_trans != NULL)
+		(void)mailbox_transaction_commit(&importer->virtual_trans);
+
+	hash_table_destroy(&importer->import_guids);
+	hash_table_destroy(&importer->import_uids);
+	array_free(&importer->maybe_expunge_uids);
+	array_free(&importer->maybe_saves);
+	array_free(&importer->wanted_uids);
+	array_free(&importer->saved_uids);
+	array_free(&importer->newmails);
+	if (array_is_created(&importer->mail_requests))
+		array_free(&importer->mail_requests);
+
+	*last_common_uid_r = importer->last_common_uid;
+	if (*changes_during_sync_r == NULL) {
+		*last_common_modseq_r = importer->remote_highest_modseq;
+		*last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq;
+	} else {
+		/* local changes occurred during dsync. we exported changes up
+		   to local_initial_highestmodseq, so all of the changes have
+		   happened after it. we want the next run to see those changes,
+		   so return it as the last common modseq */
+		*last_common_modseq_r = importer->local_initial_highestmodseq;
+		*last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq;
+	}
+	if (importer->delete_mailbox) {
+		if (mailbox_delete(importer->box) < 0) {
+			i_error("Couldn't delete mailbox %s: %s",
+				mailbox_get_vname(importer->box),
+				mailbox_get_last_internal_error(importer->box,
+								&importer->mail_error));
+			importer->failed = TRUE;
+		}
+		*last_messages_count_r = 0;
+	} else {
+		mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status);
+		*last_messages_count_r = status.messages;
+	}
+
+	i_assert(importer->failed == (importer->mail_error != 0));
+	ret = importer->failed ? -1 : 0;
+	*error_r = importer->mail_error;
+	pool_unref(&importer->pool);
+	return ret;
+}
+
+const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer)
+{
+	if (importer->search_ctx != NULL)
+		return "";
+	return t_strdup_printf("%u/%u", importer->import_pos,
+			       importer->import_count);
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
new file mode 100644
index 0000000..6a2020e
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -0,0 +1,63 @@
+#ifndef DSYNC_MAILBOX_IMPORT_H
+#define DSYNC_MAILBOX_IMPORT_H
+
+#include "mail-error.h"
+
+enum dsync_mailbox_import_flags {
+	DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN		= 0x01,
+	DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS	= 0x02,
+	DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES	= 0x04,
+	DSYNC_MAILBOX_IMPORT_FLAG_DEBUG			= 0x08,
+	DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS	= 0x10,
+	DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128	= 0x20,
+	DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY		= 0x40,
+	DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND	= 0x100
+};
+
+struct mailbox;
+struct dsync_mailbox_attribute;
+struct dsync_mail;
+struct dsync_mail_change;
+struct dsync_transaction_log_scan;
+
+struct dsync_mailbox_importer *
+dsync_mailbox_import_init(struct mailbox *box,
+			  struct mailbox *virtual_all_box,
+			  struct dsync_transaction_log_scan *log_scan,
+			  uint32_t last_common_uid,
+			  uint64_t last_common_modseq,
+			  uint64_t last_common_pvt_modseq,
+			  uint32_t remote_uid_next,
+			  uint32_t remote_first_recent_uid,
+			  uint64_t remote_highest_modseq,
+			  uint64_t remote_highest_pvt_modseq,
+			  time_t sync_since_timestamp,
+			  time_t sync_until_timestamp,
+			  uoff_t sync_max_size,
+			  const char *sync_flag,
+			  unsigned int commit_msgs_interval,
+			  enum dsync_mailbox_import_flags flags,
+			  unsigned int hdr_hash_version,
+			  const char *const *hashed_headers);
+int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
+				   const struct dsync_mailbox_attribute *attr);
+int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
+				const struct dsync_mail_change *change);
+int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer);
+const struct dsync_mail_request *
+dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer);
+int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
+			      const struct dsync_mail *mail);
+int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer,
+				bool success,
+				uint32_t *last_common_uid_r,
+				uint64_t *last_common_modseq_r,
+				uint64_t *last_common_pvt_modseq_r,
+				uint32_t *last_messages_count_r,
+				const char **changes_during_sync_r,
+				bool *require_full_resync_r,
+				enum mail_error *error_r);
+
+const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-state.c b/src/doveadm/dsync/dsync-mailbox-state.c
new file mode 100644
index 0000000..cca24d4
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-state.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "base64.h"
+#include "crc32.h"
+#include "hash.h"
+#include "dsync-mailbox-state.h"
+
+#define DSYNC_STATE_MAJOR_VERSION 1
+#define DSYNC_STATE_MINOR_VERSION 0
+
+#define V0_MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8)
+#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8 + 4)
+
+static void put_uint32(buffer_t *output, uint32_t num)
+{
+	uint8_t tmp[sizeof(uint32_t)];
+
+	cpu32_to_le_unaligned(num, tmp);
+
+	buffer_append(output, tmp, sizeof(tmp));
+}
+
+void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states,
+				 string_t *output)
+{
+	struct hash_iterate_context *iter;
+	struct dsync_mailbox_state *state;
+	uint8_t *guid;
+	buffer_t *buf = t_buffer_create(128);
+	uint32_t crc = 0;
+
+	buffer_append_c(buf, DSYNC_STATE_MAJOR_VERSION);
+	buffer_append_c(buf, DSYNC_STATE_MINOR_VERSION);
+	buffer_append_c(buf, '\0');
+	buffer_append_c(buf, '\0');
+
+	iter = hash_table_iterate_init(states);
+	while (hash_table_iterate(iter, states, &guid, &state)) {
+		buffer_append(buf, state->mailbox_guid,
+			      sizeof(state->mailbox_guid));
+		put_uint32(buf, state->last_uidvalidity);
+		put_uint32(buf, state->last_common_uid);
+		put_uint32(buf, state->last_common_modseq & 0xffffffffU);
+		put_uint32(buf, state->last_common_modseq >> 32);
+		put_uint32(buf, state->last_common_pvt_modseq & 0xffffffffU);
+		put_uint32(buf, state->last_common_pvt_modseq >> 32);
+		put_uint32(buf, state->last_messages_count); /* v1 */
+		if (buf->used % 3 == 0) {
+			crc = crc32_data_more(crc, buf->data, buf->used);
+			base64_encode(buf->data, buf->used, output);
+			buffer_set_used_size(buf, 0);
+		}
+	}
+	hash_table_iterate_deinit(&iter);
+
+	crc = crc32_data_more(crc, buf->data, buf->used);
+	put_uint32(buf, crc);
+	base64_encode(buf->data, buf->used, output);
+}
+
+static int dsync_mailbox_states_retry_import_v0(const buffer_t *buf)
+{
+	const unsigned char *data = buf->data;
+
+	/* v0 had no version header and no last_messages_count */
+
+	if ((buf->used-4) % V0_MAILBOX_SIZE != 0 ||
+	    le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4))
+		return -1;
+	/* looks like valid v0 format, silently treat it as empty state */
+	return 0;
+}
+
+int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states,
+				pool_t pool, const char *input,
+				const char **error_r)
+{
+	struct dsync_mailbox_state *state;
+	buffer_t *buf;
+	uint8_t *guid_p;
+	const unsigned char *data;
+	unsigned int i, count;
+
+	buf = t_buffer_create(strlen(input));
+	if (base64_decode(input, strlen(input), NULL, buf) < 0) {
+		*error_r = "Invalid base64 data";
+		return -1;
+	}
+	/* v1: 4 byte header, mailboxes[], CRC32 */
+	data = buf->data;
+
+	if (buf->used == 4 && le32_to_cpu_unaligned(data) == 0) {
+		/* v0: Empty state */
+		return 0;
+	}
+	if (buf->used < 8) {
+		*error_r = "Input too small";
+		return -1;
+	}
+
+	if ((buf->used-8) % MAILBOX_SIZE != 0) {
+		*error_r = "Invalid input size";
+		return dsync_mailbox_states_retry_import_v0(buf);
+	}
+
+	if (le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4)) {
+		*error_r = "CRC32 mismatch";
+		return dsync_mailbox_states_retry_import_v0(buf);
+	}
+	data += 4;
+	count = (buf->used-8) / MAILBOX_SIZE;
+
+	for (i = 0; i < count; i++, data += MAILBOX_SIZE) {
+		state = p_new(pool, struct dsync_mailbox_state, 1);
+		memcpy(state->mailbox_guid, data, GUID_128_SIZE);
+		state->last_uidvalidity = le32_to_cpu_unaligned(data + GUID_128_SIZE);
+		state->last_common_uid = le32_to_cpu_unaligned(data + GUID_128_SIZE + 4);
+		state->last_common_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 8);
+		state->last_common_pvt_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 16);
+		state->last_messages_count = le32_to_cpu_unaligned(data + GUID_128_SIZE + 24);
+		guid_p = state->mailbox_guid;
+		hash_table_insert(states, guid_p, state);
+	}
+	return 0;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-state.h b/src/doveadm/dsync/dsync-mailbox-state.h
new file mode 100644
index 0000000..a01f531
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-state.h
@@ -0,0 +1,24 @@
+#ifndef DSYNC_MAILBOX_STATE_H
+#define DSYNC_MAILBOX_STATE_H
+
+#include "guid.h"
+
+struct dsync_mailbox_state {
+	guid_128_t mailbox_guid;
+	uint32_t last_uidvalidity;
+	uint32_t last_common_uid;
+	uint64_t last_common_modseq;
+	uint64_t last_common_pvt_modseq;
+	uint32_t last_messages_count;
+	bool changes_during_sync;
+};
+ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state);
+HASH_TABLE_DEFINE_TYPE(dsync_mailbox_state, uint8_t *, struct dsync_mailbox_state *);
+
+void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states,
+				 string_t *output);
+int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states,
+				pool_t pool, const char *input,
+				const char **error_r);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-fill.c b/src/doveadm/dsync/dsync-mailbox-tree-fill.c
new file mode 100644
index 0000000..c523e6b
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c
@@ -0,0 +1,405 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "guid.h"
+#include "str.h"
+#include "wildcard-match.h"
+#include "mailbox-log.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "mailbox-list-iter.h"
+#include "dsync-mailbox-tree-private.h"
+
+static int
+dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree,
+			    const struct mailbox_info *info,
+			    struct dsync_mailbox_node **node_r)
+{
+	struct dsync_mailbox_node *node;
+
+	node = dsync_mailbox_tree_get(tree, info->vname);
+	if (node->ns == info->ns)
+		;
+	else if (node->ns == NULL) {
+		i_assert(tree->root.ns == NULL);
+		node->ns = info->ns;
+	} else {
+		i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'",
+			info->vname, node->ns->prefix, info->ns->prefix);
+		return -1;
+	}
+	*node_r = node;
+	return 0;
+}
+
+static int
+dsync_mailbox_tree_add_exists_node(struct dsync_mailbox_tree *tree,
+				   const struct mailbox_info *info,
+				   struct dsync_mailbox_node **node_r,
+				   enum mail_error *error_r)
+{
+	if (dsync_mailbox_tree_add_node(tree, info, node_r) < 0) {
+		*error_r = MAIL_ERROR_TEMP;
+		return -1;
+	}
+	(*node_r)->existence = DSYNC_MAILBOX_NODE_EXISTS;
+	return 0;
+}
+
+static int
+dsync_mailbox_tree_get_selectable(struct mailbox *box,
+				  struct mailbox_metadata *metadata_r,
+				  struct mailbox_status *status_r)
+{
+	/* try the fast path */
+	if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0)
+		return -1;
+	if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
+		return -1;
+
+	i_assert(!guid_128_is_empty(metadata_r->guid));
+	if (status_r->uidvalidity != 0)
+		return 0;
+
+	/* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */
+	if (mailbox_sync(box, 0) < 0)
+		return -1;
+	if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
+		return -1;
+	i_assert(status_r->uidvalidity != 0);
+	return 0;
+}
+
+static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree,
+				  const struct mailbox_info *info,
+				  const guid_128_t box_guid,
+				  enum mail_error *error_r)
+{
+	struct dsync_mailbox_node *node;
+	struct mailbox *box;
+	struct mailbox_metadata metadata;
+	struct mailbox_status status;
+	const char *errstr;
+	enum mail_error error;
+	int ret = 0;
+
+	if ((info->flags & MAILBOX_NONEXISTENT) != 0)
+		return 0;
+	if ((info->flags & MAILBOX_NOSELECT) != 0) {
+		return !guid_128_is_empty(box_guid) ? 0 :
+			dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r);
+	}
+
+	/* get GUID and UIDVALIDITY for selectable mailbox */
+	box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY);
+	if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) {
+		errstr = mailbox_get_last_internal_error(box, &error);
+		switch (error) {
+		case MAIL_ERROR_NOTFOUND:
+			/* mailbox was just deleted? */
+			break;
+		case MAIL_ERROR_NOTPOSSIBLE:
+			/* invalid mbox files? ignore */
+			break;
+		default:
+			i_error("Failed to access mailbox %s: %s",
+				info->vname, errstr);
+			*error_r = error;
+			ret = -1;
+		}
+		mailbox_free(&box);
+		return ret;
+	}
+	mailbox_free(&box);
+
+	if (!guid_128_is_empty(box_guid) &&
+	    !guid_128_equals(box_guid, metadata.guid)) {
+		/* unwanted mailbox */
+		return 0;
+	}
+	if (dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r) < 0)
+		return -1;
+	memcpy(node->mailbox_guid, metadata.guid,
+	       sizeof(node->mailbox_guid));
+	node->uid_validity = status.uidvalidity;
+	node->uid_next = status.uidnext;
+	return 0;
+}
+
+static struct dsync_mailbox_node *
+dsync_mailbox_tree_find_sha(struct dsync_mailbox_tree *tree,
+			    struct mail_namespace *ns, const guid_128_t sha128)
+{
+	struct dsync_mailbox_node *node;
+
+	if (!hash_table_is_created(tree->name128_hash))
+		dsync_mailbox_tree_build_name128_hash(tree);
+
+	node = hash_table_lookup(tree->name128_hash, sha128);
+	return node == NULL || node->ns != ns ? NULL : node;
+}
+
+static int
+dsync_mailbox_tree_add_change_timestamps(struct dsync_mailbox_tree *tree,
+					 struct mail_namespace *ns)
+{
+	struct dsync_mailbox_node *node;
+	struct dsync_mailbox_delete *del;
+	struct mailbox_log *log;
+	struct mailbox_log_iter *iter;
+	const struct mailbox_log_record *rec;
+	const uint8_t *guid_p;
+	time_t timestamp;
+
+	log = mailbox_list_get_changelog(ns->list);
+	if (log == NULL)
+		return 0;
+
+	iter = mailbox_log_iter_init(log);
+	while ((rec = mailbox_log_iter_next(iter)) != NULL) {
+		/* For DELETE_MAILBOX the record_guid is the mailbox GUID.
+		   Otherwise it's 128bit SHA1 of the mailbox vname. */
+		node = rec->type == MAILBOX_LOG_RECORD_DELETE_MAILBOX ? NULL :
+			dsync_mailbox_tree_find_sha(tree, ns, rec->mailbox_guid);
+
+		timestamp = mailbox_log_record_get_timestamp(rec);
+		switch (rec->type) {
+		case MAILBOX_LOG_RECORD_DELETE_MAILBOX:
+			guid_p = rec->mailbox_guid;
+			if (hash_table_lookup(tree->guid_hash, guid_p) != NULL) {
+				/* mailbox still exists. maybe it was restored
+				   from backup or something. */
+				break;
+			}
+			del = array_append_space(&tree->deletes);
+			del->type = DSYNC_MAILBOX_DELETE_TYPE_MAILBOX;
+			del->timestamp = timestamp;
+			memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
+			break;
+		case MAILBOX_LOG_RECORD_DELETE_DIR:
+			if (node != NULL &&
+			    node->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+				/* directory exists again, skip it */
+				break;
+			}
+			/* we don't know what directory name was deleted,
+			   just its hash. if the name still exists on the other
+			   dsync side, it can match this deletion to the
+			   name. */
+			del = array_append_space(&tree->deletes);
+			del->type = DSYNC_MAILBOX_DELETE_TYPE_DIR;
+			del->timestamp = timestamp;
+			memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
+			break;
+		case MAILBOX_LOG_RECORD_CREATE_DIR:
+			if (node == NULL) {
+				/* directory has been deleted again, skip it */
+				break;
+			}
+			/* notify the remote that we want to keep this
+			   directory created (unless remote has a newer delete
+			   timestamp) */
+			node->last_renamed_or_created = timestamp;
+			break;
+		case MAILBOX_LOG_RECORD_RENAME:
+			if (node != NULL)
+				node->last_renamed_or_created = timestamp;
+			break;
+		case MAILBOX_LOG_RECORD_SUBSCRIBE:
+			if (node != NULL)
+				node->last_subscription_change = timestamp;
+			break;
+		case MAILBOX_LOG_RECORD_UNSUBSCRIBE:
+			if (node != NULL) {
+				node->last_subscription_change = timestamp;
+				break;
+			}
+			/* The mailbox is already deleted, but it may still
+			   exist on the other side (even the subscription
+			   alone). */
+			del = array_append_space(&tree->deletes);
+			del->type = DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE;
+			del->timestamp = timestamp;
+			memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
+			break;
+		}
+	}
+	if (mailbox_log_iter_deinit(&iter) < 0) {
+		i_error("Mailbox log iteration for namespace '%s' failed",
+			ns->prefix);
+		return -1;
+	}
+	return 0;
+}
+
+static int
+dsync_mailbox_tree_fix_guid_duplicate(struct dsync_mailbox_tree *tree,
+				      struct dsync_mailbox_node *node1,
+				      struct dsync_mailbox_node *node2)
+{
+	struct mailbox *box;
+	struct mailbox_update update;
+	struct dsync_mailbox_node *change_node;
+	const char *change_vname;
+	int ret = 0;
+
+	i_zero(&update);
+	guid_128_generate(update.mailbox_guid);
+
+	/* just in case the duplication exists in both sides,
+	   make them choose the same node */
+	if (strcmp(dsync_mailbox_node_get_full_name(tree, node1),
+		   dsync_mailbox_node_get_full_name(tree, node2)) <= 0)
+		change_node = node1;
+	else
+		change_node = node2;
+
+	change_vname = dsync_mailbox_node_get_full_name(tree, change_node);
+	i_error("Duplicate mailbox GUID %s for mailboxes %s and %s - "
+		"giving a new GUID %s to %s",
+		guid_128_to_string(node1->mailbox_guid),
+		dsync_mailbox_node_get_full_name(tree, node1),
+		dsync_mailbox_node_get_full_name(tree, node2),
+		guid_128_to_string(update.mailbox_guid), change_vname);
+
+	i_assert(node1->ns != NULL && node2->ns != NULL);
+	box = mailbox_alloc(change_node->ns->list, change_vname, 0);
+	if (mailbox_update(box, &update) < 0) {
+		i_error("Couldn't update mailbox %s GUID: %s",
+			change_vname, mailbox_get_last_internal_error(box, NULL));
+		ret = -1;
+	} else {
+		memcpy(change_node->mailbox_guid, update.mailbox_guid,
+		       sizeof(change_node->mailbox_guid));
+	}
+	mailbox_free(&box);
+	return ret;
+}
+
+static bool
+dsync_mailbox_info_is_wanted(const struct mailbox_info *info,
+			     const char *box_name,
+			     const char *const *exclude_mailboxes)
+{
+	const char *const *info_specialuses;
+	unsigned int i;
+
+	if (exclude_mailboxes == NULL &&
+	    (box_name == NULL || box_name[0] != '\\'))
+		return TRUE;
+
+	info_specialuses = info->special_use == NULL ? NULL :
+		t_strsplit(info->special_use, " ");
+	/* include */
+	if (box_name != NULL && box_name[0] == '\\') {
+		if (info_specialuses == NULL ||
+		    !str_array_icase_find(info_specialuses, box_name))
+			return FALSE;
+	}
+	/* exclude */
+	if (exclude_mailboxes == NULL)
+		return TRUE;
+	for (i = 0; exclude_mailboxes[i] != NULL; i++) {
+		const char *exclude = exclude_mailboxes[i];
+
+		if (exclude[0] == '\\') {
+			/* special-use */
+			if (info_specialuses != NULL &&
+			    str_array_icase_find(info_specialuses, exclude))
+				return FALSE;
+		} else {
+			/* mailbox with wildcards */
+			if (wildcard_match(info->vname, exclude))
+				return FALSE;
+		}
+	}
+	return TRUE;
+}
+
+int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree,
+			    struct mail_namespace *ns, const char *box_name,
+			    const guid_128_t box_guid,
+			    const char *const *exclude_mailboxes,
+			    enum mail_error *error_r)
+{
+	const enum mailbox_list_iter_flags list_flags =
+		/* FIXME: we'll skip symlinks, because we can't handle them
+		   currently. in future we could detect them and create them
+		   by creating the symlink. */
+		MAILBOX_LIST_ITER_SKIP_ALIASES |
+		MAILBOX_LIST_ITER_NO_AUTO_BOXES;
+	const enum mailbox_list_iter_flags subs_list_flags =
+		MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+		MAILBOX_LIST_ITER_SELECT_SUBSCRIBED |
+		MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+	struct mailbox_list_iterate_context *iter;
+	struct dsync_mailbox_node *node, *dup_node1, *dup_node2;
+	const struct mailbox_info *info;
+	const char *list_pattern =
+		box_name != NULL && box_name[0] != '\\' ? box_name : "*";
+	int ret = 0;
+
+	i_assert(mail_namespace_get_sep(ns) == tree->sep);
+
+	/* assign namespace to its root, so it gets copied to children */
+	if (ns->prefix_len > 0) {
+		const char *vname = t_strndup(ns->prefix, ns->prefix_len-1);
+		node = dsync_mailbox_tree_get(tree, vname);
+		node->ns = ns;
+
+		struct mailbox_info ns_info = {
+			.vname = vname,
+			.ns = ns,
+		};
+		if (dsync_mailbox_tree_add(tree, &ns_info, box_guid, error_r) < 0)
+			return -1;
+	} else {
+		tree->root.ns = ns;
+	}
+
+	/* first add all of the existing mailboxes */
+	iter = mailbox_list_iter_init(ns->list, list_pattern, list_flags);
+	while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+		if (dsync_mailbox_info_is_wanted(info, box_name,
+						 exclude_mailboxes)) {
+			if (dsync_mailbox_tree_add(tree, info, box_guid, error_r) < 0)
+				ret = -1;
+		}
+	} T_END;
+	if (mailbox_list_iter_deinit(&iter) < 0) {
+		i_error("Mailbox listing for namespace '%s' failed: %s",
+			ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r));
+		ret = -1;
+	}
+
+	/* add subscriptions */
+	iter = mailbox_list_iter_init(ns->list, list_pattern, subs_list_flags);
+	while ((info = mailbox_list_iter_next(iter)) != NULL) {
+		if (dsync_mailbox_tree_add_node(tree, info, &node) == 0)
+			node->subscribed = TRUE;
+		else {
+			*error_r = MAIL_ERROR_TEMP;
+			ret = -1;
+		}
+	}
+	if (mailbox_list_iter_deinit(&iter) < 0) {
+		i_error("Mailbox listing for namespace '%s' failed: %s",
+			ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r));
+		ret = -1;
+	}
+	if (ret < 0)
+		return -1;
+
+	while (dsync_mailbox_tree_build_guid_hash(tree, &dup_node1,
+						  &dup_node2) < 0) {
+		if (dsync_mailbox_tree_fix_guid_duplicate(tree, dup_node1, dup_node2) < 0)
+			return -1;
+	}
+
+	/* add timestamps */
+	if (dsync_mailbox_tree_add_change_timestamps(tree, ns) < 0)
+		return -1;
+	return 0;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-private.h b/src/doveadm/dsync/dsync-mailbox-tree-private.h
new file mode 100644
index 0000000..0614151
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree-private.h
@@ -0,0 +1,38 @@
+#ifndef DSYNC_MAILBOX_TREE_PRIVATE_H
+#define DSYNC_MAILBOX_TREE_PRIVATE_H
+
+#include "dsync-mailbox-tree.h"
+
+struct dsync_mailbox_tree {
+	pool_t pool;
+	char sep, sep_str[2], remote_sep, alt_char;
+	char escape_char, remote_escape_char;
+	/* root node isn't part of the real mailbox tree. its name is "" and
+	   it has no siblings */
+	struct dsync_mailbox_node root;
+
+	unsigned int iter_count;
+
+	ARRAY(struct dsync_mailbox_delete) deletes;
+
+	/* guid_128_t => struct dsync_mailbox_node */
+	HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_hash;
+	HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_remotesep_hash;
+	HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) guid_hash;
+};
+
+void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree);
+
+int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1,
+				struct dsync_mailbox_node *const *n2);
+
+void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node,
+				    struct dsync_mailbox_node *parent);
+void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node);
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src);
+bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1,
+			       struct dsync_mailbox_tree *tree2);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
new file mode 100644
index 0000000..e4a50ae
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
@@ -0,0 +1,1473 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "str.h"
+#include "md5.h"
+#include "hex-binary.h"
+#include "aqueue.h"
+#include "hash.h"
+#include "dsync-brain-private.h"
+#include "dsync-mailbox-tree-private.h"
+
+#define TEMP_MAX_NAME_LEN 100
+#define TEMP_SUFFIX_MAX_LEN (sizeof("temp-")-1 + 8)
+#define TEMP_SUFFIX_FORMAT "temp-%x"
+
+struct dsync_mailbox_tree_bfs_iter {
+	struct dsync_mailbox_tree *tree;
+
+	ARRAY(struct dsync_mailbox_node *) queue_arr;
+	struct aqueue *queue;
+	struct dsync_mailbox_node *cur;
+};
+
+struct dsync_mailbox_tree_sync_ctx {
+	pool_t pool;
+	struct dsync_mailbox_tree *local_tree, *remote_tree;
+	enum dsync_mailbox_trees_sync_type sync_type;
+	enum dsync_mailbox_trees_sync_flags sync_flags;
+	unsigned int combined_mailboxes_count;
+
+	ARRAY(struct dsync_mailbox_tree_sync_change) changes;
+	unsigned int change_idx;
+	bool failed;
+};
+
+static struct dsync_mailbox_tree_bfs_iter *
+dsync_mailbox_tree_bfs_iter_init(struct dsync_mailbox_tree *tree)
+{
+	struct dsync_mailbox_tree_bfs_iter *iter;
+
+	iter = i_new(struct dsync_mailbox_tree_bfs_iter, 1);
+	iter->tree = tree;
+	i_array_init(&iter->queue_arr, 32);
+	iter->queue = aqueue_init(&iter->queue_arr.arr);
+	iter->cur = tree->root.first_child;
+	return iter;
+}
+
+static bool
+dsync_mailbox_tree_bfs_iter_next(struct dsync_mailbox_tree_bfs_iter *iter,
+				 struct dsync_mailbox_node **node_r)
+{
+	if (iter->cur == NULL) {
+		if (aqueue_count(iter->queue) == 0)
+			return FALSE;
+		iter->cur = array_idx_elem(&iter->queue_arr,
+					   aqueue_idx(iter->queue, 0));
+		aqueue_delete_tail(iter->queue);
+	}
+	*node_r = iter->cur;
+
+	if (iter->cur->first_child != NULL)
+		aqueue_append(iter->queue, &iter->cur->first_child);
+	iter->cur = iter->cur->next;
+	return TRUE;
+}
+
+static void
+dsync_mailbox_tree_bfs_iter_deinit(struct dsync_mailbox_tree_bfs_iter **_iter)
+{
+	struct dsync_mailbox_tree_bfs_iter *iter = *_iter;
+
+	*_iter = NULL;
+
+	aqueue_deinit(&iter->queue);
+	array_free(&iter->queue_arr);
+	i_free(iter);
+}
+
+static void
+sync_add_dir_change(struct dsync_mailbox_tree_sync_ctx *ctx,
+		    const struct dsync_mailbox_node *node,
+		    enum dsync_mailbox_tree_sync_type type)
+{
+	struct dsync_mailbox_tree_sync_change *change;
+	const char *name;
+
+	i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+
+	name = dsync_mailbox_node_get_full_name(ctx->local_tree, node);
+
+	change = array_append_space(&ctx->changes);
+	change->type = type;
+	change->ns = node->ns;
+	change->full_name = p_strdup(ctx->pool, name);
+}
+
+static void
+sync_add_create_change(struct dsync_mailbox_tree_sync_ctx *ctx,
+		       const struct dsync_mailbox_node *node, const char *name)
+{
+	struct dsync_mailbox_tree_sync_change *change;
+
+	i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+
+	change = array_append_space(&ctx->changes);
+	change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX;
+	change->ns = node->ns;
+	change->full_name = p_strdup(ctx->pool, name);
+	memcpy(change->mailbox_guid, node->mailbox_guid,
+	       sizeof(change->mailbox_guid));
+	change->uid_validity = node->uid_validity;
+}
+
+static void sort_siblings(ARRAY_TYPE(dsync_mailbox_node) *siblings)
+{
+	struct dsync_mailbox_node *const *nodes;
+	unsigned int i, count;
+
+	array_sort(siblings, dsync_mailbox_node_name_cmp);
+
+	nodes = array_get(siblings, &count);
+	if (count == 0)
+		return;
+
+	nodes[0]->parent->first_child = nodes[0];
+	for (i = 1; i < count; i++)
+		nodes[i-1]->next = nodes[i];
+	nodes[count-1]->next = NULL;
+}
+
+static void
+sync_set_node_deleted(struct dsync_mailbox_tree *tree,
+		      struct dsync_mailbox_node *node)
+{
+	const uint8_t *guid_p;
+
+	/* for the rest of this sync assume that the mailbox has
+	   already been deleted */
+	guid_p = node->mailbox_guid;
+	hash_table_remove(tree->guid_hash, guid_p);
+
+	node->existence = DSYNC_MAILBOX_NODE_DELETED;
+	memset(node->mailbox_guid, 0, sizeof(node->mailbox_guid));
+	node->uid_validity = 0;
+}
+
+static void
+sync_delete_mailbox_node(struct dsync_mailbox_tree_sync_ctx *ctx,
+			 struct dsync_mailbox_tree *tree,
+			 struct dsync_mailbox_node *node, const char *reason)
+{
+	struct dsync_mailbox_tree_sync_change *change;
+	const char *name;
+
+	if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 &&
+	    tree == ctx->local_tree) {
+		i_debug("brain %c: Deleting mailbox '%s' (GUID %s): %s",
+			(ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+			dsync_mailbox_node_get_full_name(tree, node),
+			guid_128_to_string(node->mailbox_guid), reason);
+	}
+
+	if (tree == ctx->local_tree &&
+	    node->existence != DSYNC_MAILBOX_NODE_DELETED) {
+		/* delete this mailbox locally */
+		i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+		change = array_append_space(&ctx->changes);
+		change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX;
+		change->ns = node->ns;
+		name = dsync_mailbox_node_get_full_name(tree, node);
+		change->full_name = p_strdup(ctx->pool, name);
+		memcpy(change->mailbox_guid, node->mailbox_guid,
+		       sizeof(change->mailbox_guid));
+	}
+	sync_set_node_deleted(tree, node);
+}
+
+static void
+sync_delete_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx,
+		    struct dsync_mailbox_tree *tree,
+		    struct dsync_mailbox_node *node, const char *reason)
+{
+	struct dsync_mailbox_tree *other_tree;
+	struct dsync_mailbox_node *other_node;
+	const uint8_t *guid_p;
+
+	other_tree = tree == ctx->local_tree ?
+		ctx->remote_tree : ctx->local_tree;
+	guid_p = node->mailbox_guid;
+	other_node = hash_table_lookup(other_tree->guid_hash, guid_p);
+	if (other_node == NULL) {
+		/* doesn't exist / already deleted */
+	} else {
+		sync_delete_mailbox_node(ctx, other_tree, other_node, reason);
+	}
+	sync_delete_mailbox_node(ctx, tree, node, reason);
+}
+
+static void
+sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+				    struct dsync_mailbox_tree *tree,
+				    bool twoway_sync)
+{
+	struct dsync_mailbox_tree_bfs_iter *iter;
+	struct dsync_mailbox_node *node, *parent = NULL;
+	ARRAY_TYPE(dsync_mailbox_node) siblings;
+
+	t_array_init(&siblings, 64);
+
+	iter = dsync_mailbox_tree_bfs_iter_init(tree);
+	while (dsync_mailbox_tree_bfs_iter_next(iter, &node)) {
+		if (node->parent != parent) {
+			sort_siblings(&siblings);
+			array_clear(&siblings);
+			parent = node->parent;
+		}
+		if (node->existence == DSYNC_MAILBOX_NODE_DELETED &&
+		    !dsync_mailbox_node_is_dir(node)) {
+			if (twoway_sync) {
+				/* this mailbox was deleted. delete it from the
+				   other side as well */
+				sync_delete_mailbox(ctx, tree, node,
+						    "Mailbox has been deleted");
+			} else {
+				/* treat the node as if it didn't exist. it'll
+				   get either recreated or deleted later. in
+				   any case this function must handle all
+				   existence=DELETED mailbox nodes by changing
+				   them into directories (setting GUID=0) or
+				   we'll assert-crash later */
+				sync_set_node_deleted(tree, node);
+			}
+		}
+		ctx->combined_mailboxes_count++;
+		array_push_back(&siblings, &node);
+	}
+	sort_siblings(&siblings);
+	dsync_mailbox_tree_bfs_iter_deinit(&iter);
+}
+
+static bool node_names_equal(const struct dsync_mailbox_node *n1,
+			     const struct dsync_mailbox_node *n2)
+{
+	while (n1 != NULL && n2 != NULL) {
+		if (strcmp(n1->name, n2->name) != 0)
+			return FALSE;
+		n1 = n1->parent;
+		n2 = n2->parent;
+	}
+	return n1 == NULL && n2 == NULL;
+}
+
+static void
+dsync_mailbox_tree_node_attach_sorted(struct dsync_mailbox_node *node,
+				      struct dsync_mailbox_node *parent)
+{
+	struct dsync_mailbox_node **p;
+
+	node->parent = parent;
+	for (p = &parent->first_child; *p != NULL; p = &(*p)->next) {
+		if (dsync_mailbox_node_name_cmp(p, &node) > 0)
+			break;
+	}
+	node->next = *p;
+	*p = node;
+}
+
+static void
+dsync_mailbox_tree_node_move_sorted(struct dsync_mailbox_node *node,
+				    struct dsync_mailbox_node *parent)
+{
+	/* detach from old parent */
+	dsync_mailbox_tree_node_detach(node);
+	/* attach to new parent */
+	dsync_mailbox_tree_node_attach_sorted(node, parent);
+}
+
+static struct dsync_mailbox_node *
+sorted_tree_get(struct dsync_mailbox_tree *tree, const char *name)
+{
+	struct dsync_mailbox_node *node, *parent, *ret;
+
+	node = ret = dsync_mailbox_tree_get(tree, name);
+	while (node->parent != NULL &&
+	       node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) {
+		parent = node->parent;
+		dsync_mailbox_tree_node_detach(node);
+		dsync_mailbox_tree_node_attach_sorted(node, parent);
+		node = parent;
+	}
+	return ret;
+}
+
+static struct dsync_mailbox_node *
+sync_node_new(struct dsync_mailbox_tree *tree,
+	      struct dsync_mailbox_node **pos,
+	      struct dsync_mailbox_node *parent,
+	      const struct dsync_mailbox_node *src)
+{
+	struct dsync_mailbox_node *node;
+
+	node = p_new(tree->pool, struct dsync_mailbox_node, 1);
+	node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+	node->name = p_strdup(tree->pool, src->name);
+	node->sync_temporary_name = src->sync_temporary_name;
+	node->ns = src->ns;
+	node->parent = parent;
+	node->next = *pos;
+	*pos = node;
+	return node;
+}
+
+static struct dsync_mailbox_node *
+sorted_tree_get_by_node_name(struct dsync_mailbox_tree *tree,
+			     struct dsync_mailbox_tree *other_tree,
+			     struct dsync_mailbox_node *other_node)
+{
+	const char *parent_name;
+
+	if (other_node == &other_tree->root)
+		return &tree->root;
+
+	parent_name = dsync_mailbox_node_get_full_name(other_tree, other_node);
+	return sorted_tree_get(tree, parent_name);
+}
+
+static bool node_has_child(struct dsync_mailbox_node *parent, const char *name)
+{
+	struct dsync_mailbox_node *node;
+
+	for (node = parent->first_child; node != NULL; node = node->next) {
+		if (strcmp(node->name, name) == 0)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static bool
+node_has_existent_children(struct dsync_mailbox_node *node, bool dirs_ok)
+{
+	for (node = node->first_child; node != NULL; node = node->next) {
+		if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    (dirs_ok || !dsync_mailbox_node_is_dir(node)))
+			return TRUE;
+		if (node_has_existent_children(node, dirs_ok))
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static bool node_is_existent(struct dsync_mailbox_node *node)
+{
+	if (node->existence == DSYNC_MAILBOX_NODE_EXISTS)
+		return TRUE;
+	return node_has_existent_children(node, TRUE);
+}
+
+static bool sync_node_is_namespace_prefix(struct dsync_mailbox_tree *tree,
+					  struct dsync_mailbox_node *node)
+{
+	const char *full_name;
+	size_t prefix_len = node->ns == NULL ? 0 : node->ns->prefix_len;
+
+	if (strcmp(node->name, "INBOX") == 0 && node->parent == &tree->root)
+		return TRUE;
+
+	if (prefix_len == 0)
+		return FALSE;
+
+	full_name = dsync_mailbox_node_get_full_name(tree, node);
+	if (node->ns->prefix[prefix_len-1] == mail_namespace_get_sep(node->ns))
+		prefix_len--;
+	return strncmp(full_name, node->ns->prefix, prefix_len) == 0 &&
+		full_name[prefix_len] == '\0';
+}
+
+static void
+sync_rename_node_to_temp(struct dsync_mailbox_tree_sync_ctx *ctx,
+			 struct dsync_mailbox_tree *tree,
+			 struct dsync_mailbox_node *node,
+			 struct dsync_mailbox_node *new_parent,
+			 const char **reason_r)
+{
+	struct dsync_mailbox_tree_sync_change *change;
+	const char *old_name, *new_name, *p;
+	char name[TEMP_MAX_NAME_LEN+1];
+	buffer_t buf;
+	size_t prefix_len, max_prefix_len;
+	unsigned int counter = 1;
+
+	i_assert(!sync_node_is_namespace_prefix(tree, node));
+
+	buffer_create_from_data(&buf, name, sizeof(name));
+	max_prefix_len = TEMP_MAX_NAME_LEN - TEMP_SUFFIX_MAX_LEN - 1;
+	if (node->sync_temporary_name) {
+		/* the source name was also a temporary name. drop the
+		 -<suffix> from it */
+		p = strrchr(node->name, '-');
+		i_assert(p != NULL);
+		if (max_prefix_len > (size_t)(p - node->name))
+			max_prefix_len = p - node->name;
+	}
+	str_append_max(&buf, node->name, max_prefix_len);
+	str_append_c(&buf, '-');
+	prefix_len = buf.used;
+
+	do {
+		str_truncate(&buf, prefix_len);
+		str_printfa(&buf, TEMP_SUFFIX_FORMAT, counter++);
+		/* the generated name is quite unlikely to exist,
+		   but check anyway.. */
+	} while (node_has_child(node->parent, str_c(&buf)));
+
+	old_name = tree != ctx->local_tree ? NULL :
+		dsync_mailbox_node_get_full_name(tree, node);
+
+	*reason_r = t_strdup_printf("Renamed '%s' to '%s'", node->name, str_c(&buf));
+	node->name = p_strdup(tree->pool, str_c(&buf));
+	node->sync_temporary_name = TRUE;
+	node->last_renamed_or_created = 0;
+	dsync_mailbox_tree_node_move_sorted(node, new_parent);
+
+	if (tree == ctx->local_tree && node_is_existent(node)) {
+		/* we're modifying a local tree. remember this change. */
+		new_name = dsync_mailbox_node_get_full_name(tree, node);
+
+		i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+		i_assert(strcmp(old_name, "INBOX") != 0);
+		change = array_append_space(&ctx->changes);
+		change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME;
+		change->ns = node->ns;
+		change->full_name = p_strdup(ctx->pool, old_name);
+		change->rename_dest_name = p_strdup(ctx->pool, new_name);
+	}
+}
+
+static bool node_has_parent(struct dsync_mailbox_node *node,
+			    struct dsync_mailbox_node *parent)
+{
+	for (; node != NULL; node = node->parent) {
+		if (node == parent)
+			return TRUE;
+	}
+	return FALSE;
+}
+
+static void
+sync_rename_node(struct dsync_mailbox_tree_sync_ctx *ctx,
+		 struct dsync_mailbox_tree *tree,
+		 struct dsync_mailbox_node *temp_node,
+		 struct dsync_mailbox_node *node,
+		 const struct dsync_mailbox_node *other_node,
+		 const char **reason_r)
+{
+	struct dsync_mailbox_tree_sync_change *change;
+	struct dsync_mailbox_tree *other_tree;
+	struct dsync_mailbox_node *parent;
+	const char *name, *other_name;
+
+	i_assert(node != NULL);
+	i_assert(other_node != NULL);
+
+	/* move/rename node in the tree, so that its position/name is identical
+	   to other_node (in other_tree). temp_node's name is changed to
+	   temporary name (i.e. it assumes that node's name becomes temp_node's
+	   original name) */
+	other_tree = tree == ctx->local_tree ?
+		ctx->remote_tree : ctx->local_tree;
+
+	parent = sorted_tree_get_by_node_name(tree, other_tree,
+					      other_node->parent);
+	if (node_has_parent(parent, node)) {
+		/* don't introduce a loop. temporarily rename node
+		   under root. */
+		sync_rename_node_to_temp(ctx, tree, node, &tree->root, reason_r);
+		*reason_r = t_strconcat(*reason_r, " (Don't introduce loop)", NULL);
+		return;
+	}
+	sync_rename_node_to_temp(ctx, tree, temp_node, temp_node->parent, reason_r);
+
+	/* get the old name before it's modified */
+	name = dsync_mailbox_node_get_full_name(tree, node);
+
+	/* set the new name */
+	*reason_r = t_strdup_printf("%s + Renamed '%s' to '%s'",
+				    *reason_r, name, other_node->name);
+	node->name = p_strdup(tree->pool, other_node->name);
+	node->sync_temporary_name = other_node->sync_temporary_name;
+	node->last_renamed_or_created = other_node->last_renamed_or_created;
+	/* change node's parent if necessary. in any case detach+reattach it
+	   sorted, because the nodes must be sorted by name, and the node's
+	   name (or its parent) changed. */
+	dsync_mailbox_tree_node_move_sorted(node, parent);
+
+	if (tree == ctx->local_tree && node_is_existent(node)) {
+		/* we're modifying a local tree. remember this change. */
+		other_name = dsync_mailbox_node_get_full_name(other_tree, other_node);
+
+		i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+		i_assert(strcmp(name, "INBOX") != 0);
+		change = array_append_space(&ctx->changes);
+		change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME;
+		change->ns = node->ns;
+		change->full_name = p_strdup(ctx->pool, name);
+		change->rename_dest_name = p_strdup(ctx->pool, other_name);
+	}
+}
+
+static int node_mailbox_guids_cmp(struct dsync_mailbox_node *node1,
+				  struct dsync_mailbox_node *node2)
+{
+	int ret;
+
+	while (node1 != NULL && node2 != NULL) {
+		if (node1->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    node2->existence != DSYNC_MAILBOX_NODE_EXISTS)
+			return -1;
+		if (node2->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    node1->existence != DSYNC_MAILBOX_NODE_EXISTS)
+			return 1;
+
+		ret = memcmp(node1->mailbox_guid, node2->mailbox_guid,
+			     sizeof(node1->mailbox_guid));
+		if (ret != 0)
+			return ret;
+
+		ret = node_mailbox_guids_cmp(node1->first_child,
+					     node2->first_child);
+		if (ret != 0)
+			return ret;
+		node1 = node1->next;
+		node2 = node2->next;
+	}
+	if (node1 == NULL && node2 == NULL)
+		return 0;
+	return node1 != NULL ? -1 : 1;
+}
+
+static int node_mailbox_names_cmp(struct dsync_mailbox_node *node1,
+				  struct dsync_mailbox_node *node2)
+{
+	int ret;
+
+	while (node1 != NULL && node2 != NULL) {
+		ret = strcmp(node1->name, node2->name);
+		if (ret != 0)
+			return ret;
+
+		ret = node_mailbox_names_cmp(node1->first_child,
+					     node2->first_child);
+		if (ret != 0)
+			return ret;
+		node1 = node1->next;
+		node2 = node2->next;
+	}
+	if (node1 == NULL && node2 == NULL)
+		return 0;
+	return node1 != NULL ? -1 : 1;
+}
+
+static int node_mailbox_trees_cmp(struct dsync_mailbox_node *node1,
+				  struct dsync_mailbox_node *node2)
+{
+	int ret;
+
+	ret = node_mailbox_guids_cmp(node1, node2);
+	if (ret == 0) {
+		/* only a directory name changed and all the timestamps
+		   are equal. just pick the alphabetically smaller. */
+		ret = node_mailbox_names_cmp(node1, node2);
+	}
+	i_assert(ret != 0);
+	return ret;
+}
+
+static time_t nodes_get_timestamp(struct dsync_mailbox_node *node1,
+				  struct dsync_mailbox_node *node2)
+{
+	time_t ts = 0;
+
+	/* avoid using temporary names in case all the timestamps are 0 */
+	if (node1 != NULL && !node1->sync_temporary_name)
+		ts = node1->last_renamed_or_created + 1;
+	if (node2 != NULL && !node2->sync_temporary_name &&
+	    ts <= node2->last_renamed_or_created)
+		ts = node2->last_renamed_or_created + 1;
+	return ts;
+}
+
+static bool sync_node_is_namespace_root(struct dsync_mailbox_tree *tree,
+					struct dsync_mailbox_node *node)
+{
+	if (node == NULL)
+		return FALSE;
+	if (node == &tree->root)
+		return TRUE;
+	return sync_node_is_namespace_prefix(tree, node);
+}
+
+static bool ATTR_NULL(3, 4)
+sync_rename_lower_ts(struct dsync_mailbox_tree_sync_ctx *ctx,
+		     struct dsync_mailbox_node *local_node1,
+		     struct dsync_mailbox_node *remote_node1,
+		     struct dsync_mailbox_node *local_node2,
+		     struct dsync_mailbox_node *remote_node2,
+		     const char **reason_r)
+{
+	time_t local_ts, remote_ts;
+
+	/* We're scanning the tree at the position of local_node1
+	   and remote_node2. They have identical names. We also know that
+	   local_node1&remote_node1 and local_node2&remote_node2 are "the same"
+	   either because their GUIDs or (in case of one being a directory)
+	   their childrens' GUIDs match. We don't know where local_node2 or
+	   remote_node1 are located in the mailbox tree, or if they exist
+	   at all. Note that node1 and node2 may be the same node pointers. */
+	i_assert(strcmp(local_node1->name, remote_node2->name) == 0);
+
+	if (sync_node_is_namespace_root(ctx->remote_tree, remote_node1) ||
+	    sync_node_is_namespace_root(ctx->remote_tree, remote_node2) ||
+	    sync_node_is_namespace_root(ctx->local_tree, local_node1) ||
+	    sync_node_is_namespace_root(ctx->local_tree, local_node2)) {
+		local_node1->sync_delayed_guid_change = TRUE;
+		remote_node2->sync_delayed_guid_change = TRUE;
+		*reason_r = "Can't rename namespace prefixes - will be merged later";
+		return FALSE;
+	}
+
+	local_ts = nodes_get_timestamp(local_node1, local_node2);
+	remote_ts = nodes_get_timestamp(remote_node1, remote_node2);
+
+	if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+		local_ts = remote_ts+1;
+	else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+		remote_ts = local_ts+1;
+
+	/* The algorithm must be deterministic regardless of the sync direction,
+	   so in case the timestamps are equal we need to resort to looking at
+	   the other data. We'll start by looking at the nodes' mailbox GUIDs,
+	   but if both of them don't exist continue looking into their
+	   children. */
+	if (local_ts > remote_ts ||
+	    (local_ts == remote_ts &&
+	     node_mailbox_trees_cmp(local_node1, remote_node2) < 0)) {
+		/* local nodes have a higher timestamp. we only want to do
+		   renames where the destination parent is the current node's
+		   (local_node1/remote_node2) parent. */
+
+		/* Numbers are GUIDs, letters are mailbox names:
+
+		   local 1A <-name conflict-> remote 2A
+		   local 2B <- potentially -> remote 1[BC]
+
+		   Here we want to preserve the local 1A & 2B names: */
+		if (local_node2 == NULL) {
+			/* local : 1A
+			   remote: 1B, 2A -> 2A-temp, 1A */
+			sync_rename_node(ctx, ctx->remote_tree, remote_node2,
+					 remote_node1, local_node1, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(local: local_node2=NULL)", NULL);
+			return TRUE;
+		} else if (remote_node1 == remote_node2) {
+			/* FIXME: this fixes an infinite loop when in
+			   rename2 test, think it through why :) */
+			*reason_r = "local: remote_node1=remote_node2";
+		} else if (remote_node1 != NULL) {
+			/* a) local_node1->parent == local_node2->parent
+
+			   local : 1A, 2B
+			   remote: 1B, 2A     -> 2A-temp, 1A(, 2B)
+			   remote: 1C, 2A     -> 2B, 1A
+			   remote: 1C, 2A, 3B -> 2A-temp, 1A(, 3B-temp, 2B)
+
+			   b) local_node1->parent != local_node2->parent
+
+			   local : 1X/A, 2Y/B
+			   remote: 1Y/B, 2X/A       -> 2X/A-temp, 1X/A(, 2Y/B)
+			   remote: 1Z/C, 2X/A       -> 2X/A-temp, 1X/A
+			   remote: 1Z/C, 2X/A, 3Y/B -> 2X/A-temp, 1X/A
+
+			   We can handle all of these more easily by simply
+			   always renaming 2 to a temporary name and handling
+			   it when we reach B handling. */
+			sync_rename_node(ctx, ctx->remote_tree, remote_node2,
+					 remote_node1, local_node1, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(local: remote_node1=NULL)", NULL);
+			return TRUE;
+		} else if (node_has_parent(local_node1, local_node2) &&
+			   ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+			/* node2 is a parent of node1, but it should be
+			   vice versa */
+			sync_rename_node_to_temp(ctx, ctx->local_tree,
+				local_node1, local_node2->parent, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(local: node2 parent of node1)", NULL);
+			return TRUE;
+		} else if (node_has_parent(local_node2, local_node1) &&
+			   ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+			/* node1 is a parent of node2, but it should be
+			   vice versa */
+			sync_rename_node_to_temp(ctx, ctx->local_tree,
+				local_node2, local_node1->parent, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(local: node1 parent of node2)", NULL);
+			return TRUE;
+		} else if (local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+			sync_rename_node_to_temp(ctx, ctx->remote_tree,
+				remote_node2, remote_node2->parent, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(local: local_node1 exists)", NULL);
+			return TRUE;
+		} else {
+			/* local : 1A, 2B
+			   remote:     2A     -> (2B)
+			   remote:     2A, 3B -> (3B-temp, 2B) */
+			*reason_r = "local: unchanged";
+		}
+	} else {
+		/* remote nodes have a higher timestamp */
+		if (remote_node1 == NULL) {
+			sync_rename_node(ctx, ctx->local_tree, local_node1,
+					 local_node2, remote_node2, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(remote: remote_node1=NULL)", NULL);
+			return TRUE;
+		} else if (local_node2 == local_node1) {
+			*reason_r = "remote: remote_node2=remote_node1";
+		} else if (local_node2 != NULL) {
+			sync_rename_node(ctx, ctx->local_tree, local_node1,
+					 local_node2, remote_node2, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(remote: local_node2=NULL)", NULL);
+			return TRUE;
+		} else if (node_has_parent(remote_node1, remote_node2) &&
+			   ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+			sync_rename_node_to_temp(ctx, ctx->remote_tree,
+				remote_node1, remote_node2->parent, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(remote: node2 parent of node1)", NULL);
+			return TRUE;
+		} else if (node_has_parent(remote_node2, remote_node1) &&
+			   ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+			sync_rename_node_to_temp(ctx, ctx->remote_tree,
+				remote_node2, remote_node1->parent, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(remote: node1 parent of node2)", NULL);
+			return TRUE;
+		} else if (remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+			sync_rename_node_to_temp(ctx, ctx->local_tree,
+				local_node1, local_node1->parent, reason_r);
+			*reason_r = t_strconcat(*reason_r, "(remote: remote_node2 exists)", NULL);
+			return TRUE;
+		} else {
+			*reason_r = "remote: unchanged";
+		}
+	}
+	return FALSE;
+}
+
+static bool sync_rename_conflict(struct dsync_mailbox_tree_sync_ctx *ctx,
+				 struct dsync_mailbox_node *local_node1,
+				 struct dsync_mailbox_node *remote_node2,
+				 const char **reason_r)
+{
+	struct dsync_mailbox_node *local_node2, *remote_node1;
+	const uint8_t *guid_p;
+	bool ret;
+
+	guid_p = local_node1->mailbox_guid;
+	remote_node1 = hash_table_lookup(ctx->remote_tree->guid_hash, guid_p);
+	guid_p = remote_node2->mailbox_guid;
+	local_node2 = hash_table_lookup(ctx->local_tree->guid_hash, guid_p);
+
+	if ((remote_node1 != NULL && remote_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) ||
+	    (local_node2 != NULL && local_node2->existence == DSYNC_MAILBOX_NODE_EXISTS)) {
+		/* conflicting name, rename the one with lower timestamp */
+		ret = sync_rename_lower_ts(ctx, local_node1, remote_node1,
+					   local_node2, remote_node2, reason_r);
+		*reason_r = t_strconcat("conflicting name: ", *reason_r, NULL);
+		return ret;
+	} else if (dsync_mailbox_node_is_dir(local_node1) ||
+		   dsync_mailbox_node_is_dir(remote_node2)) {
+		/* one of the nodes is a directory, and the other is a mailbox
+		   that doesn't exist on the other side. there is no conflict,
+		   we'll just need to create the mailbox later. */
+		*reason_r = "mailbox not selectable yet";
+		return FALSE;
+	} else {
+		/* both nodes are mailboxes that don't exist on the other side.
+		   we'll merge these mailboxes together later and change their
+		   GUIDs and UIDVALIDITYs to be the same */
+		local_node1->sync_delayed_guid_change = TRUE;
+		remote_node2->sync_delayed_guid_change = TRUE;
+		*reason_r = "GUIDs conflict - will be merged later";
+		return FALSE;
+	}
+}
+
+static struct dsync_mailbox_node *
+sync_find_branch(struct dsync_mailbox_tree *tree,
+		 struct dsync_mailbox_tree *other_tree,
+		 struct dsync_mailbox_node *dir_node)
+{
+	struct dsync_mailbox_node *node, *other_node;
+	const uint8_t *guid_p;
+
+	for (node = dir_node->first_child; node != NULL; node = node->next) {
+		if (dsync_mailbox_node_is_dir(node)) {
+			other_node = sync_find_branch(tree, other_tree, node);
+			if (other_node != NULL)
+				return other_node;
+		} else {
+			guid_p = node->mailbox_guid;
+			other_node = hash_table_lookup(other_tree->guid_hash,
+						       guid_p);
+			if (other_node != NULL)
+				return other_node->parent;
+		}
+	}
+	return NULL;
+}
+
+static bool sync_rename_directory(struct dsync_mailbox_tree_sync_ctx *ctx,
+				  struct dsync_mailbox_node *local_node1,
+				  struct dsync_mailbox_node *remote_node2,
+				  const char **reason_r)
+{
+	struct dsync_mailbox_node *remote_node1, *local_node2;
+
+	/* see if we can find matching mailbox branches based on the nodes'
+	   child mailboxes (with GUIDs). we can then rename the entire branch.
+	   don't try to do this for namespace prefixes though. */
+	remote_node1 = sync_find_branch(ctx->local_tree,
+					ctx->remote_tree, local_node1);
+	local_node2 = sync_find_branch(ctx->remote_tree, ctx->local_tree,
+				       remote_node2);
+	if (remote_node1 == NULL || local_node2 == NULL) {
+		*reason_r = "Directory rename branch not found";
+		return FALSE;
+	}
+	if (node_names_equal(remote_node1, local_node2)) {
+		*reason_r = "Directory name paths are equal";
+		return FALSE;
+	}
+
+	return sync_rename_lower_ts(ctx, local_node1, remote_node1,
+				    local_node2, remote_node2, reason_r);
+}
+
+static bool sync_rename_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+				  struct dsync_mailbox_node *local_parent,
+				  struct dsync_mailbox_node *remote_parent)
+{
+	struct dsync_mailbox_node **local_nodep = &local_parent->first_child;
+	struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child;
+	struct dsync_mailbox_node *local_node, *remote_node;
+	const char *reason;
+	string_t *debug = NULL;
+	bool changed;
+
+	if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0)
+		debug = t_str_new(128);
+
+	/* the nodes are sorted by their names. */
+	while (*local_nodep != NULL || *remote_nodep != NULL) {
+		local_node = *local_nodep;
+		remote_node = *remote_nodep;
+
+		if (local_node == NULL ||
+		    (remote_node != NULL &&
+		     strcmp(local_node->name, remote_node->name) > 0)) {
+			/* add a missing local node */
+			local_node = sync_node_new(ctx->local_tree, local_nodep,
+						   local_parent, remote_node);
+		}
+		if (remote_node == NULL ||
+		    strcmp(remote_node->name, local_node->name) > 0) {
+			/* add a missing remote node */
+			remote_node = sync_node_new(ctx->remote_tree, remote_nodep,
+						    remote_parent, local_node);
+		}
+		i_assert(strcmp(local_node->name, remote_node->name) == 0);
+		if (debug != NULL) {
+			str_truncate(debug, 0);
+			str_append(debug, "Mailbox ");
+			dsync_mailbox_node_append_full_name(debug, ctx->local_tree, local_node);
+			str_printfa(debug, ": local=%s/%ld/%d, remote=%s/%ld/%d",
+				    guid_128_to_string(local_node->mailbox_guid),
+				    (long)local_node->last_renamed_or_created,
+				    local_node->existence,
+				    guid_128_to_string(remote_node->mailbox_guid),
+				    (long)remote_node->last_renamed_or_created,
+				    remote_node->existence);
+		}
+
+		if (dsync_mailbox_node_is_dir(local_node) &&
+		    dsync_mailbox_node_is_dir(remote_node)) {
+			/* both nodes are directories (or other side is
+			   nonexistent). see if we can match them by their
+			   child mailboxes */
+			changed = sync_rename_directory(ctx, local_node, remote_node, &reason);
+		} else if (dsync_mailbox_node_guids_equal(local_node,
+							  remote_node)) {
+			/* mailboxes are equal, no need to rename */
+			reason = "Mailboxes are equal";
+			changed = FALSE;
+		} else {
+			/* mailbox naming conflict */
+			changed = sync_rename_conflict(ctx, local_node,
+						       remote_node, &reason);
+		}
+		/* handle children, if there are any */
+		if (debug != NULL) {
+			i_debug("brain %c: %s: %s",
+				(ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+				str_c(debug), reason);
+		}
+
+		if (!changed) T_BEGIN {
+			changed = sync_rename_mailboxes(ctx, local_node,
+							remote_node);
+		} T_END;
+		if (changed)
+			return TRUE;
+
+		local_nodep = &local_node->next;
+		remote_nodep = &remote_node->next;
+	}
+	return FALSE;
+}
+
+static bool mailbox_node_hash_first_child(struct dsync_mailbox_node *node,
+					  struct md5_context *md5)
+{
+	for (node = node->first_child; node != NULL; node = node->next) {
+		if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+			md5_update(md5, node->mailbox_guid,
+				   sizeof(node->mailbox_guid));
+			md5_update(md5, node->name, strlen(node->name));
+			return TRUE;
+		}
+		if (node->first_child != NULL) {
+			if (mailbox_node_hash_first_child(node, md5))
+				return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static const char *
+mailbox_node_generate_suffix(struct dsync_mailbox_node *node)
+{
+	struct md5_context md5;
+	unsigned char digest[MD5_RESULTLEN];
+
+	if (!dsync_mailbox_node_is_dir(node))
+		return guid_128_to_string(node->mailbox_guid);
+
+	md5_init(&md5);
+	if (!mailbox_node_hash_first_child(node, &md5))
+		i_unreached(); /* we would have deleted it */
+	md5_final(&md5, digest);
+	return binary_to_hex(digest, sizeof(digest));
+}
+
+static void suffix_inc(string_t *str)
+{
+	char *data;
+	size_t i;
+
+	data = str_c_modifiable(str) + str_len(str)-1;
+	for (i = str_len(str); i > 0; i--, data--) {
+		if ((*data >= '0' && *data < '9') ||
+		    (*data >= 'a' && *data < 'f')) {
+			*data += 1;
+			return;
+		} else if (*data == '9') {
+			*data = 'a';
+			return;
+		} else if (*data != 'f') {
+			i_unreached();
+		}
+	}
+	i_unreached();
+}
+
+static void
+sync_rename_temp_mailbox_node(struct dsync_mailbox_tree *tree,
+			      struct dsync_mailbox_node *node,
+			      const char **reason_r)
+{
+	const char *p, *new_suffix;
+	string_t *str = t_str_new(256);
+	size_t max_prefix_len;
+
+	/* The name is currently <oldname>-<temp>. Both sides need to
+	   use equivalent names, so we'll replace the <temp> if possible
+	   with a) mailbox GUID, b) sha1 of childrens' (GUID|name)s. In the
+	   very unlikely case of such name already existing, just increase
+	   the last letters until it's not found. */
+	new_suffix = mailbox_node_generate_suffix(node);
+
+	p = strrchr(node->name, '-');
+	i_assert(p != NULL);
+	p++;
+	max_prefix_len = TEMP_MAX_NAME_LEN - strlen(new_suffix) - 1;
+	if (max_prefix_len > (size_t)(p-node->name))
+		max_prefix_len = p-node->name;
+	str_append_max(str, node->name, max_prefix_len);
+	str_append(str, new_suffix);
+	while (node_has_child(node->parent, str_c(str)))
+		suffix_inc(str);
+
+	*reason_r = t_strdup_printf("Renamed '%s' to '%s'",
+		dsync_mailbox_node_get_full_name(tree, node), str_c(str));
+	node->name = p_strdup(tree->pool, str_c(str));
+
+	dsync_mailbox_tree_node_move_sorted(node, node->parent);
+	node->sync_temporary_name = FALSE;
+}
+
+static void
+sync_rename_delete_node_dirs(struct dsync_mailbox_tree_sync_ctx *ctx,
+			     struct dsync_mailbox_tree *tree,
+			     struct dsync_mailbox_node *node)
+{
+	struct dsync_mailbox_node *child;
+
+	for (child = node->first_child; child != NULL; child = child->next)
+		sync_rename_delete_node_dirs(ctx, tree, child);
+
+	if (tree == ctx->local_tree &&
+	    ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL &&
+	    node->existence != DSYNC_MAILBOX_NODE_NONEXISTENT) {
+		sync_add_dir_change(ctx, node,
+				    DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR);
+	}
+	node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+	node->sync_temporary_name = FALSE;
+}
+
+static bool
+sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+			   struct dsync_mailbox_tree *tree,
+			   struct dsync_mailbox_node *node, bool *renames_r)
+{
+	const char *reason;
+
+	for (; node != NULL; node = node->next) {
+		while (sync_rename_temp_mailboxes(ctx, tree, node->first_child, renames_r)) ;
+
+		if (!node->sync_temporary_name) {
+		} else if (dsync_mailbox_node_is_dir(node) &&
+			   (node->first_child == NULL ||
+			    !node_has_existent_children(node, FALSE))) {
+			/* we can just delete this directory and
+			   any child directories it may have */
+			if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) {
+				i_debug("brain %c: %s mailbox %s: Delete directory-only tree",
+					(ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+					tree == ctx->local_tree ? "local" : "remote",
+					dsync_mailbox_node_get_full_name(tree, node));
+			}
+			sync_rename_delete_node_dirs(ctx, tree, node);
+		} else {
+			T_BEGIN {
+				*renames_r = TRUE;
+				sync_rename_temp_mailbox_node(tree, node, &reason);
+				if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) {
+					i_debug("brain %c: %s mailbox %s: %s",
+						(ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+						tree == ctx->local_tree ? "local" : "remote",
+						dsync_mailbox_node_get_full_name(tree, node), reason);
+				}
+			} T_END;
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static int
+dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx,
+				  bool *renames_r)
+{
+	unsigned int max_renames, count = 0;
+	bool changed;
+
+	*renames_r = FALSE;
+
+	max_renames = ctx->combined_mailboxes_count * 3;
+	do {
+		T_BEGIN {
+			changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root,
+							&ctx->remote_tree->root);
+		} T_END;
+		if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 &&
+		    changed) {
+			i_debug("brain %c: -- Mailbox renamed, restart sync --",
+				(ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S');
+		}
+	} while (changed && ++count <= max_renames);
+
+	if (changed) {
+		i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting");
+		return -1;
+	}
+
+	while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root, renames_r)) ;
+	while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root, renames_r)) ;
+	return 0;
+}
+
+static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node,
+				  const struct dsync_mailbox_node *wanted_node,
+				  const char **reason_r)
+{
+	if (wanted_node->existence != DSYNC_MAILBOX_NODE_EXISTS) {
+		*reason_r = wanted_node->existence == DSYNC_MAILBOX_NODE_DELETED ?
+			"Mailbox has been deleted" : "Mailbox doesn't exist";
+		return TRUE;
+	}
+	if (node->uid_validity != wanted_node->uid_validity) {
+		*reason_r = t_strdup_printf("UIDVALIDITY changed (%u -> %u)",
+					    wanted_node->uid_validity,
+					    node->uid_validity);
+		return TRUE;
+	}
+	if (node->uid_next > wanted_node->uid_next) {
+		/* we can't lower the UIDNEXT */
+		*reason_r = t_strdup_printf("UIDNEXT is too high (%u > %u)",
+					    node->uid_next,
+					    wanted_node->uid_next);
+		return TRUE;
+	}
+	if (memcmp(node->mailbox_guid, wanted_node->mailbox_guid,
+		   sizeof(node->mailbox_guid)) != 0) {
+		*reason_r = "GUID changed";
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static void
+sync_delete_wrong_mailboxes_branch(struct dsync_mailbox_tree_sync_ctx *ctx,
+				   struct dsync_mailbox_tree *tree,
+				   const struct dsync_mailbox_tree *wanted_tree,
+				   struct dsync_mailbox_node *node,
+				   const struct dsync_mailbox_node *wanted_node)
+{
+	const char *reason;
+	int ret;
+
+	while (node != NULL && wanted_node != NULL) {
+		if (node->first_child != NULL) {
+			sync_delete_wrong_mailboxes_branch(ctx,
+				tree, wanted_tree,
+				node->first_child, wanted_node->first_child);
+		}
+		ret = strcmp(node->name, wanted_node->name);
+		if (ret < 0) {
+			/* node shouldn't exist */
+			if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+			    !dsync_mailbox_node_is_dir(node)) {
+				sync_delete_mailbox_node(ctx, tree, node,
+					"Mailbox doesn't exist");
+			}
+			node = node->next;
+		} else if (ret > 0) {
+			/* wanted_node doesn't exist. it's created later. */
+			wanted_node = wanted_node->next;
+		} else T_BEGIN {
+			if (sync_is_wrong_mailbox(node, wanted_node, &reason) &&
+			    node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+			    !dsync_mailbox_node_is_dir(node))
+				sync_delete_mailbox_node(ctx, tree, node, reason);
+			node = node->next;
+			wanted_node = wanted_node->next;
+		} T_END;
+	}
+	for (; node != NULL; node = node->next) {
+		/* node and its children shouldn't exist */
+		if (node->first_child != NULL) {
+			sync_delete_wrong_mailboxes_branch(ctx,
+				tree, wanted_tree,
+				node->first_child, NULL);
+		}
+		if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    !dsync_mailbox_node_is_dir(node))
+			sync_delete_mailbox_node(ctx, tree, node, "Mailbox doesn't exist");
+	}
+}
+
+static void
+sync_delete_wrong_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+			    struct dsync_mailbox_tree *tree,
+			    const struct dsync_mailbox_tree *wanted_tree)
+{
+	sync_delete_wrong_mailboxes_branch(ctx, tree, wanted_tree,
+					   tree->root.first_child,
+					   wanted_tree->root.first_child);
+}
+
+static void sync_create_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+				  struct dsync_mailbox_tree *tree)
+{
+	struct dsync_mailbox_tree *other_tree;
+	struct dsync_mailbox_tree_iter *iter;
+	struct dsync_mailbox_node *node, *other_node;
+	const char *name;
+	const uint8_t *guid_p;
+
+	other_tree = tree == ctx->local_tree ?
+		ctx->remote_tree : ctx->local_tree;
+
+	iter = dsync_mailbox_tree_iter_init(tree);
+	while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+		/* make sure the renaming handled everything */
+		i_assert(!node->sync_temporary_name);
+		if (dsync_mailbox_node_is_dir(node))
+			continue;
+
+		i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS);
+
+		guid_p = node->mailbox_guid;
+		other_node = hash_table_lookup(other_tree->guid_hash, guid_p);
+		if (other_node == NULL)
+			other_node = sorted_tree_get(other_tree, name);
+		if (dsync_mailbox_node_is_dir(other_node)) {
+			/* create a missing mailbox */
+			other_node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+			other_node->ns = node->ns;
+			other_node->uid_validity = node->uid_validity;
+			memcpy(other_node->mailbox_guid, node->mailbox_guid,
+			       sizeof(other_node->mailbox_guid));
+			if (other_tree == ctx->local_tree)
+				sync_add_create_change(ctx, other_node, name);
+		} else if (!guid_128_equals(node->mailbox_guid,
+					    other_node->mailbox_guid)) {
+			/* mailbox with same name exists both locally and
+			   remotely, but they have different GUIDs and neither
+			   side has the other's GUID. typically this means that
+			   both sides had autocreated some mailboxes (e.g.
+			   INBOX). we'll just change the GUID for one of
+			   them. */
+			i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS);
+			i_assert(node->ns == other_node->ns);
+
+			if (other_tree == ctx->local_tree)
+				sync_add_create_change(ctx, node, name);
+		} else {
+			/* existing mailbox. mismatching UIDVALIDITY is handled
+			   later while syncing the mailbox. */
+			i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS);
+			i_assert(node->ns == other_node->ns);
+		}
+	}
+	dsync_mailbox_tree_iter_deinit(&iter);
+}
+
+static void
+sync_subscription(struct dsync_mailbox_tree_sync_ctx *ctx,
+		  struct dsync_mailbox_node *local_node,
+		  struct dsync_mailbox_node *remote_node)
+{
+	bool use_local;
+
+	if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+		use_local = TRUE;
+	else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+		use_local = FALSE;
+	else if (local_node->last_subscription_change > remote_node->last_subscription_change)
+		use_local = TRUE;
+	else if (local_node->last_subscription_change < remote_node->last_subscription_change)
+		use_local = FALSE;
+	else {
+		/* local and remote have equal timestamps. prefer to subscribe
+		   rather than unsubscribe. */
+		use_local = local_node->subscribed;
+	}
+	if (use_local) {
+		/* use local subscription state */
+		remote_node->subscribed = local_node->subscribed;
+	} else {
+		/* use remote subscription state */
+		local_node->subscribed = remote_node->subscribed;
+		sync_add_dir_change(ctx, local_node, local_node->subscribed ?
+				    DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE :
+				    DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE);
+	}
+}
+
+static void sync_mailbox_child_dirs(struct dsync_mailbox_tree_sync_ctx *ctx,
+				    struct dsync_mailbox_node *local_parent,
+				    struct dsync_mailbox_node *remote_parent)
+{
+	struct dsync_mailbox_node **local_nodep = &local_parent->first_child;
+	struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child;
+	struct dsync_mailbox_node *local_node, *remote_node;
+	int ret;
+
+	/* NOTE: the nodes are always sorted. renaming created all of the
+	   interesting nodes, but it may have left some extra nonexistent nodes
+	   lying around, which we will delete. */
+	while (*local_nodep != NULL && *remote_nodep != NULL) {
+		local_node = *local_nodep;
+		remote_node = *remote_nodep;
+
+		ret = strcmp(local_node->name, remote_node->name);
+		if (ret < 0) {
+			i_assert(!node_is_existent(local_node));
+			*local_nodep = local_node->next;
+			continue;
+		}
+		if (ret > 0) {
+			i_assert(!node_is_existent(remote_node));
+			*remote_nodep = remote_node->next;
+			continue;
+		}
+
+		if (local_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    remote_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT &&
+		    ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+			/* create to remote */
+			remote_node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+		}
+		if (remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    local_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT &&
+		    ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+			/* create to local */
+			local_node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+			sync_add_dir_change(ctx, local_node,
+				DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR);
+		}
+
+		/* create/delete child dirs */
+		sync_mailbox_child_dirs(ctx, local_node, remote_node);
+
+		if (local_node->subscribed != remote_node->subscribed)
+			sync_subscription(ctx, local_node, remote_node);
+
+		if (local_node->existence == DSYNC_MAILBOX_NODE_DELETED &&
+		    !node_has_existent_children(local_node, TRUE) &&
+		    remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+			/* delete from remote */
+			i_assert(!node_has_existent_children(remote_node, TRUE));
+			remote_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+		}
+		if (remote_node->existence == DSYNC_MAILBOX_NODE_DELETED &&
+		    !node_has_existent_children(remote_node, TRUE) &&
+		    local_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+		    ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+			/* delete from local */
+			i_assert(!node_has_existent_children(local_node, TRUE));
+			local_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+			sync_add_dir_change(ctx, local_node,
+				DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR);
+		}
+
+		local_nodep = &local_node->next;
+		remote_nodep = &remote_node->next;
+	}
+	while (*local_nodep != NULL) {
+		i_assert(!node_is_existent(*local_nodep));
+		*local_nodep = (*local_nodep)->next;
+	}
+	while (*remote_nodep != NULL) {
+		i_assert(!node_is_existent(*remote_nodep));
+		*remote_nodep = (*remote_nodep)->next;
+	}
+}
+
+static void sync_mailbox_dirs(struct dsync_mailbox_tree_sync_ctx *ctx)
+{
+	sync_mailbox_child_dirs(ctx, &ctx->local_tree->root,
+				&ctx->remote_tree->root);
+}
+
+static void
+dsync_mailbox_tree_update_child_timestamps(struct dsync_mailbox_node *node,
+					   time_t parent_timestamp)
+{
+	time_t ts;
+
+	if (node->last_renamed_or_created < parent_timestamp)
+		node->last_renamed_or_created = parent_timestamp;
+	ts = node->last_renamed_or_created;
+
+	for (node = node->first_child; node != NULL; node = node->next)
+		dsync_mailbox_tree_update_child_timestamps(node, ts);
+}
+
+struct dsync_mailbox_tree_sync_ctx *
+dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
+			      struct dsync_mailbox_tree *remote_tree,
+			      enum dsync_mailbox_trees_sync_type sync_type,
+			      enum dsync_mailbox_trees_sync_flags sync_flags)
+{
+	struct dsync_mailbox_tree_sync_ctx *ctx;
+	unsigned int rename_counter = 0;
+	bool renames;
+	pool_t pool;
+
+	i_assert(hash_table_is_created(local_tree->guid_hash));
+	i_assert(hash_table_is_created(remote_tree->guid_hash));
+
+	pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox trees sync",
+				     1024*64);
+	ctx = p_new(pool, struct dsync_mailbox_tree_sync_ctx, 1);
+	ctx->pool = pool;
+	ctx->local_tree = local_tree;
+	ctx->remote_tree = remote_tree;
+	ctx->sync_type = sync_type;
+	ctx->sync_flags = sync_flags;
+	i_array_init(&ctx->changes, 128);
+
+again:
+	renames = FALSE;
+	ctx->combined_mailboxes_count = 0;
+	sync_tree_sort_and_delete_mailboxes(ctx, remote_tree,
+		sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY);
+	sync_tree_sort_and_delete_mailboxes(ctx, local_tree,
+		sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY);
+
+	dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0);
+	dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0);
+	if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) {
+		if (dsync_mailbox_tree_handle_renames(ctx, &renames) < 0) {
+			ctx->failed = TRUE;
+			return ctx;
+		}
+	}
+
+	/* if we're not doing a two-way sync, delete now any mailboxes, which
+	   a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY,
+	   c) has a too high UIDNEXT */
+	if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+		sync_delete_wrong_mailboxes(ctx, remote_tree, local_tree);
+	else if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+		sync_delete_wrong_mailboxes(ctx, local_tree, remote_tree);
+
+	if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+		sync_create_mailboxes(ctx, remote_tree);
+	if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+		sync_create_mailboxes(ctx, local_tree);
+	if (renames && rename_counter++ <= ctx->combined_mailboxes_count*3) {
+		/* this rename algorithm is just horrible. we're retrying this
+		   because the final sync_rename_temp_mailbox_node() calls
+		   give different names to local & remote mailbox trees.
+		   something's not right here, but this looping is better than
+		   a crash in sync_mailbox_dirs() due to trees not matching. */
+		goto again;
+	}
+	sync_mailbox_dirs(ctx);
+	return ctx;
+}
+
+const struct dsync_mailbox_tree_sync_change *
+dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx)
+{
+	if (ctx->change_idx == array_count(&ctx->changes))
+		return NULL;
+	return array_idx(&ctx->changes, ctx->change_idx++);
+}
+
+int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx)
+{
+	struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx;
+	int ret = ctx->failed ? -1 : 0;
+
+	*_ctx = NULL;
+
+	array_free(&ctx->changes);
+	pool_unref(&ctx->pool);
+	return ret;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-tree.c b/src/doveadm/dsync/dsync-mailbox-tree.c
new file mode 100644
index 0000000..e34f335
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree.c
@@ -0,0 +1,554 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "sort.h"
+#include "mailbox-list-private.h"
+#include "dsync-mailbox-tree-private.h"
+
+
+struct dsync_mailbox_tree_iter {
+	struct dsync_mailbox_tree *tree;
+
+	struct dsync_mailbox_node *cur;
+	string_t *name;
+};
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_init(char sep, char escape_char, char alt_char)
+{
+	struct dsync_mailbox_tree *tree;
+	pool_t pool;
+
+	i_assert(sep != '\0');
+	i_assert(alt_char != '\0');
+
+	pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox tree", 4096);
+	tree = p_new(pool, struct dsync_mailbox_tree, 1);
+	tree->pool = pool;
+	tree->sep = tree->sep_str[0] = sep;
+	tree->escape_char = escape_char;
+	tree->alt_char = alt_char;
+	tree->root.name = "";
+	i_array_init(&tree->deletes, 32);
+	return tree;
+}
+
+void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **_tree)
+{
+	struct dsync_mailbox_tree *tree = *_tree;
+
+	*_tree = NULL;
+	hash_table_destroy(&tree->name128_hash);
+	hash_table_destroy(&tree->guid_hash);
+	array_free(&tree->deletes);
+	pool_unref(&tree->pool);
+}
+
+static struct dsync_mailbox_node *
+dsync_mailbox_node_find(struct dsync_mailbox_node *nodes, const char *name)
+{
+	for (; nodes != NULL; nodes = nodes->next) {
+		if (strcmp(name, nodes->name) == 0)
+			return nodes;
+	}
+	return NULL;
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree,
+			  const char *full_name)
+{
+	struct dsync_mailbox_node *node = &tree->root;
+
+	T_BEGIN {
+		const char *const *path;
+
+		path = t_strsplit(full_name, tree->sep_str);
+		for (; *path != NULL && node != NULL; path++)
+			node = dsync_mailbox_node_find(node->first_child, *path);
+	} T_END;
+	return node;
+}
+
+void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node,
+				    struct dsync_mailbox_node *parent)
+{
+	node->parent = parent;
+	node->next = parent->first_child;
+	parent->first_child = node;
+}
+
+void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node)
+{
+	struct dsync_mailbox_node **p;
+
+	for (p = &node->parent->first_child;; p = &(*p)->next) {
+		if (*p == node) {
+			*p = node->next;
+			break;
+		}
+	}
+	node->parent = NULL;
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name)
+{
+	struct dsync_mailbox_node *parent = NULL, *node = &tree->root;
+
+	i_assert(tree->iter_count == 0);
+
+	T_BEGIN {
+		const char *const *path;
+
+		/* find the existing part */
+		path = t_strsplit(full_name, tree->sep_str);
+		for (; *path != NULL; path++) {
+			parent = node;
+			node = dsync_mailbox_node_find(node->first_child, *path);
+			if (node == NULL)
+				break;
+		}
+		/* create the rest */
+		for (; *path != NULL; path++) {
+			node = p_new(tree->pool, struct dsync_mailbox_node, 1);
+			node->name = p_strdup(tree->pool, *path);
+			node->ns = parent->ns;
+			dsync_mailbox_tree_node_attach(node, parent);
+			parent = node;
+		}
+	} T_END;
+	return node;
+}
+
+static void
+node_get_full_name_recurse(const struct dsync_mailbox_tree *tree,
+			   const struct dsync_mailbox_node *node, string_t *str)
+{
+	if (node->parent != &tree->root)
+		node_get_full_name_recurse(tree, node->parent, str);
+
+	str_append(str, node->name);
+	str_append_c(str, tree->sep);
+}
+
+const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree,
+					     const struct dsync_mailbox_node *node)
+{
+	string_t *str = t_str_new(128);
+	dsync_mailbox_node_append_full_name(str, tree, node);
+	return str_c(str);
+}
+
+void dsync_mailbox_node_append_full_name(string_t *str,
+					 const struct dsync_mailbox_tree *tree,
+					 const struct dsync_mailbox_node *node)
+{
+	i_assert(node->parent != NULL);
+
+	node_get_full_name_recurse(tree, node, str);
+	/* remove the trailing separator */
+	str_truncate(str, str_len(str)-1);
+}
+
+void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest,
+				  const struct dsync_mailbox_node *src)
+{
+	memcpy(dest->mailbox_guid, src->mailbox_guid,
+	       sizeof(dest->mailbox_guid));
+	dest->uid_validity = src->uid_validity;
+	dest->uid_next = src->uid_next;
+	dest->existence = src->existence;
+	dest->last_renamed_or_created = src->last_renamed_or_created;
+	dest->subscribed = src->subscribed;
+	dest->last_subscription_change = src->last_subscription_change;
+}
+
+struct dsync_mailbox_tree_iter *
+dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree)
+{
+	struct dsync_mailbox_tree_iter *iter;
+
+	iter = i_new(struct dsync_mailbox_tree_iter, 1);
+	iter->tree = tree;
+	iter->name = str_new(default_pool, 128);
+	iter->cur = &tree->root;
+
+	tree->iter_count++;
+	return iter;
+}
+
+static size_t node_get_full_name_length(struct dsync_mailbox_node *node)
+{
+	if (node->parent->parent == NULL)
+		return strlen(node->name);
+	else {
+		return strlen(node->name) + 1 +
+			node_get_full_name_length(node->parent);
+	}
+}
+
+bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter,
+				  const char **full_name_r,
+				  struct dsync_mailbox_node **node_r)
+{
+	size_t len;
+
+	if (iter->cur->first_child != NULL)
+		iter->cur = iter->cur->first_child;
+	else {
+		while (iter->cur->next == NULL) {
+			if (iter->cur == &iter->tree->root)
+				return FALSE;
+			iter->cur = iter->cur->parent;
+		}
+		iter->cur = iter->cur->next;
+		len = iter->cur->parent == &iter->tree->root ? 0 :
+			node_get_full_name_length(iter->cur->parent);
+		str_truncate(iter->name, len);
+	}
+	if (str_len(iter->name) > 0)
+		str_append_c(iter->name, iter->tree->sep);
+	str_append(iter->name, iter->cur->name);
+	*full_name_r = str_c(iter->name);
+	*node_r = iter->cur;
+	return TRUE;
+}
+
+void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **_iter)
+{
+	struct dsync_mailbox_tree_iter *iter = *_iter;
+
+	*_iter = NULL;
+
+	i_assert(iter->tree->iter_count > 0);
+	iter->tree->iter_count--;
+
+	str_free(&iter->name);
+	i_free(iter);
+}
+
+void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree)
+{
+	struct dsync_mailbox_tree_iter *iter;
+	struct dsync_mailbox_node *node;
+	const char *name;
+	guid_128_t *sha128;
+	uint8_t *guid_p;
+
+	i_assert(!hash_table_is_created(tree->name128_hash));
+
+	hash_table_create(&tree->name128_hash,
+			  tree->pool, 0, guid_128_hash, guid_128_cmp);
+	iter = dsync_mailbox_tree_iter_init(tree);
+	while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+		sha128 = p_new(tree->pool, guid_128_t, 1);
+		mailbox_name_get_sha128(name, *sha128);
+		guid_p = *sha128;
+		hash_table_insert(tree->name128_hash, guid_p, node);
+	}
+	dsync_mailbox_tree_iter_deinit(&iter);
+}
+
+static const char *
+convert_name_to_remote_sep(struct dsync_mailbox_tree *tree, const char *name)
+{
+	string_t *str = t_str_new(128);
+
+	char remote_escape_chars[3] = {
+		tree->remote_escape_char,
+		tree->remote_sep,
+		'\0'
+	};
+
+	for (;;) {
+		const char *end = strchr(name, tree->sep);
+		const char *name_part = end == NULL ? name :
+			t_strdup_until(name, end++);
+
+		if (tree->escape_char != '\0')
+			mailbox_list_name_unescape(&name_part, tree->escape_char);
+		if (remote_escape_chars[0] != '\0') {
+			/* The local name can be fully escaped to remote
+			   name and back. */
+			mailbox_list_name_escape(name_part, remote_escape_chars,
+						 str);
+		} else {
+			/* There is no remote escape char, so for conflicting
+			   separator use the alt_char. */
+			for (; *name_part != '\0'; name_part++) {
+				if (*name_part == tree->remote_sep)
+					str_append_c(str, tree->alt_char);
+				else
+					str_append_c(str, *name_part);
+			}
+		}
+		if (end == NULL)
+			break;
+		str_append_c(str, tree->remote_sep);
+	}
+	return str_c(str);
+}
+
+static void
+dsync_mailbox_tree_build_name128_remotesep_hash(struct dsync_mailbox_tree *tree)
+{
+	struct dsync_mailbox_tree_iter *iter;
+	struct dsync_mailbox_node *node;
+	const char *name;
+	guid_128_t *sha128;
+	uint8_t *guid_p;
+
+	i_assert(tree->sep != tree->remote_sep);
+	i_assert(!hash_table_is_created(tree->name128_remotesep_hash));
+
+	hash_table_create(&tree->name128_remotesep_hash, tree->pool, 0,
+			  guid_128_hash, guid_128_cmp);
+	iter = dsync_mailbox_tree_iter_init(tree);
+	while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+		sha128 = p_new(tree->pool, guid_128_t, 1);
+		T_BEGIN {
+			const char *remote_name =
+				convert_name_to_remote_sep(tree, name);
+			mailbox_name_get_sha128(remote_name, *sha128);
+		} T_END;
+		guid_p = *sha128;
+		hash_table_insert(tree->name128_remotesep_hash, guid_p, node);
+	}
+	dsync_mailbox_tree_iter_deinit(&iter);
+}
+
+int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree,
+				     struct dsync_mailbox_node *node,
+				     struct dsync_mailbox_node **old_node_r)
+{
+	struct dsync_mailbox_node *old_node;
+	uint8_t *guid = node->mailbox_guid;
+
+	if (guid_128_is_empty(node->mailbox_guid))
+		return 0;
+
+	*old_node_r = old_node = hash_table_lookup(tree->guid_hash, guid);
+	if (old_node == NULL)
+		hash_table_insert(tree->guid_hash, guid, node);
+	else if (old_node != node)
+		return -1;
+	return 0;
+}
+
+int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree,
+				       struct dsync_mailbox_node **dup_node1_r,
+				       struct dsync_mailbox_node **dup_node2_r)
+{
+	struct dsync_mailbox_tree_iter *iter;
+	struct dsync_mailbox_node *node, *old_node;
+	const char *name;
+	int ret = 0;
+
+	if (!hash_table_is_created(tree->guid_hash)) {
+		hash_table_create(&tree->guid_hash, tree->pool, 0,
+				  guid_128_hash, guid_128_cmp);
+	}
+	iter = dsync_mailbox_tree_iter_init(tree);
+	while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+		if (dsync_mailbox_tree_guid_hash_add(tree, node, &old_node) < 0) {
+			*dup_node1_r = node;
+			*dup_node2_r = old_node;
+			ret = -1;
+		}
+	}
+	dsync_mailbox_tree_iter_deinit(&iter);
+	return ret;
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree,
+			       const guid_128_t guid)
+{
+	const uint8_t *guid_p = guid;
+
+	return hash_table_lookup(tree->guid_hash, guid_p);
+}
+
+const struct dsync_mailbox_delete *
+dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree,
+			       unsigned int *count_r)
+{
+	return array_get(&tree->deletes, count_r);
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree,
+			       const struct dsync_mailbox_delete *del)
+{
+	const uint8_t *guid_p = del->guid;
+
+	i_assert(hash_table_is_created(tree->guid_hash));
+	i_assert(tree->remote_sep != '\0');
+
+	if (del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) {
+		/* find node by GUID */
+		return hash_table_lookup(tree->guid_hash, guid_p);
+	}
+
+	/* find node by name. this is a bit tricky, since the hierarchy
+	   separator may differ from ours. */
+	if (tree->sep == tree->remote_sep) {
+		if (!hash_table_is_created(tree->name128_hash))
+			dsync_mailbox_tree_build_name128_hash(tree);
+		return hash_table_lookup(tree->name128_hash, guid_p);
+	} else {
+		if (!hash_table_is_created(tree->name128_remotesep_hash))
+			dsync_mailbox_tree_build_name128_remotesep_hash(tree);
+		return hash_table_lookup(tree->name128_remotesep_hash, guid_p);
+	}
+}
+
+void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree,
+					 char remote_sep, char escape_char)
+{
+	i_assert(tree->remote_sep == '\0');
+	i_assert(tree->remote_escape_char == '\0');
+
+	tree->remote_sep = remote_sep;
+	tree->remote_escape_char = escape_char;
+}
+
+static void
+dsync_mailbox_tree_dup_nodes(struct dsync_mailbox_tree *dest_tree,
+			     const struct dsync_mailbox_node *src,
+			     string_t *path)
+{
+	size_t prefix_len = str_len(path);
+	struct dsync_mailbox_node *node;
+
+	if (prefix_len > 0) {
+		str_append_c(path, dest_tree->sep);
+		prefix_len++;
+	}
+	for (; src != NULL; src = src->next) {
+		str_truncate(path, prefix_len);
+		str_append(path, src->name);
+		node = dsync_mailbox_tree_get(dest_tree, str_c(path));
+
+		node->ns = src->ns;
+		memcpy(node->mailbox_guid, src->mailbox_guid,
+		       sizeof(node->mailbox_guid));
+		node->uid_validity = src->uid_validity;
+		node->uid_next = src->uid_next;
+		node->existence = src->existence;
+		node->last_renamed_or_created = src->last_renamed_or_created;
+		node->subscribed = src->subscribed;
+		node->last_subscription_change = src->last_subscription_change;
+
+		if (src->first_child != NULL) {
+			dsync_mailbox_tree_dup_nodes(dest_tree,
+						     src->first_child, path);
+		}
+	}
+}
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src)
+{
+	struct dsync_mailbox_tree *dest;
+	string_t *str = t_str_new(128);
+
+	dest = dsync_mailbox_tree_init(src->sep, src->escape_char,
+				       src->alt_char);
+	dsync_mailbox_tree_dup_nodes(dest, &src->root, str);
+	return dest;
+}
+
+int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1,
+				struct dsync_mailbox_node *const *n2)
+{
+	return strcmp((*n1)->name, (*n2)->name);
+}
+
+static bool
+dsync_mailbox_nodes_equal(const struct dsync_mailbox_node *node1,
+			  const struct dsync_mailbox_node *node2)
+{
+	return strcmp(node1->name, node2->name) == 0 &&
+		node1->ns == node2->ns &&
+		memcmp(node1->mailbox_guid, node2->mailbox_guid,
+		       sizeof(node1->mailbox_guid)) == 0 &&
+		node1->uid_validity == node2->uid_validity &&
+		node1->existence == node2->existence &&
+		node1->subscribed == node2->subscribed;
+}
+
+static bool
+dsync_mailbox_branches_equal(struct dsync_mailbox_node *node1,
+			     struct dsync_mailbox_node *node2)
+{
+	/* this function is used only for unit tests, so performance doesn't
+	   really matter */
+	struct dsync_mailbox_node *n, **snodes1, **snodes2;
+	unsigned int i, count;
+
+	for (n = node1, count = 0; n != NULL; n = n->next) count++;
+	for (n = node2, i = 0; n != NULL; n = n->next) i++;
+	if (i != count)
+		return FALSE;
+	if (count == 0)
+		return TRUE;
+
+	/* sort the trees by name */
+	snodes1 = t_new(struct dsync_mailbox_node *, count);
+	snodes2 = t_new(struct dsync_mailbox_node *, count);
+	for (n = node1, i = 0; n != NULL; n = n->next, i++)
+		snodes1[i] = n;
+	for (n = node2, i = 0; n != NULL; n = n->next, i++)
+		snodes2[i] = n;
+	i_qsort(snodes1, count, sizeof(*snodes1), dsync_mailbox_node_name_cmp);
+	i_qsort(snodes2, count, sizeof(*snodes2), dsync_mailbox_node_name_cmp);
+
+	for (i = 0; i < count; i++) {
+		if (!dsync_mailbox_nodes_equal(snodes1[i], snodes2[i]))
+			return FALSE;
+		if (!dsync_mailbox_branches_equal(snodes1[i]->first_child,
+						  snodes2[i]->first_child))
+			return FALSE;
+	}
+	return TRUE;
+}
+
+bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1,
+			       struct dsync_mailbox_tree *tree2)
+{
+	bool ret;
+
+	T_BEGIN {
+		ret = dsync_mailbox_branches_equal(&tree1->root, &tree2->root);
+	} T_END;
+	return ret;
+}
+
+const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node)
+{
+	return t_strdup_printf("guid=%s uid_validity=%u uid_next=%u subs=%s last_change=%ld last_subs=%ld",
+			       guid_128_to_string(node->mailbox_guid),
+			       node->uid_validity, node->uid_next,
+			       node->subscribed ? "yes" : "no",
+			       (long)node->last_renamed_or_created,
+			       (long)node->last_subscription_change);
+}
+
+const char *
+dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type)
+{
+	switch (type) {
+	case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX:
+		return "mailbox";
+	case DSYNC_MAILBOX_DELETE_TYPE_DIR:
+		return "dir";
+	case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE:
+		return "unsubscribe";
+	}
+	i_unreached();
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-tree.h b/src/doveadm/dsync/dsync-mailbox-tree.h
new file mode 100644
index 0000000..6969813
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree.h
@@ -0,0 +1,205 @@
+#ifndef DSYNC_MAILBOX_TREE_H
+#define DSYNC_MAILBOX_TREE_H
+
+#include "guid.h"
+#include "mail-error.h"
+
+struct mail_namespace;
+struct dsync_brain;
+
+enum dsync_mailbox_trees_sync_type {
+	/* two-way sync for both mailboxes */
+	DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY,
+	/* make remote tree look exactly like the local tree */
+	DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL,
+	/* make local tree look exactly like the remote tree */
+	DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE
+};
+
+enum dsync_mailbox_trees_sync_flags {
+	/* Enable debugging */
+	DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG		= 0x01,
+	/* Show ourself as "master brain" in the debug output */
+	DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN	= 0x02,
+	/* Disable mailbox renaming logic. This is just a kludge that should
+	   be removed once the renaming logic has no more bugs.. */
+	DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES	= 0x04
+};
+
+enum dsync_mailbox_node_existence {
+	/* this is just a filler node for children or for
+	   subscription deletion */
+	DSYNC_MAILBOX_NODE_NONEXISTENT = 0,
+	/* if mailbox GUID is set, the mailbox exists.
+	   otherwise the directory exists. */
+	DSYNC_MAILBOX_NODE_EXISTS,
+	/* if mailbox GUID is set, the mailbox has been deleted.
+	   otherwise the directory has been deleted. */
+	DSYNC_MAILBOX_NODE_DELETED
+};
+
+struct dsync_mailbox_node {
+	struct dsync_mailbox_node *parent, *next, *first_child;
+
+	/* namespace where this node belongs to */
+	struct mail_namespace *ns;
+	/* this node's name (not including parents) */
+	const char *name;
+	/* mailbox GUID, or full of zeros if this is about a directory name */
+	guid_128_t mailbox_guid;
+	/* mailbox's UIDVALIDITY/UIDNEXT (may be 0 if not assigned yet) */
+	uint32_t uid_validity, uid_next;
+
+	/* existence of this mailbox/directory.
+	   doesn't affect subscription state. */
+	enum dsync_mailbox_node_existence existence;
+	/* last time the mailbox/directory was created/renamed,
+	   0 if not known */
+	time_t last_renamed_or_created;
+
+	/* last time the subscription state was changed, 0 if not known */
+	time_t last_subscription_change;
+	/* is this mailbox or directory subscribed? */
+	bool subscribed:1;
+
+	/* Internal syncing flags: */
+	bool sync_delayed_guid_change:1;
+	bool sync_temporary_name:1;
+};
+ARRAY_DEFINE_TYPE(dsync_mailbox_node, struct dsync_mailbox_node *);
+
+#define dsync_mailbox_node_guids_equal(node1, node2) \
+	(memcmp((node1)->mailbox_guid, (node2)->mailbox_guid, \
+		sizeof(guid_128_t)) == 0)
+
+#define dsync_mailbox_node_is_dir(node) \
+	guid_128_is_empty((node)->mailbox_guid)
+
+enum dsync_mailbox_delete_type {
+	/* Delete mailbox by given GUID */
+	DSYNC_MAILBOX_DELETE_TYPE_MAILBOX = 1,
+	/* Delete mailbox directory by given SHA1 name */
+	DSYNC_MAILBOX_DELETE_TYPE_DIR,
+	/* Unsubscribe mailbox by given SHA1 name */
+	DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE,
+};
+
+struct dsync_mailbox_delete {
+	enum dsync_mailbox_delete_type type;
+	guid_128_t guid;
+	time_t timestamp;
+};
+
+enum dsync_mailbox_tree_sync_type {
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX,
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR,
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX,
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR,
+	/* Rename given mailbox name and its children */
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME,
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE,
+	DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE
+};
+
+struct dsync_mailbox_tree_sync_change {
+	enum dsync_mailbox_tree_sync_type type;
+
+	/* for all types: */
+	struct mail_namespace *ns;
+	const char *full_name;
+
+	/* for create_box and delete_box: */
+	guid_128_t mailbox_guid;
+	/* for create_box: */
+	uint32_t uid_validity;
+	/* for rename: */
+	const char *rename_dest_name;
+};
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_init(char sep, char escape_char, char alt_char);
+void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **tree);
+
+/* Lookup a mailbox node by name. Returns NULL if not known. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree,
+			  const char *full_name);
+/* Lookup a mailbox node by GUID. Returns NULL if not known.
+   The mailbox GUID hash must have been build before calling this. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree,
+			       const guid_128_t guid);
+/* Lookup or create a mailbox node by name. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name);
+
+/* Returns full name for the given mailbox node. */
+const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree,
+					     const struct dsync_mailbox_node *node);
+void dsync_mailbox_node_append_full_name(string_t *str,
+					 const struct dsync_mailbox_tree *tree,
+					 const struct dsync_mailbox_node *node);
+
+/* Copy everything from src to dest, except name and hierarchy pointers */
+void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest,
+				  const struct dsync_mailbox_node *src);
+
+/* Add nodes to tree from the given namespace. If box_name or box_guid is
+   non-NULL, add only that mailbox to the tree. */
+int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree,
+			    struct mail_namespace *ns, const char *box_name,
+			    const guid_128_t box_guid,
+			    const char *const *exclude_mailboxes,
+			    enum mail_error *error_r);
+
+/* Return all known deleted mailboxes and directories. */
+const struct dsync_mailbox_delete *
+dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree,
+			       unsigned int *count_r);
+/* Return mailbox node for a given delete record, or NULL if it doesn't exist.
+   The delete record is intended to come from another tree, possibly with
+   a different hierarchy separator. dsync_mailbox_tree_build_guid_hash() must
+   have been called before this. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree,
+			       const struct dsync_mailbox_delete *del);
+/* Build GUID lookup hash, if it's not already built. Returns 0 if ok, -1 if
+   there are duplicate GUIDs. The nodes with the duplicate GUIDs are
+   returned. */
+int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree,
+				       struct dsync_mailbox_node **dup_node1_r,
+				       struct dsync_mailbox_node **dup_node2_r);
+/* Manually add a new node to hash. */
+int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree,
+				     struct dsync_mailbox_node *node,
+				     struct dsync_mailbox_node **old_node_r);
+/* Set remote separator used for directory deletions in
+   dsync_mailbox_tree_find_delete() */
+void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree,
+					 char remote_sep,
+					 char remote_escape_char);
+
+/* Iterate through all nodes in a tree (depth-first) */
+struct dsync_mailbox_tree_iter *
+dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree);
+bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter,
+				  const char **full_name_r,
+				  struct dsync_mailbox_node **node_r);
+void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **iter);
+
+/* Sync local and remote trees so at the end they're exactly the same.
+   Return changes done to local tree. */
+struct dsync_mailbox_tree_sync_ctx *
+dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
+			      struct dsync_mailbox_tree *remote_tree,
+			      enum dsync_mailbox_trees_sync_type sync_type,
+			      enum dsync_mailbox_trees_sync_flags sync_flags);
+const struct dsync_mailbox_tree_sync_change *
+dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx);
+int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx);
+
+const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node);
+const char *
+dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox.c b/src/doveadm/dsync/dsync-mailbox.c
new file mode 100644
index 0000000..d6d06fd
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox.c
@@ -0,0 +1,61 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "mail-storage-private.h"
+#include "dsync-brain-private.h"
+#include "dsync-mailbox.h"
+
+void dsync_mailbox_attribute_dup(pool_t pool,
+				 const struct dsync_mailbox_attribute *src,
+				 struct dsync_mailbox_attribute *dest_r)
+{
+	dest_r->type = src->type;
+	dest_r->key = p_strdup(pool, src->key);
+	dest_r->value = p_strdup(pool, src->value);
+	if (src->value_stream != NULL) {
+		dest_r->value_stream = src->value_stream;
+		i_stream_ref(dest_r->value_stream);
+	}
+
+	dest_r->deleted = src->deleted;
+	dest_r->last_change = src->last_change;
+	dest_r->modseq = src->modseq;
+}
+
+int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box,
+		       struct file_lock **lock_r)
+{
+	const char *path, *error;
+	int ret;
+
+	/* Make sure the mailbox is open - locking requires it */
+	if (mailbox_open(box) < 0) {
+		i_error("Can't open mailbox %s: %s", mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, &brain->mail_error));
+		return -1;
+	}
+
+	ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path);
+	if (ret < 0) {
+		i_error("Can't get mailbox %s path: %s", mailbox_get_vname(box),
+			mailbox_get_last_internal_error(box, &brain->mail_error));
+		return -1;
+	}
+	if (ret == 0) {
+		/* No index files - don't do any locking. In theory we still
+		   could, but this lock is mainly meant to prevent replication
+		   problems, and replication wouldn't work without indexes. */
+		*lock_r = NULL;
+		return 0;
+	}
+
+	if (mailbox_lock_file_create(box, DSYNC_MAILBOX_LOCK_FILENAME,
+				     brain->mailbox_lock_timeout_secs,
+				     lock_r, &error) <= 0) {
+		i_error("Failed to lock mailbox %s for dsyncing: %s",
+			box->vname, error);
+		return -1;
+	}
+	return 0;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox.h b/src/doveadm/dsync/dsync-mailbox.h
new file mode 100644
index 0000000..7e81c0c
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox.h
@@ -0,0 +1,44 @@
+#ifndef DSYNC_MAILBOX_H
+#define DSYNC_MAILBOX_H
+
+#include "mail-storage.h"
+
+struct dsync_brain;
+
+/* Mailbox that is going to be synced. Its name was already sent in the
+   mailbox tree. */
+struct dsync_mailbox {
+	guid_128_t mailbox_guid;
+	bool mailbox_lost;
+	bool mailbox_ignore;
+	bool have_guids, have_save_guids, have_only_guid128;
+
+	uint32_t uid_validity, uid_next, messages_count, first_recent_uid;
+	uint64_t highest_modseq, highest_pvt_modseq;
+	ARRAY_TYPE(mailbox_cache_field) cache_fields;
+};
+
+struct dsync_mailbox_attribute {
+	enum mail_attribute_type type;
+	const char *key;
+	/* if both values are NULL = not looked up yet / deleted */
+	const char *value;
+	struct istream *value_stream;
+
+	time_t last_change; /* 0 = unknown */
+	uint64_t modseq; /* 0 = unknown */
+
+	bool deleted; /* attribute is known to have been deleted */
+	bool exported; /* internally used by exporting */
+};
+#define DSYNC_ATTR_HAS_VALUE(attr) \
+	((attr)->value != NULL || (attr)->value_stream != NULL)
+
+void dsync_mailbox_attribute_dup(pool_t pool,
+				 const struct dsync_mailbox_attribute *src,
+				 struct dsync_mailbox_attribute *dest_r);
+
+int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box,
+		       struct file_lock **lock_r);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-serializer.c b/src/doveadm/dsync/dsync-serializer.c
new file mode 100644
index 0000000..7b369b8
--- /dev/null
+++ b/src/doveadm/dsync/dsync-serializer.c
@@ -0,0 +1,117 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strescape.h"
+#include "dsync-serializer.h"
+
+struct dsync_serializer {
+	pool_t pool;
+	const char *const *keys;
+	unsigned int keys_count;
+};
+
+struct dsync_serializer_encoder {
+	pool_t pool;
+	struct dsync_serializer *serializer;
+	ARRAY_TYPE(const_string) values;
+};
+
+struct dsync_serializer *dsync_serializer_init(const char *const keys[])
+{
+	struct dsync_serializer *serializer;
+	pool_t pool;
+	const char **dup_keys;
+	unsigned int i, count;
+
+	pool = pool_alloconly_create("dsync serializer", 512);
+	serializer = p_new(pool, struct dsync_serializer, 1);
+	serializer->pool = pool;
+
+	count = str_array_length(keys);
+	dup_keys = p_new(pool, const char *, count + 1);
+	for (i = 0; i < count; i++)
+		dup_keys[i] = p_strdup(pool, keys[i]);
+	serializer->keys = dup_keys;
+	serializer->keys_count = count;
+	return serializer;
+}
+
+void dsync_serializer_deinit(struct dsync_serializer **_serializer)
+{
+	struct dsync_serializer *serializer = *_serializer;
+
+	*_serializer = NULL;
+
+	pool_unref(&serializer->pool);
+}
+
+const char *
+dsync_serializer_encode_header_line(struct dsync_serializer *serializer)
+{
+	string_t *str = t_str_new(128);
+	unsigned int i;
+
+	for (i = 0; serializer->keys[i] != NULL; i++) {
+		if (i > 0)
+			str_append_c(str, '\t');
+		str_append_tabescaped(str, serializer->keys[i]);
+	}
+	str_append_c(str, '\n');
+	return str_c(str);
+}
+
+struct dsync_serializer_encoder *
+dsync_serializer_encode_begin(struct dsync_serializer *serializer)
+{
+	struct dsync_serializer_encoder *encoder;
+	pool_t pool = pool_alloconly_create("dsync serializer encode", 1024);
+
+	encoder = p_new(pool, struct dsync_serializer_encoder, 1);
+	encoder->pool = pool;
+	encoder->serializer = serializer;
+	p_array_init(&encoder->values, pool, serializer->keys_count);
+	return encoder;
+}
+
+void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder,
+				 const char *key, const char *value)
+{
+	unsigned int i;
+
+	for (i = 0; encoder->serializer->keys[i] != NULL; i++) {
+		if (strcmp(encoder->serializer->keys[i], key) == 0) {
+			value = p_strdup(encoder->pool, value);
+			array_idx_set(&encoder->values, i, &value);
+			return;
+		}
+	}
+	i_panic("Unknown key: %s", key);
+}
+
+void dsync_serializer_encode_finish(struct dsync_serializer_encoder **_encoder,
+				    string_t *output)
+{
+	struct dsync_serializer_encoder *encoder = *_encoder;
+	const char *const *values;
+	unsigned int i, count;
+
+	*_encoder = NULL;
+
+	values = array_get(&encoder->values, &count);
+	for (i = 0; i < count; i++) {
+		if (i > 0)
+			str_append_c(output, '\t');
+		if (values[i] == NULL)
+			str_append_c(output, NULL_CHR);
+		else  {
+			if (values[i][0] == NULL_CHR)
+				str_append_c(output, NULL_CHR);
+			str_append_tabescaped(output, values[i]);
+		}
+	}
+	str_append_c(output, '\n');
+
+	pool_unref(&encoder->pool);
+}
diff --git a/src/doveadm/dsync/dsync-serializer.h b/src/doveadm/dsync/dsync-serializer.h
new file mode 100644
index 0000000..1ed6c31
--- /dev/null
+++ b/src/doveadm/dsync/dsync-serializer.h
@@ -0,0 +1,18 @@
+#ifndef DSYNC_SERIALIZER_H
+#define DSYNC_SERIALIZER_H
+
+#define NULL_CHR '\002'
+
+struct dsync_serializer *dsync_serializer_init(const char *const keys[]);
+void dsync_serializer_deinit(struct dsync_serializer **serializer);
+
+const char *
+dsync_serializer_encode_header_line(struct dsync_serializer *serializer);
+struct dsync_serializer_encoder *
+dsync_serializer_encode_begin(struct dsync_serializer *serializer);
+void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder,
+				 const char *key, const char *value);
+void dsync_serializer_encode_finish(struct dsync_serializer_encoder **encoder,
+				    string_t *output);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.c b/src/doveadm/dsync/dsync-transaction-log-scan.c
new file mode 100644
index 0000000..dd2834a
--- /dev/null
+++ b/src/doveadm/dsync/dsync-transaction-log-scan.c
@@ -0,0 +1,608 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "hash.h"
+#include "mail-index-modseq.h"
+#include "mail-storage-private.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-transaction-log-scan.h"
+
+struct dsync_transaction_log_scan {
+	pool_t pool;
+	HASH_TABLE_TYPE(dsync_uid_mail_change) changes;
+	HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
+	struct mail_index_view *view;
+	uint32_t highest_wanted_uid;
+
+	uint32_t last_log_seq;
+	uoff_t last_log_offset;
+
+	bool returned_all_changes;
+};
+
+static bool ATTR_NOWARN_UNUSED_RESULT
+export_change_get(struct dsync_transaction_log_scan *ctx, uint32_t uid,
+		  enum dsync_mail_change_type type,
+		  struct dsync_mail_change **change_r)
+{
+	struct dsync_mail_change *change;
+	const char *orig_guid;
+
+	i_assert(uid > 0);
+	i_assert(type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
+
+	*change_r = NULL;
+
+	if (uid > ctx->highest_wanted_uid)
+		return FALSE;
+
+	change = hash_table_lookup(ctx->changes, POINTER_CAST(uid));
+	if (change == NULL) {
+		/* first change for this UID */
+		change = p_new(ctx->pool, struct dsync_mail_change, 1);
+		change->uid = uid;
+		change->type = type;
+		hash_table_insert(ctx->changes, POINTER_CAST(uid), change);
+	} else if (type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+		/* expunge overrides flag changes */
+		orig_guid = change->guid;
+		i_zero(change);
+		change->type = type;
+		change->uid = uid;
+		change->guid = orig_guid;
+	} else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+		/* already expunged, this change doesn't matter */
+		return FALSE;
+	} else {
+		/* another flag update */
+	}
+	*change_r = change;
+	return TRUE;
+}
+
+static void
+log_add_expunge(struct dsync_transaction_log_scan *ctx, const void *data,
+		const struct mail_transaction_header *hdr)
+{
+	const struct mail_transaction_expunge *rec = data, *end;
+	struct dsync_mail_change *change;
+	uint32_t uid;
+
+	if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+		/* this is simply a request for expunge */
+		return;
+	}
+	end = CONST_PTR_OFFSET(data, hdr->size);
+	for (; rec != end; rec++) {
+		for (uid = rec->uid1; uid <= rec->uid2; uid++) {
+			export_change_get(ctx, uid,
+					  DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+					  &change);
+		}
+	}
+}
+
+static bool
+log_add_expunge_uid(struct dsync_transaction_log_scan *ctx, const void *data,
+		    const struct mail_transaction_header *hdr, uint32_t uid)
+{
+	const struct mail_transaction_expunge *rec = data, *end;
+	struct dsync_mail_change *change;
+
+	if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+		/* this is simply a request for expunge */
+		return FALSE;
+	}
+	end = CONST_PTR_OFFSET(data, hdr->size);
+	for (; rec != end; rec++) {
+		if (uid >= rec->uid1 && uid <= rec->uid2) {
+			export_change_get(ctx, uid,
+					  DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+					  &change);
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static void
+log_add_expunge_guid(struct dsync_transaction_log_scan *ctx,
+		     struct mail_index_view *view, const void *data,
+		     const struct mail_transaction_header *hdr)
+{
+	const struct mail_transaction_expunge_guid *rec = data, *end;
+	struct dsync_mail_change *change;
+	uint32_t seq;
+	bool external;
+
+	external = (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0;
+
+	end = CONST_PTR_OFFSET(data, hdr->size);
+	for (; rec != end; rec++) {
+		if (!external && mail_index_lookup_seq(view, rec->uid, &seq)) {
+			/* expunge request that hasn't been actually done yet.
+			   we check non-external ones because they might have
+			   the GUID while external ones don't. */
+			continue;
+		}
+		if (export_change_get(ctx, rec->uid,
+				      DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+				      &change) &&
+		    !guid_128_is_empty(rec->guid_128)) T_BEGIN {
+			change->guid = p_strdup(ctx->pool,
+				guid_128_to_string(rec->guid_128));
+		} T_END;
+	}
+}
+
+static bool
+log_add_expunge_guid_uid(struct dsync_transaction_log_scan *ctx, const void *data,
+			 const struct mail_transaction_header *hdr, uint32_t uid)
+{
+	const struct mail_transaction_expunge_guid *rec = data, *end;
+	struct dsync_mail_change *change;
+
+	/* we're assuming UID is already known to be expunged */
+	end = CONST_PTR_OFFSET(data, hdr->size);
+	for (; rec != end; rec++) {
+		if (rec->uid != uid)
+			continue;
+
+		if (!export_change_get(ctx, rec->uid,
+				       DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+				       &change))
+			i_unreached();
+		if (!guid_128_is_empty(rec->guid_128)) T_BEGIN {
+			change->guid = p_strdup(ctx->pool,
+						guid_128_to_string(rec->guid_128));
+		} T_END;
+		return TRUE;
+	}
+	return FALSE;
+}
+
+static void
+log_add_flag_update(struct dsync_transaction_log_scan *ctx, const void *data,
+		    const struct mail_transaction_header *hdr)
+{
+	const struct mail_transaction_flag_update *rec = data, *end;
+	struct dsync_mail_change *change;
+	uint32_t uid;
+
+	end = CONST_PTR_OFFSET(data, hdr->size);
+	for (; rec != end; rec++) {
+		for (uid = rec->uid1; uid <= rec->uid2; uid++) {
+			if (export_change_get(ctx, uid,
+					DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+					&change)) {
+				change->add_flags |= rec->add_flags;
+				change->remove_flags &= ENUM_NEGATE(rec->add_flags);
+				change->remove_flags |= rec->remove_flags;
+				change->add_flags &= ENUM_NEGATE(rec->remove_flags);
+			}
+		}
+	}
+}
+
+static void
+log_add_keyword_reset(struct dsync_transaction_log_scan *ctx, const void *data,
+		      const struct mail_transaction_header *hdr)
+{
+	const struct mail_transaction_keyword_reset *rec = data, *end;
+	struct dsync_mail_change *change;
+	uint32_t uid;
+
+	end = CONST_PTR_OFFSET(data, hdr->size);
+	for (; rec != end; rec++) {
+		for (uid = rec->uid1; uid <= rec->uid2; uid++) {
+			if (!export_change_get(ctx, uid,
+					DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+					&change))
+				continue;
+
+			change->keywords_reset = TRUE;
+			if (array_is_created(&change->keyword_changes))
+				array_clear(&change->keyword_changes);
+		}
+	}
+}
+
+static void
+keywords_change_remove(struct dsync_mail_change *change, const char *name)
+{
+	const char *const *changes;
+	unsigned int i, count;
+
+	changes = array_get(&change->keyword_changes, &count);
+	for (i = 0; i < count; i++) {
+		if (strcmp(changes[i]+1, name) == 0) {
+			array_delete(&change->keyword_changes, i, 1);
+			break;
+		}
+	}
+}
+
+static void
+log_add_keyword_update(struct dsync_transaction_log_scan *ctx, const void *data,
+		       const struct mail_transaction_header *hdr)
+{
+	const struct mail_transaction_keyword_update *rec = data;
+	struct dsync_mail_change *change;
+	const char *kw_name, *change_str;
+	const uint32_t *uids, *end;
+	unsigned int uids_offset;
+	uint32_t uid;
+
+	uids_offset = sizeof(*rec) + rec->name_size;
+	if ((uids_offset % 4) != 0)
+		uids_offset += 4 - (uids_offset % 4);
+
+	kw_name = t_strndup((const void *)(rec+1), rec->name_size);
+	switch (rec->modify_type) {
+	case MODIFY_ADD:
+		change_str = p_strdup_printf(ctx->pool, "%c%s",
+					     KEYWORD_CHANGE_ADD, kw_name);
+		break;
+	case MODIFY_REMOVE:
+		change_str = p_strdup_printf(ctx->pool, "%c%s",
+					     KEYWORD_CHANGE_REMOVE, kw_name);
+		break;
+	default:
+		i_unreached();
+	}
+
+	uids = CONST_PTR_OFFSET(rec, uids_offset);
+	end = CONST_PTR_OFFSET(rec, hdr->size);
+
+	for (; uids < end; uids += 2) {
+		for (uid = uids[0]; uid <= uids[1]; uid++) {
+			if (!export_change_get(ctx, uid,
+					DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+					&change))
+				continue;
+			if (!array_is_created(&change->keyword_changes)) {
+				p_array_init(&change->keyword_changes,
+					     ctx->pool, 4);
+			} else {
+				keywords_change_remove(change, kw_name);
+			}
+			array_push_back(&change->keyword_changes, &change_str);
+		}
+	}
+}
+
+static void
+log_add_modseq_update(struct dsync_transaction_log_scan *ctx, const void *data,
+		      const struct mail_transaction_header *hdr, bool pvt_scan)
+{
+	const struct mail_transaction_modseq_update *rec = data, *end;
+	struct dsync_mail_change *change;
+	uint64_t modseq;
+
+	/* update message's modseq, possibly by creating an empty flag change */
+	end = CONST_PTR_OFFSET(rec, hdr->size);
+	for (; rec != end; rec++) {
+		if (rec->uid == 0) {
+			/* highestmodseq update */
+			continue;
+		}
+
+		if (!export_change_get(ctx, rec->uid,
+				       DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+				       &change))
+			continue;
+
+		modseq = rec->modseq_low32 |
+			((uint64_t)rec->modseq_high32 << 32);
+		if (!pvt_scan) {
+			if (change->modseq < modseq)
+				change->modseq = modseq;
+		} else {
+			if (change->pvt_modseq < modseq)
+				change->pvt_modseq = modseq;
+		}
+	}
+}
+
+static void
+log_add_attribute_update_key(struct dsync_transaction_log_scan *ctx,
+			     const char *attr_change, uint64_t modseq)
+{
+	struct dsync_mailbox_attribute lookup_attr, *attr;
+
+	i_assert(strlen(attr_change) > 2); /* checked by lib-index */
+
+	lookup_attr.type = attr_change[1] == 'p' ?
+		MAIL_ATTRIBUTE_TYPE_PRIVATE : MAIL_ATTRIBUTE_TYPE_SHARED;
+	lookup_attr.key = attr_change+2;
+
+	attr = hash_table_lookup(ctx->attr_changes, &lookup_attr);
+	if (attr == NULL) {
+		attr = p_new(ctx->pool, struct dsync_mailbox_attribute, 1);
+		attr->type = lookup_attr.type;
+		attr->key = p_strdup(ctx->pool, lookup_attr.key);
+		hash_table_insert(ctx->attr_changes, attr, attr);
+	}
+	attr->deleted = attr_change[0] == '-';
+	attr->modseq = modseq;
+}
+
+static void
+log_add_attribute_update(struct dsync_transaction_log_scan *ctx,
+			 const void *data,
+			 const struct mail_transaction_header *hdr,
+			 uint64_t modseq)
+{
+	const char *attr_changes = data;
+	unsigned int i;
+
+	for (i = 0; i < hdr->size && attr_changes[i] != '\0'; ) {
+		log_add_attribute_update_key(ctx, attr_changes+i, modseq);
+		i += strlen(attr_changes+i) + 1;
+	}
+}
+
+static int
+dsync_log_set(struct dsync_transaction_log_scan *ctx,
+	      struct mail_index_view *view, bool pvt_scan,
+	      struct mail_transaction_log_view *log_view, uint64_t modseq)
+{
+	uint32_t log_seq, end_seq;
+	uoff_t log_offset, end_offset;
+	const char *reason;
+	bool reset;
+	int ret;
+
+	end_seq = view->log_file_head_seq;
+	end_offset = view->log_file_head_offset;
+
+	if (modseq != 0 &&
+	    mail_index_modseq_get_next_log_offset(view, modseq,
+						  &log_seq, &log_offset)) {
+		/* scan the view only up to end of the current view.
+		   if there are more changes, we don't care about them until
+		   the next sync. */
+		ret = mail_transaction_log_view_set(log_view,
+						    log_seq, log_offset,
+						    end_seq, end_offset,
+						    &reset, &reason);
+		if (ret != 0)
+			return ret;
+	}
+
+	/* return everything we've got (until the end of the view) */
+	if (!pvt_scan)
+		ctx->returned_all_changes = TRUE;
+	if (mail_transaction_log_view_set_all(log_view) < 0)
+		return -1;
+
+	mail_transaction_log_view_get_prev_pos(log_view, &log_seq, &log_offset);
+	if (log_seq > end_seq ||
+	    (log_seq == end_seq && log_offset > end_offset)) {
+		end_seq = log_seq;
+		end_offset = log_offset;
+	}
+	ret = mail_transaction_log_view_set(log_view,
+					    log_seq, log_offset,
+					    end_seq, end_offset,
+					    &reset, &reason);
+	if (ret == 0) {
+		/* we shouldn't get here. _view_set_all() already
+		   reserved all the log files, the _view_set() only
+		   removed unwanted ones. */
+		i_error("%s: Couldn't set transaction log view (seq %u..%u): %s",
+			view->index->filepath, log_seq, end_seq, reason);
+		ret = -1;
+	}
+	if (ret < 0)
+		return -1;
+	if (modseq != 0) {
+		/* we didn't see all the changes that we wanted to */
+		return 0;
+	}
+	return 1;
+}
+
+static int
+dsync_log_scan(struct dsync_transaction_log_scan *ctx,
+	       struct mail_index_view *view, uint64_t modseq, bool pvt_scan)
+{
+	struct mail_transaction_log_view *log_view;
+	const struct mail_transaction_header *hdr;
+	const void *data;
+	uint32_t file_seq, max_seq;
+	uoff_t file_offset, max_offset;
+	uint64_t cur_modseq;
+	int ret;
+
+	log_view = mail_transaction_log_view_open(view->index->log);
+	if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) {
+		mail_transaction_log_view_close(&log_view);
+		return -1;
+	}
+
+	/* read the log only up to current position in view */
+	max_seq = view->log_file_expunge_seq;
+	max_offset = view->log_file_expunge_offset;
+
+	mail_transaction_log_view_get_prev_pos(log_view, &file_seq,
+					       &file_offset);
+
+	while (mail_transaction_log_view_next(log_view, &hdr, &data) > 0) {
+		mail_transaction_log_view_get_prev_pos(log_view, &file_seq,
+						       &file_offset);
+		if (file_offset >= max_offset && file_seq == max_seq)
+			break;
+
+		if ((hdr->type & MAIL_TRANSACTION_SYNC) != 0) {
+			/* ignore changes done by dsync, unless we can get
+			   expunged message's GUID from it */
+			if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) !=
+			    MAIL_TRANSACTION_EXPUNGE_GUID)
+				continue;
+		}
+
+		switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+		case MAIL_TRANSACTION_EXPUNGE:
+			if (!pvt_scan)
+				log_add_expunge(ctx, data, hdr);
+			break;
+		case MAIL_TRANSACTION_EXPUNGE_GUID:
+			if (!pvt_scan)
+				log_add_expunge_guid(ctx, view, data, hdr);
+			break;
+		case MAIL_TRANSACTION_FLAG_UPDATE:
+			log_add_flag_update(ctx, data, hdr);
+			break;
+		case MAIL_TRANSACTION_KEYWORD_RESET:
+			log_add_keyword_reset(ctx, data, hdr);
+			break;
+		case MAIL_TRANSACTION_KEYWORD_UPDATE:
+			T_BEGIN {
+				log_add_keyword_update(ctx, data, hdr);
+			} T_END;
+			break;
+		case MAIL_TRANSACTION_MODSEQ_UPDATE:
+			log_add_modseq_update(ctx, data, hdr, pvt_scan);
+			break;
+		case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
+			cur_modseq = mail_transaction_log_view_get_prev_modseq(log_view);
+			log_add_attribute_update(ctx, data, hdr, cur_modseq);
+			break;
+		}
+	}
+
+	if (!pvt_scan) {
+		ctx->last_log_seq = file_seq;
+		ctx->last_log_offset = file_offset;
+	}
+	mail_transaction_log_view_close(&log_view);
+	return ret;
+}
+
+static int
+dsync_mailbox_attribute_cmp(const struct dsync_mailbox_attribute *attr1,
+			    const struct dsync_mailbox_attribute *attr2)
+{
+	if (attr1->type < attr2->type)
+		return -1;
+	if (attr1->type > attr2->type)
+		return 1;
+	return strcmp(attr1->key, attr2->key);
+}
+
+static unsigned int
+dsync_mailbox_attribute_hash(const struct dsync_mailbox_attribute *attr)
+{
+	return str_hash(attr->key) ^ attr->type;
+}
+
+int dsync_transaction_log_scan_init(struct mail_index_view *view,
+				    struct mail_index_view *pvt_view,
+				    uint32_t highest_wanted_uid,
+				    uint64_t modseq, uint64_t pvt_modseq,
+				    struct dsync_transaction_log_scan **scan_r,
+				    bool *pvt_too_old_r)
+{
+	struct dsync_transaction_log_scan *ctx;
+	pool_t pool;
+	int ret, ret2;
+
+	*pvt_too_old_r = FALSE;
+
+	pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan",
+				     10240);
+	ctx = p_new(pool, struct dsync_transaction_log_scan, 1);
+	ctx->pool = pool;
+	hash_table_create_direct(&ctx->changes, pool, 0);
+	hash_table_create(&ctx->attr_changes, pool, 0,
+			  dsync_mailbox_attribute_hash,
+			  dsync_mailbox_attribute_cmp);
+	ctx->view = view;
+	ctx->highest_wanted_uid = highest_wanted_uid;
+
+	if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0)
+		return -1;
+	if (pvt_view != NULL) {
+		if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0)
+			return -1;
+		if (ret2 == 0) {
+			ret = 0;
+			*pvt_too_old_r = TRUE;
+		}
+	}
+
+	*scan_r = ctx;
+	return ret;
+}
+
+HASH_TABLE_TYPE(dsync_uid_mail_change)
+dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan)
+{
+	return scan->changes;
+}
+
+HASH_TABLE_TYPE(dsync_attr_change)
+dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan)
+{
+	return scan->attr_changes;
+}
+
+bool
+dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan)
+{
+	return scan->returned_all_changes;
+}
+
+struct dsync_mail_change *
+dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan,
+					    uint32_t uid)
+{
+	struct mail_transaction_log_view *log_view;
+	const struct mail_transaction_header *hdr;
+	const void *data;
+	const char *reason;
+	bool reset, found = FALSE;
+
+	i_assert(uid > 0);
+
+	if (scan->highest_wanted_uid < uid)
+		scan->highest_wanted_uid = uid;
+
+	log_view = mail_transaction_log_view_open(scan->view->index->log);
+	if (mail_transaction_log_view_set(log_view,
+					  scan->last_log_seq,
+					  scan->last_log_offset,
+					  (uint32_t)-1, UOFF_T_MAX,
+					  &reset, &reason) > 0) {
+		while (!found &&
+		       mail_transaction_log_view_next(log_view, &hdr, &data) > 0) {
+			switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+			case MAIL_TRANSACTION_EXPUNGE:
+				if (log_add_expunge_uid(scan, data, hdr, uid))
+					found = TRUE;
+				break;
+			case MAIL_TRANSACTION_EXPUNGE_GUID:
+				if (log_add_expunge_guid_uid(scan, data, hdr, uid))
+					found = TRUE;
+				break;
+			}
+		}
+	}
+	mail_transaction_log_view_close(&log_view);
+
+	return !found ? NULL :
+		hash_table_lookup(scan->changes, POINTER_CAST(uid));
+}
+
+void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **_scan)
+{
+	struct dsync_transaction_log_scan *scan = *_scan;
+
+	*_scan = NULL;
+
+	hash_table_destroy(&scan->changes);
+	hash_table_destroy(&scan->attr_changes);
+	pool_unref(&scan->pool);
+}
diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.h b/src/doveadm/dsync/dsync-transaction-log-scan.h
new file mode 100644
index 0000000..458b775
--- /dev/null
+++ b/src/doveadm/dsync/dsync-transaction-log-scan.h
@@ -0,0 +1,32 @@
+#ifndef DSYNC_TRANSACTION_LOG_SCAN_H
+#define DSYNC_TRANSACTION_LOG_SCAN_H
+
+HASH_TABLE_DEFINE_TYPE(dsync_uid_mail_change,
+		       void *, struct dsync_mail_change *);
+HASH_TABLE_DEFINE_TYPE(dsync_attr_change,
+		       struct dsync_mailbox_attribute *,
+		       struct dsync_mailbox_attribute *);
+
+struct mail_index_view;
+struct dsync_transaction_log_scan;
+
+int dsync_transaction_log_scan_init(struct mail_index_view *view,
+				    struct mail_index_view *pvt_view,
+				    uint32_t highest_wanted_uid,
+				    uint64_t modseq, uint64_t pvt_modseq,
+				    struct dsync_transaction_log_scan **scan_r,
+				    bool *pvt_too_old_r);
+HASH_TABLE_TYPE(dsync_uid_mail_change)
+dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan);
+HASH_TABLE_TYPE(dsync_attr_change)
+dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan);
+/* Returns TRUE if the entire transaction log was scanned */
+bool dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan);
+/* If the given UID has been expunged after the initial log scan, create/update
+   a change record for it and return it. */
+struct dsync_mail_change *
+dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan,
+					    uint32_t uid);
+void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **scan);
+
+#endif
diff --git a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
new file mode 100644
index 0000000..b068137
--- /dev/null
+++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
@@ -0,0 +1,781 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "sha1.h"
+#include "str.h"
+#include "mailbox-list-private.h"
+#include "dsync-mailbox-tree-private.h"
+#include "test-common.h"
+
+#include <stdio.h>
+
+#define MAX_DEPTH 4
+#define TEST_NAMESPACE_NAME "INBOX"
+
+static struct mail_namespace inbox_namespace = {
+	.prefix = TEST_NAMESPACE_NAME"/",
+	.prefix_len = sizeof(TEST_NAMESPACE_NAME)-1 + 1
+};
+
+char mail_namespace_get_sep(struct mail_namespace *ns ATTR_UNUSED)
+{
+	return '/';
+}
+
+void mailbox_name_get_sha128(const char *name, guid_128_t guid_128_r)
+{
+	unsigned char sha[SHA1_RESULTLEN];
+
+	sha1_get_digest(name, strlen(name), sha);
+	memcpy(guid_128_r, sha, I_MIN(GUID_128_SIZE, sizeof(sha)));
+}
+
+void mailbox_list_name_unescape(const char **name ATTR_UNUSED,
+				char escape_char ATTR_UNUSED)
+{
+}
+
+void mailbox_list_name_escape(const char *name,
+			      const char *escape_chars ATTR_UNUSED,
+			      string_t *dest)
+{
+	str_append(dest, name);
+}
+
+static struct dsync_mailbox_node *
+node_create(struct dsync_mailbox_tree *tree, unsigned int counter,
+	    const char *name, unsigned int last_renamed_or_created)
+{
+	struct dsync_mailbox_node *node;
+
+	node = dsync_mailbox_tree_get(tree, name);
+	memcpy(node->mailbox_guid, &counter, sizeof(counter));
+	node->uid_validity = counter;
+	node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+	node->last_renamed_or_created = last_renamed_or_created;
+	return node;
+}
+
+static struct dsync_mailbox_node *
+random_node_create(struct dsync_mailbox_tree *tree, unsigned int counter,
+		   const char *name)
+{
+	return node_create(tree, counter, name, i_rand_limit(10));
+}
+
+static void nodes_create(struct dsync_mailbox_tree *tree, unsigned int *counter,
+			 const char *const *names)
+{
+	for (; *names != NULL; names++) {
+		*counter += 1;
+		node_create(tree, *counter, *names, 0);
+	}
+}
+
+static void nodes_delete(struct dsync_mailbox_tree *tree, unsigned int *counter,
+			 const char *const *names)
+{
+	struct dsync_mailbox_node *node;
+
+	for (; *names != NULL; names++) {
+		*counter += 1;
+		node = node_create(tree, *counter, *names, 0);
+		node->existence = DSYNC_MAILBOX_NODE_DELETED;
+	}
+}
+
+static void
+create_random_nodes(struct dsync_mailbox_tree *tree, const char *parent_name,
+		    unsigned int depth, unsigned int *counter)
+{
+	unsigned int parent_len, i, nodes_count = i_rand_minmax(1, 3);
+	string_t *str;
+
+	if (depth == MAX_DEPTH)
+		return;
+
+	str = t_str_new(32);
+	if (*parent_name != '\0')
+		str_printfa(str, "%s/", parent_name);
+	parent_len = str_len(str);
+
+	for (i = 0; i < nodes_count; i++) {
+		*counter += 1;
+		str_truncate(str, parent_len);
+		str_printfa(str, "%u.%u", depth, i);
+		random_node_create(tree, *counter, str_c(str));
+		create_random_nodes(tree, str_c(str), depth+1, counter);
+	}
+}
+
+static struct dsync_mailbox_tree *create_random_tree(void)
+{
+	struct dsync_mailbox_tree *tree;
+	unsigned int counter = 0;
+
+	tree = dsync_mailbox_tree_init('/', '\0', '_');
+	create_random_nodes(tree, "", 0, &counter);
+	return tree;
+}
+
+static void test_tree_nodes_fixup(struct dsync_mailbox_node **pos,
+				  unsigned int *newguid_counter)
+{
+	struct dsync_mailbox_node *node;
+
+	for (node = *pos; node != NULL; node = node->next) {
+		if (node->sync_delayed_guid_change) {
+			/* the real code will pick one of the GUIDs.
+			 we don't really care which one gets picked, so we'll
+			 just change them to the same new one */
+			memcpy(node->mailbox_guid, newguid_counter,
+			       sizeof(*newguid_counter));
+			node->uid_validity = *newguid_counter;
+			*newguid_counter += 1;
+		}
+		if (node->existence == DSYNC_MAILBOX_NODE_DELETED)
+			node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+		test_tree_nodes_fixup(&node->first_child, newguid_counter);
+		if (node->existence != DSYNC_MAILBOX_NODE_EXISTS &&
+		    node->first_child == NULL) {
+			/* nonexistent node, drop it */
+			*pos = node->next;
+		} else {
+			pos = &node->next;
+		}
+	}
+}
+
+static void test_tree_fixup(struct dsync_mailbox_tree *tree)
+{
+	unsigned int newguid_counter = INT_MAX;
+
+	test_tree_nodes_fixup(&tree->root.first_child, &newguid_counter);
+}
+
+static void nodes_dump(const struct dsync_mailbox_node *node, unsigned int depth)
+{
+	unsigned int i;
+
+	for (; node != NULL; node = node->next) {
+		for (i = 0; i < depth; i++) printf(" ");
+		printf("%-*s guid:%.5s uidv:%u %d%d %ld\n", 40-depth, node->name,
+		       guid_128_to_string(node->mailbox_guid), node->uid_validity,
+		       node->existence, node->subscribed ? 1 : 0,
+		       (long)node->last_renamed_or_created);
+		nodes_dump(node->first_child, depth+1);
+	}
+}
+
+static void trees_dump(struct dsync_mailbox_tree *tree1,
+		       struct dsync_mailbox_tree *tree2)
+{
+	printf("tree1:\n");
+	nodes_dump(tree1->root.first_child, 1);
+	printf("tree2:\n");
+	nodes_dump(tree2->root.first_child, 1);
+}
+
+static void test_trees_nofree(struct dsync_mailbox_tree *tree1,
+			      struct dsync_mailbox_tree **_tree2)
+{
+	struct dsync_mailbox_tree *tree2 = *_tree2;
+	struct dsync_mailbox_tree *orig_tree1, *orig_tree2;
+	struct dsync_mailbox_tree_sync_ctx *ctx;
+	struct dsync_mailbox_node *dup_node1, *dup_node2;
+
+	orig_tree1 = dsync_mailbox_tree_dup(tree1);
+	orig_tree2 = dsync_mailbox_tree_dup(tree2);
+
+	/* test tree1 -> tree2 */
+	dsync_mailbox_tree_build_guid_hash(tree1, &dup_node1, &dup_node2);
+	dsync_mailbox_tree_build_guid_hash(tree2, &dup_node1, &dup_node2);
+	ctx = dsync_mailbox_trees_sync_init(tree1, tree2,
+					    DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY,
+					    DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG);
+	while (dsync_mailbox_trees_sync_next(ctx) != NULL) {
+	}
+	dsync_mailbox_trees_sync_deinit(&ctx);
+	test_tree_fixup(tree1);
+	test_tree_fixup(tree2);
+	if (!dsync_mailbox_trees_equal(tree1, tree2)) {
+		test_assert(FALSE);
+		trees_dump(tree1, tree2);
+	}
+
+	/* test tree2 -> tree1 */
+	dsync_mailbox_tree_build_guid_hash(orig_tree1, &dup_node1, &dup_node2);
+	dsync_mailbox_tree_build_guid_hash(orig_tree2, &dup_node1, &dup_node2);
+	ctx = dsync_mailbox_trees_sync_init(orig_tree2, orig_tree1,
+					    DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0);
+	while (dsync_mailbox_trees_sync_next(ctx) != NULL) {
+	}
+	dsync_mailbox_trees_sync_deinit(&ctx);
+	test_tree_fixup(orig_tree1);
+	test_tree_fixup(orig_tree2);
+	if (!dsync_mailbox_trees_equal(orig_tree1, orig_tree2)) {
+		test_assert(FALSE);
+		trees_dump(orig_tree1, orig_tree2);
+	}
+
+	/* make sure both directions produced equal trees */
+	if (!dsync_mailbox_trees_equal(tree1, orig_tree1)) {
+		test_assert(FALSE);
+		trees_dump(tree1, orig_tree1);
+	}
+
+	dsync_mailbox_tree_deinit(_tree2);
+	dsync_mailbox_tree_deinit(&orig_tree1);
+	dsync_mailbox_tree_deinit(&orig_tree2);
+}
+
+static void
+test_tree_nodes_add_namespace(struct dsync_mailbox_node *node,
+			      struct mail_namespace *ns)
+{
+	for (; node != NULL; node = node->next) {
+		node->ns = ns;
+		test_tree_nodes_add_namespace(node->first_child, ns);
+	}
+}
+
+static void
+test_tree_add_namespace(struct dsync_mailbox_tree *tree,
+			struct mail_namespace *ns)
+{
+	struct dsync_mailbox_node *node, *n;
+
+	node = dsync_mailbox_tree_get(tree, TEST_NAMESPACE_NAME);
+	node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+	i_assert(tree->root.first_child == node);
+	i_assert(node->first_child == NULL);
+	node->first_child = node->next;
+	for (n = node->first_child; n != NULL; n = n->next)
+		n->parent = node;
+	node->next = NULL;
+
+	test_tree_nodes_add_namespace(&tree->root, ns);
+}
+
+static void test_trees(struct dsync_mailbox_tree *tree1,
+		       struct dsync_mailbox_tree *tree2)
+{
+	struct dsync_mailbox_tree *tree1_dup, *tree2_dup;
+
+	tree1_dup = dsync_mailbox_tree_dup(tree1);
+	tree2_dup = dsync_mailbox_tree_dup(tree2);
+
+	/* test without namespace prefix */
+	test_trees_nofree(tree1, &tree2);
+	dsync_mailbox_tree_deinit(&tree1);
+
+	/* test with namespace prefix */
+	test_tree_add_namespace(tree1_dup, &inbox_namespace);
+	test_tree_add_namespace(tree2_dup, &inbox_namespace);
+	test_trees_nofree(tree1_dup, &tree2_dup);
+	dsync_mailbox_tree_deinit(&tree1_dup);
+}
+
+static void test_dsync_mailbox_tree_sync_creates(void)
+{
+	static const char *common_nodes[] = { "foo", "foo/bar", NULL };
+	static const char *create1_nodes[] = { "bar", "foo/baz", NULL };
+	static const char *create2_nodes[] = { "foo/xyz", "foo/bar/3", NULL };
+	struct dsync_mailbox_tree *tree1, *tree2;
+	unsigned int counter = 0;
+
+	test_begin("dsync mailbox tree sync creates");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	nodes_create(tree1, &counter, common_nodes);
+	tree2 = dsync_mailbox_tree_dup(tree1);
+	nodes_create(tree1, &counter, create1_nodes);
+	nodes_create(tree2, &counter, create2_nodes);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_deletes(void)
+{
+	static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", NULL };
+	static const char *delete1_nodes[] = { "1", "2", NULL };
+	static const char *delete2_nodes[] = { "2/s1", "x/y", NULL };
+	struct dsync_mailbox_tree *tree1, *tree2;
+	unsigned int counter = 0;
+
+	test_begin("dsync mailbox tree sync deletes");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	nodes_create(tree1, &counter, common_nodes);
+	tree2 = dsync_mailbox_tree_dup(tree1);
+	nodes_delete(tree1, &counter, delete1_nodes);
+	nodes_delete(tree2, &counter, delete2_nodes);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames1(void)
+{
+	static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", "3/s3", NULL };
+	struct dsync_mailbox_tree *tree1, *tree2;
+	struct dsync_mailbox_node *node;
+	unsigned int counter = 0;
+
+	test_begin("dsync mailbox tree sync renames 1");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	nodes_create(tree1, &counter, common_nodes);
+	tree2 = dsync_mailbox_tree_dup(tree1);
+
+	node = dsync_mailbox_tree_get(tree1, "1");
+	node->name = "a";
+	node->last_renamed_or_created = 1000;
+	node = dsync_mailbox_tree_get(tree2, "2");
+	node->name = "b";
+	node->last_renamed_or_created = 1000;
+
+	node = dsync_mailbox_tree_get(tree1, "3/s3");
+	node->name = "z";
+	node->last_renamed_or_created = 1000;
+	dsync_mailbox_tree_node_detach(node);
+	dsync_mailbox_tree_node_attach(node, &tree1->root);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames2(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 2");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/1", 1);
+	node_create(tree1, 2, "0/1/2", 3);
+
+	node_create(tree2, 1, "0", 0);
+	node_create(tree2, 2, "0/1/2", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames3(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 3");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/2", 1);
+	node_create(tree1, 2, "0/3", 1);
+
+	node_create(tree2, 1, "0/4/5", 0);
+	node_create(tree2, 2, "1", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames4(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 4");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/b", 0);
+	node_create(tree1, 2, "c", 2);
+
+	node_create(tree2, 2, "0/a", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames5(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 5");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "b", 0);
+	node_create(tree1, 2, "c", 2);
+
+	node_create(tree2, 2, "0/a", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames6(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 6");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/1", 0);
+	node_create(tree1, 2, "0/2", 1);
+
+	node_create(tree2, 1, "0", 1);
+	node_create(tree2, 2, "0/3", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames7(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 7");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/2", 0);
+	node_create(tree2, 1, "1/2", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames8(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 8");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/1", 0);
+	node_create(tree1, 2, "0/2", 1);
+
+	node_create(tree2, 1, "0", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames9(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 9");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/1/2", 0);
+	node_create(tree1, 2, "0/3", 1);
+
+	node_create(tree2, 1, "0", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames10(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 10");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/1", 0);
+	node_create(tree1, 3, "0/2/3", 0);
+
+	node_create(tree2, 1, "0", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames11(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 11");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/1", 2);
+	node_create(tree1, 0, "0/1/2", 0);
+
+	node_create(tree2, 1, "0", 1);
+	node_create(tree2, 0, "0/1/2", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames12(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 12");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/2", 0);
+	node_create(tree1, 2, "1", 0);
+	node_create(tree1, 3, "1/4", 0);
+	node_create(tree1, 4, "1/4/5", 1);
+
+	node_create(tree2, 1, "1", 2);
+	node_create(tree2, 2, "1/4", 3);
+	node_create(tree2, 3, "1/4/6", 4);
+	node_create(tree2, 4, "1/3", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames13(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 13");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 4, "0.0/1.0/2.1", 0);
+	node_create(tree1, 5, "0.1", 2);
+	node_create(tree1, 6, "0.1/1.0", 2);
+	node_create(tree1, 7, "0.1/1.0/2.0", 8);
+
+	node_create(tree2, 5, "0.1/1.0", 5);
+	node_create(tree2, 6, "0.1/1.0/2.0", 8);
+	node_create(tree2, 7, "0.1/1.1", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames14(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 14");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "1", 0);
+	node_create(tree1, 2, "1/2", 0);
+	node_create(tree1, 3, "1/2/4", 1);
+
+	node_create(tree2, 1, "1/2", 3);
+	node_create(tree2, 2, "1/2/5", 4);
+	node_create(tree2, 3, "1/2/4", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames15(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 15");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "1", 0);
+	node_create(tree2, 2, "1", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames16(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 16");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "1/2", 4);
+	node_create(tree1, 2, "1", 2);
+
+	node_create(tree2, 1, "2", 1);
+	node_create(tree2, 2, "1/2", 3);
+	node_create(tree2, 3, "1", 5);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames17(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 17");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "1", 1);
+
+	node_create(tree2, 1, "1/2", 0);
+	node_create(tree2, 2, "1", 2);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames18(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 18");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 2, "a", 5);
+	node_create(tree1, 4, "a/c", 2);
+	node_create(tree1, 5, "b", 6);
+
+	node_create(tree2, 1, "a", 7);
+	node_create(tree2, 2, "b", 3);
+	node_create(tree2, 3, "b/c", 4);
+	node_create(tree2, 4, "d", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames19(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 19");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "0/2/1", 1);
+	node_create(tree1, 2, "0/4", 3);
+	node_create(tree1, 3, "0/2", 2);
+
+	node_create(tree2, 1, "1", 0);
+	node_create(tree2, 2, "1/3", 4);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames20(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 20");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "1", 0);
+	node_create(tree1, 2, "0", 0);
+	node_create(tree1, 3, "0/2", 0);
+	/* rename 0 -> 1/0 */
+	node_create(tree2, 1, "1", 0);
+	node_create(tree2, 2, "1/0", 1);
+	node_create(tree2, 3, "1/0/2", 0);
+
+	test_trees_nofree(tree1, &tree2);
+	test_assert(tree1->root.first_child != NULL &&
+		    tree1->root.first_child->next == NULL);
+	dsync_mailbox_tree_deinit(&tree1);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames21(void)
+{
+#if 0
+	/* FIXME: we can't currently test this without crashing */
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 21");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 1, "INBOX", 0);
+	node_create(tree1, 2, "foo", 0);
+	/* swap INBOX and foo - the INBOX name is important since it's
+	   treated specially */
+	node_create(tree2, 1, "foo", 0);
+	node_create(tree2, 2, "INBOX", 1);
+
+	test_trees(tree1, tree2);
+	test_end();
+#endif
+}
+
+static void test_dsync_mailbox_tree_sync_renames22(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync renames 22");
+	tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+	tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+	node_create(tree1, 3, "p/a", 0);
+	node_create(tree1, 0, "p/2", 0);
+	node_create(tree1, 5, "p/2/h", 0);
+
+	node_create(tree2, 4, "p/1/z", 0);
+	node_create(tree2, 1, "p/2", 0);
+	node_create(tree2, 2, "p/2/a", 0);
+	node_create(tree2, 5, "p/2/y", 0);
+	node_create(tree2, 3, "p/3", 0);
+
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_random(void)
+{
+	struct dsync_mailbox_tree *tree1, *tree2;
+
+	test_begin("dsync mailbox tree sync random");
+	tree1 = create_random_tree();
+	tree2 = create_random_tree();
+	test_trees(tree1, tree2);
+	test_end();
+}
+
+int main(void)
+{
+	static void (*const test_functions[])(void) = {
+		test_dsync_mailbox_tree_sync_creates,
+		test_dsync_mailbox_tree_sync_deletes,
+		test_dsync_mailbox_tree_sync_renames1,
+		test_dsync_mailbox_tree_sync_renames2,
+		test_dsync_mailbox_tree_sync_renames3,
+		test_dsync_mailbox_tree_sync_renames4,
+		test_dsync_mailbox_tree_sync_renames5,
+		test_dsync_mailbox_tree_sync_renames6,
+		test_dsync_mailbox_tree_sync_renames7,
+		test_dsync_mailbox_tree_sync_renames8,
+		test_dsync_mailbox_tree_sync_renames9,
+		test_dsync_mailbox_tree_sync_renames10,
+		test_dsync_mailbox_tree_sync_renames11,
+		test_dsync_mailbox_tree_sync_renames12,
+		test_dsync_mailbox_tree_sync_renames13,
+		test_dsync_mailbox_tree_sync_renames14,
+		test_dsync_mailbox_tree_sync_renames15,
+		test_dsync_mailbox_tree_sync_renames16,
+		test_dsync_mailbox_tree_sync_renames17,
+		test_dsync_mailbox_tree_sync_renames18,
+		test_dsync_mailbox_tree_sync_renames19,
+		test_dsync_mailbox_tree_sync_renames20,
+		test_dsync_mailbox_tree_sync_renames21,
+		test_dsync_mailbox_tree_sync_renames22,
+		test_dsync_mailbox_tree_sync_random,
+		NULL
+	};
+	return test_run(test_functions);
+}