diff options
Diffstat (limited to 'src/doveadm/dsync')
36 files changed, 16782 insertions, 0 deletions
diff --git a/src/doveadm/dsync/Makefile.am b/src/doveadm/dsync/Makefile.am new file mode 100644 index 0000000..20cbfc7 --- /dev/null +++ b/src/doveadm/dsync/Makefile.am @@ -0,0 +1,76 @@ +pkglib_LTLIBRARIES = libdovecot-dsync.la +noinst_LTLIBRARIES = libdsync.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +libdsync_la_SOURCES = \ + dsync-brain.c \ + dsync-brain-mailbox.c \ + dsync-brain-mailbox-tree.c \ + dsync-brain-mailbox-tree-sync.c \ + dsync-brain-mails.c \ + dsync-deserializer.c \ + dsync-mail.c \ + dsync-mailbox.c \ + dsync-mailbox-import.c \ + dsync-mailbox-export.c \ + dsync-mailbox-state.c \ + dsync-mailbox-tree.c \ + dsync-mailbox-tree-fill.c \ + dsync-mailbox-tree-sync.c \ + dsync-serializer.c \ + dsync-ibc.c \ + dsync-ibc-stream.c \ + dsync-ibc-pipe.c \ + dsync-transaction-log-scan.c + +libdovecot_dsync_la_SOURCES = +libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la +libdovecot_dsync_la_DEPENDENCIES = libdsync.la +libdovecot_dsync_la_LDFLAGS = -export-dynamic + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + dsync-brain.h \ + dsync-ibc.h + +noinst_HEADERS = \ + dsync-brain-private.h \ + dsync-mail.h \ + dsync-mailbox.h \ + dsync-mailbox-import.h \ + dsync-mailbox-export.h \ + dsync-mailbox-state.h \ + dsync-mailbox-tree.h \ + dsync-mailbox-tree-private.h \ + dsync-serializer.h \ + dsync-deserializer.h \ + dsync-ibc-private.h \ + dsync-transaction-log-scan.h + +test_programs = \ + test-dsync-mailbox-tree-sync + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../../lib-test/libtest.la \ + ../../lib/liblib.la + +test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c +test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs) +test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/doveadm/dsync/Makefile.in b/src/doveadm/dsync/Makefile.in new file mode 100644 index 0000000..ba39895 --- /dev/null +++ b/src/doveadm/dsync/Makefile.in @@ -0,0 +1,1019 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/doveadm/dsync +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \ + $(pkginc_lib_HEADERS) $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-dsync-mailbox-tree-sync$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkglibdir)" \ + "$(DESTDIR)$(pkginc_libdir)" +LTLIBRARIES = $(noinst_LTLIBRARIES) $(pkglib_LTLIBRARIES) +am_libdovecot_dsync_la_OBJECTS = +libdovecot_dsync_la_OBJECTS = $(am_libdovecot_dsync_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +libdovecot_dsync_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(libdovecot_dsync_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libdsync_la_LIBADD = +am_libdsync_la_OBJECTS = dsync-brain.lo dsync-brain-mailbox.lo \ + dsync-brain-mailbox-tree.lo dsync-brain-mailbox-tree-sync.lo \ + dsync-brain-mails.lo dsync-deserializer.lo dsync-mail.lo \ + dsync-mailbox.lo dsync-mailbox-import.lo \ + dsync-mailbox-export.lo dsync-mailbox-state.lo \ + dsync-mailbox-tree.lo dsync-mailbox-tree-fill.lo \ + dsync-mailbox-tree-sync.lo dsync-serializer.lo dsync-ibc.lo \ + dsync-ibc-stream.lo dsync-ibc-pipe.lo \ + dsync-transaction-log-scan.lo +libdsync_la_OBJECTS = $(am_libdsync_la_OBJECTS) +am_test_dsync_mailbox_tree_sync_OBJECTS = \ + test-dsync-mailbox-tree-sync.$(OBJEXT) +test_dsync_mailbox_tree_sync_OBJECTS = \ + $(am_test_dsync_mailbox_tree_sync_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo \ + ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo \ + ./$(DEPDIR)/dsync-brain-mailbox.Plo \ + ./$(DEPDIR)/dsync-brain-mails.Plo ./$(DEPDIR)/dsync-brain.Plo \ + ./$(DEPDIR)/dsync-deserializer.Plo \ + ./$(DEPDIR)/dsync-ibc-pipe.Plo \ + ./$(DEPDIR)/dsync-ibc-stream.Plo ./$(DEPDIR)/dsync-ibc.Plo \ + ./$(DEPDIR)/dsync-mail.Plo \ + ./$(DEPDIR)/dsync-mailbox-export.Plo \ + ./$(DEPDIR)/dsync-mailbox-import.Plo \ + ./$(DEPDIR)/dsync-mailbox-state.Plo \ + ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo \ + ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo \ + ./$(DEPDIR)/dsync-mailbox-tree.Plo \ + ./$(DEPDIR)/dsync-mailbox.Plo ./$(DEPDIR)/dsync-serializer.Plo \ + ./$(DEPDIR)/dsync-transaction-log-scan.Plo \ + ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \ + $(test_dsync_mailbox_tree_sync_SOURCES) +DIST_SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \ + $(test_dsync_mailbox_tree_sync_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +pkglib_LTLIBRARIES = libdovecot-dsync.la +noinst_LTLIBRARIES = libdsync.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +libdsync_la_SOURCES = \ + dsync-brain.c \ + dsync-brain-mailbox.c \ + dsync-brain-mailbox-tree.c \ + dsync-brain-mailbox-tree-sync.c \ + dsync-brain-mails.c \ + dsync-deserializer.c \ + dsync-mail.c \ + dsync-mailbox.c \ + dsync-mailbox-import.c \ + dsync-mailbox-export.c \ + dsync-mailbox-state.c \ + dsync-mailbox-tree.c \ + dsync-mailbox-tree-fill.c \ + dsync-mailbox-tree-sync.c \ + dsync-serializer.c \ + dsync-ibc.c \ + dsync-ibc-stream.c \ + dsync-ibc-pipe.c \ + dsync-transaction-log-scan.c + +libdovecot_dsync_la_SOURCES = +libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la +libdovecot_dsync_la_DEPENDENCIES = libdsync.la +libdovecot_dsync_la_LDFLAGS = -export-dynamic +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + dsync-brain.h \ + dsync-ibc.h + +noinst_HEADERS = \ + dsync-brain-private.h \ + dsync-mail.h \ + dsync-mailbox.h \ + dsync-mailbox-import.h \ + dsync-mailbox-export.h \ + dsync-mailbox-state.h \ + dsync-mailbox-tree.h \ + dsync-mailbox-tree-private.h \ + dsync-serializer.h \ + dsync-deserializer.h \ + dsync-ibc-private.h \ + dsync-transaction-log-scan.h + +test_programs = \ + test-dsync-mailbox-tree-sync + +test_libs = \ + ../../lib-test/libtest.la \ + ../../lib/liblib.la + +test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c +test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs) +test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES) + @$(NORMAL_INSTALL) + @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkglibdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkglibdir)" || exit 1; \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(pkglibdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(pkglibdir)"; \ + } + +uninstall-pkglibLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(pkglibdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(pkglibdir)/$$f"; \ + done + +clean-pkglibLTLIBRARIES: + -test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES) + @list='$(pkglib_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libdovecot-dsync.la: $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_DEPENDENCIES) $(EXTRA_libdovecot_dsync_la_DEPENDENCIES) + $(AM_V_CCLD)$(libdovecot_dsync_la_LINK) -rpath $(pkglibdir) $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_LIBADD) $(LIBS) + +libdsync.la: $(libdsync_la_OBJECTS) $(libdsync_la_DEPENDENCIES) $(EXTRA_libdsync_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libdsync_la_OBJECTS) $(libdsync_la_LIBADD) $(LIBS) + +test-dsync-mailbox-tree-sync$(EXEEXT): $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_DEPENDENCIES) $(EXTRA_test_dsync_mailbox_tree_sync_DEPENDENCIES) + @rm -f test-dsync-mailbox-tree-sync$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mails.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-deserializer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-pipe.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-stream.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-export.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-import.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-state.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-fill.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-serializer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-transaction-log-scan.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS clean-pkglibLTLIBRARIES mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mails.Plo + -rm -f ./$(DEPDIR)/dsync-brain.Plo + -rm -f ./$(DEPDIR)/dsync-deserializer.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo + -rm -f ./$(DEPDIR)/dsync-ibc.Plo + -rm -f ./$(DEPDIR)/dsync-mail.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-serializer.Plo + -rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo + -rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-pkglibLTLIBRARIES + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mails.Plo + -rm -f ./$(DEPDIR)/dsync-brain.Plo + -rm -f ./$(DEPDIR)/dsync-deserializer.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo + -rm -f ./$(DEPDIR)/dsync-ibc.Plo + -rm -f ./$(DEPDIR)/dsync-mail.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-serializer.Plo + -rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo + -rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS \ + clean-pkglibLTLIBRARIES cscopelist-am ctags ctags-am distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-pkginc_libHEADERS \ + install-pkglibLTLIBRARIES install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c new file mode 100644 index 0000000..2030e04 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c @@ -0,0 +1,229 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "dsync-mailbox-tree.h" +#include "dsync-brain-private.h" + +static int +sync_create_box(struct dsync_brain *brain, struct mailbox *box, + const guid_128_t mailbox_guid, uint32_t uid_validity, + enum mail_error *error_r) +{ + struct mailbox_metadata metadata; + struct mailbox_update update; + enum mail_error error; + const char *errstr; + int ret; + + i_zero(&update); + memcpy(update.mailbox_guid, mailbox_guid, sizeof(update.mailbox_guid)); + update.uid_validity = uid_validity; + + if (mailbox_create(box, &update, FALSE) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error != MAIL_ERROR_EXISTS) { + i_error("Can't create mailbox %s: %s", + mailbox_get_vname(box), errstr); + *error_r = error; + return -1; + } + } + if (brain->no_mail_sync) { + /* trust that create worked, we can't actually open it + and verify. */ + return 0; + } + /* sync the mailbox so we can look up its latest status */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + + /* verify that the GUID is what we wanted. if it's not, it probably + means that the mailbox had already been created. then we'll use the + GUID that is higher. + + mismatching UIDVALIDITY is handled later, because we choose it by + checking which mailbox has more messages */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { + i_error("Can't get mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + + ret = memcmp(mailbox_guid, metadata.guid, sizeof(metadata.guid)); + + /* if THEIR guid is bigger than OUR guid, and we are not doing + backup in either direction, OR GUID did not match and we are + receiving backup, try change the mailbox GUID. + */ + + if ((ret > 0 && !brain->backup_recv && + !brain->backup_send) || (ret != 0 && brain->backup_recv)) { + if (brain->debug) { + i_debug("brain %c: Changing mailbox %s GUID %s -> %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), + guid_128_to_string(metadata.guid), + guid_128_to_string(mailbox_guid)); + } + i_zero(&update); + memcpy(update.mailbox_guid, mailbox_guid, + sizeof(update.mailbox_guid)); + if (mailbox_update(box, &update) < 0) { + i_error("Can't update mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + /* verify that the update worked */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, + &metadata) < 0) { + i_error("Can't get mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + if (memcmp(mailbox_guid, metadata.guid, + sizeof(metadata.guid)) != 0) { + i_error("Backend didn't update mailbox %s GUID", + mailbox_get_vname(box)); + *error_r = MAIL_ERROR_TEMP; + return -1; + } + } else if (ret < 0) { + if (brain->debug) { + i_debug("brain %c: Other brain should change mailbox " + "%s GUID %s -> %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), + guid_128_to_string(mailbox_guid), + guid_128_to_string(metadata.guid)); + } + } + return 0; +} + +int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, + const struct dsync_mailbox_tree_sync_change *change, + enum mail_error *error_r) +{ + struct mailbox *box = NULL, *destbox; + const char *errstr, *func_name = NULL, *storage_name; + enum mail_error error; + int ret = -1; + + if (brain->backup_send) { + i_assert(brain->no_backup_overwrite); + return 0; + } + + switch (change->type) { + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: + /* make sure we're deleting the correct mailbox */ + ret = dsync_brain_mailbox_alloc(brain, change->mailbox_guid, + &box, &errstr, error_r); + if (ret < 0) { + i_error("Mailbox sync: Couldn't allocate mailbox %s GUID %s: %s", + change->full_name, + guid_128_to_string(change->mailbox_guid), errstr); + return -1; + } + if (ret == 0) { + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox %s GUID %s deletion conflict: %s", + change->full_name, + guid_128_to_string(change->mailbox_guid), errstr)); + return 0; + } + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR: + storage_name = mailbox_list_get_storage_name(change->ns->list, + change->full_name); + if (mailbox_list_delete_dir(change->ns->list, storage_name) == 0) + return 0; + + errstr = mailbox_list_get_last_internal_error(change->ns->list, &error); + if (error == MAIL_ERROR_NOTFOUND || + error == MAIL_ERROR_EXISTS) { + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox %s mailbox_list_delete_dir conflict: %s", + change->full_name, errstr)); + return 0; + } else { + i_error("Mailbox sync: mailbox_list_delete_dir failed: %s", + errstr); + *error_r = error; + return -1; + } + default: + box = mailbox_alloc(change->ns->list, change->full_name, 0); + break; + } + mailbox_skip_create_name_restrictions(box, TRUE); + switch (change->type) { + case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX: + ret = sync_create_box(brain, box, change->mailbox_guid, + change->uid_validity, error_r); + mailbox_free(&box); + return ret; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR: + ret = mailbox_create(box, NULL, TRUE); + if (ret < 0 && + mailbox_get_last_mail_error(box) == MAIL_ERROR_EXISTS) { + /* it doesn't matter if somebody else created this + directory or we automatically did while creating its + child mailbox. it's there now anyway and we don't + gain anything by treating this failure any + differently from success. */ + ret = 0; + } + func_name = "mailbox_create"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: + ret = mailbox_delete(box); + func_name = "mailbox_delete"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR: + i_unreached(); + case DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME: + destbox = mailbox_alloc(change->ns->list, + change->rename_dest_name, 0); + mailbox_skip_create_name_restrictions(destbox, TRUE); + ret = mailbox_rename(box, destbox); + func_name = "mailbox_rename"; + mailbox_free(&destbox); + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE: + ret = mailbox_set_subscribed(box, TRUE); + func_name = "mailbox_set_subscribed"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE: + ret = mailbox_set_subscribed(box, FALSE); + func_name = "mailbox_set_subscribed"; + break; + } + if (ret < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error == MAIL_ERROR_EXISTS || + error == MAIL_ERROR_NOTFOUND) { + /* mailbox was already created or was already deleted. + let the next sync figure out what to do */ + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox %s %s conflict: %s", + mailbox_get_vname(box), func_name, errstr)); + ret = 0; + } else { + i_error("Mailbox %s sync: %s failed: %s", + mailbox_get_vname(box), func_name, errstr); + *error_r = error; + } + } + mailbox_free(&box); + return ret; +} diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree.c b/src/doveadm/dsync/dsync-brain-mailbox-tree.c new file mode 100644 index 0000000..44b76bf --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c @@ -0,0 +1,624 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "mail-namespace.h" +#include "mailbox-list-private.h" +#include "dsync-ibc.h" +#include "dsync-mailbox-tree.h" +#include "dsync-brain-private.h" + +#include <ctype.h> + +static void dsync_brain_check_namespaces(struct dsync_brain *brain) +{ + struct mail_namespace *ns, *first_ns = NULL; + char sep, escape_char; + + i_assert(brain->hierarchy_sep == '\0'); + i_assert(brain->escape_char == '\0'); + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + sep = mail_namespace_get_sep(ns); + escape_char = mailbox_list_get_settings(ns->list)->vname_escape_char; + if (first_ns == NULL) { + brain->hierarchy_sep = sep; + brain->escape_char = escape_char; + first_ns = ns; + } else if (brain->hierarchy_sep != sep) { + i_fatal("Synced namespaces have conflicting separators " + "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")", + brain->hierarchy_sep, first_ns->prefix, + sep, ns->prefix); + } else if (brain->escape_char != escape_char) { + i_fatal("Synced namespaces have conflicting escape chars " + "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")", + brain->escape_char, first_ns->prefix, + escape_char, ns->prefix); + } + } + if (brain->hierarchy_sep != '\0') + return; + + i_fatal("All your namespaces have a location setting. " + "Only namespaces with empty location settings are converted. " + "(One namespace should default to mail_location setting)"); +} + +void dsync_brain_mailbox_trees_init(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + + dsync_brain_check_namespaces(brain); + + brain->local_mailbox_tree = + dsync_mailbox_tree_init(brain->hierarchy_sep, + brain->escape_char, brain->alt_char); + /* we'll convert remote mailbox names to use our own separator */ + brain->remote_mailbox_tree = + dsync_mailbox_tree_init(brain->hierarchy_sep, + brain->escape_char, brain->alt_char); + + /* fill the local mailbox tree */ + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + if (brain->debug) + i_debug("brain %c: Namespace %s has location %s", + brain->master_brain ? 'M' : 'S', + ns->prefix, ns->set->location); + if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, ns, + brain->sync_box, + brain->sync_box_guid, + brain->exclude_mailboxes, + &brain->mail_error) < 0) { + brain->failed = TRUE; + break; + } + } + + brain->local_tree_iter = + dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); +} + +static const char *const * +dsync_brain_mailbox_to_parts(struct dsync_brain *brain, const char *name) +{ + char sep[] = { brain->hierarchy_sep, '\0' }; + char **parts = p_strsplit(unsafe_data_stack_pool, name, sep); + for (unsigned int i = 0; parts[i] != NULL; i++) { + mailbox_list_name_unescape((const char **)&parts[i], + brain->escape_char); + } + return (const char *const *)parts; +} + +void dsync_brain_send_mailbox_tree(struct dsync_brain *brain) +{ + struct dsync_mailbox_node *node; + enum dsync_ibc_send_ret ret; + const char *full_name; + + while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, + &full_name, &node)) { + if (node->ns == NULL) { + /* This node was created when adding a namespace prefix + to the tree that has multiple hierarchical names, + but the parent names don't belong to any synced + namespace. For example when syncing "-n Shared/user/" + so "Shared/" is skipped. Or if there is e.g. + "Public/files/" namespace prefix, but no "Public/" + namespace at all. */ + continue; + } + + T_BEGIN { + const char *const *parts; + + if (brain->debug) { + i_debug("brain %c: Local mailbox tree: %s %s", + brain->master_brain ? 'M' : 'S', full_name, + dsync_mailbox_node_to_string(node)); + } + + parts = dsync_brain_mailbox_to_parts(brain, full_name); + ret = dsync_ibc_send_mailbox_tree_node(brain->ibc, + parts, node); + } T_END; + if (ret == DSYNC_IBC_SEND_RET_FULL) + return; + } + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_TREE); + + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE_DELETES; +} + +void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain) +{ + const struct dsync_mailbox_delete *deletes; + unsigned int count; + + deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree, + &count); + dsync_ibc_send_mailbox_deletes(brain->ibc, deletes, count, + brain->hierarchy_sep, + brain->escape_char); + + brain->state = DSYNC_STATE_RECV_MAILBOX_TREE; +} + +static bool +dsync_namespace_match_parts(struct mail_namespace *ns, + const char *const *name_parts) +{ + const char *part, *prefix = ns->prefix; + size_t part_len; + char ns_sep = mail_namespace_get_sep(ns); + + if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && + strcmp(name_parts[0], "INBOX") == 0 && name_parts[1] == NULL) + return TRUE; + + for (; *name_parts != NULL && *prefix != '\0'; name_parts++) { + part = *name_parts; + part_len = strlen(part); + + if (!str_begins(prefix, part)) + return FALSE; + if (prefix[part_len] != ns_sep) + return FALSE; + prefix += part_len + 1; + } + if (*name_parts != NULL) { + /* namespace prefix found with a mailbox */ + return TRUE; + } + if (*prefix == '\0') { + /* namespace prefix itself matched */ + return TRUE; + } + return FALSE; +} + +static struct mail_namespace * +dsync_find_namespace(struct dsync_brain *brain, const char *const *name_parts) +{ + struct mail_namespace *ns, *best_ns = NULL; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + if (ns->prefix_len == 0) { + /* prefix="" is the fallback namespace */ + if (best_ns == NULL) + best_ns = ns; + } else if (dsync_namespace_match_parts(ns, name_parts)) { + if (best_ns == NULL || + best_ns->prefix_len < ns->prefix_len) + best_ns = ns; + } + } + return best_ns; +} + +static bool +dsync_is_valid_name(struct mail_namespace *ns, const char *vname) +{ + struct mailbox *box; + bool ret; + + box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); + ret = mailbox_verify_create_name(box) == 0; + mailbox_free(&box); + return ret; +} + +static bool +dsync_is_valid_name_until(struct mail_namespace *ns, string_t *vname_full, + unsigned int end_pos) +{ + const char *vname; + if (end_pos == str_len(vname_full)) + vname = str_c(vname_full); + else + vname = t_strndup(str_c(vname_full), end_pos); + return dsync_is_valid_name(ns, vname); +} + +static bool +dsync_fix_mailbox_name_until(struct mail_namespace *ns, string_t *vname_full, + char alt_char, unsigned int start_pos, + unsigned int *_end_pos) +{ + unsigned int end_pos = *_end_pos; + unsigned int i; + + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + + /* 1) change any real separators to alt separators (this + wouldn't be necessary with listescape, but don't bother + detecting it) */ + char list_sep = mailbox_list_get_hierarchy_sep(ns->list); + char ns_sep = mail_namespace_get_sep(ns); + if (list_sep != ns_sep) { + char *v = str_c_modifiable(vname_full); + for (i = start_pos; i < end_pos; i++) { + if (v[i] == list_sep) + v[i] = alt_char; + } + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + } + + /* 2) '/' characters aren't valid without listescape */ + if (ns_sep != '/' && list_sep != '/') { + char *v = str_c_modifiable(vname_full); + for (i = start_pos; i < end_pos; i++) { + if (v[i] == '/') + v[i] = alt_char; + } + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + } + + /* 3) probably some reserved name (e.g. dbox-Mails or ..) */ + str_insert(vname_full, start_pos, "_"); end_pos++; *_end_pos += 1; + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + + return FALSE; +} + +static void +dsync_fix_mailbox_name(struct mail_namespace *ns, string_t *vname_str, + char alt_char) +{ + const char *old_vname; + char *vname; + char ns_sep = mail_namespace_get_sep(ns); + guid_128_t guid; + unsigned int i, start_pos; + + vname = str_c_modifiable(vname_str); + if (strncmp(vname, ns->prefix, ns->prefix_len) == 0) + start_pos = ns->prefix_len; + else + start_pos = 0; + + /* replace control chars */ + for (i = start_pos; vname[i] != '\0'; i++) { + if ((unsigned char)vname[i] < ' ') + vname[i] = alt_char; + } + /* make it valid UTF8 */ + if (!uni_utf8_str_is_valid(vname)) { + old_vname = t_strdup(vname + start_pos); + str_truncate(vname_str, start_pos); + if (uni_utf8_get_valid_data((const void *)old_vname, + strlen(old_vname), vname_str)) + i_unreached(); + vname = str_c_modifiable(vname_str); + } + if (dsync_is_valid_name(ns, vname)) + return; + + /* Check/fix each hierarchical name separately */ + const char *p; + do { + i_assert(start_pos <= str_len(vname_str)); + p = strchr(str_c(vname_str) + start_pos, ns_sep); + unsigned int end_pos; + if (p == NULL) + end_pos = str_len(vname_str); + else + end_pos = p - str_c(vname_str); + + if (!dsync_fix_mailbox_name_until(ns, vname_str, alt_char, + start_pos, &end_pos)) { + /* Couldn't fix it. Name is too long? Just give up and + generate a unique name. */ + guid_128_generate(guid); + str_truncate(vname_str, 0); + str_append(vname_str, ns->prefix); + str_append(vname_str, guid_128_to_string(guid)); + i_assert(dsync_is_valid_name(ns, str_c(vname_str))); + break; + } + start_pos = end_pos + 1; + } while (p != NULL); +} + +static int +dsync_get_mailbox_name(struct dsync_brain *brain, const char *const *name_parts, + const char **name_r, struct mail_namespace **ns_r) +{ + struct mail_namespace *ns; + const char *p; + string_t *vname; + char ns_sep; + + i_assert(*name_parts != NULL); + + ns = dsync_find_namespace(brain, name_parts); + if (ns == NULL) + return -1; + ns_sep = mail_namespace_get_sep(ns); + + /* build the mailbox name */ + char escape_chars[] = { + brain->escape_char, + ns_sep, + '\0' + }; + struct dsync_mailbox_list *dlist = DSYNC_LIST_CONTEXT(ns->list); + if (dlist != NULL && !dlist->have_orig_escape_char) { + /* The escape character was added only for dsync internally. + Normally there is no escape character configured. Change + the mailbox names so that it doesn't rely on it. */ + escape_chars[0] = '\0'; + } + vname = t_str_new(128); + for (; *name_parts != NULL; name_parts++) { + if (escape_chars[0] != '\0') { + mailbox_list_name_escape(*name_parts, escape_chars, + vname); + } else { + for (p = *name_parts; *p != '\0'; p++) { + if (*p != ns_sep) + str_append_c(vname, *p); + else + str_append_c(vname, brain->alt_char); + } + } + str_append_c(vname, ns_sep); + } + str_truncate(vname, str_len(vname)-1); + + dsync_fix_mailbox_name(ns, vname, brain->alt_char); + *name_r = str_c(vname); + *ns_r = ns; + return 0; +} + +static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain) +{ + struct dsync_mailbox_tree_sync_ctx *ctx; + const struct dsync_mailbox_tree_sync_change *change; + enum dsync_mailbox_trees_sync_type sync_type; + enum dsync_mailbox_trees_sync_flags sync_flags = + (brain->debug ? DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG : 0) | + (brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0); + int ret; + + if (brain->no_backup_overwrite) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; + else if (brain->backup_send) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL; + else if (brain->backup_recv) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE; + else + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; + + ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree, + brain->remote_mailbox_tree, + sync_type, sync_flags); + while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { + T_BEGIN { + ret = dsync_brain_mailbox_tree_sync_change( + brain, change, &brain->mail_error); + } T_END; + if (ret < 0) { + brain->failed = TRUE; + break; + } + } + if (dsync_mailbox_trees_sync_deinit(&ctx) < 0) + brain->failed = TRUE; +} + +static int +dsync_brain_recv_mailbox_tree_add(struct dsync_brain *brain, + const char *const *parts, + const struct dsync_mailbox_node *remote_node, + const char *sep) +{ + struct dsync_mailbox_node *node; + struct mail_namespace *ns; + const char *name; + + if (dsync_get_mailbox_name(brain, parts, &name, &ns) < 0) + return -1; + if (brain->debug) { + i_debug("brain %c: Remote mailbox tree: %s %s", + brain->master_brain ? 'M' : 'S', + t_strarray_join(parts, sep), + dsync_mailbox_node_to_string(remote_node)); + } + node = dsync_mailbox_tree_get(brain->remote_mailbox_tree, name); + node->ns = ns; + dsync_mailbox_node_copy_data(node, remote_node); + return 0; +} + +bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain) +{ + const struct dsync_mailbox_node *remote_node; + struct dsync_mailbox_node *dup_node1, *dup_node2; + const char *const *parts; + enum dsync_ibc_recv_ret ret; + int ret2; + char sep[2]; + bool changed = FALSE; + + sep[0] = brain->hierarchy_sep; sep[1] = '\0'; + while ((ret = dsync_ibc_recv_mailbox_tree_node(brain->ibc, &parts, + &remote_node)) > 0) { + T_BEGIN { + ret2 = dsync_brain_recv_mailbox_tree_add( + brain, parts, remote_node, sep); + } T_END; + if (ret2 < 0) { + i_error("Couldn't find namespace for mailbox %s", + t_strarray_join(parts, sep)); + brain->failed = TRUE; + return TRUE; + } + } + if (ret != DSYNC_IBC_RECV_RET_FINISHED) + return changed; + + if (dsync_mailbox_tree_build_guid_hash(brain->remote_mailbox_tree, + &dup_node1, &dup_node2) < 0) { + i_error("Remote sent duplicate mailbox GUID %s for mailboxes %s and %s", + guid_128_to_string(dup_node1->mailbox_guid), + dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree, + dup_node1), + dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree, + dup_node2)); + brain->failed = TRUE; + } + + brain->state = DSYNC_STATE_RECV_MAILBOX_TREE_DELETES; + return TRUE; +} + +static void +dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + const struct dsync_mailbox_delete *other_del, + const struct dsync_mailbox_node **node_r, + const char **status_r) +{ + const struct dsync_mailbox_node *node; + struct dsync_mailbox_node *other_node, *old_node; + const char *name; + + /* see if we can find the deletion based on mailbox tree that should + still have the mailbox */ + node = *node_r = dsync_mailbox_tree_find_delete(tree, other_del); + if (node == NULL) { + *status_r = "not found"; + return; + } + + switch (other_del->type) { + case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX: + /* mailbox is always deleted */ + break; + case DSYNC_MAILBOX_DELETE_TYPE_DIR: + if (other_del->timestamp <= node->last_renamed_or_created) { + /* we don't want to delete this directory, we already + have a newer timestamp for it */ + *status_r = "keep directory, we have a newer timestamp"; + return; + } + break; + case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE: + if (other_del->timestamp <= node->last_subscription_change) { + /* we don't want to unsubscribe, since we already have + a newer subscription timestamp */ + *status_r = "keep subscription, we have a newer timestamp"; + return; + } + break; + } + + /* make a node for it in the other mailbox tree */ + name = dsync_mailbox_node_get_full_name(tree, node); + other_node = dsync_mailbox_tree_get(other_tree, name); + + if (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + (!guid_128_is_empty(other_node->mailbox_guid) || + other_del->type != DSYNC_MAILBOX_DELETE_TYPE_MAILBOX)) { + /* other side has already created a new mailbox or + directory with this name, we can't delete it */ + *status_r = "name has already been recreated"; + return; + } + + /* ok, mark the other node deleted */ + if (other_del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) { + memcpy(other_node->mailbox_guid, node->mailbox_guid, + sizeof(other_node->mailbox_guid)); + } + if (other_node->ns != node->ns && other_node->ns != NULL) { + /* namespace mismatch for this node. this shouldn't happen + normally, but especially during some misconfigurations it's + possible that one side has created mailboxes that conflict + with another namespace's prefix. since we're here because + one of the mailboxes was deleted, we'll just ignore this. */ + *status_r = "namespace mismatch"; + return; + } + other_node->ns = node->ns; + if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) { + other_node->existence = DSYNC_MAILBOX_NODE_DELETED; + *status_r = "marked as deleted"; + } else { + other_node->last_subscription_change = other_del->timestamp; + other_node->subscribed = FALSE; + *status_r = "marked as unsubscribed"; + } + + if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node, + &old_node) < 0) + i_unreached(); +} + +bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain) +{ + const struct dsync_mailbox_node *node; + const char *status; + const struct dsync_mailbox_delete *deletes; + unsigned int i, count; + char sep, escape_char; + + if (dsync_ibc_recv_mailbox_deletes(brain->ibc, &deletes, &count, + &sep, &escape_char) == 0) + return FALSE; + + /* apply remote's mailbox deletions based on our local tree */ + dsync_mailbox_tree_set_remote_chars(brain->local_mailbox_tree, sep, + escape_char); + for (i = 0; i < count; i++) { + dsync_brain_mailbox_tree_add_delete(brain->local_mailbox_tree, + brain->remote_mailbox_tree, + &deletes[i], &node, &status); + if (brain->debug) { + const char *node_name = node == NULL ? "" : + dsync_mailbox_node_get_full_name(brain->local_mailbox_tree, node); + i_debug("brain %c: Remote mailbox tree deletion: guid=%s type=%s timestamp=%ld name=%s local update=%s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(deletes[i].guid), + dsync_mailbox_delete_type_to_string(deletes[i].type), + deletes[i].timestamp, node_name, status); + } + } + + /* apply local mailbox deletions based on remote tree */ + deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree, + &count); + dsync_mailbox_tree_set_remote_chars(brain->remote_mailbox_tree, + brain->hierarchy_sep, + brain->escape_char); + for (i = 0; i < count; i++) { + dsync_brain_mailbox_tree_add_delete(brain->remote_mailbox_tree, + brain->local_mailbox_tree, + &deletes[i], &node, &status); + } + + dsync_brain_mailbox_trees_sync(brain); + brain->state = brain->master_brain ? + DSYNC_STATE_MASTER_SEND_MAILBOX : + DSYNC_STATE_SLAVE_RECV_MAILBOX; + i_assert(brain->local_tree_iter == NULL); + brain->local_tree_iter = + dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); + return TRUE; +} diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c new file mode 100644 index 0000000..759018f --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -0,0 +1,923 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "mail-cache-private.h" +#include "mail-namespace.h" +#include "mail-storage-private.h" +#include "dsync-ibc.h" +#include "dsync-mailbox-tree.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-brain-private.h" + +static int +ns_mailbox_try_alloc(struct dsync_brain *brain, struct mail_namespace *ns, + const guid_128_t guid, struct mailbox **box_r, + const char **errstr_r, enum mail_error *error_r) +{ + enum mailbox_flags flags = 0; + struct mailbox *box; + enum mailbox_existence existence; + int ret; + + if (brain->backup_send) { + /* make sure mailbox isn't modified */ + flags |= MAILBOX_FLAG_READONLY; + } + + box = mailbox_alloc_guid(ns->list, guid, flags); + ret = mailbox_exists(box, FALSE, &existence); + if (ret < 0) { + *errstr_r = mailbox_get_last_internal_error(box, error_r); + mailbox_free(&box); + return -1; + } + if (existence != MAILBOX_EXISTENCE_SELECT) { + mailbox_free(&box); + *errstr_r = existence == MAILBOX_EXISTENCE_NONE ? + "Mailbox was already deleted" : + "Mailbox is no longer selectable"; + return 0; + } + *box_r = box; + return 1; +} + +int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, + struct mailbox **box_r, const char **errstr_r, + enum mail_error *error_r) +{ + struct mail_namespace *ns; + int ret; + + *box_r = NULL; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + if ((ret = ns_mailbox_try_alloc(brain, ns, guid, box_r, + errstr_r, error_r)) != 0) + return ret; + } + return 0; +} + +static void +dsync_mailbox_cache_field_dup(ARRAY_TYPE(mailbox_cache_field) *dest, + const ARRAY_TYPE(mailbox_cache_field) *src, + pool_t pool) +{ + const struct mailbox_cache_field *src_field; + struct mailbox_cache_field *dest_field; + + p_array_init(dest, pool, array_count(src)); + array_foreach(src, src_field) { + dest_field = array_append_space(dest); + dest_field->name = p_strdup(pool, src_field->name); + dest_field->decision = src_field->decision; + dest_field->last_used = src_field->last_used; + } +} + + +static const struct dsync_mailbox_state * +dsync_mailbox_state_find(struct dsync_brain *brain, + const guid_128_t mailbox_guid) +{ + const uint8_t *guid_p; + + guid_p = mailbox_guid; + return hash_table_lookup(brain->mailbox_states, guid_p); +} + +static void +dsync_mailbox_state_remove(struct dsync_brain *brain, + const guid_128_t mailbox_guid) +{ + const uint8_t *guid_p; + + guid_p = mailbox_guid; + if (hash_table_lookup(brain->mailbox_states, guid_p) != NULL) + hash_table_remove(brain->mailbox_states, guid_p); +} + +void dsync_brain_sync_init_box_states(struct dsync_brain *brain) +{ + if (brain->backup_send) { + /* we have an exporter, but no importer. */ + brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES; + brain->box_recv_state = brain->mail_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : + DSYNC_BOX_STATE_RECV_LAST_COMMON; + } else if (brain->backup_recv) { + /* we have an importer, but no exporter */ + brain->box_send_state = brain->mail_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : + DSYNC_BOX_STATE_DONE; + brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES; + } else { + brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES; + brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES; + } +} + +static void +dsync_brain_sync_mailbox_init(struct dsync_brain *brain, + struct mailbox *box, + struct file_lock *lock, + const struct dsync_mailbox *local_dsync_box, + bool wait_for_remote_box) +{ + const struct dsync_mailbox_state *state; + + i_assert(brain->box_importer == NULL); + i_assert(brain->box_exporter == NULL); + i_assert(box->synced); + + brain->box = box; + brain->box_lock = lock; + brain->pre_box_state = brain->state; + if (wait_for_remote_box) { + brain->box_send_state = DSYNC_BOX_STATE_MAILBOX; + brain->box_recv_state = DSYNC_BOX_STATE_MAILBOX; + } else { + dsync_brain_sync_init_box_states(brain); + } + brain->local_dsync_box = *local_dsync_box; + if (brain->dsync_box_pool != NULL) + p_clear(brain->dsync_box_pool); + else { + brain->dsync_box_pool = + pool_alloconly_create(MEMPOOL_GROWING"dsync brain box pool", 2048); + } + dsync_mailbox_cache_field_dup(&brain->local_dsync_box.cache_fields, + &local_dsync_box->cache_fields, + brain->dsync_box_pool); + i_zero(&brain->remote_dsync_box); + + state = dsync_mailbox_state_find(brain, local_dsync_box->mailbox_guid); + if (state != NULL) + brain->mailbox_state = *state; + else { + i_zero(&brain->mailbox_state); + memcpy(brain->mailbox_state.mailbox_guid, + local_dsync_box->mailbox_guid, + sizeof(brain->mailbox_state.mailbox_guid)); + brain->mailbox_state.last_uidvalidity = + local_dsync_box->uid_validity; + } +} + +static void +dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box) +{ + enum dsync_mailbox_import_flags import_flags = 0; + const struct dsync_mailbox_state *state; + uint32_t last_common_uid; + uint64_t last_common_modseq, last_common_pvt_modseq; + + i_assert(brain->box_importer == NULL); + i_assert(brain->log_scan != NULL); + + i_assert(memcmp(brain->local_dsync_box.mailbox_guid, + remote_dsync_box->mailbox_guid, + sizeof(remote_dsync_box->mailbox_guid)) == 0); + + brain->remote_dsync_box = *remote_dsync_box; + dsync_mailbox_cache_field_dup(&brain->remote_dsync_box.cache_fields, + &remote_dsync_box->cache_fields, + brain->dsync_box_pool); + + state = dsync_mailbox_state_find(brain, remote_dsync_box->mailbox_guid); + if (state != NULL) { + last_common_uid = state->last_common_uid; + last_common_modseq = state->last_common_modseq; + last_common_pvt_modseq = state->last_common_pvt_modseq; + } else { + last_common_uid = 0; + last_common_modseq = 0; + last_common_pvt_modseq = 0; + } + + if (brain->mail_requests) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS; + if (brain->master_brain) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN; + if (brain->backup_recv && !brain->no_backup_overwrite) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES; + if (brain->debug) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_DEBUG; + if (brain->local_dsync_box.have_save_guids && + (remote_dsync_box->have_save_guids || + (brain->backup_recv && remote_dsync_box->have_guids))) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; + if (brain->local_dsync_box.have_only_guid128 || + remote_dsync_box->have_only_guid128) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128; + if (brain->no_notify) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY; + if (brain->empty_hdr_workaround) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND; + + brain->box_importer = brain->backup_send ? NULL : + dsync_mailbox_import_init(brain->box, brain->virtual_all_box, + brain->log_scan, + last_common_uid, last_common_modseq, + last_common_pvt_modseq, + remote_dsync_box->uid_next, + remote_dsync_box->first_recent_uid, + remote_dsync_box->highest_modseq, + remote_dsync_box->highest_pvt_modseq, + brain->sync_since_timestamp, + brain->sync_until_timestamp, + brain->sync_max_size, + brain->sync_flag, + brain->import_commit_msgs_interval, + import_flags, brain->hdr_hash_version, + brain->hashed_headers); +} + +int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box) +{ + struct mailbox_status status; + enum dsync_mailbox_exporter_flags exporter_flags = 0; + uint32_t last_common_uid, highest_wanted_uid; + uint64_t last_common_modseq, last_common_pvt_modseq; + const char *desync_reason = ""; + bool pvt_too_old; + int ret; + + i_assert(brain->log_scan == NULL); + i_assert(brain->box_exporter == NULL); + + last_common_uid = brain->mailbox_state.last_common_uid; + last_common_modseq = brain->mailbox_state.last_common_modseq; + last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq; + highest_wanted_uid = last_common_uid == 0 ? + (uint32_t)-1 : last_common_uid; + ret = dsync_transaction_log_scan_init(brain->box->view, + brain->box->view_pvt, + highest_wanted_uid, + last_common_modseq, + last_common_pvt_modseq, + &brain->log_scan, &pvt_too_old); + if (ret < 0) { + i_error("Failed to read transaction log for mailbox %s", + mailbox_get_vname(brain->box)); + brain->failed = TRUE; + return -1; + } + + mailbox_get_open_status(brain->box, STATUS_UIDNEXT | + STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ, &status); + if (status.nonpermanent_modseqs) + status.highest_modseq = 0; + if (ret == 0) { + if (pvt_too_old) { + desync_reason = t_strdup_printf( + "Private modseq %"PRIu64" no longer in transaction log " + "(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)", + last_common_pvt_modseq, + status.highest_pvt_modseq, last_common_uid, + status.uidnext); + } else { + desync_reason = t_strdup_printf( + "Modseq %"PRIu64" no longer in transaction log " + "(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)", + last_common_modseq, + status.highest_modseq, last_common_uid, + status.uidnext); + } + } + + if (last_common_uid != 0) { + /* if last_common_* is higher than our current ones it means + that the incremental sync state is stale and we need to do + a full resync */ + if (status.uidnext < last_common_uid) { + desync_reason = t_strdup_printf("uidnext %u < %u", + status.uidnext, last_common_uid); + ret = 0; + } else if (status.highest_modseq < last_common_modseq) { + desync_reason = t_strdup_printf("highest_modseq %"PRIu64" < %"PRIu64, + status.highest_modseq, last_common_modseq); + ret = 0; + } else if (status.highest_pvt_modseq < last_common_pvt_modseq) { + desync_reason = t_strdup_printf("highest_pvt_modseq %"PRIu64" < %"PRIu64, + status.highest_pvt_modseq, last_common_pvt_modseq); + ret = 0; + } + } + if (ret == 0) { + i_warning("Failed to do incremental sync for mailbox %s, " + "retry with a full sync (%s)", + mailbox_get_vname(brain->box), desync_reason); + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Incremental sync failed: %s", desync_reason)); + brain->require_full_resync = TRUE; + return 0; + } + + if (!brain->mail_requests) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS; + if (remote_dsync_box->have_save_guids && + (brain->local_dsync_box.have_save_guids || + (brain->backup_send && brain->local_dsync_box.have_guids))) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; + if (brain->no_mail_prefetch) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL; + if (brain->sync_since_timestamp > 0 || + brain->sync_until_timestamp > 0) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS; + if (brain->sync_max_size > 0) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES; + if (remote_dsync_box->messages_count == 0) { + /* remote mailbox is empty - we don't really need to export + header hashes since they're not going to match anything + anyway. */ + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES; + } + + brain->box_exporter = brain->backup_recv ? NULL : + dsync_mailbox_export_init(brain->box, brain->log_scan, + last_common_uid, + exporter_flags, + brain->hdr_hash_version, + brain->hashed_headers); + dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box); + return 1; +} + +void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) +{ + enum mail_error error; + + i_assert(brain->box != NULL); + + array_push_back(&brain->remote_mailbox_states, &brain->mailbox_state); + if (brain->box_exporter != NULL) { + const char *errstr; + + i_assert(brain->failed || brain->require_full_resync || + brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED); + if (dsync_mailbox_export_deinit(&brain->box_exporter, + &errstr, &error) < 0) + i_error("Mailbox export failed: %s", errstr); + } + if (brain->box_importer != NULL) { + uint32_t last_common_uid, last_messages_count; + uint64_t last_common_modseq, last_common_pvt_modseq; + const char *changes_during_sync; + bool require_full_resync; + + i_assert(brain->failed); + (void)dsync_mailbox_import_deinit(&brain->box_importer, + FALSE, + &last_common_uid, + &last_common_modseq, + &last_common_pvt_modseq, + &last_messages_count, + &changes_during_sync, + &require_full_resync, + &brain->mail_error); + if (require_full_resync) + brain->require_full_resync = TRUE; + } + if (brain->log_scan != NULL) + dsync_transaction_log_scan_deinit(&brain->log_scan); + file_lock_free(&brain->box_lock); + mailbox_free(&brain->box); + + brain->state = brain->pre_box_state; +} + +static int dsync_box_get(struct mailbox *box, struct dsync_mailbox *dsync_box_r, + enum mail_error *error_r) +{ + const enum mailbox_status_items status_items = + STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | + STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ; + const enum mailbox_metadata_items metadata_items = + MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID; + struct mailbox_status status; + struct mailbox_metadata metadata; + const char *errstr; + enum mail_error error; + + /* get metadata first, since it may autocreate the mailbox */ + if (mailbox_get_metadata(box, metadata_items, &metadata) < 0 || + mailbox_get_status(box, status_items, &status) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error == MAIL_ERROR_NOTFOUND || + error == MAIL_ERROR_NOTPOSSIBLE) { + /* Mailbox isn't selectable, try the next one. We + should have already caught \Noselect mailboxes, but + check them anyway here. The NOTPOSSIBLE check is + mainly for invalid mbox files. */ + return 0; + } + i_error("Failed to access mailbox %s: %s", + mailbox_get_vname(box), errstr); + *error_r = error; + return -1; + } + if (status.nonpermanent_modseqs) + status.highest_modseq = 0; + + i_assert(status.uidvalidity != 0 || status.messages == 0); + + i_zero(dsync_box_r); + memcpy(dsync_box_r->mailbox_guid, metadata.guid, + sizeof(dsync_box_r->mailbox_guid)); + dsync_box_r->uid_validity = status.uidvalidity; + dsync_box_r->uid_next = status.uidnext; + dsync_box_r->messages_count = status.messages; + dsync_box_r->first_recent_uid = status.first_recent_uid; + dsync_box_r->highest_modseq = status.highest_modseq; + dsync_box_r->highest_pvt_modseq = status.highest_pvt_modseq; + dsync_mailbox_cache_field_dup(&dsync_box_r->cache_fields, + metadata.cache_fields, + pool_datastack_create()); + dsync_box_r->have_guids = status.have_guids; + dsync_box_r->have_save_guids = status.have_save_guids; + dsync_box_r->have_only_guid128 = status.have_only_guid128; + return 1; +} + +static bool +dsync_brain_has_mailbox_state_changed(struct dsync_brain *brain, + const struct dsync_mailbox *dsync_box) +{ + const struct dsync_mailbox_state *state; + + if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + return TRUE; + + state = dsync_mailbox_state_find(brain, dsync_box->mailbox_guid); + return state == NULL || + state->last_uidvalidity != dsync_box->uid_validity || + state->last_common_uid+1 != dsync_box->uid_next || + state->last_common_modseq != dsync_box->highest_modseq || + state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq || + state->last_messages_count != dsync_box->messages_count; +} + +static int +dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, + struct file_lock **lock_r, + struct dsync_mailbox *dsync_box_r) +{ + enum mailbox_flags flags = 0; + struct dsync_mailbox dsync_box; + struct mailbox *box; + struct file_lock *lock = NULL; + struct dsync_mailbox_node *node; + const char *vname = NULL; + enum mail_error error; + bool synced = FALSE; + int ret; + + *box_r = NULL; + + while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !guid_128_is_empty(node->mailbox_guid)) + break; + vname = NULL; + } + if (vname == NULL) { + /* no more mailboxes */ + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + return -1; + } + + if (brain->backup_send) { + /* make sure mailbox isn't modified */ + flags |= MAILBOX_FLAG_READONLY; + } + box = mailbox_alloc(node->ns->list, vname, flags); + for (;;) { + if ((ret = dsync_box_get(box, &dsync_box, &error)) <= 0) { + if (ret < 0) { + brain->mail_error = error; + brain->failed = TRUE; + } + mailbox_free(&box); + file_lock_free(&lock); + return ret; + } + + /* if mailbox's last_common_* state equals the current state, + we can skip the mailbox */ + if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) { + if (brain->debug) { + i_debug("brain %c: Skipping mailbox %s with unchanged state " + "uidvalidity=%u uidnext=%u highestmodseq=%"PRIu64" highestpvtmodseq=%"PRIu64" messages=%u", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box.mailbox_guid), + dsync_box.uid_validity, + dsync_box.uid_next, + dsync_box.highest_modseq, + dsync_box.highest_pvt_modseq, + dsync_box.messages_count); + } + mailbox_free(&box); + file_lock_free(&lock); + return 0; + } + if (synced) { + /* ok, the mailbox really changed */ + break; + } + + /* mailbox appears to have changed. do a full sync here and get the + state again. Lock before syncing. */ + if (dsync_mailbox_lock(brain, box, &lock) < 0) { + brain->failed = TRUE; + mailbox_free(&box); + return -1; + } + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + brain->failed = TRUE; + mailbox_free(&box); + file_lock_free(&lock); + return -1; + } + synced = TRUE; + } + + *box_r = box; + *lock_r = lock; + *dsync_box_r = dsync_box; + return 1; +} + +static bool +dsync_brain_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, + struct file_lock **lock_r, + struct dsync_mailbox *dsync_box_r) +{ + int ret; + + if (brain->no_mail_sync) + return FALSE; + + while ((ret = dsync_brain_try_next_mailbox(brain, box_r, lock_r, dsync_box_r)) == 0) + ; + return ret > 0; +} + +void dsync_brain_master_send_mailbox(struct dsync_brain *brain) +{ + struct dsync_mailbox dsync_box; + struct mailbox *box; + struct file_lock *lock; + + i_assert(brain->master_brain); + i_assert(brain->box == NULL); + + if (!dsync_brain_next_mailbox(brain, &box, &lock, &dsync_box)) { + brain->state = DSYNC_STATE_FINISH; + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX); + return; + } + + /* start exporting this mailbox (wait for remote to start importing) */ + dsync_ibc_send_mailbox(brain->ibc, &dsync_box); + dsync_brain_sync_mailbox_init(brain, box, lock, &dsync_box, TRUE); + brain->state = DSYNC_STATE_SYNC_MAILS; +} + +bool dsync_boxes_need_sync(struct dsync_brain *brain, + const struct dsync_mailbox *box1, + const struct dsync_mailbox *box2, + const char **reason_r) +{ + if (brain->no_mail_sync) + return FALSE; + if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_CHANGED) { + *reason_r = "Full sync"; + return TRUE; + } + if (box1->uid_validity != box2->uid_validity) + *reason_r = t_strdup_printf("UIDVALIDITY changed: %u -> %u", + box1->uid_validity, box2->uid_validity); + else if (box1->uid_next != box2->uid_next) + *reason_r = t_strdup_printf("UIDNEXT changed: %u -> %u", + box1->uid_next, box2->uid_next); + else if (box1->messages_count != box2->messages_count) + *reason_r = t_strdup_printf("Message count changed: %u -> %u", + box1->messages_count, box2->messages_count); + else if (box1->highest_modseq != box2->highest_modseq) { + *reason_r = t_strdup_printf("HIGHESTMODSEQ changed %" + PRIu64" -> %"PRIu64, + box1->highest_modseq, + box2->highest_modseq); + if (box1->highest_modseq == 0 || + box2->highest_modseq == 0) { + *reason_r = t_strdup_printf( + "%s (Permanent MODSEQs aren't supported)", + *reason_r); + } + } else if (box1->highest_pvt_modseq != box2->highest_pvt_modseq) + *reason_r = t_strdup_printf("Private HIGHESTMODSEQ changed %" + PRIu64" -> %"PRIu64, + box1->highest_pvt_modseq, + box2->highest_pvt_modseq); + else if (box1->first_recent_uid != box2->first_recent_uid) + *reason_r = t_strdup_printf("First RECENT UID changed: %u -> %u", + box1->first_recent_uid, box2->first_recent_uid); + else + return FALSE; + return TRUE; +} + +static int +mailbox_cache_field_name_cmp(const struct mailbox_cache_field *f1, + const struct mailbox_cache_field *f2) +{ + return strcmp(f1->name, f2->name); +} + +static void +dsync_cache_fields_update(const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + struct mailbox *box, + struct mailbox_update *update) +{ + ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes; + const struct mailbox_cache_field *local_fields, *remote_fields; + unsigned int li, ri, local_count, remote_count; + time_t drop_older_timestamp; + int ret; + + if (array_count(&remote_box->cache_fields) == 0) { + /* remote has no cached fields. there's nothing to update. */ + return; + } + + t_array_init(&local_sorted, array_count(&local_box->cache_fields)); + t_array_init(&remote_sorted, array_count(&remote_box->cache_fields)); + array_append_array(&local_sorted, &local_box->cache_fields); + array_append_array(&remote_sorted, &remote_box->cache_fields); + array_sort(&local_sorted, mailbox_cache_field_name_cmp); + array_sort(&remote_sorted, mailbox_cache_field_name_cmp); + + if (array_count(&local_sorted) == 0) { + /* local has no cached fields. set them to same as remote. */ + array_append_zero(&remote_sorted); + update->cache_updates = array_front(&remote_sorted); + return; + } + + /* figure out what to change */ + local_fields = array_get(&local_sorted, &local_count); + remote_fields = array_get(&remote_sorted, &remote_count); + t_array_init(&changes, local_count + remote_count); + drop_older_timestamp = ioloop_time - + box->index->optimization_set.cache.unaccessed_field_drop_secs; + + for (li = ri = 0; li < local_count || ri < remote_count; ) { + ret = li == local_count ? 1 : + ri == remote_count ? -1 : + strcmp(local_fields[li].name, remote_fields[ri].name); + if (ret == 0) { + /* field exists in both local and remote */ + const struct mailbox_cache_field *lf = &local_fields[li]; + const struct mailbox_cache_field *rf = &remote_fields[ri]; + + if (lf->last_used > rf->last_used || + (lf->last_used == rf->last_used && + lf->decision > rf->decision)) { + /* use local decision and timestamp */ + } else { + array_push_back(&changes, rf); + } + li++; ri++; + } else if (ret < 0) { + /* remote field doesn't exist */ + li++; + } else { + /* local field doesn't exist */ + if (remote_fields[ri].last_used < drop_older_timestamp) { + /* field hasn't be used for a long time, remote + will probably drop this soon as well */ + } else { + array_push_back(&changes, &remote_fields[ri]); + } + ri++; + } + } + i_assert(li == local_count && ri == remote_count); + if (array_count(&changes) > 0) { + array_append_zero(&changes); + update->cache_updates = array_front(&changes); + } +} + +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + const char **reason_r) +{ + struct mailbox_update update; + const struct dsync_mailbox_state *state; + bool ret = TRUE; + + *reason_r = NULL; + i_zero(&update); + + if (local_box->uid_validity != remote_box->uid_validity) { + /* Keep the UIDVALIDITY for the mailbox that has more + messages. If they equal, use the higher UIDVALIDITY. */ + if (remote_box->messages_count > local_box->messages_count || + (remote_box->messages_count == local_box->messages_count && + remote_box->uid_validity > local_box->uid_validity)) + update.uid_validity = remote_box->uid_validity; + + state = dsync_mailbox_state_find(brain, local_box->mailbox_guid); + if (state != NULL && state->last_common_uid > 0) { + /* we can't continue syncing this mailbox in this + session, because the other side already started + sending mailbox changes, but not for all mails. */ + dsync_mailbox_state_remove(brain, local_box->mailbox_guid); + *reason_r = "UIDVALIDITY changed during a stateful sync, need to restart"; + brain->failed = TRUE; + ret = FALSE; + } + } + + dsync_cache_fields_update(local_box, remote_box, box, &update); + + if (update.uid_validity == 0 && + update.cache_updates == NULL) { + /* no changes */ + return ret; + } + + if (mailbox_update(box, &update) < 0) { + i_error("Couldn't update mailbox %s metadata: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + brain->failed = TRUE; + } + return ret; +} + +static void +dsync_brain_slave_send_mailbox_lost(struct dsync_brain *brain, + const struct dsync_mailbox *dsync_box, + bool ignore) +{ + struct dsync_mailbox delete_box; + + if (brain->debug) { + i_debug("brain %c: We don't have mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + i_zero(&delete_box); + memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid, + sizeof(delete_box.mailbox_guid)); + t_array_init(&delete_box.cache_fields, 0); + if (ignore) + delete_box.mailbox_ignore = TRUE; + else + delete_box.mailbox_lost = TRUE; + dsync_ibc_send_mailbox(brain->ibc, &delete_box); +} + +bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain) +{ + const struct dsync_mailbox *dsync_box; + struct dsync_mailbox local_dsync_box; + struct mailbox *box; + struct file_lock *lock; + const char *errstr, *resync_reason, *reason; + enum mail_error error; + int ret; + bool resync; + + i_assert(!brain->master_brain); + i_assert(brain->box == NULL); + + if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0) + return FALSE; + if (ret < 0) { + brain->state = DSYNC_STATE_FINISH; + return TRUE; + } + + if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid, + &box, &errstr, &error) < 0) { + i_error("Couldn't allocate mailbox GUID %s: %s", + guid_128_to_string(dsync_box->mailbox_guid), errstr); + brain->mail_error = error; + brain->failed = TRUE; + return TRUE; + } + if (box == NULL) { + /* mailbox was probably deleted/renamed during sync */ + if (brain->backup_send && brain->no_backup_overwrite) { + if (brain->debug) { + i_debug("brain %c: Ignore nonexistent " + "mailbox GUID %s with -1 sync", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, TRUE); + return TRUE; + } + //FIXME: verify this from log, and if not log an error. + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox GUID %s was lost", + guid_128_to_string(dsync_box->mailbox_guid))); + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE); + return TRUE; + } + /* Lock before syncing */ + if (dsync_mailbox_lock(brain, box, &lock) < 0) { + mailbox_free(&box); + brain->failed = TRUE; + return TRUE; + } + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + file_lock_free(&lock); + mailbox_free(&box); + brain->failed = TRUE; + return TRUE; + } + + if ((ret = dsync_box_get(box, &local_dsync_box, &error)) <= 0) { + file_lock_free(&lock); + mailbox_free(&box); + if (ret < 0) { + brain->mail_error = error; + brain->failed = TRUE; + return TRUE; + } + /* another process just deleted this mailbox? */ + if (brain->debug) { + i_debug("brain %c: Skipping lost mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE); + return TRUE; + } + i_assert(local_dsync_box.uid_validity != 0); + i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid, + sizeof(dsync_box->mailbox_guid)) == 0); + + resync = !dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, + dsync_box, &resync_reason); + + if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box, &reason)) { + /* no fields appear to have changed, skip this mailbox */ + if (brain->debug) { + i_debug("brain %c: Skipping unchanged mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box); + file_lock_free(&lock); + mailbox_free(&box); + return TRUE; + } + if (brain->debug) { + i_debug("brain %c: Syncing mailbox %s: %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid), reason); + } + + /* start export/import */ + dsync_brain_sync_mailbox_init(brain, box, lock, &local_dsync_box, FALSE); + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) + return TRUE; + if (resync) + dsync_brain_set_changes_during_sync(brain, resync_reason); + if (ret == 0 || resync) { + brain->require_full_resync = TRUE; + dsync_brain_sync_mailbox_deinit(brain); + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE); + return TRUE; + } + + dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box); + brain->state = DSYNC_STATE_SYNC_MAILS; + return TRUE; +} diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c new file mode 100644 index 0000000..3feea20 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mails.c @@ -0,0 +1,438 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "dsync-ibc.h" +#include "dsync-mail.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" +#include "dsync-brain-private.h" + +const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1] = { + "mailbox", + "changes", + "attributes", + "mail_requests", + "mails", + "recv_last_common", + "done" +}; + +static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain) +{ + const struct dsync_mailbox *dsync_box; + const char *resync_reason, *reason; + enum dsync_ibc_recv_ret ret; + bool resync; + + i_assert(brain->master_brain); + + if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + i_error("Remote sent end-of-list instead of a mailbox"); + brain->failed = TRUE; + return TRUE; + } + if (memcmp(dsync_box->mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(dsync_box->mailbox_guid)) != 0) { + i_error("Remote sent mailbox with a wrong GUID"); + brain->failed = TRUE; + return TRUE; + } + + if (dsync_box->mailbox_ignore) { + /* ignore this box */ + if (brain->debug) + i_debug("brain %c: Ignoring missing remote box GUID %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + if (dsync_box->mailbox_lost) { + /* remote lost the mailbox. it's probably already deleted, but + verify it on next sync just to be sure */ + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Remote lost mailbox GUID %s (maybe it was just deleted?)", + guid_128_to_string(dsync_box->mailbox_guid))); + brain->require_full_resync = TRUE; + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + resync = !dsync_brain_mailbox_update_pre(brain, brain->box, + &brain->local_dsync_box, + dsync_box, &resync_reason); + + if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box, + &reason)) { + /* no fields appear to have changed, skip this mailbox */ + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + if (brain->debug) { + i_debug("brain %c: Syncing mailbox %s: %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid), reason); + } + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) + return TRUE; + if (resync) + dsync_brain_set_changes_during_sync(brain, resync_reason); + if (ret == 0 || resync) { + brain->require_full_resync = TRUE; + brain->failed = TRUE; + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + dsync_brain_sync_init_box_states(brain); + return TRUE; +} + +static bool dsync_brain_recv_mailbox_attribute(struct dsync_brain *brain) +{ + const struct dsync_mailbox_attribute *attr; + struct istream *input; + enum dsync_ibc_recv_ret ret; + + if ((ret = dsync_ibc_recv_mailbox_attribute(brain->ibc, &attr)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->box_recv_state = DSYNC_BOX_STATE_CHANGES; + return TRUE; + } + if (dsync_mailbox_import_attribute(brain->box_importer, attr) < 0) + brain->failed = TRUE; + input = attr->value_stream; + i_stream_unref(&input); + return TRUE; +} + +static void dsync_brain_send_end_of_list(struct dsync_brain *brain, + enum dsync_ibc_eol_type type) +{ + i_assert(!brain->failed); + dsync_ibc_send_end_of_list(brain->ibc, type); +} + +static int dsync_brain_export_deinit(struct dsync_brain *brain) +{ + const char *errstr; + enum mail_error error; + + if (dsync_mailbox_export_deinit(&brain->box_exporter, + &errstr, &error) < 0) { + i_error("Exporting mailbox %s failed: %s", + mailbox_get_vname(brain->box), errstr); + brain->mail_error = error; + brain->failed = TRUE; + return -1; + } + return 0; +} + +static void dsync_brain_send_mailbox_attribute(struct dsync_brain *brain) +{ + const struct dsync_mailbox_attribute *attr; + int ret; + + while ((ret = dsync_mailbox_export_next_attr(brain->box_exporter, &attr)) > 0) { + if (dsync_ibc_send_mailbox_attribute(brain->ibc, attr) == 0) + return; + } + if (ret < 0) { + if (dsync_brain_export_deinit(brain) == 0) + i_unreached(); + return; + } + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE); + brain->box_send_state = DSYNC_BOX_STATE_CHANGES; +} + +static bool dsync_brain_recv_mail_change(struct dsync_brain *brain) +{ + const struct dsync_mail_change *change; + enum dsync_ibc_recv_ret ret; + + if ((ret = dsync_ibc_recv_change(brain->ibc, &change)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + if (dsync_mailbox_import_changes_finish(brain->box_importer) < 0) + brain->failed = TRUE; + if (brain->mail_requests && brain->box_exporter != NULL) + brain->box_recv_state = DSYNC_BOX_STATE_MAIL_REQUESTS; + else + brain->box_recv_state = DSYNC_BOX_STATE_MAILS; + return TRUE; + } + if (dsync_mailbox_import_change(brain->box_importer, change) < 0) + brain->failed = TRUE; + return TRUE; +} + +static void dsync_brain_send_mail_change(struct dsync_brain *brain) +{ + const struct dsync_mail_change *change; + int ret; + + while ((ret = dsync_mailbox_export_next(brain->box_exporter, &change)) > 0) { + if (dsync_ibc_send_change(brain->ibc, change) == 0) + return; + } + if (ret < 0) { + if (dsync_brain_export_deinit(brain) == 0) + i_unreached(); + return; + } + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_CHANGES); + if (brain->mail_requests && brain->box_importer != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAIL_REQUESTS; + else + brain->box_send_state = DSYNC_BOX_STATE_MAILS; +} + +static bool dsync_brain_recv_mail_request(struct dsync_brain *brain) +{ + const struct dsync_mail_request *request; + enum dsync_ibc_recv_ret ret; + + i_assert(brain->mail_requests); + i_assert(brain->box_exporter != NULL); + + if ((ret = dsync_ibc_recv_mail_request(brain->ibc, &request)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->box_recv_state = brain->box_importer != NULL ? + DSYNC_BOX_STATE_MAILS : + DSYNC_BOX_STATE_RECV_LAST_COMMON; + return TRUE; + } + dsync_mailbox_export_want_mail(brain->box_exporter, request); + return TRUE; +} + +static bool dsync_brain_send_mail_request(struct dsync_brain *brain) +{ + const struct dsync_mail_request *request; + + i_assert(brain->mail_requests); + + while ((request = dsync_mailbox_import_next_request(brain->box_importer)) != NULL) { + if (dsync_ibc_send_mail_request(brain->ibc, request) == 0) + return TRUE; + } + if (brain->box_recv_state < DSYNC_BOX_STATE_MAIL_REQUESTS) + return FALSE; + + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_REQUESTS); + if (brain->box_exporter != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAILS; + else { + i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE); + brain->box_send_state = DSYNC_BOX_STATE_DONE; + } + return TRUE; +} + +static void dsync_brain_sync_half_finished(struct dsync_brain *brain) +{ + struct dsync_mailbox_state state; + const char *changes_during_sync; + bool require_full_resync; + + if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON || + brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON) + return; + + /* finished with this mailbox */ + i_zero(&state); + memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(state.mailbox_guid)); + state.last_uidvalidity = brain->local_dsync_box.uid_validity; + if (brain->box_importer == NULL) { + /* this mailbox didn't exist on remote */ + state.last_common_uid = brain->local_dsync_box.uid_next-1; + state.last_common_modseq = + brain->local_dsync_box.highest_modseq; + state.last_common_pvt_modseq = + brain->local_dsync_box.highest_pvt_modseq; + state.last_messages_count = + brain->local_dsync_box.messages_count; + } else { + if (dsync_mailbox_import_deinit(&brain->box_importer, + !brain->failed, + &state.last_common_uid, + &state.last_common_modseq, + &state.last_common_pvt_modseq, + &state.last_messages_count, + &changes_during_sync, + &require_full_resync, + &brain->mail_error) < 0) { + if (require_full_resync) { + /* don't treat this as brain failure or the + state won't be sent to the other brain. + this also means we'll continue syncing the + following mailboxes. */ + brain->require_full_resync = TRUE; + } else { + brain->failed = TRUE; + } + } + if (changes_during_sync != NULL) { + state.changes_during_sync = TRUE; + dsync_brain_set_changes_during_sync(brain, changes_during_sync); + } + } + if (brain->require_full_resync) { + state.last_uidvalidity = 0; + state.changes_during_sync = TRUE; + } + brain->mailbox_state = state; + dsync_ibc_send_mailbox_state(brain->ibc, &state); +} + +static bool dsync_brain_recv_mail(struct dsync_brain *brain) +{ + struct dsync_mail *mail; + enum dsync_ibc_recv_ret ret; + + if ((ret = dsync_ibc_recv_mail(brain->ibc, &mail)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON; + if (brain->box_exporter != NULL && + brain->box_send_state >= DSYNC_BOX_STATE_RECV_LAST_COMMON) { + if (dsync_brain_export_deinit(brain) < 0) + return TRUE; + } + dsync_brain_sync_half_finished(brain); + return TRUE; + } + if (brain->debug) { + i_debug("brain %c: import mail uid %u guid %s", + brain->master_brain ? 'M' : 'S', mail->uid, mail->guid); + } + if (dsync_mailbox_import_mail(brain->box_importer, mail) < 0) + brain->failed = TRUE; + i_stream_unref(&mail->input); + return TRUE; +} + +static bool dsync_brain_send_mail(struct dsync_brain *brain) +{ + const struct dsync_mail *mail; + + if (brain->mail_requests && + brain->box_recv_state < DSYNC_BOX_STATE_MAILS) { + /* wait for mail requests to finish. we could already start + exporting, but then we're going to do quite a lot of + separate searches. especially with pipe backend we'd do + a separate search for each mail. */ + return FALSE; + } + + while (dsync_mailbox_export_next_mail(brain->box_exporter, &mail) > 0) { + if (dsync_ibc_send_mail(brain->ibc, mail) == 0) + return TRUE; + } + + if (dsync_brain_export_deinit(brain) < 0) + return TRUE; + + brain->box_send_state = DSYNC_BOX_STATE_DONE; + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS); + + dsync_brain_sync_half_finished(brain); + return TRUE; +} + +static bool dsync_brain_recv_last_common(struct dsync_brain *brain) +{ + enum dsync_ibc_recv_ret ret; + struct dsync_mailbox_state state; + + if ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + i_error("Remote sent end-of-list instead of a mailbox state"); + brain->failed = TRUE; + return TRUE; + } + i_assert(brain->box_send_state == DSYNC_BOX_STATE_DONE); + i_assert(memcmp(state.mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(state.mailbox_guid)) == 0); + + /* normally the last_common_* values should be the same in local and + remote, but during unexpected changes they may differ. use the + values that are lower as the final state. */ + if (brain->mailbox_state.last_common_uid > state.last_common_uid) + brain->mailbox_state.last_common_uid = state.last_common_uid; + if (brain->mailbox_state.last_common_modseq > state.last_common_modseq) + brain->mailbox_state.last_common_modseq = state.last_common_modseq; + if (brain->mailbox_state.last_common_pvt_modseq > state.last_common_pvt_modseq) + brain->mailbox_state.last_common_pvt_modseq = state.last_common_pvt_modseq; + if (state.changes_during_sync) + brain->changes_during_remote_sync = TRUE; + + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; +} + +bool dsync_brain_sync_mails(struct dsync_brain *brain) +{ + bool changed = FALSE; + + i_assert(brain->box != NULL); + + switch (brain->box_recv_state) { + case DSYNC_BOX_STATE_MAILBOX: + changed = dsync_brain_master_sync_recv_mailbox(brain); + break; + case DSYNC_BOX_STATE_ATTRIBUTES: + changed = dsync_brain_recv_mailbox_attribute(brain); + break; + case DSYNC_BOX_STATE_CHANGES: + changed = dsync_brain_recv_mail_change(brain); + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + changed = dsync_brain_recv_mail_request(brain); + break; + case DSYNC_BOX_STATE_MAILS: + changed = dsync_brain_recv_mail(brain); + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + changed = dsync_brain_recv_last_common(brain); + break; + case DSYNC_BOX_STATE_DONE: + break; + } + + if (!dsync_ibc_is_send_queue_full(brain->ibc) && !brain->failed) { + switch (brain->box_send_state) { + case DSYNC_BOX_STATE_MAILBOX: + /* wait for mailbox to be received first */ + break; + case DSYNC_BOX_STATE_ATTRIBUTES: + dsync_brain_send_mailbox_attribute(brain); + changed = TRUE; + break; + case DSYNC_BOX_STATE_CHANGES: + dsync_brain_send_mail_change(brain); + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + if (dsync_brain_send_mail_request(brain)) + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAILS: + if (dsync_brain_send_mail(brain)) + changed = TRUE; + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + i_unreached(); + case DSYNC_BOX_STATE_DONE: + break; + } + } + return changed; +} diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h new file mode 100644 index 0000000..9849703 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -0,0 +1,163 @@ +#ifndef DSYNC_BRAIN_PRIVATE_H +#define DSYNC_BRAIN_PRIVATE_H + +#include "hash.h" +#include "dsync-brain.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" + +#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" +#define DSYNC_MAILBOX_LOCK_FILENAME ".dovecot-box-sync.lock" +#define DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS 30 + +struct dsync_mailbox_tree_sync_change; + +enum dsync_state { + DSYNC_STATE_MASTER_RECV_HANDSHAKE, + DSYNC_STATE_SLAVE_RECV_HANDSHAKE, + /* if sync_type=STATE, the master brain knows the saved "last common + mailbox state". this state is sent to the slave. */ + DSYNC_STATE_MASTER_SEND_LAST_COMMON, + DSYNC_STATE_SLAVE_RECV_LAST_COMMON, + + /* both sides send their mailbox trees */ + DSYNC_STATE_SEND_MAILBOX_TREE, + DSYNC_STATE_SEND_MAILBOX_TREE_DELETES, + DSYNC_STATE_RECV_MAILBOX_TREE, + DSYNC_STATE_RECV_MAILBOX_TREE_DELETES, + + /* master decides in which order mailboxes are synced (it knows the + slave's mailboxes by looking at the received mailbox tree) */ + DSYNC_STATE_MASTER_SEND_MAILBOX, + DSYNC_STATE_SLAVE_RECV_MAILBOX, + /* once mailbox is selected, the mails inside it are synced. + after the mails are synced, another mailbox is synced. */ + DSYNC_STATE_SYNC_MAILS, + + DSYNC_STATE_FINISH, + DSYNC_STATE_DONE +}; + +enum dsync_box_state { + DSYNC_BOX_STATE_MAILBOX, + DSYNC_BOX_STATE_CHANGES, + DSYNC_BOX_STATE_ATTRIBUTES, + DSYNC_BOX_STATE_MAIL_REQUESTS, + DSYNC_BOX_STATE_MAILS, + DSYNC_BOX_STATE_RECV_LAST_COMMON, + DSYNC_BOX_STATE_DONE +}; + +struct dsync_brain { + pool_t pool; + struct mail_user *user; + struct dsync_ibc *ibc; + const char *process_title_prefix; + ARRAY(struct mail_namespace *) sync_namespaces; + const char *sync_box; + struct mailbox *virtual_all_box; + guid_128_t sync_box_guid; + const char *const *exclude_mailboxes; + enum dsync_brain_sync_type sync_type; + time_t sync_since_timestamp; + time_t sync_until_timestamp; + uoff_t sync_max_size; + const char *sync_flag; + char alt_char; + unsigned int import_commit_msgs_interval; + unsigned int hdr_hash_version; + + unsigned int lock_timeout; + int lock_fd; + const char *lock_path; + struct file_lock *lock; + + char hierarchy_sep, escape_char; + struct dsync_mailbox_tree *local_mailbox_tree; + struct dsync_mailbox_tree *remote_mailbox_tree; + struct dsync_mailbox_tree_iter *local_tree_iter; + + enum dsync_state state, pre_box_state; + enum dsync_box_state box_recv_state; + enum dsync_box_state box_send_state; + unsigned int proctitle_update_counter; + + struct dsync_transaction_log_scan *log_scan; + struct dsync_mailbox_importer *box_importer; + struct dsync_mailbox_exporter *box_exporter; + + struct mailbox *box; + struct file_lock *box_lock; + unsigned int mailbox_lock_timeout_secs; + struct dsync_mailbox local_dsync_box, remote_dsync_box; + pool_t dsync_box_pool; + /* list of mailbox states + for master brain: given to brain at init and + for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */ + HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states; + /* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */ + struct hash_iterate_context *mailbox_states_iter; + /* state of the mailbox we're currently syncing, changed at + init and deinit */ + struct dsync_mailbox_state mailbox_state; + /* new states for synced mailboxes */ + ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states; + + const char *changes_during_sync; + enum mail_error mail_error; + + const char *const *hashed_headers; + + bool master_brain:1; + bool mail_requests:1; + bool backup_send:1; + bool backup_recv:1; + bool purge:1; + bool debug:1; + bool sync_visible_namespaces:1; + bool no_mail_sync:1; + bool no_backup_overwrite:1; + bool no_mail_prefetch:1; + bool changes_during_remote_sync:1; + bool require_full_resync:1; + bool verbose_proctitle:1; + bool no_notify:1; + bool failed:1; + bool empty_hdr_workaround:1; +}; + +extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1]; + +void dsync_brain_mailbox_trees_init(struct dsync_brain *brain); +void dsync_brain_send_mailbox_tree(struct dsync_brain *brain); +void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain); +bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain); +bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain); +int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, + const struct dsync_mailbox_tree_sync_change *change, + enum mail_error *error_r); + +void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain); +int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, + struct mailbox **box_r, const char **errstr_r, + enum mail_error *error_r); +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + const char **reason_r); +bool dsync_boxes_need_sync(struct dsync_brain *brain, + const struct dsync_mailbox *box1, + const struct dsync_mailbox *box2, + const char **reason_r); +void dsync_brain_sync_init_box_states(struct dsync_brain *brain); +void dsync_brain_set_changes_during_sync(struct dsync_brain *brain, + const char *reason); + +void dsync_brain_master_send_mailbox(struct dsync_brain *brain); +bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain); +int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box); +bool dsync_brain_sync_mails(struct dsync_brain *brain); + +#endif diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c new file mode 100644 index 0000000..8ff7247 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain.c @@ -0,0 +1,901 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "hostpid.h" +#include "str.h" +#include "file-create-locked.h" +#include "process-title.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "mail-namespace.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" + +#include <sys/stat.h> + +enum dsync_brain_title { + DSYNC_BRAIN_TITLE_NONE = 0, + DSYNC_BRAIN_TITLE_LOCKING, +}; + +static const char *dsync_state_names[] = { + "master_recv_handshake", + "slave_recv_handshake", + "master_send_last_common", + "slave_recv_last_common", + "send_mailbox_tree", + "send_mailbox_tree_deletes", + "recv_mailbox_tree", + "recv_mailbox_tree_deletes", + "master_send_mailbox", + "slave_recv_mailbox", + "sync_mails", + "finish", + "done" +}; + +struct dsync_mailbox_list_module dsync_mailbox_list_module = + MODULE_CONTEXT_INIT(&mailbox_list_module_register); + +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain); + +static const char * +dsync_brain_get_proctitle_full(struct dsync_brain *brain, + enum dsync_brain_title title) +{ + string_t *str = t_str_new(128); + const char *import_title, *export_title; + + str_append_c(str, '['); + if (brain->process_title_prefix != NULL) + str_append(str, brain->process_title_prefix); + str_append(str, brain->user->username); + if (brain->box == NULL) { + str_append_c(str, ' '); + str_append(str, dsync_state_names[brain->state]); + } else { + str_append_c(str, ' '); + str_append(str, mailbox_get_vname(brain->box)); + import_title = brain->box_importer == NULL ? "" : + dsync_mailbox_import_get_proctitle(brain->box_importer); + export_title = brain->box_exporter == NULL ? "" : + dsync_mailbox_export_get_proctitle(brain->box_exporter); + if (import_title[0] == '\0' && export_title[0] == '\0') { + str_printfa(str, " send:%s recv:%s", + dsync_box_state_names[brain->box_send_state], + dsync_box_state_names[brain->box_recv_state]); + } else { + if (import_title[0] != '\0') { + str_append(str, " import:"); + str_append(str, import_title); + } + if (export_title[0] != '\0') { + str_append(str, " export:"); + str_append(str, export_title); + } + } + } + switch (title) { + case DSYNC_BRAIN_TITLE_NONE: + break; + case DSYNC_BRAIN_TITLE_LOCKING: + str_append(str, " locking "DSYNC_LOCK_FILENAME); + break; + } + str_append_c(str, ']'); + return str_c(str); +} + +static const char *dsync_brain_get_proctitle(struct dsync_brain *brain) +{ + return dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_NONE); +} + +static void dsync_brain_run_io(void *context) +{ + struct dsync_brain *brain = context; + bool changed, try_pending; + + if (dsync_ibc_has_failed(brain->ibc)) { + io_loop_stop(current_ioloop); + brain->failed = TRUE; + return; + } + + try_pending = TRUE; + do { + if (!dsync_brain_run(brain, &changed)) { + io_loop_stop(current_ioloop); + break; + } + if (changed) + try_pending = TRUE; + else if (try_pending) { + if (dsync_ibc_has_pending_data(brain->ibc)) + changed = TRUE; + try_pending = FALSE; + } + } while (changed); +} + +static struct dsync_brain * +dsync_brain_common_init(struct mail_user *user, struct dsync_ibc *ibc) +{ + struct dsync_brain *brain; + const struct master_service_settings *service_set; + pool_t pool; + + service_set = master_service_settings_get(master_service); + mail_user_ref(user); + + pool = pool_alloconly_create("dsync brain", 10240); + brain = p_new(pool, struct dsync_brain, 1); + brain->pool = pool; + brain->user = user; + brain->ibc = ibc; + brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN; + brain->lock_fd = -1; + brain->verbose_proctitle = service_set->verbose_proctitle; + hash_table_create(&brain->mailbox_states, pool, 0, + guid_128_hash, guid_128_cmp); + p_array_init(&brain->remote_mailbox_states, pool, 64); + return brain; +} + +static void +dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags) +{ + brain->mail_requests = + (flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0; + brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0; + brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0; + brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0; + brain->sync_visible_namespaces = + (flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0; + brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0; + brain->no_backup_overwrite = + (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0; + brain->no_mail_prefetch = + (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; + brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0; + brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0; +} + +static void +dsync_brain_open_virtual_all_box(struct dsync_brain *brain, + const char *vname) +{ + struct mail_namespace *ns; + + ns = mail_namespace_find(brain->user->namespaces, vname); + brain->virtual_all_box = + mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); +} + +struct dsync_brain * +dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const struct dsync_brain_settings *set) +{ + struct dsync_ibc_settings ibc_set; + struct dsync_brain *brain; + struct mail_namespace *ns; + string_t *sync_ns_str = NULL; + const char *error; + + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE || + (set->state != NULL && *set->state != '\0')); + i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1); + + brain = dsync_brain_common_init(user, ibc); + brain->process_title_prefix = + p_strdup(brain->pool, set->process_title_prefix); + brain->sync_type = sync_type; + if (array_count(&set->sync_namespaces) > 0) { + sync_ns_str = t_str_new(128); + p_array_init(&brain->sync_namespaces, brain->pool, + array_count(&set->sync_namespaces)); + array_foreach_elem(&set->sync_namespaces, ns) { + str_append(sync_ns_str, ns->prefix); + str_append_c(sync_ns_str, '\n'); + array_push_back(&brain->sync_namespaces, &ns); + } + str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1); + } + brain->alt_char = set->mailbox_alt_char == '\0' ? '_' : + set->mailbox_alt_char; + brain->sync_since_timestamp = set->sync_since_timestamp; + brain->sync_until_timestamp = set->sync_until_timestamp; + brain->sync_max_size = set->sync_max_size; + brain->sync_flag = p_strdup(brain->pool, set->sync_flag); + brain->sync_box = p_strdup(brain->pool, set->sync_box); + brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(brain->pool, set->exclude_mailboxes); + memcpy(brain->sync_box_guid, set->sync_box_guid, + sizeof(brain->sync_box_guid)); + brain->lock_timeout = set->lock_timeout_secs; + if (brain->lock_timeout != 0) + brain->mailbox_lock_timeout_secs = brain->lock_timeout; + else + brain->mailbox_lock_timeout_secs = + DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS; + brain->import_commit_msgs_interval = set->import_commit_msgs_interval; + brain->master_brain = TRUE; + brain->hashed_headers = + (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers); + dsync_brain_set_flags(brain, flags); + + if (set->virtual_all_box != NULL) + dsync_brain_open_virtual_all_box(brain, set->virtual_all_box); + + if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + ; + else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool, + set->state, &error) < 0) { + hash_table_clear(brain->mailbox_states, FALSE); + i_error("Saved sync state is invalid, " + "falling back to full sync: %s", error); + brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + } else { + if (brain->debug) { + i_debug("brain %c: Imported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } + } + dsync_brain_mailbox_trees_init(brain); + + i_zero(&ibc_set); + ibc_set.hostname = my_hostdomain(); + ibc_set.sync_ns_prefixes = sync_ns_str == NULL ? + NULL : str_c(sync_ns_str); + ibc_set.sync_box = set->sync_box; + ibc_set.virtual_all_box = set->virtual_all_box; + ibc_set.exclude_mailboxes = set->exclude_mailboxes; + ibc_set.sync_since_timestamp = set->sync_since_timestamp; + ibc_set.sync_until_timestamp = set->sync_until_timestamp; + ibc_set.sync_max_size = set->sync_max_size; + ibc_set.sync_flags = set->sync_flag; + memcpy(ibc_set.sync_box_guid, set->sync_box_guid, + sizeof(ibc_set.sync_box_guid)); + ibc_set.alt_char = brain->alt_char; + ibc_set.sync_type = sync_type; + ibc_set.hdr_hash_v2 = TRUE; + ibc_set.lock_timeout = set->lock_timeout_secs; + ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval; + ibc_set.hashed_headers = set->hashed_headers; + /* reverse the backup direction for the slave */ + ibc_set.brain_flags = flags & ENUM_NEGATE(DSYNC_BRAIN_FLAG_BACKUP_SEND | + DSYNC_BRAIN_FLAG_BACKUP_RECV); + if ((flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0) + ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + else if ((flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0) + ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + dsync_ibc_send_handshake(ibc, &ibc_set); + + dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain); + brain->state = DSYNC_STATE_MASTER_RECV_HANDSHAKE; + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + return brain; +} + +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, + bool local, const char *process_title_prefix, + char default_alt_char) +{ + struct dsync_ibc_settings ibc_set; + struct dsync_brain *brain; + + i_assert(default_alt_char != '\0'); + + brain = dsync_brain_common_init(user, ibc); + brain->alt_char = default_alt_char; + brain->process_title_prefix = + p_strdup(brain->pool, process_title_prefix); + brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE; + + if (local) { + /* both master and slave are running within the same process, + update the proctitle only for master. */ + brain->verbose_proctitle = FALSE; + } + + i_zero(&ibc_set); + ibc_set.hdr_hash_v2 = TRUE; + ibc_set.hostname = my_hostdomain(); + dsync_ibc_send_handshake(ibc, &ibc_set); + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain); + return brain; +} + +static void dsync_brain_purge(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + struct mail_storage *storage; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + storage = mail_namespace_get_default_storage(ns); + if (mail_storage_purge(storage) < 0) { + i_error("Purging namespace '%s' failed: %s", ns->prefix, + mail_storage_get_last_internal_error(storage, NULL)); + } + } +} + +int dsync_brain_deinit(struct dsync_brain **_brain, enum mail_error *error_r) +{ + struct dsync_brain *brain = *_brain; + int ret; + + *_brain = NULL; + + if (dsync_ibc_has_timed_out(brain->ibc)) { + i_error("Timeout during state=%s%s", + dsync_state_names[brain->state], + brain->state != DSYNC_STATE_SYNC_MAILS ? "" : + t_strdup_printf(" (send=%s recv=%s)", + dsync_box_state_names[brain->box_send_state], + dsync_box_state_names[brain->box_recv_state])); + } + if (dsync_ibc_has_failed(brain->ibc) || + brain->state != DSYNC_STATE_DONE) + brain->failed = TRUE; + dsync_ibc_close_mail_streams(brain->ibc); + + if (brain->purge && !brain->failed) + dsync_brain_purge(brain); + + if (brain->box != NULL) + dsync_brain_sync_mailbox_deinit(brain); + if (brain->virtual_all_box != NULL) + mailbox_free(&brain->virtual_all_box); + if (brain->local_tree_iter != NULL) + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + if (brain->local_mailbox_tree != NULL) + dsync_mailbox_tree_deinit(&brain->local_mailbox_tree); + if (brain->remote_mailbox_tree != NULL) + dsync_mailbox_tree_deinit(&brain->remote_mailbox_tree); + hash_table_iterate_deinit(&brain->mailbox_states_iter); + hash_table_destroy(&brain->mailbox_states); + + pool_unref(&brain->dsync_box_pool); + + if (brain->lock_fd != -1) { + /* unlink the lock file before it gets unlocked */ + i_unlink(brain->lock_path); + if (brain->debug) { + i_debug("brain %c: Unlocked %s", + brain->master_brain ? 'M' : 'S', + brain->lock_path); + } + file_lock_free(&brain->lock); + i_close_fd(&brain->lock_fd); + } + + ret = brain->failed ? -1 : 0; + mail_user_unref(&brain->user); + + *error_r = !brain->failed ? 0 : + (brain->mail_error == 0 ? MAIL_ERROR_TEMP : brain->mail_error); + pool_unref(&brain->pool); + return ret; +} + +static int +dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname) +{ + const struct file_create_settings lock_set = { + .lock_timeout_secs = brain->lock_timeout, + .lock_settings = { + .lock_method = FILE_LOCK_METHOD_FCNTL, + }, + }; + const char *home, *error, *local_hostname = my_hostdomain(); + bool created; + int ret; + + if ((ret = strcmp(remote_hostname, local_hostname)) < 0) { + /* locking done by remote */ + if (brain->debug) { + i_debug("brain %c: Locking done by remote " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + local_hostname, remote_hostname); + } + return 0; + } + if (ret == 0 && !brain->master_brain) { + /* running dsync within the same server. + locking done by master brain. */ + if (brain->debug) { + i_debug("brain %c: Locking done by local master-brain " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + local_hostname, remote_hostname); + } + return 0; + } + + if ((ret = mail_user_get_home(brain->user, &home)) < 0) { + i_error("Couldn't look up user's home dir"); + return -1; + } + if (ret == 0) { + i_error("User has no home directory"); + return -1; + } + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING)); + brain->lock_path = p_strconcat(brain->pool, home, + "/"DSYNC_LOCK_FILENAME, NULL); + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + if (brain->lock_fd == -1 && errno == ENOENT) { + /* home directory not created */ + if (mail_user_home_mkdir(brain->user) < 0) + return -1; + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + } + if (brain->lock_fd == -1) + i_error("Couldn't lock %s: %s", brain->lock_path, error); + else if (brain->debug) { + i_debug("brain %c: Locking done locally in %s " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + brain->lock_path, local_hostname, remote_hostname); + } + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + return brain->lock_fd == -1 ? -1 : 0; +} + +static void +dsync_brain_set_hdr_hash_version(struct dsync_brain *brain, + const struct dsync_ibc_settings *ibc_set) +{ + if (ibc_set->hdr_hash_v3) + brain->hdr_hash_version = 3; + else if (ibc_set->hdr_hash_v2) + brain->hdr_hash_version = 3; + else + brain->hdr_hash_version = 1; +} + +static bool dsync_brain_master_recv_handshake(struct dsync_brain *brain) +{ + const struct dsync_ibc_settings *ibc_set; + + i_assert(brain->master_brain); + + if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0) + return FALSE; + + if (brain->lock_timeout > 0) { + if (dsync_brain_lock(brain, ibc_set->hostname) < 0) { + brain->failed = TRUE; + return FALSE; + } + } + dsync_brain_set_hdr_hash_version(brain, ibc_set); + + brain->state = brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE ? + DSYNC_STATE_MASTER_SEND_LAST_COMMON : + DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; +} + +static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) +{ + const struct dsync_ibc_settings *ibc_set; + struct mail_namespace *ns; + const char *const *prefixes; + + i_assert(!brain->master_brain); + + if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0) + return FALSE; + dsync_brain_set_hdr_hash_version(brain, ibc_set); + + if (ibc_set->lock_timeout > 0) { + brain->lock_timeout = ibc_set->lock_timeout; + brain->mailbox_lock_timeout_secs = brain->lock_timeout; + if (dsync_brain_lock(brain, ibc_set->hostname) < 0) { + brain->failed = TRUE; + return FALSE; + } + } else { + brain->mailbox_lock_timeout_secs = + DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS; + } + + if (ibc_set->sync_ns_prefixes != NULL) { + p_array_init(&brain->sync_namespaces, brain->pool, 4); + prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n"); + if (prefixes[0] == NULL) { + /* ugly workaround for strsplit API: there was one + prefix="" entry */ + static const char *empty_prefix[] = { "", NULL }; + prefixes = empty_prefix; + } + for (; *prefixes != NULL; prefixes++) { + ns = mail_namespace_find(brain->user->namespaces, + *prefixes); + if (ns == NULL) { + i_error("Namespace not found: '%s'", *prefixes); + brain->failed = TRUE; + return FALSE; + } + array_push_back(&brain->sync_namespaces, &ns); + } + } + brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box); + brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes); + brain->sync_since_timestamp = ibc_set->sync_since_timestamp; + brain->sync_until_timestamp = ibc_set->sync_until_timestamp; + brain->sync_max_size = ibc_set->sync_max_size; + brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags); + memcpy(brain->sync_box_guid, ibc_set->sync_box_guid, + sizeof(brain->sync_box_guid)); + if (ibc_set->alt_char != '\0') + brain->alt_char = ibc_set->alt_char; + i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + brain->sync_type = ibc_set->sync_type; + + dsync_brain_set_flags(brain, ibc_set->brain_flags); + if (ibc_set->hashed_headers != NULL) + brain->hashed_headers = + p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers); + /* this flag is only set on the remote slave brain */ + brain->purge = (ibc_set->brain_flags & + DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0; + + if (ibc_set->virtual_all_box != NULL) + dsync_brain_open_virtual_all_box(brain, ibc_set->virtual_all_box); + dsync_brain_mailbox_trees_init(brain); + + if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) + brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON; + else + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; +} + +static void dsync_brain_master_send_last_common(struct dsync_brain *brain) +{ + struct dsync_mailbox_state *state; + uint8_t *guid; + enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK; + + i_assert(brain->master_brain); + + if (brain->mailbox_states_iter == NULL) { + brain->mailbox_states_iter = + hash_table_iterate_init(brain->mailbox_states); + } + + for (;;) { + if (ret == DSYNC_IBC_SEND_RET_FULL) + return; + if (!hash_table_iterate(brain->mailbox_states_iter, + brain->mailbox_states, &guid, &state)) + break; + ret = dsync_ibc_send_mailbox_state(brain->ibc, state); + } + hash_table_iterate_deinit(&brain->mailbox_states_iter); + + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_STATE); + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; +} + +static void dsync_mailbox_state_add(struct dsync_brain *brain, + const struct dsync_mailbox_state *state) +{ + struct dsync_mailbox_state *dupstate; + uint8_t *guid_p; + + dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1); + *dupstate = *state; + guid_p = dupstate->mailbox_guid; + hash_table_insert(brain->mailbox_states, guid_p, dupstate); +} + +static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain) +{ + struct dsync_mailbox_state state; + enum dsync_ibc_recv_ret ret; + bool changed = FALSE; + + i_assert(!brain->master_brain); + + while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) { + dsync_mailbox_state_add(brain, &state); + changed = TRUE; + } + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + changed = TRUE; + } + return changed; +} + +static bool dsync_brain_finish(struct dsync_brain *brain) +{ + const char *error; + enum mail_error mail_error; + bool require_full_resync; + enum dsync_ibc_recv_ret ret; + + if (!brain->master_brain) { + dsync_ibc_send_finish(brain->ibc, + brain->failed ? "dsync failed" : NULL, + brain->mail_error, + brain->require_full_resync); + brain->state = DSYNC_STATE_DONE; + return TRUE; + } + ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error, + &require_full_resync); + if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN) + return FALSE; + if (error != NULL) { + i_error("Remote dsync failed: %s", error); + brain->failed = TRUE; + if (mail_error != 0 && + (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP)) + brain->mail_error = mail_error; + } + if (require_full_resync) + brain->require_full_resync = TRUE; + brain->state = DSYNC_STATE_DONE; + return TRUE; +} + +static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r) +{ + enum dsync_state orig_state = brain->state; + enum dsync_box_state orig_box_recv_state = brain->box_recv_state; + enum dsync_box_state orig_box_send_state = brain->box_send_state; + bool changed = FALSE, ret = TRUE; + + if (brain->failed) + return FALSE; + + switch (brain->state) { + case DSYNC_STATE_MASTER_RECV_HANDSHAKE: + changed = dsync_brain_master_recv_handshake(brain); + break; + case DSYNC_STATE_SLAVE_RECV_HANDSHAKE: + changed = dsync_brain_slave_recv_handshake(brain); + break; + case DSYNC_STATE_MASTER_SEND_LAST_COMMON: + dsync_brain_master_send_last_common(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_LAST_COMMON: + changed = dsync_brain_slave_recv_last_common(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE: + dsync_brain_send_mailbox_tree(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE: + changed = dsync_brain_recv_mailbox_tree(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES: + dsync_brain_send_mailbox_tree_deletes(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES: + changed = dsync_brain_recv_mailbox_tree_deletes(brain); + break; + case DSYNC_STATE_MASTER_SEND_MAILBOX: + dsync_brain_master_send_mailbox(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_MAILBOX: + changed = dsync_brain_slave_recv_mailbox(brain); + break; + case DSYNC_STATE_SYNC_MAILS: + changed = dsync_brain_sync_mails(brain); + break; + case DSYNC_STATE_FINISH: + changed = dsync_brain_finish(brain); + break; + case DSYNC_STATE_DONE: + changed = TRUE; + ret = FALSE; + break; + } + if (brain->verbose_proctitle) { + if (orig_state != brain->state || + orig_box_recv_state != brain->box_recv_state || + orig_box_send_state != brain->box_send_state || + ++brain->proctitle_update_counter % 100 == 0) + process_title_set(dsync_brain_get_proctitle(brain)); + } + *changed_r = changed; + return brain->failed ? FALSE : ret; +} + +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r) +{ + bool ret; + + *changed_r = FALSE; + + if (dsync_ibc_has_failed(brain->ibc)) { + brain->failed = TRUE; + return FALSE; + } + + T_BEGIN { + ret = dsync_brain_run_real(brain, changed_r); + } T_END; + return ret; +} + +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_state *state; + uint8_t *guid; + + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%"PRIu64" pvt_modseq=%"PRIu64" messages=%u changes_during_sync=%d", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid), + state->last_uidvalidity, + state->last_common_uid, + state->last_common_modseq, + state->last_common_pvt_modseq, + state->last_messages_count, + state->changes_during_sync ? 1 : 0); + } + hash_table_iterate_deinit(&iter); +} + +void dsync_brain_get_state(struct dsync_brain *brain, string_t *output) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_node *node; + const struct dsync_mailbox_state *new_state; + struct dsync_mailbox_state *state; + const uint8_t *guid_p; + uint8_t *guid; + + if (brain->require_full_resync) + return; + + /* update mailbox states */ + array_foreach(&brain->remote_mailbox_states, new_state) { + guid_p = new_state->mailbox_guid; + state = hash_table_lookup(brain->mailbox_states, guid_p); + if (state != NULL) + *state = *new_state; + else + dsync_mailbox_state_add(brain, new_state); + } + + /* remove nonexistent mailboxes */ + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree, + guid); + if (node == NULL || + node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + if (brain->debug) { + i_debug("brain %c: Removed state for deleted mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid)); + } + hash_table_remove(brain->mailbox_states, guid); + } + } + hash_table_iterate_deinit(&iter); + + if (brain->debug) { + i_debug("brain %c: Exported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } + dsync_mailbox_states_export(brain->mailbox_states, output); +} + +enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain) +{ + return brain->sync_type; +} + +bool dsync_brain_has_failed(struct dsync_brain *brain) +{ + return brain->failed; +} + +const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain, + bool *remote_only_r) +{ + if (brain->changes_during_sync == NULL && + brain->changes_during_remote_sync) { + *remote_only_r = TRUE; + return "Remote notified that changes happened during sync"; + } + *remote_only_r = FALSE; + return brain->changes_during_sync; +} + +static bool dsync_brain_want_shared_namespace(const struct mail_namespace *ns, + const struct mail_namespace *sync_ns) +{ + /* Include shared namespaces and all its + children in the sync (e.g. "Shared/example.com" + will be synced to "Shared/"). + This also allows "dsync -n Shared/example.com/" + with "Shared/example.com/username/" style + shared namespace config. */ + return (ns->type == MAIL_NAMESPACE_TYPE_SHARED) && + (sync_ns->type == MAIL_NAMESPACE_TYPE_SHARED) && + str_begins(ns->prefix, sync_ns->prefix); +} + +bool dsync_brain_want_namespace(struct dsync_brain *brain, + struct mail_namespace *ns) +{ + struct mail_namespace *sync_ns; + + if (array_is_created(&brain->sync_namespaces)) { + array_foreach_elem(&brain->sync_namespaces, sync_ns) { + if (ns == sync_ns) + return TRUE; + if (dsync_brain_want_shared_namespace(ns, sync_ns)) + return TRUE; + } + return FALSE; + } + if (ns->alias_for != NULL) { + /* always skip aliases */ + return FALSE; + } + if (brain->sync_visible_namespaces) { + if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0) + return TRUE; + if ((ns->flags & (NAMESPACE_FLAG_LIST_PREFIX | + NAMESPACE_FLAG_LIST_CHILDREN)) != 0) + return TRUE; + return FALSE; + } else { + return strcmp(ns->unexpanded_set->location, + SETTING_STRVAR_UNEXPANDED) == 0; + } +} + +void dsync_brain_set_changes_during_sync(struct dsync_brain *brain, + const char *reason) +{ + if (brain->debug) { + i_debug("brain %c: Change during sync: %s", + brain->master_brain ? 'M' : 'S', reason); + } + if (brain->changes_during_sync == NULL) + brain->changes_during_sync = p_strdup(brain->pool, reason); +} diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h new file mode 100644 index 0000000..5813148 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain.h @@ -0,0 +1,130 @@ +#ifndef DSYNC_BRAIN_H +#define DSYNC_BRAIN_H + +#include "module-context.h" +#include "guid.h" +#include "mail-error.h" +#include "mailbox-list-private.h" + +struct mail_namespace; +struct mail_user; +struct dsync_ibc; + +enum dsync_brain_flags { + DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS = 0x01, + DSYNC_BRAIN_FLAG_BACKUP_SEND = 0x02, + DSYNC_BRAIN_FLAG_BACKUP_RECV = 0x04, + DSYNC_BRAIN_FLAG_DEBUG = 0x08, + DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES= 0x10, + /* Sync everything but the actual mails (e.g. mailbox creates, + deletes) */ + DSYNC_BRAIN_FLAG_NO_MAIL_SYNC = 0x20, + /* Used with BACKUP_SEND/RECV: Don't force the + Use the two-way syncing algorithm, but don't actually modify + anything locally. (Useful during migration.) */ + DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40, + /* Run storage purge on the remote after syncing. + Useful with e.g. a nightly doveadm backup. */ + DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80, + /* Don't prefetch mail bodies until they're actually needed. This works + only with pipe ibc. It's useful if most of the mails can be copied + directly within filesystem without having to read them. */ + DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100, + /* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */ + DSYNC_BRAIN_FLAG_NO_NOTIFY = 0x400, + /* Workaround missing Date/Message-ID headers */ + DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND = 0x800, +}; + +enum dsync_brain_sync_type { + DSYNC_BRAIN_SYNC_TYPE_UNKNOWN, + /* Go through all mailboxes to make sure everything is synced */ + DSYNC_BRAIN_SYNC_TYPE_FULL, + /* Go through all mailboxes that have changed (based on UIDVALIDITY, + UIDNEXT, HIGHESTMODSEQ). If both sides have had equal amount of + changes in some mailbox, it may get incorrectly skipped. */ + DSYNC_BRAIN_SYNC_TYPE_CHANGED, + /* Use saved state to find out what has changed. */ + DSYNC_BRAIN_SYNC_TYPE_STATE +}; + +struct dsync_brain_settings { + const char *process_title_prefix; + /* Sync only these namespaces */ + ARRAY(struct mail_namespace *) sync_namespaces; + /* Sync only this mailbox name */ + const char *sync_box; + /* Use this virtual \All mailbox to be able to copy mails with the same + GUID instead of saving them twice. With most storages this results + in less disk space usage. */ + const char *virtual_all_box; + /* Sync only this mailbox GUID */ + guid_128_t sync_box_guid; + /* Exclude these mailboxes from the sync. They can contain '*' + wildcards and be \special-use flags. */ + const char *const *exclude_mailboxes; + /* Alternative character to use in mailbox names where the original + character cannot be used. */ + char mailbox_alt_char; + /* Sync only mails with received timestamp at least this high. */ + time_t sync_since_timestamp; + /* Sync only mails with received timestamp less or equal than this */ + time_t sync_until_timestamp; + /* Don't sync mails larger than this. */ + uoff_t sync_max_size; + /* Sync only mails which contains / doesn't contain this flag. + '-' at the beginning means this flag must not exist. */ + const char *sync_flag; + /* Headers to hash (defaults to Date, Message-ID) */ + const char *const *hashed_headers; + + /* If non-zero, use dsync lock file for this user */ + unsigned int lock_timeout_secs; + /* If non-zero, importing will attempt to commit transaction after + saving this many messages. */ + unsigned int import_commit_msgs_interval; + /* Input state for DSYNC_BRAIN_SYNC_TYPE_STATE */ + const char *state; +}; + +#define DSYNC_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, dsync_mailbox_list_module) +struct dsync_mailbox_list { + union mailbox_list_module_context module_ctx; + bool have_orig_escape_char; +}; +extern MODULE_CONTEXT_DEFINE(dsync_mailbox_list_module, + &mailbox_list_module_register); + +struct dsync_brain * +dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const struct dsync_brain_settings *set); +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, + bool local, const char *process_title_prefix, + char default_alt_char); +/* Returns 0 if everything was successful, -1 if syncing failed in some way */ +int dsync_brain_deinit(struct dsync_brain **brain, enum mail_error *error_r); + +/* Returns TRUE if brain needs to run more, FALSE if it's finished. + changed_r is TRUE if anything happened during this run. */ +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r); +/* Returns TRUE if brain has failed, and there's no point in continuing. */ +bool dsync_brain_has_failed(struct dsync_brain *brain); +/* Returns the current sync state string, which can be given as parameter to + dsync_brain_master_init() to quickly sync only the new changes. */ +void dsync_brain_get_state(struct dsync_brain *brain, string_t *output); +/* Returns the sync type that was used. Mainly useful with slave brain. */ +enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain); +/* If there were any unexpected changes during the sync, return the reason + for them. Otherwise return NULL. If remote_only_r=TRUE, this brain itself + didn't see any changes, but the remote brain did. */ +const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain, + bool *remote_only_r); +/* Returns TRUE if we want to sync this namespace. */ +bool dsync_brain_want_namespace(struct dsync_brain *brain, + struct mail_namespace *ns); + +#endif diff --git a/src/doveadm/dsync/dsync-deserializer.c b/src/doveadm/dsync/dsync-deserializer.c new file mode 100644 index 0000000..2a90c47 --- /dev/null +++ b/src/doveadm/dsync/dsync-deserializer.c @@ -0,0 +1,193 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strescape.h" +#include "dsync-serializer.h" +#include "dsync-deserializer.h" + +struct dsync_deserializer { + pool_t pool; + const char *name; + const char *const *required_fields; + const char *const *keys; + unsigned int *required_field_indexes; + unsigned int required_field_count; +}; + +struct dsync_deserializer_decoder { + pool_t pool; + struct dsync_deserializer *deserializer; + const char *const *values; + unsigned int values_count; +}; + +static bool field_find(const char *const *names, const char *name, + unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; names[i] != NULL; i++) { + if (strcmp(names[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +int dsync_deserializer_init(const char *name, const char *const *required_fields, + const char *header_line, + struct dsync_deserializer **deserializer_r, + const char **error_r) +{ + struct dsync_deserializer *deserializer; + const char **dup_required_fields; + unsigned int i, required_count; + pool_t pool; + + *deserializer_r = NULL; + + pool = pool_alloconly_create("dsync deserializer", 1024); + deserializer = p_new(pool, struct dsync_deserializer, 1); + deserializer->pool = pool; + deserializer->name = p_strdup(pool, name); + deserializer->keys = (void *)p_strsplit_tabescaped(pool, header_line); + + deserializer->required_field_count = required_count = + required_fields == NULL ? 0 : + str_array_length(required_fields); + dup_required_fields = p_new(pool, const char *, required_count + 1); + deserializer->required_field_indexes = + p_new(pool, unsigned int, required_count + 1); + for (i = 0; i < required_count; i++) { + dup_required_fields[i] = + p_strdup(pool, required_fields[i]); + if (!field_find(deserializer->keys, required_fields[i], + &deserializer->required_field_indexes[i])) { + *error_r = t_strdup_printf( + "Header missing required field %s", + required_fields[i]); + pool_unref(&pool); + return -1; + } + } + deserializer->required_fields = dup_required_fields; + + *deserializer_r = deserializer; + return 0; +} + +void dsync_deserializer_deinit(struct dsync_deserializer **_deserializer) +{ + struct dsync_deserializer *deserializer = *_deserializer; + + *_deserializer = NULL; + + pool_unref(&deserializer->pool); +} + +int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer, + const char *input, + struct dsync_deserializer_decoder **decoder_r, + const char **error_r) +{ + struct dsync_deserializer_decoder *decoder; + unsigned int i; + char **values; + pool_t pool; + + *decoder_r = NULL; + + pool = pool_alloconly_create("dsync deserializer decode", 1024); + decoder = p_new(pool, struct dsync_deserializer_decoder, 1); + decoder->pool = pool; + decoder->deserializer = deserializer; + values = p_strsplit_tabescaped(pool, input); + + /* fix NULLs */ + for (i = 0; values[i] != NULL; i++) { + if (values[i][0] == NULL_CHR) { + /* NULL? */ + if (values[i][1] == '\0') + values[i] = NULL; + else + values[i] += 1; + } + } + decoder->values_count = i; + + /* see if all required fields exist */ + for (i = 0; i < deserializer->required_field_count; i++) { + unsigned int ridx = deserializer->required_field_indexes[i]; + + if (ridx >= decoder->values_count || values[ridx] == NULL) { + *error_r = t_strdup_printf("Missing required field %s", + deserializer->required_fields[i]); + pool_unref(&pool); + return -1; + } + } + decoder->values = (void *)values; + + *decoder_r = decoder; + return 0; +} + +static bool +dsync_deserializer_find_field(struct dsync_deserializer *deserializer, + const char *key, unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; deserializer->keys[i] != NULL; i++) { + if (strcmp(deserializer->keys[i], key) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder, + const char *key, const char **value_r) +{ + unsigned int idx; + + if (!dsync_deserializer_find_field(decoder->deserializer, key, &idx) || + idx >= decoder->values_count) { + *value_r = NULL; + return FALSE; + } else { + *value_r = decoder->values[idx]; + return *value_r != NULL; + } +} + +const char * +dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder, + const char *key) +{ + const char *value; + + if (!dsync_deserializer_decode_try(decoder, key, &value)) { + i_panic("dsync_deserializer_decode_get() " + "used for non-required key %s", key); + } + return value; +} + +const char * +dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder) +{ + return decoder->deserializer->name; +} + +void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **_decoder) +{ + struct dsync_deserializer_decoder *decoder = *_decoder; + + *_decoder = NULL; + + pool_unref(&decoder->pool); +} diff --git a/src/doveadm/dsync/dsync-deserializer.h b/src/doveadm/dsync/dsync-deserializer.h new file mode 100644 index 0000000..45b19cc --- /dev/null +++ b/src/doveadm/dsync/dsync-deserializer.h @@ -0,0 +1,27 @@ +#ifndef DSYNC_DESERIALIZER_H +#define DSYNC_DESERIALIZER_H + +struct dsync_deserializer; +struct dsync_deserializer_decoder; + +int dsync_deserializer_init(const char *name, const char *const *required_fields, + const char *header_line, + struct dsync_deserializer **deserializer_r, + const char **error_r); +void dsync_deserializer_deinit(struct dsync_deserializer **deserializer); + +int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer, + const char *input, + struct dsync_deserializer_decoder **decoder_r, + const char **error_r); +bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder, + const char *key, const char **value_r); +/* key must be in required fields. The return value is never NULL. */ +const char * +dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder, + const char *key); +const char * +dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder); +void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **decoder); + +#endif diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c new file mode 100644 index 0000000..1b8886e --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc-pipe.c @@ -0,0 +1,599 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc-private.h" + +enum item_type { + ITEM_END_OF_LIST, + ITEM_HANDSHAKE, + ITEM_MAILBOX_STATE, + ITEM_MAILBOX_TREE_NODE, + ITEM_MAILBOX_DELETE, + ITEM_MAILBOX, + ITEM_MAILBOX_ATTRIBUTE, + ITEM_MAIL_CHANGE, + ITEM_MAIL_REQUEST, + ITEM_MAIL, + ITEM_FINISH +}; + +struct item { + enum item_type type; + pool_t pool; + + union { + struct dsync_ibc_settings set; + struct dsync_mailbox_state state; + struct dsync_mailbox_node node; + guid_128_t mailbox_guid; + struct dsync_mailbox dsync_box; + struct dsync_mailbox_attribute attr; + struct dsync_mail_change change; + struct dsync_mail_request request; + struct dsync_mail mail; + struct { + const struct dsync_mailbox_delete *deletes; + unsigned int count; + char hierarchy_sep; + char escape_char; + } mailbox_delete; + struct { + const char *error; + enum mail_error mail_error; + bool require_full_resync; + } finish; + } u; +}; + +struct dsync_ibc_pipe { + struct dsync_ibc ibc; + + ARRAY(pool_t) pools; + ARRAY(struct item) item_queue; + struct dsync_ibc_pipe *remote; + + pool_t pop_pool; + struct item pop_item; +}; + +static pool_t dsync_ibc_pipe_get_pool(struct dsync_ibc_pipe *pipe) +{ + pool_t *pools, ret; + unsigned int count; + + pools = array_get_modifiable(&pipe->pools, &count); + if (count == 0) + return pool_alloconly_create(MEMPOOL_GROWING"pipe item pool", 1024); + + ret = pools[count-1]; + array_delete(&pipe->pools, count-1, 1); + p_clear(ret); + return ret; +} + +static struct item * ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_pipe_push_item(struct dsync_ibc_pipe *pipe, enum item_type type) +{ + struct item *item; + + item = array_append_space(&pipe->item_queue); + item->type = type; + + switch (type) { + case ITEM_END_OF_LIST: + case ITEM_MAILBOX_STATE: + case ITEM_MAILBOX_DELETE: + break; + case ITEM_HANDSHAKE: + case ITEM_MAILBOX: + case ITEM_MAILBOX_TREE_NODE: + case ITEM_MAILBOX_ATTRIBUTE: + case ITEM_MAIL_CHANGE: + case ITEM_MAIL_REQUEST: + case ITEM_MAIL: + case ITEM_FINISH: + item->pool = dsync_ibc_pipe_get_pool(pipe); + break; + } + return item; +} + +static struct item * +dsync_ibc_pipe_pop_item(struct dsync_ibc_pipe *pipe, enum item_type type) +{ + struct item *item; + + if (array_count(&pipe->item_queue) == 0) + return NULL; + + item = array_front_modifiable(&pipe->item_queue); + i_assert(item->type == type); + pipe->pop_item = *item; + array_pop_front(&pipe->item_queue); + item = NULL; + + pool_unref(&pipe->pop_pool); + pipe->pop_pool = pipe->pop_item.pool; + return &pipe->pop_item; +} + +static bool dsync_ibc_pipe_try_pop_eol(struct dsync_ibc_pipe *pipe) +{ + const struct item *item; + + if (array_count(&pipe->item_queue) == 0) + return FALSE; + + item = array_front(&pipe->item_queue); + if (item->type != ITEM_END_OF_LIST) + return FALSE; + + array_pop_front(&pipe->item_queue); + return TRUE; +} + +static void dsync_ibc_pipe_deinit(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + pool_t pool; + + if (pipe->remote != NULL) { + i_assert(pipe->remote->remote == pipe); + pipe->remote->remote = NULL; + } + + pool_unref(&pipe->pop_pool); + array_foreach_modifiable(&pipe->item_queue, item) { + pool_unref(&item->pool); + } + array_foreach_elem(&pipe->pools, pool) + pool_unref(&pool); + array_free(&pipe->pools); + array_free(&pipe->item_queue); + i_free(pipe); +} + +static void +dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_HANDSHAKE); + item->u.set = *set; + item->u.set.sync_ns_prefixes = + p_strdup(item->pool, set->sync_ns_prefixes); + item->u.set.sync_box = p_strdup(item->pool, set->sync_box); + item->u.set.virtual_all_box = p_strdup(item->pool, set->virtual_all_box); + item->u.set.exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(item->pool, set->exclude_mailboxes); + memcpy(item->u.set.sync_box_guid, set->sync_box_guid, + sizeof(item->u.set.sync_box_guid)); + item->u.set.sync_since_timestamp = set->sync_since_timestamp; + item->u.set.sync_until_timestamp = set->sync_until_timestamp; + item->u.set.sync_max_size = set->sync_max_size; + item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_HANDSHAKE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *set_r = &item->u.set; + return DSYNC_IBC_RECV_RET_OK; +} + +static bool dsync_ibc_pipe_is_send_queue_full(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + return array_count(&pipe->remote->item_queue) > 0; +} + +static bool dsync_ibc_pipe_has_pending_data(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + return array_count(&pipe->item_queue) > 0; +} + +static void +dsync_ibc_pipe_send_end_of_list(struct dsync_ibc *ibc, + enum dsync_ibc_eol_type type ATTR_UNUSED) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + dsync_ibc_pipe_push_item(pipe->remote, ITEM_END_OF_LIST); +} + +static void +dsync_ibc_pipe_send_mailbox_state(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_STATE); + item->u.state = *state; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_state(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_STATE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *state_r = item->u.state; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_TREE_NODE); + + /* a little bit kludgy way to send it */ + item->u.node.name = (void *)p_strarray_dup(item->pool, name); + dsync_mailbox_node_copy_data(&item->u.node, node); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_TREE_NODE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *name_r = (const void *)item->u.node.name; + item->u.node.name = NULL; + + *node_r = &item->u.node; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_DELETE); + + /* we'll assume that the deletes are permanent. this works for now.. */ + /* a little bit kludgy way to send it */ + item->u.mailbox_delete.deletes = deletes; + item->u.mailbox_delete.count = count; + item->u.mailbox_delete.hierarchy_sep = hierarchy_sep; + item->u.mailbox_delete.escape_char = escape_char; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, + char *hierarchy_sep_r, + char *escape_char_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_DELETE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *deletes_r = item->u.mailbox_delete.deletes; + *count_r = item->u.mailbox_delete.count; + *hierarchy_sep_r = item->u.mailbox_delete.hierarchy_sep; + *escape_char_r = item->u.mailbox_delete.escape_char; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + const struct mailbox_cache_field *cf; + struct mailbox_cache_field *ncf; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX); + item->u.dsync_box = *dsync_box; + p_array_init(&item->u.dsync_box.cache_fields, item->pool, + array_count(&dsync_box->cache_fields)); + array_foreach(&dsync_box->cache_fields, cf) { + ncf = array_append_space(&item->u.dsync_box.cache_fields); + ncf->name = p_strdup(item->pool, cf->name); + ncf->decision = cf->decision; + ncf->last_used = cf->last_used; + } +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *dsync_box_r = &item->u.dsync_box; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_ATTRIBUTE); + dsync_mailbox_attribute_dup(item->pool, attr, &item->u.attr); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_ATTRIBUTE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *attr_r = &item->u.attr; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_change(struct dsync_ibc *ibc, + const struct dsync_mail_change *change) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_CHANGE); + dsync_mail_change_dup(item->pool, change, &item->u.change); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_change(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_CHANGE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *change_r = &item->u.change; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request *request) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_REQUEST); + item->u.request.guid = p_strdup(item->pool, request->guid); + item->u.request.uid = request->uid; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_REQUEST); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *request_r = &item->u.request; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL); + item->u.mail.guid = p_strdup(item->pool, mail->guid); + item->u.mail.uid = mail->uid; + item->u.mail.pop3_uidl = p_strdup(item->pool, mail->pop3_uidl); + item->u.mail.pop3_order = mail->pop3_order; + item->u.mail.received_date = mail->received_date; + if (mail->input != NULL) { + item->u.mail.input = mail->input; + i_stream_ref(mail->input); + } + item->u.mail.input_mail = mail->input_mail; + item->u.mail.input_mail_uid = mail->input_mail_uid; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *mail_r = &item->u.mail; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_FINISH); + item->u.finish.error = p_strdup(item->pool, error); + item->u.finish.mail_error = mail_error; + item->u.finish.require_full_resync = require_full_resync; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_FINISH); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *error_r = item->u.finish.error; + *mail_error_r = item->u.finish.mail_error; + *require_full_resync_r = item->u.finish.require_full_resync; + return DSYNC_IBC_RECV_RET_OK; +} + +static void pipe_close_mail_streams(struct dsync_ibc_pipe *pipe) +{ + struct item *item; + + if (array_count(&pipe->item_queue) > 0) { + item = array_front_modifiable(&pipe->item_queue); + if (item->type == ITEM_MAIL && + item->u.mail.input != NULL) + i_stream_unref(&item->u.mail.input); + } +} + +static void dsync_ibc_pipe_close_mail_streams(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + pipe_close_mail_streams(pipe); + pipe_close_mail_streams(pipe->remote); +} + +static const struct dsync_ibc_vfuncs dsync_ibc_pipe_vfuncs = { + dsync_ibc_pipe_deinit, + dsync_ibc_pipe_send_handshake, + dsync_ibc_pipe_recv_handshake, + dsync_ibc_pipe_send_end_of_list, + dsync_ibc_pipe_send_mailbox_state, + dsync_ibc_pipe_recv_mailbox_state, + dsync_ibc_pipe_send_mailbox_tree_node, + dsync_ibc_pipe_recv_mailbox_tree_node, + dsync_ibc_pipe_send_mailbox_deletes, + dsync_ibc_pipe_recv_mailbox_deletes, + dsync_ibc_pipe_send_mailbox, + dsync_ibc_pipe_recv_mailbox, + dsync_ibc_pipe_send_mailbox_attribute, + dsync_ibc_pipe_recv_mailbox_attribute, + dsync_ibc_pipe_send_change, + dsync_ibc_pipe_recv_change, + dsync_ibc_pipe_send_mail_request, + dsync_ibc_pipe_recv_mail_request, + dsync_ibc_pipe_send_mail, + dsync_ibc_pipe_recv_mail, + dsync_ibc_pipe_send_finish, + dsync_ibc_pipe_recv_finish, + dsync_ibc_pipe_close_mail_streams, + dsync_ibc_pipe_is_send_queue_full, + dsync_ibc_pipe_has_pending_data +}; + +static struct dsync_ibc_pipe * +dsync_ibc_pipe_alloc(void) +{ + struct dsync_ibc_pipe *pipe; + + pipe = i_new(struct dsync_ibc_pipe, 1); + pipe->ibc.v = dsync_ibc_pipe_vfuncs; + i_array_init(&pipe->pools, 4); + i_array_init(&pipe->item_queue, 4); + return pipe; +} + +void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r, struct dsync_ibc **ibc2_r) +{ + struct dsync_ibc_pipe *pipe1, *pipe2; + + pipe1 = dsync_ibc_pipe_alloc(); + pipe2 = dsync_ibc_pipe_alloc(); + pipe1->remote = pipe2; + pipe2->remote = pipe1; + *ibc1_r = &pipe1->ibc; + *ibc2_r = &pipe2->ibc; +} diff --git a/src/doveadm/dsync/dsync-ibc-private.h b/src/doveadm/dsync/dsync-ibc-private.h new file mode 100644 index 0000000..c055aca --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc-private.h @@ -0,0 +1,96 @@ +#ifndef DSYNC_IBC_PRIVATE_H +#define DSYNC_IBC_PRIVATE_H + +#include "dsync-ibc.h" + +struct dsync_ibc_vfuncs { + void (*deinit)(struct dsync_ibc *ibc); + + void (*send_handshake)(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set); + enum dsync_ibc_recv_ret + (*recv_handshake)(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r); + + void (*send_end_of_list)(struct dsync_ibc *ibc, + enum dsync_ibc_eol_type type); + + void (*send_mailbox_state)(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state); + enum dsync_ibc_recv_ret + (*recv_mailbox_state)(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r); + + void (*send_mailbox_tree_node)(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node); + enum dsync_ibc_recv_ret + (*recv_mailbox_tree_node)(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r); + + void (*send_mailbox_deletes)(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char); + enum dsync_ibc_recv_ret + (*recv_mailbox_deletes)(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, + char *hierarchy_sep_r, + char *escape_char_r); + + void (*send_mailbox)(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box); + enum dsync_ibc_recv_ret + (*recv_mailbox)(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r); + + void (*send_mailbox_attribute)(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr); + enum dsync_ibc_recv_ret + (*recv_mailbox_attribute)(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r); + + void (*send_change)(struct dsync_ibc *ibc, + const struct dsync_mail_change *change); + enum dsync_ibc_recv_ret + (*recv_change)(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r); + + void (*send_mail_request)(struct dsync_ibc *ibc, + const struct dsync_mail_request *request); + enum dsync_ibc_recv_ret + (*recv_mail_request)(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r); + + void (*send_mail)(struct dsync_ibc *ibc, + const struct dsync_mail *mail); + enum dsync_ibc_recv_ret + (*recv_mail)(struct dsync_ibc *ibc, + struct dsync_mail **mail_r); + + void (*send_finish)(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync); + enum dsync_ibc_recv_ret + (*recv_finish)(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r); + + void (*close_mail_streams)(struct dsync_ibc *ibc); + bool (*is_send_queue_full)(struct dsync_ibc *ibc); + bool (*has_pending_data)(struct dsync_ibc *ibc); +}; + +struct dsync_ibc { + struct dsync_ibc_vfuncs v; + + io_callback_t *io_callback; + void *io_context; + + bool failed:1; + bool timeout:1; +}; + +#endif diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c new file mode 100644 index 0000000..17115b0 --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc-stream.c @@ -0,0 +1,2138 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "safe-mkstemp.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-seekable.h" +#include "istream-dot.h" +#include "ostream.h" +#include "str.h" +#include "strescape.h" +#include "master-service.h" +#include "mail-cache.h" +#include "mail-storage-private.h" +#include "dsync-serializer.h" +#include "dsync-deserializer.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc-private.h" + + +#define DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE (1024*128) + +#define DSYNC_PROTOCOL_VERSION_MAJOR 3 +#define DSYNC_PROTOCOL_VERSION_MINOR 5 +#define DSYNC_HANDSHAKE_VERSION "VERSION\tdsync\t3\t5\n" + +#define DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES 1 +#define DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID 2 +#define DSYNC_PROTOCOL_MINOR_HAVE_FINISH 3 +#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2 4 +#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3 5 + +enum item_type { + ITEM_NONE, + ITEM_DONE, + + ITEM_HANDSHAKE, + ITEM_MAILBOX_STATE, + ITEM_MAILBOX_TREE_NODE, + ITEM_MAILBOX_DELETE, + ITEM_MAILBOX, + + ITEM_MAILBOX_ATTRIBUTE, + ITEM_MAIL_CHANGE, + ITEM_MAIL_REQUEST, + ITEM_MAIL, + ITEM_FINISH, + + ITEM_MAILBOX_CACHE_FIELD, + + ITEM_END_OF_LIST +}; + +#define END_OF_LIST_LINE "." +static const struct { + /* full human readable name of the item */ + const char *name; + /* unique character identifying the item */ + char chr; + const char *required_keys; + const char *optional_keys; + unsigned int min_minor_version; +} items[ITEM_END_OF_LIST+1] = { + { NULL, '\0', NULL, NULL, 0 }, + { .name = "done", + .chr = 'X', + .optional_keys = "" + }, + { .name = "handshake", + .chr = 'H', + .required_keys = "hostname", + .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type " + "debug sync_visible_namespaces exclude_mailboxes " + "send_mail_requests backup_send backup_recv lock_timeout " + "no_mail_sync no_backup_overwrite purge_remote " + "no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp " + "virtual_all_box empty_hdr_workaround import_commit_msgs_interval " + "hashed_headers alt_char" + }, + { .name = "mailbox_state", + .chr = 'S', + .required_keys = "mailbox_guid last_uidvalidity last_common_uid " + "last_common_modseq last_common_pvt_modseq", + .optional_keys = "last_messages_count changes_during_sync" + }, + { .name = "mailbox_tree_node", + .chr = 'N', + .required_keys = "name existence", + .optional_keys = "mailbox_guid uid_validity uid_next " + "last_renamed_or_created subscribed last_subscription_change" + }, + { .name = "mailbox_delete", + .chr = 'D', + .required_keys = "hierarchy_sep", + .optional_keys = "escape_char mailboxes dirs unsubscribes" + }, + { .name = "mailbox", + .chr = 'B', + .required_keys = "mailbox_guid uid_validity uid_next messages_count " + "first_recent_uid highest_modseq highest_pvt_modseq", + .optional_keys = "mailbox_lost mailbox_ignore " + "cache_fields have_guids have_save_guids have_only_guid128" + }, + { .name = "mailbox_attribute", + .chr = 'A', + .required_keys = "type key", + .optional_keys = "value stream deleted last_change modseq", + .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES + }, + { .name = "mail_change", + .chr = 'C', + .required_keys = "type uid", + .optional_keys = "guid hdr_hash modseq pvt_modseq " + "add_flags remove_flags final_flags " + "keywords_reset keyword_changes received_timestamp virtual_size" + }, + { .name = "mail_request", + .chr = 'R', + .optional_keys = "guid uid" + }, + { .name = "mail", + .chr = 'M', + .optional_keys = "guid uid pop3_uidl pop3_order received_date saved_date stream" + }, + { .name = "finish", + .chr = 'F', + .optional_keys = "error mail_error require_full_resync", + .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_FINISH + }, + { .name = "mailbox_cache_field", + .chr = 'c', + .required_keys = "name decision", + .optional_keys = "last_used" + }, + + { "end_of_list", '\0', NULL, NULL, 0 } +}; + +struct dsync_ibc_stream { + struct dsync_ibc ibc; + + char *name, *temp_path_prefix; + unsigned int timeout_secs; + struct istream *input; + struct ostream *output; + struct io *io; + struct timeout *to; + + unsigned int minor_version; + struct dsync_serializer *serializers[ITEM_END_OF_LIST]; + struct dsync_deserializer *deserializers[ITEM_END_OF_LIST]; + + pool_t ret_pool; + struct dsync_deserializer_decoder *cur_decoder; + + struct istream *value_output, *value_input; + struct dsync_mail *cur_mail; + struct dsync_mailbox_attribute *cur_attr; + char value_output_last; + + enum item_type last_recv_item, last_sent_item; + bool last_recv_item_eol:1; + bool last_sent_item_eol:1; + + bool version_received:1; + bool handshake_received:1; + bool has_pending_data:1; + bool finish_received:1; + bool done_received:1; + bool stopped:1; +}; + +static const char *dsync_ibc_stream_get_state(struct dsync_ibc_stream *ibc) +{ + if (!ibc->version_received) + return "version not received"; + else if (!ibc->handshake_received) + return "handshake not received"; + + return t_strdup_printf("last sent=%s%s, last recv=%s%s", + items[ibc->last_sent_item].name, + ibc->last_sent_item_eol ? " (EOL)" : "", + items[ibc->last_recv_item].name, + ibc->last_recv_item_eol ? " (EOL)" : ""); +} + +static void dsync_ibc_stream_stop(struct dsync_ibc_stream *ibc) +{ + ibc->stopped = TRUE; + i_stream_close(ibc->input); + o_stream_close(ibc->output); + io_loop_stop(current_ioloop); +} + +static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc) +{ + do { + i_stream_skip(ibc->value_input, + i_stream_get_data_size(ibc->value_input)); + } while (i_stream_read(ibc->value_input) > 0); + if (ibc->value_input->eof) { + if (ibc->value_input->stream_errno != 0) { + i_error("dsync(%s): read(%s) failed: %s (%s)", ibc->name, + i_stream_get_name(ibc->value_input), + i_stream_get_error(ibc->value_input), + dsync_ibc_stream_get_state(ibc)); + dsync_ibc_stream_stop(ibc); + return -1; + } + /* finished reading the mail stream */ + i_assert(ibc->value_input->eof); + i_stream_seek(ibc->value_input, 0); + ibc->has_pending_data = TRUE; + ibc->value_input = NULL; + return 1; + } + return 0; +} + +static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc) +{ + timeout_reset(ibc->to); + if (ibc->value_input != NULL) { + if (dsync_ibc_stream_read_mail_stream(ibc) == 0) + return; + } + o_stream_cork(ibc->output); + ibc->ibc.io_callback(ibc->ibc.io_context); + o_stream_uncork(ibc->output); +} + +static int dsync_ibc_stream_send_value_stream(struct dsync_ibc_stream *ibc) +{ + const unsigned char *data; + unsigned char add; + size_t i, size; + int ret; + + while ((ret = i_stream_read_more(ibc->value_output, &data, &size)) > 0) { + add = '\0'; + for (i = 0; i < size; i++) { + if (data[i] == '.' && + ((i == 0 && ibc->value_output_last == '\n') || + (i > 0 && data[i-1] == '\n'))) { + /* escape the dot */ + add = '.'; + break; + } + } + + if (i > 0) { + o_stream_nsend(ibc->output, data, i); + ibc->value_output_last = data[i-1]; + i_stream_skip(ibc->value_output, i); + } + + if (o_stream_get_buffer_used_size(ibc->output) >= 4096) { + if ((ret = o_stream_flush(ibc->output)) < 0) { + dsync_ibc_stream_stop(ibc); + return -1; + } + if (ret == 0) { + /* continue later */ + o_stream_set_flush_pending(ibc->output, TRUE); + return 0; + } + } + + if (add != '\0') { + o_stream_nsend(ibc->output, &add, 1); + ibc->value_output_last = add; + } + } + i_assert(ret == -1); + + if (ibc->value_output->stream_errno != 0) { + i_error("dsync(%s): read(%s) failed: %s (%s)", + ibc->name, i_stream_get_name(ibc->value_output), + i_stream_get_error(ibc->value_output), + dsync_ibc_stream_get_state(ibc)); + dsync_ibc_stream_stop(ibc); + return -1; + } + + /* finished sending the stream. use "CRLF." instead of "LF." just in + case we're sending binary data that ends with CR. */ + o_stream_nsend_str(ibc->output, "\r\n.\r\n"); + i_stream_unref(&ibc->value_output); + return 1; +} + +static int dsync_ibc_stream_output(struct dsync_ibc_stream *ibc) +{ + struct ostream *output = ibc->output; + int ret; + + if ((ret = o_stream_flush(output)) < 0) + ret = 1; + else if (ibc->value_output != NULL) { + if (dsync_ibc_stream_send_value_stream(ibc) < 0) + ret = 1; + } + timeout_reset(ibc->to); + + if (!dsync_ibc_is_send_queue_full(&ibc->ibc)) + ibc->ibc.io_callback(ibc->ibc.io_context); + return ret; +} + +static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc) +{ + i_error("dsync(%s): I/O has stalled, no activity for %u seconds (%s)", + ibc->name, ibc->timeout_secs, dsync_ibc_stream_get_state(ibc)); + ibc->ibc.timeout = TRUE; + dsync_ibc_stream_stop(ibc); +} + +static void dsync_ibc_stream_init(struct dsync_ibc_stream *ibc) +{ + unsigned int i; + + ibc->io = io_add_istream(ibc->input, dsync_ibc_stream_input, ibc); + io_set_pending(ibc->io); + o_stream_set_no_error_handling(ibc->output, TRUE); + o_stream_set_flush_callback(ibc->output, dsync_ibc_stream_output, ibc); + ibc->to = timeout_add(ibc->timeout_secs * 1000, + dsync_ibc_stream_timeout, ibc); + o_stream_cork(ibc->output); + o_stream_nsend_str(ibc->output, DSYNC_HANDSHAKE_VERSION); + + /* initialize serializers and send their headers to remote */ + for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) T_BEGIN { + const char *keys; + + keys = items[i].required_keys == NULL ? items[i].optional_keys : + t_strconcat(items[i].required_keys, " ", + items[i].optional_keys, NULL); + if (keys != NULL) { + i_assert(items[i].chr != '\0'); + + ibc->serializers[i] = + dsync_serializer_init(t_strsplit_spaces(keys, " ")); + o_stream_nsend(ibc->output, &items[i].chr, 1); + o_stream_nsend_str(ibc->output, + dsync_serializer_encode_header_line(ibc->serializers[i])); + } + } T_END; + o_stream_nsend_str(ibc->output, ".\n"); + o_stream_uncork(ibc->output); +} + +static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + unsigned int i; + + for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) { + if (ibc->serializers[i] != NULL) + dsync_serializer_deinit(&ibc->serializers[i]); + if (ibc->deserializers[i] != NULL) + dsync_deserializer_deinit(&ibc->deserializers[i]); + } + if (ibc->cur_decoder != NULL) + dsync_deserializer_decode_finish(&ibc->cur_decoder); + if (ibc->value_output != NULL) + i_stream_unref(&ibc->value_output); + else { + /* If the remote has not told us that they are closing we + notify remote that we're closing. this is mainly to avoid + "read() failed: EOF" errors on failing dsyncs. + + Avoid a race condition: + We do not tell the remote we are closing if they have + already told us because they close the + connection after sending ITEM_DONE and will + not be ever receive anything else from us unless + it just happens to get combined into the same packet + as a previous response and is already in the buffer. + */ + if (!ibc->done_received && !ibc->finish_received) { + o_stream_nsend_str(ibc->output, + t_strdup_printf("%c\n", items[ITEM_DONE].chr)); + } + (void)o_stream_finish(ibc->output); + } + + timeout_remove(&ibc->to); + io_remove(&ibc->io); + i_stream_destroy(&ibc->input); + o_stream_destroy(&ibc->output); + pool_unref(&ibc->ret_pool); + i_free(ibc->temp_path_prefix); + i_free(ibc->name); + i_free(ibc); +} + +static int dsync_ibc_stream_next_line(struct dsync_ibc_stream *ibc, + const char **line_r) +{ + string_t *error; + const char *line; + ssize_t ret; + + line = i_stream_next_line(ibc->input); + if (line != NULL) { + *line_r = line; + return 1; + } + /* try reading some */ + if ((ret = i_stream_read(ibc->input)) == -1) { + if (ibc->stopped) + return -1; + error = t_str_new(128); + if (ibc->input->stream_errno != 0) { + str_printfa(error, "read(%s) failed: %s", ibc->name, + i_stream_get_error(ibc->input)); + } else { + i_assert(ibc->input->eof); + str_printfa(error, "read(%s) failed: EOF", ibc->name); + } + str_printfa(error, " (%s)", dsync_ibc_stream_get_state(ibc)); + i_error("%s", str_c(error)); + dsync_ibc_stream_stop(ibc); + return -1; + } + i_assert(ret >= 0); + *line_r = i_stream_next_line(ibc->input); + if (*line_r == NULL) { + ibc->has_pending_data = FALSE; + return 0; + } + ibc->has_pending_data = TRUE; + return 1; +} + +static void ATTR_FORMAT(3, 4) ATTR_NULL(2) +dsync_ibc_input_error(struct dsync_ibc_stream *ibc, + struct dsync_deserializer_decoder *decoder, + const char *fmt, ...) +{ + va_list args; + const char *error; + + va_start(args, fmt); + error = t_strdup_vprintf(fmt, args); + if (decoder == NULL) + i_error("dsync(%s): %s", ibc->name, error); + else { + i_error("dsync(%s): %s: %s", ibc->name, + dsync_deserializer_decoder_get_name(decoder), error); + } + va_end(args); + + dsync_ibc_stream_stop(ibc); +} + +static void +dsync_ibc_stream_send_string(struct dsync_ibc_stream *ibc, + const string_t *str) +{ + i_assert(ibc->value_output == NULL); + o_stream_nsend(ibc->output, str_data(str), str_len(str)); +} + +static int seekable_fd_callback(const char **path_r, void *context) +{ + struct dsync_ibc_stream *ibc = context; + string_t *path; + int fd; + + path = t_str_new(128); + str_append(path, ibc->temp_path_prefix); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (i_unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_close_fd(&fd); + return -1; + } + + *path_r = str_c(path); + return fd; +} + +static struct istream * +dsync_ibc_stream_input_stream(struct dsync_ibc_stream *ibc) +{ + struct istream *inputs[2]; + + inputs[0] = i_stream_create_dot(ibc->input, FALSE); + inputs[1] = NULL; + ibc->value_input = i_stream_create_seekable(inputs, MAIL_READ_FULL_BLOCK_SIZE, + seekable_fd_callback, ibc); + i_stream_unref(&inputs[0]); + + return ibc->value_input; +} + +static int +dsync_ibc_check_missing_deserializers(struct dsync_ibc_stream *ibc) +{ + unsigned int i; + int ret = 0; + + for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) { + if (ibc->deserializers[i] == NULL && + ibc->minor_version >= items[i].min_minor_version && + (items[i].required_keys != NULL || + items[i].optional_keys != NULL)) { + dsync_ibc_input_error(ibc, NULL, + "Remote didn't handshake deserializer for %s", + items[i].name); + ret = -1; + } + } + return ret; +} + +static bool +dsync_ibc_stream_handshake(struct dsync_ibc_stream *ibc, const char *line) +{ + enum item_type item = ITEM_NONE; + const char *const *required_keys, *error; + unsigned int i; + + if (ibc->handshake_received) + return TRUE; + + if (!ibc->version_received) { + if (!version_string_verify_full(line, "dsync", + DSYNC_PROTOCOL_VERSION_MAJOR, + &ibc->minor_version)) { + dsync_ibc_input_error(ibc, NULL, + "Remote dsync doesn't use compatible protocol"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + ibc->version_received = TRUE; + return FALSE; + } + + if (strcmp(line, END_OF_LIST_LINE) == 0) { + /* finished handshaking */ + if (dsync_ibc_check_missing_deserializers(ibc) < 0) + return FALSE; + ibc->handshake_received = TRUE; + ibc->last_recv_item = ITEM_HANDSHAKE; + return FALSE; + } + + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (items[i].chr == line[0]) { + item = i; + break; + } + } + if (item == ITEM_NONE) { + /* unknown deserializer, ignore */ + return FALSE; + } + + required_keys = items[item].required_keys == NULL ? NULL : + t_strsplit(items[item].required_keys, " "); + if (dsync_deserializer_init(items[item].name, + required_keys, line + 1, + &ibc->deserializers[item], &error) < 0) { + dsync_ibc_input_error(ibc, NULL, + "Remote sent invalid handshake for %s: %s", + items[item].name, error); + } + return FALSE; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_input_next(struct dsync_ibc_stream *ibc, enum item_type item, + struct dsync_deserializer_decoder **decoder_r) +{ + enum item_type line_item = ITEM_NONE; + const char *line, *error; + unsigned int i; + + i_assert(ibc->value_input == NULL); + + timeout_reset(ibc->to); + + do { + if (dsync_ibc_stream_next_line(ibc, &line) <= 0) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } while (!dsync_ibc_stream_handshake(ibc, line)); + + ibc->last_recv_item = item; + ibc->last_recv_item_eol = FALSE; + + if (strcmp(line, END_OF_LIST_LINE) == 0) { + /* end of this list */ + ibc->last_recv_item_eol = TRUE; + return DSYNC_IBC_RECV_RET_FINISHED; + } + if (line[0] == items[ITEM_DONE].chr) { + /* remote cleanly closed the connection, possibly because of + some failure (which it should have logged). we don't want to + log any stream errors anyway after this. */ + ibc->done_received = TRUE; + dsync_ibc_stream_stop(ibc); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + } + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (*line == items[i].chr) { + line_item = i; + break; + } + } + if (line_item != item) { + dsync_ibc_input_error(ibc, NULL, + "Received unexpected input %c != %c", + *line, items[item].chr); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (ibc->cur_decoder != NULL) + dsync_deserializer_decode_finish(&ibc->cur_decoder); + if (dsync_deserializer_decode_begin(ibc->deserializers[item], + line+1, &ibc->cur_decoder, + &error) < 0) { + dsync_ibc_input_error(ibc, NULL, "Invalid input to %s: %s", + items[item].name, error); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *decoder_r = ibc->cur_decoder; + return DSYNC_IBC_RECV_RET_OK; +} + +static struct dsync_serializer_encoder * +dsync_ibc_send_encode_begin(struct dsync_ibc_stream *ibc, enum item_type item) +{ + ibc->last_sent_item = item; + ibc->last_sent_item_eol = FALSE; + return dsync_serializer_encode_begin(ibc->serializers[item]); +} + +static void +dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc, + const struct dsync_ibc_settings *set) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char sync_type[2]; + + str_append_c(str, items[ITEM_HANDSHAKE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_HANDSHAKE); + dsync_serializer_encode_add(encoder, "hostname", set->hostname); + if (set->sync_ns_prefixes != NULL) { + dsync_serializer_encode_add(encoder, "sync_ns_prefix", + set->sync_ns_prefixes); + } + if (set->sync_box != NULL) + dsync_serializer_encode_add(encoder, "sync_box", set->sync_box); + if (set->virtual_all_box != NULL) { + dsync_serializer_encode_add(encoder, "virtual_all_box", + set->virtual_all_box); + } + if (set->exclude_mailboxes != NULL) { + string_t *substr = t_str_new(64); + unsigned int i; + + for (i = 0; set->exclude_mailboxes[i] != NULL; i++) { + if (i != 0) + str_append_c(substr, '\t'); + str_append_tabescaped(substr, set->exclude_mailboxes[i]); + } + dsync_serializer_encode_add(encoder, "exclude_mailboxes", + str_c(substr)); + } + if (!guid_128_is_empty(set->sync_box_guid)) { + dsync_serializer_encode_add(encoder, "sync_box_guid", + guid_128_to_string(set->sync_box_guid)); + } + + sync_type[0] = sync_type[1] = '\0'; + switch (set->sync_type) { + case DSYNC_BRAIN_SYNC_TYPE_UNKNOWN: + break; + case DSYNC_BRAIN_SYNC_TYPE_FULL: + sync_type[0] = 'f'; + break; + case DSYNC_BRAIN_SYNC_TYPE_CHANGED: + sync_type[0] = 'c'; + break; + case DSYNC_BRAIN_SYNC_TYPE_STATE: + sync_type[0] = 's'; + break; + } + if (sync_type[0] != '\0') + dsync_serializer_encode_add(encoder, "sync_type", sync_type); + if (set->lock_timeout > 0) { + dsync_serializer_encode_add(encoder, "lock_timeout", + t_strdup_printf("%u", set->lock_timeout)); + } + if (set->import_commit_msgs_interval > 0) { + dsync_serializer_encode_add(encoder, "import_commit_msgs_interval", + t_strdup_printf("%u", set->import_commit_msgs_interval)); + } + if (set->sync_since_timestamp > 0) { + dsync_serializer_encode_add(encoder, "sync_since_timestamp", + t_strdup_printf("%ld", (long)set->sync_since_timestamp)); + } + if (set->sync_until_timestamp > 0) { + dsync_serializer_encode_add(encoder, "sync_until_timestamp", + t_strdup_printf("%ld", (long)set->sync_since_timestamp)); + } + if (set->sync_max_size > 0) { + dsync_serializer_encode_add(encoder, "sync_max_size", + t_strdup_printf("%"PRIu64, set->sync_max_size)); + } + if (set->sync_flags != NULL) { + dsync_serializer_encode_add(encoder, "sync_flags", + set->sync_flags); + } + if (set->alt_char != '\0') { + dsync_serializer_encode_add(encoder, "alt_char", + t_strdup_printf("%c", set->alt_char)); + } + if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0) + dsync_serializer_encode_add(encoder, "send_mail_requests", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0) + dsync_serializer_encode_add(encoder, "backup_send", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0) + dsync_serializer_encode_add(encoder, "backup_recv", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_DEBUG) != 0) + dsync_serializer_encode_add(encoder, "debug", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0) + dsync_serializer_encode_add(encoder, "sync_visible_namespaces", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0) + dsync_serializer_encode_add(encoder, "no_mail_sync", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0) + dsync_serializer_encode_add(encoder, "no_backup_overwrite", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0) + dsync_serializer_encode_add(encoder, "purge_remote", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0) + dsync_serializer_encode_add(encoder, "no_notify", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0) + dsync_serializer_encode_add(encoder, "empty_hdr_workaround", ""); + /* this can be NULL in slave */ + string_t *str2 = t_str_new(32); + if (set->hashed_headers != NULL) { + for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) { + str_append_tabescaped(str2, *ptr); + str_append_c(str2, '\t'); + } + } + dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2)); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc, + const struct dsync_ibc_settings **set_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + struct dsync_ibc_settings *set; + const char *value; + pool_t pool = ibc->ret_pool; + enum dsync_ibc_recv_ret ret; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_HANDSHAKE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) { + if (ret != DSYNC_IBC_RECV_RET_TRYAGAIN) { + i_error("dsync(%s): Unexpected input in handshake", + ibc->name); + dsync_ibc_stream_stop(ibc); + } + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + p_clear(pool); + set = p_new(pool, struct dsync_ibc_settings, 1); + + value = dsync_deserializer_decode_get(decoder, "hostname"); + set->hostname = p_strdup(pool, value); + /* now that we know the remote's hostname, use it for the + stream's name */ + i_free(ibc->name); + ibc->name = i_strdup(set->hostname); + + if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value)) + set->sync_ns_prefixes = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "sync_box", &value)) + set->sync_box = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "virtual_all_box", &value)) + set->virtual_all_box = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "sync_box_guid", &value) && + guid_128_from_string(value, set->sync_box_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_box_guid: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "exclude_mailboxes", &value) && + *value != '\0') { + char **boxes = p_strsplit_tabescaped(pool, value); + set->exclude_mailboxes = (const void *)boxes; + } + if (dsync_deserializer_decode_try(decoder, "sync_type", &value)) { + switch (value[0]) { + case 'f': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + break; + case 'c': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + break; + case 's': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE; + break; + default: + dsync_ibc_input_error(ibc, decoder, + "Unknown sync_type: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "lock_timeout", &value)) { + if (str_to_uint(value, &set->lock_timeout) < 0 || + set->lock_timeout == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid lock_timeout: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "import_commit_msgs_interval", &value)) { + if (str_to_uint(value, &set->import_commit_msgs_interval) < 0 || + set->import_commit_msgs_interval == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid import_commit_msgs_interval: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) { + if (str_to_time(value, &set->sync_since_timestamp) < 0 || + set->sync_since_timestamp == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_since_timestamp: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_until_timestamp", &value)) { + if (str_to_time(value, &set->sync_until_timestamp) < 0 || + set->sync_until_timestamp == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_until_timestamp: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) { + if (str_to_uoff(value, &set->sync_max_size) < 0 || + set->sync_max_size == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_max_size: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_flags", &value)) + set->sync_flags = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "alt_char", &value)) + set->alt_char = value[0]; + if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS; + if (dsync_deserializer_decode_try(decoder, "backup_send", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + if (dsync_deserializer_decode_try(decoder, "backup_recv", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + if (dsync_deserializer_decode_try(decoder, "debug", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; + if (dsync_deserializer_decode_try(decoder, "sync_visible_namespaces", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES; + if (dsync_deserializer_decode_try(decoder, "no_mail_sync", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC; + if (dsync_deserializer_decode_try(decoder, "no_backup_overwrite", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; + if (dsync_deserializer_decode_try(decoder, "purge_remote", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; + if (dsync_deserializer_decode_try(decoder, "no_notify", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY; + if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; + if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value)) + set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value); + set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2; + set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3; + + *set_r = set; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_end_of_list(struct dsync_ibc *_ibc, + enum dsync_ibc_eol_type type) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + i_assert(ibc->value_output == NULL); + + switch (type) { + case DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE: + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) + return; + break; + default: + break; + } + + ibc->last_sent_item_eol = TRUE; + o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n"); +} + +static void +dsync_ibc_stream_send_mailbox_state(struct dsync_ibc *_ibc, + const struct dsync_mailbox_state *state) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_MAILBOX_STATE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_STATE); + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(state->mailbox_guid)); + dsync_serializer_encode_add(encoder, "last_uidvalidity", + dec2str(state->last_uidvalidity)); + dsync_serializer_encode_add(encoder, "last_common_uid", + dec2str(state->last_common_uid)); + dsync_serializer_encode_add(encoder, "last_common_modseq", + dec2str(state->last_common_modseq)); + dsync_serializer_encode_add(encoder, "last_common_pvt_modseq", + dec2str(state->last_common_pvt_modseq)); + dsync_serializer_encode_add(encoder, "last_messages_count", + dec2str(state->last_messages_count)); + if (state->changes_during_sync) + dsync_serializer_encode_add(encoder, "changes_during_sync", ""); + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_state(struct dsync_ibc *_ibc, + struct dsync_mailbox_state *state_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + const char *value; + enum dsync_ibc_recv_ret ret; + + i_zero(state_r); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_STATE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "mailbox_guid"); + if (guid_128_from_string(value, state_r->mailbox_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_uidvalidity"); + if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_uidvalidity"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_uid"); + if (str_to_uint32(value, &state_r->last_common_uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_modseq"); + if (str_to_uint64(value, &state_r->last_common_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_pvt_modseq"); + if (str_to_uint64(value, &state_r->last_common_pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_messages_count", &value) && + str_to_uint32(value, &state_r->last_messages_count) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "changes_during_sync", &value)) + state_r->changes_during_sync = TRUE; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mailbox_tree_node(struct dsync_ibc *_ibc, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str, *namestr; + + i_assert(*name != NULL); + + str = t_str_new(128); + str_append_c(str, items[ITEM_MAILBOX_TREE_NODE].chr); + + /* convert all hierarchy separators to tabs. mailbox names really + aren't supposed to have any tabs, but escape them anyway if there + are. */ + namestr = t_str_new(128); + for (; *name != NULL; name++) { + str_append_tabescaped(namestr, *name); + str_append_c(namestr, '\t'); + } + str_truncate(namestr, str_len(namestr)-1); + + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_TREE_NODE); + dsync_serializer_encode_add(encoder, "name", str_c(namestr)); + switch (node->existence) { + case DSYNC_MAILBOX_NODE_NONEXISTENT: + dsync_serializer_encode_add(encoder, "existence", "n"); + break; + case DSYNC_MAILBOX_NODE_EXISTS: + dsync_serializer_encode_add(encoder, "existence", "y"); + break; + case DSYNC_MAILBOX_NODE_DELETED: + dsync_serializer_encode_add(encoder, "existence", "d"); + break; + } + + if (!guid_128_is_empty(node->mailbox_guid)) { + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(node->mailbox_guid)); + } + if (node->uid_validity != 0) { + dsync_serializer_encode_add(encoder, "uid_validity", + dec2str(node->uid_validity)); + } + if (node->uid_next != 0) { + dsync_serializer_encode_add(encoder, "uid_next", + dec2str(node->uid_next)); + } + if (node->last_renamed_or_created != 0) { + dsync_serializer_encode_add(encoder, "last_renamed_or_created", + dec2str(node->last_renamed_or_created)); + } + if (node->last_subscription_change != 0) { + dsync_serializer_encode_add(encoder, "last_subscription_change", + dec2str(node->last_subscription_change)); + } + if (node->subscribed) + dsync_serializer_encode_add(encoder, "subscribed", ""); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_tree_node(struct dsync_ibc *_ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox_node *node; + const char *value; + enum dsync_ibc_recv_ret ret; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_TREE_NODE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + p_clear(ibc->ret_pool); + node = p_new(ibc->ret_pool, struct dsync_mailbox_node, 1); + + value = dsync_deserializer_decode_get(decoder, "name"); + if (*value == '\0') { + dsync_ibc_input_error(ibc, decoder, "Empty name"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *name_r = (void *)p_strsplit_tabescaped(ibc->ret_pool, value); + + value = dsync_deserializer_decode_get(decoder, "existence"); + switch (*value) { + case 'n': + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + break; + case 'y': + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + break; + case 'd': + node->existence = DSYNC_MAILBOX_NODE_DELETED; + break; + } + + if (dsync_deserializer_decode_try(decoder, "mailbox_guid", &value) && + guid_128_from_string(value, node->mailbox_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "uid_validity", &value) && + str_to_uint32(value, &node->uid_validity) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "uid_next", &value) && + str_to_uint32(value, &node->uid_next) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_next"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_renamed_or_created", &value) && + str_to_time(value, &node->last_renamed_or_created) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_renamed_or_created"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_subscription_change", &value) && + str_to_time(value, &node->last_subscription_change) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_subscription_change"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "subscribed", &value)) + node->subscribed = TRUE; + + *node_r = node; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_encode_delete(string_t *str, + struct dsync_serializer_encoder *encoder, + const struct dsync_mailbox_delete *deletes, + unsigned int count, const char *key, + enum dsync_mailbox_delete_type type) +{ + unsigned int i; + + str_truncate(str, 0); + for (i = 0; i < count; i++) { + if (deletes[i].type == type) { + str_append(str, guid_128_to_string(deletes[i].guid)); + str_printfa(str, " %ld ", (long)deletes[i].timestamp); + } + } + if (str_len(str) > 0) { + str_truncate(str, str_len(str)-1); + dsync_serializer_encode_add(encoder, key, str_c(str)); + } +} + +static void +dsync_ibc_stream_send_mailbox_deletes(struct dsync_ibc *_ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str, *substr; + char sep[2]; + + str = t_str_new(128); + str_append_c(str, items[ITEM_MAILBOX_DELETE].chr); + + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_DELETE); + sep[0] = hierarchy_sep; sep[1] = '\0'; + dsync_serializer_encode_add(encoder, "hierarchy_sep", sep); + sep[0] = escape_char; sep[1] = '\0'; + dsync_serializer_encode_add(encoder, "escape_char", sep); + + substr = t_str_new(128); + dsync_ibc_stream_encode_delete(substr, encoder, deletes, count, + "mailboxes", + DSYNC_MAILBOX_DELETE_TYPE_MAILBOX); + dsync_ibc_stream_encode_delete(substr, encoder, deletes, count, + "dirs", + DSYNC_MAILBOX_DELETE_TYPE_DIR); + dsync_ibc_stream_encode_delete(substr, encoder, deletes, count, + "unsubscribes", + DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +ARRAY_DEFINE_TYPE(dsync_mailbox_delete, struct dsync_mailbox_delete); +static int +decode_mailbox_deletes(ARRAY_TYPE(dsync_mailbox_delete) *deletes, + const char *value, enum dsync_mailbox_delete_type type) +{ + struct dsync_mailbox_delete *del; + const char *const *tmp; + unsigned int i; + + tmp = t_strsplit(value, " "); + for (i = 0; tmp[i] != NULL; i += 2) { + del = array_append_space(deletes); + del->type = type; + if (guid_128_from_string(tmp[i], del->guid) < 0) + return -1; + if (tmp[i+1] == NULL || + str_to_time(tmp[i+1], &del->timestamp) < 0) + return -1; + } + return 0; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_deletes(struct dsync_ibc *_ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r, + char *escape_char_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + ARRAY_TYPE(dsync_mailbox_delete) deletes; + const char *value; + enum dsync_ibc_recv_ret ret; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_DELETE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + p_clear(ibc->ret_pool); + p_array_init(&deletes, ibc->ret_pool, 16); + + value = dsync_deserializer_decode_get(decoder, "hierarchy_sep"); + if (strlen(value) != 1) { + dsync_ibc_input_error(ibc, decoder, "Invalid hierarchy_sep"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *hierarchy_sep_r = value[0]; + + if (!dsync_deserializer_decode_try(decoder, "escape_char", &value)) + *escape_char_r = '\0'; + else { + if (strlen(value) > 1) { + dsync_ibc_input_error(ibc, decoder, "Invalid escape_char '%s'", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *escape_char_r = value[0]; + } + + if (dsync_deserializer_decode_try(decoder, "mailboxes", &value) && + decode_mailbox_deletes(&deletes, value, + DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailboxes"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "dirs", &value) && + decode_mailbox_deletes(&deletes, value, + DSYNC_MAILBOX_DELETE_TYPE_DIR) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid dirs"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "unsubscribes", &value) && + decode_mailbox_deletes(&deletes, value, + DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid dirs"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *deletes_r = array_get(&deletes, count_r); + return DSYNC_IBC_RECV_RET_OK; +} + +static const char * +get_cache_fields(struct dsync_ibc_stream *ibc, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_serializer_encoder *encoder; + string_t *str; + const struct mailbox_cache_field *cache_fields; + unsigned int i, count; + char decision[3]; + + cache_fields = array_get(&dsync_box->cache_fields, &count); + if (count == 0) + return ""; + + str = t_str_new(128); + for (i = 0; i < count; i++) { + const struct mailbox_cache_field *field = &cache_fields[i]; + + encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_CACHE_FIELD]); + dsync_serializer_encode_add(encoder, "name", field->name); + + memset(decision, 0, sizeof(decision)); + switch (field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) { + case MAIL_CACHE_DECISION_NO: + decision[0] = 'n'; + break; + case MAIL_CACHE_DECISION_TEMP: + decision[0] = 't'; + break; + case MAIL_CACHE_DECISION_YES: + decision[0] = 'y'; + break; + } + i_assert(decision[0] != '\0'); + if ((field->decision & MAIL_CACHE_DECISION_FORCED) != 0) + decision[1] = 'F'; + dsync_serializer_encode_add(encoder, "decision", decision); + if (field->last_used != 0) { + dsync_serializer_encode_add(encoder, "last_used", + dec2str(field->last_used)); + } + dsync_serializer_encode_finish(&encoder, str); + } + if (i > 0) { + /* remove the trailing LF */ + str_truncate(str, str_len(str)-1); + } + return str_c(str); +} + +static void +dsync_ibc_stream_send_mailbox(struct dsync_ibc *_ibc, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + const char *value; + + str_append_c(str, items[ITEM_MAILBOX].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX); + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(dsync_box->mailbox_guid)); + + if (dsync_box->mailbox_lost) + dsync_serializer_encode_add(encoder, "mailbox_lost", ""); + if (dsync_box->mailbox_ignore) + dsync_serializer_encode_add(encoder, "mailbox_ignore", ""); + if (dsync_box->have_guids) + dsync_serializer_encode_add(encoder, "have_guids", ""); + if (dsync_box->have_save_guids) + dsync_serializer_encode_add(encoder, "have_save_guids", ""); + if (dsync_box->have_only_guid128) + dsync_serializer_encode_add(encoder, "have_only_guid128", ""); + dsync_serializer_encode_add(encoder, "uid_validity", + dec2str(dsync_box->uid_validity)); + dsync_serializer_encode_add(encoder, "uid_next", + dec2str(dsync_box->uid_next)); + dsync_serializer_encode_add(encoder, "messages_count", + dec2str(dsync_box->messages_count)); + dsync_serializer_encode_add(encoder, "first_recent_uid", + dec2str(dsync_box->first_recent_uid)); + dsync_serializer_encode_add(encoder, "highest_modseq", + dec2str(dsync_box->highest_modseq)); + dsync_serializer_encode_add(encoder, "highest_pvt_modseq", + dec2str(dsync_box->highest_pvt_modseq)); + + value = get_cache_fields(ibc, dsync_box); + if (value != NULL) + dsync_serializer_encode_add(encoder, "cache_fields", value); + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static int +parse_cache_field(struct dsync_ibc_stream *ibc, struct dsync_mailbox *box, + const char *value) +{ + struct dsync_deserializer_decoder *decoder; + struct mailbox_cache_field field; + const char *error; + int ret = 0; + + if (dsync_deserializer_decode_begin(ibc->deserializers[ITEM_MAILBOX_CACHE_FIELD], + value, &decoder, &error) < 0) { + dsync_ibc_input_error(ibc, NULL, + "cache_field: Invalid input: %s", error); + return -1; + } + + i_zero(&field); + value = dsync_deserializer_decode_get(decoder, "name"); + field.name = p_strdup(ibc->ret_pool, value); + + value = dsync_deserializer_decode_get(decoder, "decision"); + switch (*value) { + case 'n': + field.decision = MAIL_CACHE_DECISION_NO; + break; + case 't': + field.decision = MAIL_CACHE_DECISION_TEMP; + break; + case 'y': + field.decision = MAIL_CACHE_DECISION_YES; + break; + default: + dsync_ibc_input_error(ibc, decoder, "Invalid decision: %s", + value); + ret = -1; + break; + } + if (value[1] == 'F') + field.decision |= MAIL_CACHE_DECISION_FORCED; + + if (dsync_deserializer_decode_try(decoder, "last_used", &value) && + str_to_time(value, &field.last_used) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_used"); + ret = -1; + } + array_push_back(&box->cache_fields, &field); + + dsync_deserializer_decode_finish(&decoder); + return ret; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox(struct dsync_ibc *_ibc, + const struct dsync_mailbox **dsync_box_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox *box; + const char *value; + enum dsync_ibc_recv_ret ret; + + p_clear(pool); + box = p_new(pool, struct dsync_mailbox, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "mailbox_guid"); + if (guid_128_from_string(value, box->mailbox_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "mailbox_lost", &value)) + box->mailbox_lost = TRUE; + if (dsync_deserializer_decode_try(decoder, "mailbox_ignore", &value)) + box->mailbox_ignore = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_guids", &value)) + box->have_guids = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_save_guids", &value) || + (box->have_guids && ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID)) + box->have_save_guids = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_only_guid128", &value)) + box->have_only_guid128 = TRUE; + value = dsync_deserializer_decode_get(decoder, "uid_validity"); + if (str_to_uint32(value, &box->uid_validity) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "uid_next"); + if (str_to_uint32(value, &box->uid_next) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_next"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "messages_count"); + if (str_to_uint32(value, &box->messages_count) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid messages_count"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "first_recent_uid"); + if (str_to_uint32(value, &box->first_recent_uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid first_recent_uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "highest_modseq"); + if (str_to_uint64(value, &box->highest_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid highest_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "highest_pvt_modseq"); + if (str_to_uint64(value, &box->highest_pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid highest_pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + p_array_init(&box->cache_fields, pool, 32); + if (dsync_deserializer_decode_try(decoder, "cache_fields", &value)) { + const char *const *fields = t_strsplit(value, "\n"); + for (; *fields != NULL; fields++) { + if (parse_cache_field(ibc, box, *fields) < 0) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + + *dsync_box_r = box; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mailbox_attribute(struct dsync_ibc *_ibc, + const struct dsync_mailbox_attribute *attr) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char type[2]; + + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) + return; + + str_append_c(str, items[ITEM_MAILBOX_ATTRIBUTE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_ATTRIBUTE); + + type[0] = type[1] = '\0'; + switch (attr->type) { + case MAIL_ATTRIBUTE_TYPE_PRIVATE: + type[0] = 'p'; + break; + case MAIL_ATTRIBUTE_TYPE_SHARED: + type[0] = 's'; + break; + } + i_assert(type[0] != '\0'); + dsync_serializer_encode_add(encoder, "type", type); + dsync_serializer_encode_add(encoder, "key", attr->key); + if (attr->value != NULL) + dsync_serializer_encode_add(encoder, "value", attr->value); + else if (attr->value_stream != NULL) + dsync_serializer_encode_add(encoder, "stream", ""); + + if (attr->deleted) + dsync_serializer_encode_add(encoder, "deleted", ""); + if (attr->last_change != 0) { + dsync_serializer_encode_add(encoder, "last_change", + dec2str(attr->last_change)); + } + if (attr->modseq != 0) { + dsync_serializer_encode_add(encoder, "modseq", + dec2str(attr->modseq)); + } + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); + + if (attr->value_stream != NULL) { + ibc->value_output_last = '\0'; + ibc->value_output = attr->value_stream; + i_stream_ref(ibc->value_output); + (void)dsync_ibc_stream_send_value_stream(ibc); + } +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_attribute(struct dsync_ibc *_ibc, + const struct dsync_mailbox_attribute **attr_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox_attribute *attr; + const char *value; + enum dsync_ibc_recv_ret ret; + + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) + return DSYNC_IBC_RECV_RET_FINISHED; + + if (ibc->value_input != NULL) { + /* wait until the mail's stream has been read */ + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (ibc->cur_attr != NULL) { + /* finished reading the stream, return the mail now */ + *attr_r = ibc->cur_attr; + ibc->cur_attr = NULL; + return DSYNC_IBC_RECV_RET_OK; + } + + p_clear(pool); + attr = p_new(pool, struct dsync_mailbox_attribute, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_ATTRIBUTE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "type"); + switch (*value) { + case 'p': + attr->type = MAIL_ATTRIBUTE_TYPE_PRIVATE; + break; + case 's': + attr->type = MAIL_ATTRIBUTE_TYPE_SHARED; + break; + default: + dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + value = dsync_deserializer_decode_get(decoder, "key"); + attr->key = p_strdup(pool, value); + + if (dsync_deserializer_decode_try(decoder, "deleted", &value)) + attr->deleted = TRUE; + if (dsync_deserializer_decode_try(decoder, "last_change", &value) && + str_to_time(value, &attr->last_change) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_change"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "modseq", &value) && + str_to_uint64(value, &attr->modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + /* NOTE: stream reading must be the last here, because reading a large + stream will be finished later by return TRYAGAIN. We need to + deserialize all the other fields before that or they'll get lost. */ + if (dsync_deserializer_decode_try(decoder, "stream", &value)) { + attr->value_stream = dsync_ibc_stream_input_stream(ibc); + if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { + ibc->cur_attr = attr; + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + /* already finished reading the stream */ + i_assert(ibc->value_input == NULL); + } else if (dsync_deserializer_decode_try(decoder, "value", &value)) + attr->value = p_strdup(pool, value); + + *attr_r = attr; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_change(struct dsync_ibc *_ibc, + const struct dsync_mail_change *change) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char type[2]; + + str_append_c(str, items[ITEM_MAIL_CHANGE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_CHANGE); + + type[0] = type[1] = '\0'; + switch (change->type) { + case DSYNC_MAIL_CHANGE_TYPE_SAVE: + type[0] = 's'; + break; + case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE: + type[0] = 'e'; + break; + case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE: + type[0] = 'f'; + break; + } + i_assert(type[0] != '\0'); + dsync_serializer_encode_add(encoder, "type", type); + dsync_serializer_encode_add(encoder, "uid", dec2str(change->uid)); + if (change->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", change->guid); + if (change->hdr_hash != NULL) { + dsync_serializer_encode_add(encoder, "hdr_hash", + change->hdr_hash); + } + if (change->modseq != 0) { + dsync_serializer_encode_add(encoder, "modseq", + dec2str(change->modseq)); + } + if (change->pvt_modseq != 0) { + dsync_serializer_encode_add(encoder, "pvt_modseq", + dec2str(change->pvt_modseq)); + } + if (change->add_flags != 0) { + dsync_serializer_encode_add(encoder, "add_flags", + t_strdup_printf("%x", change->add_flags)); + } + if (change->remove_flags != 0) { + dsync_serializer_encode_add(encoder, "remove_flags", + t_strdup_printf("%x", change->remove_flags)); + } + if (change->final_flags != 0) { + dsync_serializer_encode_add(encoder, "final_flags", + t_strdup_printf("%x", change->final_flags)); + } + if (change->keywords_reset) + dsync_serializer_encode_add(encoder, "keywords_reset", ""); + + if (array_is_created(&change->keyword_changes) && + array_count(&change->keyword_changes) > 0) { + string_t *kw_str = t_str_new(128); + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + str_append_tabescaped(kw_str, changes[0]); + for (i = 1; i < count; i++) { + str_append_c(kw_str, '\t'); + str_append_tabescaped(kw_str, changes[i]); + } + dsync_serializer_encode_add(encoder, "keyword_changes", + str_c(kw_str)); + } + if (change->received_timestamp > 0) { + dsync_serializer_encode_add(encoder, "received_timestamp", + t_strdup_printf("%"PRIxTIME_T, change->received_timestamp)); + } + if (change->virtual_size > 0) { + dsync_serializer_encode_add(encoder, "virtual_size", + t_strdup_printf("%llx", (unsigned long long)change->virtual_size)); + } + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc, + const struct dsync_mail_change **change_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail_change *change; + const char *value; + unsigned int uintval; + unsigned long long ullongval; + enum dsync_ibc_recv_ret ret; + + p_clear(pool); + change = p_new(pool, struct dsync_mail_change, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_CHANGE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "type"); + switch (*value) { + case 's': + change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE; + break; + case 'e': + change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE; + break; + case 'f': + change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE; + break; + default: + dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + value = dsync_deserializer_decode_get(decoder, "uid"); + if (str_to_uint32(value, &change->uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + change->guid = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "hdr_hash", &value)) + change->hdr_hash = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "modseq", &value) && + str_to_uint64(value, &change->modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "pvt_modseq", &value) && + str_to_uint64(value, &change->pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid add_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->add_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid remove_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->remove_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid final_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->final_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "keywords_reset", &value)) + change->keywords_reset = TRUE; + + if (dsync_deserializer_decode_try(decoder, "keyword_changes", &value) && + *value != '\0') { + const char *const *changes = t_strsplit_tabescaped(value); + unsigned int i, count = str_array_length(changes); + + p_array_init(&change->keyword_changes, pool, count); + for (i = 0; i < count; i++) { + value = p_strdup(pool, changes[i]); + array_push_back(&change->keyword_changes, &value); + } + } + if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value)) { + if (str_to_ullong_hex(value, &ullongval) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->received_timestamp = ullongval; + } + if (dsync_deserializer_decode_try(decoder, "virtual_size", &value)) { + if (str_to_ullong_hex(value, &ullongval) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->virtual_size = ullongval; + } + + *change_r = change; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mail_request(struct dsync_ibc *_ibc, + const struct dsync_mail_request *request) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_MAIL_REQUEST].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_REQUEST); + if (request->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", request->guid); + if (request->uid != 0) { + dsync_serializer_encode_add(encoder, "uid", + dec2str(request->uid)); + } + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mail_request(struct dsync_ibc *_ibc, + const struct dsync_mail_request **request_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail_request *request; + const char *value; + enum dsync_ibc_recv_ret ret; + + p_clear(ibc->ret_pool); + request = p_new(ibc->ret_pool, struct dsync_mail_request, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_REQUEST, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + request->guid = p_strdup(ibc->ret_pool, value); + if (dsync_deserializer_decode_try(decoder, "uid", &value) && + str_to_uint32(value, &request->uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + *request_r = request; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mail(struct dsync_ibc *_ibc, + const struct dsync_mail *mail) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + i_assert(!mail->minimal_fields); + i_assert(ibc->value_output == NULL); + + str_append_c(str, items[ITEM_MAIL].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL); + if (mail->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", mail->guid); + if (mail->uid != 0) + dsync_serializer_encode_add(encoder, "uid", dec2str(mail->uid)); + if (mail->pop3_uidl != NULL) { + dsync_serializer_encode_add(encoder, "pop3_uidl", + mail->pop3_uidl); + } + if (mail->pop3_order > 0) { + dsync_serializer_encode_add(encoder, "pop3_order", + dec2str(mail->pop3_order)); + } + if (mail->received_date > 0) { + dsync_serializer_encode_add(encoder, "received_date", + dec2str(mail->received_date)); + } + if (mail->saved_date != 0) { + dsync_serializer_encode_add(encoder, "saved_date", + dec2str(mail->saved_date)); + } + if (mail->input != NULL) + dsync_serializer_encode_add(encoder, "stream", ""); + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); + + if (mail->input != NULL) { + ibc->value_output_last = '\0'; + ibc->value_output = mail->input; + i_stream_ref(ibc->value_output); + (void)dsync_ibc_stream_send_value_stream(ibc); + } +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail *mail; + const char *value; + enum dsync_ibc_recv_ret ret; + + if (ibc->value_input != NULL) { + /* wait until the mail's stream has been read */ + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (ibc->cur_mail != NULL) { + /* finished reading the stream, return the mail now */ + *mail_r = ibc->cur_mail; + ibc->cur_mail = NULL; + return DSYNC_IBC_RECV_RET_OK; + } + + p_clear(pool); + mail = p_new(pool, struct dsync_mail, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + mail->guid = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "uid", &value) && + str_to_uint32(value, &mail->uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "pop3_uidl", &value)) + mail->pop3_uidl = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "pop3_order", &value) && + str_to_uint32(value, &mail->pop3_order) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid pop3_order"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "received_date", &value) && + str_to_time(value, &mail->received_date) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid received_date"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "saved_date", &value) && + str_to_time(value, &mail->saved_date) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid saved_date"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "stream", &value)) { + mail->input = dsync_ibc_stream_input_stream(ibc); + if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { + ibc->cur_mail = mail; + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + /* already finished reading the stream */ + i_assert(ibc->value_input == NULL); + } + + *mail_r = mail; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_FINISH].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_FINISH); + if (error != NULL) + dsync_serializer_encode_add(encoder, "error", error); + if (mail_error != 0) { + dsync_serializer_encode_add(encoder, "mail_error", + dec2str(mail_error)); + } + if (require_full_resync) + dsync_serializer_encode_add(encoder, "require_full_resync", ""); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + const char *value; + enum dsync_ibc_recv_ret ret; + int i = 0; + + *error_r = NULL; + *mail_error_r = 0; + *require_full_resync_r = FALSE; + + p_clear(ibc->ret_pool); + + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_FINISH) + return DSYNC_IBC_RECV_RET_OK; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_FINISH, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "error", &value)) + *error_r = p_strdup(ibc->ret_pool, value); + if (dsync_deserializer_decode_try(decoder, "mail_error", &value) && + str_to_int(value, &i) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mail_error"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "require_full_resync", &value)) + *require_full_resync_r = TRUE; + *mail_error_r = i; + + ibc->finish_received = TRUE; + return DSYNC_IBC_RECV_RET_OK; +} + +static void dsync_ibc_stream_close_mail_streams(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + if (ibc->value_output != NULL) { + i_stream_unref(&ibc->value_output); + dsync_ibc_stream_stop(ibc); + } +} + +static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + size_t bytes; + + if (ibc->value_output != NULL) + return TRUE; + + bytes = o_stream_get_buffer_used_size(ibc->output); + if (bytes < DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE) + return FALSE; + + o_stream_set_flush_pending(ibc->output, TRUE); + return TRUE; +} + +static bool dsync_ibc_stream_has_pending_data(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + return ibc->has_pending_data; +} + +static const struct dsync_ibc_vfuncs dsync_ibc_stream_vfuncs = { + dsync_ibc_stream_deinit, + dsync_ibc_stream_send_handshake, + dsync_ibc_stream_recv_handshake, + dsync_ibc_stream_send_end_of_list, + dsync_ibc_stream_send_mailbox_state, + dsync_ibc_stream_recv_mailbox_state, + dsync_ibc_stream_send_mailbox_tree_node, + dsync_ibc_stream_recv_mailbox_tree_node, + dsync_ibc_stream_send_mailbox_deletes, + dsync_ibc_stream_recv_mailbox_deletes, + dsync_ibc_stream_send_mailbox, + dsync_ibc_stream_recv_mailbox, + dsync_ibc_stream_send_mailbox_attribute, + dsync_ibc_stream_recv_mailbox_attribute, + dsync_ibc_stream_send_change, + dsync_ibc_stream_recv_change, + dsync_ibc_stream_send_mail_request, + dsync_ibc_stream_recv_mail_request, + dsync_ibc_stream_send_mail, + dsync_ibc_stream_recv_mail, + dsync_ibc_stream_send_finish, + dsync_ibc_stream_recv_finish, + dsync_ibc_stream_close_mail_streams, + dsync_ibc_stream_is_send_queue_full, + dsync_ibc_stream_has_pending_data +}; + +struct dsync_ibc * +dsync_ibc_init_stream(struct istream *input, struct ostream *output, + const char *name, const char *temp_path_prefix, + unsigned int timeout_secs) +{ + struct dsync_ibc_stream *ibc; + + ibc = i_new(struct dsync_ibc_stream, 1); + ibc->ibc.v = dsync_ibc_stream_vfuncs; + ibc->input = input; + ibc->output = output; + i_stream_ref(ibc->input); + o_stream_ref(ibc->output); + ibc->name = i_strdup(name); + ibc->temp_path_prefix = i_strdup(temp_path_prefix); + ibc->timeout_secs = timeout_secs; + ibc->ret_pool = pool_alloconly_create("ibc stream data", 2048); + dsync_ibc_stream_init(ibc); + return &ibc->ibc; +} diff --git a/src/doveadm/dsync/dsync-ibc.c b/src/doveadm/dsync/dsync-ibc.c new file mode 100644 index 0000000..ede7b83 --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dsync-mail.h" +#include "dsync-ibc-private.h" + +void dsync_ibc_deinit(struct dsync_ibc **_ibc) +{ + struct dsync_ibc *ibc = *_ibc; + + *_ibc = NULL; + ibc->v.deinit(ibc); +} + +void dsync_ibc_set_io_callback(struct dsync_ibc *ibc, + io_callback_t *callback, void *context) +{ + ibc->io_callback = callback; + ibc->io_context = context; +} + +void dsync_ibc_send_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set) +{ + ibc->v.send_handshake(ibc, set); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r) +{ + return ibc->v.recv_handshake(ibc, set_r); +} + +static enum dsync_ibc_send_ret +dsync_ibc_send_ret(struct dsync_ibc *ibc) +{ + return ibc->v.is_send_queue_full(ibc) ? + DSYNC_IBC_SEND_RET_FULL : + DSYNC_IBC_SEND_RET_OK; +} + +enum dsync_ibc_send_ret +dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type) +{ + ibc->v.send_end_of_list(ibc, type); + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state) +{ + T_BEGIN { + ibc->v.send_mailbox_state(ibc, state); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r) +{ + return ibc->v.recv_mailbox_state(ibc, state_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + i_assert(*name != NULL); + + T_BEGIN { + ibc->v.send_mailbox_tree_node(ibc, name, node); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + return ibc->v.recv_mailbox_tree_node(ibc, name_r, node_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char) +{ + T_BEGIN { + ibc->v.send_mailbox_deletes(ibc, deletes, count, + hierarchy_sep, escape_char); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r, + char *escape_char_r) +{ + return ibc->v.recv_mailbox_deletes(ibc, deletes_r, count_r, + hierarchy_sep_r, escape_char_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box) +{ + T_BEGIN { + ibc->v.send_mailbox(ibc, dsync_box); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r) +{ + return ibc->v.recv_mailbox(ibc, dsync_box_r); +} + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr) +{ + T_BEGIN { + ibc->v.send_mailbox_attribute(ibc, attr); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r) +{ + return ibc->v.recv_mailbox_attribute(ibc, attr_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_change(struct dsync_ibc *ibc, + const struct dsync_mail_change *change) +{ + i_assert(change->uid > 0); + + T_BEGIN { + ibc->v.send_change(ibc, change); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_change(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r) +{ + return ibc->v.recv_change(ibc, change_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request *request) +{ + i_assert(request->guid != NULL || request->uid != 0); + + T_BEGIN { + ibc->v.send_mail_request(ibc, request); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r) +{ + return ibc->v.recv_mail_request(ibc, request_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail) +{ + i_assert(*mail->guid != '\0' || mail->uid != 0); + + T_BEGIN { + ibc->v.send_mail(ibc, mail); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r) +{ + return ibc->v.recv_mail(ibc, mail_r); +} + +void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync) +{ + ibc->v.send_finish(ibc, error, mail_error, require_full_resync); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r) +{ + return ibc->v.recv_finish(ibc, error_r, mail_error_r, + require_full_resync_r); +} + +void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc) +{ + ibc->v.close_mail_streams(ibc); +} + +bool dsync_ibc_has_failed(struct dsync_ibc *ibc) +{ + return ibc->failed; +} + +bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc) +{ + return ibc->timeout; +} + +bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc) +{ + return ibc->v.is_send_queue_full(ibc); +} + +bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc) +{ + return ibc->v.has_pending_data(ibc); +} diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h new file mode 100644 index 0000000..dc85560 --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc.h @@ -0,0 +1,178 @@ +#ifndef DSYNC_IBC_H +#define DSYNC_IBC_H + +/* dsync inter-brain communicator */ + +#include "ioloop.h" +#include "guid.h" +#include "mail-error.h" +#include "dsync-brain.h" + +struct dsync_mailbox; +struct dsync_mailbox_state; +struct dsync_mailbox_node; +struct dsync_mailbox_delete; +struct dsync_mailbox_attribute; +struct dsync_mail; +struct dsync_mail_change; +struct dsync_mail_request; + +enum dsync_ibc_send_ret { + DSYNC_IBC_SEND_RET_OK = 1, + /* send queue is full, stop sending more */ + DSYNC_IBC_SEND_RET_FULL = 0 +}; + +enum dsync_ibc_recv_ret { + DSYNC_IBC_RECV_RET_FINISHED = -1, + /* try again / error (the error handling delayed until io callback) */ + DSYNC_IBC_RECV_RET_TRYAGAIN = 0, + DSYNC_IBC_RECV_RET_OK = 1 +}; + +enum dsync_ibc_eol_type { + DSYNC_IBC_EOL_MAILBOX_STATE, + DSYNC_IBC_EOL_MAILBOX_TREE, + DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE, + DSYNC_IBC_EOL_MAILBOX, + DSYNC_IBC_EOL_MAIL_CHANGES, + DSYNC_IBC_EOL_MAIL_REQUESTS, + DSYNC_IBC_EOL_MAILS +}; + +struct dsync_ibc_settings { + /* Server hostname. Used for determining which server does the + locking. */ + const char *hostname; + /* if non-NULL, sync only these namespaces (LF-separated) */ + const char *sync_ns_prefixes; + /* if non-NULL, sync only this mailbox name */ + const char *sync_box; + /* if non-NULL, use this mailbox for finding messages with GUIDs and + copying them instead of saving them again. */ + const char *virtual_all_box; + /* if non-empty, sync only this mailbox GUID */ + guid_128_t sync_box_guid; + /* Exclude these mailboxes from the sync. They can contain '*' + wildcards and be \special-use flags. */ + const char *const *exclude_mailboxes; + /* Sync only mails with received timestamp at least this high. */ + time_t sync_since_timestamp; + /* Sync only mails with received timestamp less or equal than this */ + time_t sync_until_timestamp; + /* Don't sync mails larger than this. */ + uoff_t sync_max_size; + /* Sync only mails with specified flags. */ + const char *sync_flags; + /* Hashed headers */ + const char *const *hashed_headers; + + char alt_char; + enum dsync_brain_sync_type sync_type; + enum dsync_brain_flags brain_flags; + bool hdr_hash_v2; + bool hdr_hash_v3; + unsigned int lock_timeout; + unsigned int import_commit_msgs_interval; +}; + +void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r, + struct dsync_ibc **ibc2_r); +struct dsync_ibc * +dsync_ibc_init_stream(struct istream *input, struct ostream *output, + const char *name, const char *temp_path_prefix, + unsigned int timeout_secs); +void dsync_ibc_deinit(struct dsync_ibc **ibc); + +/* I/O callback is called whenever new data is available. It's also called on + errors, so check first the error status. */ +void dsync_ibc_set_io_callback(struct dsync_ibc *ibc, + io_callback_t *callback, void *context); + +void dsync_ibc_send_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set); +enum dsync_ibc_recv_ret +dsync_ibc_recv_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r, + char *escape_char_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_change(struct dsync_ibc *ibc, + const struct dsync_mail_change *change); +enum dsync_ibc_recv_ret +dsync_ibc_recv_change(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request *request); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r); + +void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync); +enum dsync_ibc_recv_ret +dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r); + +/* Close any mail input streams that are kept open. This needs to be called + before the mail is attempted to be freed (usually on error conditions). */ +void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc); + +bool dsync_ibc_has_failed(struct dsync_ibc *ibc); +bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc); +bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc); +bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc); + +#endif diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c new file mode 100644 index 0000000..b82818f --- /dev/null +++ b/src/doveadm/dsync/dsync-mail.c @@ -0,0 +1,156 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hex-binary.h" +#include "md5.h" +#include "istream.h" +#include "istream-crlf.h" +#include "message-header-hash.h" +#include "message-size.h" +#include "mail-storage.h" +#include "dsync-mail.h" + +struct mailbox_header_lookup_ctx * +dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers) +{ + return mailbox_header_lookup_init(box, hashed_headers); +} + +int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, + const char *const *hashed_headers, const char **hdr_hash_r) +{ + struct istream *hdr_input, *input; + struct mailbox_header_lookup_ctx *hdr_ctx; + struct message_header_hash_context hash_ctx; + struct md5_context md5_ctx; + unsigned char md5_result[MD5_RESULTLEN]; + const unsigned char *data; + size_t size; + ssize_t sret; + int ret = 0; + + hdr_ctx = mailbox_header_lookup_init(mail->box, hashed_headers); + ret = mail_get_header_stream(mail, hdr_ctx, &hdr_input); + mailbox_header_lookup_unref(&hdr_ctx); + if (ret < 0) + return -1; + + input = i_stream_create_lf(hdr_input); + + md5_init(&md5_ctx); + i_zero(&hash_ctx); + while ((sret = i_stream_read_more(input, &data, &size)) > 0) { + message_header_hash_more(&hash_ctx, &hash_method_md5, &md5_ctx, + version, data, size); + i_stream_skip(input, size); + } + i_assert(sret == -1); + if (input->stream_errno != 0) + ret = -1; + i_stream_unref(&input); + + md5_final(&md5_ctx, md5_result); + *hdr_hash_r = binary_to_hex(md5_result, sizeof(md5_result)); + return ret; +} + +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r) +{ + const char *guid; + + i_zero(dmail_r); + + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) { + *error_field_r = "GUID"; + return -1; + } + dmail_r->guid = guid; + dmail_r->uid = mail->uid; + + dmail_r->input_mail = mail; + dmail_r->input_mail_uid = mail->uid; + + if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { + *error_field_r = "saved-date"; + return -1; + } + if (!minimal_fill) + return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r); + dmail_r->minimal_fields = TRUE; + return 0; +} + +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r) +{ + const char *str; + + if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) { + *error_field_r = "body"; + return -1; + } + + if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail_r->pop3_uidl) < 0) { + *error_field_r = "pop3-uidl"; + return -1; + } + if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) { + *error_field_r = "pop3-order"; + return -1; + } + if (*str != '\0') { + if (str_to_uint32(str, &dmail_r->pop3_order) < 0) + i_unreached(); + } + if (mail_get_received_date(mail, &dmail_r->received_date) < 0) { + *error_field_r = "received-date"; + return -1; + } + return 0; +} + +static void +const_string_array_dup(pool_t pool, const ARRAY_TYPE(const_string) *src, + ARRAY_TYPE(const_string) *dest) +{ + const char *const *strings, *str; + unsigned int i, count; + + if (!array_is_created(src)) + return; + + strings = array_get(src, &count); + if (count == 0) + return; + + p_array_init(dest, pool, count); + for (i = 0; i < count; i++) { + str = p_strdup(pool, strings[i]); + array_push_back(dest, &str); + } +} + +void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, + struct dsync_mail_change *dest_r) +{ + dest_r->type = src->type; + dest_r->uid = src->uid; + if (src->guid != NULL) { + dest_r->guid = *src->guid == '\0' ? "" : + p_strdup(pool, src->guid); + } + dest_r->hdr_hash = p_strdup(pool, src->hdr_hash); + dest_r->modseq = src->modseq; + dest_r->pvt_modseq = src->pvt_modseq; + + dest_r->add_flags = src->add_flags; + dest_r->remove_flags = src->remove_flags; + dest_r->final_flags = src->final_flags; + dest_r->keywords_reset = src->keywords_reset; + const_string_array_dup(pool, &src->keyword_changes, + &dest_r->keyword_changes); + dest_r->received_timestamp = src->received_timestamp; + dest_r->virtual_size = src->virtual_size; +} diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h new file mode 100644 index 0000000..fc00059 --- /dev/null +++ b/src/doveadm/dsync/dsync-mail.h @@ -0,0 +1,108 @@ +#ifndef DSYNC_MAIL_H +#define DSYNC_MAIL_H + +#include "mail-types.h" + +struct md5_context; +struct mail; +struct mailbox; + +struct dsync_mail { + /* either GUID="" or uid=0 */ + const char *guid; + uint32_t uid; + time_t saved_date; + + /* If non-NULL, we're syncing within the dsync process using ibc-pipe. + This mail can be used to mailbox_copy() the mail. */ + struct mail *input_mail; + /* Verify that this equals to input_mail->uid */ + uint32_t input_mail_uid; + + /* TRUE if the following fields aren't set, because minimal_fill=TRUE + parameter was used. */ + bool minimal_fields; + + const char *pop3_uidl; + uint32_t pop3_order; + time_t received_date; + /* Input stream containing the message text, or NULL if all instances + of the message were already expunged from this mailbox. */ + struct istream *input; +}; + +struct dsync_mail_request { + /* either GUID=NULL or uid=0 */ + const char *guid; + uint32_t uid; +}; + +enum dsync_mail_change_type { + DSYNC_MAIL_CHANGE_TYPE_SAVE, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE +}; + +#define KEYWORD_CHANGE_ADD '+' +#define KEYWORD_CHANGE_REMOVE '-' +#define KEYWORD_CHANGE_FINAL '=' +#define KEYWORD_CHANGE_ADD_AND_FINAL '&' + +struct dsync_mail_change { + enum dsync_mail_change_type type; + + uint32_t uid; + /* Message's GUID: + - for expunges either 128bit hex or NULL if unknown + - "" if backend doesn't support GUIDs */ + const char *guid; + /* If GUID is "", this contains hash of the message header, + otherwise NULL */ + const char *hdr_hash; + + /* Message's current modseq (saves, flag changes) */ + uint64_t modseq; + /* Message's current private modseq (for private flags in + shared mailboxes, otherwise 0) */ + uint64_t pvt_modseq; + + /* List of flag/keyword changes: (saves, flag changes) */ + + /* Flags added/removed since last sync, and final flags containing + flags that exist now but haven't changed */ + uint8_t add_flags, remove_flags, final_flags; + uint8_t add_pvt_flags, remove_pvt_flags; + /* Remove all keywords before applying changes. This is used only with + old transaction logs, new ones never reset keywords (just explicitly + remove unwanted keywords) */ + bool keywords_reset; + /* +add, -remove, =final, &add_and_final. */ + ARRAY_TYPE(const_string) keyword_changes; + + /* Received timestamp for saves, if brain.sync_since/until_timestamp is set */ + time_t received_timestamp; + /* Mail's size for saves if brain.sync_max_size is set, + UOFF_T_MAX otherwise. */ + uoff_t virtual_size; +}; + +struct mailbox_header_lookup_ctx * +dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers); + +int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, + const char *const *hashed_headers, const char **hdr_hash_r); +static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash) +{ + /* md5(\n) */ + return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0; +} + +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r); +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r); + +void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, + struct dsync_mail_change *dest_r); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c new file mode 100644 index 0000000..a58fbec --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-export.c @@ -0,0 +1,961 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "istream.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-export.h" + +struct dsync_mail_guid_instances { + ARRAY_TYPE(seq_range) seqs; + bool requested; + bool searched; +}; + +struct dsync_mailbox_exporter { + pool_t pool; + struct mailbox *box; + struct dsync_transaction_log_scan *log_scan; + uint32_t last_common_uid; + + struct mailbox_header_lookup_ctx *wanted_headers; + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + unsigned int search_pos, search_count; + unsigned int hdr_hash_version; + + const char *const *hashed_headers; + + /* GUID => instances */ + HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids; + ARRAY_TYPE(seq_range) requested_uids; + ARRAY_TYPE(seq_range) search_uids; + + ARRAY_TYPE(seq_range) expunged_seqs; + ARRAY_TYPE(const_string) expunged_guids; + unsigned int expunged_guid_idx; + + /* uint32_t UID => struct dsync_mail_change */ + HASH_TABLE(void *, struct dsync_mail_change *) changes; + /* changes sorted by UID */ + ARRAY(struct dsync_mail_change *) sorted_changes; + unsigned int change_idx; + uint32_t highest_changed_uid; + + struct mailbox_attribute_iter *attr_iter; + struct hash_iterate_context *attr_change_iter; + enum mail_attribute_type attr_type; + struct dsync_mailbox_attribute attr; + + struct dsync_mail_change change; + struct dsync_mail dsync_mail; + + const char *error; + enum mail_error mail_error; + + bool body_search_initialized:1; + bool auto_export_mails:1; + bool mails_have_guids:1; + bool minimal_dmail_fill:1; + bool return_all_mails:1; + bool export_received_timestamps:1; + bool export_virtual_sizes:1; + bool no_hdr_hashes:1; +}; + +static int dsync_mail_error(struct dsync_mailbox_exporter *exporter, + struct mail *mail, const char *field) +{ + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_internal_error(exporter->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return 0; + + exporter->mail_error = error; + exporter->error = p_strdup_printf(exporter->pool, + "Can't lookup %s for UID=%u: %s", + field, mail->uid, errstr); + return -1; +} + +static bool +final_keyword_check(struct dsync_mail_change *change, const char *name, + char *type_r) +{ + const char *const *changes; + unsigned int i, count; + + *type_r = KEYWORD_CHANGE_FINAL; + + changes = array_get(&change->keyword_changes, &count); + for (i = 0; i < count; i++) { + if (strcmp(changes[i]+1, name) != 0) + continue; + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + /* replace with ADD_AND_FINAL */ + array_delete(&change->keyword_changes, i, 1); + *type_r = KEYWORD_CHANGE_ADD_AND_FINAL; + return FALSE; + case KEYWORD_CHANGE_REMOVE: + /* a final keyword is marked as removed. + this shouldn't normally happen. */ + array_delete(&change->keyword_changes, i, 1); + return FALSE; + case KEYWORD_CHANGE_ADD_AND_FINAL: + case KEYWORD_CHANGE_FINAL: + /* no change */ + return TRUE; + } + } + return FALSE; +} + +static void +search_update_flag_changes(struct dsync_mailbox_exporter *exporter, + struct mail *mail, struct dsync_mail_change *change) +{ + const char *const *keywords; + unsigned int i; + char type; + + i_assert((change->add_flags & change->remove_flags) == 0); + + change->modseq = mail_get_modseq(mail); + change->pvt_modseq = mail_get_pvt_modseq(mail); + change->final_flags = mail_get_flags(mail); + + keywords = mail_get_keywords(mail); + if (!array_is_created(&change->keyword_changes) && + keywords[0] != NULL) { + p_array_init(&change->keyword_changes, exporter->pool, + str_array_length(keywords)); + } + for (i = 0; keywords[i] != NULL; i++) { + /* add the final keyword if it's not already there + as +keyword */ + if (!final_keyword_check(change, keywords[i], &type)) { + const char *keyword_change = + p_strdup_printf(exporter->pool, "%c%s", + type, keywords[i]); + array_push_back(&change->keyword_changes, + &keyword_change); + } + } +} + +static int +exporter_get_guids(struct dsync_mailbox_exporter *exporter, + struct mail *mail, const char **guid_r, + const char **hdr_hash_r) +{ + *guid_r = ""; + *hdr_hash_r = NULL; + + /* always try to get GUID, even if we're also getting header hash */ + if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0) + return dsync_mail_error(exporter, mail, "GUID"); + + if (!exporter->mails_have_guids) { + /* get header hash also */ + if (exporter->no_hdr_hashes) { + *hdr_hash_r = ""; + return 1; + } + if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version, + exporter->hashed_headers, hdr_hash_r) < 0) + return dsync_mail_error(exporter, mail, "hdr-stream"); + return 1; + } else if (**guid_r == '\0') { + exporter->mail_error = MAIL_ERROR_TEMP; + exporter->error = "Backend doesn't support GUIDs, " + "sync with header hashes instead"; + return -1; + } else { + /* GUIDs are required, we don't need header hash */ + return 1; + } +} + +static int +search_update_flag_change_guid(struct dsync_mailbox_exporter *exporter, + struct mail *mail) +{ + struct dsync_mail_change *change, *log_change; + const char *guid, *hdr_hash; + int ret; + + change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid)); + if (change != NULL) { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE); + } else { + i_assert(exporter->return_all_mails); + + change = p_new(exporter->pool, struct dsync_mail_change, 1); + change->uid = mail->uid; + change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE; + hash_table_insert(exporter->changes, + POINTER_CAST(mail->uid), change); + } + + if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0) + return -1; + if (ret == 0) { + /* the message was expunged during export */ + i_zero(change); + change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE; + change->uid = mail->uid; + + /* find its GUID from log if possible */ + log_change = dsync_transaction_log_scan_find_new_expunge( + exporter->log_scan, mail->uid); + if (log_change != NULL) + change->guid = log_change->guid; + } else { + change->guid = *guid == '\0' ? "" : + p_strdup(exporter->pool, guid); + change->hdr_hash = p_strdup(exporter->pool, hdr_hash); + search_update_flag_changes(exporter, mail, change); + } + return 0; +} + +static struct dsync_mail_change * +export_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid) +{ + struct dsync_mail_change *change; + + change = hash_table_lookup(exporter->changes, POINTER_CAST(uid)); + if (change == NULL) { + change = p_new(exporter->pool, struct dsync_mail_change, 1); + change->uid = uid; + hash_table_insert(exporter->changes, POINTER_CAST(uid), change); + } else { + /* move flag changes into a save. this happens only when + last_common_uid isn't known */ + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE); + i_assert(exporter->last_common_uid == 0); + } + + change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE; + return change; +} + +static void +export_add_mail_instance(struct dsync_mailbox_exporter *exporter, + struct dsync_mail_change *change, uint32_t seq) +{ + struct dsync_mail_guid_instances *instances; + + if (exporter->auto_export_mails && !exporter->mails_have_guids) { + /* GUIDs not supported, mail is requested by UIDs */ + seq_range_array_add(&exporter->requested_uids, change->uid); + return; + } + if (*change->guid == '\0') { + /* mail UIDs are manually requested */ + i_assert(!exporter->mails_have_guids); + return; + } + + instances = hash_table_lookup(exporter->export_guids, change->guid); + if (instances == NULL) { + instances = p_new(exporter->pool, + struct dsync_mail_guid_instances, 1); + p_array_init(&instances->seqs, exporter->pool, 2); + hash_table_insert(exporter->export_guids, + p_strdup(exporter->pool, change->guid), + instances); + if (exporter->auto_export_mails) + instances->requested = TRUE; + } + seq_range_array_add(&instances->seqs, seq); +} + +static int +search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail) +{ + struct dsync_mail_change *change; + const char *guid, *hdr_hash; + enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID; + time_t received_timestamp = 0; + uoff_t virtual_size = UOFF_T_MAX; + int ret; + + /* update wanted fields in case we didn't already set them for the + search */ + if (exporter->export_received_timestamps) + wanted_fields |= MAIL_FETCH_RECEIVED_DATE; + if (exporter->export_virtual_sizes) + wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE; + mail_add_temp_wanted_fields(mail, wanted_fields, + exporter->wanted_headers); + + /* If message is already expunged here, just skip it */ + if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0) + return ret; + + if (exporter->export_received_timestamps) { + if (mail_get_received_date(mail, &received_timestamp) < 0) + return dsync_mail_error(exporter, mail, "received-time"); + if (received_timestamp == 0) { + /* don't allow timestamps to be zero. we want to have + asserts verify that the timestamp is set properly. */ + received_timestamp = 1; + } + } + if (exporter->export_virtual_sizes) { + if (mail_get_virtual_size(mail, &virtual_size) < 0) + return dsync_mail_error(exporter, mail, "virtual-size"); + i_assert(virtual_size != UOFF_T_MAX); + } + + change = export_save_change_get(exporter, mail->uid); + change->guid = *guid == '\0' ? "" : + p_strdup(exporter->pool, guid); + change->hdr_hash = p_strdup(exporter->pool, hdr_hash); + change->received_timestamp = received_timestamp; + change->virtual_size = virtual_size; + search_update_flag_changes(exporter, mail, change); + + export_add_mail_instance(exporter, change, mail->seq); + return 1; +} + +static void +dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter, + ARRAY_TYPE(seq_range) *uids) +{ + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change; + + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, exporter->changes, &key, &change)) { + if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE) + seq_range_array_add(uids, change->uid); + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter) +{ + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change; + + /* any flag changes for UIDs above last_common_uid weren't found by + mailbox search, which means they were already expunged. for some + reason the log scanner found flag changes for the message, but not + the expunge. just remove these. */ + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, exporter->changes, &key, &change)) { + if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE && + change->uid > exporter->last_common_uid) + hash_table_remove(exporter->changes, key); + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter) +{ + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + struct mail *mail; + enum mail_fetch_field wanted_fields = 0; + struct mailbox_header_lookup_ctx *wanted_headers = NULL; + int ret = 0; + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&sarg->value.seqset, search_args->pool, 1); + + if (exporter->return_all_mails || exporter->last_common_uid == 0) { + /* we want to know about all mails */ + seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1); + } else { + /* lookup GUIDs for messages with flag changes */ + dsync_mailbox_export_add_flagchange_uids(exporter, + &sarg->value.seqset); + /* lookup new messages */ + seq_range_array_add_range(&sarg->value.seqset, + exporter->last_common_uid + 1, + (uint32_t)-1); + } + + if (exporter->last_common_uid == 0) { + /* we're syncing all mails, so we can request the wanted + fields for all the mails */ + wanted_fields = MAIL_FETCH_GUID; + wanted_headers = exporter->wanted_headers; + } + + exporter->trans = mailbox_transaction_begin(exporter->box, + MAILBOX_TRANSACTION_FLAG_SYNC, + __func__); + search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, + wanted_fields, wanted_headers); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + T_BEGIN { + if (mail->uid <= exporter->last_common_uid) + ret = search_update_flag_change_guid(exporter, mail); + else + ret = search_add_save(exporter, mail); + } T_END; + if (ret < 0) + break; + } + i_assert(ret >= 0 || exporter->error != NULL); + + dsync_mailbox_export_drop_expunged_flag_changes(exporter); + + if (mailbox_search_deinit(&search_ctx) < 0 && + exporter->error == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Mail search failed: %s", + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + } +} + +static int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1, + struct dsync_mail_change *const *c2) +{ + if ((*c1)->uid < (*c2)->uid) + return -1; + if ((*c1)->uid > (*c2)->uid) + return 1; + return 0; +} + +static void +dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter) +{ + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change; + + p_array_init(&exporter->sorted_changes, exporter->pool, + hash_table_count(exporter->changes)); + + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, exporter->changes, &key, &change)) + array_push_back(&exporter->sorted_changes, &change); + hash_table_iterate_deinit(&iter); + array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp); +} + +static void +dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter, + enum mail_attribute_type type) +{ + exporter->attr_iter = + mailbox_attribute_iter_init(exporter->box, type, ""); + exporter->attr_type = type; +} + +static void +dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter, + struct dsync_transaction_log_scan *log_scan) +{ + HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes; + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change, *dup_change; + + log_changes = dsync_transaction_log_scan_get_hash(log_scan); + if (dsync_transaction_log_scan_has_all_changes(log_scan)) { + /* we tried to access too old/invalid modseqs. to make sure + no changes get lost, we need to send all of the messages */ + exporter->return_all_mails = TRUE; + } + + /* clone the hash table, since we're changing it. */ + hash_table_create_direct(&exporter->changes, exporter->pool, + hash_table_count(log_changes)); + iter = hash_table_iterate_init(log_changes); + while (hash_table_iterate(iter, log_changes, &key, &change)) { + dup_change = p_new(exporter->pool, struct dsync_mail_change, 1); + *dup_change = *change; + hash_table_insert(exporter->changes, key, dup_change); + if (exporter->highest_changed_uid < change->uid) + exporter->highest_changed_uid = change->uid; + } + hash_table_iterate_deinit(&iter); +} + +struct dsync_mailbox_exporter * +dsync_mailbox_export_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + enum dsync_mailbox_exporter_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers) +{ + struct dsync_mailbox_exporter *exporter; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export", + 4096); + exporter = p_new(pool, struct dsync_mailbox_exporter, 1); + exporter->pool = pool; + exporter->box = box; + exporter->log_scan = log_scan; + exporter->last_common_uid = last_common_uid; + exporter->auto_export_mails = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0; + exporter->mails_have_guids = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0; + exporter->minimal_dmail_fill = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0; + exporter->export_received_timestamps = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0; + exporter->export_virtual_sizes = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0; + exporter->hdr_hash_version = hdr_hash_version; + exporter->no_hdr_hashes = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0; + exporter->hashed_headers = hashed_headers; + + p_array_init(&exporter->requested_uids, pool, 16); + p_array_init(&exporter->search_uids, pool, 16); + hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp); + p_array_init(&exporter->expunged_seqs, pool, 16); + p_array_init(&exporter->expunged_guids, pool, 16); + + if (!exporter->mails_have_guids && !exporter->no_hdr_hashes) + exporter->wanted_headers = + dsync_mail_get_hash_headers(box, exporter->hashed_headers); + + /* first scan transaction log and save any expunges and flag changes */ + dsync_mailbox_export_log_scan(exporter, log_scan); + /* get saves and also find GUIDs for flag changes */ + dsync_mailbox_export_search(exporter); + /* get the changes sorted by UID */ + dsync_mailbox_export_sort_changes(exporter); + + dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE); + return exporter; +} + +static int +dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter) +{ + struct dsync_mailbox_attribute *attr; + struct mail_attribute_value value; + + while (hash_table_iterate(exporter->attr_change_iter, + dsync_transaction_log_scan_get_attr_hash(exporter->log_scan), + &attr, &attr)) { + if (attr->exported || !attr->deleted) + continue; + + /* lookup the value mainly to get its last_change value. */ + if (mailbox_attribute_get_stream(exporter->box, attr->type, + attr->key, &value) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute %s lookup failed: %s", attr->key, + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + break; + } + if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { + i_stream_unref(&value.value_stream); + continue; + } + + attr->last_change = value.last_change; + if (value.value != NULL || value.value_stream != NULL) { + attr->value = p_strdup(exporter->pool, value.value); + attr->value_stream = value.value_stream; + attr->deleted = FALSE; + } + attr->exported = TRUE; + exporter->attr = *attr; + return 1; + } + hash_table_iterate_deinit(&exporter->attr_change_iter); + return 0; +} + +static int +dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter) +{ + HASH_TABLE_TYPE(dsync_attr_change) attr_changes; + struct dsync_mailbox_attribute lookup_attr, *attr; + struct dsync_mailbox_attribute *attr_change; + const char *key; + struct mail_attribute_value value; + bool export_all_attrs; + + export_all_attrs = exporter->return_all_mails || + exporter->last_common_uid == 0; + attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan); + lookup_attr.type = exporter->attr_type; + + /* note that the order of processing may be important for some + attributes. for example sieve can't set a script active until it's + first been created */ + while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) { + lookup_attr.key = key; + attr_change = hash_table_lookup(attr_changes, &lookup_attr); + if (attr_change == NULL && !export_all_attrs) + continue; + + if (mailbox_attribute_get_stream(exporter->box, + exporter->attr_type, key, + &value) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute %s lookup failed: %s", key, + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + return -1; + } + if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { + /* readonly attributes can't be changed, + no point in exporting them */ + if (value.value_stream != NULL) + i_stream_unref(&value.value_stream); + continue; + } + if (value.value == NULL && value.value_stream == NULL && + (attr_change == NULL || !attr_change->deleted)) { + /* the attribute was just deleted? + skip for this sync. */ + continue; + } + if (attr_change != NULL && attr_change->exported) { + /* duplicate attribute returned. + shouldn't normally happen, but don't crash. */ + i_warning("Ignoring duplicate attributes '%s'", key); + continue; + } + + attr = &exporter->attr; + i_zero(attr); + attr->type = exporter->attr_type; + attr->value = p_strdup(exporter->pool, value.value); + attr->value_stream = value.value_stream; + attr->last_change = value.last_change; + if (attr_change != NULL) { + attr_change->exported = TRUE; + attr->key = attr_change->key; + attr->deleted = attr_change->deleted && + !DSYNC_ATTR_HAS_VALUE(attr); + attr->modseq = attr_change->modseq; + } else { + attr->key = p_strdup(exporter->pool, key); + } + return 1; + } + if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute iteration failed: %s", + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + return -1; + } + if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) { + /* export shared attributes */ + dsync_mailbox_export_attr_init(exporter, + MAIL_ATTRIBUTE_TYPE_SHARED); + return dsync_mailbox_export_iter_next_attr(exporter); + } + exporter->attr_change_iter = hash_table_iterate_init(attr_changes); + return dsync_mailbox_export_iter_next_nonexistent_attr(exporter); +} + +int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter, + const struct dsync_mailbox_attribute **attr_r) +{ + int ret; + + if (exporter->error != NULL) + return -1; + + i_stream_unref(&exporter->attr.value_stream); + + if (exporter->attr_iter != NULL) { + ret = dsync_mailbox_export_iter_next_attr(exporter); + } else { + ret = dsync_mailbox_export_iter_next_nonexistent_attr(exporter); + } + if (ret > 0) + *attr_r = &exporter->attr; + return ret; +} + +int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_change **change_r) +{ + struct dsync_mail_change *const *changes; + unsigned int count; + + if (exporter->error != NULL) + return -1; + + changes = array_get(&exporter->sorted_changes, &count); + if (exporter->change_idx == count) + return 0; + *change_r = changes[exporter->change_idx++]; + return 1; +} + +static int +dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter) +{ + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + struct hash_iterate_context *iter; + const struct seq_range *uids; + char *guid; + const char *const_guid; + enum mail_fetch_field wanted_fields; + struct dsync_mail_guid_instances *instances; + const struct seq_range *range; + unsigned int i, count; + uint32_t seq, seq1, seq2; + + i_assert(exporter->search_ctx == NULL); + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_SEQSET); + p_array_init(&sarg->value.seqset, search_args->pool, 128); + + /* get a list of messages we want to fetch. if there are more than one + instance for a GUID, use the first one. */ + iter = hash_table_iterate_init(exporter->export_guids); + while (hash_table_iterate(iter, exporter->export_guids, + &guid, &instances)) { + if (!instances->requested || + array_count(&instances->seqs) == 0) + continue; + + uids = array_front(&instances->seqs); + seq = uids[0].seq1; + if (!instances->searched) { + instances->searched = TRUE; + seq_range_array_add(&sarg->value.seqset, seq); + } else if (seq_range_exists(&exporter->expunged_seqs, seq)) { + /* we're on a second round, refetching expunged + messages */ + seq_range_array_remove(&instances->seqs, seq); + seq_range_array_remove(&exporter->expunged_seqs, seq); + if (array_count(&instances->seqs) == 0) { + /* no instances left */ + const_guid = guid; + array_push_back(&exporter->expunged_guids, + &const_guid); + continue; + } + uids = array_front(&instances->seqs); + seq = uids[0].seq1; + seq_range_array_add(&sarg->value.seqset, seq); + } + } + hash_table_iterate_deinit(&iter); + + /* add requested UIDs */ + range = array_get(&exporter->requested_uids, &count); + for (i = 0; i < count; i++) { + mailbox_get_seq_range(exporter->box, + range[i].seq1, range[i].seq2, + &seq1, &seq2); + seq_range_array_add_range(&sarg->value.seqset, + seq1, seq2); + } + array_clear(&exporter->search_uids); + array_append_array(&exporter->search_uids, &exporter->requested_uids); + array_clear(&exporter->requested_uids); + + wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE; + if (!exporter->minimal_dmail_fill) { + wanted_fields |= MAIL_FETCH_RECEIVED_DATE | + MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER | + MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY; + } + exporter->search_count += seq_range_count(&sarg->value.seqset); + exporter->search_ctx = + mailbox_search_init(exporter->trans, search_args, NULL, + wanted_fields, NULL); + mail_search_args_unref(&search_args); + return array_count(&sarg->value.seqset) > 0 ? 1 : 0; +} + +static void +dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter) +{ + if (exporter->search_ctx == NULL) + return; + + if (mailbox_search_deinit(&exporter->search_ctx) < 0 && + exporter->error == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Mail search failed: %s", + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + } +} + +static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter, + struct mail *mail) +{ + struct dsync_mail_guid_instances *instances; + const char *error_field; + + if (dsync_mail_fill(mail, exporter->minimal_dmail_fill, + &exporter->dsync_mail, &error_field) < 0) + return dsync_mail_error(exporter, mail, error_field); + + instances = *exporter->dsync_mail.guid == '\0' ? NULL : + hash_table_lookup(exporter->export_guids, + exporter->dsync_mail.guid); + if (instances != NULL) { + /* GUID found */ + } else if (exporter->dsync_mail.uid != 0) { + /* mail requested by UID */ + } else { + exporter->mail_error = MAIL_ERROR_TEMP; + exporter->error = p_strdup_printf(exporter->pool, + "GUID unexpectedly changed for UID=%u GUID=%s", + mail->uid, exporter->dsync_mail.guid); + return -1; + } + + if (!seq_range_exists(&exporter->search_uids, mail->uid)) + exporter->dsync_mail.uid = 0; + else + exporter->dsync_mail.guid = ""; + + /* this message was successfully returned, don't try retrying it */ + if (instances != NULL) + array_clear(&instances->seqs); + return 1; +} + +void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_request *request) +{ + struct dsync_mail_guid_instances *instances; + + i_assert(!exporter->auto_export_mails); + + if (request->guid == NULL) { + i_assert(request->uid > 0); + seq_range_array_add(&exporter->requested_uids, request->uid); + return; + } + + instances = hash_table_lookup(exporter->export_guids, request->guid); + if (instances == NULL) { + exporter->mail_error = MAIL_ERROR_TEMP; + exporter->error = p_strdup_printf(exporter->pool, + "Remote requested unexpected GUID %s", request->guid); + return; + } + instances->requested = TRUE; +} + +int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail **mail_r) +{ + struct mail *mail; + const char *const *guids; + unsigned int count; + int ret; + + if (exporter->error != NULL) + return -1; + if (!exporter->body_search_initialized) { + exporter->body_search_initialized = TRUE; + if (dsync_mailbox_export_body_search_init(exporter) < 0) { + i_assert(exporter->error != NULL); + return -1; + } + } + + while (mailbox_search_next(exporter->search_ctx, &mail)) { + exporter->search_pos++; + if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) { + *mail_r = &exporter->dsync_mail; + return 1; + } + if (ret < 0) { + i_assert(exporter->error != NULL); + return -1; + } + /* the message was expunged. if the GUID has another instance, + try sending it later. */ + seq_range_array_add(&exporter->expunged_seqs, mail->seq); + } + /* if some instances of messages were expunged, retry fetching them + with other instances */ + dsync_mailbox_export_body_search_deinit(exporter); + if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) { + i_assert(exporter->error != NULL); + return -1; + } + if (ret > 0) { + /* not finished yet */ + return dsync_mailbox_export_next_mail(exporter, mail_r); + } + + /* finished with messages. if there are any expunged messages, + return them */ + guids = array_get(&exporter->expunged_guids, &count); + if (exporter->expunged_guid_idx < count) { + i_zero(&exporter->dsync_mail); + exporter->dsync_mail.guid = + guids[exporter->expunged_guid_idx++]; + *mail_r = &exporter->dsync_mail; + return 1; + } + return 0; +} + +int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter, + const char **errstr_r, enum mail_error *error_r) +{ + struct dsync_mailbox_exporter *exporter = *_exporter; + + *_exporter = NULL; + + if (exporter->attr_iter != NULL) + (void)mailbox_attribute_iter_deinit(&exporter->attr_iter); + dsync_mailbox_export_body_search_deinit(exporter); + (void)mailbox_transaction_commit(&exporter->trans); + mailbox_header_lookup_unref(&exporter->wanted_headers); + + i_stream_unref(&exporter->attr.value_stream); + hash_table_destroy(&exporter->export_guids); + hash_table_destroy(&exporter->changes); + + i_assert((exporter->error != NULL) == (exporter->mail_error != 0)); + + *error_r = exporter->mail_error; + *errstr_r = t_strdup(exporter->error); + pool_unref(&exporter->pool); + return *errstr_r != NULL ? -1 : 0; +} + +const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter) +{ + if (exporter->search_ctx == NULL) + return ""; + return t_strdup_printf("%u/%u", exporter->search_pos, + exporter->search_count); +} diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h new file mode 100644 index 0000000..67e3216 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-export.h @@ -0,0 +1,37 @@ +#ifndef DSYNC_MAILBOX_EXPORT_H +#define DSYNC_MAILBOX_EXPORT_H + +enum dsync_mailbox_exporter_flags { + DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01, + DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02, + DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04, + DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS = 0x08, + DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES = 0x20, + DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES = 0x40, +}; + +struct dsync_mailbox_exporter * +dsync_mailbox_export_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + enum dsync_mailbox_exporter_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers); +/* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */ +int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter, + const struct dsync_mailbox_attribute **attr_r); +/* Returns 1 if change was returned, 0 if no more changes, -1 on error */ +int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_change **change_r); + +void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_request *request); +/* Returns 1 if mail was returned, 0 if no more mails, -1 on error */ +int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail **mail_r); +int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **exporter, + const char **errstr_r, enum mail_error *error_r); + +const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c new file mode 100644 index 0000000..8a8fbe4 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -0,0 +1,2997 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "hex-binary.h" +#include "istream.h" +#include "seq-range-array.h" +#include "imap-util.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-import.h" + +struct importer_mail { + const char *guid; + uint32_t uid; +}; + +struct importer_new_mail { + /* linked list of mails for this GUID */ + struct importer_new_mail *next; + /* if non-NULL, this mail exists in both local and remote. this link + points to the other side. */ + struct importer_new_mail *link; + + const char *guid; + struct dsync_mail_change *change; + + /* the final UID for the message */ + uint32_t final_uid; + /* the original local UID, or 0 if exists only remotely */ + uint32_t local_uid; + /* the original remote UID, or 0 if exists only remotely */ + uint32_t remote_uid; + /* UID for the mail in the virtual \All mailbox */ + uint32_t virtual_all_uid; + + bool uid_in_local:1; + bool uid_is_usable:1; + bool skip:1; + bool expunged:1; + bool copy_failed:1; + bool saved:1; +}; + +/* for quickly testing that two-way sync doesn't actually do any unexpected + modifications. */ +#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/ + +HASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *); +HASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *); + +struct dsync_mailbox_importer { + pool_t pool; + struct mailbox *box; + uint32_t last_common_uid; + uint64_t last_common_modseq, last_common_pvt_modseq; + uint32_t remote_uid_next; + uint32_t remote_first_recent_uid; + uint64_t remote_highest_modseq, remote_highest_pvt_modseq; + time_t sync_since_timestamp; + time_t sync_until_timestamp; + uoff_t sync_max_size; + enum mailbox_transaction_flags transaction_flags; + unsigned int hdr_hash_version; + unsigned int commit_msgs_interval; + + const char *const *hashed_headers; + + enum mail_flags sync_flag; + const char *sync_keyword; + bool sync_flag_dontwant; + + struct mailbox_transaction_context *trans, *ext_trans; + struct mail_search_context *search_ctx; + struct mail *mail, *ext_mail; + + struct mailbox *virtual_all_box; + struct mailbox_transaction_context *virtual_trans; + struct mail *virtual_mail; + + struct mail *cur_mail; + const char *cur_guid; + const char *cur_hdr_hash; + + /* UID => struct dsync_mail_change */ + HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes; + HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes; + + ARRAY_TYPE(seq_range) maybe_expunge_uids; + ARRAY(struct dsync_mail_change *) maybe_saves; + + /* GUID => struct importer_new_mail */ + HASH_TABLE_TYPE(guid_new_mail) import_guids; + /* UID => struct importer_new_mail */ + HASH_TABLE_TYPE(uid_new_mail) import_uids; + + ARRAY(struct importer_new_mail *) newmails; + ARRAY_TYPE(uint32_t) wanted_uids; + ARRAY_TYPE(uint32_t) saved_uids; + uint32_t highest_wanted_uid; + + ARRAY(struct dsync_mail_request) mail_requests; + unsigned int mail_request_idx; + + uint32_t prev_uid, next_local_seq, local_uid_next; + uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq; + unsigned int import_pos, import_count; + unsigned int first_unsaved_idx, saves_since_commit; + + enum mail_error mail_error; + + bool failed:1; + bool require_full_resync:1; + bool debug:1; + bool stateful_import:1; + bool last_common_uid_found:1; + bool cur_uid_has_change:1; + bool cur_mail_skip:1; + bool local_expunged_guids_set:1; + bool new_uids_assigned:1; + bool want_mail_requests:1; + bool master_brain:1; + bool revert_local_changes:1; + bool mails_have_guids:1; + bool mails_use_guid128:1; + bool delete_mailbox:1; + bool empty_hdr_workaround:1; +}; + +static const char *dsync_mail_change_type_names[] = { + "save", "expunge", "flag-change" +}; + +static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *all_newmails, + bool remote_mail); +static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, + bool final); + +static void ATTR_FORMAT(2, 3) +imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...) +{ + va_list args; + + if (importer->debug) T_BEGIN { + va_start(args, fmt); + i_debug("brain %c: Import %s: %s", + importer->master_brain ? 'M' : 'S', + mailbox_get_vname(importer->box), + t_strdup_vprintf(fmt, args)); + va_end(args); + } T_END; +} + +static void +dsync_import_unexpected_state(struct dsync_mailbox_importer *importer, + const char *error) +{ + if (!importer->stateful_import) { + i_error("Mailbox %s: %s", mailbox_get_vname(importer->box), + error); + } else { + i_warning("Mailbox %s doesn't match previous state: %s " + "(dsync must be run again without the state)", + mailbox_get_vname(importer->box), error); + } + importer->require_full_resync = TRUE; +} + +static void +dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer) +{ + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&sarg->value.seqset, search_args->pool, 128); + seq_range_array_add_range(&sarg->value.seqset, + importer->last_common_uid+1, (uint32_t)-1); + + importer->search_ctx = + mailbox_search_init(importer->trans, search_args, NULL, + 0, NULL); + mail_search_args_unref(&search_args); + + if (mailbox_search_next(importer->search_ctx, &importer->cur_mail)) + importer->next_local_seq = importer->cur_mail->seq; + /* this flag causes cur_guid to be looked up later */ + importer->cur_mail_skip = TRUE; +} + +static void +dsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer) +{ + const enum mailbox_transaction_flags ext_trans_flags = + importer->transaction_flags | + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; + + importer->trans = mailbox_transaction_begin(importer->box, + importer->transaction_flags, + "dsync import"); + importer->ext_trans = mailbox_transaction_begin(importer->box, + ext_trans_flags, + "dsync ext import"); + importer->mail = mail_alloc(importer->trans, 0, NULL); + importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL); +} + +struct dsync_mailbox_importer * +dsync_mailbox_import_init(struct mailbox *box, + struct mailbox *virtual_all_box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + uint64_t last_common_pvt_modseq, + uint32_t remote_uid_next, + uint32_t remote_first_recent_uid, + uint64_t remote_highest_modseq, + uint64_t remote_highest_pvt_modseq, + time_t sync_since_timestamp, + time_t sync_until_timestamp, + uoff_t sync_max_size, + const char *sync_flag, + unsigned int commit_msgs_interval, + enum dsync_mailbox_import_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers) +{ + struct dsync_mailbox_importer *importer; + struct mailbox_status status; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer", + 10240); + importer = p_new(pool, struct dsync_mailbox_importer, 1); + importer->pool = pool; + importer->box = box; + importer->virtual_all_box = virtual_all_box; + importer->last_common_uid = last_common_uid; + importer->last_common_modseq = last_common_modseq; + importer->last_common_pvt_modseq = last_common_pvt_modseq; + importer->last_common_uid_found = + last_common_uid != 0 || last_common_modseq != 0; + importer->remote_uid_next = remote_uid_next; + importer->remote_first_recent_uid = remote_first_recent_uid; + importer->remote_highest_modseq = remote_highest_modseq; + importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq; + importer->sync_since_timestamp = sync_since_timestamp; + importer->sync_until_timestamp = sync_until_timestamp; + importer->sync_max_size = sync_max_size; + importer->stateful_import = importer->last_common_uid_found; + importer->hashed_headers = hashed_headers; + + if (sync_flag != NULL) { + if (sync_flag[0] == '-') { + importer->sync_flag_dontwant = TRUE; + sync_flag++; + } + if (sync_flag[0] == '\\') + importer->sync_flag = imap_parse_system_flag(sync_flag); + else + importer->sync_keyword = p_strdup(pool, sync_flag); + } + importer->commit_msgs_interval = commit_msgs_interval; + importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC; + if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0) + importer->transaction_flags |= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY; + + hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp); + hash_table_create_direct(&importer->import_uids, pool, 0); + i_array_init(&importer->maybe_expunge_uids, 16); + i_array_init(&importer->maybe_saves, 128); + i_array_init(&importer->newmails, 128); + i_array_init(&importer->wanted_uids, 128); + i_array_init(&importer->saved_uids, 128); + + dsync_mailbox_import_transaction_begin(importer); + + if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) { + i_array_init(&importer->mail_requests, 128); + importer->want_mail_requests = TRUE; + } + importer->master_brain = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0; + importer->revert_local_changes = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0; + importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0; + importer->mails_have_guids = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0; + importer->mails_use_guid128 = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0; + importer->hdr_hash_version = hdr_hash_version; + importer->empty_hdr_workaround = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0; + + mailbox_get_open_status(importer->box, STATUS_UIDNEXT | + STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ, + &status); + if (status.nonpermanent_modseqs) + status.highest_modseq = 0; + importer->local_uid_next = status.uidnext; + importer->local_initial_highestmodseq = status.highest_modseq; + importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq; + dsync_mailbox_import_search_init(importer); + + if (!importer->stateful_import) + ; + else if (importer->local_uid_next <= last_common_uid) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local UIDNEXT %u <= last common UID %u", + importer->local_uid_next, last_common_uid)); + } else if (importer->local_initial_highestmodseq < last_common_modseq) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64, + importer->local_initial_highestmodseq, + last_common_modseq)); + } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64, + importer->local_initial_highestpvtmodseq, + last_common_pvt_modseq)); + } + + importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan); + importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan); + return importer; +} + +static int +dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer, + enum mail_attribute_type type, const char *key, + struct dsync_mailbox_attribute **attr_r) +{ + struct dsync_mailbox_attribute lookup_attr, *attr; + const struct dsync_mailbox_attribute *attr_change; + struct mail_attribute_value value; + + *attr_r = NULL; + + if (mailbox_attribute_get_stream(importer->box, type, key, &value) < 0) { + i_error("Mailbox %s: Failed to get attribute %s: %s", + mailbox_get_vname(importer->box), key, + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + return -1; + } + + lookup_attr.type = type; + lookup_attr.key = key; + + attr_change = hash_table_lookup(importer->local_attr_changes, + &lookup_attr); + if (attr_change == NULL && + value.value == NULL && value.value_stream == NULL) { + /* we have no knowledge of this attribute */ + return 0; + } + attr = t_new(struct dsync_mailbox_attribute, 1); + attr->type = type; + attr->key = key; + attr->value = value.value; + attr->value_stream = value.value_stream; + attr->last_change = value.last_change; + if (attr_change != NULL) { + attr->deleted = attr_change->deleted && + !DSYNC_ATTR_HAS_VALUE(attr); + attr->modseq = attr_change->modseq; + } + *attr_r = attr; + return 0; +} + +static int +dsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r) +{ + const unsigned char *data1, *data2; + size_t size1, size2, size; + + *cmp_r = -1; /* quiet gcc */ + + for (;;) { + (void)i_stream_read_more(input1, &data1, &size1); + (void)i_stream_read_more(input2, &data2, &size2); + + if (size1 == 0 || size2 == 0) + break; + size = I_MIN(size1, size2); + *cmp_r = memcmp(data1, data2, size); + if (*cmp_r != 0) + return 0; + i_stream_skip(input1, size); + i_stream_skip(input2, size); + } + if (input1->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(input1), + i_stream_get_error(input1)); + return -1; + } + if (input2->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(input2), + i_stream_get_error(input2)); + return -1; + } + if (size1 == 0 && size2 == 0) + *cmp_r = 0; + else + *cmp_r = size1 == 0 ? -1 : 1; + return 0; +} + +static int +dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1, + const struct dsync_mailbox_attribute *attr2, + int *cmp_r) +{ + struct istream *input1, *input2; + int ret; + + i_assert(attr1->value_stream != NULL || attr1->value != NULL); + i_assert(attr2->value_stream != NULL || attr2->value != NULL); + + if (attr1->value != NULL && attr2->value != NULL) { + *cmp_r = strcmp(attr1->value, attr2->value); + return 0; + } + /* at least one of them is a stream. make both of them streams. */ + input1 = attr1->value_stream != NULL ? attr1->value_stream : + i_stream_create_from_data(attr1->value, strlen(attr1->value)); + input2 = attr2->value_stream != NULL ? attr2->value_stream : + i_stream_create_from_data(attr2->value, strlen(attr2->value)); + i_stream_seek(input1, 0); + i_stream_seek(input2, 0); + ret = dsync_istreams_cmp(input1, input2, cmp_r); + if (attr1->value_stream == NULL) + i_stream_unref(&input1); + if (attr2->value_stream == NULL) + i_stream_unref(&input2); + return ret; +} + +static int +dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr, + const struct dsync_mailbox_attribute *local_attr, + int *cmp_r) +{ + if (DSYNC_ATTR_HAS_VALUE(attr) && + !DSYNC_ATTR_HAS_VALUE(local_attr)) { + /* remote has a value and local doesn't -> use it */ + *cmp_r = 1; + return 0; + } else if (!DSYNC_ATTR_HAS_VALUE(attr) && + DSYNC_ATTR_HAS_VALUE(local_attr)) { + /* remote doesn't have a value, bt local does -> skip */ + *cmp_r = -1; + return 0; + } + + return dsync_attributes_cmp_values(attr, local_attr, cmp_r); +} + +static int +dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer, + const struct dsync_mailbox_attribute *attr, + const struct dsync_mailbox_attribute *local_attr, + const char **result_r) +{ + struct mail_attribute_value value; + int cmp; + bool ignore = FALSE; + + i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted); + + if (attr->deleted && + (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) { + /* attribute doesn't exist on either side -> ignore */ + *result_r = "Nonexistent in both sides"; + return 0; + } + if (local_attr == NULL) { + /* we haven't seen this locally -> use whatever remote has */ + *result_r = "Nonexistent locally"; + } else if (local_attr->modseq <= importer->last_common_modseq && + attr->modseq > importer->last_common_modseq && + importer->last_common_modseq > 0) { + /* we're doing incremental syncing, and we can see that the + attribute was changed remotely, but not locally -> use it */ + *result_r = "Changed remotely"; + } else if (local_attr->modseq > importer->last_common_modseq && + attr->modseq <= importer->last_common_modseq && + importer->last_common_modseq > 0) { + /* we're doing incremental syncing, and we can see that the + attribute was changed locally, but not remotely -> ignore */ + *result_r = "Changed locally"; + ignore = TRUE; + } else if (attr->last_change > local_attr->last_change) { + /* remote has a newer timestamp -> use it */ + *result_r = "Remote has newer timestamp"; + } else if (attr->last_change < local_attr->last_change) { + /* remote has an older timestamp -> ignore */ + *result_r = "Local has newer timestamp"; + ignore = TRUE; + } else { + /* the timestamps are the same. now we're down to guessing + the right answer, unless the values are actually equal, + so check that first. next try to use modseqs, but if even + they are the same, fallback to just picking one based on the + value. */ + if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) { + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + return -1; + } + if (cmp == 0) { + /* identical scripts */ + *result_r = "Unchanged value"; + return 0; + } + + if (attr->modseq > local_attr->modseq) { + /* remote has a higher modseq -> use it */ + *result_r = "Remote has newer modseq"; + } else if (attr->modseq < local_attr->modseq) { + /* remote has an older modseq -> ignore */ + *result_r = "Local has newer modseq"; + ignore = TRUE; + } else if (cmp < 0) { + ignore = TRUE; + *result_r = "Value changed, but unknown which is newer - picking local"; + } else { + *result_r = "Value changed, but unknown which is newer - picking remote"; + } + } + if (ignore) + return 0; + + i_zero(&value); + value.value = attr->value; + value.value_stream = attr->value_stream; + value.last_change = attr->last_change; + if (mailbox_attribute_set(importer->trans, attr->type, + attr->key, &value) < 0) { + i_error("Mailbox %s: Failed to set attribute %s: %s", + mailbox_get_vname(importer->box), attr->key, + mailbox_get_last_internal_error(importer->box, NULL)); + /* the attributes aren't vital, don't fail everything just + because of them. */ + } + return 0; +} + +int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, + const struct dsync_mailbox_attribute *attr) +{ + struct dsync_mailbox_attribute *local_attr; + const char *result = ""; + int ret; + + if (dsync_mailbox_import_lookup_attr(importer, attr->type, + attr->key, &local_attr) < 0) + ret = -1; + else { + ret = dsync_mailbox_import_attribute_real(importer, attr, + local_attr, &result); + if (local_attr != NULL && local_attr->value_stream != NULL) + i_stream_unref(&local_attr->value_stream); + } + imp_debug(importer, "Import attribute %s: %s", attr->key, + ret < 0 ? "failed" : result); + return ret; +} + +static void dsync_mail_error(struct dsync_mailbox_importer *importer, + struct mail *mail, const char *field) +{ + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_internal_error(importer->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return; + + i_error("Mailbox %s: Can't lookup %s for UID=%u: %s", + mailbox_get_vname(mail->box), field, mail->uid, errstr); + importer->mail_error = error; + importer->failed = TRUE; +} + +static bool +dsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char *guid, const char **cmp_guid_r) +{ + guid_128_t guid_128, change_guid_128; + + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + if (guid_128_from_string(change->guid, change_guid_128) < 0) + i_unreached(); + } else if (importer->mails_use_guid128) { + mail_generate_guid_128_hash(change->guid, change_guid_128); + } else { + if (cmp_guid_r != NULL) + *cmp_guid_r = change->guid; + return strcmp(change->guid, guid) == 0; + } + + mail_generate_guid_128_hash(guid, guid_128); + if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { + if (cmp_guid_r != NULL) { + *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)", + binary_to_hex(change_guid_128, sizeof(change_guid_128)), + change->guid); + } + return FALSE; + } + return TRUE; +} + +static int +importer_try_next_mail(struct dsync_mailbox_importer *importer, + uint32_t wanted_uid) +{ + struct mail_private *pmail; + const char *hdr_hash; + + if (importer->cur_mail == NULL) { + /* end of search */ + return -1; + } + while (importer->cur_mail->seq < importer->next_local_seq || + importer->cur_mail->uid < wanted_uid) { + if (!importer->cur_uid_has_change && + !importer->last_common_uid_found) { + /* this message exists locally, but remote didn't send + expunge-change for it. if the message's + uid <= last-common-uid, it should be deleted */ + seq_range_array_add(&importer->maybe_expunge_uids, + importer->cur_mail->uid); + } + + importer->cur_mail_skip = FALSE; + if (!mailbox_search_next(importer->search_ctx, + &importer->cur_mail)) { + importer->cur_mail = NULL; + importer->cur_guid = NULL; + importer->cur_hdr_hash = NULL; + return -1; + } + importer->cur_uid_has_change = FALSE; + } + importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid; + if (importer->mails_have_guids) { + if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID, + &importer->cur_guid) < 0) { + dsync_mail_error(importer, importer->cur_mail, "GUID"); + return 0; + } + } else { + if (dsync_mail_get_hdr_hash(importer->cur_mail, + importer->hdr_hash_version, + importer->hashed_headers, + &hdr_hash) < 0) { + dsync_mail_error(importer, importer->cur_mail, + "header hash"); + return 0; + } + pmail = (struct mail_private *)importer->cur_mail; + importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash); + importer->cur_guid = ""; + } + /* make sure next_local_seq gets updated in case we came here + because of min_uid */ + importer->next_local_seq = importer->cur_mail->seq; + return 1; +} + +static bool +importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid) +{ + int ret; + + for (;;) { + T_BEGIN { + ret = importer_try_next_mail(importer, wanted_uid); + } T_END; + if (ret != 0 || importer->failed) + break; + importer->next_local_seq = importer->cur_mail->seq + 1; + } + return ret > 0; +} + +static int +importer_mail_cmp(const struct importer_mail *m1, + const struct importer_mail *m2) +{ + int ret; + + if (m1->guid == NULL) + return 1; + if (m2->guid == NULL) + return -1; + + ret = strcmp(m1->guid, m2->guid); + if (ret != 0) + return ret; + + if (m1->uid < m2->uid) + return -1; + if (m1->uid > m2->uid) + return 1; + return 0; +} + +static void newmail_link(struct dsync_mailbox_importer *importer, + struct importer_new_mail *newmail, uint32_t remote_uid) +{ + struct importer_new_mail *first_mail, **last, *mail, *link = NULL; + + if (*newmail->guid != '\0') { + first_mail = hash_table_lookup(importer->import_guids, + newmail->guid); + if (first_mail == NULL) { + /* first mail for this GUID */ + hash_table_insert(importer->import_guids, + newmail->guid, newmail); + return; + } + } else { + if (remote_uid == 0) { + /* mail exists locally. we don't want to request + it, and we'll assume it has no duplicate + instances. */ + return; + } + first_mail = hash_table_lookup(importer->import_uids, + POINTER_CAST(remote_uid)); + if (first_mail == NULL) { + /* first mail for this UID */ + hash_table_insert(importer->import_uids, + POINTER_CAST(remote_uid), newmail); + return; + } + } + /* 1) add the newmail to the end of the linked list + 2) find our link + + FIXME: this loop is slow if the same GUID has a ton of instances. + Could it be improved in some way? */ + last = &first_mail->next; + for (mail = first_mail; mail != NULL; mail = mail->next) { + if (mail->final_uid == newmail->final_uid) + mail->uid_is_usable = TRUE; + if (link == NULL && mail->link == NULL && + mail->uid_in_local != newmail->uid_in_local) + link = mail; + last = &mail->next; + } + *last = newmail; + if (link != NULL && newmail->link == NULL) { + link->link = newmail; + newmail->link = link; + } +} + +static void +dsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer, + uint32_t uid, const char *reason) +{ + i_assert(importer->revert_local_changes); + + /* UID either already exists or UIDNEXT is too high. we can't set the + wanted UID, so we'll need to delete the whole mailbox and resync */ + i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s", + mailbox_get_vname(importer->box), uid, reason); + importer->delete_mailbox = TRUE; + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; +} + +static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + struct importer_mail m1, m2; + struct importer_new_mail *newmail; + int diff; + bool remote_saved; + + i_zero(&m1); + if (importer->cur_mail != NULL) { + m1.guid = importer->mails_have_guids ? + importer->cur_guid : importer->cur_hdr_hash; + m1.uid = importer->cur_mail->uid; + } + i_zero(&m2); + if (save_change != NULL) { + m2.guid = importer->mails_have_guids ? + save_change->guid : save_change->hdr_hash; + m2.uid = save_change->uid; + i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + } + + if (importer->empty_hdr_workaround && !importer->mails_have_guids && + importer->cur_mail != NULL && save_change != NULL && + (dsync_mail_hdr_hash_is_empty(m1.guid) || + dsync_mail_hdr_hash_is_empty(m2.guid))) { + /* one of the headers is empty. assume it's broken and that + the header matches what we have currently. */ + diff = 0; + } else { + diff = importer_mail_cmp(&m1, &m2); + } + if (diff < 0) { + /* add a record for local mail */ + i_assert(importer->cur_mail != NULL); + if (importer->revert_local_changes) { + if (save_change == NULL && + importer->cur_mail->uid >= importer->remote_uid_next) { + dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid, + t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next)); + return TRUE; + } + mail_expunge(importer->cur_mail); + importer->cur_mail_skip = TRUE; + importer->next_local_seq++; + return FALSE; + } + newmail = p_new(importer->pool, struct importer_new_mail, 1); + newmail->guid = p_strdup(importer->pool, importer->cur_guid); + newmail->final_uid = importer->cur_mail->uid; + newmail->local_uid = importer->cur_mail->uid; + newmail->uid_in_local = TRUE; + newmail->uid_is_usable = + newmail->final_uid >= importer->remote_uid_next; + remote_saved = FALSE; + } else if (diff > 0) { + i_assert(save_change != NULL); + newmail = p_new(importer->pool, struct importer_new_mail, 1); + newmail->guid = save_change->guid; + newmail->final_uid = save_change->uid; + newmail->remote_uid = save_change->uid; + newmail->uid_in_local = FALSE; + newmail->uid_is_usable = + newmail->final_uid >= importer->local_uid_next; + if (!newmail->uid_is_usable && importer->revert_local_changes) { + dsync_mailbox_revert_existing_uid(importer, newmail->final_uid, + t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next)); + return TRUE; + } + remote_saved = TRUE; + } else { + /* identical */ + i_assert(importer->cur_mail != NULL); + i_assert(save_change != NULL); + newmail = p_new(importer->pool, struct importer_new_mail, 1); + newmail->guid = save_change->guid; + newmail->final_uid = importer->cur_mail->uid; + newmail->local_uid = importer->cur_mail->uid; + newmail->remote_uid = save_change->uid; + newmail->uid_in_local = TRUE; + newmail->uid_is_usable = TRUE; + newmail->link = newmail; + remote_saved = TRUE; + } + + if (newmail->uid_in_local) { + importer->cur_mail_skip = TRUE; + importer->next_local_seq++; + } + /* NOTE: assumes save_change is allocated from importer pool */ + newmail->change = save_change; + + array_push_back(&importer->newmails, &newmail); + if (newmail->uid_in_local) + newmail_link(importer, newmail, 0); + else { + i_assert(save_change != NULL); + newmail_link(importer, newmail, save_change->uid); + } + return remote_saved; +} + +static bool ATTR_NULL(2) +dsync_mailbox_try_save(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + if (importer->cur_mail_skip) { + if (!importer_next_mail(importer, 0) && save_change == NULL) + return FALSE; + } + return dsync_mailbox_try_save_cur(importer, save_change); +} + +static void dsync_mailbox_save(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + while (!dsync_mailbox_try_save(importer, save_change)) ; +} + +static bool +dsync_import_set_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *guid, *cmp_guid; + + if (!mail_set_uid(importer->mail, change->uid)) + return FALSE; + if (change->guid == NULL) { + /* GUID is unknown */ + return TRUE; + } + if (*change->guid == '\0') { + /* backend doesn't support GUIDs. if hdr_hash is set, we could + verify it, but since this message really is supposed to + match, it's probably too much trouble. */ + return TRUE; + } + + /* verify that GUID matches, just in case */ + if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) { + dsync_mail_error(importer, importer->mail, "GUID"); + return FALSE; + } + if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "Unexpected GUID mismatch for UID=%u: %s != %s", + change->uid, guid, cmp_guid)); + return FALSE; + } + return TRUE; +} + +static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *cmp_guid; + + if (change->guid == NULL || change->guid[0] == '\0' || + importer->cur_guid[0] == '\0') + return TRUE; + if (!dsync_mail_change_guid_equals(importer, change, + importer->cur_guid, &cmp_guid)) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "Unexpected GUID mismatch (2) for UID=%u: %s != %s", + change->uid, importer->cur_guid, cmp_guid)); + return FALSE; + } + return TRUE; +} + +static void +merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove, + uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove, + uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote, + uint32_t *change_add_r, uint32_t *change_remove_r, + bool *remote_changed, bool *remote_pvt_changed) +{ + uint32_t combined_add, combined_remove, conflict_flags; + uint32_t local_wanted, remote_wanted, conflict_pvt_flags; + + /* resolve conflicts */ + conflict_flags = local_add & remote_remove; + if (conflict_flags != 0) { + conflict_pvt_flags = conflict_flags & pvt_mask; + conflict_flags &= ~pvt_mask; + if (prefer_remote) + local_add &= ~conflict_flags; + else + remote_remove &= ~conflict_flags; + if (prefer_pvt_remote) + local_add &= ~conflict_pvt_flags; + else + remote_remove &= ~conflict_pvt_flags; + } + conflict_flags = local_remove & remote_add; + if (conflict_flags != 0) { + conflict_pvt_flags = conflict_flags & pvt_mask; + conflict_flags &= ~pvt_mask; + if (prefer_remote) + local_remove &= ~conflict_flags; + else + remote_add &= ~conflict_flags; + if (prefer_pvt_remote) + local_remove &= ~conflict_pvt_flags; + else + remote_add &= ~conflict_pvt_flags; + } + + combined_add = local_add|remote_add; + combined_remove = local_remove|remote_remove; + i_assert((combined_add & combined_remove) == 0); + + /* don't change flags that are currently identical in both sides */ + conflict_flags = local_final ^ remote_final; + combined_add &= conflict_flags; + combined_remove &= conflict_flags; + + /* see if there are conflicting final flags */ + local_wanted = (local_final|combined_add) & ~combined_remove; + remote_wanted = (remote_final|combined_add) & ~combined_remove; + + conflict_flags = local_wanted ^ remote_wanted; + if (conflict_flags != 0) { + if (prefer_remote && prefer_pvt_remote) + local_wanted = remote_wanted; + else if (prefer_remote && !prefer_pvt_remote) { + local_wanted = (local_wanted & pvt_mask) | + (remote_wanted & ~pvt_mask); + } else if (!prefer_remote && prefer_pvt_remote) { + local_wanted = (local_wanted & ~pvt_mask) | + (remote_wanted & pvt_mask); + } + } + + *change_add_r = local_wanted & ~local_final; + *change_remove_r = local_final & ~local_wanted; + if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask)) + *remote_changed = TRUE; + if ((local_wanted & pvt_mask) != (remote_final & pvt_mask)) + *remote_pvt_changed = TRUE; +} + +static bool +keyword_find(ARRAY_TYPE(const_string) *keywords, const char *name, + unsigned int *idx_r) +{ + const char *const *names; + unsigned int i, count; + + names = array_get(keywords, &count); + for (i = 0; i < count; i++) { + if (strcmp(names[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +static void keywords_append(ARRAY_TYPE(const_string) *dest, + const ARRAY_TYPE(const_string) *keywords, + uint32_t bits, unsigned int start_idx) +{ + const char *name; + unsigned int i; + + for (i = 0; i < 32; i++) { + if ((bits & (1U << i)) == 0) + continue; + + name = array_idx_elem(keywords, start_idx+i); + array_push_back(dest, &name); + } +} + +static void +merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes, + const ARRAY_TYPE(const_string) *remote_changes, + bool prefer_remote, + bool *remote_changed, bool *remote_pvt_changed) +{ + /* local_changes and remote_changes are assumed to have no + duplicates names */ + uint32_t *local_add, *local_remove, *local_final; + uint32_t *remote_add, *remote_remove, *remote_final; + uint32_t *change_add, *change_remove; + ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords; + const char *const *changes, *name, *const *local_keywords; + struct mail_keywords *kw; + unsigned int i, count, name_idx, array_size; + + local_keywords = mail_get_keywords(mail); + + /* we'll assign a common index for each keyword name and place + the changes to separate bit arrays. */ + if (array_is_created(remote_changes)) + changes = array_get(remote_changes, &count); + else { + changes = NULL; + count = 0; + } + + array_size = str_array_length(local_keywords) + count; + if (array_is_created(local_changes)) + array_size += array_count(local_changes); + if (array_size == 0) { + /* this message has no keywords */ + return; + } + t_array_init(&all_keywords, array_size); + t_array_init(&add_keywords, array_size); + t_array_init(&remove_keywords, array_size); + + /* @UNSAFE: create large enough arrays to fit all keyword indexes. */ + array_size = (array_size+31)/32; + local_add = t_new(uint32_t, array_size); + local_remove = t_new(uint32_t, array_size); + local_final = t_new(uint32_t, array_size); + remote_add = t_new(uint32_t, array_size); + remote_remove = t_new(uint32_t, array_size); + remote_final = t_new(uint32_t, array_size); + change_add = t_new(uint32_t, array_size); + change_remove = t_new(uint32_t, array_size); + + /* get remote changes */ + for (i = 0; i < count; i++) { + name = changes[i]+1; + name_idx = array_count(&all_keywords); + array_push_back(&all_keywords, &name); + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + remote_add[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_REMOVE: + remote_remove[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_FINAL: + remote_final[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_ADD_AND_FINAL: + remote_add[name_idx/32] |= 1U << (name_idx%32); + remote_final[name_idx/32] |= 1U << (name_idx%32); + break; + } + } + + /* get local changes. use existing indexes for names when they exist. */ + if (array_is_created(local_changes)) + changes = array_get(local_changes, &count); + else { + changes = NULL; + count = 0; + } + for (i = 0; i < count; i++) { + name = changes[i]+1; + if (!keyword_find(&all_keywords, name, &name_idx)) { + name_idx = array_count(&all_keywords); + array_push_back(&all_keywords, &name); + } + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + case KEYWORD_CHANGE_ADD_AND_FINAL: + local_add[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_REMOVE: + local_remove[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_FINAL: + break; + } + } + for (i = 0; local_keywords[i] != NULL; i++) { + name = local_keywords[i]; + if (!keyword_find(&all_keywords, name, &name_idx)) { + name_idx = array_count(&all_keywords); + array_push_back(&all_keywords, &name); + } + local_final[name_idx/32] |= 1U << (name_idx%32); + } + i_assert(array_count(&all_keywords) <= array_size*32); + array_size = (array_count(&all_keywords)+31) / 32; + + /* merge keywords */ + for (i = 0; i < array_size; i++) { + merge_flags(local_final[i], local_add[i], local_remove[i], + remote_final[i], remote_add[i], remote_remove[i], + 0, prefer_remote, prefer_remote, + &change_add[i], &change_remove[i], + remote_changed, remote_pvt_changed); + if (change_add[i] != 0) { + keywords_append(&add_keywords, &all_keywords, + change_add[i], i*32); + } + if (change_remove[i] != 0) { + keywords_append(&remove_keywords, &all_keywords, + change_remove[i], i*32); + } + } + + /* apply changes */ + if (array_count(&add_keywords) > 0) { + array_append_zero(&add_keywords); + kw = mailbox_keywords_create_valid(mail->box, + array_front(&add_keywords)); + mail_update_keywords(mail, MODIFY_ADD, kw); + mailbox_keywords_unref(&kw); + } + if (array_count(&remove_keywords) > 0) { + array_append_zero(&remove_keywords); + kw = mailbox_keywords_create_valid(mail->box, + array_front(&remove_keywords)); + mail_update_keywords(mail, MODIFY_REMOVE, kw); + mailbox_keywords_unref(&kw); + } +} + +static void +dsync_mailbox_import_replace_flags(struct mail *mail, + const struct dsync_mail_change *change) +{ + ARRAY_TYPE(const_string) keywords; + struct mail_keywords *kw; + const char *const *changes, *name; + unsigned int i, count; + + if (array_is_created(&change->keyword_changes)) + changes = array_get(&change->keyword_changes, &count); + else { + changes = NULL; + count = 0; + } + t_array_init(&keywords, count+1); + for (i = 0; i < count; i++) { + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + case KEYWORD_CHANGE_FINAL: + case KEYWORD_CHANGE_ADD_AND_FINAL: + name = changes[i]+1; + array_push_back(&keywords, &name); + break; + case KEYWORD_CHANGE_REMOVE: + break; + } + } + array_append_zero(&keywords); + + kw = mailbox_keywords_create_valid(mail->box, array_front(&keywords)); + mail_update_keywords(mail, MODIFY_REPLACE, kw); + mailbox_keywords_unref(&kw); + + mail_update_flags(mail, MODIFY_REPLACE, + change->add_flags | change->final_flags); + if (mail_get_modseq(mail) < change->modseq) + mail_update_modseq(mail, change->modseq); + if (mail_get_pvt_modseq(mail) < change->pvt_modseq) + mail_update_pvt_modseq(mail, change->pvt_modseq); +} + +static void +dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const struct dsync_mail_change *local_change; + enum mail_flags local_add, local_remove; + uint32_t change_add, change_remove; + uint64_t new_modseq; + ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT; + struct mail *mail; + bool prefer_remote, prefer_pvt_remote; + bool remote_changed = FALSE, remote_pvt_changed = FALSE; + + i_assert((change->add_flags & change->remove_flags) == 0); + + if (importer->cur_mail != NULL && + importer->cur_mail->uid == change->uid) { + if (!dsync_check_cur_guid(importer, change)) + return; + mail = importer->cur_mail; + } else { + if (!dsync_import_set_mail(importer, change)) + return; + mail = importer->mail; + } + + if (importer->revert_local_changes) { + /* dsync backup: just make the local look like remote. */ + dsync_mailbox_import_replace_flags(mail, change); + return; + } + + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL) { + local_add = local_remove = 0; + } else { + local_add = local_change->add_flags; + local_remove = local_change->remove_flags; + local_keyword_changes = local_change->keyword_changes; + } + + if (mail_get_modseq(mail) < change->modseq) + prefer_remote = TRUE; + else if (mail_get_modseq(mail) > change->modseq) + prefer_remote = FALSE; + else { + /* identical modseq, we'll just have to pick one. + Note that both brains need to pick the same one, otherwise + they become unsynced. */ + prefer_remote = !importer->master_brain; + } + if (mail_get_pvt_modseq(mail) < change->pvt_modseq) + prefer_pvt_remote = TRUE; + else if (mail_get_pvt_modseq(mail) > change->pvt_modseq) + prefer_pvt_remote = FALSE; + else + prefer_pvt_remote = !importer->master_brain; + + /* merge flags */ + merge_flags(mail_get_flags(mail), local_add, local_remove, + change->final_flags, change->add_flags, change->remove_flags, + mailbox_get_private_flags_mask(mail->box), + prefer_remote, prefer_pvt_remote, + &change_add, &change_remove, + &remote_changed, &remote_pvt_changed); + + if (change_add != 0) + mail_update_flags(mail, MODIFY_ADD, change_add); + if (change_remove != 0) + mail_update_flags(mail, MODIFY_REMOVE, change_remove); + + /* merge keywords */ + merge_keywords(mail, &local_keyword_changes, &change->keyword_changes, + prefer_remote, &remote_changed, &remote_pvt_changed); + + /* update modseqs. try to anticipate when we have to increase modseq + to get it closer to what remote has (although we can't guess it + exactly correctly) */ + new_modseq = change->modseq; + if (remote_changed && new_modseq <= importer->remote_highest_modseq) + new_modseq = importer->remote_highest_modseq+1; + if (mail_get_modseq(mail) < new_modseq) + mail_update_modseq(mail, new_modseq); + + new_modseq = change->pvt_modseq; + if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq) + new_modseq = importer->remote_highest_pvt_modseq+1; + if (mail_get_pvt_modseq(mail) < new_modseq) + mail_update_pvt_modseq(mail, new_modseq); +} + +static bool +dsync_mail_change_have_keyword(const struct dsync_mail_change *change, + const char *keyword) +{ + const char *str; + + if (!array_is_created(&change->keyword_changes)) + return FALSE; + + array_foreach_elem(&change->keyword_changes, str) { + switch (str[0]) { + case KEYWORD_CHANGE_FINAL: + case KEYWORD_CHANGE_ADD_AND_FINAL: + if (strcasecmp(str+1, keyword) == 0) + return TRUE; + break; + default: + break; + } + } + return FALSE; +} + +static bool +dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + if (importer->sync_since_timestamp > 0) { + i_assert(change->received_timestamp > 0); + if (change->received_timestamp < importer->sync_since_timestamp) { + /* mail has too old timestamp - skip it */ + *result_r = "Ignoring missing local mail with too old timestamp"; + return FALSE; + } + } + if (importer->sync_until_timestamp > 0) { + i_assert(change->received_timestamp > 0); + if (change->received_timestamp > importer->sync_until_timestamp) { + /* mail has too new timestamp - skip it */ + *result_r = "Ignoring missing local mail with too new timestamp"; + return FALSE; + } + } + if (importer->sync_max_size > 0) { + i_assert(change->virtual_size != UOFF_T_MAX); + if (change->virtual_size > importer->sync_max_size) { + /* mail is too large - skip it */ + *result_r = "Ignoring missing local mail with too large size"; + return FALSE; + } + } + if (importer->sync_flag != 0) { + bool have_flag = (change->final_flags & importer->sync_flag) != 0; + + if (have_flag && importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that doesn't have wanted flags"; + return FALSE; + } + if (!have_flag && !importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that has unwanted flags"; + return FALSE; + } + } + if (importer->sync_keyword != NULL) { + bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword); + + if (have_kw && importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that doesn't have wanted keywords"; + return FALSE; + } + if (!have_kw && !importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that has unwanted keywords"; + return FALSE; + } + } + return TRUE; +} + +static void +dsync_mailbox_import_save(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + struct dsync_mail_change *save; + const char *result; + + i_assert(change->guid != NULL); + + if (change->uid == importer->last_common_uid) { + /* we've already verified that the GUID matches. + apply flag changes if there are any. */ + i_assert(!importer->last_common_uid_found); + dsync_mailbox_import_flag_change(importer, change); + return; + } + if (!dsync_mailbox_import_want_change(importer, change, &result)) + return; + + save = p_new(importer->pool, struct dsync_mail_change, 1); + dsync_mail_change_dup(importer->pool, change, save); + + if (importer->last_common_uid_found) { + /* this is a new mail. its UID may or may not conflict with + an existing local mail, we'll figure it out later. */ + i_assert(change->uid > importer->last_common_uid); + dsync_mailbox_save(importer, save); + } else { + /* the local mail is expunged. we'll decide later if we want + to save this mail locally or expunge it form remote. */ + i_assert(change->uid > importer->last_common_uid); + i_assert(importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid); + array_push_back(&importer->maybe_saves, &save); + } +} + +static void +dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + + if (importer->last_common_uid_found) { + /* expunge the message, unless its GUID unexpectedly doesn't + match */ + i_assert(change->uid <= importer->last_common_uid); + if (dsync_import_set_mail(importer, change)) + mail_expunge(importer->mail); + } else if (importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid) { + /* already expunged locally, we can ignore this. + uid=last_common_uid if we managed to verify from + transaction log that the GUIDs match */ + i_assert(change->uid >= importer->last_common_uid); + } else if (change->uid == importer->last_common_uid) { + /* already verified that the GUID matches */ + i_assert(importer->cur_mail->uid == change->uid); + if (dsync_check_cur_guid(importer, change)) + mail_expunge(importer->cur_mail); + } else { + /* we don't know yet if we should expunge this + message or not. queue it until we do. */ + i_assert(change->uid > importer->last_common_uid); + seq_range_array_add(&importer->maybe_expunge_uids, change->uid); + } +} + +static void +dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer) +{ + /* If there are local mails after last_common_uid which we skipped + while trying to match the next message, we need to now go back */ + if (importer->cur_mail != NULL && + importer->cur_mail->uid <= importer->last_common_uid+1) + return; + + importer->cur_mail = NULL; + importer->cur_guid = NULL; + importer->cur_hdr_hash = NULL; + importer->next_local_seq = 0; + + (void)mailbox_search_deinit(&importer->search_ctx); + dsync_mailbox_import_search_init(importer); +} + +static void +dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer) +{ + struct dsync_mail_change *const *saves; + struct seq_range_iter iter; + unsigned int n, i, count; + uint32_t uid; + + if (importer->debug) T_BEGIN { + string_t *expunges = t_str_new(64); + + imap_write_seq_range(expunges, &importer->maybe_expunge_uids); + imp_debug(importer, "Last common UID=%u. Delayed expunges=%s", + importer->last_common_uid, str_c(expunges)); + } T_END; + + importer->last_common_uid_found = TRUE; + dsync_mailbox_rewind_search(importer); + + /* expunge the messages whose expunge-decision we delayed previously */ + seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + if (uid > importer->last_common_uid) { + /* we expunge messages only up to last_common_uid, + ignore the rest */ + break; + } + + if (mail_set_uid(importer->mail, uid)) + mail_expunge(importer->mail); + } + + /* handle pending saves */ + saves = array_get(&importer->maybe_saves, &count); + for (i = 0; i < count; i++) { + if (saves[i]->uid > importer->last_common_uid) { + imp_debug(importer, "Delayed save UID=%u: Save", + saves[i]->uid); + dsync_mailbox_save(importer, saves[i]); + } else { + imp_debug(importer, "Delayed save UID=%u: Ignore", + saves[i]->uid); + } + } +} + +static int +dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + const char *hdr_hash, *cmp_guid; + + if (*change->guid != '\0' && *importer->cur_guid != '\0') { + /* we have GUIDs, verify them */ + if (dsync_mail_change_guid_equals(importer, change, + importer->cur_guid, &cmp_guid)) { + *result_r = "GUIDs match"; + return 1; + } else { + *result_r = t_strdup_printf("GUIDs don't match (%s vs %s)", + importer->cur_guid, cmp_guid); + return 0; + } + } + + /* verify hdr_hash if it exists */ + if (change->hdr_hash == NULL) { + i_assert(*importer->cur_guid == '\0'); + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* the message was already expunged, so we don't know + its header. return "unknown". */ + *result_r = "Unknown match for expunge"; + return -1; + } + i_error("Mailbox %s: GUIDs not supported, " + "sync with header hashes instead", + mailbox_get_vname(importer->box)); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + *result_r = "Error, invalid parameters"; + return -1; + } + + if (dsync_mail_get_hdr_hash(importer->cur_mail, + importer->hdr_hash_version, + importer->hashed_headers, &hdr_hash) < 0) { + dsync_mail_error(importer, importer->cur_mail, "hdr-stream"); + *result_r = "Error fetching header stream"; + return -1; + } + if (importer->empty_hdr_workaround && + (dsync_mail_hdr_hash_is_empty(change->hdr_hash) || + dsync_mail_hdr_hash_is_empty(hdr_hash))) { + *result_r = "Empty headers found with workaround enabled - assuming a match"; + return 1; + } else if (strcmp(change->hdr_hash, hdr_hash) == 0) { + *result_r = "Headers hashes match"; + return 1; + } else { + *result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)", + change->hdr_hash, hdr_hash); + return 0; + } +} + +static bool +dsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + const struct dsync_mail_change *local_change; + + if (*change->guid == '\0') { + /* remote doesn't support GUIDs, can't verify expunge */ + *result_r = "GUIDs not supported, can't verify expunge"; + return FALSE; + } + + /* local message is expunged. see if we can find its GUID from + transaction log and check if the GUIDs match. The GUID in + log is a 128bit GUID, so we may need to convert the remote's + GUID string to 128bit GUID first. */ + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL || local_change->guid == NULL) { + *result_r = "Expunged local mail's GUID not found"; + return FALSE; + } + + i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + if (dsync_mail_change_guid_equals(importer, local_change, + change->guid, NULL)) { + importer->last_common_uid = change->uid; + *result_r = "Expunged local mail's GUID matches remote"; + } else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + dsync_mailbox_common_uid_found(importer); + *result_r = "Expunged local mail's GUID doesn't match remote GUID"; + } else { + /* GUID mismatch for two expunged mails. dsync can't update + GUIDs for already expunged messages, so we can't immediately + determine that the rest of the messages are a mismatch. so + for now we'll just skip over this pair. */ + *result_r = "Expunged mails' GUIDs don't match - delaying decision"; + /* NOTE: the return value here doesn't matter, because the only + caller that checks for it never reaches this code path */ + } + return TRUE; +} + +static void +dsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + i_assert(importer->revert_local_changes); + + /* mail exists on remote, but not locally. we'll need to + insert this mail back, which means deleting the whole + mailbox and resyncing. */ + i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally", + mailbox_get_vname(importer->box), + change->uid, change->guid); + importer->delete_mailbox = TRUE; + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; +} + +static void +dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + int ret; + + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || + ((change->received_timestamp > 0 || + (importer->sync_since_timestamp == 0 && + importer->sync_until_timestamp == 0)) && + (change->virtual_size != UOFF_T_MAX || importer->sync_max_size == 0))); + + /* try to find the matching local mail */ + if (!importer_next_mail(importer, change->uid)) { + /* no more local mails. we can still try to match + expunged mails though. */ + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* mail doesn't exist remotely either, don't bother + looking it up locally. */ + *result_r = "Expunged mail not found locally"; + return; + } + i_assert(change->guid != NULL); + if (!dsync_mailbox_import_want_change(importer, change, result_r)) + ; + else if (importer->local_uid_next <= change->uid) { + dsync_mailbox_common_uid_found(importer); + *result_r = "Mail's UID is above local UIDNEXT"; + } else if (importer->revert_local_changes) { + dsync_mailbox_revert_missing(importer, change); + *result_r = "Reverting local change by deleting mailbox"; + } else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) { + /* it's unknown if this mail existed locally and was + expunged. since we don't want to lose any mails, + assume that we need to preserve the mail. use the + last message with a matching GUID as the last common + UID. */ + dsync_mailbox_common_uid_found(importer); + } + *result_r = t_strdup_printf("%s - No more local mails found", *result_r); + return; + } + + if (change->guid == NULL) { + /* we can't know if this UID matches */ + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + *result_r = "Expunged mail has no GUID, can't verify it"; + return; + } + if (importer->cur_mail->uid == change->uid) { + /* we have a matching local UID. check GUID to see if it's + really the same mail or not */ + if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) { + /* unknown */ + return; + } + if (ret > 0) { + importer->last_common_uid = change->uid; + } else if (!importer->revert_local_changes) { + /* mismatch - found the first non-common UID */ + dsync_mailbox_common_uid_found(importer); + } else { + /* mismatch and we want to revert local changes - + need to delete the mailbox. */ + dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r); + } + return; + } + /* mail exists remotely, but doesn't exist locally. */ + if (!dsync_mailbox_import_want_change(importer, change, result_r)) + return; + if (importer->revert_local_changes && + change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + dsync_mailbox_revert_missing(importer, change); + *result_r = "Reverting local change by deleting mailbox"; + } else { + (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r); + } + *result_r = t_strdup_printf("%s (next local mail UID=%u)", + *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid); +} + +int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *result; + + i_assert(!importer->new_uids_assigned); + i_assert(importer->prev_uid < change->uid); + + importer->prev_uid = change->uid; + + if (importer->failed) + return -1; + if (importer->require_full_resync) + return 0; + + if (!importer->last_common_uid_found) { + result = NULL; + dsync_mailbox_find_common_uid(importer, change, &result); + i_assert(result != NULL); + } else { + result = "New mail"; + } + + imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s", + dsync_mail_change_type_names[change->type], + change->guid != NULL ? change->guid : "<unknown>", change->uid, + change->hdr_hash != NULL ? change->hdr_hash : "", result); + + if (importer->failed) + return -1; + if (importer->require_full_resync) + return 0; + + if (importer->last_common_uid_found) { + /* a) uid <= last_common_uid for flag changes and expunges. + this happens only when last_common_uid was originally given + as parameter to importer. + + when we're finding the last_common_uid ourself, + uid>last_common_uid always in here, because + last_common_uid_found=TRUE only after we find the first + mismatch. + + b) uid > last_common_uid for i) new messages, ii) expunges + that were sent "just in case" */ + if (change->uid <= importer->last_common_uid) { + i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE); + } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* ignore */ + return 0; + } else { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE); + } + } else { + /* a) uid < last_common_uid can never happen */ + i_assert(change->uid >= importer->last_common_uid); + /* b) uid = last_common_uid if we've verified that the + messages' GUIDs match so far. + + c) uid > last_common_uid: i) TYPE_EXPUNGE change has + GUID=NULL, so we couldn't verify yet if it matches our + local message, ii) local message is expunged and we couldn't + find its GUID */ + if (change->uid > importer->last_common_uid) { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || + importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid); + } + } + + switch (change->type) { + case DSYNC_MAIL_CHANGE_TYPE_SAVE: + dsync_mailbox_import_save(importer, change); + break; + case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE: + dsync_mailbox_import_expunge(importer, change); + break; + case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE: + i_assert(importer->last_common_uid_found); + dsync_mailbox_import_flag_change(importer, change); + break; + } + return importer->failed ? -1 : 0; +} + +static int +importer_new_mail_final_uid_cmp(struct importer_new_mail *const *newmail1, + struct importer_new_mail *const *newmail2) +{ + if ((*newmail1)->final_uid < (*newmail2)->final_uid) + return -1; + if ((*newmail1)->final_uid > (*newmail2)->final_uid) + return 1; + return 0; +} + +static void +dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer) +{ + struct importer_new_mail *newmail; + uint32_t common_uid_next, new_uid; + + common_uid_next = I_MAX(importer->local_uid_next, + importer->remote_uid_next); + array_foreach_elem(&importer->newmails, newmail) { + if (newmail->skip) { + /* already assigned */ + i_assert(newmail->final_uid != 0); + continue; + } + + /* figure out what UID to use for the mail */ + if (newmail->uid_is_usable) { + /* keep the UID */ + new_uid = newmail->final_uid; + } else if (newmail->link != NULL && + newmail->link->uid_is_usable) { + /* we can use the linked message's UID and expunge + this mail */ + new_uid = newmail->link->final_uid; + } else { + i_assert(!importer->revert_local_changes); + new_uid = common_uid_next++; + imp_debug(importer, "UID %u isn't usable, assigning new UID %u", + newmail->final_uid, new_uid); + } + + newmail->final_uid = new_uid; + if (newmail->link != NULL && newmail->link != newmail) { + /* skip processing the linked mail */ + newmail->link->skip = TRUE; + } + } + importer->last_common_uid = common_uid_next-1; + importer->new_uids_assigned = TRUE; + /* Sort the newmails by their final_uid. This is used for tracking + whether an intermediate commit is allowed. */ + array_sort(&importer->newmails, importer_new_mail_final_uid_cmp); +} + +static int +dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer, + struct mail *mail, uint32_t uid, const char *guid, + struct dsync_mail *dmail_r) +{ + const char *error_field, *errstr; + enum mail_error error; + + if (!mail_set_uid(mail, uid)) + return 0; + + /* NOTE: Errors are logged, but they don't cause the entire import + to fail. */ + if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) { + errstr = mailbox_get_last_internal_error(mail->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return 0; + + i_error("Mailbox %s: Can't lookup %s for UID=%u: %s", + mailbox_get_vname(importer->box), + error_field, uid, errstr); + return -1; + } + if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "Unexpected GUID mismatch (3) for UID=%u: %s != %s", + uid, dmail_r->guid, guid)); + return -1; + } + return 1; +} + +static void +dsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer, + uint32_t uid) +{ + i_assert(importer->search_ctx == NULL); + + if (importer->highest_wanted_uid < uid) + importer->highest_wanted_uid = uid; + array_push_back(&importer->wanted_uids, &uid); +} + +static void +dsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer) +{ + struct importer_new_mail *const *newmails; + unsigned int count; + + newmails = array_get(&importer->newmails, &count); + while (importer->first_unsaved_idx < count) { + if (!newmails[importer->first_unsaved_idx]->saved) + break; + importer->first_unsaved_idx++; + } +} + +static void +dsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer, + struct importer_new_mail *newmail) +{ + dsync_mailbox_import_saved_uid(importer, newmail->final_uid); + newmail->saved = TRUE; + + dsync_mailbox_import_update_first_saved(importer); + importer->saves_since_commit++; + /* we can commit only if all the upcoming mails will have UIDs that + are larger than we're committing. + + Note that if any existing UIDs have been changed, the new UID is + usually higher than anything that is being saved so we can't do + an intermediate commit. It's too much extra work to try to handle + that situation. So here this never happens, because then + array_count(wanted_uids) is always higher than first_unsaved_idx. */ + if (importer->saves_since_commit >= importer->commit_msgs_interval && + importer->first_unsaved_idx == array_count(&importer->wanted_uids)) { + if (dsync_mailbox_import_commit(importer, FALSE) < 0) + importer->failed = TRUE; + importer->saves_since_commit = 0; + } +} + +static bool +dsync_msg_change_uid(struct dsync_mailbox_importer *importer, + uint32_t old_uid, uint32_t new_uid) +{ + struct mail_save_context *save_ctx; + + IMPORTER_DEBUG_CHANGE(importer); + + if (!mail_set_uid(importer->mail, old_uid)) + return FALSE; + + save_ctx = mailbox_save_alloc(importer->ext_trans); + mailbox_save_copy_flags(save_ctx, importer->mail); + mailbox_save_set_uid(save_ctx, new_uid); + if (mailbox_move(&save_ctx, importer->mail) < 0) + return FALSE; + dsync_mailbox_import_saved_uid(importer, new_uid); + return TRUE; +} + +static bool +dsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer, + ARRAY_TYPE(seq_range) *unwanted_uids, + uint32_t wanted_uid) +{ + const struct seq_range *range; + unsigned int count, n; + struct seq_range_iter iter; + uint32_t uid; + + /* optimize by first trying to use the latest UID */ + range = array_get(unwanted_uids, &count); + if (count == 0) + return FALSE; + if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) { + seq_range_array_remove(unwanted_uids, range[count-1].seq2); + return TRUE; + } + if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED) + seq_range_array_remove(unwanted_uids, range[count-1].seq2); + + /* now try to use any of them by iterating through them. (would be + easier&faster to just iterate backwards, but probably too much + trouble to add such API) */ + n = 0; seq_range_array_iter_init(&iter, unwanted_uids); + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + if (dsync_msg_change_uid(importer, uid, wanted_uid)) { + seq_range_array_remove(unwanted_uids, uid); + return TRUE; + } + if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED) + seq_range_array_remove(unwanted_uids, uid); + } + return FALSE; +} + +static bool +dsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer, + struct importer_new_mail *all_newmails, + ARRAY_TYPE(seq_range) *local_uids, + ARRAY_TYPE(seq_range) *wanted_uids) +{ + ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids; + struct seq_range_iter local_iter, wanted_iter; + unsigned int local_n, wanted_n; + uint32_t local_uid, wanted_uid; + struct importer_new_mail *mail; + struct dsync_mail dmail; + + if (array_count(local_uids) == 0) + return FALSE; + + local_n = wanted_n = 0; + seq_range_array_iter_init(&local_iter, local_uids); + seq_range_array_iter_init(&wanted_iter, wanted_uids); + + /* wanted_uids contains UIDs that need to exist at the end. those that + don't already exist in local_uids have a higher UID than any + existing local UID */ + t_array_init(&assigned_uids, array_count(wanted_uids)); + t_array_init(&unwanted_uids, 8); + while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) { + if (seq_range_array_iter_nth(&wanted_iter, wanted_n, + &wanted_uid)) { + if (local_uid == wanted_uid) { + /* we have exactly the UID we want. keep it. */ + seq_range_array_add(&assigned_uids, wanted_uid); + wanted_n++; + continue; + } + i_assert(local_uid < wanted_uid); + } + /* we no longer want this local UID. */ + seq_range_array_add(&unwanted_uids, local_uid); + } + + /* reuse as many existing messages as possible by changing their UIDs */ + while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) { + if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids, + wanted_uid)) + break; + seq_range_array_add(&assigned_uids, wanted_uid); + wanted_n++; + } + + /* expunge all unwanted messages */ + local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids); + while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) { + IMPORTER_DEBUG_CHANGE(importer); + if (mail_set_uid(importer->mail, local_uid)) + mail_expunge(importer->mail); + } + + /* mark mails whose UIDs we got to be skipped over later */ + for (mail = all_newmails; mail != NULL; mail = mail->next) { + if (!mail->skip && + seq_range_exists(&assigned_uids, mail->final_uid)) + mail->skip = TRUE; + } + + if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) { + /* we've assigned all wanted UIDs */ + return TRUE; + } + + /* try to find one existing message that we can use to copy to the + other instances */ + local_n = 0; seq_range_array_iter_init(&local_iter, local_uids); + while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) { + if (dsync_mailbox_import_local_uid(importer, importer->mail, + local_uid, all_newmails->guid, + &dmail) > 0) { + if (dsync_mailbox_save_newmails(importer, &dmail, + all_newmails, FALSE)) + return TRUE; + } + } + return FALSE; +} + +static bool +dsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer, + struct importer_new_mail *all_newmails) +{ + struct dsync_mail dmail; + + if (all_newmails->virtual_all_uid == 0) + return FALSE; + + if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail, + all_newmails->virtual_all_uid, + all_newmails->guid, &dmail) > 0) { + if (dsync_mailbox_save_newmails(importer, &dmail, + all_newmails, FALSE)) + return TRUE; + } + return FALSE; +} + +static bool +dsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer, + struct importer_new_mail *all_newmails) +{ + ARRAY_TYPE(seq_range) local_uids, wanted_uids; + struct dsync_mail_request *request; + struct importer_new_mail *mail; + const char *request_guid = NULL; + uint32_t request_uid = 0; + + i_assert(all_newmails != NULL); + + /* get the list of the current local UIDs and the wanted UIDs. + find the first remote instance that we can request in case there are + no local instances */ + t_array_init(&local_uids, 8); + t_array_init(&wanted_uids, 8); + for (mail = all_newmails; mail != NULL; mail = mail->next) { + if (mail->uid_in_local) + seq_range_array_add(&local_uids, mail->local_uid); + else if (request_guid == NULL) { + if (*mail->guid != '\0') + request_guid = mail->guid; + request_uid = mail->remote_uid; + i_assert(request_uid != 0); + } + if (!mail->skip) + seq_range_array_add(&wanted_uids, mail->final_uid); + } + i_assert(array_count(&wanted_uids) > 0); + + if (!dsync_mailbox_import_try_local(importer, all_newmails, + &local_uids, &wanted_uids) && + !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) { + /* no local instance. request from remote */ + IMPORTER_DEBUG_CHANGE(importer); + if (importer->want_mail_requests) { + request = array_append_space(&importer->mail_requests); + request->guid = request_guid; + request->uid = request_uid; + } + return FALSE; + } + /* successfully handled all the mails locally */ + importer->import_pos++; + return TRUE; +} + +static void +dsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer) +{ + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct importer_new_mail *newmail; + struct mail *mail; + const char *guid; + + if (mailbox_sync(importer->virtual_all_box, 0) < 0) { + i_error("Couldn't sync \\All mailbox '%s': %s", + mailbox_get_vname(importer->virtual_all_box), + mailbox_get_last_internal_error(importer->virtual_all_box, NULL)); + return; + } + + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + + importer->virtual_trans = + mailbox_transaction_begin(importer->virtual_all_box, + importer->transaction_flags, + __func__); + search_ctx = mailbox_search_init(importer->virtual_trans, search_args, + NULL, MAIL_FETCH_GUID, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) { + /* ignore errors */ + continue; + } + newmail = hash_table_lookup(importer->import_guids, guid); + if (newmail != NULL && newmail->virtual_all_uid == 0) + newmail->virtual_all_uid = mail->uid; + } + if (mailbox_search_deinit(&search_ctx) < 0) { + i_error("Couldn't search \\All mailbox '%s': %s", + mailbox_get_vname(importer->virtual_all_box), + mailbox_get_last_internal_error(importer->virtual_all_box, NULL)); + } + + importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL); +} + +static void +dsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer) +{ + struct hash_iterate_context *iter; + const char *key; + void *key2; + struct importer_new_mail *mail; + + if (importer->virtual_all_box != NULL && + hash_table_count(importer->import_guids) > 0) { + /* find UIDs in \All mailbox for all wanted GUIDs. */ + dsync_mailbox_import_find_virtual_uids(importer); + } + + iter = hash_table_iterate_init(importer->import_guids); + while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) { + T_BEGIN { + if (dsync_mailbox_import_handle_mail(importer, mail)) + hash_table_remove(importer->import_guids, key); + } T_END; + } + hash_table_iterate_deinit(&iter); + + iter = hash_table_iterate_init(importer->import_uids); + while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) { + T_BEGIN { + if (dsync_mailbox_import_handle_mail(importer, mail)) + hash_table_remove(importer->import_uids, key2); + } T_END; + } + hash_table_iterate_deinit(&iter); + if (!importer->mails_have_guids) { + array_foreach_elem(&importer->newmails, mail) { + if (mail->uid_in_local) + (void)dsync_mailbox_import_handle_mail(importer, mail); + } + } +} + +int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer) +{ + i_assert(!importer->new_uids_assigned); + + if (!importer->last_common_uid_found) { + /* handle pending expunges and flag updates */ + dsync_mailbox_common_uid_found(importer); + } + /* skip common local mails */ + (void)importer_next_mail(importer, importer->last_common_uid+1); + /* if there are any local mails left, add them to newmails list */ + while (importer->cur_mail != NULL && !importer->failed) + (void)dsync_mailbox_try_save(importer, NULL); + + if (importer->search_ctx != NULL) { + if (mailbox_search_deinit(&importer->search_ctx) < 0) { + i_error("Mailbox %s: Search failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } + } + importer->import_count = hash_table_count(importer->import_guids) + + hash_table_count(importer->import_uids); + + dsync_mailbox_import_assign_new_uids(importer); + /* save mails from local sources where possible, + request the rest from remote */ + if (!importer->failed) + dsync_mailbox_import_handle_local_mails(importer); + return importer->failed ? -1 : 0; +} + +const struct dsync_mail_request * +dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer) +{ + const struct dsync_mail_request *requests; + unsigned int count; + + requests = array_get(&importer->mail_requests, &count); + if (importer->mail_request_idx == count) + return NULL; + return &requests[importer->mail_request_idx++]; +} + +static const char *const * +dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change) +{ + ARRAY_TYPE(const_string) keywords; + const char *const *changes; + unsigned int i, count; + + if (!array_is_created(&change->keyword_changes)) + return NULL; + + changes = array_get(&change->keyword_changes, &count); + t_array_init(&keywords, count); + for (i = 0; i < count; i++) { + if (changes[i][0] == KEYWORD_CHANGE_ADD || + changes[i][0] == KEYWORD_CHANGE_FINAL || + changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) { + const char *name = changes[i]+1; + + array_push_back(&keywords, &name); + } + } + if (array_count(&keywords) == 0) + return NULL; + + array_append_zero(&keywords); + return array_front(&keywords); +} + +static void +dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer, + struct mail_save_context *save_ctx, + const struct dsync_mail_change *change) +{ + const char *const *keyword_names; + struct mail_keywords *keywords; + + keyword_names = dsync_mailbox_get_final_keywords(change); + keywords = keyword_names == NULL ? NULL : + mailbox_keywords_create_valid(importer->box, + keyword_names); + mailbox_save_set_flags(save_ctx, change->final_flags, keywords); + if (keywords != NULL) + mailbox_keywords_unref(&keywords); + + if (change->modseq > 1) { + (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE); + mailbox_save_set_min_modseq(save_ctx, change->modseq); + } + /* FIXME: if there already are private flags, they get lost because + saving can't handle updating private index. they get added on the + next sync though. if this is fixed here, set min_pvt_modseq also. */ +} + +static int +dsync_msg_try_copy(struct dsync_mailbox_importer *importer, + struct mail_save_context **save_ctx_p, + struct importer_new_mail **all_newmails_forcopy) +{ + struct importer_new_mail *inst; + + for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) { + if (inst->uid_in_local && !inst->copy_failed && + mail_set_uid(importer->mail, inst->local_uid)) { + if (mailbox_copy(save_ctx_p, importer->mail) < 0) { + enum mail_error error; + const char *errstr; + + errstr = mailbox_get_last_internal_error(importer->box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_warning("Failed to copy mail from UID=%u: " + "%s - falling back to other means", + inst->local_uid, errstr); + } + inst->copy_failed = TRUE; + return -1; + } + *all_newmails_forcopy = inst; + return 1; + } + } + *all_newmails_forcopy = NULL; + return 0; +} + +static void +dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx, + const struct dsync_mail *mail) +{ + if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') + mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); + if (mail->pop3_order > 0) + mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); + mailbox_save_set_received_date(save_ctx, mail->received_date, 0); +} + +static struct mail_save_context * +dsync_mailbox_save_init(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail) +{ + struct mail_save_context *save_ctx; + + save_ctx = mailbox_save_alloc(importer->ext_trans); + mailbox_save_set_uid(save_ctx, newmail->final_uid); + if (*mail->guid != '\0') + mailbox_save_set_guid(save_ctx, mail->guid); + if (mail->saved_date != 0) + mailbox_save_set_save_date(save_ctx, mail->saved_date); + dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); + + if (!mail->minimal_fields) + dsync_mailbox_save_set_nonminimal(save_ctx, mail); + return save_ctx; +} + +static bool +dsync_mailbox_save_body(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail, + struct importer_new_mail **all_newmails_forcopy, + bool remote_mail) +{ + struct mail_save_context *save_ctx; + struct istream *input; + ssize_t ret; + bool save_failed = FALSE; + + /* try to save the mail by copying an existing mail */ + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) { + if (save_ctx == NULL) + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + } + if (ret <= 0 && mail->input_mail != NULL) { + /* copy using the source mail */ + i_assert(mail->input_mail->uid == mail->input_mail_uid); + if (mailbox_copy(&save_ctx, mail->input_mail) == 0) + ret = 1; + else { + enum mail_error error; + const char *errstr; + + errstr = mailbox_get_last_internal_error(importer->box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_warning("Failed to copy source UID=%u mail: " + "%s - falling back to regular saving", + mail->input_mail->uid, errstr); + } + ret = -1; + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + } + + } + if (ret > 0) { + i_assert(save_ctx == NULL); + dsync_mailbox_import_saved_newmail(importer, newmail); + return TRUE; + } + /* fallback to saving from remote stream */ + if (!remote_mail) { + /* the mail isn't remote yet. we were just trying to copy a + local mail to avoid downloading the remote mail. */ + mailbox_save_cancel(&save_ctx); + return FALSE; + } + if (mail->minimal_fields) { + struct dsync_mail mail2; + const char *error_field; + + i_assert(mail->input_mail != NULL); + + if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2, + &error_field) < 0) { + i_error("Mailbox %s: Failed to read mail %s uid=%u: %s", + mailbox_get_vname(importer->box), + error_field, mail->uid, + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + mailbox_save_cancel(&save_ctx); + return TRUE; + } + dsync_mailbox_save_set_nonminimal(save_ctx, &mail2); + input = mail2.input; + } else { + input = mail->input; + } + + if (input == NULL) { + /* it was just expunged in remote, skip it */ + mailbox_save_cancel(&save_ctx); + return TRUE; + } + + i_stream_seek(input, 0); + if (mailbox_save_begin(&save_ctx, input) < 0) { + i_error("Mailbox %s: Saving failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + return TRUE; + } + while ((ret = i_stream_read(input)) > 0 || ret == -2) { + if (mailbox_save_continue(save_ctx) < 0) { + save_failed = TRUE; + ret = -1; + break; + } + } + i_assert(ret == -1); + + if (input->stream_errno != 0) { + i_error("Mailbox %s: read(msg input) failed: %s", + mailbox_get_vname(importer->box), + i_stream_get_error(input)); + mailbox_save_cancel(&save_ctx); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } else if (save_failed) { + i_error("Mailbox %s: Saving failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + mailbox_save_cancel(&save_ctx); + importer->failed = TRUE; + } else { + i_assert(input->eof); + if (mailbox_save_finish(&save_ctx) < 0) { + i_error("Mailbox %s: Saving failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } else { + dsync_mailbox_import_saved_newmail(importer, newmail); + } + } + return TRUE; +} + +static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *all_newmails, + bool remote_mail) +{ + struct importer_new_mail *newmail, *all_newmails_forcopy; + bool ret = TRUE; + + /* if all_newmails list is large, avoid scanning through the + uninteresting ones for each newmail */ + all_newmails_forcopy = all_newmails; + + /* save all instances of the message */ + for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) { + if (!newmail->skip) T_BEGIN { + if (!dsync_mailbox_save_body(importer, mail, newmail, + &all_newmails_forcopy, + remote_mail)) + ret = FALSE; + } T_END; + } + return ret; +} + +int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail) +{ + struct importer_new_mail *all_newmails; + + i_assert(mail->input == NULL || mail->input->seekable); + i_assert(importer->new_uids_assigned); + + if (importer->failed) + return -1; + if (importer->require_full_resync) + return 0; + + imp_debug(importer, "Import mail body for GUID=%s UID=%u", + mail->guid, mail->uid); + + all_newmails = *mail->guid != '\0' ? + hash_table_lookup(importer->import_guids, mail->guid) : + hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid)); + if (all_newmails == NULL) { + if (importer->want_mail_requests) { + i_error("Mailbox %s: Remote sent unwanted message body for " + "GUID=%s UID=%u", + mailbox_get_vname(importer->box), + mail->guid, mail->uid); + } else { + imp_debug(importer, "Skip unwanted mail body for " + "GUID=%s UID=%u", mail->guid, mail->uid); + } + return 0; + } + if (*mail->guid != '\0') + hash_table_remove(importer->import_guids, mail->guid); + else { + hash_table_remove(importer->import_uids, + POINTER_CAST(mail->uid)); + } + importer->import_pos++; + if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE)) + i_unreached(); + return importer->failed ? -1 : 0; +} + +static int +reassign_uids_in_seq_range(struct dsync_mailbox_importer *importer, + const ARRAY_TYPE(seq_range) *unwanted_uids) +{ + struct mailbox *box = importer->box; + const enum mailbox_transaction_flags trans_flags = + importer->transaction_flags | + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; + struct mailbox_transaction_context *trans; + struct mail_search_args *search_args; + struct mail_search_arg *arg; + struct mail_search_context *search_ctx; + struct mail_save_context *save_ctx; + struct mail *mail; + unsigned int renumber_count = 0; + int ret = 1; + + if (array_count(unwanted_uids) == 0) + return 1; + + if (importer->debug) T_BEGIN { + string_t *str = t_str_new(256); + imap_write_seq_range(str, unwanted_uids); + imp_debug(importer, "Reassign UIDs: %s", str_c(str)); + } T_END; + + search_args = mail_search_build_init(); + arg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&arg->value.seqset, search_args->pool, + array_count(unwanted_uids)); + array_append_array(&arg->value.seqset, unwanted_uids); + + trans = mailbox_transaction_begin(box, trans_flags, __func__); + search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + save_ctx = mailbox_save_alloc(trans); + mailbox_save_copy_flags(save_ctx, mail); + if (mailbox_move(&save_ctx, mail) < 0) { + i_error("Mailbox %s: Couldn't move mail within mailbox: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &importer->mail_error)); + ret = -1; + } else if (ret > 0) { + ret = 0; + } + renumber_count++; + } + if (mailbox_search_deinit(&search_ctx) < 0) { + i_error("Mailbox %s: mail search failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &importer->mail_error)); + ret = -1; + } + + if (mailbox_transaction_commit(&trans) < 0) { + i_error("Mailbox %s: UID reassign commit failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &importer->mail_error)); + ret = -1; + } + if (ret == 0) { + imp_debug(importer, "Mailbox %s: Change during sync: " + "Renumbered %u of %u unwanted UIDs", + mailbox_get_vname(box), + renumber_count, array_count(unwanted_uids)); + } + return ret; +} + +static int +reassign_unwanted_uids(struct dsync_mailbox_importer *importer, + const char **changes_during_sync_r) +{ + ARRAY_TYPE(seq_range) unwanted_uids; + const uint32_t *wanted_uids, *saved_uids; + uint32_t highest_seen_uid; + unsigned int i, wanted_count, saved_count; + int ret = 0; + + wanted_uids = array_get(&importer->wanted_uids, &wanted_count); + saved_uids = array_get(&importer->saved_uids, &saved_count); + i_assert(wanted_count == saved_count); + if (wanted_count == 0) + return 0; + /* wanted_uids contains the UIDs we tried to save mails with. + if nothing changed during dsync, we should have the expected UIDs + (saved_uids) and all is well. + + if any new messages got inserted during dsync, we'll need to fix up + the UIDs and let the next dsync fix up the other side. for example: + + remote uids = 5,7,9 = wanted_uids + remote uidnext = 12 + locally added new uid=5 -> + saved_uids = 10,7,9 + + we'll now need to reassign UIDs 5 and 10. to be fully future-proof + we'll reassign all UIDs between [original local uidnext .. highest + UID we think we know] that aren't in saved_uids. */ + + /* create uidset for the list of UIDs we don't want to exist */ + t_array_init(&unwanted_uids, 8); + highest_seen_uid = I_MAX(importer->remote_uid_next-1, + importer->highest_wanted_uid); + i_assert(importer->local_uid_next <= highest_seen_uid); + seq_range_array_add_range(&unwanted_uids, + importer->local_uid_next, highest_seen_uid); + for (i = 0; i < wanted_count; i++) { + i_assert(i < wanted_count); + if (saved_uids[i] == wanted_uids[i]) + seq_range_array_remove(&unwanted_uids, saved_uids[i]); + } + + ret = reassign_uids_in_seq_range(importer, &unwanted_uids); + if (ret == 0) { + *changes_during_sync_r = t_strdup_printf( + "%u UIDs changed due to UID conflicts", + seq_range_count(&unwanted_uids)); + /* conflicting changes during sync, revert our last-common-uid + back to a safe value. */ + importer->last_common_uid = importer->local_uid_next - 1; + } + return ret < 0 ? -1 : 0; +} + +static int +dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final) +{ + struct mail_transaction_commit_changes changes; + struct seq_range_iter iter; + uint32_t uid; + unsigned int n; + int ret = importer->failed ? -1 : 0; + + mail_free(&importer->mail); + mail_free(&importer->ext_mail); + + /* commit saves */ + if (mailbox_transaction_commit_get_changes(&importer->ext_trans, + &changes) < 0) { + i_error("Mailbox %s: Save commit failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, &importer->mail_error)); + /* removed wanted_uids that weren't actually saved */ + array_delete(&importer->wanted_uids, + array_count(&importer->saved_uids), + array_count(&importer->wanted_uids) - + array_count(&importer->saved_uids)); + mailbox_transaction_rollback(&importer->trans); + ret = -1; + } else { + /* remember the UIDs that were successfully saved */ + if (importer->debug) T_BEGIN { + string_t *str = t_str_new(256); + imap_write_seq_range(str, &changes.saved_uids); + imp_debug(importer, "Saved UIDs: %s", str_c(str)); + } T_END; + seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) + array_push_back(&importer->saved_uids, &uid); + pool_unref(&changes.pool); + + /* commit flag changes and expunges */ + if (mailbox_transaction_commit(&importer->trans) < 0) { + i_error("Mailbox %s: Commit failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + ret = -1; + } + } + + if (!final) + dsync_mailbox_import_transaction_begin(importer); + return ret; +} + +static int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer, + const char **changes_during_sync_r) +{ + struct mailbox_update update; + int ret; + + ret = dsync_mailbox_import_commit(importer, TRUE); + + if (ret == 0) { + /* update mailbox metadata if we successfully saved + everything. */ + i_zero(&update); + update.min_next_uid = importer->remote_uid_next; + update.min_first_recent_uid = + I_MIN(importer->last_common_uid+1, + importer->remote_first_recent_uid); + update.min_highest_modseq = importer->remote_highest_modseq; + update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq; + + imp_debug(importer, "Finish update: min_next_uid=%u " + "min_first_recent_uid=%u min_highest_modseq=%"PRIu64" " + "min_highest_pvt_modseq=%"PRIu64, + update.min_next_uid, update.min_first_recent_uid, + update.min_highest_modseq, + update.min_highest_pvt_modseq); + + if (mailbox_update(importer->box, &update) < 0) { + i_error("Mailbox %s: Update failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + ret = -1; + } + } + + /* sync mailbox to finish flag changes and expunges. */ + if (mailbox_sync(importer->box, 0) < 0) { + i_error("Mailbox %s: Sync failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + ret = -1; + } + if (ret == 0) { + /* give new UIDs to messages that got saved with unwanted UIDs. + do it only if the whole transaction succeeded. */ + if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0) + ret = -1; + } + return ret; +} + +static void +dsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer) +{ + struct hash_iterate_context *iter; + const char *key; + struct importer_new_mail *mail; + + iter = hash_table_iterate_init(importer->import_guids); + while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) { + for (; mail != NULL; mail = mail->next) { + if (mail->skip) + continue; + + i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)", + mailbox_get_vname(importer->box), + mail->guid, mail->remote_uid); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer) +{ + struct hash_iterate_context *iter; + void *key; + struct importer_new_mail *mail; + + iter = hash_table_iterate_init(importer->import_uids); + while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) { + for (; mail != NULL; mail = mail->next) { + if (mail->skip) + continue; + + i_error("Mailbox %s: Remote didn't send mail UID=%u", + mailbox_get_vname(importer->box), + mail->remote_uid); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } + } + hash_table_iterate_deinit(&iter); +} + +int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer, + bool success, + uint32_t *last_common_uid_r, + uint64_t *last_common_modseq_r, + uint64_t *last_common_pvt_modseq_r, + uint32_t *last_messages_count_r, + const char **changes_during_sync_r, + bool *require_full_resync_r, + enum mail_error *error_r) +{ + struct dsync_mailbox_importer *importer = *_importer; + struct mailbox_status status; + int ret; + + *_importer = NULL; + *changes_during_sync_r = NULL; + *require_full_resync_r = importer->require_full_resync; + + if ((!success || importer->require_full_resync) && !importer->failed) { + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } + + if (!importer->new_uids_assigned && !importer->failed) + dsync_mailbox_import_assign_new_uids(importer); + + if (!importer->failed) { + dsync_mailbox_import_check_missing_guid_imports(importer); + dsync_mailbox_import_check_missing_uid_imports(importer); + } + + if (importer->search_ctx != NULL) { + if (mailbox_search_deinit(&importer->search_ctx) < 0) { + i_error("Mailbox %s: Search failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } + } + if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0) + importer->failed = TRUE; + + if (importer->virtual_mail != NULL) + mail_free(&importer->virtual_mail); + if (importer->virtual_trans != NULL) + (void)mailbox_transaction_commit(&importer->virtual_trans); + + hash_table_destroy(&importer->import_guids); + hash_table_destroy(&importer->import_uids); + array_free(&importer->maybe_expunge_uids); + array_free(&importer->maybe_saves); + array_free(&importer->wanted_uids); + array_free(&importer->saved_uids); + array_free(&importer->newmails); + if (array_is_created(&importer->mail_requests)) + array_free(&importer->mail_requests); + + *last_common_uid_r = importer->last_common_uid; + if (*changes_during_sync_r == NULL) { + *last_common_modseq_r = importer->remote_highest_modseq; + *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq; + } else { + /* local changes occurred during dsync. we exported changes up + to local_initial_highestmodseq, so all of the changes have + happened after it. we want the next run to see those changes, + so return it as the last common modseq */ + *last_common_modseq_r = importer->local_initial_highestmodseq; + *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq; + } + if (importer->delete_mailbox) { + if (mailbox_delete(importer->box) < 0) { + i_error("Couldn't delete mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } + *last_messages_count_r = 0; + } else { + mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status); + *last_messages_count_r = status.messages; + } + + i_assert(importer->failed == (importer->mail_error != 0)); + ret = importer->failed ? -1 : 0; + *error_r = importer->mail_error; + pool_unref(&importer->pool); + return ret; +} + +const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer) +{ + if (importer->search_ctx != NULL) + return ""; + return t_strdup_printf("%u/%u", importer->import_pos, + importer->import_count); +} diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h new file mode 100644 index 0000000..6a2020e --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -0,0 +1,63 @@ +#ifndef DSYNC_MAILBOX_IMPORT_H +#define DSYNC_MAILBOX_IMPORT_H + +#include "mail-error.h" + +enum dsync_mailbox_import_flags { + DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN = 0x01, + DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02, + DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x04, + DSYNC_MAILBOX_IMPORT_FLAG_DEBUG = 0x08, + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10, + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128 = 0x20, + DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY = 0x40, + DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND = 0x100 +}; + +struct mailbox; +struct dsync_mailbox_attribute; +struct dsync_mail; +struct dsync_mail_change; +struct dsync_transaction_log_scan; + +struct dsync_mailbox_importer * +dsync_mailbox_import_init(struct mailbox *box, + struct mailbox *virtual_all_box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + uint64_t last_common_pvt_modseq, + uint32_t remote_uid_next, + uint32_t remote_first_recent_uid, + uint64_t remote_highest_modseq, + uint64_t remote_highest_pvt_modseq, + time_t sync_since_timestamp, + time_t sync_until_timestamp, + uoff_t sync_max_size, + const char *sync_flag, + unsigned int commit_msgs_interval, + enum dsync_mailbox_import_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers); +int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, + const struct dsync_mailbox_attribute *attr); +int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change); +int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer); +const struct dsync_mail_request * +dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer); +int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail); +int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer, + bool success, + uint32_t *last_common_uid_r, + uint64_t *last_common_modseq_r, + uint64_t *last_common_pvt_modseq_r, + uint32_t *last_messages_count_r, + const char **changes_during_sync_r, + bool *require_full_resync_r, + enum mail_error *error_r); + +const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-state.c b/src/doveadm/dsync/dsync-mailbox-state.c new file mode 100644 index 0000000..cca24d4 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-state.c @@ -0,0 +1,127 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "base64.h" +#include "crc32.h" +#include "hash.h" +#include "dsync-mailbox-state.h" + +#define DSYNC_STATE_MAJOR_VERSION 1 +#define DSYNC_STATE_MINOR_VERSION 0 + +#define V0_MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8) +#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8 + 4) + +static void put_uint32(buffer_t *output, uint32_t num) +{ + uint8_t tmp[sizeof(uint32_t)]; + + cpu32_to_le_unaligned(num, tmp); + + buffer_append(output, tmp, sizeof(tmp)); +} + +void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states, + string_t *output) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_state *state; + uint8_t *guid; + buffer_t *buf = t_buffer_create(128); + uint32_t crc = 0; + + buffer_append_c(buf, DSYNC_STATE_MAJOR_VERSION); + buffer_append_c(buf, DSYNC_STATE_MINOR_VERSION); + buffer_append_c(buf, '\0'); + buffer_append_c(buf, '\0'); + + iter = hash_table_iterate_init(states); + while (hash_table_iterate(iter, states, &guid, &state)) { + buffer_append(buf, state->mailbox_guid, + sizeof(state->mailbox_guid)); + put_uint32(buf, state->last_uidvalidity); + put_uint32(buf, state->last_common_uid); + put_uint32(buf, state->last_common_modseq & 0xffffffffU); + put_uint32(buf, state->last_common_modseq >> 32); + put_uint32(buf, state->last_common_pvt_modseq & 0xffffffffU); + put_uint32(buf, state->last_common_pvt_modseq >> 32); + put_uint32(buf, state->last_messages_count); /* v1 */ + if (buf->used % 3 == 0) { + crc = crc32_data_more(crc, buf->data, buf->used); + base64_encode(buf->data, buf->used, output); + buffer_set_used_size(buf, 0); + } + } + hash_table_iterate_deinit(&iter); + + crc = crc32_data_more(crc, buf->data, buf->used); + put_uint32(buf, crc); + base64_encode(buf->data, buf->used, output); +} + +static int dsync_mailbox_states_retry_import_v0(const buffer_t *buf) +{ + const unsigned char *data = buf->data; + + /* v0 had no version header and no last_messages_count */ + + if ((buf->used-4) % V0_MAILBOX_SIZE != 0 || + le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4)) + return -1; + /* looks like valid v0 format, silently treat it as empty state */ + return 0; +} + +int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states, + pool_t pool, const char *input, + const char **error_r) +{ + struct dsync_mailbox_state *state; + buffer_t *buf; + uint8_t *guid_p; + const unsigned char *data; + unsigned int i, count; + + buf = t_buffer_create(strlen(input)); + if (base64_decode(input, strlen(input), NULL, buf) < 0) { + *error_r = "Invalid base64 data"; + return -1; + } + /* v1: 4 byte header, mailboxes[], CRC32 */ + data = buf->data; + + if (buf->used == 4 && le32_to_cpu_unaligned(data) == 0) { + /* v0: Empty state */ + return 0; + } + if (buf->used < 8) { + *error_r = "Input too small"; + return -1; + } + + if ((buf->used-8) % MAILBOX_SIZE != 0) { + *error_r = "Invalid input size"; + return dsync_mailbox_states_retry_import_v0(buf); + } + + if (le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4)) { + *error_r = "CRC32 mismatch"; + return dsync_mailbox_states_retry_import_v0(buf); + } + data += 4; + count = (buf->used-8) / MAILBOX_SIZE; + + for (i = 0; i < count; i++, data += MAILBOX_SIZE) { + state = p_new(pool, struct dsync_mailbox_state, 1); + memcpy(state->mailbox_guid, data, GUID_128_SIZE); + state->last_uidvalidity = le32_to_cpu_unaligned(data + GUID_128_SIZE); + state->last_common_uid = le32_to_cpu_unaligned(data + GUID_128_SIZE + 4); + state->last_common_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 8); + state->last_common_pvt_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 16); + state->last_messages_count = le32_to_cpu_unaligned(data + GUID_128_SIZE + 24); + guid_p = state->mailbox_guid; + hash_table_insert(states, guid_p, state); + } + return 0; +} diff --git a/src/doveadm/dsync/dsync-mailbox-state.h b/src/doveadm/dsync/dsync-mailbox-state.h new file mode 100644 index 0000000..a01f531 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-state.h @@ -0,0 +1,24 @@ +#ifndef DSYNC_MAILBOX_STATE_H +#define DSYNC_MAILBOX_STATE_H + +#include "guid.h" + +struct dsync_mailbox_state { + guid_128_t mailbox_guid; + uint32_t last_uidvalidity; + uint32_t last_common_uid; + uint64_t last_common_modseq; + uint64_t last_common_pvt_modseq; + uint32_t last_messages_count; + bool changes_during_sync; +}; +ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state); +HASH_TABLE_DEFINE_TYPE(dsync_mailbox_state, uint8_t *, struct dsync_mailbox_state *); + +void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states, + string_t *output); +int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states, + pool_t pool, const char *input, + const char **error_r); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-tree-fill.c b/src/doveadm/dsync/dsync-mailbox-tree-fill.c new file mode 100644 index 0000000..c523e6b --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c @@ -0,0 +1,405 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "guid.h" +#include "str.h" +#include "wildcard-match.h" +#include "mailbox-log.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mailbox-list-iter.h" +#include "dsync-mailbox-tree-private.h" + +static int +dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + struct dsync_mailbox_node **node_r) +{ + struct dsync_mailbox_node *node; + + node = dsync_mailbox_tree_get(tree, info->vname); + if (node->ns == info->ns) + ; + else if (node->ns == NULL) { + i_assert(tree->root.ns == NULL); + node->ns = info->ns; + } else { + i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'", + info->vname, node->ns->prefix, info->ns->prefix); + return -1; + } + *node_r = node; + return 0; +} + +static int +dsync_mailbox_tree_add_exists_node(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + struct dsync_mailbox_node **node_r, + enum mail_error *error_r) +{ + if (dsync_mailbox_tree_add_node(tree, info, node_r) < 0) { + *error_r = MAIL_ERROR_TEMP; + return -1; + } + (*node_r)->existence = DSYNC_MAILBOX_NODE_EXISTS; + return 0; +} + +static int +dsync_mailbox_tree_get_selectable(struct mailbox *box, + struct mailbox_metadata *metadata_r, + struct mailbox_status *status_r) +{ + /* try the fast path */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0) + return -1; + if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0) + return -1; + + i_assert(!guid_128_is_empty(metadata_r->guid)); + if (status_r->uidvalidity != 0) + return 0; + + /* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */ + if (mailbox_sync(box, 0) < 0) + return -1; + if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0) + return -1; + i_assert(status_r->uidvalidity != 0); + return 0; +} + +static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + const guid_128_t box_guid, + enum mail_error *error_r) +{ + struct dsync_mailbox_node *node; + struct mailbox *box; + struct mailbox_metadata metadata; + struct mailbox_status status; + const char *errstr; + enum mail_error error; + int ret = 0; + + if ((info->flags & MAILBOX_NONEXISTENT) != 0) + return 0; + if ((info->flags & MAILBOX_NOSELECT) != 0) { + return !guid_128_is_empty(box_guid) ? 0 : + dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r); + } + + /* get GUID and UIDVALIDITY for selectable mailbox */ + box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY); + if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + switch (error) { + case MAIL_ERROR_NOTFOUND: + /* mailbox was just deleted? */ + break; + case MAIL_ERROR_NOTPOSSIBLE: + /* invalid mbox files? ignore */ + break; + default: + i_error("Failed to access mailbox %s: %s", + info->vname, errstr); + *error_r = error; + ret = -1; + } + mailbox_free(&box); + return ret; + } + mailbox_free(&box); + + if (!guid_128_is_empty(box_guid) && + !guid_128_equals(box_guid, metadata.guid)) { + /* unwanted mailbox */ + return 0; + } + if (dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r) < 0) + return -1; + memcpy(node->mailbox_guid, metadata.guid, + sizeof(node->mailbox_guid)); + node->uid_validity = status.uidvalidity; + node->uid_next = status.uidnext; + return 0; +} + +static struct dsync_mailbox_node * +dsync_mailbox_tree_find_sha(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const guid_128_t sha128) +{ + struct dsync_mailbox_node *node; + + if (!hash_table_is_created(tree->name128_hash)) + dsync_mailbox_tree_build_name128_hash(tree); + + node = hash_table_lookup(tree->name128_hash, sha128); + return node == NULL || node->ns != ns ? NULL : node; +} + +static int +dsync_mailbox_tree_add_change_timestamps(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns) +{ + struct dsync_mailbox_node *node; + struct dsync_mailbox_delete *del; + struct mailbox_log *log; + struct mailbox_log_iter *iter; + const struct mailbox_log_record *rec; + const uint8_t *guid_p; + time_t timestamp; + + log = mailbox_list_get_changelog(ns->list); + if (log == NULL) + return 0; + + iter = mailbox_log_iter_init(log); + while ((rec = mailbox_log_iter_next(iter)) != NULL) { + /* For DELETE_MAILBOX the record_guid is the mailbox GUID. + Otherwise it's 128bit SHA1 of the mailbox vname. */ + node = rec->type == MAILBOX_LOG_RECORD_DELETE_MAILBOX ? NULL : + dsync_mailbox_tree_find_sha(tree, ns, rec->mailbox_guid); + + timestamp = mailbox_log_record_get_timestamp(rec); + switch (rec->type) { + case MAILBOX_LOG_RECORD_DELETE_MAILBOX: + guid_p = rec->mailbox_guid; + if (hash_table_lookup(tree->guid_hash, guid_p) != NULL) { + /* mailbox still exists. maybe it was restored + from backup or something. */ + break; + } + del = array_append_space(&tree->deletes); + del->type = DSYNC_MAILBOX_DELETE_TYPE_MAILBOX; + del->timestamp = timestamp; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + case MAILBOX_LOG_RECORD_DELETE_DIR: + if (node != NULL && + node->existence == DSYNC_MAILBOX_NODE_EXISTS) { + /* directory exists again, skip it */ + break; + } + /* we don't know what directory name was deleted, + just its hash. if the name still exists on the other + dsync side, it can match this deletion to the + name. */ + del = array_append_space(&tree->deletes); + del->type = DSYNC_MAILBOX_DELETE_TYPE_DIR; + del->timestamp = timestamp; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + case MAILBOX_LOG_RECORD_CREATE_DIR: + if (node == NULL) { + /* directory has been deleted again, skip it */ + break; + } + /* notify the remote that we want to keep this + directory created (unless remote has a newer delete + timestamp) */ + node->last_renamed_or_created = timestamp; + break; + case MAILBOX_LOG_RECORD_RENAME: + if (node != NULL) + node->last_renamed_or_created = timestamp; + break; + case MAILBOX_LOG_RECORD_SUBSCRIBE: + if (node != NULL) + node->last_subscription_change = timestamp; + break; + case MAILBOX_LOG_RECORD_UNSUBSCRIBE: + if (node != NULL) { + node->last_subscription_change = timestamp; + break; + } + /* The mailbox is already deleted, but it may still + exist on the other side (even the subscription + alone). */ + del = array_append_space(&tree->deletes); + del->type = DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE; + del->timestamp = timestamp; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + } + } + if (mailbox_log_iter_deinit(&iter) < 0) { + i_error("Mailbox log iteration for namespace '%s' failed", + ns->prefix); + return -1; + } + return 0; +} + +static int +dsync_mailbox_tree_fix_guid_duplicate(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + struct mailbox *box; + struct mailbox_update update; + struct dsync_mailbox_node *change_node; + const char *change_vname; + int ret = 0; + + i_zero(&update); + guid_128_generate(update.mailbox_guid); + + /* just in case the duplication exists in both sides, + make them choose the same node */ + if (strcmp(dsync_mailbox_node_get_full_name(tree, node1), + dsync_mailbox_node_get_full_name(tree, node2)) <= 0) + change_node = node1; + else + change_node = node2; + + change_vname = dsync_mailbox_node_get_full_name(tree, change_node); + i_error("Duplicate mailbox GUID %s for mailboxes %s and %s - " + "giving a new GUID %s to %s", + guid_128_to_string(node1->mailbox_guid), + dsync_mailbox_node_get_full_name(tree, node1), + dsync_mailbox_node_get_full_name(tree, node2), + guid_128_to_string(update.mailbox_guid), change_vname); + + i_assert(node1->ns != NULL && node2->ns != NULL); + box = mailbox_alloc(change_node->ns->list, change_vname, 0); + if (mailbox_update(box, &update) < 0) { + i_error("Couldn't update mailbox %s GUID: %s", + change_vname, mailbox_get_last_internal_error(box, NULL)); + ret = -1; + } else { + memcpy(change_node->mailbox_guid, update.mailbox_guid, + sizeof(change_node->mailbox_guid)); + } + mailbox_free(&box); + return ret; +} + +static bool +dsync_mailbox_info_is_wanted(const struct mailbox_info *info, + const char *box_name, + const char *const *exclude_mailboxes) +{ + const char *const *info_specialuses; + unsigned int i; + + if (exclude_mailboxes == NULL && + (box_name == NULL || box_name[0] != '\\')) + return TRUE; + + info_specialuses = info->special_use == NULL ? NULL : + t_strsplit(info->special_use, " "); + /* include */ + if (box_name != NULL && box_name[0] == '\\') { + if (info_specialuses == NULL || + !str_array_icase_find(info_specialuses, box_name)) + return FALSE; + } + /* exclude */ + if (exclude_mailboxes == NULL) + return TRUE; + for (i = 0; exclude_mailboxes[i] != NULL; i++) { + const char *exclude = exclude_mailboxes[i]; + + if (exclude[0] == '\\') { + /* special-use */ + if (info_specialuses != NULL && + str_array_icase_find(info_specialuses, exclude)) + return FALSE; + } else { + /* mailbox with wildcards */ + if (wildcard_match(info->vname, exclude)) + return FALSE; + } + } + return TRUE; +} + +int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const char *box_name, + const guid_128_t box_guid, + const char *const *exclude_mailboxes, + enum mail_error *error_r) +{ + const enum mailbox_list_iter_flags list_flags = + /* FIXME: we'll skip symlinks, because we can't handle them + currently. in future we could detect them and create them + by creating the symlink. */ + MAILBOX_LIST_ITER_SKIP_ALIASES | + MAILBOX_LIST_ITER_NO_AUTO_BOXES; + const enum mailbox_list_iter_flags subs_list_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_SELECT_SUBSCRIBED | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct mailbox_list_iterate_context *iter; + struct dsync_mailbox_node *node, *dup_node1, *dup_node2; + const struct mailbox_info *info; + const char *list_pattern = + box_name != NULL && box_name[0] != '\\' ? box_name : "*"; + int ret = 0; + + i_assert(mail_namespace_get_sep(ns) == tree->sep); + + /* assign namespace to its root, so it gets copied to children */ + if (ns->prefix_len > 0) { + const char *vname = t_strndup(ns->prefix, ns->prefix_len-1); + node = dsync_mailbox_tree_get(tree, vname); + node->ns = ns; + + struct mailbox_info ns_info = { + .vname = vname, + .ns = ns, + }; + if (dsync_mailbox_tree_add(tree, &ns_info, box_guid, error_r) < 0) + return -1; + } else { + tree->root.ns = ns; + } + + /* first add all of the existing mailboxes */ + iter = mailbox_list_iter_init(ns->list, list_pattern, list_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (dsync_mailbox_info_is_wanted(info, box_name, + exclude_mailboxes)) { + if (dsync_mailbox_tree_add(tree, info, box_guid, error_r) < 0) + ret = -1; + } + } T_END; + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Mailbox listing for namespace '%s' failed: %s", + ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r)); + ret = -1; + } + + /* add subscriptions */ + iter = mailbox_list_iter_init(ns->list, list_pattern, subs_list_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if (dsync_mailbox_tree_add_node(tree, info, &node) == 0) + node->subscribed = TRUE; + else { + *error_r = MAIL_ERROR_TEMP; + ret = -1; + } + } + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Mailbox listing for namespace '%s' failed: %s", + ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r)); + ret = -1; + } + if (ret < 0) + return -1; + + while (dsync_mailbox_tree_build_guid_hash(tree, &dup_node1, + &dup_node2) < 0) { + if (dsync_mailbox_tree_fix_guid_duplicate(tree, dup_node1, dup_node2) < 0) + return -1; + } + + /* add timestamps */ + if (dsync_mailbox_tree_add_change_timestamps(tree, ns) < 0) + return -1; + return 0; +} diff --git a/src/doveadm/dsync/dsync-mailbox-tree-private.h b/src/doveadm/dsync/dsync-mailbox-tree-private.h new file mode 100644 index 0000000..0614151 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree-private.h @@ -0,0 +1,38 @@ +#ifndef DSYNC_MAILBOX_TREE_PRIVATE_H +#define DSYNC_MAILBOX_TREE_PRIVATE_H + +#include "dsync-mailbox-tree.h" + +struct dsync_mailbox_tree { + pool_t pool; + char sep, sep_str[2], remote_sep, alt_char; + char escape_char, remote_escape_char; + /* root node isn't part of the real mailbox tree. its name is "" and + it has no siblings */ + struct dsync_mailbox_node root; + + unsigned int iter_count; + + ARRAY(struct dsync_mailbox_delete) deletes; + + /* guid_128_t => struct dsync_mailbox_node */ + HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_hash; + HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_remotesep_hash; + HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) guid_hash; +}; + +void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree); + +int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1, + struct dsync_mailbox_node *const *n2); + +void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent); +void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node); + +struct dsync_mailbox_tree * +dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src); +bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c new file mode 100644 index 0000000..e4a50ae --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c @@ -0,0 +1,1473 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "str.h" +#include "md5.h" +#include "hex-binary.h" +#include "aqueue.h" +#include "hash.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox-tree-private.h" + +#define TEMP_MAX_NAME_LEN 100 +#define TEMP_SUFFIX_MAX_LEN (sizeof("temp-")-1 + 8) +#define TEMP_SUFFIX_FORMAT "temp-%x" + +struct dsync_mailbox_tree_bfs_iter { + struct dsync_mailbox_tree *tree; + + ARRAY(struct dsync_mailbox_node *) queue_arr; + struct aqueue *queue; + struct dsync_mailbox_node *cur; +}; + +struct dsync_mailbox_tree_sync_ctx { + pool_t pool; + struct dsync_mailbox_tree *local_tree, *remote_tree; + enum dsync_mailbox_trees_sync_type sync_type; + enum dsync_mailbox_trees_sync_flags sync_flags; + unsigned int combined_mailboxes_count; + + ARRAY(struct dsync_mailbox_tree_sync_change) changes; + unsigned int change_idx; + bool failed; +}; + +static struct dsync_mailbox_tree_bfs_iter * +dsync_mailbox_tree_bfs_iter_init(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_bfs_iter *iter; + + iter = i_new(struct dsync_mailbox_tree_bfs_iter, 1); + iter->tree = tree; + i_array_init(&iter->queue_arr, 32); + iter->queue = aqueue_init(&iter->queue_arr.arr); + iter->cur = tree->root.first_child; + return iter; +} + +static bool +dsync_mailbox_tree_bfs_iter_next(struct dsync_mailbox_tree_bfs_iter *iter, + struct dsync_mailbox_node **node_r) +{ + if (iter->cur == NULL) { + if (aqueue_count(iter->queue) == 0) + return FALSE; + iter->cur = array_idx_elem(&iter->queue_arr, + aqueue_idx(iter->queue, 0)); + aqueue_delete_tail(iter->queue); + } + *node_r = iter->cur; + + if (iter->cur->first_child != NULL) + aqueue_append(iter->queue, &iter->cur->first_child); + iter->cur = iter->cur->next; + return TRUE; +} + +static void +dsync_mailbox_tree_bfs_iter_deinit(struct dsync_mailbox_tree_bfs_iter **_iter) +{ + struct dsync_mailbox_tree_bfs_iter *iter = *_iter; + + *_iter = NULL; + + aqueue_deinit(&iter->queue); + array_free(&iter->queue_arr); + i_free(iter); +} + +static void +sync_add_dir_change(struct dsync_mailbox_tree_sync_ctx *ctx, + const struct dsync_mailbox_node *node, + enum dsync_mailbox_tree_sync_type type) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *name; + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + + name = dsync_mailbox_node_get_full_name(ctx->local_tree, node); + + change = array_append_space(&ctx->changes); + change->type = type; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); +} + +static void +sync_add_create_change(struct dsync_mailbox_tree_sync_ctx *ctx, + const struct dsync_mailbox_node *node, const char *name) +{ + struct dsync_mailbox_tree_sync_change *change; + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); + memcpy(change->mailbox_guid, node->mailbox_guid, + sizeof(change->mailbox_guid)); + change->uid_validity = node->uid_validity; +} + +static void sort_siblings(ARRAY_TYPE(dsync_mailbox_node) *siblings) +{ + struct dsync_mailbox_node *const *nodes; + unsigned int i, count; + + array_sort(siblings, dsync_mailbox_node_name_cmp); + + nodes = array_get(siblings, &count); + if (count == 0) + return; + + nodes[0]->parent->first_child = nodes[0]; + for (i = 1; i < count; i++) + nodes[i-1]->next = nodes[i]; + nodes[count-1]->next = NULL; +} + +static void +sync_set_node_deleted(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + const uint8_t *guid_p; + + /* for the rest of this sync assume that the mailbox has + already been deleted */ + guid_p = node->mailbox_guid; + hash_table_remove(tree->guid_hash, guid_p); + + node->existence = DSYNC_MAILBOX_NODE_DELETED; + memset(node->mailbox_guid, 0, sizeof(node->mailbox_guid)); + node->uid_validity = 0; +} + +static void +sync_delete_mailbox_node(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, const char *reason) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *name; + + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && + tree == ctx->local_tree) { + i_debug("brain %c: Deleting mailbox '%s' (GUID %s): %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + dsync_mailbox_node_get_full_name(tree, node), + guid_128_to_string(node->mailbox_guid), reason); + } + + if (tree == ctx->local_tree && + node->existence != DSYNC_MAILBOX_NODE_DELETED) { + /* delete this mailbox locally */ + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX; + change->ns = node->ns; + name = dsync_mailbox_node_get_full_name(tree, node); + change->full_name = p_strdup(ctx->pool, name); + memcpy(change->mailbox_guid, node->mailbox_guid, + sizeof(change->mailbox_guid)); + } + sync_set_node_deleted(tree, node); +} + +static void +sync_delete_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, const char *reason) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *other_node; + const uint8_t *guid_p; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + guid_p = node->mailbox_guid; + other_node = hash_table_lookup(other_tree->guid_hash, guid_p); + if (other_node == NULL) { + /* doesn't exist / already deleted */ + } else { + sync_delete_mailbox_node(ctx, other_tree, other_node, reason); + } + sync_delete_mailbox_node(ctx, tree, node, reason); +} + +static void +sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + bool twoway_sync) +{ + struct dsync_mailbox_tree_bfs_iter *iter; + struct dsync_mailbox_node *node, *parent = NULL; + ARRAY_TYPE(dsync_mailbox_node) siblings; + + t_array_init(&siblings, 64); + + iter = dsync_mailbox_tree_bfs_iter_init(tree); + while (dsync_mailbox_tree_bfs_iter_next(iter, &node)) { + if (node->parent != parent) { + sort_siblings(&siblings); + array_clear(&siblings); + parent = node->parent; + } + if (node->existence == DSYNC_MAILBOX_NODE_DELETED && + !dsync_mailbox_node_is_dir(node)) { + if (twoway_sync) { + /* this mailbox was deleted. delete it from the + other side as well */ + sync_delete_mailbox(ctx, tree, node, + "Mailbox has been deleted"); + } else { + /* treat the node as if it didn't exist. it'll + get either recreated or deleted later. in + any case this function must handle all + existence=DELETED mailbox nodes by changing + them into directories (setting GUID=0) or + we'll assert-crash later */ + sync_set_node_deleted(tree, node); + } + } + ctx->combined_mailboxes_count++; + array_push_back(&siblings, &node); + } + sort_siblings(&siblings); + dsync_mailbox_tree_bfs_iter_deinit(&iter); +} + +static bool node_names_equal(const struct dsync_mailbox_node *n1, + const struct dsync_mailbox_node *n2) +{ + while (n1 != NULL && n2 != NULL) { + if (strcmp(n1->name, n2->name) != 0) + return FALSE; + n1 = n1->parent; + n2 = n2->parent; + } + return n1 == NULL && n2 == NULL; +} + +static void +dsync_mailbox_tree_node_attach_sorted(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + struct dsync_mailbox_node **p; + + node->parent = parent; + for (p = &parent->first_child; *p != NULL; p = &(*p)->next) { + if (dsync_mailbox_node_name_cmp(p, &node) > 0) + break; + } + node->next = *p; + *p = node; +} + +static void +dsync_mailbox_tree_node_move_sorted(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + /* detach from old parent */ + dsync_mailbox_tree_node_detach(node); + /* attach to new parent */ + dsync_mailbox_tree_node_attach_sorted(node, parent); +} + +static struct dsync_mailbox_node * +sorted_tree_get(struct dsync_mailbox_tree *tree, const char *name) +{ + struct dsync_mailbox_node *node, *parent, *ret; + + node = ret = dsync_mailbox_tree_get(tree, name); + while (node->parent != NULL && + node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) { + parent = node->parent; + dsync_mailbox_tree_node_detach(node); + dsync_mailbox_tree_node_attach_sorted(node, parent); + node = parent; + } + return ret; +} + +static struct dsync_mailbox_node * +sync_node_new(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node **pos, + struct dsync_mailbox_node *parent, + const struct dsync_mailbox_node *src) +{ + struct dsync_mailbox_node *node; + + node = p_new(tree->pool, struct dsync_mailbox_node, 1); + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + node->name = p_strdup(tree->pool, src->name); + node->sync_temporary_name = src->sync_temporary_name; + node->ns = src->ns; + node->parent = parent; + node->next = *pos; + *pos = node; + return node; +} + +static struct dsync_mailbox_node * +sorted_tree_get_by_node_name(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + struct dsync_mailbox_node *other_node) +{ + const char *parent_name; + + if (other_node == &other_tree->root) + return &tree->root; + + parent_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + return sorted_tree_get(tree, parent_name); +} + +static bool node_has_child(struct dsync_mailbox_node *parent, const char *name) +{ + struct dsync_mailbox_node *node; + + for (node = parent->first_child; node != NULL; node = node->next) { + if (strcmp(node->name, name) == 0) + return TRUE; + } + return FALSE; +} + +static bool +node_has_existent_children(struct dsync_mailbox_node *node, bool dirs_ok) +{ + for (node = node->first_child; node != NULL; node = node->next) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + (dirs_ok || !dsync_mailbox_node_is_dir(node))) + return TRUE; + if (node_has_existent_children(node, dirs_ok)) + return TRUE; + } + return FALSE; +} + +static bool node_is_existent(struct dsync_mailbox_node *node) +{ + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) + return TRUE; + return node_has_existent_children(node, TRUE); +} + +static bool sync_node_is_namespace_prefix(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + const char *full_name; + size_t prefix_len = node->ns == NULL ? 0 : node->ns->prefix_len; + + if (strcmp(node->name, "INBOX") == 0 && node->parent == &tree->root) + return TRUE; + + if (prefix_len == 0) + return FALSE; + + full_name = dsync_mailbox_node_get_full_name(tree, node); + if (node->ns->prefix[prefix_len-1] == mail_namespace_get_sep(node->ns)) + prefix_len--; + return strncmp(full_name, node->ns->prefix, prefix_len) == 0 && + full_name[prefix_len] == '\0'; +} + +static void +sync_rename_node_to_temp(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node *new_parent, + const char **reason_r) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *old_name, *new_name, *p; + char name[TEMP_MAX_NAME_LEN+1]; + buffer_t buf; + size_t prefix_len, max_prefix_len; + unsigned int counter = 1; + + i_assert(!sync_node_is_namespace_prefix(tree, node)); + + buffer_create_from_data(&buf, name, sizeof(name)); + max_prefix_len = TEMP_MAX_NAME_LEN - TEMP_SUFFIX_MAX_LEN - 1; + if (node->sync_temporary_name) { + /* the source name was also a temporary name. drop the + -<suffix> from it */ + p = strrchr(node->name, '-'); + i_assert(p != NULL); + if (max_prefix_len > (size_t)(p - node->name)) + max_prefix_len = p - node->name; + } + str_append_max(&buf, node->name, max_prefix_len); + str_append_c(&buf, '-'); + prefix_len = buf.used; + + do { + str_truncate(&buf, prefix_len); + str_printfa(&buf, TEMP_SUFFIX_FORMAT, counter++); + /* the generated name is quite unlikely to exist, + but check anyway.. */ + } while (node_has_child(node->parent, str_c(&buf))); + + old_name = tree != ctx->local_tree ? NULL : + dsync_mailbox_node_get_full_name(tree, node); + + *reason_r = t_strdup_printf("Renamed '%s' to '%s'", node->name, str_c(&buf)); + node->name = p_strdup(tree->pool, str_c(&buf)); + node->sync_temporary_name = TRUE; + node->last_renamed_or_created = 0; + dsync_mailbox_tree_node_move_sorted(node, new_parent); + + if (tree == ctx->local_tree && node_is_existent(node)) { + /* we're modifying a local tree. remember this change. */ + new_name = dsync_mailbox_node_get_full_name(tree, node); + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + i_assert(strcmp(old_name, "INBOX") != 0); + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, old_name); + change->rename_dest_name = p_strdup(ctx->pool, new_name); + } +} + +static bool node_has_parent(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + for (; node != NULL; node = node->parent) { + if (node == parent) + return TRUE; + } + return FALSE; +} + +static void +sync_rename_node(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *temp_node, + struct dsync_mailbox_node *node, + const struct dsync_mailbox_node *other_node, + const char **reason_r) +{ + struct dsync_mailbox_tree_sync_change *change; + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *parent; + const char *name, *other_name; + + i_assert(node != NULL); + i_assert(other_node != NULL); + + /* move/rename node in the tree, so that its position/name is identical + to other_node (in other_tree). temp_node's name is changed to + temporary name (i.e. it assumes that node's name becomes temp_node's + original name) */ + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + + parent = sorted_tree_get_by_node_name(tree, other_tree, + other_node->parent); + if (node_has_parent(parent, node)) { + /* don't introduce a loop. temporarily rename node + under root. */ + sync_rename_node_to_temp(ctx, tree, node, &tree->root, reason_r); + *reason_r = t_strconcat(*reason_r, " (Don't introduce loop)", NULL); + return; + } + sync_rename_node_to_temp(ctx, tree, temp_node, temp_node->parent, reason_r); + + /* get the old name before it's modified */ + name = dsync_mailbox_node_get_full_name(tree, node); + + /* set the new name */ + *reason_r = t_strdup_printf("%s + Renamed '%s' to '%s'", + *reason_r, name, other_node->name); + node->name = p_strdup(tree->pool, other_node->name); + node->sync_temporary_name = other_node->sync_temporary_name; + node->last_renamed_or_created = other_node->last_renamed_or_created; + /* change node's parent if necessary. in any case detach+reattach it + sorted, because the nodes must be sorted by name, and the node's + name (or its parent) changed. */ + dsync_mailbox_tree_node_move_sorted(node, parent); + + if (tree == ctx->local_tree && node_is_existent(node)) { + /* we're modifying a local tree. remember this change. */ + other_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + i_assert(strcmp(name, "INBOX") != 0); + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); + change->rename_dest_name = p_strdup(ctx->pool, other_name); + } +} + +static int node_mailbox_guids_cmp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + int ret; + + while (node1 != NULL && node2 != NULL) { + if (node1->existence == DSYNC_MAILBOX_NODE_EXISTS && + node2->existence != DSYNC_MAILBOX_NODE_EXISTS) + return -1; + if (node2->existence == DSYNC_MAILBOX_NODE_EXISTS && + node1->existence != DSYNC_MAILBOX_NODE_EXISTS) + return 1; + + ret = memcmp(node1->mailbox_guid, node2->mailbox_guid, + sizeof(node1->mailbox_guid)); + if (ret != 0) + return ret; + + ret = node_mailbox_guids_cmp(node1->first_child, + node2->first_child); + if (ret != 0) + return ret; + node1 = node1->next; + node2 = node2->next; + } + if (node1 == NULL && node2 == NULL) + return 0; + return node1 != NULL ? -1 : 1; +} + +static int node_mailbox_names_cmp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + int ret; + + while (node1 != NULL && node2 != NULL) { + ret = strcmp(node1->name, node2->name); + if (ret != 0) + return ret; + + ret = node_mailbox_names_cmp(node1->first_child, + node2->first_child); + if (ret != 0) + return ret; + node1 = node1->next; + node2 = node2->next; + } + if (node1 == NULL && node2 == NULL) + return 0; + return node1 != NULL ? -1 : 1; +} + +static int node_mailbox_trees_cmp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + int ret; + + ret = node_mailbox_guids_cmp(node1, node2); + if (ret == 0) { + /* only a directory name changed and all the timestamps + are equal. just pick the alphabetically smaller. */ + ret = node_mailbox_names_cmp(node1, node2); + } + i_assert(ret != 0); + return ret; +} + +static time_t nodes_get_timestamp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + time_t ts = 0; + + /* avoid using temporary names in case all the timestamps are 0 */ + if (node1 != NULL && !node1->sync_temporary_name) + ts = node1->last_renamed_or_created + 1; + if (node2 != NULL && !node2->sync_temporary_name && + ts <= node2->last_renamed_or_created) + ts = node2->last_renamed_or_created + 1; + return ts; +} + +static bool sync_node_is_namespace_root(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + if (node == NULL) + return FALSE; + if (node == &tree->root) + return TRUE; + return sync_node_is_namespace_prefix(tree, node); +} + +static bool ATTR_NULL(3, 4) +sync_rename_lower_ts(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node1, + struct dsync_mailbox_node *remote_node1, + struct dsync_mailbox_node *local_node2, + struct dsync_mailbox_node *remote_node2, + const char **reason_r) +{ + time_t local_ts, remote_ts; + + /* We're scanning the tree at the position of local_node1 + and remote_node2. They have identical names. We also know that + local_node1&remote_node1 and local_node2&remote_node2 are "the same" + either because their GUIDs or (in case of one being a directory) + their childrens' GUIDs match. We don't know where local_node2 or + remote_node1 are located in the mailbox tree, or if they exist + at all. Note that node1 and node2 may be the same node pointers. */ + i_assert(strcmp(local_node1->name, remote_node2->name) == 0); + + if (sync_node_is_namespace_root(ctx->remote_tree, remote_node1) || + sync_node_is_namespace_root(ctx->remote_tree, remote_node2) || + sync_node_is_namespace_root(ctx->local_tree, local_node1) || + sync_node_is_namespace_root(ctx->local_tree, local_node2)) { + local_node1->sync_delayed_guid_change = TRUE; + remote_node2->sync_delayed_guid_change = TRUE; + *reason_r = "Can't rename namespace prefixes - will be merged later"; + return FALSE; + } + + local_ts = nodes_get_timestamp(local_node1, local_node2); + remote_ts = nodes_get_timestamp(remote_node1, remote_node2); + + if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + local_ts = remote_ts+1; + else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + remote_ts = local_ts+1; + + /* The algorithm must be deterministic regardless of the sync direction, + so in case the timestamps are equal we need to resort to looking at + the other data. We'll start by looking at the nodes' mailbox GUIDs, + but if both of them don't exist continue looking into their + children. */ + if (local_ts > remote_ts || + (local_ts == remote_ts && + node_mailbox_trees_cmp(local_node1, remote_node2) < 0)) { + /* local nodes have a higher timestamp. we only want to do + renames where the destination parent is the current node's + (local_node1/remote_node2) parent. */ + + /* Numbers are GUIDs, letters are mailbox names: + + local 1A <-name conflict-> remote 2A + local 2B <- potentially -> remote 1[BC] + + Here we want to preserve the local 1A & 2B names: */ + if (local_node2 == NULL) { + /* local : 1A + remote: 1B, 2A -> 2A-temp, 1A */ + sync_rename_node(ctx, ctx->remote_tree, remote_node2, + remote_node1, local_node1, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: local_node2=NULL)", NULL); + return TRUE; + } else if (remote_node1 == remote_node2) { + /* FIXME: this fixes an infinite loop when in + rename2 test, think it through why :) */ + *reason_r = "local: remote_node1=remote_node2"; + } else if (remote_node1 != NULL) { + /* a) local_node1->parent == local_node2->parent + + local : 1A, 2B + remote: 1B, 2A -> 2A-temp, 1A(, 2B) + remote: 1C, 2A -> 2B, 1A + remote: 1C, 2A, 3B -> 2A-temp, 1A(, 3B-temp, 2B) + + b) local_node1->parent != local_node2->parent + + local : 1X/A, 2Y/B + remote: 1Y/B, 2X/A -> 2X/A-temp, 1X/A(, 2Y/B) + remote: 1Z/C, 2X/A -> 2X/A-temp, 1X/A + remote: 1Z/C, 2X/A, 3Y/B -> 2X/A-temp, 1X/A + + We can handle all of these more easily by simply + always renaming 2 to a temporary name and handling + it when we reach B handling. */ + sync_rename_node(ctx, ctx->remote_tree, remote_node2, + remote_node1, local_node1, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: remote_node1=NULL)", NULL); + return TRUE; + } else if (node_has_parent(local_node1, local_node2) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* node2 is a parent of node1, but it should be + vice versa */ + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node1, local_node2->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: node2 parent of node1)", NULL); + return TRUE; + } else if (node_has_parent(local_node2, local_node1) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* node1 is a parent of node2, but it should be + vice versa */ + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node2, local_node1->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: node1 parent of node2)", NULL); + return TRUE; + } else if (local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node2, remote_node2->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: local_node1 exists)", NULL); + return TRUE; + } else { + /* local : 1A, 2B + remote: 2A -> (2B) + remote: 2A, 3B -> (3B-temp, 2B) */ + *reason_r = "local: unchanged"; + } + } else { + /* remote nodes have a higher timestamp */ + if (remote_node1 == NULL) { + sync_rename_node(ctx, ctx->local_tree, local_node1, + local_node2, remote_node2, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: remote_node1=NULL)", NULL); + return TRUE; + } else if (local_node2 == local_node1) { + *reason_r = "remote: remote_node2=remote_node1"; + } else if (local_node2 != NULL) { + sync_rename_node(ctx, ctx->local_tree, local_node1, + local_node2, remote_node2, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: local_node2=NULL)", NULL); + return TRUE; + } else if (node_has_parent(remote_node1, remote_node2) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node1, remote_node2->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: node2 parent of node1)", NULL); + return TRUE; + } else if (node_has_parent(remote_node2, remote_node1) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node2, remote_node1->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: node1 parent of node2)", NULL); + return TRUE; + } else if (remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS) { + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node1, local_node1->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: remote_node2 exists)", NULL); + return TRUE; + } else { + *reason_r = "remote: unchanged"; + } + } + return FALSE; +} + +static bool sync_rename_conflict(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node1, + struct dsync_mailbox_node *remote_node2, + const char **reason_r) +{ + struct dsync_mailbox_node *local_node2, *remote_node1; + const uint8_t *guid_p; + bool ret; + + guid_p = local_node1->mailbox_guid; + remote_node1 = hash_table_lookup(ctx->remote_tree->guid_hash, guid_p); + guid_p = remote_node2->mailbox_guid; + local_node2 = hash_table_lookup(ctx->local_tree->guid_hash, guid_p); + + if ((remote_node1 != NULL && remote_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) || + (local_node2 != NULL && local_node2->existence == DSYNC_MAILBOX_NODE_EXISTS)) { + /* conflicting name, rename the one with lower timestamp */ + ret = sync_rename_lower_ts(ctx, local_node1, remote_node1, + local_node2, remote_node2, reason_r); + *reason_r = t_strconcat("conflicting name: ", *reason_r, NULL); + return ret; + } else if (dsync_mailbox_node_is_dir(local_node1) || + dsync_mailbox_node_is_dir(remote_node2)) { + /* one of the nodes is a directory, and the other is a mailbox + that doesn't exist on the other side. there is no conflict, + we'll just need to create the mailbox later. */ + *reason_r = "mailbox not selectable yet"; + return FALSE; + } else { + /* both nodes are mailboxes that don't exist on the other side. + we'll merge these mailboxes together later and change their + GUIDs and UIDVALIDITYs to be the same */ + local_node1->sync_delayed_guid_change = TRUE; + remote_node2->sync_delayed_guid_change = TRUE; + *reason_r = "GUIDs conflict - will be merged later"; + return FALSE; + } +} + +static struct dsync_mailbox_node * +sync_find_branch(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + struct dsync_mailbox_node *dir_node) +{ + struct dsync_mailbox_node *node, *other_node; + const uint8_t *guid_p; + + for (node = dir_node->first_child; node != NULL; node = node->next) { + if (dsync_mailbox_node_is_dir(node)) { + other_node = sync_find_branch(tree, other_tree, node); + if (other_node != NULL) + return other_node; + } else { + guid_p = node->mailbox_guid; + other_node = hash_table_lookup(other_tree->guid_hash, + guid_p); + if (other_node != NULL) + return other_node->parent; + } + } + return NULL; +} + +static bool sync_rename_directory(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node1, + struct dsync_mailbox_node *remote_node2, + const char **reason_r) +{ + struct dsync_mailbox_node *remote_node1, *local_node2; + + /* see if we can find matching mailbox branches based on the nodes' + child mailboxes (with GUIDs). we can then rename the entire branch. + don't try to do this for namespace prefixes though. */ + remote_node1 = sync_find_branch(ctx->local_tree, + ctx->remote_tree, local_node1); + local_node2 = sync_find_branch(ctx->remote_tree, ctx->local_tree, + remote_node2); + if (remote_node1 == NULL || local_node2 == NULL) { + *reason_r = "Directory rename branch not found"; + return FALSE; + } + if (node_names_equal(remote_node1, local_node2)) { + *reason_r = "Directory name paths are equal"; + return FALSE; + } + + return sync_rename_lower_ts(ctx, local_node1, remote_node1, + local_node2, remote_node2, reason_r); +} + +static bool sync_rename_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_parent, + struct dsync_mailbox_node *remote_parent) +{ + struct dsync_mailbox_node **local_nodep = &local_parent->first_child; + struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child; + struct dsync_mailbox_node *local_node, *remote_node; + const char *reason; + string_t *debug = NULL; + bool changed; + + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) + debug = t_str_new(128); + + /* the nodes are sorted by their names. */ + while (*local_nodep != NULL || *remote_nodep != NULL) { + local_node = *local_nodep; + remote_node = *remote_nodep; + + if (local_node == NULL || + (remote_node != NULL && + strcmp(local_node->name, remote_node->name) > 0)) { + /* add a missing local node */ + local_node = sync_node_new(ctx->local_tree, local_nodep, + local_parent, remote_node); + } + if (remote_node == NULL || + strcmp(remote_node->name, local_node->name) > 0) { + /* add a missing remote node */ + remote_node = sync_node_new(ctx->remote_tree, remote_nodep, + remote_parent, local_node); + } + i_assert(strcmp(local_node->name, remote_node->name) == 0); + if (debug != NULL) { + str_truncate(debug, 0); + str_append(debug, "Mailbox "); + dsync_mailbox_node_append_full_name(debug, ctx->local_tree, local_node); + str_printfa(debug, ": local=%s/%ld/%d, remote=%s/%ld/%d", + guid_128_to_string(local_node->mailbox_guid), + (long)local_node->last_renamed_or_created, + local_node->existence, + guid_128_to_string(remote_node->mailbox_guid), + (long)remote_node->last_renamed_or_created, + remote_node->existence); + } + + if (dsync_mailbox_node_is_dir(local_node) && + dsync_mailbox_node_is_dir(remote_node)) { + /* both nodes are directories (or other side is + nonexistent). see if we can match them by their + child mailboxes */ + changed = sync_rename_directory(ctx, local_node, remote_node, &reason); + } else if (dsync_mailbox_node_guids_equal(local_node, + remote_node)) { + /* mailboxes are equal, no need to rename */ + reason = "Mailboxes are equal"; + changed = FALSE; + } else { + /* mailbox naming conflict */ + changed = sync_rename_conflict(ctx, local_node, + remote_node, &reason); + } + /* handle children, if there are any */ + if (debug != NULL) { + i_debug("brain %c: %s: %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + str_c(debug), reason); + } + + if (!changed) T_BEGIN { + changed = sync_rename_mailboxes(ctx, local_node, + remote_node); + } T_END; + if (changed) + return TRUE; + + local_nodep = &local_node->next; + remote_nodep = &remote_node->next; + } + return FALSE; +} + +static bool mailbox_node_hash_first_child(struct dsync_mailbox_node *node, + struct md5_context *md5) +{ + for (node = node->first_child; node != NULL; node = node->next) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) { + md5_update(md5, node->mailbox_guid, + sizeof(node->mailbox_guid)); + md5_update(md5, node->name, strlen(node->name)); + return TRUE; + } + if (node->first_child != NULL) { + if (mailbox_node_hash_first_child(node, md5)) + return TRUE; + } + } + return FALSE; +} + +static const char * +mailbox_node_generate_suffix(struct dsync_mailbox_node *node) +{ + struct md5_context md5; + unsigned char digest[MD5_RESULTLEN]; + + if (!dsync_mailbox_node_is_dir(node)) + return guid_128_to_string(node->mailbox_guid); + + md5_init(&md5); + if (!mailbox_node_hash_first_child(node, &md5)) + i_unreached(); /* we would have deleted it */ + md5_final(&md5, digest); + return binary_to_hex(digest, sizeof(digest)); +} + +static void suffix_inc(string_t *str) +{ + char *data; + size_t i; + + data = str_c_modifiable(str) + str_len(str)-1; + for (i = str_len(str); i > 0; i--, data--) { + if ((*data >= '0' && *data < '9') || + (*data >= 'a' && *data < 'f')) { + *data += 1; + return; + } else if (*data == '9') { + *data = 'a'; + return; + } else if (*data != 'f') { + i_unreached(); + } + } + i_unreached(); +} + +static void +sync_rename_temp_mailbox_node(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + const char **reason_r) +{ + const char *p, *new_suffix; + string_t *str = t_str_new(256); + size_t max_prefix_len; + + /* The name is currently <oldname>-<temp>. Both sides need to + use equivalent names, so we'll replace the <temp> if possible + with a) mailbox GUID, b) sha1 of childrens' (GUID|name)s. In the + very unlikely case of such name already existing, just increase + the last letters until it's not found. */ + new_suffix = mailbox_node_generate_suffix(node); + + p = strrchr(node->name, '-'); + i_assert(p != NULL); + p++; + max_prefix_len = TEMP_MAX_NAME_LEN - strlen(new_suffix) - 1; + if (max_prefix_len > (size_t)(p-node->name)) + max_prefix_len = p-node->name; + str_append_max(str, node->name, max_prefix_len); + str_append(str, new_suffix); + while (node_has_child(node->parent, str_c(str))) + suffix_inc(str); + + *reason_r = t_strdup_printf("Renamed '%s' to '%s'", + dsync_mailbox_node_get_full_name(tree, node), str_c(str)); + node->name = p_strdup(tree->pool, str_c(str)); + + dsync_mailbox_tree_node_move_sorted(node, node->parent); + node->sync_temporary_name = FALSE; +} + +static void +sync_rename_delete_node_dirs(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_node *child; + + for (child = node->first_child; child != NULL; child = child->next) + sync_rename_delete_node_dirs(ctx, tree, child); + + if (tree == ctx->local_tree && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL && + node->existence != DSYNC_MAILBOX_NODE_NONEXISTENT) { + sync_add_dir_change(ctx, node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR); + } + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + node->sync_temporary_name = FALSE; +} + +static bool +sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, bool *renames_r) +{ + const char *reason; + + for (; node != NULL; node = node->next) { + while (sync_rename_temp_mailboxes(ctx, tree, node->first_child, renames_r)) ; + + if (!node->sync_temporary_name) { + } else if (dsync_mailbox_node_is_dir(node) && + (node->first_child == NULL || + !node_has_existent_children(node, FALSE))) { + /* we can just delete this directory and + any child directories it may have */ + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) { + i_debug("brain %c: %s mailbox %s: Delete directory-only tree", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + tree == ctx->local_tree ? "local" : "remote", + dsync_mailbox_node_get_full_name(tree, node)); + } + sync_rename_delete_node_dirs(ctx, tree, node); + } else { + T_BEGIN { + *renames_r = TRUE; + sync_rename_temp_mailbox_node(tree, node, &reason); + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) { + i_debug("brain %c: %s mailbox %s: %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + tree == ctx->local_tree ? "local" : "remote", + dsync_mailbox_node_get_full_name(tree, node), reason); + } + } T_END; + return TRUE; + } + } + return FALSE; +} + +static int +dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx, + bool *renames_r) +{ + unsigned int max_renames, count = 0; + bool changed; + + *renames_r = FALSE; + + max_renames = ctx->combined_mailboxes_count * 3; + do { + T_BEGIN { + changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root, + &ctx->remote_tree->root); + } T_END; + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && + changed) { + i_debug("brain %c: -- Mailbox renamed, restart sync --", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S'); + } + } while (changed && ++count <= max_renames); + + if (changed) { + i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting"); + return -1; + } + + while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root, renames_r)) ; + while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root, renames_r)) ; + return 0; +} + +static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node, + const struct dsync_mailbox_node *wanted_node, + const char **reason_r) +{ + if (wanted_node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + *reason_r = wanted_node->existence == DSYNC_MAILBOX_NODE_DELETED ? + "Mailbox has been deleted" : "Mailbox doesn't exist"; + return TRUE; + } + if (node->uid_validity != wanted_node->uid_validity) { + *reason_r = t_strdup_printf("UIDVALIDITY changed (%u -> %u)", + wanted_node->uid_validity, + node->uid_validity); + return TRUE; + } + if (node->uid_next > wanted_node->uid_next) { + /* we can't lower the UIDNEXT */ + *reason_r = t_strdup_printf("UIDNEXT is too high (%u > %u)", + node->uid_next, + wanted_node->uid_next); + return TRUE; + } + if (memcmp(node->mailbox_guid, wanted_node->mailbox_guid, + sizeof(node->mailbox_guid)) != 0) { + *reason_r = "GUID changed"; + return TRUE; + } + return FALSE; +} + +static void +sync_delete_wrong_mailboxes_branch(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_tree *wanted_tree, + struct dsync_mailbox_node *node, + const struct dsync_mailbox_node *wanted_node) +{ + const char *reason; + int ret; + + while (node != NULL && wanted_node != NULL) { + if (node->first_child != NULL) { + sync_delete_wrong_mailboxes_branch(ctx, + tree, wanted_tree, + node->first_child, wanted_node->first_child); + } + ret = strcmp(node->name, wanted_node->name); + if (ret < 0) { + /* node shouldn't exist */ + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) { + sync_delete_mailbox_node(ctx, tree, node, + "Mailbox doesn't exist"); + } + node = node->next; + } else if (ret > 0) { + /* wanted_node doesn't exist. it's created later. */ + wanted_node = wanted_node->next; + } else T_BEGIN { + if (sync_is_wrong_mailbox(node, wanted_node, &reason) && + node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) + sync_delete_mailbox_node(ctx, tree, node, reason); + node = node->next; + wanted_node = wanted_node->next; + } T_END; + } + for (; node != NULL; node = node->next) { + /* node and its children shouldn't exist */ + if (node->first_child != NULL) { + sync_delete_wrong_mailboxes_branch(ctx, + tree, wanted_tree, + node->first_child, NULL); + } + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) + sync_delete_mailbox_node(ctx, tree, node, "Mailbox doesn't exist"); + } +} + +static void +sync_delete_wrong_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_tree *wanted_tree) +{ + sync_delete_wrong_mailboxes_branch(ctx, tree, wanted_tree, + tree->root.first_child, + wanted_tree->root.first_child); +} + +static void sync_create_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node, *other_node; + const char *name; + const uint8_t *guid_p; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + /* make sure the renaming handled everything */ + i_assert(!node->sync_temporary_name); + if (dsync_mailbox_node_is_dir(node)) + continue; + + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + + guid_p = node->mailbox_guid; + other_node = hash_table_lookup(other_tree->guid_hash, guid_p); + if (other_node == NULL) + other_node = sorted_tree_get(other_tree, name); + if (dsync_mailbox_node_is_dir(other_node)) { + /* create a missing mailbox */ + other_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + other_node->ns = node->ns; + other_node->uid_validity = node->uid_validity; + memcpy(other_node->mailbox_guid, node->mailbox_guid, + sizeof(other_node->mailbox_guid)); + if (other_tree == ctx->local_tree) + sync_add_create_change(ctx, other_node, name); + } else if (!guid_128_equals(node->mailbox_guid, + other_node->mailbox_guid)) { + /* mailbox with same name exists both locally and + remotely, but they have different GUIDs and neither + side has the other's GUID. typically this means that + both sides had autocreated some mailboxes (e.g. + INBOX). we'll just change the GUID for one of + them. */ + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + i_assert(node->ns == other_node->ns); + + if (other_tree == ctx->local_tree) + sync_add_create_change(ctx, node, name); + } else { + /* existing mailbox. mismatching UIDVALIDITY is handled + later while syncing the mailbox. */ + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + i_assert(node->ns == other_node->ns); + } + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +static void +sync_subscription(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node, + struct dsync_mailbox_node *remote_node) +{ + bool use_local; + + if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + use_local = TRUE; + else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + use_local = FALSE; + else if (local_node->last_subscription_change > remote_node->last_subscription_change) + use_local = TRUE; + else if (local_node->last_subscription_change < remote_node->last_subscription_change) + use_local = FALSE; + else { + /* local and remote have equal timestamps. prefer to subscribe + rather than unsubscribe. */ + use_local = local_node->subscribed; + } + if (use_local) { + /* use local subscription state */ + remote_node->subscribed = local_node->subscribed; + } else { + /* use remote subscription state */ + local_node->subscribed = remote_node->subscribed; + sync_add_dir_change(ctx, local_node, local_node->subscribed ? + DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE : + DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE); + } +} + +static void sync_mailbox_child_dirs(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_parent, + struct dsync_mailbox_node *remote_parent) +{ + struct dsync_mailbox_node **local_nodep = &local_parent->first_child; + struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child; + struct dsync_mailbox_node *local_node, *remote_node; + int ret; + + /* NOTE: the nodes are always sorted. renaming created all of the + interesting nodes, but it may have left some extra nonexistent nodes + lying around, which we will delete. */ + while (*local_nodep != NULL && *remote_nodep != NULL) { + local_node = *local_nodep; + remote_node = *remote_nodep; + + ret = strcmp(local_node->name, remote_node->name); + if (ret < 0) { + i_assert(!node_is_existent(local_node)); + *local_nodep = local_node->next; + continue; + } + if (ret > 0) { + i_assert(!node_is_existent(remote_node)); + *remote_nodep = remote_node->next; + continue; + } + + if (local_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + remote_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + /* create to remote */ + remote_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + } + if (remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + local_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* create to local */ + local_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + sync_add_dir_change(ctx, local_node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR); + } + + /* create/delete child dirs */ + sync_mailbox_child_dirs(ctx, local_node, remote_node); + + if (local_node->subscribed != remote_node->subscribed) + sync_subscription(ctx, local_node, remote_node); + + if (local_node->existence == DSYNC_MAILBOX_NODE_DELETED && + !node_has_existent_children(local_node, TRUE) && + remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + /* delete from remote */ + i_assert(!node_has_existent_children(remote_node, TRUE)); + remote_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + } + if (remote_node->existence == DSYNC_MAILBOX_NODE_DELETED && + !node_has_existent_children(remote_node, TRUE) && + local_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* delete from local */ + i_assert(!node_has_existent_children(local_node, TRUE)); + local_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + sync_add_dir_change(ctx, local_node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR); + } + + local_nodep = &local_node->next; + remote_nodep = &remote_node->next; + } + while (*local_nodep != NULL) { + i_assert(!node_is_existent(*local_nodep)); + *local_nodep = (*local_nodep)->next; + } + while (*remote_nodep != NULL) { + i_assert(!node_is_existent(*remote_nodep)); + *remote_nodep = (*remote_nodep)->next; + } +} + +static void sync_mailbox_dirs(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + sync_mailbox_child_dirs(ctx, &ctx->local_tree->root, + &ctx->remote_tree->root); +} + +static void +dsync_mailbox_tree_update_child_timestamps(struct dsync_mailbox_node *node, + time_t parent_timestamp) +{ + time_t ts; + + if (node->last_renamed_or_created < parent_timestamp) + node->last_renamed_or_created = parent_timestamp; + ts = node->last_renamed_or_created; + + for (node = node->first_child; node != NULL; node = node->next) + dsync_mailbox_tree_update_child_timestamps(node, ts); +} + +struct dsync_mailbox_tree_sync_ctx * +dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, + struct dsync_mailbox_tree *remote_tree, + enum dsync_mailbox_trees_sync_type sync_type, + enum dsync_mailbox_trees_sync_flags sync_flags) +{ + struct dsync_mailbox_tree_sync_ctx *ctx; + unsigned int rename_counter = 0; + bool renames; + pool_t pool; + + i_assert(hash_table_is_created(local_tree->guid_hash)); + i_assert(hash_table_is_created(remote_tree->guid_hash)); + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox trees sync", + 1024*64); + ctx = p_new(pool, struct dsync_mailbox_tree_sync_ctx, 1); + ctx->pool = pool; + ctx->local_tree = local_tree; + ctx->remote_tree = remote_tree; + ctx->sync_type = sync_type; + ctx->sync_flags = sync_flags; + i_array_init(&ctx->changes, 128); + +again: + renames = FALSE; + ctx->combined_mailboxes_count = 0; + sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, + sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + sync_tree_sort_and_delete_mailboxes(ctx, local_tree, + sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + + dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0); + dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0); + if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) { + if (dsync_mailbox_tree_handle_renames(ctx, &renames) < 0) { + ctx->failed = TRUE; + return ctx; + } + } + + /* if we're not doing a two-way sync, delete now any mailboxes, which + a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY, + c) has a too high UIDNEXT */ + if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + sync_delete_wrong_mailboxes(ctx, remote_tree, local_tree); + else if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + sync_delete_wrong_mailboxes(ctx, local_tree, remote_tree); + + if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + sync_create_mailboxes(ctx, remote_tree); + if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + sync_create_mailboxes(ctx, local_tree); + if (renames && rename_counter++ <= ctx->combined_mailboxes_count*3) { + /* this rename algorithm is just horrible. we're retrying this + because the final sync_rename_temp_mailbox_node() calls + give different names to local & remote mailbox trees. + something's not right here, but this looping is better than + a crash in sync_mailbox_dirs() due to trees not matching. */ + goto again; + } + sync_mailbox_dirs(ctx); + return ctx; +} + +const struct dsync_mailbox_tree_sync_change * +dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + if (ctx->change_idx == array_count(&ctx->changes)) + return NULL; + return array_idx(&ctx->changes, ctx->change_idx++); +} + +int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx) +{ + struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx; + int ret = ctx->failed ? -1 : 0; + + *_ctx = NULL; + + array_free(&ctx->changes); + pool_unref(&ctx->pool); + return ret; +} diff --git a/src/doveadm/dsync/dsync-mailbox-tree.c b/src/doveadm/dsync/dsync-mailbox-tree.c new file mode 100644 index 0000000..e34f335 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree.c @@ -0,0 +1,554 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "sort.h" +#include "mailbox-list-private.h" +#include "dsync-mailbox-tree-private.h" + + +struct dsync_mailbox_tree_iter { + struct dsync_mailbox_tree *tree; + + struct dsync_mailbox_node *cur; + string_t *name; +}; + +struct dsync_mailbox_tree * +dsync_mailbox_tree_init(char sep, char escape_char, char alt_char) +{ + struct dsync_mailbox_tree *tree; + pool_t pool; + + i_assert(sep != '\0'); + i_assert(alt_char != '\0'); + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox tree", 4096); + tree = p_new(pool, struct dsync_mailbox_tree, 1); + tree->pool = pool; + tree->sep = tree->sep_str[0] = sep; + tree->escape_char = escape_char; + tree->alt_char = alt_char; + tree->root.name = ""; + i_array_init(&tree->deletes, 32); + return tree; +} + +void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **_tree) +{ + struct dsync_mailbox_tree *tree = *_tree; + + *_tree = NULL; + hash_table_destroy(&tree->name128_hash); + hash_table_destroy(&tree->guid_hash); + array_free(&tree->deletes); + pool_unref(&tree->pool); +} + +static struct dsync_mailbox_node * +dsync_mailbox_node_find(struct dsync_mailbox_node *nodes, const char *name) +{ + for (; nodes != NULL; nodes = nodes->next) { + if (strcmp(name, nodes->name) == 0) + return nodes; + } + return NULL; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree, + const char *full_name) +{ + struct dsync_mailbox_node *node = &tree->root; + + T_BEGIN { + const char *const *path; + + path = t_strsplit(full_name, tree->sep_str); + for (; *path != NULL && node != NULL; path++) + node = dsync_mailbox_node_find(node->first_child, *path); + } T_END; + return node; +} + +void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + node->parent = parent; + node->next = parent->first_child; + parent->first_child = node; +} + +void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_node **p; + + for (p = &node->parent->first_child;; p = &(*p)->next) { + if (*p == node) { + *p = node->next; + break; + } + } + node->parent = NULL; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name) +{ + struct dsync_mailbox_node *parent = NULL, *node = &tree->root; + + i_assert(tree->iter_count == 0); + + T_BEGIN { + const char *const *path; + + /* find the existing part */ + path = t_strsplit(full_name, tree->sep_str); + for (; *path != NULL; path++) { + parent = node; + node = dsync_mailbox_node_find(node->first_child, *path); + if (node == NULL) + break; + } + /* create the rest */ + for (; *path != NULL; path++) { + node = p_new(tree->pool, struct dsync_mailbox_node, 1); + node->name = p_strdup(tree->pool, *path); + node->ns = parent->ns; + dsync_mailbox_tree_node_attach(node, parent); + parent = node; + } + } T_END; + return node; +} + +static void +node_get_full_name_recurse(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node, string_t *str) +{ + if (node->parent != &tree->root) + node_get_full_name_recurse(tree, node->parent, str); + + str_append(str, node->name); + str_append_c(str, tree->sep); +} + +const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node) +{ + string_t *str = t_str_new(128); + dsync_mailbox_node_append_full_name(str, tree, node); + return str_c(str); +} + +void dsync_mailbox_node_append_full_name(string_t *str, + const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node) +{ + i_assert(node->parent != NULL); + + node_get_full_name_recurse(tree, node, str); + /* remove the trailing separator */ + str_truncate(str, str_len(str)-1); +} + +void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, + const struct dsync_mailbox_node *src) +{ + memcpy(dest->mailbox_guid, src->mailbox_guid, + sizeof(dest->mailbox_guid)); + dest->uid_validity = src->uid_validity; + dest->uid_next = src->uid_next; + dest->existence = src->existence; + dest->last_renamed_or_created = src->last_renamed_or_created; + dest->subscribed = src->subscribed; + dest->last_subscription_change = src->last_subscription_change; +} + +struct dsync_mailbox_tree_iter * +dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + + iter = i_new(struct dsync_mailbox_tree_iter, 1); + iter->tree = tree; + iter->name = str_new(default_pool, 128); + iter->cur = &tree->root; + + tree->iter_count++; + return iter; +} + +static size_t node_get_full_name_length(struct dsync_mailbox_node *node) +{ + if (node->parent->parent == NULL) + return strlen(node->name); + else { + return strlen(node->name) + 1 + + node_get_full_name_length(node->parent); + } +} + +bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter, + const char **full_name_r, + struct dsync_mailbox_node **node_r) +{ + size_t len; + + if (iter->cur->first_child != NULL) + iter->cur = iter->cur->first_child; + else { + while (iter->cur->next == NULL) { + if (iter->cur == &iter->tree->root) + return FALSE; + iter->cur = iter->cur->parent; + } + iter->cur = iter->cur->next; + len = iter->cur->parent == &iter->tree->root ? 0 : + node_get_full_name_length(iter->cur->parent); + str_truncate(iter->name, len); + } + if (str_len(iter->name) > 0) + str_append_c(iter->name, iter->tree->sep); + str_append(iter->name, iter->cur->name); + *full_name_r = str_c(iter->name); + *node_r = iter->cur; + return TRUE; +} + +void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **_iter) +{ + struct dsync_mailbox_tree_iter *iter = *_iter; + + *_iter = NULL; + + i_assert(iter->tree->iter_count > 0); + iter->tree->iter_count--; + + str_free(&iter->name); + i_free(iter); +} + +void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + guid_128_t *sha128; + uint8_t *guid_p; + + i_assert(!hash_table_is_created(tree->name128_hash)); + + hash_table_create(&tree->name128_hash, + tree->pool, 0, guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + sha128 = p_new(tree->pool, guid_128_t, 1); + mailbox_name_get_sha128(name, *sha128); + guid_p = *sha128; + hash_table_insert(tree->name128_hash, guid_p, node); + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +static const char * +convert_name_to_remote_sep(struct dsync_mailbox_tree *tree, const char *name) +{ + string_t *str = t_str_new(128); + + char remote_escape_chars[3] = { + tree->remote_escape_char, + tree->remote_sep, + '\0' + }; + + for (;;) { + const char *end = strchr(name, tree->sep); + const char *name_part = end == NULL ? name : + t_strdup_until(name, end++); + + if (tree->escape_char != '\0') + mailbox_list_name_unescape(&name_part, tree->escape_char); + if (remote_escape_chars[0] != '\0') { + /* The local name can be fully escaped to remote + name and back. */ + mailbox_list_name_escape(name_part, remote_escape_chars, + str); + } else { + /* There is no remote escape char, so for conflicting + separator use the alt_char. */ + for (; *name_part != '\0'; name_part++) { + if (*name_part == tree->remote_sep) + str_append_c(str, tree->alt_char); + else + str_append_c(str, *name_part); + } + } + if (end == NULL) + break; + str_append_c(str, tree->remote_sep); + } + return str_c(str); +} + +static void +dsync_mailbox_tree_build_name128_remotesep_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + guid_128_t *sha128; + uint8_t *guid_p; + + i_assert(tree->sep != tree->remote_sep); + i_assert(!hash_table_is_created(tree->name128_remotesep_hash)); + + hash_table_create(&tree->name128_remotesep_hash, tree->pool, 0, + guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + sha128 = p_new(tree->pool, guid_128_t, 1); + T_BEGIN { + const char *remote_name = + convert_name_to_remote_sep(tree, name); + mailbox_name_get_sha128(remote_name, *sha128); + } T_END; + guid_p = *sha128; + hash_table_insert(tree->name128_remotesep_hash, guid_p, node); + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node **old_node_r) +{ + struct dsync_mailbox_node *old_node; + uint8_t *guid = node->mailbox_guid; + + if (guid_128_is_empty(node->mailbox_guid)) + return 0; + + *old_node_r = old_node = hash_table_lookup(tree->guid_hash, guid); + if (old_node == NULL) + hash_table_insert(tree->guid_hash, guid, node); + else if (old_node != node) + return -1; + return 0; +} + +int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node **dup_node1_r, + struct dsync_mailbox_node **dup_node2_r) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node, *old_node; + const char *name; + int ret = 0; + + if (!hash_table_is_created(tree->guid_hash)) { + hash_table_create(&tree->guid_hash, tree->pool, 0, + guid_128_hash, guid_128_cmp); + } + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + if (dsync_mailbox_tree_guid_hash_add(tree, node, &old_node) < 0) { + *dup_node1_r = node; + *dup_node2_r = old_node; + ret = -1; + } + } + dsync_mailbox_tree_iter_deinit(&iter); + return ret; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree, + const guid_128_t guid) +{ + const uint8_t *guid_p = guid; + + return hash_table_lookup(tree->guid_hash, guid_p); +} + +const struct dsync_mailbox_delete * +dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree, + unsigned int *count_r) +{ + return array_get(&tree->deletes, count_r); +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_delete *del) +{ + const uint8_t *guid_p = del->guid; + + i_assert(hash_table_is_created(tree->guid_hash)); + i_assert(tree->remote_sep != '\0'); + + if (del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) { + /* find node by GUID */ + return hash_table_lookup(tree->guid_hash, guid_p); + } + + /* find node by name. this is a bit tricky, since the hierarchy + separator may differ from ours. */ + if (tree->sep == tree->remote_sep) { + if (!hash_table_is_created(tree->name128_hash)) + dsync_mailbox_tree_build_name128_hash(tree); + return hash_table_lookup(tree->name128_hash, guid_p); + } else { + if (!hash_table_is_created(tree->name128_remotesep_hash)) + dsync_mailbox_tree_build_name128_remotesep_hash(tree); + return hash_table_lookup(tree->name128_remotesep_hash, guid_p); + } +} + +void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree, + char remote_sep, char escape_char) +{ + i_assert(tree->remote_sep == '\0'); + i_assert(tree->remote_escape_char == '\0'); + + tree->remote_sep = remote_sep; + tree->remote_escape_char = escape_char; +} + +static void +dsync_mailbox_tree_dup_nodes(struct dsync_mailbox_tree *dest_tree, + const struct dsync_mailbox_node *src, + string_t *path) +{ + size_t prefix_len = str_len(path); + struct dsync_mailbox_node *node; + + if (prefix_len > 0) { + str_append_c(path, dest_tree->sep); + prefix_len++; + } + for (; src != NULL; src = src->next) { + str_truncate(path, prefix_len); + str_append(path, src->name); + node = dsync_mailbox_tree_get(dest_tree, str_c(path)); + + node->ns = src->ns; + memcpy(node->mailbox_guid, src->mailbox_guid, + sizeof(node->mailbox_guid)); + node->uid_validity = src->uid_validity; + node->uid_next = src->uid_next; + node->existence = src->existence; + node->last_renamed_or_created = src->last_renamed_or_created; + node->subscribed = src->subscribed; + node->last_subscription_change = src->last_subscription_change; + + if (src->first_child != NULL) { + dsync_mailbox_tree_dup_nodes(dest_tree, + src->first_child, path); + } + } +} + +struct dsync_mailbox_tree * +dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src) +{ + struct dsync_mailbox_tree *dest; + string_t *str = t_str_new(128); + + dest = dsync_mailbox_tree_init(src->sep, src->escape_char, + src->alt_char); + dsync_mailbox_tree_dup_nodes(dest, &src->root, str); + return dest; +} + +int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1, + struct dsync_mailbox_node *const *n2) +{ + return strcmp((*n1)->name, (*n2)->name); +} + +static bool +dsync_mailbox_nodes_equal(const struct dsync_mailbox_node *node1, + const struct dsync_mailbox_node *node2) +{ + return strcmp(node1->name, node2->name) == 0 && + node1->ns == node2->ns && + memcmp(node1->mailbox_guid, node2->mailbox_guid, + sizeof(node1->mailbox_guid)) == 0 && + node1->uid_validity == node2->uid_validity && + node1->existence == node2->existence && + node1->subscribed == node2->subscribed; +} + +static bool +dsync_mailbox_branches_equal(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + /* this function is used only for unit tests, so performance doesn't + really matter */ + struct dsync_mailbox_node *n, **snodes1, **snodes2; + unsigned int i, count; + + for (n = node1, count = 0; n != NULL; n = n->next) count++; + for (n = node2, i = 0; n != NULL; n = n->next) i++; + if (i != count) + return FALSE; + if (count == 0) + return TRUE; + + /* sort the trees by name */ + snodes1 = t_new(struct dsync_mailbox_node *, count); + snodes2 = t_new(struct dsync_mailbox_node *, count); + for (n = node1, i = 0; n != NULL; n = n->next, i++) + snodes1[i] = n; + for (n = node2, i = 0; n != NULL; n = n->next, i++) + snodes2[i] = n; + i_qsort(snodes1, count, sizeof(*snodes1), dsync_mailbox_node_name_cmp); + i_qsort(snodes2, count, sizeof(*snodes2), dsync_mailbox_node_name_cmp); + + for (i = 0; i < count; i++) { + if (!dsync_mailbox_nodes_equal(snodes1[i], snodes2[i])) + return FALSE; + if (!dsync_mailbox_branches_equal(snodes1[i]->first_child, + snodes2[i]->first_child)) + return FALSE; + } + return TRUE; +} + +bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2) +{ + bool ret; + + T_BEGIN { + ret = dsync_mailbox_branches_equal(&tree1->root, &tree2->root); + } T_END; + return ret; +} + +const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node) +{ + return t_strdup_printf("guid=%s uid_validity=%u uid_next=%u subs=%s last_change=%ld last_subs=%ld", + guid_128_to_string(node->mailbox_guid), + node->uid_validity, node->uid_next, + node->subscribed ? "yes" : "no", + (long)node->last_renamed_or_created, + (long)node->last_subscription_change); +} + +const char * +dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type) +{ + switch (type) { + case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX: + return "mailbox"; + case DSYNC_MAILBOX_DELETE_TYPE_DIR: + return "dir"; + case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE: + return "unsubscribe"; + } + i_unreached(); +} diff --git a/src/doveadm/dsync/dsync-mailbox-tree.h b/src/doveadm/dsync/dsync-mailbox-tree.h new file mode 100644 index 0000000..6969813 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree.h @@ -0,0 +1,205 @@ +#ifndef DSYNC_MAILBOX_TREE_H +#define DSYNC_MAILBOX_TREE_H + +#include "guid.h" +#include "mail-error.h" + +struct mail_namespace; +struct dsync_brain; + +enum dsync_mailbox_trees_sync_type { + /* two-way sync for both mailboxes */ + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, + /* make remote tree look exactly like the local tree */ + DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL, + /* make local tree look exactly like the remote tree */ + DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE +}; + +enum dsync_mailbox_trees_sync_flags { + /* Enable debugging */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG = 0x01, + /* Show ourself as "master brain" in the debug output */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN = 0x02, + /* Disable mailbox renaming logic. This is just a kludge that should + be removed once the renaming logic has no more bugs.. */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES = 0x04 +}; + +enum dsync_mailbox_node_existence { + /* this is just a filler node for children or for + subscription deletion */ + DSYNC_MAILBOX_NODE_NONEXISTENT = 0, + /* if mailbox GUID is set, the mailbox exists. + otherwise the directory exists. */ + DSYNC_MAILBOX_NODE_EXISTS, + /* if mailbox GUID is set, the mailbox has been deleted. + otherwise the directory has been deleted. */ + DSYNC_MAILBOX_NODE_DELETED +}; + +struct dsync_mailbox_node { + struct dsync_mailbox_node *parent, *next, *first_child; + + /* namespace where this node belongs to */ + struct mail_namespace *ns; + /* this node's name (not including parents) */ + const char *name; + /* mailbox GUID, or full of zeros if this is about a directory name */ + guid_128_t mailbox_guid; + /* mailbox's UIDVALIDITY/UIDNEXT (may be 0 if not assigned yet) */ + uint32_t uid_validity, uid_next; + + /* existence of this mailbox/directory. + doesn't affect subscription state. */ + enum dsync_mailbox_node_existence existence; + /* last time the mailbox/directory was created/renamed, + 0 if not known */ + time_t last_renamed_or_created; + + /* last time the subscription state was changed, 0 if not known */ + time_t last_subscription_change; + /* is this mailbox or directory subscribed? */ + bool subscribed:1; + + /* Internal syncing flags: */ + bool sync_delayed_guid_change:1; + bool sync_temporary_name:1; +}; +ARRAY_DEFINE_TYPE(dsync_mailbox_node, struct dsync_mailbox_node *); + +#define dsync_mailbox_node_guids_equal(node1, node2) \ + (memcmp((node1)->mailbox_guid, (node2)->mailbox_guid, \ + sizeof(guid_128_t)) == 0) + +#define dsync_mailbox_node_is_dir(node) \ + guid_128_is_empty((node)->mailbox_guid) + +enum dsync_mailbox_delete_type { + /* Delete mailbox by given GUID */ + DSYNC_MAILBOX_DELETE_TYPE_MAILBOX = 1, + /* Delete mailbox directory by given SHA1 name */ + DSYNC_MAILBOX_DELETE_TYPE_DIR, + /* Unsubscribe mailbox by given SHA1 name */ + DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE, +}; + +struct dsync_mailbox_delete { + enum dsync_mailbox_delete_type type; + guid_128_t guid; + time_t timestamp; +}; + +enum dsync_mailbox_tree_sync_type { + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX, + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR, + /* Rename given mailbox name and its children */ + DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME, + DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE, + DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE +}; + +struct dsync_mailbox_tree_sync_change { + enum dsync_mailbox_tree_sync_type type; + + /* for all types: */ + struct mail_namespace *ns; + const char *full_name; + + /* for create_box and delete_box: */ + guid_128_t mailbox_guid; + /* for create_box: */ + uint32_t uid_validity; + /* for rename: */ + const char *rename_dest_name; +}; + +struct dsync_mailbox_tree * +dsync_mailbox_tree_init(char sep, char escape_char, char alt_char); +void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **tree); + +/* Lookup a mailbox node by name. Returns NULL if not known. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree, + const char *full_name); +/* Lookup a mailbox node by GUID. Returns NULL if not known. + The mailbox GUID hash must have been build before calling this. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree, + const guid_128_t guid); +/* Lookup or create a mailbox node by name. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name); + +/* Returns full name for the given mailbox node. */ +const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node); +void dsync_mailbox_node_append_full_name(string_t *str, + const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node); + +/* Copy everything from src to dest, except name and hierarchy pointers */ +void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, + const struct dsync_mailbox_node *src); + +/* Add nodes to tree from the given namespace. If box_name or box_guid is + non-NULL, add only that mailbox to the tree. */ +int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const char *box_name, + const guid_128_t box_guid, + const char *const *exclude_mailboxes, + enum mail_error *error_r); + +/* Return all known deleted mailboxes and directories. */ +const struct dsync_mailbox_delete * +dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree, + unsigned int *count_r); +/* Return mailbox node for a given delete record, or NULL if it doesn't exist. + The delete record is intended to come from another tree, possibly with + a different hierarchy separator. dsync_mailbox_tree_build_guid_hash() must + have been called before this. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_delete *del); +/* Build GUID lookup hash, if it's not already built. Returns 0 if ok, -1 if + there are duplicate GUIDs. The nodes with the duplicate GUIDs are + returned. */ +int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node **dup_node1_r, + struct dsync_mailbox_node **dup_node2_r); +/* Manually add a new node to hash. */ +int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node **old_node_r); +/* Set remote separator used for directory deletions in + dsync_mailbox_tree_find_delete() */ +void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree, + char remote_sep, + char remote_escape_char); + +/* Iterate through all nodes in a tree (depth-first) */ +struct dsync_mailbox_tree_iter * +dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree); +bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter, + const char **full_name_r, + struct dsync_mailbox_node **node_r); +void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **iter); + +/* Sync local and remote trees so at the end they're exactly the same. + Return changes done to local tree. */ +struct dsync_mailbox_tree_sync_ctx * +dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, + struct dsync_mailbox_tree *remote_tree, + enum dsync_mailbox_trees_sync_type sync_type, + enum dsync_mailbox_trees_sync_flags sync_flags); +const struct dsync_mailbox_tree_sync_change * +dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx); +int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx); + +const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node); +const char * +dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox.c b/src/doveadm/dsync/dsync-mailbox.c new file mode 100644 index 0000000..d6d06fd --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox.c @@ -0,0 +1,61 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "mail-storage-private.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox.h" + +void dsync_mailbox_attribute_dup(pool_t pool, + const struct dsync_mailbox_attribute *src, + struct dsync_mailbox_attribute *dest_r) +{ + dest_r->type = src->type; + dest_r->key = p_strdup(pool, src->key); + dest_r->value = p_strdup(pool, src->value); + if (src->value_stream != NULL) { + dest_r->value_stream = src->value_stream; + i_stream_ref(dest_r->value_stream); + } + + dest_r->deleted = src->deleted; + dest_r->last_change = src->last_change; + dest_r->modseq = src->modseq; +} + +int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box, + struct file_lock **lock_r) +{ + const char *path, *error; + int ret; + + /* Make sure the mailbox is open - locking requires it */ + if (mailbox_open(box) < 0) { + i_error("Can't open mailbox %s: %s", mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + return -1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path); + if (ret < 0) { + i_error("Can't get mailbox %s path: %s", mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + return -1; + } + if (ret == 0) { + /* No index files - don't do any locking. In theory we still + could, but this lock is mainly meant to prevent replication + problems, and replication wouldn't work without indexes. */ + *lock_r = NULL; + return 0; + } + + if (mailbox_lock_file_create(box, DSYNC_MAILBOX_LOCK_FILENAME, + brain->mailbox_lock_timeout_secs, + lock_r, &error) <= 0) { + i_error("Failed to lock mailbox %s for dsyncing: %s", + box->vname, error); + return -1; + } + return 0; +} diff --git a/src/doveadm/dsync/dsync-mailbox.h b/src/doveadm/dsync/dsync-mailbox.h new file mode 100644 index 0000000..7e81c0c --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox.h @@ -0,0 +1,44 @@ +#ifndef DSYNC_MAILBOX_H +#define DSYNC_MAILBOX_H + +#include "mail-storage.h" + +struct dsync_brain; + +/* Mailbox that is going to be synced. Its name was already sent in the + mailbox tree. */ +struct dsync_mailbox { + guid_128_t mailbox_guid; + bool mailbox_lost; + bool mailbox_ignore; + bool have_guids, have_save_guids, have_only_guid128; + + uint32_t uid_validity, uid_next, messages_count, first_recent_uid; + uint64_t highest_modseq, highest_pvt_modseq; + ARRAY_TYPE(mailbox_cache_field) cache_fields; +}; + +struct dsync_mailbox_attribute { + enum mail_attribute_type type; + const char *key; + /* if both values are NULL = not looked up yet / deleted */ + const char *value; + struct istream *value_stream; + + time_t last_change; /* 0 = unknown */ + uint64_t modseq; /* 0 = unknown */ + + bool deleted; /* attribute is known to have been deleted */ + bool exported; /* internally used by exporting */ +}; +#define DSYNC_ATTR_HAS_VALUE(attr) \ + ((attr)->value != NULL || (attr)->value_stream != NULL) + +void dsync_mailbox_attribute_dup(pool_t pool, + const struct dsync_mailbox_attribute *src, + struct dsync_mailbox_attribute *dest_r); + +int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box, + struct file_lock **lock_r); + +#endif diff --git a/src/doveadm/dsync/dsync-serializer.c b/src/doveadm/dsync/dsync-serializer.c new file mode 100644 index 0000000..7b369b8 --- /dev/null +++ b/src/doveadm/dsync/dsync-serializer.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "dsync-serializer.h" + +struct dsync_serializer { + pool_t pool; + const char *const *keys; + unsigned int keys_count; +}; + +struct dsync_serializer_encoder { + pool_t pool; + struct dsync_serializer *serializer; + ARRAY_TYPE(const_string) values; +}; + +struct dsync_serializer *dsync_serializer_init(const char *const keys[]) +{ + struct dsync_serializer *serializer; + pool_t pool; + const char **dup_keys; + unsigned int i, count; + + pool = pool_alloconly_create("dsync serializer", 512); + serializer = p_new(pool, struct dsync_serializer, 1); + serializer->pool = pool; + + count = str_array_length(keys); + dup_keys = p_new(pool, const char *, count + 1); + for (i = 0; i < count; i++) + dup_keys[i] = p_strdup(pool, keys[i]); + serializer->keys = dup_keys; + serializer->keys_count = count; + return serializer; +} + +void dsync_serializer_deinit(struct dsync_serializer **_serializer) +{ + struct dsync_serializer *serializer = *_serializer; + + *_serializer = NULL; + + pool_unref(&serializer->pool); +} + +const char * +dsync_serializer_encode_header_line(struct dsync_serializer *serializer) +{ + string_t *str = t_str_new(128); + unsigned int i; + + for (i = 0; serializer->keys[i] != NULL; i++) { + if (i > 0) + str_append_c(str, '\t'); + str_append_tabescaped(str, serializer->keys[i]); + } + str_append_c(str, '\n'); + return str_c(str); +} + +struct dsync_serializer_encoder * +dsync_serializer_encode_begin(struct dsync_serializer *serializer) +{ + struct dsync_serializer_encoder *encoder; + pool_t pool = pool_alloconly_create("dsync serializer encode", 1024); + + encoder = p_new(pool, struct dsync_serializer_encoder, 1); + encoder->pool = pool; + encoder->serializer = serializer; + p_array_init(&encoder->values, pool, serializer->keys_count); + return encoder; +} + +void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder, + const char *key, const char *value) +{ + unsigned int i; + + for (i = 0; encoder->serializer->keys[i] != NULL; i++) { + if (strcmp(encoder->serializer->keys[i], key) == 0) { + value = p_strdup(encoder->pool, value); + array_idx_set(&encoder->values, i, &value); + return; + } + } + i_panic("Unknown key: %s", key); +} + +void dsync_serializer_encode_finish(struct dsync_serializer_encoder **_encoder, + string_t *output) +{ + struct dsync_serializer_encoder *encoder = *_encoder; + const char *const *values; + unsigned int i, count; + + *_encoder = NULL; + + values = array_get(&encoder->values, &count); + for (i = 0; i < count; i++) { + if (i > 0) + str_append_c(output, '\t'); + if (values[i] == NULL) + str_append_c(output, NULL_CHR); + else { + if (values[i][0] == NULL_CHR) + str_append_c(output, NULL_CHR); + str_append_tabescaped(output, values[i]); + } + } + str_append_c(output, '\n'); + + pool_unref(&encoder->pool); +} diff --git a/src/doveadm/dsync/dsync-serializer.h b/src/doveadm/dsync/dsync-serializer.h new file mode 100644 index 0000000..1ed6c31 --- /dev/null +++ b/src/doveadm/dsync/dsync-serializer.h @@ -0,0 +1,18 @@ +#ifndef DSYNC_SERIALIZER_H +#define DSYNC_SERIALIZER_H + +#define NULL_CHR '\002' + +struct dsync_serializer *dsync_serializer_init(const char *const keys[]); +void dsync_serializer_deinit(struct dsync_serializer **serializer); + +const char * +dsync_serializer_encode_header_line(struct dsync_serializer *serializer); +struct dsync_serializer_encoder * +dsync_serializer_encode_begin(struct dsync_serializer *serializer); +void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder, + const char *key, const char *value); +void dsync_serializer_encode_finish(struct dsync_serializer_encoder **encoder, + string_t *output); + +#endif diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.c b/src/doveadm/dsync/dsync-transaction-log-scan.c new file mode 100644 index 0000000..dd2834a --- /dev/null +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c @@ -0,0 +1,608 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-transaction-log-scan.h" + +struct dsync_transaction_log_scan { + pool_t pool; + HASH_TABLE_TYPE(dsync_uid_mail_change) changes; + HASH_TABLE_TYPE(dsync_attr_change) attr_changes; + struct mail_index_view *view; + uint32_t highest_wanted_uid; + + uint32_t last_log_seq; + uoff_t last_log_offset; + + bool returned_all_changes; +}; + +static bool ATTR_NOWARN_UNUSED_RESULT +export_change_get(struct dsync_transaction_log_scan *ctx, uint32_t uid, + enum dsync_mail_change_type type, + struct dsync_mail_change **change_r) +{ + struct dsync_mail_change *change; + const char *orig_guid; + + i_assert(uid > 0); + i_assert(type != DSYNC_MAIL_CHANGE_TYPE_SAVE); + + *change_r = NULL; + + if (uid > ctx->highest_wanted_uid) + return FALSE; + + change = hash_table_lookup(ctx->changes, POINTER_CAST(uid)); + if (change == NULL) { + /* first change for this UID */ + change = p_new(ctx->pool, struct dsync_mail_change, 1); + change->uid = uid; + change->type = type; + hash_table_insert(ctx->changes, POINTER_CAST(uid), change); + } else if (type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* expunge overrides flag changes */ + orig_guid = change->guid; + i_zero(change); + change->type = type; + change->uid = uid; + change->guid = orig_guid; + } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* already expunged, this change doesn't matter */ + return FALSE; + } else { + /* another flag update */ + } + *change_r = change; + return TRUE; +} + +static void +log_add_expunge(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_expunge *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + return; + } + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change); + } + } +} + +static bool +log_add_expunge_uid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, uint32_t uid) +{ + const struct mail_transaction_expunge *rec = data, *end; + struct dsync_mail_change *change; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + return FALSE; + } + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (uid >= rec->uid1 && uid <= rec->uid2) { + export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change); + return TRUE; + } + } + return FALSE; +} + +static void +log_add_expunge_guid(struct dsync_transaction_log_scan *ctx, + struct mail_index_view *view, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_expunge_guid *rec = data, *end; + struct dsync_mail_change *change; + uint32_t seq; + bool external; + + external = (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (!external && mail_index_lookup_seq(view, rec->uid, &seq)) { + /* expunge request that hasn't been actually done yet. + we check non-external ones because they might have + the GUID while external ones don't. */ + continue; + } + if (export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change) && + !guid_128_is_empty(rec->guid_128)) T_BEGIN { + change->guid = p_strdup(ctx->pool, + guid_128_to_string(rec->guid_128)); + } T_END; + } +} + +static bool +log_add_expunge_guid_uid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, uint32_t uid) +{ + const struct mail_transaction_expunge_guid *rec = data, *end; + struct dsync_mail_change *change; + + /* we're assuming UID is already known to be expunged */ + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (rec->uid != uid) + continue; + + if (!export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change)) + i_unreached(); + if (!guid_128_is_empty(rec->guid_128)) T_BEGIN { + change->guid = p_strdup(ctx->pool, + guid_128_to_string(rec->guid_128)); + } T_END; + return TRUE; + } + return FALSE; +} + +static void +log_add_flag_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_flag_update *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + if (export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) { + change->add_flags |= rec->add_flags; + change->remove_flags &= ENUM_NEGATE(rec->add_flags); + change->remove_flags |= rec->remove_flags; + change->add_flags &= ENUM_NEGATE(rec->remove_flags); + } + } + } +} + +static void +log_add_keyword_reset(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_keyword_reset *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + if (!export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + + change->keywords_reset = TRUE; + if (array_is_created(&change->keyword_changes)) + array_clear(&change->keyword_changes); + } + } +} + +static void +keywords_change_remove(struct dsync_mail_change *change, const char *name) +{ + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + for (i = 0; i < count; i++) { + if (strcmp(changes[i]+1, name) == 0) { + array_delete(&change->keyword_changes, i, 1); + break; + } + } +} + +static void +log_add_keyword_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_keyword_update *rec = data; + struct dsync_mail_change *change; + const char *kw_name, *change_str; + const uint32_t *uids, *end; + unsigned int uids_offset; + uint32_t uid; + + uids_offset = sizeof(*rec) + rec->name_size; + if ((uids_offset % 4) != 0) + uids_offset += 4 - (uids_offset % 4); + + kw_name = t_strndup((const void *)(rec+1), rec->name_size); + switch (rec->modify_type) { + case MODIFY_ADD: + change_str = p_strdup_printf(ctx->pool, "%c%s", + KEYWORD_CHANGE_ADD, kw_name); + break; + case MODIFY_REMOVE: + change_str = p_strdup_printf(ctx->pool, "%c%s", + KEYWORD_CHANGE_REMOVE, kw_name); + break; + default: + i_unreached(); + } + + uids = CONST_PTR_OFFSET(rec, uids_offset); + end = CONST_PTR_OFFSET(rec, hdr->size); + + for (; uids < end; uids += 2) { + for (uid = uids[0]; uid <= uids[1]; uid++) { + if (!export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + if (!array_is_created(&change->keyword_changes)) { + p_array_init(&change->keyword_changes, + ctx->pool, 4); + } else { + keywords_change_remove(change, kw_name); + } + array_push_back(&change->keyword_changes, &change_str); + } + } +} + +static void +log_add_modseq_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, bool pvt_scan) +{ + const struct mail_transaction_modseq_update *rec = data, *end; + struct dsync_mail_change *change; + uint64_t modseq; + + /* update message's modseq, possibly by creating an empty flag change */ + end = CONST_PTR_OFFSET(rec, hdr->size); + for (; rec != end; rec++) { + if (rec->uid == 0) { + /* highestmodseq update */ + continue; + } + + if (!export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + + modseq = rec->modseq_low32 | + ((uint64_t)rec->modseq_high32 << 32); + if (!pvt_scan) { + if (change->modseq < modseq) + change->modseq = modseq; + } else { + if (change->pvt_modseq < modseq) + change->pvt_modseq = modseq; + } + } +} + +static void +log_add_attribute_update_key(struct dsync_transaction_log_scan *ctx, + const char *attr_change, uint64_t modseq) +{ + struct dsync_mailbox_attribute lookup_attr, *attr; + + i_assert(strlen(attr_change) > 2); /* checked by lib-index */ + + lookup_attr.type = attr_change[1] == 'p' ? + MAIL_ATTRIBUTE_TYPE_PRIVATE : MAIL_ATTRIBUTE_TYPE_SHARED; + lookup_attr.key = attr_change+2; + + attr = hash_table_lookup(ctx->attr_changes, &lookup_attr); + if (attr == NULL) { + attr = p_new(ctx->pool, struct dsync_mailbox_attribute, 1); + attr->type = lookup_attr.type; + attr->key = p_strdup(ctx->pool, lookup_attr.key); + hash_table_insert(ctx->attr_changes, attr, attr); + } + attr->deleted = attr_change[0] == '-'; + attr->modseq = modseq; +} + +static void +log_add_attribute_update(struct dsync_transaction_log_scan *ctx, + const void *data, + const struct mail_transaction_header *hdr, + uint64_t modseq) +{ + const char *attr_changes = data; + unsigned int i; + + for (i = 0; i < hdr->size && attr_changes[i] != '\0'; ) { + log_add_attribute_update_key(ctx, attr_changes+i, modseq); + i += strlen(attr_changes+i) + 1; + } +} + +static int +dsync_log_set(struct dsync_transaction_log_scan *ctx, + struct mail_index_view *view, bool pvt_scan, + struct mail_transaction_log_view *log_view, uint64_t modseq) +{ + uint32_t log_seq, end_seq; + uoff_t log_offset, end_offset; + const char *reason; + bool reset; + int ret; + + end_seq = view->log_file_head_seq; + end_offset = view->log_file_head_offset; + + if (modseq != 0 && + mail_index_modseq_get_next_log_offset(view, modseq, + &log_seq, &log_offset)) { + /* scan the view only up to end of the current view. + if there are more changes, we don't care about them until + the next sync. */ + ret = mail_transaction_log_view_set(log_view, + log_seq, log_offset, + end_seq, end_offset, + &reset, &reason); + if (ret != 0) + return ret; + } + + /* return everything we've got (until the end of the view) */ + if (!pvt_scan) + ctx->returned_all_changes = TRUE; + if (mail_transaction_log_view_set_all(log_view) < 0) + return -1; + + mail_transaction_log_view_get_prev_pos(log_view, &log_seq, &log_offset); + if (log_seq > end_seq || + (log_seq == end_seq && log_offset > end_offset)) { + end_seq = log_seq; + end_offset = log_offset; + } + ret = mail_transaction_log_view_set(log_view, + log_seq, log_offset, + end_seq, end_offset, + &reset, &reason); + if (ret == 0) { + /* we shouldn't get here. _view_set_all() already + reserved all the log files, the _view_set() only + removed unwanted ones. */ + i_error("%s: Couldn't set transaction log view (seq %u..%u): %s", + view->index->filepath, log_seq, end_seq, reason); + ret = -1; + } + if (ret < 0) + return -1; + if (modseq != 0) { + /* we didn't see all the changes that we wanted to */ + return 0; + } + return 1; +} + +static int +dsync_log_scan(struct dsync_transaction_log_scan *ctx, + struct mail_index_view *view, uint64_t modseq, bool pvt_scan) +{ + struct mail_transaction_log_view *log_view; + const struct mail_transaction_header *hdr; + const void *data; + uint32_t file_seq, max_seq; + uoff_t file_offset, max_offset; + uint64_t cur_modseq; + int ret; + + log_view = mail_transaction_log_view_open(view->index->log); + if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) { + mail_transaction_log_view_close(&log_view); + return -1; + } + + /* read the log only up to current position in view */ + max_seq = view->log_file_expunge_seq; + max_offset = view->log_file_expunge_offset; + + mail_transaction_log_view_get_prev_pos(log_view, &file_seq, + &file_offset); + + while (mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { + mail_transaction_log_view_get_prev_pos(log_view, &file_seq, + &file_offset); + if (file_offset >= max_offset && file_seq == max_seq) + break; + + if ((hdr->type & MAIL_TRANSACTION_SYNC) != 0) { + /* ignore changes done by dsync, unless we can get + expunged message's GUID from it */ + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != + MAIL_TRANSACTION_EXPUNGE_GUID) + continue; + } + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + if (!pvt_scan) + log_add_expunge(ctx, data, hdr); + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + if (!pvt_scan) + log_add_expunge_guid(ctx, view, data, hdr); + break; + case MAIL_TRANSACTION_FLAG_UPDATE: + log_add_flag_update(ctx, data, hdr); + break; + case MAIL_TRANSACTION_KEYWORD_RESET: + log_add_keyword_reset(ctx, data, hdr); + break; + case MAIL_TRANSACTION_KEYWORD_UPDATE: + T_BEGIN { + log_add_keyword_update(ctx, data, hdr); + } T_END; + break; + case MAIL_TRANSACTION_MODSEQ_UPDATE: + log_add_modseq_update(ctx, data, hdr, pvt_scan); + break; + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + cur_modseq = mail_transaction_log_view_get_prev_modseq(log_view); + log_add_attribute_update(ctx, data, hdr, cur_modseq); + break; + } + } + + if (!pvt_scan) { + ctx->last_log_seq = file_seq; + ctx->last_log_offset = file_offset; + } + mail_transaction_log_view_close(&log_view); + return ret; +} + +static int +dsync_mailbox_attribute_cmp(const struct dsync_mailbox_attribute *attr1, + const struct dsync_mailbox_attribute *attr2) +{ + if (attr1->type < attr2->type) + return -1; + if (attr1->type > attr2->type) + return 1; + return strcmp(attr1->key, attr2->key); +} + +static unsigned int +dsync_mailbox_attribute_hash(const struct dsync_mailbox_attribute *attr) +{ + return str_hash(attr->key) ^ attr->type; +} + +int dsync_transaction_log_scan_init(struct mail_index_view *view, + struct mail_index_view *pvt_view, + uint32_t highest_wanted_uid, + uint64_t modseq, uint64_t pvt_modseq, + struct dsync_transaction_log_scan **scan_r, + bool *pvt_too_old_r) +{ + struct dsync_transaction_log_scan *ctx; + pool_t pool; + int ret, ret2; + + *pvt_too_old_r = FALSE; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan", + 10240); + ctx = p_new(pool, struct dsync_transaction_log_scan, 1); + ctx->pool = pool; + hash_table_create_direct(&ctx->changes, pool, 0); + hash_table_create(&ctx->attr_changes, pool, 0, + dsync_mailbox_attribute_hash, + dsync_mailbox_attribute_cmp); + ctx->view = view; + ctx->highest_wanted_uid = highest_wanted_uid; + + if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0) + return -1; + if (pvt_view != NULL) { + if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0) + return -1; + if (ret2 == 0) { + ret = 0; + *pvt_too_old_r = TRUE; + } + } + + *scan_r = ctx; + return ret; +} + +HASH_TABLE_TYPE(dsync_uid_mail_change) +dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan) +{ + return scan->changes; +} + +HASH_TABLE_TYPE(dsync_attr_change) +dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan) +{ + return scan->attr_changes; +} + +bool +dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan) +{ + return scan->returned_all_changes; +} + +struct dsync_mail_change * +dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan, + uint32_t uid) +{ + struct mail_transaction_log_view *log_view; + const struct mail_transaction_header *hdr; + const void *data; + const char *reason; + bool reset, found = FALSE; + + i_assert(uid > 0); + + if (scan->highest_wanted_uid < uid) + scan->highest_wanted_uid = uid; + + log_view = mail_transaction_log_view_open(scan->view->index->log); + if (mail_transaction_log_view_set(log_view, + scan->last_log_seq, + scan->last_log_offset, + (uint32_t)-1, UOFF_T_MAX, + &reset, &reason) > 0) { + while (!found && + mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + if (log_add_expunge_uid(scan, data, hdr, uid)) + found = TRUE; + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + if (log_add_expunge_guid_uid(scan, data, hdr, uid)) + found = TRUE; + break; + } + } + } + mail_transaction_log_view_close(&log_view); + + return !found ? NULL : + hash_table_lookup(scan->changes, POINTER_CAST(uid)); +} + +void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **_scan) +{ + struct dsync_transaction_log_scan *scan = *_scan; + + *_scan = NULL; + + hash_table_destroy(&scan->changes); + hash_table_destroy(&scan->attr_changes); + pool_unref(&scan->pool); +} diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.h b/src/doveadm/dsync/dsync-transaction-log-scan.h new file mode 100644 index 0000000..458b775 --- /dev/null +++ b/src/doveadm/dsync/dsync-transaction-log-scan.h @@ -0,0 +1,32 @@ +#ifndef DSYNC_TRANSACTION_LOG_SCAN_H +#define DSYNC_TRANSACTION_LOG_SCAN_H + +HASH_TABLE_DEFINE_TYPE(dsync_uid_mail_change, + void *, struct dsync_mail_change *); +HASH_TABLE_DEFINE_TYPE(dsync_attr_change, + struct dsync_mailbox_attribute *, + struct dsync_mailbox_attribute *); + +struct mail_index_view; +struct dsync_transaction_log_scan; + +int dsync_transaction_log_scan_init(struct mail_index_view *view, + struct mail_index_view *pvt_view, + uint32_t highest_wanted_uid, + uint64_t modseq, uint64_t pvt_modseq, + struct dsync_transaction_log_scan **scan_r, + bool *pvt_too_old_r); +HASH_TABLE_TYPE(dsync_uid_mail_change) +dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan); +HASH_TABLE_TYPE(dsync_attr_change) +dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan); +/* Returns TRUE if the entire transaction log was scanned */ +bool dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan); +/* If the given UID has been expunged after the initial log scan, create/update + a change record for it and return it. */ +struct dsync_mail_change * +dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan, + uint32_t uid); +void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **scan); + +#endif diff --git a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c new file mode 100644 index 0000000..b068137 --- /dev/null +++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c @@ -0,0 +1,781 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "sha1.h" +#include "str.h" +#include "mailbox-list-private.h" +#include "dsync-mailbox-tree-private.h" +#include "test-common.h" + +#include <stdio.h> + +#define MAX_DEPTH 4 +#define TEST_NAMESPACE_NAME "INBOX" + +static struct mail_namespace inbox_namespace = { + .prefix = TEST_NAMESPACE_NAME"/", + .prefix_len = sizeof(TEST_NAMESPACE_NAME)-1 + 1 +}; + +char mail_namespace_get_sep(struct mail_namespace *ns ATTR_UNUSED) +{ + return '/'; +} + +void mailbox_name_get_sha128(const char *name, guid_128_t guid_128_r) +{ + unsigned char sha[SHA1_RESULTLEN]; + + sha1_get_digest(name, strlen(name), sha); + memcpy(guid_128_r, sha, I_MIN(GUID_128_SIZE, sizeof(sha))); +} + +void mailbox_list_name_unescape(const char **name ATTR_UNUSED, + char escape_char ATTR_UNUSED) +{ +} + +void mailbox_list_name_escape(const char *name, + const char *escape_chars ATTR_UNUSED, + string_t *dest) +{ + str_append(dest, name); +} + +static struct dsync_mailbox_node * +node_create(struct dsync_mailbox_tree *tree, unsigned int counter, + const char *name, unsigned int last_renamed_or_created) +{ + struct dsync_mailbox_node *node; + + node = dsync_mailbox_tree_get(tree, name); + memcpy(node->mailbox_guid, &counter, sizeof(counter)); + node->uid_validity = counter; + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + node->last_renamed_or_created = last_renamed_or_created; + return node; +} + +static struct dsync_mailbox_node * +random_node_create(struct dsync_mailbox_tree *tree, unsigned int counter, + const char *name) +{ + return node_create(tree, counter, name, i_rand_limit(10)); +} + +static void nodes_create(struct dsync_mailbox_tree *tree, unsigned int *counter, + const char *const *names) +{ + for (; *names != NULL; names++) { + *counter += 1; + node_create(tree, *counter, *names, 0); + } +} + +static void nodes_delete(struct dsync_mailbox_tree *tree, unsigned int *counter, + const char *const *names) +{ + struct dsync_mailbox_node *node; + + for (; *names != NULL; names++) { + *counter += 1; + node = node_create(tree, *counter, *names, 0); + node->existence = DSYNC_MAILBOX_NODE_DELETED; + } +} + +static void +create_random_nodes(struct dsync_mailbox_tree *tree, const char *parent_name, + unsigned int depth, unsigned int *counter) +{ + unsigned int parent_len, i, nodes_count = i_rand_minmax(1, 3); + string_t *str; + + if (depth == MAX_DEPTH) + return; + + str = t_str_new(32); + if (*parent_name != '\0') + str_printfa(str, "%s/", parent_name); + parent_len = str_len(str); + + for (i = 0; i < nodes_count; i++) { + *counter += 1; + str_truncate(str, parent_len); + str_printfa(str, "%u.%u", depth, i); + random_node_create(tree, *counter, str_c(str)); + create_random_nodes(tree, str_c(str), depth+1, counter); + } +} + +static struct dsync_mailbox_tree *create_random_tree(void) +{ + struct dsync_mailbox_tree *tree; + unsigned int counter = 0; + + tree = dsync_mailbox_tree_init('/', '\0', '_'); + create_random_nodes(tree, "", 0, &counter); + return tree; +} + +static void test_tree_nodes_fixup(struct dsync_mailbox_node **pos, + unsigned int *newguid_counter) +{ + struct dsync_mailbox_node *node; + + for (node = *pos; node != NULL; node = node->next) { + if (node->sync_delayed_guid_change) { + /* the real code will pick one of the GUIDs. + we don't really care which one gets picked, so we'll + just change them to the same new one */ + memcpy(node->mailbox_guid, newguid_counter, + sizeof(*newguid_counter)); + node->uid_validity = *newguid_counter; + *newguid_counter += 1; + } + if (node->existence == DSYNC_MAILBOX_NODE_DELETED) + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + test_tree_nodes_fixup(&node->first_child, newguid_counter); + if (node->existence != DSYNC_MAILBOX_NODE_EXISTS && + node->first_child == NULL) { + /* nonexistent node, drop it */ + *pos = node->next; + } else { + pos = &node->next; + } + } +} + +static void test_tree_fixup(struct dsync_mailbox_tree *tree) +{ + unsigned int newguid_counter = INT_MAX; + + test_tree_nodes_fixup(&tree->root.first_child, &newguid_counter); +} + +static void nodes_dump(const struct dsync_mailbox_node *node, unsigned int depth) +{ + unsigned int i; + + for (; node != NULL; node = node->next) { + for (i = 0; i < depth; i++) printf(" "); + printf("%-*s guid:%.5s uidv:%u %d%d %ld\n", 40-depth, node->name, + guid_128_to_string(node->mailbox_guid), node->uid_validity, + node->existence, node->subscribed ? 1 : 0, + (long)node->last_renamed_or_created); + nodes_dump(node->first_child, depth+1); + } +} + +static void trees_dump(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2) +{ + printf("tree1:\n"); + nodes_dump(tree1->root.first_child, 1); + printf("tree2:\n"); + nodes_dump(tree2->root.first_child, 1); +} + +static void test_trees_nofree(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree **_tree2) +{ + struct dsync_mailbox_tree *tree2 = *_tree2; + struct dsync_mailbox_tree *orig_tree1, *orig_tree2; + struct dsync_mailbox_tree_sync_ctx *ctx; + struct dsync_mailbox_node *dup_node1, *dup_node2; + + orig_tree1 = dsync_mailbox_tree_dup(tree1); + orig_tree2 = dsync_mailbox_tree_dup(tree2); + + /* test tree1 -> tree2 */ + dsync_mailbox_tree_build_guid_hash(tree1, &dup_node1, &dup_node2); + dsync_mailbox_tree_build_guid_hash(tree2, &dup_node1, &dup_node2); + ctx = dsync_mailbox_trees_sync_init(tree1, tree2, + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, + DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG); + while (dsync_mailbox_trees_sync_next(ctx) != NULL) { + } + dsync_mailbox_trees_sync_deinit(&ctx); + test_tree_fixup(tree1); + test_tree_fixup(tree2); + if (!dsync_mailbox_trees_equal(tree1, tree2)) { + test_assert(FALSE); + trees_dump(tree1, tree2); + } + + /* test tree2 -> tree1 */ + dsync_mailbox_tree_build_guid_hash(orig_tree1, &dup_node1, &dup_node2); + dsync_mailbox_tree_build_guid_hash(orig_tree2, &dup_node1, &dup_node2); + ctx = dsync_mailbox_trees_sync_init(orig_tree2, orig_tree1, + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0); + while (dsync_mailbox_trees_sync_next(ctx) != NULL) { + } + dsync_mailbox_trees_sync_deinit(&ctx); + test_tree_fixup(orig_tree1); + test_tree_fixup(orig_tree2); + if (!dsync_mailbox_trees_equal(orig_tree1, orig_tree2)) { + test_assert(FALSE); + trees_dump(orig_tree1, orig_tree2); + } + + /* make sure both directions produced equal trees */ + if (!dsync_mailbox_trees_equal(tree1, orig_tree1)) { + test_assert(FALSE); + trees_dump(tree1, orig_tree1); + } + + dsync_mailbox_tree_deinit(_tree2); + dsync_mailbox_tree_deinit(&orig_tree1); + dsync_mailbox_tree_deinit(&orig_tree2); +} + +static void +test_tree_nodes_add_namespace(struct dsync_mailbox_node *node, + struct mail_namespace *ns) +{ + for (; node != NULL; node = node->next) { + node->ns = ns; + test_tree_nodes_add_namespace(node->first_child, ns); + } +} + +static void +test_tree_add_namespace(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns) +{ + struct dsync_mailbox_node *node, *n; + + node = dsync_mailbox_tree_get(tree, TEST_NAMESPACE_NAME); + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + i_assert(tree->root.first_child == node); + i_assert(node->first_child == NULL); + node->first_child = node->next; + for (n = node->first_child; n != NULL; n = n->next) + n->parent = node; + node->next = NULL; + + test_tree_nodes_add_namespace(&tree->root, ns); +} + +static void test_trees(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2) +{ + struct dsync_mailbox_tree *tree1_dup, *tree2_dup; + + tree1_dup = dsync_mailbox_tree_dup(tree1); + tree2_dup = dsync_mailbox_tree_dup(tree2); + + /* test without namespace prefix */ + test_trees_nofree(tree1, &tree2); + dsync_mailbox_tree_deinit(&tree1); + + /* test with namespace prefix */ + test_tree_add_namespace(tree1_dup, &inbox_namespace); + test_tree_add_namespace(tree2_dup, &inbox_namespace); + test_trees_nofree(tree1_dup, &tree2_dup); + dsync_mailbox_tree_deinit(&tree1_dup); +} + +static void test_dsync_mailbox_tree_sync_creates(void) +{ + static const char *common_nodes[] = { "foo", "foo/bar", NULL }; + static const char *create1_nodes[] = { "bar", "foo/baz", NULL }; + static const char *create2_nodes[] = { "foo/xyz", "foo/bar/3", NULL }; + struct dsync_mailbox_tree *tree1, *tree2; + unsigned int counter = 0; + + test_begin("dsync mailbox tree sync creates"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + nodes_create(tree1, &counter, common_nodes); + tree2 = dsync_mailbox_tree_dup(tree1); + nodes_create(tree1, &counter, create1_nodes); + nodes_create(tree2, &counter, create2_nodes); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_deletes(void) +{ + static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", NULL }; + static const char *delete1_nodes[] = { "1", "2", NULL }; + static const char *delete2_nodes[] = { "2/s1", "x/y", NULL }; + struct dsync_mailbox_tree *tree1, *tree2; + unsigned int counter = 0; + + test_begin("dsync mailbox tree sync deletes"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + nodes_create(tree1, &counter, common_nodes); + tree2 = dsync_mailbox_tree_dup(tree1); + nodes_delete(tree1, &counter, delete1_nodes); + nodes_delete(tree2, &counter, delete2_nodes); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames1(void) +{ + static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", "3/s3", NULL }; + struct dsync_mailbox_tree *tree1, *tree2; + struct dsync_mailbox_node *node; + unsigned int counter = 0; + + test_begin("dsync mailbox tree sync renames 1"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + nodes_create(tree1, &counter, common_nodes); + tree2 = dsync_mailbox_tree_dup(tree1); + + node = dsync_mailbox_tree_get(tree1, "1"); + node->name = "a"; + node->last_renamed_or_created = 1000; + node = dsync_mailbox_tree_get(tree2, "2"); + node->name = "b"; + node->last_renamed_or_created = 1000; + + node = dsync_mailbox_tree_get(tree1, "3/s3"); + node->name = "z"; + node->last_renamed_or_created = 1000; + dsync_mailbox_tree_node_detach(node); + dsync_mailbox_tree_node_attach(node, &tree1->root); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames2(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 2"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 1); + node_create(tree1, 2, "0/1/2", 3); + + node_create(tree2, 1, "0", 0); + node_create(tree2, 2, "0/1/2", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames3(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 3"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2", 1); + node_create(tree1, 2, "0/3", 1); + + node_create(tree2, 1, "0/4/5", 0); + node_create(tree2, 2, "1", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames4(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 4"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/b", 0); + node_create(tree1, 2, "c", 2); + + node_create(tree2, 2, "0/a", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames5(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 5"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "b", 0); + node_create(tree1, 2, "c", 2); + + node_create(tree2, 2, "0/a", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames6(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 6"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 0); + node_create(tree1, 2, "0/2", 1); + + node_create(tree2, 1, "0", 1); + node_create(tree2, 2, "0/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames7(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 7"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2", 0); + node_create(tree2, 1, "1/2", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames8(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 8"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 0); + node_create(tree1, 2, "0/2", 1); + + node_create(tree2, 1, "0", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames9(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 9"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1/2", 0); + node_create(tree1, 2, "0/3", 1); + + node_create(tree2, 1, "0", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames10(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 10"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 0); + node_create(tree1, 3, "0/2/3", 0); + + node_create(tree2, 1, "0", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames11(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 11"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 2); + node_create(tree1, 0, "0/1/2", 0); + + node_create(tree2, 1, "0", 1); + node_create(tree2, 0, "0/1/2", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames12(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 12"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2", 0); + node_create(tree1, 2, "1", 0); + node_create(tree1, 3, "1/4", 0); + node_create(tree1, 4, "1/4/5", 1); + + node_create(tree2, 1, "1", 2); + node_create(tree2, 2, "1/4", 3); + node_create(tree2, 3, "1/4/6", 4); + node_create(tree2, 4, "1/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames13(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 13"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 4, "0.0/1.0/2.1", 0); + node_create(tree1, 5, "0.1", 2); + node_create(tree1, 6, "0.1/1.0", 2); + node_create(tree1, 7, "0.1/1.0/2.0", 8); + + node_create(tree2, 5, "0.1/1.0", 5); + node_create(tree2, 6, "0.1/1.0/2.0", 8); + node_create(tree2, 7, "0.1/1.1", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames14(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 14"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 0); + node_create(tree1, 2, "1/2", 0); + node_create(tree1, 3, "1/2/4", 1); + + node_create(tree2, 1, "1/2", 3); + node_create(tree2, 2, "1/2/5", 4); + node_create(tree2, 3, "1/2/4", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames15(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 15"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 0); + node_create(tree2, 2, "1", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames16(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 16"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1/2", 4); + node_create(tree1, 2, "1", 2); + + node_create(tree2, 1, "2", 1); + node_create(tree2, 2, "1/2", 3); + node_create(tree2, 3, "1", 5); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames17(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 17"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 1); + + node_create(tree2, 1, "1/2", 0); + node_create(tree2, 2, "1", 2); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames18(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 18"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 2, "a", 5); + node_create(tree1, 4, "a/c", 2); + node_create(tree1, 5, "b", 6); + + node_create(tree2, 1, "a", 7); + node_create(tree2, 2, "b", 3); + node_create(tree2, 3, "b/c", 4); + node_create(tree2, 4, "d", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames19(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 19"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2/1", 1); + node_create(tree1, 2, "0/4", 3); + node_create(tree1, 3, "0/2", 2); + + node_create(tree2, 1, "1", 0); + node_create(tree2, 2, "1/3", 4); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames20(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 20"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 0); + node_create(tree1, 2, "0", 0); + node_create(tree1, 3, "0/2", 0); + /* rename 0 -> 1/0 */ + node_create(tree2, 1, "1", 0); + node_create(tree2, 2, "1/0", 1); + node_create(tree2, 3, "1/0/2", 0); + + test_trees_nofree(tree1, &tree2); + test_assert(tree1->root.first_child != NULL && + tree1->root.first_child->next == NULL); + dsync_mailbox_tree_deinit(&tree1); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames21(void) +{ +#if 0 + /* FIXME: we can't currently test this without crashing */ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 21"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "INBOX", 0); + node_create(tree1, 2, "foo", 0); + /* swap INBOX and foo - the INBOX name is important since it's + treated specially */ + node_create(tree2, 1, "foo", 0); + node_create(tree2, 2, "INBOX", 1); + + test_trees(tree1, tree2); + test_end(); +#endif +} + +static void test_dsync_mailbox_tree_sync_renames22(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 22"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 3, "p/a", 0); + node_create(tree1, 0, "p/2", 0); + node_create(tree1, 5, "p/2/h", 0); + + node_create(tree2, 4, "p/1/z", 0); + node_create(tree2, 1, "p/2", 0); + node_create(tree2, 2, "p/2/a", 0); + node_create(tree2, 5, "p/2/y", 0); + node_create(tree2, 3, "p/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_random(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync random"); + tree1 = create_random_tree(); + tree2 = create_random_tree(); + test_trees(tree1, tree2); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_dsync_mailbox_tree_sync_creates, + test_dsync_mailbox_tree_sync_deletes, + test_dsync_mailbox_tree_sync_renames1, + test_dsync_mailbox_tree_sync_renames2, + test_dsync_mailbox_tree_sync_renames3, + test_dsync_mailbox_tree_sync_renames4, + test_dsync_mailbox_tree_sync_renames5, + test_dsync_mailbox_tree_sync_renames6, + test_dsync_mailbox_tree_sync_renames7, + test_dsync_mailbox_tree_sync_renames8, + test_dsync_mailbox_tree_sync_renames9, + test_dsync_mailbox_tree_sync_renames10, + test_dsync_mailbox_tree_sync_renames11, + test_dsync_mailbox_tree_sync_renames12, + test_dsync_mailbox_tree_sync_renames13, + test_dsync_mailbox_tree_sync_renames14, + test_dsync_mailbox_tree_sync_renames15, + test_dsync_mailbox_tree_sync_renames16, + test_dsync_mailbox_tree_sync_renames17, + test_dsync_mailbox_tree_sync_renames18, + test_dsync_mailbox_tree_sync_renames19, + test_dsync_mailbox_tree_sync_renames20, + test_dsync_mailbox_tree_sync_renames21, + test_dsync_mailbox_tree_sync_renames22, + test_dsync_mailbox_tree_sync_random, + NULL + }; + return test_run(test_functions); +} |