diff options
Diffstat (limited to 'src/doveadm')
120 files changed, 42367 insertions, 0 deletions
diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am new file mode 100644 index 0000000..440af73 --- /dev/null +++ b/src/doveadm/Makefile.am @@ -0,0 +1,201 @@ +doveadm_moduledir = $(moduledir)/doveadm +pkglibexecdir = $(libexecdir)/dovecot + +SUBDIRS = dsync + +bin_PROGRAMS = doveadm +pkglibexec_PROGRAMS = doveadm-server + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-auth \ + -I$(top_srcdir)/src/lib-compression \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-imap-storage \ + -I$(top_srcdir)/src/lib-http \ + -I$(top_srcdir)/src/lib-dcrypt \ + -I$(top_srcdir)/src/auth \ + -I$(top_srcdir)/src/stats \ + -DMODULEDIR=\""$(moduledir)"\" \ + -DAUTH_MODULE_DIR=\""$(moduledir)/auth"\" \ + -DDOVEADM_MODULEDIR=\""$(doveadm_moduledir)"\" \ + -DPKG_RUNDIR=\""$(rundir)"\" \ + -DPKG_STATEDIR=\""$(statedir)"\" \ + -DPKG_LIBEXECDIR=\""$(pkglibexecdir)"\" \ + -DBINDIR=\""$(bindir)"\" \ + -DMANDIR=\""$(mandir)"\" \ + $(BINARY_CFLAGS) + +cmd_pw_libs = \ + ../auth/libpassword.la \ + ../lib-otp/libotp.la + +libs = \ + dsync/libdsync.la \ + ../lib-compression/libcompression.la + +doveadm_LDADD = \ + $(libs) \ + $(cmd_pw_libs) \ + $(CRYPT_LIBS) \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(LIBSODIUM_LIBS) \ + $(BINARY_LDFLAGS) \ + -lm + +doveadm_DEPENDENCIES = \ + $(libs) \ + $(cmd_pw_libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) + +doveadm_server_LDADD = \ + $(libs) \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(BINARY_LDFLAGS) \ + -lm + +doveadm_server_DEPENDENCIES = \ + $(libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) + +doveadm_common_cmds = \ + doveadm-auth.c \ + doveadm-dict.c \ + doveadm-director.c \ + doveadm-fs.c \ + doveadm-instance.c \ + doveadm-kick.c \ + doveadm-log.c \ + doveadm-master.c \ + doveadm-mutf7.c \ + doveadm-penalty.c \ + doveadm-proxy.c \ + doveadm-replicator.c \ + doveadm-sis.c \ + doveadm-stats.c \ + doveadm-oldstats.c \ + doveadm-who.c + +doveadm_common_mail_cmds = \ + doveadm-dsync.c \ + doveadm-mail.c \ + doveadm-mail-altmove.c \ + doveadm-mail-batch.c \ + doveadm-mail-deduplicate.c \ + doveadm-mail-expunge.c \ + doveadm-mail-fetch.c \ + doveadm-mail-flags.c \ + doveadm-mail-import.c \ + doveadm-mail-index.c \ + doveadm-mail-iter.c \ + doveadm-mail-mailbox.c \ + doveadm-mail-mailbox-metadata.c \ + doveadm-mail-mailbox-status.c \ + doveadm-mail-copymove.c \ + doveadm-mailbox-list-iter.c \ + doveadm-mail-save.c \ + doveadm-mail-search.c \ + doveadm-mail-server.c \ + doveadm-mail-mailbox-cache.c \ + doveadm-mail-rebuild.c + +# these aren't actually useful in doveadm-server, but plugins may implement +# both dumping and some other commands inside a single plugin. not having the +# dump functions in doveadm-server fails to load such plugins. +doveadm_common_dump_cmds = \ + doveadm-dump.c \ + doveadm-dump-dbox.c \ + doveadm-dump-index.c \ + doveadm-dump-log.c \ + doveadm-dump-mailboxlog.c \ + doveadm-dump-thread.c \ + doveadm-dump-dcrypt-file.c \ + doveadm-dump-dcrypt-key.c \ + doveadm-zlib.c + +common = \ + $(doveadm_common_cmds) \ + $(doveadm_common_mail_cmds) \ + $(doveadm_common_dump_cmds) \ + doveadm-cmd.c \ + doveadm-print.c \ + doveadm-settings.c \ + doveadm-util.c \ + server-connection.c \ + doveadm-print-formatted.c + +doveadm_SOURCES = \ + $(common) \ + doveadm.c \ + doveadm-print-flow.c \ + doveadm-print-pager.c \ + doveadm-print-tab.c \ + doveadm-print-table.c \ + doveadm-print-json.c \ + doveadm-pw.c + +doveadm_server_SOURCES = \ + $(common) \ + doveadm-auth-server.c \ + client-connection.c \ + client-connection-tcp.c \ + client-connection-http.c \ + doveadm-print-server.c \ + doveadm-print-json.c \ + main.c + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + doveadm.h \ + doveadm-cmd.h \ + doveadm-dsync.h \ + doveadm-dump.h \ + doveadm-mail.h \ + doveadm-mail-iter.h \ + doveadm-mailbox-list-iter.h \ + doveadm-print.h \ + doveadm-print-private.h \ + doveadm-settings.h \ + doveadm-util.h + +noinst_HEADERS = \ + client-connection.h \ + client-connection-private.h \ + server-connection.h \ + doveadm-server.h \ + doveadm-who.h + +install-exec-local: + rm -f $(DESTDIR)$(bindir)/dsync + $(LN_S) doveadm $(DESTDIR)$(bindir)/dsync + +test_programs = \ + test-doveadm-util +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la +test_deps = $(noinst_LTLIBRARIES) $(test_libs) + +test_doveadm_util_SOURCES = doveadm-util.c test-doveadm-util.c +test_doveadm_util_LDADD = $(test_libs) $(MODULE_LIBS) +test_doveadm_util_DEPENDENCIES = $(test_deps) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/doveadm/Makefile.in b/src/doveadm/Makefile.in new file mode 100644 index 0000000..b4e04fd --- /dev/null +++ b/src/doveadm/Makefile.in @@ -0,0 +1,1515 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +bin_PROGRAMS = doveadm$(EXEEXT) +pkglibexec_PROGRAMS = doveadm-server$(EXEEXT) +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/doveadm +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \ + $(pkginc_lib_HEADERS) $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkglibexecdir)" \ + "$(DESTDIR)$(pkginc_libdir)" +am__EXEEXT_1 = test-doveadm-util$(EXEEXT) +PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) $(pkglibexec_PROGRAMS) +am__objects_1 = doveadm-auth.$(OBJEXT) doveadm-dict.$(OBJEXT) \ + doveadm-director.$(OBJEXT) doveadm-fs.$(OBJEXT) \ + doveadm-instance.$(OBJEXT) doveadm-kick.$(OBJEXT) \ + doveadm-log.$(OBJEXT) doveadm-master.$(OBJEXT) \ + doveadm-mutf7.$(OBJEXT) doveadm-penalty.$(OBJEXT) \ + doveadm-proxy.$(OBJEXT) doveadm-replicator.$(OBJEXT) \ + doveadm-sis.$(OBJEXT) doveadm-stats.$(OBJEXT) \ + doveadm-oldstats.$(OBJEXT) doveadm-who.$(OBJEXT) +am__objects_2 = doveadm-dsync.$(OBJEXT) doveadm-mail.$(OBJEXT) \ + doveadm-mail-altmove.$(OBJEXT) doveadm-mail-batch.$(OBJEXT) \ + doveadm-mail-deduplicate.$(OBJEXT) \ + doveadm-mail-expunge.$(OBJEXT) doveadm-mail-fetch.$(OBJEXT) \ + doveadm-mail-flags.$(OBJEXT) doveadm-mail-import.$(OBJEXT) \ + doveadm-mail-index.$(OBJEXT) doveadm-mail-iter.$(OBJEXT) \ + doveadm-mail-mailbox.$(OBJEXT) \ + doveadm-mail-mailbox-metadata.$(OBJEXT) \ + doveadm-mail-mailbox-status.$(OBJEXT) \ + doveadm-mail-copymove.$(OBJEXT) \ + doveadm-mailbox-list-iter.$(OBJEXT) \ + doveadm-mail-save.$(OBJEXT) doveadm-mail-search.$(OBJEXT) \ + doveadm-mail-server.$(OBJEXT) \ + doveadm-mail-mailbox-cache.$(OBJEXT) \ + doveadm-mail-rebuild.$(OBJEXT) +am__objects_3 = doveadm-dump.$(OBJEXT) doveadm-dump-dbox.$(OBJEXT) \ + doveadm-dump-index.$(OBJEXT) doveadm-dump-log.$(OBJEXT) \ + doveadm-dump-mailboxlog.$(OBJEXT) \ + doveadm-dump-thread.$(OBJEXT) \ + doveadm-dump-dcrypt-file.$(OBJEXT) \ + doveadm-dump-dcrypt-key.$(OBJEXT) doveadm-zlib.$(OBJEXT) +am__objects_4 = $(am__objects_1) $(am__objects_2) $(am__objects_3) \ + doveadm-cmd.$(OBJEXT) doveadm-print.$(OBJEXT) \ + doveadm-settings.$(OBJEXT) doveadm-util.$(OBJEXT) \ + server-connection.$(OBJEXT) doveadm-print-formatted.$(OBJEXT) +am_doveadm_OBJECTS = $(am__objects_4) doveadm.$(OBJEXT) \ + doveadm-print-flow.$(OBJEXT) doveadm-print-pager.$(OBJEXT) \ + doveadm-print-tab.$(OBJEXT) doveadm-print-table.$(OBJEXT) \ + doveadm-print-json.$(OBJEXT) doveadm-pw.$(OBJEXT) +doveadm_OBJECTS = $(am_doveadm_OBJECTS) +am__DEPENDENCIES_1 = +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +am_doveadm_server_OBJECTS = $(am__objects_4) \ + doveadm-auth-server.$(OBJEXT) client-connection.$(OBJEXT) \ + client-connection-tcp.$(OBJEXT) \ + client-connection-http.$(OBJEXT) \ + doveadm-print-server.$(OBJEXT) doveadm-print-json.$(OBJEXT) \ + main.$(OBJEXT) +doveadm_server_OBJECTS = $(am_doveadm_server_OBJECTS) +am_test_doveadm_util_OBJECTS = doveadm-util.$(OBJEXT) \ + test-doveadm-util.$(OBJEXT) +test_doveadm_util_OBJECTS = $(am_test_doveadm_util_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/client-connection-http.Po \ + ./$(DEPDIR)/client-connection-tcp.Po \ + ./$(DEPDIR)/client-connection.Po \ + ./$(DEPDIR)/doveadm-auth-server.Po ./$(DEPDIR)/doveadm-auth.Po \ + ./$(DEPDIR)/doveadm-cmd.Po ./$(DEPDIR)/doveadm-dict.Po \ + ./$(DEPDIR)/doveadm-director.Po ./$(DEPDIR)/doveadm-dsync.Po \ + ./$(DEPDIR)/doveadm-dump-dbox.Po \ + ./$(DEPDIR)/doveadm-dump-dcrypt-file.Po \ + ./$(DEPDIR)/doveadm-dump-dcrypt-key.Po \ + ./$(DEPDIR)/doveadm-dump-index.Po \ + ./$(DEPDIR)/doveadm-dump-log.Po \ + ./$(DEPDIR)/doveadm-dump-mailboxlog.Po \ + ./$(DEPDIR)/doveadm-dump-thread.Po ./$(DEPDIR)/doveadm-dump.Po \ + ./$(DEPDIR)/doveadm-fs.Po ./$(DEPDIR)/doveadm-instance.Po \ + ./$(DEPDIR)/doveadm-kick.Po ./$(DEPDIR)/doveadm-log.Po \ + ./$(DEPDIR)/doveadm-mail-altmove.Po \ + ./$(DEPDIR)/doveadm-mail-batch.Po \ + ./$(DEPDIR)/doveadm-mail-copymove.Po \ + ./$(DEPDIR)/doveadm-mail-deduplicate.Po \ + ./$(DEPDIR)/doveadm-mail-expunge.Po \ + ./$(DEPDIR)/doveadm-mail-fetch.Po \ + ./$(DEPDIR)/doveadm-mail-flags.Po \ + ./$(DEPDIR)/doveadm-mail-import.Po \ + ./$(DEPDIR)/doveadm-mail-index.Po \ + ./$(DEPDIR)/doveadm-mail-iter.Po \ + ./$(DEPDIR)/doveadm-mail-mailbox-cache.Po \ + ./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po \ + ./$(DEPDIR)/doveadm-mail-mailbox-status.Po \ + ./$(DEPDIR)/doveadm-mail-mailbox.Po \ + ./$(DEPDIR)/doveadm-mail-rebuild.Po \ + ./$(DEPDIR)/doveadm-mail-save.Po \ + ./$(DEPDIR)/doveadm-mail-search.Po \ + ./$(DEPDIR)/doveadm-mail-server.Po ./$(DEPDIR)/doveadm-mail.Po \ + ./$(DEPDIR)/doveadm-mailbox-list-iter.Po \ + ./$(DEPDIR)/doveadm-master.Po ./$(DEPDIR)/doveadm-mutf7.Po \ + ./$(DEPDIR)/doveadm-oldstats.Po ./$(DEPDIR)/doveadm-penalty.Po \ + ./$(DEPDIR)/doveadm-print-flow.Po \ + ./$(DEPDIR)/doveadm-print-formatted.Po \ + ./$(DEPDIR)/doveadm-print-json.Po \ + ./$(DEPDIR)/doveadm-print-pager.Po \ + ./$(DEPDIR)/doveadm-print-server.Po \ + ./$(DEPDIR)/doveadm-print-tab.Po \ + ./$(DEPDIR)/doveadm-print-table.Po \ + ./$(DEPDIR)/doveadm-print.Po ./$(DEPDIR)/doveadm-proxy.Po \ + ./$(DEPDIR)/doveadm-pw.Po ./$(DEPDIR)/doveadm-replicator.Po \ + ./$(DEPDIR)/doveadm-settings.Po ./$(DEPDIR)/doveadm-sis.Po \ + ./$(DEPDIR)/doveadm-stats.Po ./$(DEPDIR)/doveadm-util.Po \ + ./$(DEPDIR)/doveadm-who.Po ./$(DEPDIR)/doveadm-zlib.Po \ + ./$(DEPDIR)/doveadm.Po ./$(DEPDIR)/main.Po \ + ./$(DEPDIR)/server-connection.Po \ + ./$(DEPDIR)/test-doveadm-util.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(doveadm_SOURCES) $(doveadm_server_SOURCES) \ + $(test_doveadm_util_SOURCES) +DIST_SOURCES = $(doveadm_SOURCES) $(doveadm_server_SOURCES) \ + $(test_doveadm_util_SOURCES) +RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \ + ctags-recursive dvi-recursive html-recursive info-recursive \ + install-data-recursive install-dvi-recursive \ + install-exec-recursive install-html-recursive \ + install-info-recursive install-pdf-recursive \ + install-ps-recursive install-recursive installcheck-recursive \ + installdirs-recursive pdf-recursive ps-recursive \ + tags-recursive uninstall-recursive +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS) +RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \ + distclean-recursive maintainer-clean-recursive +am__recursive_targets = \ + $(RECURSIVE_TARGETS) \ + $(RECURSIVE_CLEAN_TARGETS) \ + $(am__extra_recursive_targets) +AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \ + distdir distdir-am +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +DIST_SUBDIRS = $(SUBDIRS) +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +am__relativize = \ + dir0=`pwd`; \ + sed_first='s,^\([^/]*\)/.*$$,\1,'; \ + sed_rest='s,^[^/]*/*,,'; \ + sed_last='s,^.*/\([^/]*\)$$,\1,'; \ + sed_butlast='s,/*[^/]*$$,,'; \ + while test -n "$$dir1"; do \ + first=`echo "$$dir1" | sed -e "$$sed_first"`; \ + if test "$$first" != "."; then \ + if test "$$first" = ".."; then \ + dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \ + dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \ + else \ + first2=`echo "$$dir2" | sed -e "$$sed_first"`; \ + if test "$$first2" = "$$first"; then \ + dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \ + else \ + dir2="../$$dir2"; \ + fi; \ + dir0="$$dir0"/"$$first"; \ + fi; \ + fi; \ + dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \ + done; \ + reldir="$$dir2" +pkglibexecdir = $(libexecdir)/dovecot +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +doveadm_moduledir = $(moduledir)/doveadm +SUBDIRS = dsync +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-auth \ + -I$(top_srcdir)/src/lib-compression \ + -I$(top_srcdir)/src/lib-dict \ + -I$(top_srcdir)/src/lib-fs \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage \ + -I$(top_srcdir)/src/lib-imap-storage \ + -I$(top_srcdir)/src/lib-http \ + -I$(top_srcdir)/src/lib-dcrypt \ + -I$(top_srcdir)/src/auth \ + -I$(top_srcdir)/src/stats \ + -DMODULEDIR=\""$(moduledir)"\" \ + -DAUTH_MODULE_DIR=\""$(moduledir)/auth"\" \ + -DDOVEADM_MODULEDIR=\""$(doveadm_moduledir)"\" \ + -DPKG_RUNDIR=\""$(rundir)"\" \ + -DPKG_STATEDIR=\""$(statedir)"\" \ + -DPKG_LIBEXECDIR=\""$(pkglibexecdir)"\" \ + -DBINDIR=\""$(bindir)"\" \ + -DMANDIR=\""$(mandir)"\" \ + $(BINARY_CFLAGS) + +cmd_pw_libs = \ + ../auth/libpassword.la \ + ../lib-otp/libotp.la + +libs = \ + dsync/libdsync.la \ + ../lib-compression/libcompression.la + +doveadm_LDADD = \ + $(libs) \ + $(cmd_pw_libs) \ + $(CRYPT_LIBS) \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(LIBSODIUM_LIBS) \ + $(BINARY_LDFLAGS) \ + -lm + +doveadm_DEPENDENCIES = \ + $(libs) \ + $(cmd_pw_libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) + +doveadm_server_LDADD = \ + $(libs) \ + $(LIBDOVECOT_STORAGE) \ + $(LIBDOVECOT) \ + $(BINARY_LDFLAGS) \ + -lm + +doveadm_server_DEPENDENCIES = \ + $(libs) \ + $(LIBDOVECOT_STORAGE_DEPS) \ + $(LIBDOVECOT_DEPS) + +doveadm_common_cmds = \ + doveadm-auth.c \ + doveadm-dict.c \ + doveadm-director.c \ + doveadm-fs.c \ + doveadm-instance.c \ + doveadm-kick.c \ + doveadm-log.c \ + doveadm-master.c \ + doveadm-mutf7.c \ + doveadm-penalty.c \ + doveadm-proxy.c \ + doveadm-replicator.c \ + doveadm-sis.c \ + doveadm-stats.c \ + doveadm-oldstats.c \ + doveadm-who.c + +doveadm_common_mail_cmds = \ + doveadm-dsync.c \ + doveadm-mail.c \ + doveadm-mail-altmove.c \ + doveadm-mail-batch.c \ + doveadm-mail-deduplicate.c \ + doveadm-mail-expunge.c \ + doveadm-mail-fetch.c \ + doveadm-mail-flags.c \ + doveadm-mail-import.c \ + doveadm-mail-index.c \ + doveadm-mail-iter.c \ + doveadm-mail-mailbox.c \ + doveadm-mail-mailbox-metadata.c \ + doveadm-mail-mailbox-status.c \ + doveadm-mail-copymove.c \ + doveadm-mailbox-list-iter.c \ + doveadm-mail-save.c \ + doveadm-mail-search.c \ + doveadm-mail-server.c \ + doveadm-mail-mailbox-cache.c \ + doveadm-mail-rebuild.c + + +# these aren't actually useful in doveadm-server, but plugins may implement +# both dumping and some other commands inside a single plugin. not having the +# dump functions in doveadm-server fails to load such plugins. +doveadm_common_dump_cmds = \ + doveadm-dump.c \ + doveadm-dump-dbox.c \ + doveadm-dump-index.c \ + doveadm-dump-log.c \ + doveadm-dump-mailboxlog.c \ + doveadm-dump-thread.c \ + doveadm-dump-dcrypt-file.c \ + doveadm-dump-dcrypt-key.c \ + doveadm-zlib.c + +common = \ + $(doveadm_common_cmds) \ + $(doveadm_common_mail_cmds) \ + $(doveadm_common_dump_cmds) \ + doveadm-cmd.c \ + doveadm-print.c \ + doveadm-settings.c \ + doveadm-util.c \ + server-connection.c \ + doveadm-print-formatted.c + +doveadm_SOURCES = \ + $(common) \ + doveadm.c \ + doveadm-print-flow.c \ + doveadm-print-pager.c \ + doveadm-print-tab.c \ + doveadm-print-table.c \ + doveadm-print-json.c \ + doveadm-pw.c + +doveadm_server_SOURCES = \ + $(common) \ + doveadm-auth-server.c \ + client-connection.c \ + client-connection-tcp.c \ + client-connection-http.c \ + doveadm-print-server.c \ + doveadm-print-json.c \ + main.c + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + doveadm.h \ + doveadm-cmd.h \ + doveadm-dsync.h \ + doveadm-dump.h \ + doveadm-mail.h \ + doveadm-mail-iter.h \ + doveadm-mailbox-list-iter.h \ + doveadm-print.h \ + doveadm-print-private.h \ + doveadm-settings.h \ + doveadm-util.h + +noinst_HEADERS = \ + client-connection.h \ + client-connection-private.h \ + server-connection.h \ + doveadm-server.h \ + doveadm-who.h + +test_programs = \ + test-doveadm-util + +test_libs = \ + ../lib-test/libtest.la \ + ../lib/liblib.la + +test_deps = $(noinst_LTLIBRARIES) $(test_libs) +test_doveadm_util_SOURCES = doveadm-util.c test-doveadm-util.c +test_doveadm_util_LDADD = $(test_libs) $(MODULE_LIBS) +test_doveadm_util_DEPENDENCIES = $(test_deps) +all: all-recursive + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/doveadm/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/doveadm/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): +install-binPROGRAMS: $(bin_PROGRAMS) + @$(NORMAL_INSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \ + fi; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p \ + || test -f $$p1 \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-binPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' \ + `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(bindir)" && rm -f $$files + +clean-binPROGRAMS: + @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list +install-pkglibexecPROGRAMS: $(pkglibexec_PROGRAMS) + @$(NORMAL_INSTALL) + @list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkglibexecdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkglibexecdir)" || exit 1; \ + fi; \ + for p in $$list; do echo "$$p $$p"; done | \ + sed 's/$(EXEEXT)$$//' | \ + while read p p1; do if test -f $$p \ + || test -f $$p1 \ + ; then echo "$$p"; echo "$$p"; else :; fi; \ + done | \ + sed -e 'p;s,.*/,,;n;h' \ + -e 's|.*|.|' \ + -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ + sed 'N;N;N;s,\n, ,g' | \ + $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ + { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ + if ($$2 == $$4) files[d] = files[d] " " $$1; \ + else { print "f", $$3 "/" $$4, $$1; } } \ + END { for (d in files) print "f", d, files[d] }' | \ + while read type dir files; do \ + if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ + test -z "$$files" || { \ + echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(pkglibexecdir)$$dir'"; \ + $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(pkglibexecdir)$$dir" || exit $$?; \ + } \ + ; done + +uninstall-pkglibexecPROGRAMS: + @$(NORMAL_UNINSTALL) + @list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \ + files=`for p in $$list; do echo "$$p"; done | \ + sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ + -e 's/$$/$(EXEEXT)/' \ + `; \ + test -n "$$list" || exit 0; \ + echo " ( cd '$(DESTDIR)$(pkglibexecdir)' && rm -f" $$files ")"; \ + cd "$(DESTDIR)$(pkglibexecdir)" && rm -f $$files + +clean-pkglibexecPROGRAMS: + @list='$(pkglibexec_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +doveadm$(EXEEXT): $(doveadm_OBJECTS) $(doveadm_DEPENDENCIES) $(EXTRA_doveadm_DEPENDENCIES) + @rm -f doveadm$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(doveadm_OBJECTS) $(doveadm_LDADD) $(LIBS) + +doveadm-server$(EXEEXT): $(doveadm_server_OBJECTS) $(doveadm_server_DEPENDENCIES) $(EXTRA_doveadm_server_DEPENDENCIES) + @rm -f doveadm-server$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(doveadm_server_OBJECTS) $(doveadm_server_LDADD) $(LIBS) + +test-doveadm-util$(EXEEXT): $(test_doveadm_util_OBJECTS) $(test_doveadm_util_DEPENDENCIES) $(EXTRA_test_doveadm_util_DEPENDENCIES) + @rm -f test-doveadm-util$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_doveadm_util_OBJECTS) $(test_doveadm_util_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-connection-http.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-connection-tcp.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-connection.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-auth-server.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-auth.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-cmd.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dict.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-director.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dsync.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-dbox.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-dcrypt-file.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-dcrypt-key.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-index.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-log.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-mailboxlog.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-thread.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-fs.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-instance.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-kick.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-log.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-altmove.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-batch.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-copymove.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-deduplicate.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-expunge.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-fetch.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-flags.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-import.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-index.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-iter.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox-cache.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox-status.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-rebuild.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-save.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-search.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-server.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mailbox-list-iter.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-master.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mutf7.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-oldstats.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-penalty.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-flow.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-formatted.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-json.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-pager.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-server.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-tab.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-table.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-proxy.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-pw.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-replicator.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-settings.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-sis.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-stats.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-util.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-who.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-zlib.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/server-connection.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-doveadm-util.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +# This directory's subdirectories are mostly independent; you can cd +# into them and run 'make' without going through this Makefile. +# To change the values of 'make' variables: instead of editing Makefiles, +# (1) if the variable is set in 'config.status', edit 'config.status' +# (which will cause the Makefiles to be regenerated when you run 'make'); +# (2) otherwise, pass the desired values on the 'make' command line. +$(am__recursive_targets): + @fail=; \ + if $(am__make_keepgoing); then \ + failcom='fail=yes'; \ + else \ + failcom='exit 1'; \ + fi; \ + dot_seen=no; \ + target=`echo $@ | sed s/-recursive//`; \ + case "$@" in \ + distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \ + *) list='$(SUBDIRS)' ;; \ + esac; \ + for subdir in $$list; do \ + echo "Making $$target in $$subdir"; \ + if test "$$subdir" = "."; then \ + dot_seen=yes; \ + local_target="$$target-am"; \ + else \ + local_target="$$target"; \ + fi; \ + ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \ + || eval $$failcom; \ + done; \ + if test "$$dot_seen" = "no"; then \ + $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \ + fi; test -z "$$fail" + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-recursive +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \ + include_option=--etags-include; \ + empty_fix=.; \ + else \ + include_option=--include; \ + empty_fix=; \ + fi; \ + list='$(SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + test ! -f $$subdir/TAGS || \ + set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \ + fi; \ + done; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-recursive + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-recursive + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done + @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \ + if test "$$subdir" = .; then :; else \ + $(am__make_dryrun) \ + || test -d "$(distdir)/$$subdir" \ + || $(MKDIR_P) "$(distdir)/$$subdir" \ + || exit 1; \ + dir1=$$subdir; dir2="$(distdir)/$$subdir"; \ + $(am__relativize); \ + new_distdir=$$reldir; \ + dir1=$$subdir; dir2="$(top_distdir)"; \ + $(am__relativize); \ + new_top_distdir=$$reldir; \ + echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \ + echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \ + ($(am__cd) $$subdir && \ + $(MAKE) $(AM_MAKEFLAGS) \ + top_distdir="$$new_top_distdir" \ + distdir="$$new_distdir" \ + am__remove_distdir=: \ + am__skip_length_check=: \ + am__skip_mode_fix=: \ + distdir) \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-recursive +all-am: Makefile $(PROGRAMS) $(HEADERS) +installdirs: installdirs-recursive +installdirs-am: + for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkglibexecdir)" "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-recursive +install-exec: install-exec-recursive +install-data: install-data-recursive +uninstall: uninstall-recursive + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-recursive +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-recursive + +clean-am: clean-binPROGRAMS clean-generic clean-libtool \ + clean-noinstPROGRAMS clean-pkglibexecPROGRAMS mostlyclean-am + +distclean: distclean-recursive + -rm -f ./$(DEPDIR)/client-connection-http.Po + -rm -f ./$(DEPDIR)/client-connection-tcp.Po + -rm -f ./$(DEPDIR)/client-connection.Po + -rm -f ./$(DEPDIR)/doveadm-auth-server.Po + -rm -f ./$(DEPDIR)/doveadm-auth.Po + -rm -f ./$(DEPDIR)/doveadm-cmd.Po + -rm -f ./$(DEPDIR)/doveadm-dict.Po + -rm -f ./$(DEPDIR)/doveadm-director.Po + -rm -f ./$(DEPDIR)/doveadm-dsync.Po + -rm -f ./$(DEPDIR)/doveadm-dump-dbox.Po + -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-file.Po + -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-key.Po + -rm -f ./$(DEPDIR)/doveadm-dump-index.Po + -rm -f ./$(DEPDIR)/doveadm-dump-log.Po + -rm -f ./$(DEPDIR)/doveadm-dump-mailboxlog.Po + -rm -f ./$(DEPDIR)/doveadm-dump-thread.Po + -rm -f ./$(DEPDIR)/doveadm-dump.Po + -rm -f ./$(DEPDIR)/doveadm-fs.Po + -rm -f ./$(DEPDIR)/doveadm-instance.Po + -rm -f ./$(DEPDIR)/doveadm-kick.Po + -rm -f ./$(DEPDIR)/doveadm-log.Po + -rm -f ./$(DEPDIR)/doveadm-mail-altmove.Po + -rm -f ./$(DEPDIR)/doveadm-mail-batch.Po + -rm -f ./$(DEPDIR)/doveadm-mail-copymove.Po + -rm -f ./$(DEPDIR)/doveadm-mail-deduplicate.Po + -rm -f ./$(DEPDIR)/doveadm-mail-expunge.Po + -rm -f ./$(DEPDIR)/doveadm-mail-fetch.Po + -rm -f ./$(DEPDIR)/doveadm-mail-flags.Po + -rm -f ./$(DEPDIR)/doveadm-mail-import.Po + -rm -f ./$(DEPDIR)/doveadm-mail-index.Po + -rm -f ./$(DEPDIR)/doveadm-mail-iter.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-cache.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-status.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox.Po + -rm -f ./$(DEPDIR)/doveadm-mail-rebuild.Po + -rm -f ./$(DEPDIR)/doveadm-mail-save.Po + -rm -f ./$(DEPDIR)/doveadm-mail-search.Po + -rm -f ./$(DEPDIR)/doveadm-mail-server.Po + -rm -f ./$(DEPDIR)/doveadm-mail.Po + -rm -f ./$(DEPDIR)/doveadm-mailbox-list-iter.Po + -rm -f ./$(DEPDIR)/doveadm-master.Po + -rm -f ./$(DEPDIR)/doveadm-mutf7.Po + -rm -f ./$(DEPDIR)/doveadm-oldstats.Po + -rm -f ./$(DEPDIR)/doveadm-penalty.Po + -rm -f ./$(DEPDIR)/doveadm-print-flow.Po + -rm -f ./$(DEPDIR)/doveadm-print-formatted.Po + -rm -f ./$(DEPDIR)/doveadm-print-json.Po + -rm -f ./$(DEPDIR)/doveadm-print-pager.Po + -rm -f ./$(DEPDIR)/doveadm-print-server.Po + -rm -f ./$(DEPDIR)/doveadm-print-tab.Po + -rm -f ./$(DEPDIR)/doveadm-print-table.Po + -rm -f ./$(DEPDIR)/doveadm-print.Po + -rm -f ./$(DEPDIR)/doveadm-proxy.Po + -rm -f ./$(DEPDIR)/doveadm-pw.Po + -rm -f ./$(DEPDIR)/doveadm-replicator.Po + -rm -f ./$(DEPDIR)/doveadm-settings.Po + -rm -f ./$(DEPDIR)/doveadm-sis.Po + -rm -f ./$(DEPDIR)/doveadm-stats.Po + -rm -f ./$(DEPDIR)/doveadm-util.Po + -rm -f ./$(DEPDIR)/doveadm-who.Po + -rm -f ./$(DEPDIR)/doveadm-zlib.Po + -rm -f ./$(DEPDIR)/doveadm.Po + -rm -f ./$(DEPDIR)/main.Po + -rm -f ./$(DEPDIR)/server-connection.Po + -rm -f ./$(DEPDIR)/test-doveadm-util.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-recursive + +dvi-am: + +html: html-recursive + +html-am: + +info: info-recursive + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-recursive + +install-dvi-am: + +install-exec-am: install-binPROGRAMS install-exec-local \ + install-pkglibexecPROGRAMS + +install-html: install-html-recursive + +install-html-am: + +install-info: install-info-recursive + +install-info-am: + +install-man: + +install-pdf: install-pdf-recursive + +install-pdf-am: + +install-ps: install-ps-recursive + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-recursive + -rm -f ./$(DEPDIR)/client-connection-http.Po + -rm -f ./$(DEPDIR)/client-connection-tcp.Po + -rm -f ./$(DEPDIR)/client-connection.Po + -rm -f ./$(DEPDIR)/doveadm-auth-server.Po + -rm -f ./$(DEPDIR)/doveadm-auth.Po + -rm -f ./$(DEPDIR)/doveadm-cmd.Po + -rm -f ./$(DEPDIR)/doveadm-dict.Po + -rm -f ./$(DEPDIR)/doveadm-director.Po + -rm -f ./$(DEPDIR)/doveadm-dsync.Po + -rm -f ./$(DEPDIR)/doveadm-dump-dbox.Po + -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-file.Po + -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-key.Po + -rm -f ./$(DEPDIR)/doveadm-dump-index.Po + -rm -f ./$(DEPDIR)/doveadm-dump-log.Po + -rm -f ./$(DEPDIR)/doveadm-dump-mailboxlog.Po + -rm -f ./$(DEPDIR)/doveadm-dump-thread.Po + -rm -f ./$(DEPDIR)/doveadm-dump.Po + -rm -f ./$(DEPDIR)/doveadm-fs.Po + -rm -f ./$(DEPDIR)/doveadm-instance.Po + -rm -f ./$(DEPDIR)/doveadm-kick.Po + -rm -f ./$(DEPDIR)/doveadm-log.Po + -rm -f ./$(DEPDIR)/doveadm-mail-altmove.Po + -rm -f ./$(DEPDIR)/doveadm-mail-batch.Po + -rm -f ./$(DEPDIR)/doveadm-mail-copymove.Po + -rm -f ./$(DEPDIR)/doveadm-mail-deduplicate.Po + -rm -f ./$(DEPDIR)/doveadm-mail-expunge.Po + -rm -f ./$(DEPDIR)/doveadm-mail-fetch.Po + -rm -f ./$(DEPDIR)/doveadm-mail-flags.Po + -rm -f ./$(DEPDIR)/doveadm-mail-import.Po + -rm -f ./$(DEPDIR)/doveadm-mail-index.Po + -rm -f ./$(DEPDIR)/doveadm-mail-iter.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-cache.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-status.Po + -rm -f ./$(DEPDIR)/doveadm-mail-mailbox.Po + -rm -f ./$(DEPDIR)/doveadm-mail-rebuild.Po + -rm -f ./$(DEPDIR)/doveadm-mail-save.Po + -rm -f ./$(DEPDIR)/doveadm-mail-search.Po + -rm -f ./$(DEPDIR)/doveadm-mail-server.Po + -rm -f ./$(DEPDIR)/doveadm-mail.Po + -rm -f ./$(DEPDIR)/doveadm-mailbox-list-iter.Po + -rm -f ./$(DEPDIR)/doveadm-master.Po + -rm -f ./$(DEPDIR)/doveadm-mutf7.Po + -rm -f ./$(DEPDIR)/doveadm-oldstats.Po + -rm -f ./$(DEPDIR)/doveadm-penalty.Po + -rm -f ./$(DEPDIR)/doveadm-print-flow.Po + -rm -f ./$(DEPDIR)/doveadm-print-formatted.Po + -rm -f ./$(DEPDIR)/doveadm-print-json.Po + -rm -f ./$(DEPDIR)/doveadm-print-pager.Po + -rm -f ./$(DEPDIR)/doveadm-print-server.Po + -rm -f ./$(DEPDIR)/doveadm-print-tab.Po + -rm -f ./$(DEPDIR)/doveadm-print-table.Po + -rm -f ./$(DEPDIR)/doveadm-print.Po + -rm -f ./$(DEPDIR)/doveadm-proxy.Po + -rm -f ./$(DEPDIR)/doveadm-pw.Po + -rm -f ./$(DEPDIR)/doveadm-replicator.Po + -rm -f ./$(DEPDIR)/doveadm-settings.Po + -rm -f ./$(DEPDIR)/doveadm-sis.Po + -rm -f ./$(DEPDIR)/doveadm-stats.Po + -rm -f ./$(DEPDIR)/doveadm-util.Po + -rm -f ./$(DEPDIR)/doveadm-who.Po + -rm -f ./$(DEPDIR)/doveadm-zlib.Po + -rm -f ./$(DEPDIR)/doveadm.Po + -rm -f ./$(DEPDIR)/main.Po + -rm -f ./$(DEPDIR)/server-connection.Po + -rm -f ./$(DEPDIR)/test-doveadm-util.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-recursive + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-recursive + +pdf-am: + +ps: ps-recursive + +ps-am: + +uninstall-am: uninstall-binPROGRAMS uninstall-pkginc_libHEADERS \ + uninstall-pkglibexecPROGRAMS + +.MAKE: $(am__recursive_targets) check-am install-am install-strip + +.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am \ + am--depfiles check check-am check-local clean \ + clean-binPROGRAMS clean-generic clean-libtool \ + clean-noinstPROGRAMS clean-pkglibexecPROGRAMS cscopelist-am \ + ctags ctags-am distclean distclean-compile distclean-generic \ + distclean-libtool distclean-tags distdir dvi dvi-am html \ + html-am info info-am install install-am install-binPROGRAMS \ + install-data install-data-am install-dvi install-dvi-am \ + install-exec install-exec-am install-exec-local install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-pkginc_libHEADERS \ + install-pkglibexecPROGRAMS install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + installdirs-am maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-binPROGRAMS uninstall-pkginc_libHEADERS \ + uninstall-pkglibexecPROGRAMS + +.PRECIOUS: Makefile + + +install-exec-local: + rm -f $(DESTDIR)$(bindir)/dsync + $(LN_S) doveadm $(DESTDIR)$(bindir)/dsync + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c new file mode 100644 index 0000000..c59d9dd --- /dev/null +++ b/src/doveadm/client-connection-http.c @@ -0,0 +1,1227 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "compat.h" +#include "lib-signals.h" +#include "base64.h" +#include "ioloop.h" +#include "str.h" +#include "str-sanitize.h" +#include "istream.h" +#include "ostream.h" +#include "strescape.h" +#include "settings-parser.h" +#include "iostream-ssl.h" +#include "iostream-temp.h" +#include "istream-seekable.h" +#include "master-service.h" +#include "master-service-ssl.h" +#include "master-service-settings.h" +#include "mail-storage-service.h" +#include "http-server.h" +#include "http-request.h" +#include "http-response.h" +#include "http-url.h" +#include "doveadm-util.h" +#include "doveadm-server.h" +#include "doveadm-mail.h" +#include "doveadm-print.h" +#include "doveadm-settings.h" +#include "client-connection-private.h" +#include "json-parser.h" + +#include <unistd.h> +#include <ctype.h> + +enum client_request_parse_state { + CLIENT_REQUEST_PARSE_INIT, + CLIENT_REQUEST_PARSE_CMD, + CLIENT_REQUEST_PARSE_CMD_NAME, + CLIENT_REQUEST_PARSE_CMD_PARAMS, + CLIENT_REQUEST_PARSE_CMD_PARAM_KEY, + CLIENT_REQUEST_PARSE_CMD_PARAM_VALUE, + CLIENT_REQUEST_PARSE_CMD_PARAM_ARRAY, + CLIENT_REQUEST_PARSE_CMD_PARAM_ISTREAM, + CLIENT_REQUEST_PARSE_CMD_ID, + CLIENT_REQUEST_PARSE_CMD_DONE, + CLIENT_REQUEST_PARSE_DONE +}; + +struct client_request_http { + pool_t pool; + struct client_connection_http *conn; + + struct http_server_request *http_request; + + struct io *io; + struct istream *input; + struct ostream *output; + + struct json_parser *json_parser; + + const struct doveadm_cmd_ver2 *cmd; + struct doveadm_cmd_param *cmd_param; + struct ioloop *ioloop; + ARRAY_TYPE(doveadm_cmd_param_arr_t) pargv; + int method_err; + char *method_id; + bool first_row; + bool value_is_array; + + enum client_request_parse_state parse_state; +}; + +struct client_connection_http { + struct client_connection conn; + + struct http_server_connection *http_conn; + + struct client_request_http *request; +}; + +typedef void doveadm_server_handler_t(struct client_request_http *req); + +struct doveadm_http_server_mount { + const char *verb; + const char *path; + doveadm_server_handler_t *handler; + bool auth; +}; + +static struct http_server *doveadm_http_server; + +static void doveadm_http_server_send_response(struct client_request_http *req); + +/* + * API + */ + +static void doveadm_http_server_options_handler(struct client_request_http *); +static void doveadm_http_server_print_mounts(struct client_request_http *); +static void doveadm_http_server_send_api_v1(struct client_request_http *); +static void doveadm_http_server_read_request_v1(struct client_request_http *); + +static struct doveadm_http_server_mount doveadm_http_server_mounts[] = { +{ + .verb = "OPTIONS", + .path = NULL, + .handler = doveadm_http_server_options_handler, + .auth = FALSE +},{ + .verb = "GET", + .path = "/", + .handler = doveadm_http_server_print_mounts, + .auth = TRUE +},{ + .verb = "GET", + .path = "/doveadm/v1", + .handler = doveadm_http_server_send_api_v1, + .auth = TRUE +},{ + .verb = "POST", + .path = "/doveadm/v1", + .handler = doveadm_http_server_read_request_v1, + .auth = TRUE +} +}; + +static void doveadm_http_server_json_error(void *context, const char *error) +{ + struct client_request_http *req = context; + struct ostream *output = req->output; + string_t *escaped; + + escaped = str_new(req->pool, 10); + + o_stream_nsend_str(output, "[\"error\",{\"type\":\""); + json_append_escaped(escaped, error); + o_stream_nsend_str(output, str_c(escaped)); + o_stream_nsend_str(output, "\", \"exitCode\":"); + str_truncate(escaped,0); + str_printfa(escaped, "%d", doveadm_exit_code); + o_stream_nsend_str(output, str_c(escaped)); + o_stream_nsend_str(output, "},\""); + str_truncate(escaped,0); + if (req->method_id != NULL) { + json_append_escaped(escaped, req->method_id); + o_stream_nsend_str(output, str_c(escaped)); + } + o_stream_nsend_str(output, "\"]"); +} + +static void doveadm_http_server_json_success(void *context, struct istream *result) +{ + struct client_request_http *req = context; + struct ostream *output = req->output; + string_t *escaped; + + escaped = str_new(req->pool, 10); + + o_stream_nsend_str(output, "[\"doveadmResponse\","); + o_stream_nsend_istream(output, result); + o_stream_nsend_str(output, ",\""); + if (req->method_id != NULL) { + json_append_escaped(escaped, req->method_id); + o_stream_nsend_str(output, str_c(escaped)); + } + o_stream_nsend_str(output, "\"]"); +} + +static void +doveadm_http_server_command_execute(struct client_request_http *req) +{ + struct client_connection_http *conn = req->conn; + struct doveadm_cmd_context cctx; + struct istream *is; + const char *user; + struct ioloop *ioloop, *prev_ioloop; + + /* final preflight check */ + if (req->method_err == 0 && + !doveadm_client_is_allowed_command(conn->conn.set, + req->cmd->name)) + req->method_err = 403; + if (req->method_err != 0) { + if (req->method_err == 404) { + doveadm_http_server_json_error(req, "unknownMethod"); + } else if (req->method_err == 403) { + doveadm_http_server_json_error(req, "unAuthorized"); + } else if (req->method_err == 400) { + doveadm_http_server_json_error(req, "invalidRequest"); + } else { + doveadm_http_server_json_error(req, "internalError"); + } + return; + } + + prev_ioloop = current_ioloop; + i_zero(&cctx); + cctx.conn_type = conn->conn.type; + cctx.input = req->input; + cctx.output = req->output; + + // create iostream + doveadm_print_ostream = iostream_temp_create("/tmp/doveadm.", 0); + cctx.cmd = req->cmd; + + if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0) + doveadm_print_init(DOVEADM_PRINT_TYPE_JSON); + + /* then call it */ + doveadm_cmd_params_null_terminate_arrays(&req->pargv); + cctx.argv = array_get(&req->pargv, (unsigned int*)&cctx.argc); + ioloop = io_loop_create(); + doveadm_exit_code = 0; + + cctx.local_ip = conn->conn.local_ip; + cctx.local_port = conn->conn.local_port; + cctx.remote_ip = conn->conn.remote_ip; + cctx.remote_port = conn->conn.remote_port; + + if (doveadm_cmd_param_str(&cctx, "user", &user)) + i_info("Executing command '%s' as '%s'", cctx.cmd->name, user); + else + i_info("Executing command '%s'", cctx.cmd->name); + client_connection_set_proctitle(&conn->conn, cctx.cmd->name); + cctx.cmd->cmd(&cctx); + client_connection_set_proctitle(&conn->conn, ""); + + o_stream_switch_ioloop_to(req->output, prev_ioloop); + io_loop_destroy(&ioloop); + + if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0) + doveadm_print_deinit(); + if (o_stream_finish(doveadm_print_ostream) < 0) { + i_info("Error writing output in command %s: %s", + req->cmd->name, + o_stream_get_error(req->output)); + doveadm_exit_code = EX_TEMPFAIL; + } + + is = iostream_temp_finish(&doveadm_print_ostream, 4096); + + if (req->first_row == TRUE) + req->first_row = FALSE; + else + o_stream_nsend_str(req->output,","); + + if (doveadm_exit_code != 0) { + if (doveadm_exit_code == 0 || doveadm_exit_code == EX_TEMPFAIL) + i_error("Command %s failed", req->cmd->name); + doveadm_http_server_json_error(req, "exitCode"); + } else { + doveadm_http_server_json_success(req, is); + } + i_stream_unref(&is); +} + +static int +request_json_parse_init(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type != JSON_TYPE_ARRAY) { + /* request must be a JSON array */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Request must be a JSON array"); + return -1; + } + req->first_row = TRUE; + o_stream_nsend_str(req->output,"["); + + /* next: parse the next command */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD; + return 1; +} + +static int +request_json_parse_cmd(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type == JSON_TYPE_ARRAY_END) { + /* end of command list */ + req->parse_state = CLIENT_REQUEST_PARSE_DONE; + return 1; + } + if (type != JSON_TYPE_ARRAY) { + /* command must be an array */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Command must be a JSON array"); + return -1; + } + req->method_err = 0; + p_free_and_null(req->pool, req->method_id); + req->cmd = NULL; + doveadm_cmd_params_clean(&req->pargv); + + /* next: parse the command name */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_NAME; + return 1; +} + +static int +request_json_parse_cmd_name(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + const struct doveadm_cmd_ver2 *ccmd; + struct doveadm_cmd_param *param; + bool found; + int pargc, ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type != JSON_TYPE_STRING) { + /* command name must be a string */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Command name must be a string"); + return -1; + } + + /* see if we can find it */ + found = FALSE; + array_foreach(&doveadm_cmds_ver2, ccmd) { + if (i_strccdascmp(ccmd->name, value) == 0) { + req->cmd = ccmd; + found = TRUE; + break; + } + } + if (!found) { + /* command not found; skip to the command ID */ + json_parse_skip_next(req->json_parser); + req->method_err = 404; + req->parse_state = CLIENT_REQUEST_PARSE_CMD_ID; + return 1; + } + + /* initialize pargv */ + for (pargc = 0; req->cmd->parameters[pargc].name != NULL; pargc++) { + param = array_append_space(&req->pargv); + *param = req->cmd->parameters[pargc]; + param->value_set = FALSE; + } + + /* next: parse the command parameters */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAMS; + return 1; +} + +static int +request_json_parse_cmd_params(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type == JSON_TYPE_OBJECT_END) { + /* empty command parameters object; parse command ID next */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_ID; + return 1; + } + if (type != JSON_TYPE_OBJECT) { + /* parameters must be contained in an object */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Parameters must be contained in a JSON object"); + return -1; + } + + /* next: parse parameter key */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY; + return 1; +} + +static int +request_json_parse_cmd_param_key(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + struct doveadm_cmd_param *par; + bool found; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type == JSON_TYPE_OBJECT_END) { + /* end of parameters; parse command ID next */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_ID; + return 1; + } + i_assert(type == JSON_TYPE_OBJECT_KEY); + /* find the parameter */ + found = FALSE; + array_foreach_modifiable(&req->pargv, par) { + if (i_strccdascmp(par->name, value) == 0) { + req->cmd_param = par; + found = TRUE; + break; + } + } + if (found && req->cmd_param->value_set) { + /* it's already set, cannot have same key twice in json */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Parameter `%s' is duplicated", + req->cmd_param->name); + return -1; + } + /* skip remaining parameters if error has already occurred */ + if (!found || req->method_err != 0) { + json_parse_skip_next(req->json_parser); + req->method_err = 400; + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY; + return 1; + } + + /* next: parse parameter value */ + req->value_is_array = FALSE; + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_VALUE; + return 1; +} + +static int +request_json_parse_param_value(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if (req->cmd_param->type == CMD_PARAM_ISTREAM) { + struct istream* is[2] = {0}; + + /* read the value as a stream */ + ret = json_parse_next_stream(req->json_parser, &is[0]); + if (ret <= 0) + return ret; + + req->cmd_param->value.v_istream = + i_stream_create_seekable_path(is, + IO_BLOCK_SIZE, "/tmp/doveadm."); + i_stream_unref(&is[0]); + req->cmd_param->value_set = TRUE; + + /* read the seekable stream to its end so that the underlying + json istream is read to its conclusion. */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_ISTREAM; + return 1; + } + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (req->cmd_param->type == CMD_PARAM_ARRAY) { + const char *tmp; + + /* expects either a singular value or an array of values */ + p_array_init(&req->cmd_param->value.v_array, req->pool, 1); + req->cmd_param->value_set = TRUE; + if (type == JSON_TYPE_ARRAY) { + /* start of array */ + req->value_is_array = TRUE; + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_ARRAY; + return 1; + } + /* singular value */ + if (type != JSON_TYPE_STRING) { + /* FIXME: should handle other than string too */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Parameter `%s' must be string or array", + req->cmd_param->name); + return -1; + } + tmp = p_strdup(req->pool, value); + array_push_back(&req->cmd_param->value.v_array, &tmp); + + /* next: continue with the next parameter */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY; + return 1; + } + + /* expects just a value */ + req->cmd_param->value_set = TRUE; + switch(req->cmd_param->type) { + case CMD_PARAM_BOOL: + req->cmd_param->value.v_bool = (strcmp(value, "true") == 0); + break; + case CMD_PARAM_INT64: + if (str_to_int64(value, &req->cmd_param->value.v_int64) != 0) { + req->method_err = 400; + } + break; + case CMD_PARAM_IP: + if (net_addr2ip(value, &req->cmd_param->value.v_ip) != 0) { + req->method_err = 400; + } + break; + case CMD_PARAM_STR: + req->cmd_param->value.v_string = p_strdup(req->pool, value); + break; + default: + break; + } + + /* next: continue with the next parameter */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY; + return 1; +} + +static int +request_json_parse_param_array(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type == JSON_TYPE_ARRAY_END) { + /* end of array: continue with next parameter */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY; + return 1; + } + if (type != JSON_TYPE_STRING) { + /* array items must be string */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Command parameter array can only contain" + "string values"); + return -1; + } + + /* record entry */ + value = p_strdup(req->pool, value); + array_push_back(&req->cmd_param->value.v_array, &value); + + /* next: continue with the next array item */ + return 1; +} + +static int +request_json_parse_param_istream(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + struct istream *v_input = req->cmd_param->value.v_istream; + const unsigned char *data; + size_t size; + + while (i_stream_read_more(v_input, &data, &size) > 0) + i_stream_skip(v_input, size); + if (!v_input->eof) { + /* more to read */ + return 0; + } + + if (v_input->stream_errno != 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(v_input), + i_stream_get_error(v_input)); + req->method_err = 400; + if (req->input->stream_errno == 0) { + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Failed to read command parameter data"); + } + return -1; + } + + /* next: continue with the next parameter */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY; + return 1; +} + +static int +request_json_parse_cmd_id(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type != JSON_TYPE_STRING) { + /* command ID must be a string */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Command ID must be a string"); + return -1; + } + + /* next: parse end of command */ + req->method_id = p_strdup(req->pool, value); + req->parse_state = CLIENT_REQUEST_PARSE_CMD_DONE; + return 1; +} + +static int +request_json_parse_cmd_done(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + if (type != JSON_TYPE_ARRAY_END) { + /* command array must end here */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Unexpected JSON element at end of command"); + return -1; + } + + /* execute command */ + doveadm_http_server_command_execute(req); + + /* next: parse next command */ + req->parse_state = CLIENT_REQUEST_PARSE_CMD; + return 1; +} + +static int +request_json_parse_done(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + enum json_type type; + const char *value; + int ret; + + if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0) + return ret; + /* only gets here when there is spurious additional JSON */ + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "Unexpected JSON element in input"); + return -1; +} + +static int +doveadm_http_server_json_parse_v1(struct client_request_http *req) +{ + /* parser state machine */ + switch (req->parse_state) { + /* command list: '[' */ + case CLIENT_REQUEST_PARSE_INIT: + return request_json_parse_init(req); + /* command begin: '[' */ + case CLIENT_REQUEST_PARSE_CMD: + return request_json_parse_cmd(req); + /* command name: string */ + case CLIENT_REQUEST_PARSE_CMD_NAME: + return request_json_parse_cmd_name(req); + /* command parameters: '{' */ + case CLIENT_REQUEST_PARSE_CMD_PARAMS: + return request_json_parse_cmd_params(req); + /* parameter key: string */ + case CLIENT_REQUEST_PARSE_CMD_PARAM_KEY: + return request_json_parse_cmd_param_key(req); + /* parameter value */ + case CLIENT_REQUEST_PARSE_CMD_PARAM_VALUE: + return request_json_parse_param_value(req); + /* parameter array value */ + case CLIENT_REQUEST_PARSE_CMD_PARAM_ARRAY: + return request_json_parse_param_array(req); + /* parameter istream value */ + case CLIENT_REQUEST_PARSE_CMD_PARAM_ISTREAM: + return request_json_parse_param_istream(req); + /* command ID: string */ + case CLIENT_REQUEST_PARSE_CMD_ID: + return request_json_parse_cmd_id(req); + /* command end: ']' */ + case CLIENT_REQUEST_PARSE_CMD_DONE: + return request_json_parse_cmd_done(req); + /* finished parsing request (seen final ']') */ + case CLIENT_REQUEST_PARSE_DONE: + return request_json_parse_done(req); + default: + break; + } + i_unreached(); +} + +static void +doveadm_http_server_read_request_v1(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + const char *error; + int ret; + + if (req->json_parser == NULL) { + req->json_parser = json_parser_init_flags( + req->input, JSON_PARSER_NO_ROOT_OBJECT); + } + + while ((ret=doveadm_http_server_json_parse_v1(req)) > 0); + + if (http_server_request_get_response(http_sreq) != NULL) { + /* already responded */ + io_remove(&req->io); + i_stream_destroy(&req->input); + return; + } + if (!req->input->eof && ret == 0) + return; + io_remove(&req->io); + + doveadm_cmd_params_clean(&req->pargv); + + if (req->input->stream_errno != 0) { + http_server_request_fail_close(http_sreq, + 400, "Client disconnected"); + i_info("read(%s) failed: %s", + i_stream_get_name(req->input), + i_stream_get_error(req->input)); + return; + } + + if (json_parser_deinit(&req->json_parser, &error) != 0) { + http_server_request_fail_text(http_sreq, + 400, "Bad Request", + "JSON parse error: %s", error); + return; + } + + i_stream_destroy(&req->input); + o_stream_nsend_str(req->output,"]"); + + doveadm_http_server_send_response(req); +} + +static void doveadm_http_server_camelcase_value(string_t *value) +{ + size_t i, k; + char *ptr = str_c_modifiable(value); + + for (i = 0, k = 0; i < strlen(ptr);) { + if (ptr[i] == ' ' || ptr[i] == '-') { + i++; + ptr[k++] = i_toupper(ptr[i++]); + } else { + ptr[k++] = ptr[i++]; + } + } + str_truncate(value, k); +} + +static void +doveadm_http_server_send_api_v1(struct client_request_http *req) +{ + struct ostream *output = req->output; + const struct doveadm_cmd_ver2 *cmd; + const struct doveadm_cmd_param *par; + unsigned int i, k; + string_t *tmp; + bool sent; + + tmp = str_new(req->pool, 8); + + o_stream_nsend_str(output,"[\n"); + for (i = 0; i < array_count(&doveadm_cmds_ver2); i++) { + cmd = array_idx(&doveadm_cmds_ver2, i); + if (i > 0) + o_stream_nsend_str(output, ",\n"); + o_stream_nsend_str(output, "\t{\"command\":\""); + json_append_escaped(tmp, cmd->name); + doveadm_http_server_camelcase_value(tmp); + o_stream_nsend_str(output, str_c(tmp)); + o_stream_nsend_str(output, "\", \"parameters\":["); + + sent = FALSE; + for (k = 0; cmd->parameters[k].name != NULL; k++) { + str_truncate(tmp, 0); + par = &(cmd->parameters[k]); + if ((par->flags & CMD_PARAM_FLAG_DO_NOT_EXPOSE) != 0) + continue; + if (sent) + o_stream_nsend_str(output, ",\n"); + else + o_stream_nsend_str(output, "\n"); + sent = TRUE; + o_stream_nsend_str(output, "\t\t{\"name\":\""); + json_append_escaped(tmp, par->name); + doveadm_http_server_camelcase_value(tmp); + o_stream_nsend_str(output, str_c(tmp)); + o_stream_nsend_str(output, "\",\"type\":\""); + switch(par->type) { + case CMD_PARAM_BOOL: + o_stream_nsend_str(output, "boolean"); + break; + case CMD_PARAM_INT64: + o_stream_nsend_str(output, "integer"); + break; + case CMD_PARAM_ARRAY: + o_stream_nsend_str(output, "array"); + break; + case CMD_PARAM_IP: + case CMD_PARAM_ISTREAM: + case CMD_PARAM_STR: + o_stream_nsend_str(output, "string"); + } + o_stream_nsend_str(output, "\"}"); + } + if (k > 0) + o_stream_nsend_str(output,"\n\t"); + o_stream_nsend_str(output,"]}"); + str_truncate(tmp, 0); + } + o_stream_nsend_str(output,"\n]"); + doveadm_http_server_send_response(req); +} + +static void +doveadm_http_server_options_handler(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + struct http_server_response *http_resp; + + http_resp = http_server_response_create(http_sreq, 200, "OK"); + http_server_response_add_header(http_resp, + "Access-Control-Allow-Origin", "*"); + http_server_response_add_header(http_resp, + "Access-Control-Allow-Methods", "GET, POST, OPTIONS"); + http_server_response_add_header(http_resp, + "Access-Control-Allow-Request-Headers", + "Content-Type, X-API-Key, Authorization"); + http_server_response_add_header(http_resp, + "Access-Control-Allow-Headers", + "Content-Type, WWW-Authenticate"); + http_server_response_submit(http_resp); +} + +static void +doveadm_http_server_print_mounts(struct client_request_http *req) +{ + struct ostream *output = req->output; + unsigned int i; + + o_stream_nsend_str(output, "[\n"); + for (i = 0; i < N_ELEMENTS(doveadm_http_server_mounts); i++) { + if (i > 0) + o_stream_nsend_str(output, ",\n"); + o_stream_nsend_str(output, "{\"method\":\""); + if (doveadm_http_server_mounts[i].verb == NULL) + o_stream_nsend_str(output, "*"); + else { + o_stream_nsend_str(output, + doveadm_http_server_mounts[i].verb); + } + o_stream_nsend_str(output, "\",\"path\":\""); + if (doveadm_http_server_mounts[i].path == NULL) + o_stream_nsend_str(output, "*"); + else { + o_stream_nsend_str(output, + doveadm_http_server_mounts[i].path); + } + o_stream_nsend_str(output, "\"}"); + } + o_stream_nsend_str(output, "\n]"); + doveadm_http_server_send_response(req); +} + +/* + * Request + */ + +static void doveadm_http_server_send_response(struct client_request_http *req) +{ + struct http_server_request *http_sreq = req->http_request; + struct http_server_response *http_resp; + struct istream *payload = NULL; + + if (req->output != NULL) { + if (o_stream_finish(req->output) == -1) { + i_info("error writing output: %s", + o_stream_get_error(req->output)); + o_stream_destroy(&req->output); + http_server_request_fail(http_sreq, + 500, "Internal server error"); + return; + } + + payload = iostream_temp_finish(&req->output, + IO_BLOCK_SIZE); + } + + http_resp = http_server_response_create(http_sreq, 200, "OK"); + http_server_response_add_header(http_resp, "Content-Type", + "application/json; charset=utf-8"); + + if (payload != NULL) { + http_server_response_set_payload(http_resp, payload); + i_stream_unref(&payload); + } + + http_server_response_submit(http_resp); +} + +static void +doveadm_http_server_request_destroy(struct client_request_http *req) +{ + struct client_connection_http *conn = req->conn; + struct http_server_request *http_sreq = req->http_request; + const struct http_request *http_req = + http_server_request_get(http_sreq); + struct http_server_response *http_resp = + http_server_request_get_response(http_sreq); + + i_assert(conn->request == req); + + if (http_resp != NULL) { + const char *agent, *url, *reason; + uoff_t size; + int status; + + http_server_response_get_status(http_resp, &status, &reason); + size = http_server_response_get_total_size(http_resp); + agent = http_request_header_get(http_req, "User-Agent"); + if (agent == NULL) agent = ""; + + url = http_url_create(http_req->target.url); + i_info("doveadm: %s %s %s \"%s %s " + "HTTP/%d.%d\" %d %"PRIuUOFF_T" \"%s\" \"%s\"", + net_ip2addr(&conn->conn.remote_ip), "-", "-", + http_req->method, http_req->target.url->path, + http_req->version_major, http_req->version_minor, + status, size, url, agent); + } + if (req->json_parser != NULL) { + const char *error ATTR_UNUSED; + (void)json_parser_deinit(&req->json_parser, &error); + // we've already failed, ignore error + } + if (req->output != NULL) + o_stream_set_no_error_handling(req->output, TRUE); + io_remove(&req->io); + o_stream_destroy(&req->output); + i_stream_destroy(&req->input); + + http_server_request_unref(&req->http_request); + http_server_switch_ioloop(doveadm_http_server); + + pool_unref(&req->pool); + conn->request = NULL; +} + +static bool +doveadm_http_server_auth_basic(struct client_request_http *req, + const struct http_auth_credentials *creds) +{ + struct client_connection_http *conn = req->conn; + const struct doveadm_settings *set = conn->conn.set; + string_t *b64_value; + char *value; + + if (*set->doveadm_password == '\0') { + i_error("Invalid authentication attempt to HTTP API: " + "Basic authentication scheme not enabled"); + return FALSE; + } + + b64_value = str_new(conn->conn.pool, 32); + value = p_strdup_printf(conn->conn.pool, + "doveadm:%s", set->doveadm_password); + base64_encode(value, strlen(value), b64_value); + if (creds->data != NULL && strcmp(creds->data, str_c(b64_value)) == 0) + return TRUE; + + i_error("Invalid authentication attempt to HTTP API " + "(using Basic authentication scheme)"); + return FALSE; +} + +static bool +doveadm_http_server_auth_api_key(struct client_request_http *req, + const struct http_auth_credentials *creds) +{ + struct client_connection_http *conn = req->conn; + const struct doveadm_settings *set = doveadm_settings; + string_t *b64_value; + + if (*set->doveadm_api_key == '\0') { + i_error("Invalid authentication attempt to HTTP API: " + "X-Dovecot-API authentication scheme not enabled"); + return FALSE; + } + + b64_value = str_new(conn->conn.pool, 32); + base64_encode(set->doveadm_api_key, + strlen(set->doveadm_api_key), b64_value); + if (creds->data != NULL && strcmp(creds->data, str_c(b64_value)) == 0) + return TRUE; + + i_error("Invalid authentication attempt to HTTP API " + "(using X-Dovecot-API authentication scheme)"); + return FALSE; +} + + +static bool +doveadm_http_server_auth_verify(struct client_request_http *req, + const struct http_auth_credentials *creds) +{ + /* see if the mech is supported */ + if (strcasecmp(creds->scheme, "Basic") == 0) + return doveadm_http_server_auth_basic(req, creds); + if (strcasecmp(creds->scheme, "X-Dovecot-API") == 0) + return doveadm_http_server_auth_api_key(req, creds); + + i_error("Unsupported authentication scheme to HTTP API: %s", + str_sanitize(creds->scheme, 128)); + return FALSE; +} + +static bool +doveadm_http_server_authorize_request(struct client_request_http *req) +{ + struct client_connection_http *conn = req->conn; + struct http_server_request *http_sreq = req->http_request; + bool auth = FALSE; + struct http_auth_credentials creds; + + /* no authentication specified */ + if (doveadm_settings->doveadm_api_key[0] == '\0' && + *conn->conn.set->doveadm_password == '\0') { + http_server_request_fail(http_sreq, + 500, "Internal Server Error"); + i_error("No authentication defined in configuration. " + "Add API key or password"); + return FALSE; + } + if (http_server_request_get_auth(http_sreq, &creds) > 0) + auth = doveadm_http_server_auth_verify(req, &creds); + if (!auth) { + struct http_server_response *http_resp; + + http_resp = http_server_response_create(http_sreq, + 401, "Authentication required"); + if (doveadm_settings->doveadm_api_key[0] != '\0') { + http_server_response_add_header(http_resp, + "WWW-Authenticate", "X-Dovecot-API" + ); + } + if (*conn->conn.set->doveadm_password != '\0') { + http_server_response_add_header(http_resp, + "WWW-Authenticate", "Basic Realm=\"doveadm\"" + ); + } + http_server_response_submit(http_resp); + } + return auth; +} + +static void +doveadm_http_server_handle_request(void *context, + struct http_server_request *http_sreq) +{ + struct client_connection_http *conn = context; + struct client_request_http *req; + const struct http_request *http_req = + http_server_request_get(http_sreq); + struct doveadm_http_server_mount *ep = NULL; + pool_t pool; + unsigned int i; + + /* no pipelining possible due to synchronous handling of requests */ + i_assert(conn->request == NULL); + + pool = pool_alloconly_create("doveadm request", 1024*16); + req = p_new(pool, struct client_request_http, 1); + req->pool = pool; + req->conn = conn; + + req->http_request = http_sreq; + http_server_request_ref(req->http_request); + + http_server_request_connection_close(http_sreq, TRUE); + http_server_request_set_destroy_callback(http_sreq, + doveadm_http_server_request_destroy, req); + + conn->request = req; + + for (i = 0; i < N_ELEMENTS(doveadm_http_server_mounts); i++) { + if (doveadm_http_server_mounts[i].verb == NULL || + strcmp(http_req->method, + doveadm_http_server_mounts[i].verb) == 0) { + if (doveadm_http_server_mounts[i].path == NULL || + strcmp(http_req->target.url->path, + doveadm_http_server_mounts[i].path) == 0) { + ep = &doveadm_http_server_mounts[i]; + break; + } + } + } + + if (ep == NULL) { + http_server_request_fail(http_sreq, + 404, "Path Not Found"); + return; + } + + if (ep->auth == TRUE && !doveadm_http_server_authorize_request(req)) + return; + + if (strcmp(http_req->method, "POST") == 0) { + /* handle request */ + req->input = http_req->payload; + i_stream_set_name(req->input, + net_ip2addr(&conn->conn.remote_ip)); + i_stream_ref(req->input); + req->io = io_add_istream(req->input, *ep->handler, req); + req->output = iostream_temp_create_named( + "/tmp/doveadm.", 0, net_ip2addr(&conn->conn.remote_ip)); + p_array_init(&req->pargv, req->pool, 5); + ep->handler(req); + } else { + req->output = iostream_temp_create_named( + "/tmp/doveadm.", 0, net_ip2addr(&conn->conn.remote_ip)); + ep->handler(req); + } +} + +/* + * Connection + */ + +static void doveadm_http_server_connection_destroy(void *context, + const char *reason); + +static const struct http_server_callbacks doveadm_http_callbacks = { + .connection_destroy = doveadm_http_server_connection_destroy, + .handle_request = doveadm_http_server_handle_request +}; + +static void +client_connection_http_free(struct client_connection *_conn) +{ + struct client_connection_http *conn = + (struct client_connection_http *)_conn; + + if (conn->http_conn != NULL) { + /* We're not in the lib-http/server's connection destroy + callback. */ + http_server_connection_close(&conn->http_conn, + "Server shutting down"); + } +} + +struct client_connection * +client_connection_http_create(int fd, bool ssl) +{ + struct client_connection_http *conn; + pool_t pool; + + pool = pool_alloconly_create("doveadm client", 1024); + conn = p_new(pool, struct client_connection_http, 1); + + if (client_connection_init(&conn->conn, + DOVEADM_CONNECTION_TYPE_HTTP, pool, fd) < 0) { + pool_unref(&conn->conn.pool); + return NULL; + } + conn->conn.free = client_connection_http_free; + + conn->http_conn = http_server_connection_create(doveadm_http_server, + fd, fd, ssl, &doveadm_http_callbacks, conn); + return &conn->conn; +} + +static void +doveadm_http_server_connection_destroy(void *context, + const char *reason ATTR_UNUSED) +{ + struct client_connection_http *conn = + (struct client_connection_http *)context; + struct client_connection *bconn = &conn->conn; + + if (conn->http_conn == NULL) { + /* already destroying client directly */ + return; + } + + /* HTTP connection is destroyed already now */ + conn->http_conn = NULL; + + /* destroy the connection itself */ + client_connection_destroy(&bconn); +} + +/* + * Server + */ + +void doveadm_http_server_init(void) +{ + struct http_server_settings http_set = { + .rawlog_dir = doveadm_settings->doveadm_http_rawlog_dir, + }; + + doveadm_http_server = http_server_init(&http_set); +} + +void doveadm_http_server_deinit(void) +{ + http_server_deinit(&doveadm_http_server); +} diff --git a/src/doveadm/client-connection-private.h b/src/doveadm/client-connection-private.h new file mode 100644 index 0000000..8205531 --- /dev/null +++ b/src/doveadm/client-connection-private.h @@ -0,0 +1,22 @@ +#ifndef CLIENT_CONNECTION_PRIVATE_H +#define CLIENT_CONNECTION_PRIVATE_H + +#include "client-connection.h" + +bool doveadm_client_is_allowed_command(const struct doveadm_settings *set, + const char *cmd_name); + +int client_connection_init(struct client_connection *conn, + enum doveadm_client_type type, pool_t pool, int fd); +void client_connection_destroy(struct client_connection **_conn); + +void client_connection_set_proctitle(struct client_connection *conn, + const char *text); + +void doveadm_http_server_init(void); +void doveadm_http_server_deinit(void); + +void doveadm_server_init(void); +void doveadm_server_deinit(void); + +#endif diff --git a/src/doveadm/client-connection-tcp.c b/src/doveadm/client-connection-tcp.c new file mode 100644 index 0000000..b80d674 --- /dev/null +++ b/src/doveadm/client-connection-tcp.c @@ -0,0 +1,558 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "lib-signals.h" +#include "str.h" +#include "base64.h" +#include "ioloop.h" +#include "istream.h" +#include "ostream.h" +#include "strescape.h" +#include "iostream-ssl.h" +#include "ostream-multiplex.h" +#include "master-service.h" +#include "master-service-ssl.h" +#include "mail-storage-service.h" +#include "doveadm-util.h" +#include "doveadm-mail.h" +#include "doveadm-print.h" +#include "doveadm-server.h" +#include "client-connection-private.h" + +#include <unistd.h> + +#define MAX_INBUF_SIZE (1024*1024) + +struct client_connection_tcp { + struct client_connection conn; + + int fd; + struct io *io; + struct istream *input; + struct ostream *output; + struct ostream *log_out; + struct ssl_iostream *ssl_iostream; + struct ioloop *ioloop; + + bool handshaked:1; + bool preauthenticated:1; + bool authenticated:1; + bool io_setup:1; + bool use_multiplex:1; +}; + +static void +client_connection_tcp_input(struct client_connection_tcp *conn); +static void +client_connection_tcp_send_auth_handshake(struct client_connection_tcp *conn); +static void +client_connection_tcp_destroy(struct client_connection_tcp **_conn); +static int +client_connection_tcp_init_ssl(struct client_connection_tcp *conn); + +static failure_callback_t *orig_error_callback, *orig_fatal_callback; +static failure_callback_t *orig_info_callback, *orig_debug_callback = NULL; + +static bool log_recursing = FALSE; + +static void ATTR_FORMAT(2, 0) +doveadm_server_log_handler(const struct failure_context *ctx, + const char *format, va_list args) +{ + struct client_connection_tcp *conn = NULL; + + if (doveadm_client != NULL && + doveadm_client->type == DOVEADM_CONNECTION_TYPE_TCP) + conn = (struct client_connection_tcp *)doveadm_client; + + if (!log_recursing && conn != NULL && + conn->log_out != NULL) T_BEGIN { + struct ioloop *prev_ioloop = current_ioloop; + struct ostream *log_out = conn->log_out; + char c; + const char *ptr; + bool corked; + va_list va; + + /* prevent re-entering this code if + any of the following code causes logging */ + log_recursing = TRUE; + + /* since we can get here from just about anywhere, make sure + the log ostream uses the connection's ioloop. */ + if (conn->ioloop != NULL) + io_loop_set_current(conn->ioloop); + + const char *log_prefix = + ctx->log_prefix != NULL ? ctx->log_prefix : + i_get_failure_prefix(); + size_t log_prefix_len = strlen(log_prefix); + c = doveadm_log_type_to_char(ctx->type); + corked = o_stream_is_corked(log_out); + + va_copy(va, args); + const char *str = t_strdup_vprintf(format, va); + va_end(va); + + if (!corked) + o_stream_cork(log_out); + for (;;) { + ptr = strchr(str, '\n'); + size_t len = ptr == NULL ? strlen(str) : + (size_t)(ptr - str); + + o_stream_nsend(log_out, &c, 1); + o_stream_nsend(log_out, log_prefix, log_prefix_len); + o_stream_nsend(log_out, str, len); + o_stream_nsend(log_out, "\n", 1); + + if (ptr == NULL) + break; + str = ptr+1; + } + o_stream_uncork(log_out); + if (corked) + o_stream_cork(log_out); + io_loop_set_current(prev_ioloop); + + log_recursing = FALSE; + } T_END; + + switch(ctx->type) { + case LOG_TYPE_DEBUG: + orig_debug_callback(ctx, format, args); + break; + case LOG_TYPE_INFO: + orig_info_callback(ctx, format, args); + break; + case LOG_TYPE_WARNING: + case LOG_TYPE_ERROR: + orig_error_callback(ctx, format, args); + break; + default: + i_unreached(); + } +} + +static void doveadm_server_capture_logs(void) +{ + i_assert(orig_debug_callback == NULL); + i_get_failure_handlers(&orig_fatal_callback, &orig_error_callback, + &orig_info_callback, &orig_debug_callback); + i_set_error_handler(doveadm_server_log_handler); + i_set_info_handler(doveadm_server_log_handler); + i_set_debug_handler(doveadm_server_log_handler); +} + +static void doveadm_server_restore_logs(void) +{ + i_assert(orig_debug_callback != NULL); + i_set_error_handler(orig_error_callback); + i_set_info_handler(orig_info_callback); + i_set_debug_handler(orig_debug_callback); + orig_fatal_callback = NULL; + orig_error_callback = NULL; + orig_info_callback = NULL; + orig_debug_callback = NULL; +} + +static void +doveadm_cmd_server_post(struct client_connection_tcp *conn, const char *cmd_name) +{ + const char *str = NULL; + + if (doveadm_exit_code == 0) { + o_stream_nsend(conn->output, "\n+\n", 3); + return; + } + + str = doveadm_exit_code_to_str(doveadm_exit_code); + + if (str != NULL) { + o_stream_nsend_str(conn->output, + t_strdup_printf("\n-%s\n", str)); + } else { + o_stream_nsend_str(conn->output, "\n-\n"); + i_error("BUG: Command '%s' returned unknown error code %d", + cmd_name, doveadm_exit_code); + } +} + +static void +doveadm_cmd_server_run_ver2(struct client_connection_tcp *conn, + int argc, const char *const argv[], + struct doveadm_cmd_context *cctx) +{ + i_getopt_reset(); + if (doveadm_cmd_run_ver2(argc, argv, cctx) < 0) + doveadm_exit_code = EX_USAGE; + doveadm_cmd_server_post(conn, cctx->cmd->name); +} + +static int doveadm_cmd_handle(struct client_connection_tcp *conn, + const char *cmd_name, + int argc, const char *const argv[], + struct doveadm_cmd_context *cctx) +{ + struct ioloop *prev_ioloop = current_ioloop; + const struct doveadm_cmd_ver2 *cmd_ver2; + + if ((cmd_ver2 = doveadm_cmd_find_with_args_ver2(cmd_name, &argc, &argv)) == NULL) { + i_error("doveadm: Client sent unknown command: %s", cmd_name); + return -1; + } + cctx->cmd = cmd_ver2; + + /* some commands will want to call io_loop_run(), but we're already + running one and we can't call the original one recursively, so + create a new ioloop. */ + conn->ioloop = io_loop_create(); + o_stream_switch_ioloop(conn->output); + if (conn->log_out != NULL) + o_stream_switch_ioloop(conn->log_out); + + doveadm_cmd_server_run_ver2(conn, argc, argv, cctx); + + o_stream_switch_ioloop_to(conn->output, prev_ioloop); + if (conn->log_out != NULL) + o_stream_switch_ioloop_to(conn->log_out, prev_ioloop); + io_loop_destroy(&conn->ioloop); + + /* clear all headers */ + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); + + /* We already sent the success/failure reply to the client. Return 0 + so caller never adds another failure reply. */ + return 0; +} + +static bool client_handle_command(struct client_connection_tcp *conn, + const char *const *args) +{ + struct doveadm_cmd_context cctx; + const char *flags, *cmd_name; + unsigned int argc = str_array_length(args); + + if (argc < 3) { + i_error("doveadm client: No command given"); + return FALSE; + } + i_zero(&cctx); + cctx.conn_type = conn->conn.type; + cctx.input = conn->input; + cctx.output = conn->output; + cctx.local_ip = conn->conn.local_ip; + cctx.remote_ip = conn->conn.remote_ip; + cctx.local_port = conn->conn.local_port; + cctx.remote_port = conn->conn.remote_port; + doveadm_exit_code = 0; + + flags = args[0]; + cctx.username = args[1]; + cmd_name = args[2]; + + doveadm_debug = FALSE; + doveadm_verbose = FALSE; + + for (; *flags != '\0'; flags++) { + switch (*flags) { + case 'D': + doveadm_debug = TRUE; + doveadm_verbose = TRUE; + break; + case 'v': + doveadm_verbose = TRUE; + break; + default: + i_error("doveadm client: Unknown flag: %c", *flags); + return FALSE; + } + } + + if (!doveadm_client_is_allowed_command(conn->conn.set, cmd_name)) { + i_error("doveadm client isn't allowed to use command: %s", + cmd_name); + return FALSE; + } + + client_connection_set_proctitle(&conn->conn, cmd_name); + o_stream_cork(conn->output); + /* Disable IO while running a command. This is required for commands + that do IO themselves (e.g. dsync-server). */ + io_remove(&conn->io); + if (doveadm_cmd_handle(conn, cmd_name, argc-2, args+2, &cctx) < 0) + o_stream_nsend(conn->output, "\n-\n", 3); + o_stream_uncork(conn->output); + conn->io = io_add_istream(conn->input, client_connection_tcp_input, conn); + client_connection_set_proctitle(&conn->conn, ""); + + /* Try to flush the output. It might finish later. */ + (void)o_stream_flush(conn->output); + return TRUE; +} + +static int +client_connection_tcp_authenticate(struct client_connection_tcp *conn) +{ + const struct doveadm_settings *set = conn->conn.set; + const char *line, *pass; + buffer_t *plain; + const unsigned char *data; + size_t size; + + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof) + return -1; + return 0; + } + + if (*set->doveadm_password == '\0') { + i_error("doveadm_password not set, " + "remote authentication disabled"); + return -1; + } + + if (strcmp(line, "STARTTLS") == 0) { + io_remove(&conn->io); + if (client_connection_tcp_init_ssl(conn) < 0) + return -1; + conn->io = io_add_istream(conn->input, client_connection_tcp_input, conn); + return 0; + } + + /* FIXME: some day we should probably let auth process do this and + support all kinds of authentication */ + if (!str_begins(line, "PLAIN\t")) { + i_error("doveadm client attempted non-PLAIN authentication: %s", line); + return -1; + } + + plain = t_buffer_create(128); + if (base64_decode(line + 6, strlen(line + 6), NULL, plain) < 0) { + i_error("doveadm client sent invalid base64 auth PLAIN data"); + return -1; + } + data = plain->data; + size = plain->used; + + if (size < 10 || data[0] != '\0' || + memcmp(data+1, "doveadm", 7) != 0 || data[8] != '\0') { + i_error("doveadm client didn't authenticate as 'doveadm'"); + return -1; + } + pass = t_strndup(data + 9, size - 9); + if (strlen(pass) != strlen(set->doveadm_password) || + !mem_equals_timing_safe(pass, set->doveadm_password, + strlen(pass))) { + i_error("doveadm client authenticated with wrong password"); + return -1; + } + return 1; +} + +static void client_log_disconnect_error(struct client_connection_tcp *conn) +{ + const char *error; + + error = conn->ssl_iostream == NULL ? NULL : + ssl_iostream_get_last_error(conn->ssl_iostream); + if (error == NULL) { + error = conn->input->stream_errno == 0 ? "EOF" : + strerror(conn->input->stream_errno); + } + i_error("doveadm client disconnected before handshake: %s", error); +} + +static void +client_connection_tcp_input(struct client_connection_tcp *conn) +{ + const char *line; + bool ok = TRUE; + int ret; + unsigned int minor; + + if (!conn->handshaked) { + if ((line = i_stream_read_next_line(conn->input)) == NULL) { + if (conn->input->eof || conn->input->stream_errno != 0) { + client_log_disconnect_error(conn); + client_connection_tcp_destroy(&conn); + } + return; + } + if (!version_string_verify_full(line, "doveadm-server", + DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR, &minor)) { + i_error("doveadm client not compatible with this server " + "(mixed old and new binaries?)"); + client_connection_tcp_destroy(&conn); + return; + } + if (minor > 0) { + /* send version reply */ + o_stream_nsend_str(conn->output, + DOVEADM_CLIENT_PROTOCOL_VERSION_LINE"\n"); + conn->use_multiplex = TRUE; + } + client_connection_tcp_send_auth_handshake(conn); + conn->handshaked = TRUE; + } + if (!conn->authenticated) { + if ((ret = client_connection_tcp_authenticate(conn)) <= 0) { + if (ret < 0) { + o_stream_nsend(conn->output, "-\n", 2); + client_connection_tcp_destroy(&conn); + } + return; + } + o_stream_nsend(conn->output, "+\n", 2); + conn->authenticated = TRUE; + } + + if (!conn->io_setup) { + conn->io_setup = TRUE; + if (conn->use_multiplex) { + struct ostream *os = conn->output; + conn->output = o_stream_create_multiplex(os, SIZE_MAX); + o_stream_set_name(conn->output, o_stream_get_name(os)); + o_stream_set_no_error_handling(conn->output, TRUE); + o_stream_unref(&os); + conn->log_out = + o_stream_multiplex_add_channel(conn->output, + DOVEADM_LOG_CHANNEL_ID); + o_stream_set_no_error_handling(conn->log_out, TRUE); + o_stream_set_name(conn->log_out, t_strdup_printf("%s (log)", + o_stream_get_name(conn->output))); + doveadm_server_capture_logs(); + } + doveadm_print_ostream = conn->output; + } + + while (ok && !conn->input->closed && + (line = i_stream_read_next_line(conn->input)) != NULL) { + T_BEGIN { + const char *const *args; + + args = t_strsplit_tabescaped(line); + ok = client_handle_command(conn, args); + } T_END; + } + if (conn->input->eof || conn->input->stream_errno != 0 || !ok) + client_connection_tcp_destroy(&conn); +} + +static int +client_connection_tcp_init_ssl(struct client_connection_tcp *conn) +{ + const char *error; + + if (master_service_ssl_init(master_service, + &conn->input, &conn->output, + &conn->ssl_iostream, &error) < 0) { + i_error("SSL init failed: %s", error); + return -1; + } + if (ssl_iostream_handshake(conn->ssl_iostream) < 0) { + i_error("SSL handshake failed: %s", + ssl_iostream_get_last_error(conn->ssl_iostream)); + return -1; + } + return 0; +} + +static bool +client_connection_is_preauthenticated(int listen_fd) +{ + const char *listen_path; + struct stat st; + + /* we'll have to do this with stat(), because at least in Linux + fstat() always returns mode as 0777 */ + return net_getunixname(listen_fd, &listen_path) == 0 && + stat(listen_path, &st) == 0 && S_ISSOCK(st.st_mode) && + (st.st_mode & 0777) == 0600; +} + +static void +client_connection_tcp_send_auth_handshake(struct client_connection_tcp *conn) +{ + if (conn->preauthenticated) { + /* no need for client to authenticate */ + conn->authenticated = TRUE; + o_stream_nsend(conn->output, "+\n", 2); + } else { + o_stream_nsend(conn->output, "-\n", 2); + } +} + +static void +client_connection_tcp_free(struct client_connection *_conn) +{ + struct client_connection_tcp *conn = + (struct client_connection_tcp *)_conn; + + i_assert(_conn->type == DOVEADM_CONNECTION_TYPE_TCP); + + doveadm_print_deinit(); + doveadm_print_ostream = NULL; + + if (conn->log_out != NULL) { + doveadm_server_restore_logs(); + o_stream_unref(&conn->log_out); + } + ssl_iostream_destroy(&conn->ssl_iostream); + + io_remove(&conn->io); + o_stream_destroy(&conn->output); + i_stream_destroy(&conn->input); + i_close_fd(&conn->fd); +} + +struct client_connection * +client_connection_tcp_create(int fd, int listen_fd, bool ssl) +{ + struct client_connection_tcp *conn; + pool_t pool; + + pool = pool_alloconly_create("doveadm client", 1024*16); + conn = p_new(pool, struct client_connection_tcp, 1); + conn->fd = fd; + + if (client_connection_init(&conn->conn, + DOVEADM_CONNECTION_TYPE_TCP, pool, fd) < 0) { + client_connection_tcp_destroy(&conn); + return NULL; + } + conn->conn.free = client_connection_tcp_free; + + doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER); + + conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE); + conn->output = o_stream_create_fd(fd, SIZE_MAX); + i_stream_set_name(conn->input, conn->conn.name); + o_stream_set_name(conn->output, conn->conn.name); + o_stream_set_no_error_handling(conn->output, TRUE); + + if (ssl) { + if (client_connection_tcp_init_ssl(conn) < 0) { + client_connection_tcp_destroy(&conn); + return NULL; + } + } + /* add IO after SSL istream is created */ + conn->io = io_add_istream(conn->input, client_connection_tcp_input, conn); + conn->preauthenticated = + client_connection_is_preauthenticated(listen_fd); + client_connection_set_proctitle(&conn->conn, ""); + + return &conn->conn; +} + +static void +client_connection_tcp_destroy(struct client_connection_tcp **_conn) +{ + struct client_connection_tcp *conn = *_conn; + struct client_connection *bconn = &conn->conn; + + *_conn = NULL; + client_connection_destroy(&bconn); +} diff --git a/src/doveadm/client-connection.c b/src/doveadm/client-connection.c new file mode 100644 index 0000000..02a64a6 --- /dev/null +++ b/src/doveadm/client-connection.c @@ -0,0 +1,127 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "process-title.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "doveadm.h" +#include "doveadm-settings.h" +#include "doveadm-server.h" +#include "client-connection-private.h" + +bool doveadm_client_is_allowed_command(const struct doveadm_settings *set, + const char *cmd_name) +{ + bool ret = FALSE; + + if (*set->doveadm_allowed_commands == '\0') + return TRUE; + + T_BEGIN { + const char *const *cmds = + t_strsplit(set->doveadm_allowed_commands, ","); + for (; *cmds != NULL; cmds++) { + if (strcmp(*cmds, cmd_name) == 0) { + ret = TRUE; + break; + } + } + } T_END; + return ret; +} + +static int client_connection_read_settings(struct client_connection *conn) +{ + const struct setting_parser_info *set_roots[] = { + &doveadm_setting_parser_info, + NULL + }; + struct master_service_settings_input input; + struct master_service_settings_output output; + const char *error; + void *set; + + i_zero(&input); + input.roots = set_roots; + input.service = "doveadm"; + input.local_ip = conn->local_ip; + input.remote_ip = conn->remote_ip; + + if (master_service_settings_read(master_service, &input, + &output, &error) < 0) { + i_error("Error reading configuration: %s", error); + return -1; + } + set = master_service_settings_get_others(master_service)[0]; + conn->set = settings_dup(&doveadm_setting_parser_info, set, conn->pool); + return 0; +} + +int client_connection_init(struct client_connection *conn, + enum doveadm_client_type type, pool_t pool, int fd) +{ + const char *ip; + + i_assert(type != DOVEADM_CONNECTION_TYPE_CLI); + + conn->type = type; + conn->pool = pool; + + (void)net_getsockname(fd, &conn->local_ip, &conn->local_port); + (void)net_getpeername(fd, &conn->remote_ip, &conn->remote_port); + + ip = net_ip2addr(&conn->remote_ip); + if (ip[0] != '\0') + i_set_failure_prefix("doveadm(%s): ", ip); + + conn->name = conn->remote_ip.family == 0 ? "<local>" : + p_strdup(pool, net_ip2addr(&conn->remote_ip)); + + return client_connection_read_settings(conn); +} + +void client_connection_destroy(struct client_connection **_conn) +{ + struct client_connection *conn = *_conn; + + *_conn = NULL; + + if (conn->free != NULL) + conn->free(conn); + + doveadm_client = NULL; + master_service_client_connection_destroyed(master_service); + + if (doveadm_verbose_proctitle) + process_title_set("[idling]"); + + pool_unref(&conn->pool); +} + +void client_connection_set_proctitle(struct client_connection *conn, + const char *text) +{ + const char *str; + + if (!doveadm_verbose_proctitle) + return; + + if (text[0] == '\0') + str = t_strdup_printf("[%s]", conn->name); + else + str = t_strdup_printf("[%s %s]", conn->name, text); + process_title_set(str); +} + +void doveadm_server_init(void) +{ + doveadm_http_server_init(); +} + +void doveadm_server_deinit(void) +{ + if (doveadm_client != NULL) + client_connection_destroy(&doveadm_client); + doveadm_http_server_deinit(); +} diff --git a/src/doveadm/client-connection.h b/src/doveadm/client-connection.h new file mode 100644 index 0000000..bb3797e --- /dev/null +++ b/src/doveadm/client-connection.h @@ -0,0 +1,26 @@ +#ifndef CLIENT_CONNECTION_H +#define CLIENT_CONNECTION_H + +#include "net.h" + +#define DOVEADM_LOG_CHANNEL_ID 'L' + +struct client_connection { + pool_t pool; + enum doveadm_client_type type; + const char *name; + + struct ip_addr local_ip, remote_ip; + in_port_t local_port, remote_port; + + const struct doveadm_settings *set; + + void (*free)(struct client_connection *conn); +}; + +struct client_connection * +client_connection_tcp_create(int fd, int listen_fd, bool ssl); +struct client_connection * +client_connection_http_create(int fd, bool ssl); + +#endif diff --git a/src/doveadm/doveadm-auth-server.c b/src/doveadm/doveadm-auth-server.c new file mode 100644 index 0000000..f50fa5f --- /dev/null +++ b/src/doveadm/doveadm-auth-server.c @@ -0,0 +1,517 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "str.h" +#include "var-expand.h" +#include "wildcard-match.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "auth-client.h" +#include "auth-master.h" +#include "master-auth.h" +#include "master-login-auth.h" +#include "mail-storage-service.h" +#include "mail-user.h" +#include "ostream.h" +#include "json-parser.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> + +struct authtest_input { + pool_t pool; + const char *username; + const char *master_user; + const char *password; + struct auth_user_info info; + bool success; + + struct auth_client_request *request; + struct master_auth_request master_auth_req; + + unsigned int auth_id; + unsigned int auth_pid; + const char *auth_cookie; + +}; + +static struct auth_master_connection * +doveadm_get_auth_master_conn(const char *auth_socket_path) +{ + enum auth_master_flags flags = 0; + + if (doveadm_debug) + flags |= AUTH_MASTER_FLAG_DEBUG; + return auth_master_init(auth_socket_path, flags); +} + +static int +cmd_user_input(struct auth_master_connection *conn, + const struct authtest_input *input, + const char *show_field, bool userdb) +{ + const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup"; + pool_t pool; + const char *updated_username = NULL, *const *fields, *p; + int ret; + + pool = pool_alloconly_create("auth master lookup", 1024); + + if (userdb) { + ret = auth_master_user_lookup(conn, input->username, &input->info, + pool, &updated_username, &fields); + } else { + ret = auth_master_pass_lookup(conn, input->username, &input->info, + pool, &fields); + } + if (ret < 0) { + const char *msg; + if (fields[0] == NULL) { + msg = t_strdup_printf("\"error\":\"%s failed\"", + lookup_name); + } else { + msg = t_strdup_printf("\"error\":\"%s failed: %s\"", + lookup_name, + fields[0]); + } + o_stream_nsend_str(doveadm_print_ostream, msg); + ret = -1; + } else if (ret == 0) { + o_stream_nsend_str(doveadm_print_ostream, + t_strdup_printf("\"error\":\"%s: user doesn't exist\"", + lookup_name)); + } else if (show_field != NULL) { + size_t show_field_len = strlen(show_field); + string_t *json_field = t_str_new(show_field_len+1); + json_append_escaped(json_field, show_field); + o_stream_nsend_str(doveadm_print_ostream, t_strdup_printf("\"%s\":", str_c(json_field))); + for (; *fields != NULL; fields++) { + if (strncmp(*fields, show_field, show_field_len) == 0 && + (*fields)[show_field_len] == '=') { + string_t *jsonval = t_str_new(32); + json_append_escaped(jsonval, *fields + show_field_len + 1); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } + } + } else { + string_t *jsonval = t_str_new(64); + o_stream_nsend_str(doveadm_print_ostream, "\"source\":\""); + o_stream_nsend_str(doveadm_print_ostream, userdb ? "userdb\"" : "passdb\""); + + if (updated_username != NULL) { + o_stream_nsend_str(doveadm_print_ostream, ",\"updated_username\":\""); + str_truncate(jsonval, 0); + json_append_escaped(jsonval, updated_username); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } + for (; *fields != NULL; fields++) { + const char *field = *fields; + if (*field == '\0') continue; + p = strchr(*fields, '='); + str_truncate(jsonval, 0); + if (p != NULL) { + field = t_strcut(*fields, '='); + } + str_truncate(jsonval, 0); + json_append_escaped(jsonval, field); + o_stream_nsend_str(doveadm_print_ostream, ",\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\":"); + if (p != NULL) { + str_truncate(jsonval, 0); + json_append_escaped(jsonval, p+1); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } else { + o_stream_nsend_str(doveadm_print_ostream, "true"); + } + } + } + return ret; +} + +static void auth_user_info_parse(struct auth_user_info *info, const char *arg) +{ + if (str_begins(arg, "service=")) + info->service = arg + 8; + else if (str_begins(arg, "lip=")) { + if (net_addr2ip(arg + 4, &info->local_ip) < 0) + i_fatal("lip: Invalid ip"); + } else if (str_begins(arg, "rip=")) { + if (net_addr2ip(arg + 4, &info->remote_ip) < 0) + i_fatal("rip: Invalid ip"); + } else if (str_begins(arg, "lport=")) { + if (net_str2port(arg + 6, &info->local_port) < 0) + i_fatal("lport: Invalid port number"); + } else if (str_begins(arg, "rport=")) { + if (net_str2port(arg + 6, &info->remote_port) < 0) + i_fatal("rport: Invalid port number"); + } else { + i_fatal("Unknown -x argument: %s", arg); + } +} + +static void +cmd_user_list(struct auth_master_connection *conn, + const struct authtest_input *input, + char *const *users) +{ + struct auth_master_user_list_ctx *ctx; + const char *username, *user_mask = "*"; + string_t *escaped = t_str_new(256); + bool first = TRUE; + unsigned int i; + + if (users[0] != NULL && users[1] == NULL) + user_mask = users[0]; + + o_stream_nsend_str(doveadm_print_ostream, "{\"userList\":["); + + ctx = auth_master_user_list_init(conn, user_mask, &input->info); + while ((username = auth_master_user_list_next(ctx)) != NULL) { + for (i = 0; users[i] != NULL; i++) { + if (wildcard_match_icase(username, users[i])) + break; + } + if (users[i] != NULL) { + if (first) + first = FALSE; + else + o_stream_nsend_str(doveadm_print_ostream, ","); + str_truncate(escaped, 0); + str_append_c(escaped, '"'); + json_append_escaped(escaped, username); + str_append_c(escaped, '"'); + o_stream_nsend(doveadm_print_ostream, escaped->data, escaped->used); + } + } + if (auth_master_user_list_deinit(&ctx) < 0) { + i_error("user listing failed"); + doveadm_exit_code = EX_DATAERR; + } + + o_stream_nsend_str(doveadm_print_ostream, "]}"); +} + +static void cmd_auth_cache_flush(struct doveadm_cmd_context *cctx) +{ + const char *master_socket_path, *const *users; + struct auth_master_connection *conn; + unsigned int count; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &master_socket_path)) { + master_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-master", NULL); + } + if (!doveadm_cmd_param_array(cctx, "user", &users)) + i_fatal("Missing user parameter"); + + conn = doveadm_get_auth_master_conn(master_socket_path); + if (auth_master_cache_flush(conn, users, &count) < 0) { + i_error("Cache flush failed"); + doveadm_exit_code = EX_TEMPFAIL; + } else { + doveadm_print_init("formatted"); + doveadm_print_formatted_set_format("%{entries} cache entries flushed\n"); + doveadm_print_header_simple("entries"); + doveadm_print_num(count); + } + auth_master_deinit(&conn); +} + +static void cmd_user_mail_input_field(const char *key, const char *value, + const char *show_field, bool *first) +{ + string_t *jvalue = t_str_new(128); + if (show_field != NULL && strcmp(show_field, key) != 0) return; + /* do not emit comma on first field. we need to keep track + of when the first field actually gets printed as it + might change due to show_field */ + if (!*first) + o_stream_nsend_str(doveadm_print_ostream, ","); + *first = FALSE; + json_append_escaped(jvalue, key); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue)); + o_stream_nsend_str(doveadm_print_ostream, "\":\""); + str_truncate(jvalue, 0); + json_append_escaped(jvalue, value); + o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue)); + o_stream_nsend_str(doveadm_print_ostream, "\""); +} + +static void +cmd_user_mail_print_fields(const struct authtest_input *input, + struct mail_user *user, + const char *const *userdb_fields, + const char *show_field) +{ + const struct mail_storage_settings *mail_set; + const char *key, *value; + unsigned int i; + bool first = TRUE; + + if (strcmp(input->username, user->username) != 0) + cmd_user_mail_input_field("user", user->username, show_field, &first); + cmd_user_mail_input_field("uid", user->set->mail_uid, show_field, &first); + cmd_user_mail_input_field("gid", user->set->mail_gid, show_field, &first); + cmd_user_mail_input_field("home", user->set->mail_home, show_field, &first); + + mail_set = mail_user_set_get_storage_set(user); + cmd_user_mail_input_field("mail", mail_set->mail_location, show_field, &first); + + if (userdb_fields != NULL) { + for (i = 0; userdb_fields[i] != NULL; i++) { + value = strchr(userdb_fields[i], '='); + if (value != NULL) + key = t_strdup_until(userdb_fields[i], value++); + else { + key = userdb_fields[i]; + value = ""; + } + if (strcmp(key, "uid") != 0 && + strcmp(key, "gid") != 0 && + strcmp(key, "home") != 0 && + strcmp(key, "mail") != 0 && + *key != '\0') { + cmd_user_mail_input_field(key, value, show_field, &first); + } + } + } +} + +static int +cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, + const struct authtest_input *input, + const char *show_field, const char *expand_field) +{ + struct mail_storage_service_input service_input; + struct mail_storage_service_user *service_user; + struct mail_user *user; + const char *error, *const *userdb_fields; + pool_t pool; + int ret; + + i_zero(&service_input); + service_input.module = "mail"; + service_input.service = input->info.service; + service_input.username = input->username; + service_input.local_ip = input->info.local_ip; + service_input.local_port = input->info.local_port; + service_input.remote_ip = input->info.remote_ip; + service_input.remote_port = input->info.remote_port; + service_input.debug = input->info.debug; + + pool = pool_alloconly_create("userdb fields", 1024); + mail_storage_service_save_userdb_fields(storage_service, pool, + &userdb_fields); + + if ((ret = mail_storage_service_lookup_next(storage_service, &service_input, + &service_user, &user, + &error)) <= 0) { + pool_unref(&pool); + if (ret < 0) + return -1; + string_t *username = t_str_new(32); + json_append_escaped(username, input->username); + o_stream_nsend_str(doveadm_print_ostream, + t_strdup_printf("\"error\":\"userdb lookup: user %s doesn't exist\"", str_c(username)) + ); + return 0; + } + + if (expand_field == NULL) + cmd_user_mail_print_fields(input, user, userdb_fields, show_field); + else { + string_t *str = t_str_new(128); + if (var_expand_with_funcs(str, expand_field, + mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user, + &error) <= 0) { + string_t *str = t_str_new(128); + str_printfa(str, "\"error\":\"Failed to expand field: "); + json_append_escaped(str, error); + str_append_c(str, '"'); + o_stream_nsend(doveadm_print_ostream, str_data(str), str_len(str)); + } else { + string_t *value = t_str_new(128); + json_append_escaped(value, expand_field); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(value)); + o_stream_nsend_str(doveadm_print_ostream, "\":\""); + str_truncate(value, 0); + json_append_escaped(value, str_c(str)); + o_stream_nsend_str(doveadm_print_ostream, str_c(value)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + } + + } + + mail_user_deinit(&user); + mail_storage_service_user_unref(&service_user); + pool_unref(&pool); + return 1; +} + +static void cmd_user_ver2(struct doveadm_cmd_context *cctx) +{ + const char * const *optval; + + const char *auth_socket_path = NULL; + struct auth_master_connection *conn; + struct authtest_input input; + const char *show_field = NULL, *expand_field = NULL; + struct mail_storage_service_ctx *storage_service = NULL; + bool have_wildcards, userdb_only = FALSE, first = TRUE; + int ret; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path)) + auth_socket_path = doveadm_settings->auth_socket_path; + + (void)doveadm_cmd_param_str(cctx, "expand-field", &expand_field); + (void)doveadm_cmd_param_str(cctx, "field", &show_field); + (void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only); + + i_zero(&input); + if (doveadm_cmd_param_array(cctx, "auth-info", &optval)) + for(;*optval != NULL; optval++) + auth_user_info_parse(&input.info, *optval); + + if (!doveadm_cmd_param_array(cctx, "user-mask", &optval)) { + doveadm_exit_code = EX_USAGE; + i_error("No user(s) specified"); + return; + } + + if (expand_field != NULL && userdb_only) { + i_error("-e can't be used with -u"); + doveadm_exit_code = EX_USAGE; + return; + } + if (expand_field != NULL && show_field != NULL) { + i_error("-e can't be used with -f"); + doveadm_exit_code = EX_USAGE; + return; + } + + conn = doveadm_get_auth_master_conn(auth_socket_path); + + have_wildcards = FALSE; + + for(const char *const *val = optval; *val != NULL; val++) { + if (strchr(*val, '*') != NULL || + strchr(*val, '?') != NULL) { + have_wildcards = TRUE; + break; + } + } + + if (have_wildcards) { + cmd_user_list(conn, &input, (char*const*)optval); + auth_master_deinit(&conn); + return; + } + + if (!userdb_only) { + storage_service = mail_storage_service_init(master_service, NULL, + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP | + MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR | + MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | + MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | + MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | + MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS); + mail_storage_service_set_auth_conn(storage_service, conn); + conn = NULL; + } + + string_t *json = t_str_new(64); + o_stream_nsend_str(doveadm_print_ostream, "{"); + + input.info.local_ip = cctx->local_ip; + input.info.local_port = cctx->local_port; + input.info.remote_ip = cctx->remote_ip; + input.info.remote_port = cctx->remote_port; + + for(const char *const *val = optval; *val != NULL; val++) { + str_truncate(json, 0); + json_append_escaped(json, *val); + + input.username = *val; + if (first) + first = FALSE; + else + o_stream_nsend_str(doveadm_print_ostream, ","); + + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, str_c(json)); + o_stream_nsend_str(doveadm_print_ostream, "\""); + o_stream_nsend_str(doveadm_print_ostream, ":{"); + + ret = !userdb_only ? + cmd_user_mail_input(storage_service, &input, show_field, expand_field) : + cmd_user_input(conn, &input, show_field, TRUE); + + o_stream_nsend_str(doveadm_print_ostream, "}"); + + switch (ret) { + case -1: + doveadm_exit_code = EX_TEMPFAIL; + break; + case 0: + doveadm_exit_code = EX_NOUSER; + break; + } + } + + o_stream_nsend_str(doveadm_print_ostream,"}"); + + if (storage_service != NULL) + mail_storage_service_deinit(&storage_service); + if (conn != NULL) + auth_master_deinit(&conn); +} + +static +struct doveadm_cmd_ver2 doveadm_cmd_auth_server[] = { +{ + .name = "auth cache flush", + .cmd = cmd_auth_cache_flush, + .usage = "[-a <master socket path>] [<user> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "user", + .cmd = cmd_user_ver2, + .usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] [-e <value>] [-u] <user mask> [...]", + .flags = CMD_FLAG_NO_PRINT, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('e', "expand-field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('u', "userdb-only", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +void doveadm_register_auth_server_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth_server); i++) { + doveadm_cmd_register_ver2(&doveadm_cmd_auth_server[i]); + } +} diff --git a/src/doveadm/doveadm-auth.c b/src/doveadm/doveadm-auth.c new file mode 100644 index 0000000..d89befe --- /dev/null +++ b/src/doveadm/doveadm-auth.c @@ -0,0 +1,787 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "array.h" +#include "askpass.h" +#include "base64.h" +#include "hex-binary.h" +#include "str.h" +#include "strescape.h" +#include "var-expand.h" +#include "wildcard-match.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "auth-client.h" +#include "auth-master.h" +#include "master-auth.h" +#include "master-login-auth.h" +#include "mail-storage-service.h" +#include "mail-user.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> + +static struct event_category event_category_auth = { + .name = "auth", +}; + +struct authtest_input { + pool_t pool; + const char *username; + const char *master_user; + const char *password; + struct auth_user_info info; + bool success; + + struct auth_client_request *request; + struct master_auth_request master_auth_req; + + unsigned int auth_id; + unsigned int auth_pid; + const char *auth_cookie; + +}; + +static void auth_cmd_help(struct doveadm_cmd_context *cctx); + +static struct auth_master_connection * +doveadm_get_auth_master_conn(const char *auth_socket_path) +{ + enum auth_master_flags flags = 0; + + if (doveadm_debug) + flags |= AUTH_MASTER_FLAG_DEBUG; + return auth_master_init(auth_socket_path, flags); +} + +static int +cmd_user_input(struct auth_master_connection *conn, + const struct authtest_input *input, + const char *show_field, bool userdb) +{ + const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup"; + pool_t pool; + const char *updated_username = NULL, *const *fields, *p; + int ret; + + pool = pool_alloconly_create("auth master lookup", 1024); + + if (userdb) { + ret = auth_master_user_lookup(conn, input->username, &input->info, + pool, &updated_username, &fields); + } else { + ret = auth_master_pass_lookup(conn, input->username, &input->info, + pool, &fields); + } + if (ret < 0) { + if (fields[0] == NULL) + i_error("%s failed for %s", lookup_name, input->username); + else { + i_error("%s failed for %s: %s", lookup_name, + input->username, fields[0]); + } + ret = -1; + } else if (ret == 0) { + fprintf(show_field == NULL ? stdout : stderr, + "%s: user %s doesn't exist\n", lookup_name, + input->username); + } else if (show_field != NULL) { + size_t show_field_len = strlen(show_field); + + for (; *fields != NULL; fields++) { + if (strncmp(*fields, show_field, show_field_len) == 0 && + (*fields)[show_field_len] == '=') + printf("%s\n", *fields + show_field_len + 1); + } + } else { + printf("%s: %s\n", userdb ? "userdb" : "passdb", input->username); + + if (updated_username != NULL) + printf(" %-10s: %s\n", "user", updated_username); + for (; *fields != NULL; fields++) { + p = strchr(*fields, '='); + if (p == NULL) + printf(" %-10s\n", *fields); + else { + printf(" %-10s: %s\n", + t_strcut(*fields, '='), p + 1); + } + } + } + pool_unref(&pool); + return ret; +} + +static void +auth_callback(struct auth_client_request *request ATTR_UNUSED, + enum auth_request_status status, + const char *data_base64 ATTR_UNUSED, + const char *const *args, void *context) +{ + struct authtest_input *input = context; + + input->request = NULL; + input->auth_id = auth_client_request_get_id(request); + input->auth_pid = auth_client_request_get_server_pid(request); + input->auth_cookie = input->pool == NULL ? NULL : + p_strdup(input->pool, auth_client_request_get_cookie(request)); + + if (!io_loop_is_running(current_ioloop)) + return; + + if (status == 0) + i_fatal("passdb expects SASL continuation"); + + switch (status) { + case AUTH_REQUEST_STATUS_ABORT: + i_unreached(); + case AUTH_REQUEST_STATUS_INTERNAL_FAIL: + case AUTH_REQUEST_STATUS_FAIL: + printf("passdb: %s auth failed\n", input->username); + break; + case AUTH_REQUEST_STATUS_CONTINUE: + printf("passdb: %s auth unexpectedly requested continuation\n", + input->username); + break; + case AUTH_REQUEST_STATUS_OK: + input->success = TRUE; + printf("passdb: %s auth succeeded\n", input->username); + break; + } + + if (args != NULL && *args != NULL) { + printf("extra fields:\n"); + for (; *args != NULL; args++) + printf(" %s\n", *args); + } + io_loop_stop(current_ioloop); +} + +static void auth_connected(struct auth_client *client, + bool connected, void *context) +{ + struct event *event_auth; + struct authtest_input *input = context; + struct auth_request_info info; + string_t *init_resp, *base64_resp; + + if (!connected) + i_fatal("Couldn't connect to auth socket"); + event_auth = event_create(NULL); + event_add_category(event_auth, &event_category_auth); + + init_resp = t_str_new(128); + str_append(init_resp, input->username); + str_append_c(init_resp, '\0'); + if (input->master_user != NULL) + str_append(init_resp, input->master_user); + else + str_append(init_resp, input->username); + str_append_c(init_resp, '\0'); + str_append(init_resp, input->password); + + base64_resp = t_str_new(128); + base64_encode(str_data(init_resp), str_len(init_resp), base64_resp); + + i_zero(&info); + info.mech = "PLAIN"; + info.service = input->info.service; + info.session_id = input->info.session_id; + info.local_name = input->info.local_name; + info.local_ip = input->info.local_ip; + info.local_port = input->info.local_port; + info.remote_ip = input->info.remote_ip; + info.remote_port = input->info.remote_port; + info.real_local_ip = input->info.real_local_ip; + info.real_remote_ip = input->info.real_remote_ip; + info.real_local_port = input->info.real_local_port; + info.real_remote_port = input->info.real_remote_port; + info.extra_fields = input->info.extra_fields; + info.forward_fields = input->info.forward_fields; + info.initial_resp_base64 = str_c(base64_resp); + if (doveadm_settings->auth_debug || + event_want_debug_log(event_auth)) + info.flags |= AUTH_REQUEST_FLAG_DEBUG; + + input->request = auth_client_request_new(client, &info, + auth_callback, input); + event_unref(&event_auth); +} + +static void +cmd_auth_input(const char *auth_socket_path, struct authtest_input *input) +{ + struct auth_client *client; + + if (auth_socket_path == NULL) { + auth_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-client", NULL); + } + + client = auth_client_init(auth_socket_path, getpid(), FALSE); + auth_client_connect(client); + auth_client_set_connect_notify(client, auth_connected, input); + + if (!auth_client_is_disconnected(client)) + io_loop_run(current_ioloop); + + auth_client_set_connect_notify(client, NULL, NULL); + auth_client_deinit(&client); +} + +static void +auth_user_info_parse_arg(struct auth_user_info *info, const char *arg) +{ + if (str_begins(arg, "service=")) + info->service = arg + 8; + else if (str_begins(arg, "session=")) + info->session_id = arg + 8; + else if (str_begins(arg, "local_name=")) + info->local_name = arg + 11; + else if (str_begins(arg, "lip=")) { + if (net_addr2ip(arg + 4, &info->local_ip) < 0) + i_fatal("lip: Invalid ip"); + } else if (str_begins(arg, "rip=")) { + if (net_addr2ip(arg + 4, &info->remote_ip) < 0) + i_fatal("rip: Invalid ip"); + } else if (str_begins(arg, "lport=")) { + if (net_str2port(arg + 6, &info->local_port) < 0) + i_fatal("lport: Invalid port number"); + } else if (str_begins(arg, "rport=")) { + if (net_str2port(arg + 6, &info->remote_port) < 0) + i_fatal("rport: Invalid port number"); + } else if (str_begins(arg, "real_lip=")) { + if (net_addr2ip(arg + 9, &info->real_local_ip) < 0) + i_fatal("real_lip: Invalid ip"); + } else if (str_begins(arg, "real_rip=")) { + if (net_addr2ip(arg + 9, &info->real_remote_ip) < 0) + i_fatal("real_rip: Invalid ip"); + } else if (str_begins(arg, "real_lport=")) { + if (net_str2port(arg + 11, &info->real_local_port) < 0) + i_fatal("real_lport: Invalid port number"); + } else if (str_begins(arg, "real_rport=")) { + if (net_str2port(arg + 11, &info->real_remote_port) < 0) + i_fatal("real_rport: Invalid port number"); + } else if (str_begins(arg, "forward_")) { + const char *key = arg+8; + const char *value = strchr(arg+8, '='); + + if (value == NULL) + value = ""; + else + key = t_strdup_until(key, value++); + key = str_tabescape(key); + value = str_tabescape(value); + if (info->forward_fields == NULL) { + info->forward_fields = + t_strdup_printf("%s=%s", key, value); + } else { + info->forward_fields = + t_strdup_printf("%s\t%s=%s", info->forward_fields, key, value); + } + } else { + if (!array_is_created(&info->extra_fields)) + t_array_init(&info->extra_fields, 4); + array_push_back(&info->extra_fields, &arg); + } +} + +static void +auth_user_info_parse(struct auth_user_info *info, const char *const *args) +{ + for (unsigned int i = 0; args[i] != NULL; i++) + auth_user_info_parse_arg(info, args[i]); +} + +static void +cmd_user_list(struct auth_master_connection *conn, + const struct authtest_input *input, + const char *const *users) +{ + struct auth_master_user_list_ctx *ctx; + const char *username, *user_mask = "*"; + unsigned int i; + + if (users[0] != NULL && users[1] == NULL) + user_mask = users[0]; + + ctx = auth_master_user_list_init(conn, user_mask, &input->info); + while ((username = auth_master_user_list_next(ctx)) != NULL) { + for (i = 0; users[i] != NULL; i++) { + if (wildcard_match_icase(username, users[i])) + break; + } + if (users[i] != NULL) + printf("%s\n", username); + } + if (auth_master_user_list_deinit(&ctx) < 0) + i_fatal("user listing failed"); +} + +static void cmd_auth_cache_flush(struct doveadm_cmd_context *cctx) +{ + const char *master_socket_path; + struct auth_master_connection *conn; + const char *const *users = NULL; + unsigned int count; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &master_socket_path)) { + master_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-master", NULL); + } + (void)doveadm_cmd_param_array(cctx, "user", &users); + + conn = doveadm_get_auth_master_conn(master_socket_path); + if (auth_master_cache_flush(conn, users, &count) < 0) { + i_error("Cache flush failed"); + doveadm_exit_code = EX_TEMPFAIL; + } else { + printf("%u cache entries flushed\n", count); + } + auth_master_deinit(&conn); +} + +static void authtest_input_init(struct authtest_input *input) +{ + i_zero(input); + input->info.service = "doveadm"; + input->info.debug = doveadm_settings->auth_debug; +} + +static void cmd_auth_test(struct doveadm_cmd_context *cctx) +{ + const char *auth_socket_path = NULL; + const char *const *auth_info; + struct authtest_input input; + + authtest_input_init(&input); + (void)doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path); + (void)doveadm_cmd_param_str(cctx, "master-user", &input.master_user); + if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info)) + auth_user_info_parse(&input.info, auth_info); + + if (!doveadm_cmd_param_str(cctx, "user", &input.username)) + auth_cmd_help(cctx); + if (!doveadm_cmd_param_str(cctx, "password", &input.password)) + input.password = t_askpass("Password: "); + cmd_auth_input(auth_socket_path, &input); + if (!input.success) + doveadm_exit_code = EX_NOPERM; +} + +static void +master_auth_callback(const char *const *auth_args, + const char *errormsg, void *context) +{ + struct authtest_input *input = context; + unsigned int i; + + io_loop_stop(current_ioloop); + if (errormsg != NULL) { + i_error("userdb lookup failed: %s", errormsg); + return; + } + printf("userdb extra fields:\n"); + for (i = 0; auth_args[i] != NULL; i++) + printf(" %s\n", auth_args[i]); + input->success = TRUE; +} + +static void +cmd_auth_master_input(const char *auth_master_socket_path, + struct authtest_input *input) +{ + struct master_login_auth *master_auth; + struct master_auth_request master_auth_req; + buffer_t buf; + + i_zero(&master_auth_req); + master_auth_req.tag = 1; + master_auth_req.auth_pid = input->auth_pid; + master_auth_req.auth_id = input->auth_id; + master_auth_req.client_pid = getpid(); + master_auth_req.local_ip = input->info.local_ip; + master_auth_req.remote_ip = input->info.remote_ip; + + buffer_create_from_data(&buf, master_auth_req.cookie, + sizeof(master_auth_req.cookie)); + if (strlen(input->auth_cookie) == MASTER_AUTH_COOKIE_SIZE*2) + (void)hex_to_binary(input->auth_cookie, &buf); + + input->success = FALSE; + master_auth = master_login_auth_init(auth_master_socket_path, FALSE); + io_loop_set_running(current_ioloop); + master_login_auth_request(master_auth, &master_auth_req, + master_auth_callback, input); + if (io_loop_is_running(current_ioloop)) + io_loop_run(current_ioloop); + master_login_auth_deinit(&master_auth); +} + +static void cmd_auth_login(struct doveadm_cmd_context *cctx) +{ + const char *auth_login_socket_path, *auth_master_socket_path; + const char *const *auth_info; + struct auth_client *auth_client; + struct authtest_input input; + + authtest_input_init(&input); + if (!doveadm_cmd_param_str(cctx, "auth-login-socket-path", + &auth_login_socket_path)) { + auth_login_socket_path = + t_strconcat(doveadm_settings->base_dir, + "/auth-login", NULL); + } + if (!doveadm_cmd_param_str(cctx, "auth-master-socket-path", + &auth_master_socket_path)) { + auth_master_socket_path = + t_strconcat(doveadm_settings->base_dir, + "/auth-master", NULL); + } + (void)doveadm_cmd_param_str(cctx, "master-user", &input.master_user); + if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info)) + auth_user_info_parse(&input.info, auth_info); + if (!doveadm_cmd_param_str(cctx, "user", &input.username)) + auth_cmd_help(cctx); + if (!doveadm_cmd_param_str(cctx, "password", &input.password)) + input.password = t_askpass("Password: "); + + input.pool = pool_alloconly_create("auth login", 256); + /* authenticate */ + auth_client = auth_client_init(auth_login_socket_path, getpid(), FALSE); + auth_client_connect(auth_client); + auth_client_set_connect_notify(auth_client, auth_connected, &input); + if (!auth_client_is_disconnected(auth_client)) + io_loop_run(current_ioloop); + auth_client_set_connect_notify(auth_client, NULL, NULL); + /* finish login with userdb lookup */ + if (input.success) + cmd_auth_master_input(auth_master_socket_path, &input); + if (!input.success) + doveadm_exit_code = EX_NOPERM; + auth_client_deinit(&auth_client); + pool_unref(&input.pool); +} + +static void cmd_auth_lookup(struct doveadm_cmd_context *cctx) +{ + const char *auth_socket_path; + struct auth_master_connection *conn; + struct authtest_input input; + const char *show_field = NULL; + const char *const *auth_info, *const *users; + bool first = TRUE; + int ret; + + authtest_input_init(&input); + if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path)) + auth_socket_path = doveadm_settings->auth_socket_path; + (void)doveadm_cmd_param_str(cctx, "field", &show_field); + if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info)) + auth_user_info_parse(&input.info, auth_info); + if (!doveadm_cmd_param_array(cctx, "user", &users)) + auth_cmd_help(cctx); + + conn = doveadm_get_auth_master_conn(auth_socket_path); + for (unsigned int i = 0; users[i] != NULL; i++) { + input.username = users[i]; + if (first) + first = FALSE; + else + putchar('\n'); + + ret = cmd_user_input(conn, &input, show_field, FALSE); + switch (ret) { + case -1: + doveadm_exit_code = EX_TEMPFAIL; + break; + case 0: + doveadm_exit_code = EX_NOUSER; + break; + } + } + auth_master_deinit(&conn); +} + +static void cmd_user_mail_input_field(const char *key, const char *value, + const char *show_field) +{ + if (show_field == NULL) { + doveadm_print(key); + doveadm_print(value); + } else if (strcmp(show_field, key) == 0) { + printf("%s\n", value); + } +} + +static void +cmd_user_mail_print_fields(const struct authtest_input *input, + struct mail_user *user, + const char *const *userdb_fields, + const char *show_field) +{ + const struct mail_storage_settings *mail_set; + const char *key, *value; + unsigned int i; + + if (strcmp(input->username, user->username) != 0) + cmd_user_mail_input_field("user", user->username, show_field); + cmd_user_mail_input_field("uid", user->set->mail_uid, show_field); + cmd_user_mail_input_field("gid", user->set->mail_gid, show_field); + cmd_user_mail_input_field("home", user->set->mail_home, show_field); + + mail_set = mail_user_set_get_storage_set(user); + cmd_user_mail_input_field("mail", mail_set->mail_location, show_field); + + if (userdb_fields != NULL) { + for (i = 0; userdb_fields[i] != NULL; i++) { + value = strchr(userdb_fields[i], '='); + if (value != NULL) + key = t_strdup_until(userdb_fields[i], value++); + else { + key = userdb_fields[i]; + value = ""; + } + if (strcmp(key, "uid") != 0 && + strcmp(key, "gid") != 0 && + strcmp(key, "home") != 0 && + strcmp(key, "mail") != 0) + cmd_user_mail_input_field(key, value, show_field); + } + } +} + +static int +cmd_user_mail_input(struct mail_storage_service_ctx *storage_service, + const struct authtest_input *input, + const char *show_field, const char *expand_field) +{ + struct mail_storage_service_input service_input; + struct mail_storage_service_user *service_user; + struct mail_user *user; + const char *error, *const *userdb_fields; + pool_t pool; + int ret; + + i_zero(&service_input); + service_input.module = "mail"; + service_input.service = input->info.service; + service_input.username = input->username; + service_input.local_ip = input->info.local_ip; + service_input.local_port = input->info.local_port; + service_input.remote_ip = input->info.remote_ip; + service_input.remote_port = input->info.remote_port; + service_input.debug = input->info.debug; + + pool = pool_alloconly_create("userdb fields", 1024); + mail_storage_service_save_userdb_fields(storage_service, pool, + &userdb_fields); + + if ((ret = mail_storage_service_lookup_next(storage_service, &service_input, + &service_user, &user, + &error)) <= 0) { + pool_unref(&pool); + if (ret < 0) + return -1; + fprintf(show_field == NULL && expand_field == NULL ? stdout : stderr, + "\nuserdb lookup: user %s doesn't exist\n", + input->username); + return 0; + } + + if (expand_field == NULL) + cmd_user_mail_print_fields(input, user, userdb_fields, show_field); + else { + string_t *str = t_str_new(128); + if (var_expand_with_funcs(str, expand_field, + mail_user_var_expand_table(user), + mail_user_var_expand_func_table, user, + &error) <= 0) { + i_error("Failed to expand %s: %s", expand_field, error); + } else { + printf("%s\n", str_c(str)); + } + } + + mail_user_deinit(&user); + mail_storage_service_user_unref(&service_user); + pool_unref(&pool); + return 1; +} + +static void cmd_user(struct doveadm_cmd_context *cctx) +{ + const char *auth_socket_path; + struct auth_master_connection *conn; + struct authtest_input input; + const char *show_field = NULL, *expand_field = NULL; + const char *const *user_masks, *const *auth_info; + struct mail_storage_service_ctx *storage_service = NULL; + unsigned int i; + bool have_wildcards, userdb_only = FALSE, first = TRUE; + int ret; + + authtest_input_init(&input); + if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path)) + auth_socket_path = doveadm_settings->auth_socket_path; + (void)doveadm_cmd_param_str(cctx, "field", &show_field); + (void)doveadm_cmd_param_str(cctx, "expand-field", &expand_field); + (void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only); + if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info)) + auth_user_info_parse(&input.info, auth_info); + if (!doveadm_cmd_param_array(cctx, "user-mask", &user_masks)) + auth_cmd_help(cctx); + + if (expand_field != NULL && userdb_only) { + i_error("-e can't be used with -u"); + doveadm_exit_code = EX_USAGE; + return; + } + if (expand_field != NULL && show_field != NULL) { + i_error("-e can't be used with -f"); + doveadm_exit_code = EX_USAGE; + return; + } + + conn = doveadm_get_auth_master_conn(auth_socket_path); + + have_wildcards = FALSE; + for (i = 0; user_masks[i] != NULL; i++) { + if (strchr(user_masks[i], '*') != NULL || + strchr(user_masks[i], '?') != NULL) { + have_wildcards = TRUE; + break; + } + } + + if (have_wildcards) { + cmd_user_list(conn, &input, user_masks); + auth_master_deinit(&conn); + return; + } + + if (!userdb_only) { + storage_service = mail_storage_service_init(master_service, NULL, + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP | + MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR | + MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT | + MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS | + MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | + MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS); + mail_storage_service_set_auth_conn(storage_service, conn); + conn = NULL; + if (show_field == NULL && expand_field == NULL) { + doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); + doveadm_print_header_simple("field"); + doveadm_print_header_simple("value"); + } + } + + for (i = 0; user_masks[i] != NULL; i++) { + input.username = user_masks[i]; + if (first) + first = FALSE; + else + putchar('\n'); + + ret = !userdb_only ? + cmd_user_mail_input(storage_service, &input, show_field, expand_field) : + cmd_user_input(conn, &input, show_field, TRUE); + switch (ret) { + case -1: + doveadm_exit_code = EX_TEMPFAIL; + break; + case 0: + doveadm_exit_code = EX_NOUSER; + break; + } + } + if (storage_service != NULL) + mail_storage_service_deinit(&storage_service); + if (conn != NULL) + auth_master_deinit(&conn); +} + +struct doveadm_cmd_ver2 doveadm_cmd_auth[] = { +{ + .cmd = cmd_auth_test, + .name = "auth test", + .usage = "[-a <auth socket path>] [-x <auth info>] [-M <master user>] <user> [<password>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('M', "master-user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "password", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .cmd = cmd_auth_login, + .name = "auth login", + .usage = "[-a <auth-login socket path>] [-m <auth-master socket path>] [-x <auth info>] [-M <master user>] <user> [<password>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "auth-login-socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('m', "auth-master-socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('M', "master-user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "password", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .cmd = cmd_auth_lookup, + .name = "auth lookup", + .usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] <user> [<user> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .cmd = cmd_auth_cache_flush, + .name = "auth cache flush", + .usage = "[-a <master socket path>] [<user> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .cmd = cmd_user, + .name = "user", + .usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] [-e <value>] [-u] <user mask> [<user mask> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('e', "expand-field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('u', "userdb-only", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +static void auth_cmd_help(struct doveadm_cmd_context *cctx) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth); i++) { + if (doveadm_cmd_auth[i].cmd == cctx->cmd->cmd) + help_ver2(&doveadm_cmd_auth[i]); + } + i_unreached(); +} + +void doveadm_register_auth_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_auth[i]); +} diff --git a/src/doveadm/doveadm-cmd.c b/src/doveadm/doveadm-cmd.c new file mode 100644 index 0000000..5de976d --- /dev/null +++ b/src/doveadm/doveadm-cmd.c @@ -0,0 +1,469 @@ +/* Copyright (c) 2009-2r016 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "str.h" +#include "net.h" +#include "doveadm.h" +#include "doveadm-cmd.h" + +#include <stdio.h> +#include <unistd.h> +#include <getopt.h> + +static struct doveadm_cmd_ver2 *doveadm_commands_ver2[] = { + &doveadm_cmd_mailbox_mutf7, + &doveadm_cmd_service_stop_ver2, + &doveadm_cmd_service_status_ver2, + &doveadm_cmd_sis_deduplicate, + &doveadm_cmd_sis_find, + &doveadm_cmd_process_status_ver2, + &doveadm_cmd_stop_ver2, + &doveadm_cmd_reload_ver2, + &doveadm_cmd_stats_dump_ver2, + &doveadm_cmd_stats_add_ver2, + &doveadm_cmd_stats_remove_ver2, + &doveadm_cmd_oldstats_dump_ver2, + &doveadm_cmd_oldstats_reset_ver2, + &doveadm_cmd_penalty_ver2, + &doveadm_cmd_kick_ver2, + &doveadm_cmd_who_ver2 +}; + +static const struct exit_code_str { + int code; + const char *str; +} exit_code_strings[] = { + { DOVEADM_EX_UNKNOWN, "UNKNOWN" }, + { EX_TEMPFAIL, "TEMPFAIL" }, + { EX_USAGE, "USAGE" }, + { EX_NOUSER, "NOUSER" }, + { EX_NOPERM, "NOPERM" }, + { EX_PROTOCOL, "PROTOCOL" }, + { EX_DATAERR, "DATAERR" }, + { DOVEADM_EX_NOREPLICATE, "NOREPLICATE" }, + { DOVEADM_EX_NOTFOUND, "NOTFOUND" } +}; + +ARRAY_TYPE(doveadm_cmd_ver2) doveadm_cmds_ver2; +ARRAY_DEFINE_TYPE(getopt_option_array, struct option); + +const char *doveadm_exit_code_to_str(int code) +{ + for(size_t i = 0; i < N_ELEMENTS(exit_code_strings); i++) { + const struct exit_code_str *ptr = &exit_code_strings[i]; + if (ptr->code == code) + return ptr->str; + } + return "UNKNOWN"; +} + +int doveadm_str_to_exit_code(const char *reason) +{ + for(size_t i = 0; i < N_ELEMENTS(exit_code_strings); i++) { + const struct exit_code_str *ptr = &exit_code_strings[i]; + if (strcmp(ptr->str, reason) == 0) + return ptr->code; + } + return DOVEADM_EX_UNKNOWN; +} + +void doveadm_cmd_register_ver2(struct doveadm_cmd_ver2 *cmd) +{ + if (cmd->cmd == NULL) { + if (cmd->mail_cmd != NULL) + cmd->cmd = doveadm_cmd_ver2_to_mail_cmd_wrapper; + else i_unreached(); + } + array_push_back(&doveadm_cmds_ver2, cmd); +} + +const struct doveadm_cmd_ver2 *doveadm_cmd_find_ver2(const char *cmd_name) +{ + const struct doveadm_cmd_ver2 *cmd; + + array_foreach(&doveadm_cmds_ver2, cmd) { + if (strcmp(cmd_name, cmd->name) == 0) + return cmd; + } + return NULL; +} + +const struct doveadm_cmd_ver2 * +doveadm_cmd_find_with_args_ver2(const char *cmd_name, int *argc, + const char *const *argv[]) +{ + int i, k; + const struct doveadm_cmd_ver2 *cmd; + const char *cptr; + + for (i = 0; i < *argc; i++) { + if (strcmp((*argv)[i], cmd_name) == 0) + break; + } + + i_assert(i != *argc); + + array_foreach(&doveadm_cmds_ver2, cmd) { + cptr = cmd->name; + /* cannot reuse i here because this needs be + done more than once */ + for (k = 0; *cptr != '\0' && i + k < *argc; k++) { + size_t alen = strlen((*argv)[i + k]); + /* make sure we don't overstep */ + if (strlen(cptr) < alen) + break; + /* did not match */ + if (strncmp(cptr, (*argv)[i+k], alen) != 0) + break; + /* do not accept abbreviations */ + if (cptr[alen] != ' ' && cptr[alen] != '\0') + break; + cptr += alen; + if (*cptr != '\0') + cptr++; /* consume space */ + } + /* name was fully consumed */ + if (*cptr == '\0') { + if (k > 1) { + *argc -= k-1; + *argv += k-1; + } + return cmd; + } + } + + return NULL; +} + +void doveadm_cmds_init(void) +{ + unsigned int i; + + i_array_init(&doveadm_cmds_ver2, 2); + + for (i = 0; i < N_ELEMENTS(doveadm_commands_ver2); i++) + doveadm_cmd_register_ver2(doveadm_commands_ver2[i]); + + doveadm_register_director_commands(); + doveadm_register_instance_commands(); + doveadm_register_proxy_commands(); + doveadm_register_log_commands(); + doveadm_register_replicator_commands(); + doveadm_register_dict_commands(); + doveadm_register_fs_commands(); +} + +void doveadm_cmds_deinit(void) +{ + array_free(&doveadm_cmds_ver2); +} + +static const struct doveadm_cmd_param * +doveadm_cmd_param_get(const struct doveadm_cmd_context *cctx, + const char *name) +{ + i_assert(cctx != NULL); + i_assert(cctx->argv != NULL); + for(int i = 0; i < cctx->argc; i++) { + if (strcmp(cctx->argv[i].name, name) == 0 && + cctx->argv[i].value_set) + return &cctx->argv[i]; + } + return NULL; +} + +bool doveadm_cmd_param_bool(const struct doveadm_cmd_context *cctx, + const char *name, bool *value_r) +{ + const struct doveadm_cmd_param *param; + if ((param = doveadm_cmd_param_get(cctx, name)) == NULL) + return FALSE; + + if (param->type == CMD_PARAM_BOOL) { + *value_r = param->value.v_bool; + return TRUE; + } + return FALSE; +} + +bool doveadm_cmd_param_int64(const struct doveadm_cmd_context *cctx, + const char *name, int64_t *value_r) +{ + const struct doveadm_cmd_param *param; + if ((param = doveadm_cmd_param_get(cctx, name)) == NULL) + return FALSE; + + if (param->type == CMD_PARAM_INT64) { + *value_r = param->value.v_int64; + return TRUE; + } + return FALSE; +} + +bool doveadm_cmd_param_str(const struct doveadm_cmd_context *cctx, + const char *name, const char **value_r) +{ + const struct doveadm_cmd_param *param; + if ((param = doveadm_cmd_param_get(cctx, name)) == NULL) + return FALSE; + + if (param->type == CMD_PARAM_STR) { + *value_r = param->value.v_string; + return TRUE; + } + return FALSE; +} + +bool doveadm_cmd_param_ip(const struct doveadm_cmd_context *cctx, + const char *name, struct ip_addr *value_r) +{ + const struct doveadm_cmd_param *param; + if ((param = doveadm_cmd_param_get(cctx, name)) == NULL) + return FALSE; + + if (param->type == CMD_PARAM_IP) { + memcpy(value_r, ¶m->value.v_ip, sizeof(struct ip_addr)); + return TRUE; + } + return FALSE; +} + +bool doveadm_cmd_param_array(const struct doveadm_cmd_context *cctx, + const char *name, const char *const **value_r) +{ + const struct doveadm_cmd_param *param; + unsigned int count; + + if ((param = doveadm_cmd_param_get(cctx, name)) == NULL) + return FALSE; + if (param->type == CMD_PARAM_ARRAY) { + *value_r = array_get(¶m->value.v_array, &count); + /* doveadm_cmd_params_null_terminate_arrays() should have been + called, which guarantees that we're NULL-terminated */ + i_assert((*value_r)[count] == NULL); + return TRUE; + } + return FALSE; +} + +bool doveadm_cmd_param_istream(const struct doveadm_cmd_context *cctx, + const char *name, struct istream **value_r) +{ + const struct doveadm_cmd_param *param; + if ((param = doveadm_cmd_param_get(cctx, name)) == NULL) + return FALSE; + + if (param->type == CMD_PARAM_ISTREAM) { + *value_r = param->value.v_istream; + return TRUE; + } + return FALSE; +} + +void doveadm_cmd_params_clean(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv) +{ + struct doveadm_cmd_param *param; + + array_foreach_modifiable(pargv, param) { + if (param->type == CMD_PARAM_ISTREAM && + param->value.v_istream != NULL) + i_stream_destroy(¶m->value.v_istream); + } + array_clear(pargv); +} + +void doveadm_cmd_params_null_terminate_arrays( + ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv) +{ + struct doveadm_cmd_param *param; + + array_foreach_modifiable(pargv, param) { + if (param->type == CMD_PARAM_ARRAY && + array_is_created(¶m->value.v_array)) { + array_append_zero(¶m->value.v_array); + array_pop_back(¶m->value.v_array); + } + } +} + +static void +doveadm_build_options(const struct doveadm_cmd_param par[], + string_t *shortopts, + ARRAY_TYPE(getopt_option_array) *longopts) +{ + for (size_t i = 0; par[i].name != NULL; i++) { + struct option longopt; + + i_zero(&longopt); + longopt.name = par[i].name; + if (par[i].short_opt != '\0') { + longopt.val = par[i].short_opt; + str_append_c(shortopts, par[i].short_opt); + if (par[i].type != CMD_PARAM_BOOL) + str_append_c(shortopts, ':'); + } + if (par[i].type != CMD_PARAM_BOOL) + longopt.has_arg = 1; + array_push_back(longopts, &longopt); + } + array_append_zero(longopts); +} + +static void +doveadm_fill_param(struct doveadm_cmd_param *param, + const char *value, pool_t pool) +{ + param->value_set = TRUE; + switch (param->type) { + case CMD_PARAM_BOOL: + param->value.v_bool = TRUE; + break; + case CMD_PARAM_INT64: + if (str_to_int64(value, ¶m->value.v_int64) != 0) + param->value_set = FALSE; + break; + case CMD_PARAM_IP: + if (net_addr2ip(value, ¶m->value.v_ip) != 0) + param->value_set = FALSE; + break; + case CMD_PARAM_STR: + param->value.v_string = p_strdup(pool, value); + break; + case CMD_PARAM_ARRAY: + if (!array_is_created(¶m->value.v_array)) + p_array_init(¶m->value.v_array, pool, 8); + const char *val = p_strdup(pool, value); + array_push_back(¶m->value.v_array, &val); + break; + case CMD_PARAM_ISTREAM: { + struct istream *is; + if (strcmp(value,"-") == 0) + is = i_stream_create_fd(STDIN_FILENO, IO_BLOCK_SIZE); + else + is = i_stream_create_file(value, IO_BLOCK_SIZE); + param->value.v_istream = is; + break; + } + } +} + +bool doveadm_cmd_try_run_ver2(const char *cmd_name, + int argc, const char *const argv[], + struct doveadm_cmd_context *cctx) +{ + const struct doveadm_cmd_ver2 *cmd; + + cmd = doveadm_cmd_find_with_args_ver2(cmd_name, &argc, &argv); + if (cmd == NULL) + return FALSE; + + cctx->cmd = cmd; + if (doveadm_cmd_run_ver2(argc, argv, cctx) < 0) + doveadm_exit_code = EX_USAGE; + return TRUE; +} + +static int +doveadm_cmd_process_options(int argc, const char *const argv[], + struct doveadm_cmd_context *cctx, pool_t pool, + ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv) +{ + struct doveadm_cmd_param *param; + ARRAY_TYPE(getopt_option_array) opts; + string_t *optbuf = str_new(pool, 64); + + p_array_init(&opts, pool, 4); + + // build parameters + if ((cctx->cmd->flags & CMD_FLAG_NO_UNORDERED_OPTIONS) != 0) + str_append_c(optbuf, '+'); + doveadm_build_options(cctx->cmd->parameters, optbuf, &opts); + + unsigned int pargc; + for (pargc = 0; cctx->cmd->parameters[pargc].name != NULL; pargc++) { + param = array_append_space(pargv); + memcpy(param, &cctx->cmd->parameters[pargc], + sizeof(struct doveadm_cmd_param)); + param->value_set = FALSE; + } + i_assert(pargc == array_count(&opts)-1); /* opts is NULL-terminated */ + + if ((cctx->cmd->flags & CMD_FLAG_NO_OPTIONS) != 0) { + /* process -parameters as if they were regular parameters */ + optind = 1; + return 0; + } + + int c, li; + while ((c = getopt_long(argc, (char *const *)argv, str_c(optbuf), + array_front(&opts), &li)) > -1) { + switch (c) { + case 0: + for (unsigned int i = 0; i < array_count(pargv); i++) { + const struct option *opt = array_idx(&opts, li); + param = array_idx_modifiable(pargv, i); + if (opt->name == param->name) + doveadm_fill_param(param, optarg, pool); + } + break; + case '?': + case ':': + doveadm_cmd_params_clean(pargv); + return -1; + default: + // hunt the option + for (unsigned int i = 0; i < pargc; i++) { + const struct option *longopt = + array_idx(&opts, i); + if (longopt->val == c) + doveadm_fill_param(array_idx_modifiable(pargv, i), + optarg, pool); + } + } + } + return 0; +} + +int doveadm_cmd_run_ver2(int argc, const char *const argv[], + struct doveadm_cmd_context *cctx) +{ + ARRAY_TYPE(doveadm_cmd_param_arr_t) pargv; + unsigned int pargc; + pool_t pool = pool_datastack_create(); + + p_array_init(&pargv, pool, 20); + if (doveadm_cmd_process_options(argc, argv, cctx, pool, &pargv) < 0) + return -1; + + /* process positional arguments */ + for (; optind < argc; optind++) { + struct doveadm_cmd_param *ptr; + bool found = FALSE; + array_foreach_modifiable(&pargv, ptr) { + if ((ptr->flags & CMD_PARAM_FLAG_POSITIONAL) != 0 && + (ptr->value_set == FALSE || + ptr->type == CMD_PARAM_ARRAY)) { + doveadm_fill_param(ptr, argv[optind], pool); + found = TRUE; + break; + } + } + if (!found) { + i_error("Extraneous arguments found: %s", + t_strarray_join(argv + optind, " ")); + doveadm_cmd_params_clean(&pargv); + return -1; + } + } + + doveadm_cmd_params_null_terminate_arrays(&pargv); + cctx->argv = array_get_modifiable(&pargv, &pargc); + cctx->argc = pargc; + + cctx->cmd->cmd(cctx); + + doveadm_cmd_params_clean(&pargv); + return 0; +} diff --git a/src/doveadm/doveadm-cmd.h b/src/doveadm/doveadm-cmd.h new file mode 100644 index 0000000..a988575 --- /dev/null +++ b/src/doveadm/doveadm-cmd.h @@ -0,0 +1,155 @@ +#ifndef DOVEADM_CMD_H +#define DOVEADM_CMD_H + +#include "net.h" + +#define DOVEADM_CMD_PARAMS_START .parameters = (const struct doveadm_cmd_param[]){ +#define DOVEADM_CMD_PARAM(optP, nameP, typeP, flagP ) { .short_opt = optP, .name = nameP, .type = typeP, .flags = flagP }, +#define DOVEADM_CMD_PARAMS_END { .short_opt = '\0', .name = NULL, .type = CMD_PARAM_BOOL, .flags = CMD_PARAM_FLAG_NONE } } + +struct doveadm_cmd_ver2; +struct doveadm_cmd_context; +struct doveadm_mail_cmd_context; + +typedef void doveadm_command_t(int argc, char *argv[]); + +typedef enum { + CMD_PARAM_BOOL = 0, /* value will contain 1 (not pointer) */ + CMD_PARAM_INT64, /* ditto but contains number (not pointer) */ + CMD_PARAM_IP, /* value contains struct ip_addr */ + CMD_PARAM_STR, /* value contains const char* */ + CMD_PARAM_ARRAY, /* value contains const char*[] */ + CMD_PARAM_ISTREAM /* value contains struct istream* */ +} doveadm_cmd_param_t; + +typedef enum { + CMD_PARAM_FLAG_NONE = 0x0, + CMD_PARAM_FLAG_POSITIONAL = 0x1, + CMD_PARAM_FLAG_DO_NOT_EXPOSE = 0x2, +} doveadm_cmd_param_flag_t; + +typedef enum { + CMD_FLAG_NONE = 0x0, + CMD_FLAG_HIDDEN = 0x1, + CMD_FLAG_NO_PRINT = 0x2, + /* Don't parse any -options for the command. */ + CMD_FLAG_NO_OPTIONS = 0x4, + /* Prevent GNU getopt() from finding options after the first + non-option is seen (e.g. "-1 arg -2" would parse -1 but not -2 + as option). */ + CMD_FLAG_NO_UNORDERED_OPTIONS = 0x8, +} doveadm_cmd_flag_t; + +struct doveadm_cmd_param { + char short_opt; + const char *name; + doveadm_cmd_param_t type; + bool value_set; + struct { + bool v_bool; + int64_t v_int64; + const char* v_string; + ARRAY_TYPE(const_string) v_array; + struct ip_addr v_ip; + struct istream* v_istream; + } value; + doveadm_cmd_param_flag_t flags; +}; +ARRAY_DEFINE_TYPE(doveadm_cmd_param_arr_t, struct doveadm_cmd_param); + +typedef void doveadm_command_ver2_t(struct doveadm_cmd_context *cctx); + +struct doveadm_cmd_ver2 { + doveadm_command_ver2_t *cmd; + struct doveadm_mail_cmd_context *(*mail_cmd)(void); + const char *name; + const char *usage; + doveadm_cmd_flag_t flags; + const struct doveadm_cmd_param *parameters; +}; + +struct doveadm_cmd_context { + const struct doveadm_cmd_ver2 *cmd; /* for help */ + + int argc; + const struct doveadm_cmd_param *argv; + + const char *username; + struct ip_addr local_ip, remote_ip; + in_port_t local_port, remote_port; + + enum doveadm_client_type conn_type; + struct istream *input; + struct ostream *output; +}; + +ARRAY_DEFINE_TYPE(doveadm_cmd_ver2, struct doveadm_cmd_ver2); +extern ARRAY_TYPE(doveadm_cmd_ver2) doveadm_cmds_ver2; + +void doveadm_register_auth_commands(void); +void doveadm_register_auth_server_commands(void); +void doveadm_register_director_commands(void); +void doveadm_register_proxy_commands(void); +void doveadm_register_log_commands(void); +void doveadm_register_instance_commands(void); +void doveadm_register_mount_commands(void); +void doveadm_register_replicator_commands(void); +void doveadm_register_dict_commands(void); +void doveadm_register_fs_commands(void); + +void doveadm_cmds_init(void); +void doveadm_cmds_deinit(void); + +void doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx); + +void doveadm_cmd_register_ver2(struct doveadm_cmd_ver2 *cmd); +const struct doveadm_cmd_ver2 * +doveadm_cmd_find_with_args_ver2(const char *cmd_name, int *argc, + const char *const *argv[]); +const struct doveadm_cmd_ver2 *doveadm_cmd_find_ver2(const char *cmd_name); +/* Returns FALSE if cmd_name doesn't exist, TRUE if it exists. */ +bool doveadm_cmd_try_run_ver2(const char *cmd_name, + int argc, const char *const argv[], + struct doveadm_cmd_context *cctx); +/* Returns 0 if success, -1 if parameters were invalid. */ +int doveadm_cmd_run_ver2(int argc, const char *const argv[], + struct doveadm_cmd_context *cctx); + +bool doveadm_cmd_param_bool(const struct doveadm_cmd_context *cctx, + const char *name, bool *value_r); +bool doveadm_cmd_param_int64(const struct doveadm_cmd_context *cctx, + const char *name, int64_t *value_r); +bool doveadm_cmd_param_str(const struct doveadm_cmd_context *cctx, + const char *name, const char **value_r); +bool doveadm_cmd_param_ip(const struct doveadm_cmd_context *cctx, + const char *name, struct ip_addr *value_r); +bool doveadm_cmd_param_array(const struct doveadm_cmd_context *cctx, + const char *name, const char *const **value_r); +bool doveadm_cmd_param_istream(const struct doveadm_cmd_context *cctx, + const char *name, struct istream **value_r); + +void doveadm_cmd_params_clean(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv); +void doveadm_cmd_params_null_terminate_arrays(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv); + +extern struct doveadm_cmd_ver2 doveadm_cmd_dump; +extern struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_service_status_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_process_status_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_stop_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_reload_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_stats_dump_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_stats_add_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_stats_remove_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_mutf7; +extern struct doveadm_cmd_ver2 doveadm_cmd_oldstats_reset_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_oldstats_dump_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_oldstats_top_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_penalty_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_pw; +extern struct doveadm_cmd_ver2 doveadm_cmd_kick_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_who_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_sis_deduplicate; +extern struct doveadm_cmd_ver2 doveadm_cmd_sis_find; +extern struct doveadm_cmd_ver2 doveadm_cmd_zlibconnect; + +#endif diff --git a/src/doveadm/doveadm-dict.c b/src/doveadm/doveadm-dict.c new file mode 100644 index 0000000..12b901a --- /dev/null +++ b/src/doveadm/doveadm-dict.c @@ -0,0 +1,329 @@ +/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dict.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> + +static int +cmd_dict_init_full(struct doveadm_cmd_context *cctx, + doveadm_command_ver2_t *cmd ATTR_UNUSED, enum dict_iterate_flags *iter_flags, + struct dict **dict_r, struct dict_op_settings *dopset_r) +{ + struct dict_settings dict_set; + struct dict *dict; + bool set = FALSE; + const char *dict_uri, *error, *key, *username = ""; + i_zero(dopset_r); + + if (doveadm_cmd_param_bool(cctx, "exact", &set) && set) + *iter_flags |= DICT_ITERATE_FLAG_EXACT_KEY; + if (doveadm_cmd_param_bool(cctx, "recurse", &set) && set) + *iter_flags |= DICT_ITERATE_FLAG_RECURSE; + if (doveadm_cmd_param_bool(cctx, "no-value", &set) && set) + *iter_flags |= DICT_ITERATE_FLAG_NO_VALUE; + (void)doveadm_cmd_param_str(cctx, "user", &username); + dopset_r->username = username; + + if (!doveadm_cmd_param_str(cctx, "dict-uri", &dict_uri)) { + i_error("dictionary URI must be specified"); + doveadm_exit_code = EX_USAGE; + return -1; + } + + if (!doveadm_cmd_param_str(cctx, "prefix", &key) && + !doveadm_cmd_param_str(cctx, "key", &key)) + key = ""; + + if (!str_begins(key, DICT_PATH_PRIVATE) && + !str_begins(key, DICT_PATH_SHARED)) { + i_error("Key must begin with '"DICT_PATH_PRIVATE + "' or '"DICT_PATH_SHARED"': %s", key); + doveadm_exit_code = EX_USAGE; + return -1; + } + if (username[0] == '\0' && + str_begins(key, DICT_PATH_PRIVATE)) { + i_error("-u must be specified for "DICT_PATH_PRIVATE" keys"); + doveadm_exit_code = EX_USAGE; + return -1; + } + + dict_drivers_register_builtin(); + i_zero(&dict_set); + dict_set.base_dir = doveadm_settings->base_dir; + if (dict_init(dict_uri, &dict_set, &dict, &error) < 0) { + i_error("dict_init(%s) failed: %s", dict_uri, error); + doveadm_exit_code = EX_TEMPFAIL; + return -1; + } + *dict_r = dict; + return 0; +} + +static int +cmd_dict_init(struct doveadm_cmd_context *cctx, + doveadm_command_ver2_t *cmd, struct dict **dict_r, + struct dict_op_settings *set_r) +{ + enum dict_iterate_flags iter_flags = 0; + return cmd_dict_init_full(cctx, cmd, &iter_flags, dict_r, set_r); +} + +struct doveadm_dict_ctx { + pool_t pool; + int ret; + const char *const *values; + const char *error; +}; + +static void dict_lookup_callback(const struct dict_lookup_result *result, + struct doveadm_dict_ctx *ctx) +{ + ctx->ret = result->ret; + ctx->values = result->values == NULL ? NULL : + p_strarray_dup(ctx->pool, result->values); + ctx->error = p_strdup(ctx->pool, result->error); +} + +static void cmd_dict_get(struct doveadm_cmd_context *cctx) +{ + struct doveadm_dict_ctx ctx; + struct dict *dict; + const char *key; + struct dict_op_settings set; + + if (!doveadm_cmd_param_str(cctx, "key", &key)) { + i_error("dict-get: Missing key"); + doveadm_exit_code = EX_USAGE; + return; + } + + if (cmd_dict_init(cctx, cmd_dict_get, &dict, &set) < 0) + return; + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("value", "", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + + i_zero(&ctx); + ctx.pool = pool_alloconly_create("doveadm dict lookup", 512); + ctx.ret = -2; + dict_lookup_async(dict, &set, key, dict_lookup_callback, &ctx); + while (ctx.ret == -2) + dict_wait(dict); + if (ctx.ret < 0) { + i_error("dict_lookup(%s) failed: %s", key, ctx.error); + doveadm_exit_code = EX_TEMPFAIL; + } else if (ctx.ret == 0) { + i_error("%s doesn't exist", key); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else { + unsigned int i, values_count = str_array_length(ctx.values); + + for (i = 1; i < values_count; i++) + doveadm_print_header("value", "", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + for (i = 0; i < values_count; i++) + doveadm_print(ctx.values[i]); + } + pool_unref(&ctx.pool); + dict_deinit(&dict); +} + +static void cmd_dict_set(struct doveadm_cmd_context *cctx) +{ + struct dict *dict; + struct dict_transaction_context *trans; + const char *error; + const char *key, *value = ""; + struct dict_op_settings set; + + if (!doveadm_cmd_param_str(cctx, "key", &key) || + !doveadm_cmd_param_str(cctx, "value", &value)) { + i_error("dict set: Missing parameters"); + doveadm_exit_code = EX_USAGE; + return; + } + + if (cmd_dict_init(cctx, cmd_dict_set, &dict, &set) < 0) + return; + + trans = dict_transaction_begin(dict, &set); + dict_set(trans, key, value); + if (dict_transaction_commit(&trans, &error) <= 0) { + i_error("dict_transaction_commit() failed: %s", error); + doveadm_exit_code = EX_TEMPFAIL; + } + dict_deinit(&dict); +} + +static void cmd_dict_unset(struct doveadm_cmd_context *cctx) +{ + struct dict *dict; + struct dict_transaction_context *trans; + const char *error; + const char *key; + struct dict_op_settings set; + + if (!doveadm_cmd_param_str(cctx, "key", &key)) { + i_error("dict unset: Missing key"); + doveadm_exit_code = EX_USAGE; + return; + } + + if (cmd_dict_init(cctx, cmd_dict_unset, &dict, &set) < 0) + return; + + trans = dict_transaction_begin(dict, &set); + dict_unset(trans, key); + if (dict_transaction_commit(&trans, &error) <= 0) { + i_error("dict_transaction_commit() failed: %s", error); + doveadm_exit_code = EX_TEMPFAIL; + } + dict_deinit(&dict); +} + +static void cmd_dict_inc(struct doveadm_cmd_context *cctx) +{ + struct dict *dict; + struct dict_transaction_context *trans; + const char *error; + const char *key; + int64_t diff; + int ret; + struct dict_op_settings set; + + if (!doveadm_cmd_param_str(cctx, "key", &key) || + !doveadm_cmd_param_int64(cctx, "difference", &diff)) { + i_error("dict-inc: Missing parameters"); + doveadm_exit_code = EX_USAGE; + return; + } + + if (cmd_dict_init(cctx, cmd_dict_inc, &dict, &set) < 0) + return; + + trans = dict_transaction_begin(dict, &set); + dict_atomic_inc(trans, key, diff); + ret = dict_transaction_commit(&trans, &error); + if (ret < 0) { + i_error("dict_transaction_commit() failed: %s", error); + doveadm_exit_code = EX_TEMPFAIL; + } else if (ret == 0) { + i_error("%s doesn't exist", key); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } + dict_deinit(&dict); +} + +static void cmd_dict_iter(struct doveadm_cmd_context *cctx) +{ + struct dict *dict; + struct dict_iterate_context *iter; + enum dict_iterate_flags iter_flags = 0; + const char *prefix, *key, *const *values, *error; + bool header_printed = FALSE; + struct dict_op_settings set; + + if (!doveadm_cmd_param_str(cctx, "prefix", &prefix)) { + i_error("dict-iter: Missing prefix"); + doveadm_exit_code = EX_USAGE; + return; + } + + if (cmd_dict_init_full(cctx, cmd_dict_iter, &iter_flags, &dict, &set) < 0) + return; + + doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); + doveadm_print_header_simple("key"); + if ((iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) + doveadm_print_header_simple("value"); + + iter = dict_iterate_init(dict, &set, prefix, iter_flags); + while (dict_iterate_values(iter, &key, &values)) { + unsigned int values_count = str_array_length(values); + if (!header_printed) { + for (unsigned int i = 1; i < values_count; i++) + doveadm_print_header_simple("value"); + header_printed = TRUE; + } + doveadm_print(key); + if ((iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) { + for (unsigned int i = 0; i < values_count; i++) + doveadm_print(values[i]); + } + } + if (dict_iterate_deinit(&iter, &error) < 0) { + i_error("dict_iterate_deinit(%s) failed: %s", prefix, error); + doveadm_exit_code = EX_TEMPFAIL; + } + dict_deinit(&dict); +} + +static struct doveadm_cmd_ver2 doveadm_cmd_dict[] = { +{ + .name = "dict get", + .cmd = cmd_dict_get, + .usage = "[-u <user>] <dict uri> <key>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "dict set", + .cmd = cmd_dict_set, + .usage = "[-u <user>] <dict uri> <key> <value>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "value", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "dict unset", + .cmd = cmd_dict_unset, + .usage = "[-u <user>] <dict uri> <key>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "dict inc", + .cmd = cmd_dict_inc, + .usage = "[-u <user>] <dict uri> <key> <diff>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "difference", CMD_PARAM_INT64, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "dict iter", + .cmd = cmd_dict_iter, + .usage = "[-u <user>] [-1RV] <dict uri> <prefix>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('1', "exact", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('R', "recurse", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('V', "no-value", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "prefix", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +void doveadm_register_dict_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_dict); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_dict[i]); +} diff --git a/src/doveadm/doveadm-director.c b/src/doveadm/doveadm-director.c new file mode 100644 index 0000000..167b091 --- /dev/null +++ b/src/doveadm/doveadm-director.c @@ -0,0 +1,1084 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "md5.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "net.h" +#include "istream.h" +#include "write-full.h" +#include "master-service.h" +#include "auth-master.h" +#include "mail-user-hash.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> + +struct director_context { + const char *socket_path; + const char *users_path; + const char *tag; + const char *user; + const char *host; + const char *ip; + const char *port; + const char *vhost_count; + const char *passdb_field; + + struct istream *users_input; + struct istream *input; + bool explicit_socket_path; + bool hash_map, user_map, force_flush; + int64_t max_parallel; +}; + +struct user_list { + struct user_list *next; + const char *name; +}; + +HASH_TABLE_DEFINE_TYPE(user_list, void *, struct user_list *); + +static void director_cmd_help(const struct doveadm_cmd_ver2 *); +static int director_get_host(const char *host, struct ip_addr **ips_r, + unsigned int *ips_count_r) ATTR_WARN_UNUSED_RESULT; +static void +director_send(struct director_context *ctx, const char *data) +{ + if (write_full(i_stream_get_fd(ctx->input), data, strlen(data)) < 0) + i_fatal("write(%s) failed: %m", ctx->socket_path); +} + +static void director_connect(struct director_context *ctx) +{ +#define DIRECTOR_HANDSHAKE "VERSION\tdirector-doveadm\t1\t0\n" + const char *line; + int fd; + + fd = doveadm_connect(ctx->socket_path); + net_set_nonblock(fd, FALSE); + + ctx->input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + director_send(ctx, DIRECTOR_HANDSHAKE); + + alarm(5); + line = i_stream_read_next_line(ctx->input); + alarm(0); + if (line == NULL) { + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } else if (ctx->input->eof) { + i_fatal("%s disconnected", ctx->socket_path); + } else { + i_fatal("read(%s) timed out (is director configured?)", + ctx->socket_path); + } + } + if (!version_string_verify(line, "director-doveadm", 1)) { + i_fatal_status(EX_PROTOCOL, + "%s not a compatible director-doveadm socket", + ctx->socket_path); + } +} + +static void director_disconnect(struct director_context *ctx) +{ + if (ctx->input != NULL) { + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } + i_stream_destroy(&ctx->input); + } +} + +static struct director_context * +cmd_director_init(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + ctx = t_new(struct director_context, 1); + if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx->socket_path))) + ctx->socket_path = t_strconcat(doveadm_settings->base_dir, + "/director-admin", NULL); + else + ctx->explicit_socket_path = TRUE; + if (!doveadm_cmd_param_bool(cctx, "user-map", &(ctx->user_map))) + ctx->user_map = FALSE; + if (!doveadm_cmd_param_bool(cctx, "hash-map", &(ctx->hash_map))) + ctx->hash_map = FALSE; + if (!doveadm_cmd_param_bool(cctx, "force-flush", &(ctx->force_flush))) + ctx->force_flush = FALSE; + if (!doveadm_cmd_param_istream(cctx, "users-file", &(ctx->users_input))) + ctx->users_input = NULL; + if (!doveadm_cmd_param_str(cctx, "tag", &(ctx->tag))) + ctx->tag = NULL; + if (!doveadm_cmd_param_str(cctx, "user", &(ctx->user))) + ctx->user = NULL; + if (!doveadm_cmd_param_str(cctx, "host", &(ctx->host))) + ctx->host = NULL; + if (!doveadm_cmd_param_str(cctx, "ip", &(ctx->ip))) + ctx->ip = NULL; + if (!doveadm_cmd_param_str(cctx, "port", &(ctx->port))) + ctx->port = NULL; + if (!doveadm_cmd_param_str(cctx, "vhost-count", &(ctx->vhost_count))) + ctx->vhost_count = NULL; + if (!doveadm_cmd_param_str(cctx, "passdb-field", &(ctx->passdb_field))) + ctx->passdb_field = NULL; + if (!doveadm_cmd_param_int64(cctx, "max-parallel", &(ctx->max_parallel))) + ctx->max_parallel = 0; + if (!ctx->user_map) + director_connect(ctx); + return ctx; +} + +static void director_disconnected(struct director_context *ctx) +{ + i_assert(ctx->input->eof); + if (ctx->input->stream_errno != 0) { + i_error("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } else { + i_error("%s unexpectedly disconnected", ctx->socket_path); + } + doveadm_exit_code = EX_TEMPFAIL; +} + +static void +cmd_director_status_user(struct director_context *ctx) +{ + const char *line, *const *args; + time_t expires; + + director_send(ctx, t_strdup_printf("USER-LOOKUP\t%s\t%s\n", ctx->user, + ctx->tag != NULL ? ctx->tag : "")); + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + director_disconnected(ctx); + return; + } + + args = t_strsplit_tabescaped(line); + if (str_array_length(args) != 4 || + str_to_time(args[1], &expires) < 0) { + i_error("Invalid reply from director"); + doveadm_exit_code = EX_PROTOCOL; + return; + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + + doveadm_print_header_simple("status"); + doveadm_print_header_simple("expires"); + doveadm_print_header_simple("hashed"); + doveadm_print_header_simple("initial-config"); + + doveadm_print_formatted_set_format("Current: %{status} (expires %{expires})\n" \ + "Hashed: %{hashed}\n" \ + "Initial config: %{initial-config}\n"); + + if (args[0][0] != '\0') { + doveadm_print(args[0]); + doveadm_print(unixdate2str(expires)); + } else { + doveadm_print("n/a"); + doveadm_print("-1"); + } + doveadm_print(args[2]); + doveadm_print(args[3]); + + director_disconnect(ctx); +} + +static void cmd_director_status(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + const char *line, *const *args; + + ctx = cmd_director_init(cctx); + if (ctx->user != NULL) { + cmd_director_status_user(ctx); + return; + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header_simple("mail server ip"); + doveadm_print_header_simple("tag"); + doveadm_print_header_simple("vhosts"); + doveadm_print_header_simple("state"); + doveadm_print_header("state-changed", "state changed", 0); + doveadm_print_header_simple("users"); + + director_send(ctx, "HOST-LIST\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + unsigned int arg_count; + time_t ts; + + args = t_strsplit_tabescaped(line); + arg_count = str_array_length(args); + if (arg_count >= 6) { + /* ip vhosts users tag updown updown-ts */ + doveadm_print(args[0]); + doveadm_print(args[3]); + doveadm_print(args[1]); + doveadm_print(args[4][0] == 'D' ? "down" : "up"); + if (str_to_time(args[5], &ts) < 0 || + ts <= 0) + doveadm_print("-"); + else + doveadm_print(unixdate2str(ts)); + doveadm_print(args[2]); + } + } T_END; + } + if (line == NULL) + director_disconnected(ctx); + director_disconnect(ctx); +} + +static bool user_hash_expand(const char *username, unsigned int *hash_r) +{ + const char *error; + + if (!mail_user_hash(username, doveadm_settings->director_username_hash, + hash_r, &error)) { + i_error("Failed to expand director_username_hash=%s: %s", + doveadm_settings->director_username_hash, error); + return FALSE; + } + return TRUE; +} + +static void +user_list_add(const char *username, pool_t pool, + HASH_TABLE_TYPE(user_list) users) +{ + struct user_list *user, *old_user; + unsigned int user_hash; + + if (!user_hash_expand(username, &user_hash)) + return; + + user = p_new(pool, struct user_list, 1); + user->name = p_strdup(pool, username); + + old_user = hash_table_lookup(users, POINTER_CAST(user_hash)); + if (old_user != NULL) + user->next = old_user; + hash_table_update(users, POINTER_CAST(user_hash), user); +} + +static void ATTR_NULL(1) +userdb_get_user_list(const char *auth_socket_path, pool_t pool, + HASH_TABLE_TYPE(user_list) users) +{ + struct auth_master_user_list_ctx *ctx; + struct auth_master_connection *conn; + const char *username; + + if (auth_socket_path == NULL) { + auth_socket_path = t_strconcat(doveadm_settings->base_dir, + "/auth-userdb", NULL); + } + + conn = auth_master_init(auth_socket_path, 0); + ctx = auth_master_user_list_init(conn, "", NULL); + while ((username = auth_master_user_list_next(ctx)) != NULL) + user_list_add(username, pool, users); + if (auth_master_user_list_deinit(&ctx) < 0) { + i_error("user listing failed"); + doveadm_exit_code = EX_TEMPFAIL; + } + auth_master_deinit(&conn); +} + +static void +user_file_get_user_list(struct istream *input, pool_t pool, + HASH_TABLE_TYPE(user_list) users) +{ + const char *username; + + while ((username = i_stream_read_next_line(input)) != NULL) + user_list_add(username, pool, users); +} + +static int director_get_host(const char *host, struct ip_addr **ips_r, + unsigned int *ips_count_r) +{ + struct ip_addr ip; + int ret = 0; + + if (net_addr2ip(host, &ip) == 0) { + *ips_r = t_new(struct ip_addr, 1); + **ips_r = ip; + *ips_count_r = 1; + } else { + ret = net_gethostbyname(host, ips_r, ips_count_r); + if (ret != 0) { + i_error("gethostname(%s) failed: %s", host, + net_gethosterror(ret)); + doveadm_exit_code = EX_TEMPFAIL; + return ret; + } + } + + return ret; +} + +static bool ip_find(const struct ip_addr *ips, unsigned int ips_count, + const struct ip_addr *match_ip) +{ + unsigned int i; + + for (i = 0; i < ips_count; i++) { + if (net_ip_compare(&ips[i], match_ip)) + return TRUE; + } + return FALSE; +} + +static void cmd_director_map(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + const char *line, *const *args; + struct ip_addr *ips, user_ip; + pool_t pool; + HASH_TABLE_TYPE(user_list) users; + struct user_list *user; + unsigned int ips_count, user_hash; + time_t expires; + + ctx = cmd_director_init(cctx); + + if ((ctx->hash_map || ctx->user_map) && ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } + + if (ctx->user_map) { + /* user -> hash mapping */ + if (user_hash_expand(ctx->host, &user_hash)) { + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("hash", "hash", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + doveadm_print(t_strdup_printf("%u", user_hash)); + } + director_disconnect(ctx); + return; + } + + if (ctx->host == NULL || ctx->hash_map) + ips_count = 0; + else if (director_get_host(ctx->host, &ips, &ips_count) != 0) { + director_disconnect(ctx); + return; + } + + pool = pool_alloconly_create("director map users", 1024*128); + hash_table_create_direct(&users, pool, 0); + if (ctx->users_input == NULL) + userdb_get_user_list(NULL, pool, users); + else + user_file_get_user_list(ctx->users_input, pool, users); + + if (ctx->hash_map) { + /* hash -> usernames mapping */ + if (str_to_uint(ctx->host, &user_hash) < 0) + i_fatal("Invalid username hash: %s", ctx->host); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + user = hash_table_lookup(users, POINTER_CAST(user_hash)); + for (; user != NULL; user = user->next) + doveadm_print(user->name); + goto deinit; + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("hash"); + doveadm_print_header_simple("mail server ip"); + doveadm_print_header_simple("expire time"); + + if (ips_count != 1) + director_send(ctx, "USER-LIST\n"); + else { + director_send(ctx, t_strdup_printf( + "USER-LIST\t%s\n", net_ip2addr(&ips[0]))); + } + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + args = t_strsplit_tabescaped(line); + if (str_array_length(args) < 3 || + str_to_uint(args[0], &user_hash) < 0 || + str_to_time(args[1], &expires) < 0 || + net_addr2ip(args[2], &user_ip) < 0) { + i_error("Invalid USER-LIST reply: %s", line); + doveadm_exit_code = EX_PROTOCOL; + } else if (ips_count == 0 || + ip_find(ips, ips_count, &user_ip)) { + user = hash_table_lookup(users, + POINTER_CAST(user_hash)); + if (user == NULL) { + doveadm_print("<unknown>"); + doveadm_print(args[0]); + doveadm_print(args[2]); + doveadm_print(unixdate2str(expires)); + } + for (; user != NULL; user = user->next) { + doveadm_print(user->name); + doveadm_print(args[0]); + doveadm_print(args[2]); + doveadm_print(unixdate2str(expires)); + } + } + } T_END; + } + if (line == NULL) + director_disconnected(ctx); +deinit: + director_disconnect(ctx); + hash_table_destroy(&users); + pool_unref(&pool); +} + +static void +cmd_director_add_or_update(struct doveadm_cmd_context *cctx, bool update) +{ + const char *director_cmd = update ? "HOST-UPDATE" : "HOST-SET"; + struct director_context *ctx; + struct ip_addr *ips; + unsigned int i, ips_count, vhost_count = UINT_MAX; + const char *line, *host; + string_t *cmd; + + ctx = cmd_director_init(cctx); + if (ctx->tag != NULL && ctx->tag[0] == '\0') + ctx->tag = NULL; + if (ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } + if (ctx->vhost_count != NULL) { + if (str_to_uint(ctx->vhost_count, &vhost_count) < 0) { + director_cmd_help(cctx->cmd); + return; + } + } else if (update) { + director_cmd_help(cctx->cmd); + return; + } + if (str_to_uint(ctx->host, &i) == 0) { + /* host is a number. this would translate to an IP address, + which is probably a mistake. */ + i_error("Invalid host '%s'", ctx->host); + director_cmd_help(cctx->cmd); + return; + } + + host = ctx->host; + if (ctx->tag == NULL) { + ctx->tag = strchr(ctx->host, '@'); + if (ctx->tag != NULL) + host = t_strdup_until(ctx->host, ctx->tag++); + } + if (director_get_host(host, &ips, &ips_count) != 0) { + director_disconnect(ctx); + return; + } + cmd = t_str_new(128); + for (i = 0; i < ips_count; i++) { + str_truncate(cmd, 0); + str_printfa(cmd, "%s\t%s", director_cmd, net_ip2addr(&ips[i])); + if (ctx->tag != NULL) + str_printfa(cmd, "@%s", ctx->tag); + if (vhost_count != UINT_MAX) + str_printfa(cmd, "\t%u", vhost_count); + str_append_c(cmd, '\n'); + director_send(ctx, str_c(cmd)); + } + for (i = 0; i < ips_count; i++) { + line = i_stream_read_next_line(ctx->input); + if (line == NULL) + director_disconnected(ctx); + else if (strcmp(line, "OK") != 0) { + i_error("%s: %s", net_ip2addr(&ips[i]), + strcmp(line, "NOTFOUND") == 0 ? + "doesn't exist" : line); + doveadm_exit_code = EX_TEMPFAIL; + } else if (doveadm_verbose) { + i_info("%s: OK", net_ip2addr(&ips[i])); + } + } + director_disconnect(ctx); +} + +static void cmd_director_add(struct doveadm_cmd_context *cctx) +{ + cmd_director_add_or_update(cctx, FALSE); +} + +static void cmd_director_update(struct doveadm_cmd_context *cctx) +{ + cmd_director_add_or_update(cctx, TRUE); +} + +static void +cmd_director_ipcmd(const char *cmd_name, const char *success_result, + struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + struct ip_addr *ips; + unsigned int i, ips_count; + const char *host, *line; + + ctx = cmd_director_init(cctx); + host = ctx->host; + if (host == NULL) { + director_cmd_help(cctx->cmd); + return; + } + + if (director_get_host(host, &ips, &ips_count) != 0) { + director_disconnect(ctx); + return; + } + for (i = 0; i < ips_count; i++) { + director_send(ctx, t_strdup_printf( + "%s\t%s\n", cmd_name, net_ip2addr(&ips[i]))); + } + for (i = 0; i < ips_count; i++) { + line = i_stream_read_next_line(ctx->input); + if (line != NULL && strcmp(line, "NOTFOUND") == 0) { + i_error("%s: doesn't exist", + net_ip2addr(&ips[i])); + if (doveadm_exit_code == 0) + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else if (line == NULL) { + director_disconnected(ctx); + } else if (strcmp(line, "OK") != 0) { + i_error("%s: %s", net_ip2addr(&ips[i]), line); + doveadm_exit_code = EX_TEMPFAIL; + } else if (doveadm_verbose) { + i_info("%s: %s", net_ip2addr(&ips[i]), success_result); + } + } + director_disconnect(ctx); +} + +static void cmd_director_remove(struct doveadm_cmd_context *cctx) +{ + cmd_director_ipcmd("HOST-REMOVE", "removed", cctx); +} + +static void cmd_director_up(struct doveadm_cmd_context *cctx) +{ + cmd_director_ipcmd("HOST-UP", "up", cctx); +} + +static void cmd_director_down(struct doveadm_cmd_context *cctx) +{ + cmd_director_ipcmd("HOST-DOWN", "down", cctx); +} + +static void cmd_director_move(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + struct ip_addr *ips; + unsigned int ips_count, user_hash; + const char *line, *ip_str; + + ctx = cmd_director_init(cctx); + if (ctx->user == NULL || ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } + + if (!user_hash_expand(ctx->user, &user_hash) || + director_get_host(ctx->host, &ips, &ips_count) != 0) { + director_disconnect(ctx); + return; + } + ip_str = net_ip2addr(&ips[0]); + director_send(ctx, t_strdup_printf( + "USER-MOVE\t%u\t%s\n", user_hash, ip_str)); + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + director_disconnected(ctx); + } else if (strcmp(line, "OK") == 0) { + if (doveadm_verbose) + i_info("User hash %u moved to %s", user_hash, ip_str); + } else if (strcmp(line, "NOTFOUND") == 0) { + i_error("Host '%s' doesn't exist", ip_str); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else if (strcmp(line, "TRYAGAIN") == 0) { + i_error("User is already being moved, " + "wait a while for it to be finished"); + doveadm_exit_code = EX_TEMPFAIL; + } else { + i_error("failed: %s", line); + doveadm_exit_code = EX_TEMPFAIL; + } + director_disconnect(ctx); +} + +static void cmd_director_kick(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + const char *line; + string_t *cmd = t_str_new(64); + + ctx = cmd_director_init(cctx); + if (ctx->user == NULL) { + director_cmd_help(cctx->cmd); + return; + } + + if (ctx->passdb_field == NULL) { + str_append(cmd, "USER-KICK\t"); + str_append_tabescaped(cmd, ctx->user); + str_append_c(cmd, '\n'); + } else { + str_append(cmd, "USER-KICK-ALT\t"); + str_append_tabescaped(cmd, ctx->passdb_field); + str_append_c(cmd, '\t'); + str_append_tabescaped(cmd, ctx->user); + str_append_c(cmd, '\n'); + } + director_send(ctx, str_c(cmd)); + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + director_disconnected(ctx); + } else if (strcmp(line, "OK") == 0) { + if (doveadm_verbose) + i_info("User %s kicked", ctx->user); + } else { + i_error("failed: %s", line); + doveadm_exit_code = EX_TEMPFAIL; + } + director_disconnect(ctx); +} + +static void cmd_director_flush_all(struct director_context *ctx) +{ + const char *line; + + if (ctx->force_flush) + line = "HOST-FLUSH\n"; + else if (ctx->max_parallel > 0) { + line = t_strdup_printf("HOST-RESET-USERS\t\t%lld\n", + (long long)ctx->max_parallel); + } else { + line = "HOST-RESET-USERS\n"; + } + director_send(ctx, line); + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + director_disconnected(ctx); + } else if (strcmp(line, "OK") != 0) { + i_error("failed: %s", line); + doveadm_exit_code = EX_TEMPFAIL; + } else if (doveadm_verbose) + i_info("flushed"); + director_disconnect(ctx); +} + +static void cmd_director_flush(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + struct ip_addr *ips; + unsigned int i, ips_count; + struct ip_addr ip; + const char *line; + string_t *cmd; + + ctx = cmd_director_init(cctx); + if (ctx->host == NULL) { + director_cmd_help(cctx->cmd); + return; + } + + if (strcmp(ctx->host, "all") == 0) { + cmd_director_flush_all(ctx); + return; + } + if (net_addr2ip(ctx->host, &ip) == 0) { + ips = &ip; + ips_count = 1; + } else if (director_get_host(ctx->host, &ips, &ips_count) != 0) { + director_disconnect(ctx); + return; + } + + cmd = t_str_new(64); + for (i = 0; i < ips_count; i++) { + ip = ips[i]; + str_truncate(cmd, 0); + if (ctx->force_flush) + str_printfa(cmd, "HOST-FLUSH\t%s\n", net_ip2addr(&ip)); + else { + str_printfa(cmd, "HOST-RESET-USERS\t%s", net_ip2addr(&ip)); + if (ctx->max_parallel > 0) { + str_printfa(cmd, "\t%lld", + (long long)ctx->max_parallel); + } + str_append_c(cmd, '\n'); + } + director_send(ctx, str_c(cmd)); + } + for (i = 0; i < ips_count; i++) { + line = i_stream_read_next_line(ctx->input); + if (line != NULL && strcmp(line, "NOTFOUND") == 0) { + i_warning("%s: doesn't exist", + net_ip2addr(&ips[i])); + if (doveadm_exit_code == 0) + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else if (line == NULL) { + director_disconnected(ctx); + } else if (strcmp(line, "OK") != 0) { + i_warning("%s: %s", net_ip2addr(&ips[i]), line); + doveadm_exit_code = EX_TEMPFAIL; + } else if (doveadm_verbose) { + i_info("%s: flushed", net_ip2addr(&ips[i])); + } + } + director_disconnect(ctx); +} + +static void cmd_director_dump(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + const char *line, *const *args; + + ctx = cmd_director_init(cctx); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + if (ctx->explicit_socket_path) + doveadm_print_formatted_set_format("doveadm director %{command} -a %{socket-path} %{host} %{vhost_count}\n"); + else + doveadm_print_formatted_set_format("doveadm director %{command} %{host} %{vhost_count}\n"); + + doveadm_print_header_simple("command"); + doveadm_print_header_simple("socket-path"); + doveadm_print_header_simple("host"); + doveadm_print_header_simple("vhost_count"); + + director_send(ctx, "HOST-LIST\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + args = t_strsplit_tabescaped(line); + if (str_array_length(args) >= 2) { + const char *host = args[0]; + const char *tag = args[3]; + /* this is guaranteed to be at least NULL */ + if (tag != NULL && + *tag != '\0') + host = t_strdup_printf("%s@%s", host, + tag); + doveadm_print("add"); + doveadm_print(ctx->socket_path); + doveadm_print(host); + doveadm_print(args[1]); + } + } T_END; + } + + director_send(ctx, "HOST-LIST-REMOVED\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + doveadm_print("remove"); + doveadm_print(ctx->socket_path); + doveadm_print(line); + doveadm_print(""); + } + if (line == NULL) + director_disconnected(ctx); + director_disconnect(ctx); +} + + +static void director_read_ok_reply(struct director_context *ctx) +{ + const char *line; + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + director_disconnected(ctx); + } else if (strcmp(line, "NOTFOUND") == 0) { + i_error("Not found"); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else if (strcmp(line, "OK") != 0) { + i_error("Failed: %s", line); + doveadm_exit_code = EX_TEMPFAIL; + } +} + +static void cmd_director_ring_add(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + struct ip_addr ip; + in_port_t port = 0; + string_t *str = t_str_new(64); + + ctx = cmd_director_init(cctx); + if (ctx->ip == NULL || + net_addr2ip(ctx->ip, &ip) < 0 || + (ctx->port != 0 && net_str2port(ctx->port, &port) < 0)) { + director_cmd_help(cctx->cmd); + return; + } + + str_printfa(str, "DIRECTOR-ADD\t%s", net_ip2addr(&ip)); + if (port != 0) + str_printfa(str, "\t%u", port); + str_append_c(str, '\n'); + director_send(ctx, str_c(str)); + director_read_ok_reply(ctx); + director_disconnect(ctx); +} + +static void cmd_director_ring_remove(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + struct ip_addr ip; + string_t *str = t_str_new(64); + in_port_t port = 0; + + ctx = cmd_director_init(cctx); + if (ctx->ip == NULL || + net_addr2ip(ctx->ip, &ip) < 0 || + (ctx->port != NULL && net_str2port(ctx->port, &port) < 0)) { + director_cmd_help(cctx->cmd); + return; + } + + str_printfa(str, "DIRECTOR-REMOVE\t%s", net_ip2addr(&ip)); + if (port != 0) + str_printfa(str, "\t%u", port); + str_append_c(str, '\n'); + director_send(ctx, str_c(str)); + director_read_ok_reply(ctx); + director_disconnect(ctx); +} + +static void cmd_director_ring_status(struct doveadm_cmd_context *cctx) +{ + struct director_context *ctx; + const char *line, *const *args; + + ctx = cmd_director_init(cctx); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header_simple("director ip"); + doveadm_print_header_simple("port"); + doveadm_print_header_simple("type"); + doveadm_print_header_simple("last failed"); + doveadm_print_header_simple("status"); + doveadm_print_header_simple("ping ms"); + doveadm_print_header_simple("input"); + doveadm_print_header_simple("output"); + doveadm_print_header_simple("buffered"); + doveadm_print_header_simple("buffered peak"); + doveadm_print_header_simple("last read"); + doveadm_print_header_simple("last write"); + + director_send(ctx, "DIRECTOR-LIST\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + unsigned int i; + time_t ts; + + args = t_strsplit_tabescaped(line); + for (i = 0; i < 12 && args[i] != NULL; i++) { + if ((i == 3 || i == 10 || i == 11) && + str_to_time(args[i], &ts) == 0) { + if (ts == 0) + doveadm_print("never"); + else + doveadm_print(unixdate2str(ts)); + } else { + doveadm_print(args[i]); + } + } + for (; i < 12; i++) + doveadm_print("-"); + } T_END; + } + if (line == NULL) + director_disconnected(ctx); + director_disconnect(ctx); +} + +struct doveadm_cmd_ver2 doveadm_cmd_director[] = { +{ + .name = "director status", + .cmd = cmd_director_status, + .usage = "[-a <director socket path>] [<user>] [<tag>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "tag", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director map", + .cmd = cmd_director_map, + .usage = "[-a <director socket path>] [-f <users file>] [-h | -u] [<host>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('f', "users-file", CMD_PARAM_ISTREAM, 0) +DOVEADM_CMD_PARAM('h', "hash-map", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('u', "user-map", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director add", + .cmd = cmd_director_add, + .usage = "[-a <director socket path>] [-t <tag>] <host> [<vhost count>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('t', "tag", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director update", + .cmd = cmd_director_update, + .usage = "[-a <director socket path>] <host> <vhost count>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director up", + .cmd = cmd_director_up, + .usage = "[-a <director socket path>] <host>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director down", + .cmd = cmd_director_down, + .usage = "[-a <director socket path>] <host>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director remove", + .cmd = cmd_director_remove, + .usage = "[-a <director socket path>] <host>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director move", + .cmd = cmd_director_move, + .usage = "[-a <director socket path>] <user> <host>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director kick", + .cmd = cmd_director_kick, + .usage = "[-a <director socket path>] [-f <passdb field>] <user>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('f', "passdb-field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director flush", + .cmd = cmd_director_flush, + .usage = "[-a <director socket path>] [-F] [--max-parallel <n>] <host>|all", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('F', "force-flush", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "max-parallel", CMD_PARAM_INT64, 0) +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director dump", + .cmd = cmd_director_dump, + .usage = "[-a <director socket path>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director ring add", + .cmd = cmd_director_ring_add, + .usage = "[-a <director socket path>] <ip> [<port>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director ring remove", + .cmd = cmd_director_ring_remove, + .usage = "[-a <director socket path>] <ip> [<port>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "director ring status", + .cmd = cmd_director_ring_status, + .usage = "[-a <director socket path>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +} +}; + +static void director_cmd_help(const struct doveadm_cmd_ver2 *cmd) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++) { + if (doveadm_cmd_director[i].cmd == cmd->cmd) + help_ver2(&doveadm_cmd_director[i]); + } + i_unreached(); +} + +void doveadm_register_director_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_director[i]); +} diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c new file mode 100644 index 0000000..e490def --- /dev/null +++ b/src/doveadm/doveadm-dsync.c @@ -0,0 +1,1562 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "lib-signals.h" +#include "array.h" +#include "execv-const.h" +#include "child-wait.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-ssl.h" +#include "iostream-rawlog.h" +#include "write-full.h" +#include "str.h" +#include "strescape.h" +#include "var-expand.h" +#include "process-title.h" +#include "settings-parser.h" +#include "imap-util.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "master-service-ssl-settings.h" +#include "mail-storage.h" +#include "mail-storage-service.h" +#include "mail-user.h" +#include "mail-namespace.h" +#include "mailbox-list-private.h" +#include "doveadm-settings.h" +#include "doveadm-mail.h" +#include "doveadm-print.h" +#include "doveadm-server.h" +#include "client-connection.h" +#include "server-connection.h" +#include "dsync/dsync-brain.h" +#include "dsync/dsync-ibc.h" +#include "doveadm-dsync.h" + +#include <stdio.h> +#include <unistd.h> +#include <ctype.h> +#include <sys/wait.h> + +#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:I:l:m:n:NO:Pr:Rs:t:e:T:Ux:" +#define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30 +/* The default vname_escape_char to use unless overridden by BROKENCHAR + setting. Note that it's only used for internal dsync names, so it won't end + up in permanent storage names. The only requirement for it is that it's not + the same as the hierarchy separator. */ +#define DSYNC_LIST_VNAME_ESCAPE_CHAR '%' +/* In case DSYNC_LIST_VNAME_ESCAPE_CHAR is the hierarchy separator, + use this instead. */ +#define DSYNC_LIST_VNAME_ALT_ESCAPE_CHAR '~' + +#define DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS (60*10) + +enum dsync_run_type { + DSYNC_RUN_TYPE_LOCAL, + DSYNC_RUN_TYPE_STREAM, + DSYNC_RUN_TYPE_CMD +}; + +struct dsync_cmd_context { + struct doveadm_mail_cmd_context ctx; + enum dsync_brain_sync_type sync_type; + const char *mailbox; + const char *sync_flags; + const char *virtual_all_box; + guid_128_t mailbox_guid; + const char *state_input, *rawlog_path; + ARRAY_TYPE(const_string) exclude_mailboxes; + ARRAY_TYPE(const_string) namespace_prefixes; + time_t sync_since_timestamp; + time_t sync_until_timestamp; + uoff_t sync_max_size; + unsigned int io_timeout_secs; + + const char *remote_name; + const char *local_location; + pid_t remote_pid; + const char *const *remote_cmd_args; + struct child_wait *child_wait; + int exit_status; + + int fd_in, fd_out, fd_err; + struct io *io_err; + struct istream *input, *err_stream; + struct ostream *output; + size_t input_orig_bufsize, output_orig_bufsize; + + struct ssl_iostream_context *ssl_ctx; + struct ssl_iostream *ssl_iostream; + + enum dsync_run_type run_type; + struct server_connection *tcp_conn; + const char *error; + + unsigned int lock_timeout; + unsigned int import_commit_msgs_interval; + + bool lock:1; + bool purge_remote:1; + bool sync_visible_namespaces:1; + bool default_replica_location:1; + bool oneway:1; + bool backup:1; + bool reverse_backup:1; + bool remote_user_prefix:1; + bool no_mail_sync:1; + bool local_location_from_arg:1; + bool replicator_notify:1; + bool exited:1; + bool empty_hdr_workaround:1; +}; + +static bool legacy_dsync = FALSE; + +static void dsync_cmd_switch_ioloop_to(struct dsync_cmd_context *ctx, + struct ioloop *ioloop) +{ + if (ctx->input != NULL) + i_stream_switch_ioloop_to(ctx->input, ioloop); + if (ctx->output != NULL) + o_stream_switch_ioloop_to(ctx->output, ioloop); +} + +static void remote_error_input(struct dsync_cmd_context *ctx) +{ + const unsigned char *data; + size_t size; + const char *line; + + switch (i_stream_read(ctx->err_stream)) { + case -2: + data = i_stream_get_data(ctx->err_stream, &size); + fprintf(stderr, "%.*s", (int)size, data); + i_stream_skip(ctx->err_stream, size); + break; + case -1: + io_remove(&ctx->io_err); + break; + default: + while ((line = i_stream_next_line(ctx->err_stream)) != NULL) + fprintf(stderr, "%s\n", line); + break; + } +} + +static void +run_cmd(struct dsync_cmd_context *ctx, const char *const *args) +{ + struct doveadm_cmd_context *cctx = ctx->ctx.cctx; + int fd_in[2], fd_out[2], fd_err[2]; + + ctx->remote_cmd_args = p_strarray_dup(ctx->ctx.pool, args); + + if (pipe(fd_in) < 0 || pipe(fd_out) < 0 || pipe(fd_err) < 0) + i_fatal("pipe() failed: %m"); + + ctx->remote_pid = fork(); + switch (ctx->remote_pid) { + case -1: + i_fatal("fork() failed: %m"); + case 0: + /* child, which will execute the proxy server. stdin/stdout + goes to pipes which we'll pass to proxy client. */ + if (dup2(fd_in[0], STDIN_FILENO) < 0 || + dup2(fd_out[1], STDOUT_FILENO) < 0 || + dup2(fd_err[1], STDERR_FILENO) < 0) + i_fatal("dup2() failed: %m"); + + i_close_fd(&fd_in[0]); + i_close_fd(&fd_in[1]); + i_close_fd(&fd_out[0]); + i_close_fd(&fd_out[1]); + i_close_fd(&fd_err[0]); + i_close_fd(&fd_err[1]); + + execvp_const(args[0], args); + default: + /* parent */ + break; + } + + i_close_fd(&fd_in[0]); + i_close_fd(&fd_out[1]); + i_close_fd(&fd_err[1]); + ctx->fd_in = fd_out[0]; + ctx->fd_out = fd_in[1]; + ctx->fd_err = fd_err[0]; + + if (ctx->remote_user_prefix) { + const char *prefix = + t_strdup_printf("%s\n", cctx->username); + if (write_full(ctx->fd_out, prefix, strlen(prefix)) < 0) + i_fatal("write(remote out) failed: %m"); + } + + fd_set_nonblock(ctx->fd_err, TRUE); + ctx->err_stream = i_stream_create_fd(ctx->fd_err, IO_BLOCK_SIZE); + i_stream_set_return_partial_line(ctx->err_stream, TRUE); +} + +static void +mirror_get_remote_cmd_line(const char *const *argv, + const char *const **cmd_args_r) +{ + ARRAY_TYPE(const_string) cmd_args; + unsigned int i; + const char *p; + + i_assert(argv[0] != NULL); + + t_array_init(&cmd_args, 16); + for (i = 0; argv[i] != NULL; i++) { + p = argv[i]; + array_push_back(&cmd_args, &p); + } + + if (legacy_dsync) { + /* we're executing dsync */ + p = "server"; + } else if (i > 0 && strcmp(argv[i-1], "dsync-server") == 0) { + /* Caller already specified dsync-server in parameters. + This is a common misconfiguration, so just allow it. */ + p = NULL; + } else { + /* we're executing doveadm */ + p = "dsync-server"; + } + if (p != NULL) + array_push_back(&cmd_args, &p); + array_append_zero(&cmd_args); + *cmd_args_r = array_front(&cmd_args); +} + +static const char *const * +get_ssh_cmd_args(const char *host, const char *login, const char *mail_user) +{ + static struct var_expand_table static_tab[] = { + { 'u', NULL, "user" }, + { '\0', NULL, "login" }, + { '\0', NULL, "host" }, + { '\0', NULL, NULL } + }; + struct var_expand_table *tab; + ARRAY_TYPE(const_string) cmd_args; + string_t *str, *str2; + const char *value, *const *args, *error; + + tab = t_malloc_no0(sizeof(static_tab)); + memcpy(tab, static_tab, sizeof(static_tab)); + + tab[0].value = mail_user; + tab[1].value = login; + tab[2].value = host; + + t_array_init(&cmd_args, 8); + str = t_str_new(128); + str2 = t_str_new(128); + args = t_strsplit(doveadm_settings->dsync_remote_cmd, " "); + for (; *args != NULL; args++) { + if (strchr(*args, '%') == NULL) + value = *args; + else { + /* some automation: if parameter's all %variables + expand to empty, but the %variable isn't the only + text in the parameter, skip it. */ + str_truncate(str, 0); + str_truncate(str2, 0); + if (var_expand(str, *args, tab, &error) <= 0 || + var_expand(str2, *args, static_tab, &error) <= 0) { + i_error("Failed to expand dsync_remote_cmd=%s: %s", + *args, error); + } + if (strcmp(str_c(str), str_c(str2)) == 0 && + str_len(str) > 0) + continue; + value = t_strdup(str_c(str)); + } + array_push_back(&cmd_args, &value); + } + array_append_zero(&cmd_args); + return array_front(&cmd_args); +} + +static bool mirror_get_remote_cmd(struct dsync_cmd_context *ctx, + const char *user, + const char *const **cmd_args_r) +{ + const char *p, *host, *const *argv = ctx->ctx.args; + + if (argv[1] != NULL) { + /* more than one parameter, so it contains a full command + (e.g. ssh host dsync) */ + mirror_get_remote_cmd_line(argv, cmd_args_r); + return TRUE; + } + + /* if it begins with /[a-z0-9]+:/, it's a mail location + (e.g. mdbox:~/mail) */ + for (p = argv[0]; *p != '\0'; p++) { + if (!i_isalnum(*p)) { + if (*p == ':') + return FALSE; + break; + } + } + + if (strchr(argv[0], ' ') != NULL || strchr(argv[0], '/') != NULL) { + /* a) the whole command is in one string. this is mainly for + backwards compatibility. + b) script/path */ + mirror_get_remote_cmd_line(t_strsplit(argv[0], " "), + cmd_args_r); + return TRUE; + } + + /* [user@]host */ + host = strchr(argv[0], '@'); + if (host != NULL) + user = t_strdup_until(argv[0], host++); + else + host = argv[0]; + + /* we'll assume virtual users, so in user@host it really means not to + give ssh a username, but to give dsync -u user parameter. */ + *cmd_args_r = get_ssh_cmd_args(host, "", user); + return TRUE; +} + +static void doveadm_user_init_dsync(struct mail_user *user) +{ + struct mail_namespace *ns; + char ns_sep = mail_namespaces_get_root_sep(user->namespaces); + + user->dsyncing = TRUE; + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + struct dsync_mailbox_list *dlist = + p_new(ns->list->pool, struct dsync_mailbox_list, 1); + MODULE_CONTEXT_SET(ns->list, dsync_mailbox_list_module, dlist); + + if (ns->list->set.vname_escape_char == '\0') { + ns->list->set.vname_escape_char = + ns_sep != DSYNC_LIST_VNAME_ESCAPE_CHAR ? + DSYNC_LIST_VNAME_ESCAPE_CHAR : + DSYNC_LIST_VNAME_ALT_ESCAPE_CHAR; + } else { + dlist->have_orig_escape_char = TRUE; + } + } +} + +static bool +paths_are_equal(struct mail_namespace *ns1, struct mail_namespace *ns2, + enum mailbox_list_path_type type) +{ + const char *path1, *path2; + + return mailbox_list_get_root_path(ns1->list, type, &path1) && + mailbox_list_get_root_path(ns2->list, type, &path2) && + strcmp(path1, path2) == 0; +} + +static int +get_dsync_verify_namespace(struct dsync_cmd_context *ctx, + struct mail_user *user, struct mail_namespace **ns_r) +{ + struct mail_namespace *ns; + + /* Use the first -n namespace if given */ + if (array_count(&ctx->namespace_prefixes) > 0) { + const char *prefix = + array_idx_elem(&ctx->namespace_prefixes, 0); + ns = mail_namespace_find(user->namespaces, prefix); + if (ns == NULL) { + i_error("Namespace not found: '%s'", prefix); + ctx->ctx.exit_code = DOVEADM_EX_NOTFOUND; + return -1; + } + *ns_r = ns; + return 0; + } + + /* Prefer prefix="" namespace over inbox=yes namespace. Either it uses + the global mail_location, which is good, or it might have + overwritten location in case of e.g. using subscriptions file for + all namespaces. This isn't necessarily obvious, so lets make it + clearer by failing if it happens. */ + if ((user->namespaces->flags & NAMESPACE_FLAG_UNUSABLE) == 0) { + *ns_r = user->namespaces; + i_assert((*ns_r)->prefix_len == 0); + } else { + /* fallback to inbox=yes */ + *ns_r = mail_namespace_find_inbox(user->namespaces); + } + return 0; +} + +static int +cmd_dsync_run_local(struct dsync_cmd_context *ctx, struct mail_user *user, + struct dsync_brain *brain, struct dsync_ibc *ibc2, + const char **changes_during_sync_r, + enum mail_error *mail_error_r) +{ + struct dsync_brain *brain2; + struct mail_user *user2; + struct mail_namespace *ns, *ns2; + struct setting_parser_context *set_parser; + const char *location, *error; + bool brain1_running, brain2_running, changed1, changed2; + bool remote_only_changes; + int ret; + + *mail_error_r = 0; + + if (ctx->local_location_from_arg) + location = ctx->ctx.args[0]; + else { + i_assert(ctx->local_location != NULL); + location = ctx->local_location; + } + + i_set_failure_prefix("dsync(%s): ", user->username); + + /* update mail_location and create another user for the + second location. */ + set_parser = mail_storage_service_user_get_settings_parser(ctx->ctx.cur_service_user); + if (settings_parse_keyvalue(set_parser, "mail_location", location) < 0) + i_unreached(); + ret = mail_storage_service_next(ctx->ctx.storage_service, + ctx->ctx.cur_service_user, + &user2, &error); + if (ret < 0) { + i_error("Failed to initialize user: %s", error); + ctx->ctx.exit_code = ret == -1 ? EX_TEMPFAIL : EX_CONFIG; + return -1; + } + doveadm_user_init_dsync(user2); + + if (get_dsync_verify_namespace(ctx, user, &ns) < 0 || + get_dsync_verify_namespace(ctx, user2, &ns2) < 0) + return -1; + if (mail_namespace_get_sep(ns) != mail_namespace_get_sep(ns2)) { + i_error("Mail locations must use the same hierarchy separator " + "(specify namespace prefix=\"%s\" " + "{ separator } explicitly)", ns->prefix); + ctx->ctx.exit_code = EX_CONFIG; + mail_user_deinit(&user2); + return -1; + } + if (paths_are_equal(ns, ns2, MAILBOX_LIST_PATH_TYPE_MAILBOX) && + paths_are_equal(ns, ns2, MAILBOX_LIST_PATH_TYPE_INDEX)) { + i_error("Both source and destination mail_location " + "points to same directory: %s (namespace " + "prefix=\"%s\" { location } is set explicitly?)", + mailbox_list_get_root_forced(user->namespaces->list, + MAILBOX_LIST_PATH_TYPE_MAILBOX), + ns->prefix); + ctx->ctx.exit_code = EX_CONFIG; + mail_user_deinit(&user2); + return -1; + } + + brain2 = dsync_brain_slave_init(user2, ibc2, TRUE, "", + doveadm_settings->dsync_alt_char[0]); + mail_user_unref(&user2); + + brain1_running = brain2_running = TRUE; + changed1 = changed2 = TRUE; + while (brain1_running || brain2_running) { + if (dsync_brain_has_failed(brain) || + dsync_brain_has_failed(brain2)) + break; + if (doveadm_is_killed()) { + i_warning("Killed with signal %d", doveadm_killed_signo()); + break; + } + + i_assert(changed1 || changed2); + brain1_running = dsync_brain_run(brain, &changed1); + brain2_running = dsync_brain_run(brain2, &changed2); + } + *changes_during_sync_r = t_strdup(dsync_brain_get_unexpected_changes_reason(brain2, &remote_only_changes)); + if (dsync_brain_deinit(&brain2, mail_error_r) < 0) + return -1; + return doveadm_is_killed() ? -1 : 0; +} + +static void cmd_dsync_remote_exited(const struct child_wait_status *status, + struct dsync_cmd_context *ctx) +{ + ctx->exited = TRUE; + ctx->exit_status = status->status; + io_loop_stop(current_ioloop); +} + +static void cmd_dsync_wait_remote(struct dsync_cmd_context *ctx) +{ + struct timeout *to; + + /* wait in ioloop for the remote process to die. while we're running + we're also reading and printing all errors that still coming from + it. */ + to = timeout_add(DSYNC_REMOTE_CMD_EXIT_WAIT_SECS*1000, + io_loop_stop, current_ioloop); + io_loop_run(current_ioloop); + timeout_remove(&to); + + /* io_loop_run() deactivates the context - put it back */ + mail_storage_service_io_activate_user(ctx->ctx.cur_service_user); + + if (!ctx->exited) { + i_error("Remote command process isn't dying, killing it"); + if (kill(ctx->remote_pid, SIGKILL) < 0 && errno != ESRCH) { + i_error("kill(%ld, SIGKILL) failed: %m", + (long)ctx->remote_pid); + } + } +} + +static void cmd_dsync_log_remote_status(int status, bool remote_errors_logged, + const char *const *remote_cmd_args) +{ + if (status == -1) + ; + else if (WIFSIGNALED(status)) { + i_error("Remote command died with signal %d: %s", WTERMSIG(status), + t_strarray_join(remote_cmd_args, " ")); + } else if (!WIFEXITED(status)) { + i_error("Remote command failed with status %d: %s", status, + t_strarray_join(remote_cmd_args, " ")); + } else if (WEXITSTATUS(status) == EX_TEMPFAIL && remote_errors_logged) { + /* remote most likely already logged the error. + don't bother logging another line about it */ + } else if (WEXITSTATUS(status) != 0) { + i_error("Remote command returned error %d: %s", WEXITSTATUS(status), + t_strarray_join(remote_cmd_args, " ")); + } +} + +static void cmd_dsync_run_remote(struct mail_user *user) +{ + i_set_failure_prefix("dsync-local(%s)<%s>: ", user->username, user->session_id); + io_loop_run(current_ioloop); +} + +static const char *const * +parse_ssh_location(const char *location, const char *username) +{ + const char *host, *login; + + host = strrchr(location, '@'); + if (host != NULL) + login = t_strdup_until(location, host++); + else { + host = location; + login = ""; + } + return get_ssh_cmd_args(host, login, username); +} + +static struct dsync_ibc * +cmd_dsync_ibc_stream_init(struct dsync_cmd_context *ctx, + const char *name, const char *temp_prefix) +{ + if (ctx->input == NULL) { + fd_set_nonblock(ctx->fd_in, TRUE); + fd_set_nonblock(ctx->fd_out, TRUE); + ctx->input = i_stream_create_fd(ctx->fd_in, SIZE_MAX); + ctx->output = o_stream_create_fd(ctx->fd_out, SIZE_MAX); + } else { + i_assert(ctx->fd_in == -1 && ctx->fd_out == -1); + ctx->fd_in = i_stream_get_fd(ctx->input); + ctx->fd_out = o_stream_get_fd(ctx->output); + ctx->input_orig_bufsize = i_stream_get_max_buffer_size(ctx->input); + ctx->output_orig_bufsize = o_stream_get_max_buffer_size(ctx->output); + i_stream_set_max_buffer_size(ctx->input, SIZE_MAX); + o_stream_set_max_buffer_size(ctx->output, SIZE_MAX); + } + if (ctx->rawlog_path != NULL) { + iostream_rawlog_create_path(ctx->rawlog_path, + &ctx->input, &ctx->output); + } + return dsync_ibc_init_stream(ctx->input, ctx->output, + name, temp_prefix, ctx->io_timeout_secs); +} + +static void +dsync_replicator_notify(struct dsync_cmd_context *ctx, + enum dsync_brain_sync_type sync_type, + const char *state_str) +{ +#define REPLICATOR_HANDSHAKE "VERSION\treplicator-doveadm-client\t1\t0\n" + const char *path; + string_t *str; + int fd; + + path = t_strdup_printf("%s/replicator-doveadm", + ctx->ctx.cur_mail_user->set->base_dir); + fd = net_connect_unix(path); + if (fd == -1) { + if (errno == ECONNREFUSED || errno == ENOENT) { + /* replicator not running on this server. ignore. */ + return; + } + i_error("net_connect_unix(%s) failed: %m", path); + return; + } + fd_set_nonblock(fd, FALSE); + + str = t_str_new(128); + str_append(str, REPLICATOR_HANDSHAKE"NOTIFY\t"); + str_append_tabescaped(str, ctx->ctx.cur_mail_user->username); + str_append_c(str, '\t'); + if (sync_type == DSYNC_BRAIN_SYNC_TYPE_FULL) + str_append_c(str, 'f'); + str_append_c(str, '\t'); + str_append_tabescaped(str, state_str); + str_append_c(str, '\n'); + if (write_full(fd, str_data(str), str_len(str)) < 0) + i_error("write(%s) failed: %m", path); + /* we only wanted to notify replicator. we don't care enough about the + answer to wait for it. */ + if (close(fd) < 0) + i_error("close(%s) failed: %m", path); +} + +static int +cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + struct doveadm_cmd_context *cctx = _ctx->cctx; + struct dsync_ibc *ibc, *ibc2 = NULL; + struct dsync_brain *brain; + struct dsync_brain_settings set; + struct mail_namespace *ns; + const char *const *strp; + enum dsync_brain_flags brain_flags; + enum mail_error mail_error = 0, mail_error2; + bool remote_errors_logged = FALSE; + bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI); + const char *changes_during_sync, *changes_during_sync2 = NULL; + bool remote_only_changes; + int ret = 0; + + /* replicator_notify indicates here automated attempt, + we still want to allow manual sync/backup */ + if (!cli && ctx->replicator_notify && + mail_user_plugin_getenv_bool(_ctx->cur_mail_user, "noreplicate")) { + ctx->ctx.exit_code = DOVEADM_EX_NOREPLICATE; + return -1; + } + + i_zero(&set); + if (cctx->remote_ip.family != 0) { + /* include the doveadm client's IP address in the ps output */ + set.process_title_prefix = t_strdup_printf( + "%s ", net_ip2addr(&cctx->remote_ip)); + } + set.sync_since_timestamp = ctx->sync_since_timestamp; + set.sync_until_timestamp = ctx->sync_until_timestamp; + + if (set.sync_since_timestamp > 0 && set.sync_until_timestamp > 0 && + set.sync_since_timestamp > set.sync_until_timestamp) { + i_fatal("start date is later than end date"); + } + + set.sync_max_size = ctx->sync_max_size; + set.sync_box = ctx->mailbox; + set.sync_flag = ctx->sync_flags; + set.virtual_all_box = ctx->virtual_all_box; + memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid)); + set.lock_timeout_secs = ctx->lock_timeout; + set.import_commit_msgs_interval = ctx->import_commit_msgs_interval; + set.state = ctx->state_input; + set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0]; + if (*doveadm_settings->dsync_hashed_headers == '\0') { + i_error("dsync_hashed_headers must not be empty"); + ctx->ctx.exit_code = EX_USAGE; + return -1; + } + set.hashed_headers = + t_strsplit_spaces(doveadm_settings->dsync_hashed_headers, " ,"); + if (array_count(&ctx->exclude_mailboxes) > 0) { + /* array is NULL-terminated in init() */ + set.exclude_mailboxes = array_front(&ctx->exclude_mailboxes); + } + doveadm_user_init_dsync(user); + + t_array_init(&set.sync_namespaces, array_count(&ctx->namespace_prefixes)); + array_foreach(&ctx->namespace_prefixes, strp) { + ns = mail_namespace_find(user->namespaces, *strp); + if (ns == NULL) { + i_error("Namespace not found: '%s'", *strp); + ctx->ctx.exit_code = EX_USAGE; + return -1; + } + array_push_back(&set.sync_namespaces, &ns); + } + + if (ctx->run_type == DSYNC_RUN_TYPE_LOCAL) + dsync_ibc_init_pipe(&ibc, &ibc2); + else { + string_t *temp_prefix = t_str_new(64); + mail_user_set_get_temp_prefix(temp_prefix, user->set); + ibc = cmd_dsync_ibc_stream_init(ctx, ctx->remote_name, + str_c(temp_prefix)); + if (ctx->fd_err != -1) { + ctx->io_err = io_add(ctx->fd_err, IO_READ, + remote_error_input, ctx); + } + } + + brain_flags = DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS; + if (ctx->sync_visible_namespaces) + brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES; + if (ctx->purge_remote) + brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; + + if (ctx->backup) { + if (ctx->reverse_backup) + brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + else + brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + } + + if (ctx->no_mail_sync) + brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC; + if (ctx->oneway) + brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; + if (ctx->empty_hdr_workaround) + brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; + if (doveadm_debug) + brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; + + child_wait_init(); + brain = dsync_brain_master_init(user, ibc, ctx->sync_type, + brain_flags, &set); + + switch (ctx->run_type) { + case DSYNC_RUN_TYPE_LOCAL: + if (cmd_dsync_run_local(ctx, user, brain, ibc2, + &changes_during_sync2, &mail_error) < 0) + ret = -1; + break; + case DSYNC_RUN_TYPE_CMD: + ctx->child_wait = child_wait_new_with_pid(ctx->remote_pid, + cmd_dsync_remote_exited, ctx); + /* fall through */ + case DSYNC_RUN_TYPE_STREAM: + cmd_dsync_run_remote(user); + /* io_loop_run() deactivates the context - put it back */ + mail_storage_service_io_activate_user(ctx->ctx.cur_service_user); + break; + } + + if (ctx->state_input != NULL) { + string_t *state_str = t_str_new(128); + dsync_brain_get_state(brain, state_str); + doveadm_print(str_c(state_str)); + } + + changes_during_sync = dsync_brain_get_unexpected_changes_reason(brain, &remote_only_changes); + if (changes_during_sync != NULL || changes_during_sync2 != NULL) { + /* don't log a warning when running via doveadm server + (e.g. called by replicator) */ + if (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI) { + i_warning("Mailbox changes caused a desync. " + "You may want to run dsync again: %s", + changes_during_sync == NULL || + (remote_only_changes && changes_during_sync2 != NULL) ? + changes_during_sync2 : changes_during_sync); + } + ctx->ctx.exit_code = 2; + } + if (dsync_brain_deinit(&brain, &mail_error2) < 0) + ret = -1; + if (ret < 0) { + /* tempfail is the default error. prefer to use a non-tempfail + if that exists. */ + if (mail_error2 != 0 && + (mail_error == 0 || mail_error == MAIL_ERROR_TEMP)) + mail_error = mail_error2; + doveadm_mail_failed_error(&ctx->ctx, mail_error); + } + dsync_ibc_deinit(&ibc); + if (ibc2 != NULL) + dsync_ibc_deinit(&ibc2); + ssl_iostream_destroy(&ctx->ssl_iostream); + if (ctx->ssl_ctx != NULL) + ssl_iostream_context_unref(&ctx->ssl_ctx); + if (ctx->input != NULL) { + i_stream_set_max_buffer_size(ctx->input, ctx->input_orig_bufsize); + i_stream_unref(&ctx->input); + } + if (ctx->output != NULL) { + o_stream_set_max_buffer_size(ctx->output, ctx->output_orig_bufsize); + o_stream_unref(&ctx->output); + } + if (ctx->fd_in != -1) { + if (ctx->fd_out != ctx->fd_in) + i_close_fd(&ctx->fd_out); + i_close_fd(&ctx->fd_in); + } + /* print any final errors after the process has died. not closing + stdin/stdout before wait() may cause the process to hang, but stderr + shouldn't (at least with ssh) and we need stderr to be open to be + able to print the final errors */ + if (ctx->run_type == DSYNC_RUN_TYPE_CMD) { + cmd_dsync_wait_remote(ctx); + remote_error_input(ctx); + remote_errors_logged = ctx->err_stream->v_offset > 0; + i_stream_destroy(&ctx->err_stream); + cmd_dsync_log_remote_status(ctx->exit_status, remote_errors_logged, + ctx->remote_cmd_args); + } else { + i_assert(ctx->err_stream == NULL); + } + io_remove(&ctx->io_err); + i_close_fd(&ctx->fd_err); + + if (ctx->child_wait != NULL) + child_wait_free(&ctx->child_wait); + child_wait_deinit(); + return ret; +} + +static void dsync_connected_callback(int exit_code, const char *error, + void *context) +{ + struct dsync_cmd_context *ctx = context; + + ctx->ctx.exit_code = exit_code; + switch (exit_code) { + case 0: + server_connection_extract(ctx->tcp_conn, &ctx->input, + &ctx->output, &ctx->ssl_iostream); + break; + case SERVER_EXIT_CODE_DISCONNECTED: + ctx->ctx.exit_code = EX_TEMPFAIL; + ctx->error = p_strdup_printf(ctx->ctx.pool, + "Disconnected from remote: %s", error); + break; + case EX_NOUSER: + ctx->error = "Unknown user in remote"; + break; + case DOVEADM_EX_NOREPLICATE: + if (doveadm_debug) + i_debug("user is disabled for replication"); + break; + default: + ctx->error = p_strdup_printf(ctx->ctx.pool, + "Failed to start remote dsync-server command: " + "Remote exit_code=%u %s", + exit_code, error == NULL ? "" : error); + break; + } + io_loop_stop(current_ioloop); +} + +static int dsync_init_ssl_ctx(struct dsync_cmd_context *ctx, + const struct master_service_ssl_settings *ssl_set, + const char **error_r) +{ + struct ssl_iostream_settings ssl_ctx_set; + + if (ctx->ssl_ctx != NULL) + return 0; + + master_service_ssl_client_settings_to_iostream_set(ssl_set, + pool_datastack_create(), &ssl_ctx_set); + return ssl_iostream_client_context_cache_get(&ssl_ctx_set, + &ctx->ssl_ctx, error_r); +} + +static void dsync_server_run_command(struct dsync_cmd_context *ctx, + struct server_connection *conn) +{ + struct doveadm_cmd_context *cctx = ctx->ctx.cctx; + /* <flags> <username> <command> [<args>] */ + string_t *cmd = t_str_new(256); + if (doveadm_debug) + str_append_c(cmd, 'D'); + str_append_c(cmd, '\t'); + str_append_tabescaped(cmd, cctx->username); + str_append(cmd, "\tdsync-server\t-u"); + str_append_tabescaped(cmd, cctx->username); + if (ctx->replicator_notify) + str_append(cmd, "\t-U"); + str_append_c(cmd, '\n'); + + ctx->tcp_conn = conn; + server_connection_cmd(conn, str_c(cmd), NULL, + dsync_connected_callback, ctx); + io_loop_run(current_ioloop); + ctx->tcp_conn = NULL; +} + +static int +dsync_connect_tcp(struct dsync_cmd_context *ctx, + const struct master_service_ssl_settings *ssl_set, + const char *target, bool ssl, const char **error_r) +{ + struct doveadm_server *server; + struct server_connection *conn; + struct ioloop *prev_ioloop, *ioloop; + const char *p, *error; + + server = p_new(ctx->ctx.pool, struct doveadm_server, 1); + server->name = p_strdup(ctx->ctx.pool, target); + p = strrchr(server->name, ':'); + server->hostname = p == NULL ? server->name : + p_strdup_until(ctx->ctx.pool, server->name, p); + if (ssl) { + if (dsync_init_ssl_ctx(ctx, ssl_set, &error) < 0) { + *error_r = t_strdup_printf( + "Couldn't initialize SSL context: %s", error); + return -1; + } + server->ssl_flags = PROXY_SSL_FLAG_YES; + server->ssl_ctx = ctx->ssl_ctx; + } + p_array_init(&server->connections, ctx->ctx.pool, 1); + p_array_init(&server->queue, ctx->ctx.pool, 1); + + prev_ioloop = current_ioloop; + ioloop = io_loop_create(); + dsync_cmd_switch_ioloop_to(ctx, ioloop); + + if (doveadm_verbose_proctitle) { + process_title_set(t_strdup_printf( + "[dsync - connecting to %s]", server->name)); + } + if (server_connection_create(server, &conn, &error) < 0) { + ctx->error = p_strdup_printf(ctx->ctx.pool, + "Couldn't create server connection: %s", error); + } else { + if (doveadm_verbose_proctitle) { + process_title_set(t_strdup_printf( + "[dsync - running dsync-server on %s]", server->name)); + } + + dsync_server_run_command(ctx, conn); + } + + if (array_count(&server->connections) > 0) + server_connection_destroy(&conn); + + dsync_cmd_switch_ioloop_to(ctx, prev_ioloop); + io_loop_destroy(&ioloop); + + if (ctx->error != NULL) { + ssl_iostream_context_unref(&ctx->ssl_ctx); + *error_r = ctx->error; + ctx->error = NULL; + return -1; + } + ctx->run_type = DSYNC_RUN_TYPE_STREAM; + return 0; +} + +static int +parse_location(struct dsync_cmd_context *ctx, + const struct master_service_ssl_settings *ssl_set, + const char *location, + const char *const **remote_cmd_args_r, const char **error_r) +{ + struct doveadm_cmd_context *cctx = ctx->ctx.cctx; + + if (str_begins(location, "tcp:")) { + /* TCP connection to remote dsync */ + ctx->remote_name = location+4; + return dsync_connect_tcp(ctx, ssl_set, ctx->remote_name, + FALSE, error_r); + } + if (str_begins(location, "tcps:")) { + /* TCP+SSL connection to remote dsync */ + ctx->remote_name = location+5; + return dsync_connect_tcp(ctx, ssl_set, ctx->remote_name, + TRUE, error_r); + } + + if (str_begins(location, "remote:")) { + /* this is a remote (ssh) command */ + ctx->remote_name = location+7; + } else if (str_begins(location, "remoteprefix:")) { + /* this is a remote (ssh) command with a "user\n" + prefix sent before dsync actually starts */ + ctx->remote_name = location+13; + ctx->remote_user_prefix = TRUE; + } else { + /* local with e.g. maildir:path */ + ctx->remote_name = NULL; + return 0; + } + *remote_cmd_args_r = + parse_ssh_location(ctx->remote_name, cctx->username); + return 0; +} + +static int cmd_dsync_prerun(struct doveadm_mail_cmd_context *_ctx, + struct mail_storage_service_user *service_user, + const char **error_r) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + struct doveadm_cmd_context *cctx = _ctx->cctx; + const char *const *remote_cmd_args = NULL; + const struct mail_user_settings *user_set; + const struct master_service_ssl_settings *ssl_set; + const char *username = ""; + + user_set = mail_storage_service_user_get_set(service_user)[0]; + ssl_set = mail_storage_service_user_get_ssl_settings(service_user); + + ctx->fd_in = -1; + ctx->fd_out = -1; + ctx->fd_err = -1; + ctx->run_type = DSYNC_RUN_TYPE_LOCAL; + ctx->remote_name = "remote"; + + if (ctx->default_replica_location) { + ctx->local_location = + mail_user_set_plugin_getenv(user_set, "mail_replica"); + if (ctx->local_location == NULL || + *ctx->local_location == '\0') { + *error_r = "User has no mail_replica in userdb"; + _ctx->exit_code = DOVEADM_EX_NOTFOUND; + return -1; + } + } else { + /* if we're executing remotely, give -u parameter if we also + did a userdb lookup. */ + if ((_ctx->service_flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) != 0) + username = cctx->username; + + if (!mirror_get_remote_cmd(ctx, username, &remote_cmd_args)) { + /* it's a mail_location */ + if (_ctx->args[1] != NULL) + doveadm_mail_help_name(_ctx->cmd->name); + ctx->local_location = _ctx->args[0]; + ctx->local_location_from_arg = TRUE; + } + } + + if (remote_cmd_args == NULL && ctx->local_location != NULL) { + if (parse_location(ctx, ssl_set, ctx->local_location, + &remote_cmd_args, error_r) < 0) + return -1; + } + + if (remote_cmd_args != NULL) { + /* do this before mail_storage_service_next() in case it + drops process privileges */ + run_cmd(ctx, remote_cmd_args); + ctx->run_type = DSYNC_RUN_TYPE_CMD; + } + + if (ctx->sync_visible_namespaces && + ctx->run_type == DSYNC_RUN_TYPE_LOCAL) + i_fatal("-N parameter requires syncing with remote host"); + return 0; +} + +static void cmd_dsync_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + + if (ctx->default_replica_location) { + if (args[0] != NULL) + i_error("Don't give mail location with -d parameter"); + } else { + if (args[0] == NULL) + doveadm_mail_help_name(_ctx->cmd->name); + } + if (array_count(&ctx->exclude_mailboxes) > 0) + array_append_zero(&ctx->exclude_mailboxes); + + lib_signals_ignore(SIGHUP, TRUE); +} + +static void cmd_dsync_preinit(struct doveadm_mail_cmd_context *ctx) +{ + if ((ctx->service_flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) == 0) + ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR; +} + +static bool +cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + const char *str, *error; + bool utc; + + switch (c) { + case '1': + ctx->oneway = TRUE; + ctx->backup = TRUE; + break; + case 'a': + ctx->virtual_all_box = optarg; + break; + case 'd': + ctx->default_replica_location = TRUE; + break; + case 'E': + /* dsync wrapper detection flag */ + legacy_dsync = TRUE; + break; + case 'f': + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + break; + case 'O': { + const char *str = optarg; + + if (strchr(str, ' ') != NULL) + i_fatal("-O parameter doesn't support multiple flags currently"); + if (str[0] == '-') + str++; + if (str[0] == '\\' && imap_parse_system_flag(str) == 0) + i_fatal("Invalid system flag given for -O parameter: '%s'", str); + ctx->sync_flags = optarg; + break; + } + case 'g': + if (optarg[0] == '\0') + ctx->no_mail_sync = TRUE; + else if (guid_128_from_string(optarg, ctx->mailbox_guid) < 0 || + guid_128_is_empty(ctx->mailbox_guid)) + i_fatal("Invalid -g parameter: %s", optarg); + break; + case 'l': + ctx->lock = TRUE; + if (str_to_uint(optarg, &ctx->lock_timeout) < 0) + i_fatal("Invalid -l parameter: %s", optarg); + break; + case 'm': + if (optarg[0] == '\0') + ctx->no_mail_sync = TRUE; + else + ctx->mailbox = optarg; + break; + case 'x': + str = optarg; + array_push_back(&ctx->exclude_mailboxes, &str); + break; + case 'n': + str = optarg; + array_push_back(&ctx->namespace_prefixes, &str); + break; + case 'N': + ctx->sync_visible_namespaces = TRUE; + break; + case 'P': + ctx->purge_remote = TRUE; + break; + case 'r': + ctx->rawlog_path = optarg; + break; + case 'R': + ctx->reverse_backup = TRUE; + break; + case 's': + if (ctx->sync_type != DSYNC_BRAIN_SYNC_TYPE_FULL && + *optarg != '\0') + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE; + ctx->state_input = optarg; + break; + case 't': + if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp, &utc) < 0) + i_fatal("Invalid -t parameter: %s", optarg); + break; + case 'e': + if (mail_parse_human_timestamp(optarg, &ctx->sync_until_timestamp, &utc) < 0) + i_fatal("Invalid -e parameter: %s", optarg); + break; + case 'I': + if (settings_get_size(optarg, &ctx->sync_max_size, &error) < 0) + i_fatal("Invalid -I parameter '%s': %s", optarg, error); + break; + case 'T': + if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0) + i_fatal("Invalid -T parameter: %s", optarg); + break; + case 'U': + ctx->replicator_notify = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_dsync_alloc(void) +{ + struct dsync_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); + ctx->io_timeout_secs = DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS; + ctx->ctx.getopt_args = DSYNC_COMMON_GETOPT_ARGS; + ctx->ctx.v.parse_arg = cmd_mailbox_dsync_parse_arg; + ctx->ctx.v.preinit = cmd_dsync_preinit; + ctx->ctx.v.init = cmd_dsync_init; + ctx->ctx.v.prerun = cmd_dsync_prerun; + ctx->ctx.v.run = cmd_dsync_run; + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + doveadm_print_header("state", "state", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + p_array_init(&ctx->exclude_mailboxes, ctx->ctx.pool, 4); + p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4); + if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0) + ctx->empty_hdr_workaround = TRUE; + ctx->import_commit_msgs_interval = doveadm_settings->dsync_commit_msgs_interval; + return &ctx->ctx; +} + +static struct doveadm_mail_cmd_context *cmd_dsync_backup_alloc(void) +{ + struct doveadm_mail_cmd_context *_ctx; + struct dsync_cmd_context *ctx; + + _ctx = cmd_dsync_alloc(); + ctx = (struct dsync_cmd_context *)_ctx; + ctx->backup = TRUE; + return _ctx; +} + +static int +cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + struct doveadm_cmd_context *cctx = _ctx->cctx; + bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI); + struct dsync_ibc *ibc; + struct dsync_brain *brain; + string_t *temp_prefix, *state_str = NULL; + enum dsync_brain_sync_type sync_type; + const char *name, *process_title_prefix = ""; + enum mail_error mail_error; + + if (!cli) { + /* replicator_notify indicates here automated attempt, + we still want to allow manual sync/backup */ + if (ctx->replicator_notify && + mail_user_plugin_getenv_bool(_ctx->cur_mail_user, "noreplicate")) { + _ctx->exit_code = DOVEADM_EX_NOREPLICATE; + return -1; + } + + /* doveadm-server connection. start with a success reply. + after that follows the regular dsync protocol. */ + ctx->fd_in = ctx->fd_out = -1; + ctx->input = cctx->input; + ctx->output = cctx->output; + i_stream_ref(ctx->input); + o_stream_ref(ctx->output); + o_stream_set_finish_also_parent(ctx->output, FALSE); + o_stream_nsend(ctx->output, "\n+\n", 3); + i_set_failure_prefix("dsync-server(%s): ", user->username); + name = i_stream_get_name(ctx->input); + + if (cctx->remote_ip.family != 0) { + /* include the doveadm client's IP address in the ps output */ + process_title_prefix = t_strdup_printf( + "%s ", net_ip2addr(&cctx->remote_ip)); + } + } else { + /* the log messages go via stderr to the remote dsync, + so the names are reversed */ + i_set_failure_prefix("dsync-remote(%s)<%s>: ", user->username, user->session_id); + name = "local"; + } + + doveadm_user_init_dsync(user); + + temp_prefix = t_str_new(64); + mail_user_set_get_temp_prefix(temp_prefix, user->set); + + ibc = cmd_dsync_ibc_stream_init(ctx, name, str_c(temp_prefix)); + brain = dsync_brain_slave_init(user, ibc, FALSE, process_title_prefix, + doveadm_settings->dsync_alt_char[0]); + + io_loop_run(current_ioloop); + /* io_loop_run() deactivates the context - put it back */ + mail_storage_service_io_activate_user(ctx->ctx.cur_service_user); + + if (ctx->replicator_notify) { + state_str = t_str_new(128); + dsync_brain_get_state(brain, state_str); + } + sync_type = dsync_brain_get_sync_type(brain); + + if (dsync_brain_deinit(&brain, &mail_error) < 0) + doveadm_mail_failed_error(_ctx, mail_error); + dsync_ibc_deinit(&ibc); + + if (!cli) { + /* make sure nothing more is written by the generic doveadm + connection code */ + o_stream_close(cctx->output); + } + i_stream_unref(&ctx->input); + o_stream_unref(&ctx->output); + + if (ctx->replicator_notify && _ctx->exit_code == 0) + dsync_replicator_notify(ctx, sync_type, str_c(state_str)); + return _ctx->exit_code == 0 ? 0 : -1; +} + +static bool +cmd_mailbox_dsync_server_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx; + + switch (c) { + case 'E': + /* dsync wrapper detection flag */ + legacy_dsync = TRUE; + break; + case 'r': + ctx->rawlog_path = optarg; + break; + case 'T': + if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0) + i_fatal("Invalid -T parameter: %s", optarg); + break; + case 'U': + ctx->replicator_notify = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_dsync_server_alloc(void) +{ + struct dsync_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context); + ctx->io_timeout_secs = DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS; + ctx->ctx.getopt_args = "Er:T:U"; + ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg; + ctx->ctx.v.run = cmd_dsync_server_run; + ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + + ctx->fd_in = STDIN_FILENO; + ctx->fd_out = STDOUT_FILENO; + return &ctx->ctx; +} + +#define DSYNC_COMMON_PARAMS \ +DOVEADM_CMD_MAIL_COMMON \ +DOVEADM_CMD_PARAM('f', "full-sync", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('P', "purge-remote", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('R', "reverse-sync", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('U', "replicator-notify", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('l', "lock-timeout", CMD_PARAM_INT64, 0) \ +DOVEADM_CMD_PARAM('r', "rawlog", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('m', "mailbox", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('n', "namespace", CMD_PARAM_ARRAY, 0) \ +DOVEADM_CMD_PARAM('N', "all-namespaces", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('x', "exclude-mailbox", CMD_PARAM_ARRAY, 0) \ +DOVEADM_CMD_PARAM('a', "all-mailbox", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('s', "state", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('t', "sync-since-time", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('e', "sync-until-time", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('O', "sync-flags", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('I', "sync-max-size", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('T', "timeout", CMD_PARAM_INT64, 0) \ +DOVEADM_CMD_PARAM('d', "default-destination", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('E', "legacy-dsync", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('\0', "destination", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) + +#define DSYNC_COMMON_USAGE \ + "[-l <secs>] [-r <rawlog path>] " \ + "[-m <mailbox>] [-g <mailbox guid>] [-n <namespace> | -N] " \ + "[-x <exclude>] [-a <all mailbox>] [-s <state>] [-T <secs>] " \ + "[-t <start date>] [-e <end date>] [-O <sync flag>] [-I <max size>] " \ + "-d|<dest>" + +struct doveadm_cmd_ver2 doveadm_cmd_dsync_mirror = { + .mail_cmd = cmd_dsync_alloc, + .name = "sync", + .usage = "[-1fPRU] "DSYNC_COMMON_USAGE, + .flags = CMD_FLAG_NO_UNORDERED_OPTIONS, +DOVEADM_CMD_PARAMS_START +DSYNC_COMMON_PARAMS +DOVEADM_CMD_PARAM('1', "oneway-sync", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAMS_END +}; +struct doveadm_cmd_ver2 doveadm_cmd_dsync_backup = { + .mail_cmd = cmd_dsync_backup_alloc, + .name = "backup", + .usage = "[-fPRU] "DSYNC_COMMON_USAGE, + .flags = CMD_FLAG_NO_UNORDERED_OPTIONS, +DOVEADM_CMD_PARAMS_START +DSYNC_COMMON_PARAMS +DOVEADM_CMD_PARAMS_END +}; +struct doveadm_cmd_ver2 doveadm_cmd_dsync_server = { + .mail_cmd = cmd_dsync_server_alloc, + .name = "dsync-server", + .usage = "[-E] [-r <rawlog path>] [-T <timeout secs>] [-U]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('E', "legacy-dsync", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('r', "rawlog", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('T', "timeout", CMD_PARAM_INT64, 0) +DOVEADM_CMD_PARAM('U', "replicator-notify", CMD_PARAM_BOOL, 0) +/* previously dsync-server could have been added twice to the parameters */ +DOVEADM_CMD_PARAM('\0', "ignore-arg", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +void doveadm_dsync_main(int *_argc, char **_argv[]) +{ + int argc = *_argc; + const char *getopt_str; + char **argv = *_argv; + char **new_argv, *mailbox = NULL, *alt_char = NULL, *username = NULL; + char *p, *dup, new_flags[6]; + int max_argc, src, dest, i, j; + bool flag_f = FALSE, flag_R = FALSE, flag_m, flag_u, flag_C, has_arg; + bool dsync_server = FALSE; + + p = strrchr(argv[0], '/'); + if (p == NULL) p = argv[0]; + if (strstr(p, "dsync") == NULL) + return; + + /* @UNSAFE: this is called when the "doveadm" binary is called as + "dsync" (for backwards compatibility) */ + max_argc = argc + 7; + new_argv = t_new(char *, max_argc); + new_argv[0] = argv[0]; + dest = 1; + getopt_str = master_service_getopt_string(); + + /* add global doveadm flags */ + for (src = 1; src < argc; src++) { + if (argv[src][0] != '-') + break; + + flag_m = FALSE; flag_C = FALSE; has_arg = FALSE; flag_u = FALSE; + dup = t_strdup_noconst(argv[src]); + for (i = j = 1; argv[src][i] != '\0'; i++) { + switch (argv[src][i]) { + case 'C': + flag_C = TRUE; + break; + case 'f': + flag_f = TRUE; + break; + case 'R': + flag_R = TRUE; + break; + case 'm': + flag_m = TRUE; + break; + case 'u': + flag_u = TRUE; + break; + default: + p = strchr(getopt_str, argv[src][i]); + if (p != NULL && p[1] == ':') + has_arg = TRUE; + dup[j++] = argv[src][i]; + break; + } + } + if (j > 1) { + dup[j++] = '\0'; + new_argv[dest++] = dup; + if (has_arg && src+1 < argc) + new_argv[dest++] = argv[++src]; + } + if (flag_m) { + if (src+1 == argc) + i_fatal("-m missing parameter"); + mailbox = argv[++src]; + } + if (flag_u) { + if (src+1 == argc) + i_fatal("-u missing parameter"); + username = argv[++src]; + } + if (flag_C) { + if (src+1 == argc) + i_fatal("-C missing parameter"); + alt_char = argv[++src]; + } + } + if (alt_char != NULL) { + new_argv[dest++] = "-o"; + new_argv[dest++] = + p_strconcat(pool_datastack_create(), + "dsync_alt_char=", alt_char, NULL); + } + + /* mirror|backup|server */ + if (src == argc) + i_fatal("Missing mirror or backup parameter"); + if (strcmp(argv[src], "sync") == 0 || + strcmp(argv[src], "dsync-server") == 0) { + /* we're re-executing dsync due to doveconf. + "backup" re-exec detection is later. */ + return; + } + if (strcmp(argv[src], "mirror") == 0) + new_argv[dest] = "sync"; + else if (strcmp(argv[src], "backup") == 0) + new_argv[dest] = "backup"; + else if (strcmp(argv[src], "server") == 0) { + new_argv[dest] = "dsync-server"; + dsync_server = TRUE; + } else + i_fatal("Invalid parameter: %s", argv[src]); + src++; dest++; + + if (src < argc && str_begins(argv[src], "-E")) { + /* we're re-executing dsync due to doveconf */ + return; + } + + /* dsync flags */ + new_flags[0] = '-'; + new_flags[1] = 'E'; i = 2; + if (!dsync_server) { + if (flag_f) + new_flags[i++] = 'f'; + if (flag_R) + new_flags[i++] = 'R'; + if (mailbox != NULL) + new_flags[i++] = 'm'; + } + i_assert((unsigned int)i < sizeof(new_flags)); + new_flags[i] = '\0'; + + if (i > 1) { + new_argv[dest++] = strdup(new_flags); + if (mailbox != NULL) + new_argv[dest++] = mailbox; + } + if (username != NULL) { + new_argv[dest++] = "-u"; + new_argv[dest++] = username; + } + + /* rest of the parameters */ + for (; src < argc; src++) + new_argv[dest++] = argv[src]; + i_assert(dest < max_argc); + new_argv[dest] = NULL; + + legacy_dsync = TRUE; + *_argc = dest; + *_argv = new_argv; + i_getopt_reset(); +} diff --git a/src/doveadm/doveadm-dsync.h b/src/doveadm/doveadm-dsync.h new file mode 100644 index 0000000..c4bf08b --- /dev/null +++ b/src/doveadm/doveadm-dsync.h @@ -0,0 +1,10 @@ +#ifndef DOVEADM_DSYNC_H +#define DOVEADM_DSYNC_H + +extern struct doveadm_cmd_ver2 doveadm_cmd_dsync_mirror; +extern struct doveadm_cmd_ver2 doveadm_cmd_dsync_backup; +extern struct doveadm_cmd_ver2 doveadm_cmd_dsync_server; + +void doveadm_dsync_main(int *_argc, char **_argv[]); + +#endif diff --git a/src/doveadm/doveadm-dump-dbox.c b/src/doveadm/doveadm-dump-dbox.c new file mode 100644 index 0000000..429b4a1 --- /dev/null +++ b/src/doveadm/doveadm-dump-dbox.c @@ -0,0 +1,231 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hex-dec.h" +#include "istream.h" +#include "index/dbox-common/dbox-file.h" +#include "doveadm-dump.h" + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> + +static void +dump_timestamp(struct istream *input, const char *name, const char *value) +{ + time_t t; + + if (strcmp(value, "0") == 0) + t = 0; + else { + t = hex2dec((const void *)value, strlen(value)); + if (t == 0) { + i_fatal("Invalid %s at %"PRIuUOFF_T": %s", + name, input->v_offset, value); + } + } + printf("%s = %ld (%s)\n", name, (long)t, unixdate2str(t)); +} + +static uoff_t +dump_size(struct istream *input, const char *name, const char *value) +{ + uoff_t size; + + if (strcmp(value, "0") == 0) + size = 0; + else { + size = hex2dec((const void *)value, strlen(value)); + if (size == 0) { + i_fatal("Invalid %s at %"PRIuUOFF_T": %s", + name, input->v_offset, value); + } + } + printf("%s = %"PRIuUOFF_T"\n", name, size); + return size; +} + +static unsigned int dump_file_hdr(struct istream *input) +{ + const char *line, *const *arg, *version; + unsigned int msg_hdr_size = 0; + + if ((line = i_stream_read_next_line(input)) == NULL) + i_fatal("Empty file"); + arg = t_strsplit(line, " "); + + /* check version */ + version = *arg; + if (version == NULL || !str_is_numeric(version, ' ')) + i_fatal("%s is not a dbox file", i_stream_get_name(input)); + if (strcmp(version, "2") != 0) + i_fatal("Unsupported dbox file version %s", version); + arg++; + + for (; *arg != NULL; arg++) { + switch (**arg) { + case DBOX_HEADER_MSG_HEADER_SIZE: + msg_hdr_size = hex2dec((const void *)(*arg + 1), + strlen(*arg + 1)); + if (msg_hdr_size == 0) { + i_fatal("Invalid msg_header_size header: %s", + *arg + 1); + } + printf("file.msg_header_size = %u\n", msg_hdr_size); + break; + case DBOX_HEADER_CREATE_STAMP: + dump_timestamp(input, "file.create_stamp", *arg + 1); + break; + default: + printf("file.unknown-%c = %s\n", **arg, *arg + 1); + break; + } + } + if (msg_hdr_size == 0) + i_fatal("Missing msg_header_size in file header"); + return msg_hdr_size; +} + +static bool +dump_msg_hdr(struct istream *input, unsigned int hdr_size, uoff_t *msg_size_r) +{ + struct dbox_message_header hdr; + const unsigned char *data; + size_t size; + uoff_t msg_size; + + if (i_stream_read_bytes(input, &data, &size, hdr_size) <= 0) { + if (size == 0) + return FALSE; + i_fatal("Partial message header read at %"PRIuUOFF_T": %zu bytes", + input->v_offset, size); + } + printf("offset %"PRIuUOFF_T":\n", input->v_offset); + + if (hdr_size < sizeof(hdr)) + i_fatal("file.hdr_size too small: %u", hdr_size); + memcpy(&hdr, data, sizeof(hdr)); + + if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0) + i_fatal("dbox wrong pre-magic at %"PRIuUOFF_T, input->v_offset); + + msg_size = dump_size(input, "msg.size", + t_strndup(hdr.message_size_hex, sizeof(hdr.message_size_hex))); + + i_stream_skip(input, hdr_size); + *msg_size_r = msg_size; + return TRUE; +} + +static void dump_msg_metadata(struct istream *input) +{ + struct dbox_metadata_header hdr; + const unsigned char *data; + size_t size; + const char *line; + + /* verify magic */ + if (i_stream_read_bytes(input, &data, &size, sizeof(hdr)) <= 0) { + i_fatal("dbox missing metadata at %"PRIuUOFF_T, + input->v_offset); + } + memcpy(&hdr, data, sizeof(hdr)); + if (memcmp(hdr.magic_post, DBOX_MAGIC_POST, sizeof(hdr.magic_post)) != 0) + i_fatal("dbox wrong post-magic at %"PRIuUOFF_T, input->v_offset); + i_stream_skip(input, sizeof(hdr)); + + /* dump the metadata */ + for (;;) { + if ((line = i_stream_read_next_line(input)) == NULL) + i_fatal("dbox metadata ended unexpectedly at EOF"); + if (*line == '\0') + break; + + switch (*line) { + case DBOX_METADATA_GUID: + printf("msg.guid = %s\n", line + 1); + break; + case DBOX_METADATA_POP3_UIDL: + printf("msg.pop3-uidl = %s\n", line + 1); + break; + case DBOX_METADATA_POP3_ORDER: + printf("msg.pop3-order = %s\n", line + 1); + break; + case DBOX_METADATA_RECEIVED_TIME: + dump_timestamp(input, "msg.received", line + 1); + break; + case DBOX_METADATA_PHYSICAL_SIZE: + (void)dump_size(input, "msg.physical-size", line + 1); + break; + case DBOX_METADATA_VIRTUAL_SIZE: + (void)dump_size(input, "msg.virtual-size", line + 1); + break; + case DBOX_METADATA_EXT_REF: + printf("msg.ext-ref = %s\n", line + 1); + break; + case DBOX_METADATA_ORIG_MAILBOX: + printf("msg.orig-mailbox = %s\n", line + 1); + break; + + case DBOX_METADATA_OLDV1_EXPUNGED: + case DBOX_METADATA_OLDV1_FLAGS: + case DBOX_METADATA_OLDV1_KEYWORDS: + case DBOX_METADATA_OLDV1_SAVE_TIME: + case DBOX_METADATA_OLDV1_SPACE: + printf("msg.obsolete-%c = %s\n", *line, line + 1); + break; + } + } +} + +static bool dump_msg(struct istream *input, unsigned int hdr_size) +{ + uoff_t msg_size; + + if (!dump_msg_hdr(input, hdr_size, &msg_size)) + return FALSE; + i_stream_skip(input, msg_size); + dump_msg_metadata(input); + return TRUE; +} + +static void cmd_dump_dbox(const char *path, const char *const *args ATTR_UNUSED) +{ + struct istream *input; + int fd; + unsigned int hdr_size; + bool ret; + + fd = open(path, O_RDONLY); + if (fd < 0) + i_fatal("open(%s) failed: %m", path); + + input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + i_stream_set_name(input, path); + hdr_size = dump_file_hdr(input); + do { + printf("\n"); + T_BEGIN { + ret = dump_msg(input, hdr_size); + } T_END; + } while (ret); + i_stream_destroy(&input); +} + +static bool test_dump_dbox(const char *path) +{ + const char *p; + + p = strrchr(path, '/'); + if (p == NULL) + p = path; + else + p++; + return str_begins(p, "m.") || str_begins(p, "u."); +} + +struct doveadm_cmd_dump doveadm_cmd_dump_dbox = { + "dbox", + test_dump_dbox, + cmd_dump_dbox +}; diff --git a/src/doveadm/doveadm-dump-dcrypt-file.c b/src/doveadm/doveadm-dump-dcrypt-file.c new file mode 100644 index 0000000..3703bf4 --- /dev/null +++ b/src/doveadm/doveadm-dump-dcrypt-file.c @@ -0,0 +1,93 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dcrypt.h" +#include "istream.h" +#include "istream-decrypt.h" +#include "dcrypt-iostream.h" +#include "doveadm-dump.h" +#include <stdio.h> + +static int get_digest(const char *digest, + struct dcrypt_private_key **priv_key_r ATTR_UNUSED, + const char **error_r ATTR_UNUSED, + void *context) +{ + const char **digest_r = (const char**)context; + *digest_r = t_strdup(digest); + return 0; +} + +static void dcrypt_istream_dump_metadata(const struct istream *stream) +{ + enum io_stream_encrypt_flags flags = i_stream_encrypt_get_flags(stream); + if ((flags & IO_STREAM_ENC_INTEGRITY_HMAC) != 0) + printf("flags: IO_STREAM_ENC_INTEGRITY_HMAC\n"); + if ((flags & IO_STREAM_ENC_INTEGRITY_AEAD) != 0) + printf("flags: IO_STREAM_ENC_INTEGRITY_AEAD\n"); + if ((flags & IO_STREAM_ENC_INTEGRITY_NONE) != 0) + printf("flags: IO_STREAM_ENC_INTEGRITY_NONE\n"); + if ((flags & IO_STREAM_ENC_VERSION_1) != 0) + printf("flags: IO_STREAM_ENC_VERSION_1\n"); + + enum decrypt_istream_format format = i_stream_encrypt_get_format(stream); + switch (format) { + case DECRYPT_FORMAT_V1: + printf("format: DECRYPT_FORMAT_V1\n"); + break; + case DECRYPT_FORMAT_V2: + printf("format: DECRYPT_FORMAT_V2\n"); + break; + } +} + +static bool dcrypt_file_dump_metadata(const char *filename, bool print) +{ + bool ret = FALSE; + struct istream *is = i_stream_create_file(filename, IO_BLOCK_SIZE); + const char *key_digest = NULL; + struct istream *ds = i_stream_create_decrypt_callback(is, + get_digest, &key_digest); + + ssize_t size = i_stream_read(ds); + i_assert(size < 0); + + if (key_digest != NULL) { + ret = TRUE; + if (print) { + dcrypt_istream_dump_metadata(ds); + printf("decrypt key digest: %s\n", key_digest); + } + } else if (print && ds->stream_errno != 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(ds), + i_stream_get_error(ds)); + } + + i_stream_unref(&ds); + i_stream_unref(&is); + return ret; +} + +static bool test_dump_dcrypt_file(const char *path) +{ + if (!dcrypt_initialize("openssl", NULL, NULL)) + return FALSE; + bool ret = dcrypt_file_dump_metadata(path, FALSE); + return ret; +} + +static void +cmd_dump_dcrypt_file(const char *path, const char *const *args ATTR_UNUSED) +{ + const char *error = NULL; + if (!dcrypt_initialize("openssl", NULL, &error)) + i_fatal("dcrypt_initialize failed: %s", error); + (void)dcrypt_file_dump_metadata(path, TRUE); +} + +struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_file = { + "dcrypt-file", + test_dump_dcrypt_file, + cmd_dump_dcrypt_file +}; diff --git a/src/doveadm/doveadm-dump-dcrypt-key.c b/src/doveadm/doveadm-dump-dcrypt-key.c new file mode 100644 index 0000000..cecd27f --- /dev/null +++ b/src/doveadm/doveadm-dump-dcrypt-key.c @@ -0,0 +1,215 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dcrypt.h" +#include "dcrypt-iostream.h" +#include "ostream-encrypt.h" +#include "istream-private.h" +#include "istream-decrypt.h" +#include "doveadm-dump.h" +#include "hex-binary.h" +#include "str.h" +#include <stdio.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> +#include <fcntl.h> + +#define KEY_BUF_SIZE 4096 + +static void dcrypt_dump_public_key_metadata(const char *buf) +{ + const char *error = NULL; + struct dcrypt_public_key *pub_key; + + bool ret = dcrypt_key_load_public(&pub_key, buf, &error); + if (ret == FALSE) { + i_error("dcrypt_key_load_public failed: %s", error); + return; + } + enum dcrypt_key_type key_type = dcrypt_key_type_public(pub_key); + if (key_type == DCRYPT_KEY_RSA) + printf("key type: DCRYPT_KEY_RSA\n"); + else if (key_type == DCRYPT_KEY_EC) + printf("key type: DCRYPT_KEY_EC\n"); + + string_t *hash = t_str_new(128); + if (!dcrypt_key_id_public(pub_key, "sha256", hash, &error)) { + i_error("dcrypt_key_id_public failed: %s", error); + } else { + const char *v2_hash = binary_to_hex(hash->data, hash->used); + printf("v2 hash: %s\n", v2_hash); + + if (key_type == DCRYPT_KEY_EC) { + buffer_set_used_size(hash, 0); + if (!dcrypt_key_id_public_old(pub_key, hash, &error)) { + i_error("dcrypt_key_id_public_old failed: %s", + error); + } else { + const char *v1_hash = binary_to_hex(hash->data, + hash->used); + printf("v1 hash: %s\n", v1_hash); + } + } + } + dcrypt_key_unref_public(&pub_key); +} + +static void dcrypt_dump_private_key_metadata(const char *buf) +{ + const char *error = NULL; + struct dcrypt_private_key *priv_key; + + bool ret = dcrypt_key_load_private(&priv_key, buf, NULL, NULL, + &error); + if (ret == FALSE) { + i_error("dcrypt_key_load_private failed: %s", error); + return; + } + enum dcrypt_key_type key_type = dcrypt_key_type_private(priv_key); + if (key_type == DCRYPT_KEY_RSA) + printf("key type: DCRYPT_KEY_RSA\n"); + else if (key_type == DCRYPT_KEY_EC) + printf("key type: DCRYPT_KEY_EC\n"); + + string_t *hash = t_str_new(128); + if (!dcrypt_key_id_private(priv_key, "sha256", hash, &error)) { + i_error("dcrypt_key_id_private failed: %s", error); + } else { + const char *v2_hash = binary_to_hex(hash->data, hash->used); + printf("v2 hash: %s\n", v2_hash); + + if (key_type == DCRYPT_KEY_EC) { + buffer_set_used_size(hash, 0); + if (!dcrypt_key_id_private_old(priv_key, hash, &error)) { + i_error("dcrypt_key_id_private_old failed: %s", error); + } else { + const char *v1_hash = binary_to_hex(hash->data, + hash->used); + printf("v1 hash: %s\n", v1_hash); + } + } + } + dcrypt_key_unref_private(&priv_key); +} + +static bool dcrypt_key_dump_metadata(const char *filename, bool print) +{ + bool ret = TRUE; + int fd = open(filename, O_RDONLY); + if (fd < 0) { + if (print) i_error("open(%s) failed: %m", filename); + return FALSE; + } + + char buf[KEY_BUF_SIZE+1]; + ssize_t res = read(fd, buf, KEY_BUF_SIZE); + if (res < 0) { + if (print) i_error("read(%s) failed: %m", filename); + i_close_fd(&fd); + return FALSE; + } + i_close_fd(&fd); + + buf[res] = '\0'; + enum dcrypt_key_format format; + enum dcrypt_key_version version; + enum dcrypt_key_kind kind; + enum dcrypt_key_encryption_type encryption_type; + const char *encryption_key_hash; + const char *key_hash; + const char *error; + + ret = dcrypt_key_string_get_info(buf, &format, &version, + &kind, &encryption_type, &encryption_key_hash, + &key_hash, &error); + if (ret == FALSE) { + if (print) i_error("dcrypt_key_string_get_info failed: %s", error); + return FALSE; + } + if (!print) return TRUE; + + switch (format) { + case DCRYPT_FORMAT_PEM: + printf("format: DCRYPT_FORMAT_PEM\n"); + break; + case DCRYPT_FORMAT_DOVECOT: + printf("format: DCRYPT_FORMAT_DOVECOT\n"); + break; + case DCRYPT_FORMAT_JWK: + printf("format: DCRYPT_FORMAT_JWK\n"); + } + + switch (version) { + case DCRYPT_KEY_VERSION_1: + printf("version: DCRYPT_KEY_VERSION_1\n"); + break; + case DCRYPT_KEY_VERSION_2: + printf("version: DCRYPT_KEY_VERSION_2\n"); + break; + case DCRYPT_KEY_VERSION_NA: + printf("version: DCRYPT_KEY_VERSION_NA\n"); + break; + } + + switch (kind) { + case DCRYPT_KEY_KIND_PUBLIC: + printf("kind: DCRYPT_KEY_KIND_PUBLIC\n"); + break; + case DCRYPT_KEY_KIND_PRIVATE: + printf("kind: DCRYPT_KEY_KIND_PRIVATE\n"); + break; + } + + switch (encryption_type) { + case DCRYPT_KEY_ENCRYPTION_TYPE_NONE: + printf("encryption_type: DCRYPT_KEY_ENCRYPTION_TYPE_NONE\n"); + break; + case DCRYPT_KEY_ENCRYPTION_TYPE_KEY: + printf("encryption_type: DCRYPT_KEY_ENCRYPTION_TYPE_KEY\n"); + break; + case DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD: + printf("encryption_type: DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD\n"); + break; + } + + if (encryption_key_hash != NULL) + printf("encryption_key_hash: %s\n", encryption_key_hash); + if (key_hash != NULL) + printf("key_hash: %s\n", key_hash); + + const char *data = t_str_rtrim(buf, "\r\n\t "); + switch (kind) { + case DCRYPT_KEY_KIND_PUBLIC: + dcrypt_dump_public_key_metadata(data); + break; + case DCRYPT_KEY_KIND_PRIVATE: + if (encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE) + dcrypt_dump_private_key_metadata(data); + break; + } + return TRUE; +} + +static bool test_dump_dcrypt_key(const char *path) +{ + if (!dcrypt_initialize("openssl", NULL, NULL)) + return FALSE; + bool ret = dcrypt_key_dump_metadata(path, FALSE); + return ret; +} + +static void +cmd_dump_dcrypt_key(const char *path, const char *const *args ATTR_UNUSED) +{ + const char *error = NULL; + if (!dcrypt_initialize("openssl", NULL, &error)) + i_fatal("dcrypt_initialize: %s", error); + (void)dcrypt_key_dump_metadata(path, TRUE); +} + +struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_key = { + "dcrypt-key", + test_dump_dcrypt_key, + cmd_dump_dcrypt_key +}; diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c new file mode 100644 index 0000000..ec21406 --- /dev/null +++ b/src/doveadm/doveadm-dump-index.c @@ -0,0 +1,833 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "hex-binary.h" +#include "file-lock.h" +#include "message-parser.h" +#include "message-part-serialize.h" +#include "mail-cache-private.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "doveadm-dump.h" + +#include <stdio.h> +#include <time.h> + +struct index_vsize_header { + uint64_t vsize; + uint32_t highest_uid; + uint32_t message_count; +}; +struct maildir_index_header { + uint32_t new_check_time, new_mtime, new_mtime_nsecs; + uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs; + uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size; +}; +struct mbox_index_header { + uint64_t sync_size; + uint32_t sync_mtime; + uint8_t dirty_flag; + uint8_t unused[3]; + uint8_t mailbox_guid[16]; +}; +struct sdbox_index_header { + uint32_t rebuild_count; + guid_128_t mailbox_guid; + uint8_t flags; + uint8_t unused[3]; +}; +struct mdbox_index_header { + uint32_t map_uid_validity; + guid_128_t mailbox_guid; + uint8_t flags; + uint8_t unused[3]; +}; +struct mdbox_mail_index_record { + uint32_t map_uid; + uint32_t save_date; +}; +struct obox_mail_index_record { + unsigned char guid[GUID_128_SIZE]; + unsigned char oid[GUID_128_SIZE]; +}; +struct mobox_mail_index_header { + uint32_t rebuild_count; + uint32_t map_uid_validity; + uint8_t unused[4]; + guid_128_t mailbox_guid; +}; +struct mobox_mail_index_record { + uint32_t map_uid; + uint32_t save_date; +}; +struct mobox_map_mail_index_header { + uint32_t rebuild_count; +}; + +struct mobox_map_mail_index_record { + uint32_t offset; + uint32_t size; + guid_128_t oid; +}; +struct mailbox_list_index_header { + uint8_t refresh_flag; + /* array of { uint32_t id; char name[]; } */ +}; +struct mailbox_list_index_record { + uint32_t name_id; + uint32_t parent_uid; + guid_128_t guid; + uint32_t uid_validity; +}; +struct mailbox_list_index_msgs_record { + uint32_t messages; + uint32_t unseen; + uint32_t recent; + uint32_t uidnext; +}; + +struct fts_index_header { + uint32_t last_indexed_uid; + uint32_t settings_checksum; + uint32_t unused; +}; +struct virtual_mail_index_header { + uint32_t change_counter; + uint32_t mailbox_count; + uint32_t highest_mailbox_id; + uint32_t search_args_crc32; +}; +struct virtual_mail_index_mailbox_record { + uint32_t id; + uint32_t name_len; + uint32_t uid_validity; + uint32_t next_uid; + uint64_t highest_modseq; +}; +struct virtual_mail_index_record { + uint32_t mailbox_id; + uint32_t real_uid; +}; + +struct mdbox_mail_index_map_record { + uint32_t file_id; + uint32_t offset; + uint32_t size; +}; + +static void dump_hdr(struct mail_index *index) +{ + const struct mail_index_header *hdr = &index->map->hdr; + unsigned int i; + + printf("version .................. = %u.%u\n", hdr->major_version, hdr->minor_version); + printf("base header size ......... = %u\n", hdr->base_header_size); + printf("header size .............. = %u\n", hdr->header_size); + printf("record size .............. = %u\n", hdr->record_size); + printf("compat flags ............. = %u\n", hdr->compat_flags); + printf("index id ................. = %u (%s)\n", hdr->indexid, unixdate2str(hdr->indexid)); + printf("flags .................... = %u\n", hdr->flags); + printf("uid validity ............. = %u (%s)\n", hdr->uid_validity, unixdate2str(hdr->uid_validity)); + printf("next uid ................. = %u\n", hdr->next_uid); + printf("messages count ........... = %u\n", hdr->messages_count); + printf("seen messages count ...... = %u\n", hdr->seen_messages_count); + printf("deleted messages count ... = %u\n", hdr->deleted_messages_count); + printf("first recent uid ......... = %u\n", hdr->first_recent_uid); + printf("first unseen uid lowwater = %u\n", hdr->first_unseen_uid_lowwater); + printf("first deleted uid lowwater = %u\n", hdr->first_deleted_uid_lowwater); + printf("log file seq ............. = %u\n", hdr->log_file_seq); + if (hdr->minor_version == 0) { + printf("log file int offset ...... = %u\n", hdr->log_file_tail_offset); + printf("log file ext offset ...... = %u\n", hdr->log_file_head_offset); + } else { + printf("log file tail offset ..... = %u\n", hdr->log_file_tail_offset); + printf("log file head offset ..... = %u\n", hdr->log_file_head_offset); + } + if (hdr->minor_version >= 3) { + printf("log2 rotate time ......... = %u (%s)\n", hdr->log2_rotate_time, unixdate2str(hdr->log2_rotate_time)); + printf("last temp file scan ...... = %u (%s)\n", hdr->last_temp_file_scan, unixdate2str(hdr->last_temp_file_scan)); + } + printf("day stamp ................ = %u (%s)\n", hdr->day_stamp, unixdate2str(hdr->day_stamp)); + for (i = 0; i < N_ELEMENTS(hdr->day_first_uid); i++) + printf("day first uid[%u] ......... = %u\n", i, hdr->day_first_uid[i]); +} + +static void dump_list_header(const void *data, size_t size) +{ + const struct mailbox_list_index_header *hdr = data; + const void *name_start, *p; + size_t i, len; + uint32_t id; + + printf(" - refresh_flag = %d\n", hdr->refresh_flag); + for (i = sizeof(*hdr); i < size; ) { + /* get id */ + if (i + sizeof(id) > size) { + printf(" - corrupted\n"); + break; + } + memcpy(&id, CONST_PTR_OFFSET(data, i), sizeof(id)); + i += sizeof(id); + + if (id == 0) + break; + + /* get name */ + p = memchr(CONST_PTR_OFFSET(data, i), '\0', size-i); + if (p == NULL) { + printf(" - corrupted\n"); + break; + } + name_start = CONST_PTR_OFFSET(data, i); + len = (const char *)p - (const char *)name_start; + + printf(" - %d : %.*s\n", id, (int)len, (const char *)name_start); + + i += len + 1; + } +} + +static void dump_box_name_header(const void *data, size_t size) +{ + char *dest = t_malloc0(size + 1); + memcpy(dest, data, size); + for (size_t i = 0; i < size; i++) { + if (dest[i] == '\0') + dest[i] = '\n'; + } + printf(" %s\n", t_strarray_join(t_strsplit(dest, "\n"), "\n ")); +} + +static void dump_extension_header(struct mail_index *index, + const struct mail_index_ext *ext) +{ + const void *data; + void *buf; + + if (strcmp(ext->name, MAIL_INDEX_EXT_KEYWORDS) == 0) + return; + + /* add some padding, since we don't bother to handle undersized + headers correctly */ + buf = t_malloc0(MALLOC_ADD(ext->hdr_size, 128)); + data = MAIL_INDEX_MAP_HDR_OFFSET(index->map, ext->hdr_offset); + memcpy(buf, data, ext->hdr_size); + data = buf; + + if (strcmp(ext->name, "hdr-vsize") == 0) { + const struct index_vsize_header *hdr = data; + + printf("header\n"); + printf(" - highest uid . = %u\n", hdr->highest_uid); + printf(" - message count = %u\n", hdr->message_count); + printf(" - vsize ....... = %"PRIu64"\n", hdr->vsize); + } else if (strcmp(ext->name, "maildir") == 0) { + const struct maildir_index_header *hdr = data; + + printf("header\n"); + printf(" - new_check_time .... = %s\n", unixdate2str(hdr->new_check_time)); + printf(" - new_mtime ......... = %s\n", unixdate2str(hdr->new_mtime)); + printf(" - new_mtime_nsecs ... = %u\n", hdr->new_mtime_nsecs); + printf(" - cur_check_time .... = %s\n", unixdate2str(hdr->cur_check_time)); + printf(" - cur_mtime ......... = %s\n", unixdate2str(hdr->cur_mtime)); + printf(" - cur_mtime_nsecs.... = %u\n", hdr->cur_mtime_nsecs); + printf(" - uidlist_mtime ..... = %s\n", unixdate2str(hdr->uidlist_mtime)); + printf(" - uidlist_mtime_nsecs = %u\n", hdr->uidlist_mtime_nsecs); + printf(" - uidlist_size ...... = %u\n", hdr->uidlist_size); + } else if (strcmp(ext->name, "mbox") == 0) { + const struct mbox_index_header *hdr = data; + + printf("header\n"); + printf(" - sync_mtime . = %s\n", unixdate2str(hdr->sync_mtime)); + printf(" - sync_size .. = %"PRIu64"\n", hdr->sync_size); + printf(" - dirty_flag . = %d\n", hdr->dirty_flag); + printf(" - mailbox_guid = %s\n", + guid_128_to_string(hdr->mailbox_guid)); + } else if (strcmp(ext->name, "mdbox-hdr") == 0) { + const struct mdbox_index_header *hdr = data; + + printf("header\n"); + printf(" - map_uid_validity .. = %u\n", hdr->map_uid_validity); + printf(" - mailbox_guid ...... = %s\n", + guid_128_to_string(hdr->mailbox_guid)); + printf(" - flags ............. = 0x%x\n", hdr->flags); + } else if (strcmp(ext->name, "dbox-hdr") == 0) { + const struct sdbox_index_header *hdr = data; + + printf("header\n"); + printf(" - rebuild_count . = %u\n", hdr->rebuild_count); + printf(" - mailbox_guid .. = %s\n", + guid_128_to_string(hdr->mailbox_guid)); + printf(" - flags ......... = 0x%x\n", hdr->flags); + } else if (strcmp(ext->name, "mobox-hdr") == 0) { + const struct mobox_mail_index_header *hdr = data; + + printf("header\n"); + printf(" - rebuild_count .. = %u\n", hdr->rebuild_count); + printf(" - map_uid_validity .. = %u\n", hdr->map_uid_validity); + printf(" - mailbox_guid ...... = %s\n", + guid_128_to_string(hdr->mailbox_guid)); + } else if (strcmp(ext->name, "mobox-map") == 0) { + const struct mobox_map_mail_index_header *hdr = data; + + printf("header\n"); + printf(" - rebuild_count .. = %u\n", hdr->rebuild_count); + } else if (strcmp(ext->name, "modseq") == 0) { + const struct mail_index_modseq_header *hdr = data; + + printf("header\n"); + printf(" - highest_modseq = %"PRIu64"\n", hdr->highest_modseq); + printf(" - log_seq ...... = %u\n", hdr->log_seq); + printf(" - log_offset ... = %u\n", hdr->log_offset); + } else if (strcmp(ext->name, "fts") == 0) { + const struct fts_index_header *hdr = data; + + printf("header\n"); + printf(" - last_indexed_uid ..... = %u\n", + hdr->last_indexed_uid); + printf(" - settings_checksum .... = %u\n", + hdr->settings_checksum); + } else if (strcmp(ext->name, "virtual") == 0) { + const struct virtual_mail_index_header *hdr = data; + const struct virtual_mail_index_mailbox_record *rec; + const unsigned char *name; + unsigned int i; + + printf("header\n"); + printf(" - change_counter ... = %u\n", hdr->change_counter); + printf(" - mailbox_count .... = %u\n", hdr->mailbox_count); + printf(" - highest_mailbox_id = %u\n", hdr->highest_mailbox_id); + printf(" - search_args_crc32 = %u\n", hdr->search_args_crc32); + + rec = CONST_PTR_OFFSET(hdr, sizeof(*hdr)); + name = CONST_PTR_OFFSET(rec, sizeof(*rec) * hdr->mailbox_count); + for (i = 0; i < hdr->mailbox_count; i++, rec++) { + printf("mailbox %s:\n", t_strndup(name, rec->name_len)); + printf(" - id ........... = %u\n", rec->id); + printf(" - uid_validity . = %u\n", rec->uid_validity); + printf(" - next_uid ..... = %u\n", rec->next_uid); + printf(" - highest_modseq = %"PRIu64"\n", + rec->highest_modseq); + + name += rec->name_len; + } + } else if (strcmp(ext->name, "list") == 0) { + printf("header ........ = %s\n", + binary_to_hex(data, ext->hdr_size)); + dump_list_header(data, ext->hdr_size); + } else if (strcmp(ext->name, "box-name") == 0) { + printf("header ........ = %s\n", + binary_to_hex(data, ext->hdr_size)); + dump_box_name_header(data, ext->hdr_size); + } else if (strcmp(ext->name, "hdr-pop3-uidl") == 0) { + const struct mailbox_index_pop3_uidl *hdr = data; + + printf("header\n"); + printf(" - max_uid_with_pop3_uidl = %u\n", + hdr->max_uid_with_pop3_uidl); + } else { + printf("header ........ = %s\n", + binary_to_hex(data, ext->hdr_size)); + } +} + +static void dump_extensions(struct mail_index *index) +{ + const struct mail_index_ext *extensions; + unsigned int i, count; + + if (array_is_created(&index->map->extensions)) + extensions = array_get(&index->map->extensions, &count); + else + count = 0; + if (count == 0) { + printf("no extensions\n"); + return; + } + + for (i = 0; i < count; i++) { + const struct mail_index_ext *ext = &extensions[i]; + + printf("-- Extension %u --\n", i); + printf("name ........ = %s\n", ext->name); + printf("hdr_size .... = %u\n", ext->hdr_size); + printf("reset_id .... = %u\n", ext->reset_id); + printf("record_offset = %u\n", ext->record_offset); + printf("record_size . = %u\n", ext->record_size); + printf("record_align = %u\n", ext->record_align); + if (ext->hdr_size > 0) T_BEGIN { + dump_extension_header(index, ext); + } T_END; + } +} + +static void dump_keywords(struct mail_index *index) +{ + const unsigned int *kw_indexes; + const char *const *keywords; + unsigned int i, count; + + printf("-- Keywords --\n"); + if (!array_is_created(&index->map->keyword_idx_map)) + return; + + kw_indexes = array_get(&index->map->keyword_idx_map, &count); + if (count == 0) + return; + + keywords = array_front(&index->keywords); + for (i = 0; i < count; i++) + printf("%3u = %s\n", i, keywords[kw_indexes[i]]); +} + +static const char *cache_decision2str(enum mail_cache_decision_type type) +{ + const char *str; + + switch (type & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) { + case MAIL_CACHE_DECISION_NO: + str = "no"; + break; + case MAIL_CACHE_DECISION_TEMP: + str = "tmp"; + break; + case MAIL_CACHE_DECISION_YES: + str = "yes"; + break; + default: + return t_strdup_printf("0x%x", type); + } + + if ((type & MAIL_CACHE_DECISION_FORCED) != 0) + str = t_strconcat(str, "!", NULL); + return str; +} + +#define CACHE_TYPE_IS_FIXED_SIZE(type) \ + ((type) == MAIL_CACHE_FIELD_FIXED_SIZE || \ + (type) == MAIL_CACHE_FIELD_BITMASK) +static const char *cache_type2str(enum mail_cache_field_type type) +{ + switch (type) { + case MAIL_CACHE_FIELD_FIXED_SIZE: + return "fix"; + case MAIL_CACHE_FIELD_VARIABLE_SIZE: + return "var"; + case MAIL_CACHE_FIELD_STRING: + return "str"; + case MAIL_CACHE_FIELD_BITMASK: + return "bit"; + case MAIL_CACHE_FIELD_HEADER: + return "hdr"; + default: + return t_strdup_printf("0x%x", type); + } +} + +static void dump_cache_hdr(struct mail_cache *cache) +{ + const struct mail_cache_header *hdr; + const struct mail_cache_field *fields, *field; + unsigned int i, count, cache_idx; + + (void)mail_cache_open_and_verify(cache); + if (MAIL_CACHE_IS_UNUSABLE(cache)) { + printf("cache is unusable\n"); + return; + } + + hdr = cache->hdr; + printf("major version ........ = %u\n", hdr->major_version); + printf("minor version ........ = %u\n", hdr->minor_version); + printf("indexid .............. = %u (%s)\n", hdr->indexid, unixdate2str(hdr->indexid)); + printf("file_seq ............. = %u (%s) (%d purges)\n", + hdr->file_seq, unixdate2str(hdr->file_seq), + hdr->file_seq - hdr->indexid); + printf("continued_record_count = %u\n", hdr->continued_record_count); + printf("record_count ......... = %u\n", hdr->record_count); + printf("used_file_size (old) . = %u\n", hdr->backwards_compat_used_file_size); + printf("deleted_record_count . = %u\n", hdr->deleted_record_count); + printf("field_header_offset .. = %u (0x%08x nontranslated)\n", + mail_index_offset_to_uint32(hdr->field_header_offset), + hdr->field_header_offset); + + printf("-- Cache fields --\n"); + fields = mail_cache_register_get_list(cache, pool_datastack_create(), + &count); + printf( +" # Name Type Size Dec Last used\n"); + for (i = 0; i < cache->file_fields_count; i++) { + cache_idx = cache->file_field_map[i]; + field = &fields[cache_idx]; + + printf("%2u: %-44s %-4s ", i, field->name, + cache_type2str(field->type)); + if (field->field_size != (uint32_t)-1 || + CACHE_TYPE_IS_FIXED_SIZE(field->type)) + printf("%4u ", field->field_size); + else + printf(" - "); + printf("%-4s %.16s\n", + cache_decision2str(field->decision), + unixdate2str(field->last_used)); + } +} + +static void dump_message_part(string_t *str, const struct message_part *part) +{ + for (; part != NULL; part = part->next) { + str_append_c(str, '('); + str_printfa(str, "pos=%"PRIuUOFF_T" ", part->physical_pos); + str_printfa(str, "hdr.p=%"PRIuUOFF_T" ", part->header_size.physical_size); + str_printfa(str, "hdr.v=%"PRIuUOFF_T" ", part->header_size.virtual_size); + str_printfa(str, "body.p=%"PRIuUOFF_T" ", part->body_size.physical_size); + str_printfa(str, "body.v=%"PRIuUOFF_T" ", part->body_size.virtual_size); + str_printfa(str, "flags=%x", part->flags); + if (part->children != NULL) { + str_append_c(str, ' '); + dump_message_part(str, part->children); + } + str_append_c(str, ')'); + } +} + +static void +dump_cache_mime_parts(string_t *str, const void *data, unsigned int size) +{ + const struct message_part *part; + const char *error; + + str_append_c(str, ' '); + + part = message_part_deserialize(pool_datastack_create(), data, size, &error); + if (part == NULL) { + str_printfa(str, "error: %s", error); + return; + } + + dump_message_part(str, part); +} + +static void +dump_cache_append_string(string_t *str, const unsigned char *data, + unsigned int size) +{ + /* cached strings end with NUL */ + if (size > 0 && data[size-1] == '\0') + size--; + str_append_data(str, data, size); +} + +static void +dump_cache_snippet(string_t *str, const unsigned char *data, unsigned int size) +{ + if (size == 0) + return; + str_printfa(str, " (version=%u: ", data[0]); + dump_cache_append_string(str, data+1, size-1); + str_append_c(str, ')'); +} + +static void dump_cache(struct mail_cache_view *cache_view, unsigned int seq) +{ + struct mail_cache_lookup_iterate_ctx iter; + const struct mail_cache_record *prev_rec = NULL; + const struct mail_cache_field *field; + struct mail_cache_iterate_field iter_field; + const void *data; + unsigned int size; + string_t *str; + int ret; + + str = t_str_new(512); + mail_cache_lookup_iter_init(cache_view, seq, &iter); + while ((ret = mail_cache_lookup_iter_next(&iter, &iter_field)) > 0) { + if (iter.rec != prev_rec) { + printf(" - cache offset=%u size=%u, prev_offset = %u\n", + iter.offset, iter.rec->size, + iter.rec->prev_offset); + prev_rec = iter.rec; + } + + field = &cache_view->cache->fields[iter_field.field_idx].field; + data = iter_field.data; + size = iter_field.size; + + str_truncate(str, 0); + str_printfa(str, " - %s: ", field->name); + switch (field->type) { + case MAIL_CACHE_FIELD_FIXED_SIZE: + if (size == sizeof(uint32_t)) { + uint32_t value; + memcpy(&value, data, sizeof(value)); + str_printfa(str, "%u ", value); + } else if (size == sizeof(uint64_t)) { + uint64_t value; + memcpy(&value, data, sizeof(value)); + str_printfa(str, "%"PRIu64, value); + } + /* fall through */ + case MAIL_CACHE_FIELD_VARIABLE_SIZE: + case MAIL_CACHE_FIELD_BITMASK: + str_printfa(str, "(%s)", binary_to_hex(data, size)); + if (strcmp(field->name, "mime.parts") == 0) + dump_cache_mime_parts(str, data, size); + else if (strcmp(field->name, "body.snippet") == 0) + dump_cache_snippet(str, data, size); + break; + case MAIL_CACHE_FIELD_STRING: + dump_cache_append_string(str, data, size); + break; + case MAIL_CACHE_FIELD_HEADER: { + const uint32_t *lines = data; + int i; + + for (i = 0;; i++) { + if (size < sizeof(uint32_t)) { + if (i == 0 && size == 0) { + /* header doesn't exist */ + break; + } + + str_append(str, "\n - BROKEN: header field doesn't end with 0 line"); + size = 0; + break; + } + + size -= sizeof(uint32_t); + data = CONST_PTR_OFFSET(data, sizeof(uint32_t)); + if (lines[i] == 0) + break; + + if (i > 0) + str_append(str, ", "); + str_printfa(str, "%u", lines[i]); + } + + if (i == 1 && size > 0 && + ((const char *)data)[size-1] == '\n') + size--; + if (size > 0) + str_printfa(str, ": %.*s", (int)size, (const char *)data); + break; + } + case MAIL_CACHE_FIELD_COUNT: + i_unreached(); + break; + } + + fwrite(str_data(str), 1, str_len(str), stdout); + putchar('\n'); + } + if (ret < 0) + printf(" - broken cache\n"); +} + +static const char *flags2str(enum mail_flags flags) +{ + string_t *str; + + str = t_str_new(64); + str_append_c(str, '('); + if ((flags & MAIL_SEEN) != 0) + str_append(str, "Seen "); + if ((flags & MAIL_ANSWERED) != 0) + str_append(str, "Answered "); + if ((flags & MAIL_FLAGGED) != 0) + str_append(str, "Flagged "); + if ((flags & MAIL_DELETED) != 0) + str_append(str, "Deleted "); + if ((flags & MAIL_DRAFT) != 0) + str_append(str, "Draft "); + if (str_len(str) == 1) + return ""; + + str_truncate(str, str_len(str)-1); + str_append_c(str, ')'); + return str_c(str); +} + +static void dump_record(struct mail_index_view *view, unsigned int seq) +{ + struct mail_index *index = mail_index_view_get_index(view); + const struct mail_index_record *rec; + const struct mail_index_registered_ext *ext; + const void *data; + unsigned int i, ext_count; + string_t *str; + bool expunged; + + rec = mail_index_lookup(view, seq); + printf("RECORD: seq=%u, uid=%u, flags=0x%02x %s\n", + seq, rec->uid, rec->flags, flags2str(rec->flags)); + + str = t_str_new(256); + ext = array_get(&index->extensions, &ext_count); + for (i = 0; i < ext_count; i++) { + mail_index_lookup_ext(view, seq, i, &data, &expunged); + if (data == NULL || ext[i].record_size == 0) + continue; + + str_truncate(str, 0); + str_printfa(str, " - ext %d %-10s: ", i, ext[i].name); + if (ext[i].record_size == sizeof(uint16_t) && + ext[i].record_align == sizeof(uint16_t)) + str_printfa(str, "%10u", *((const uint16_t *)data)); + else if (ext[i].record_size == sizeof(uint32_t) && + ext[i].record_align == sizeof(uint32_t)) + str_printfa(str, "%10u", *((const uint32_t *)data)); + else if (ext[i].record_size == sizeof(uint64_t) && + ext[i].record_align == sizeof(uint64_t)) { + uint64_t value = *((const uint64_t *)data); + str_printfa(str, "%10"PRIu64, value); + } else { + str_append(str, " "); + } + str_printfa(str, " (%s)", + binary_to_hex(data, ext[i].record_size)); + printf("%s\n", str_c(str)); + if (strcmp(ext[i].name, "virtual") == 0) { + const struct virtual_mail_index_record *vrec = data; + printf(" : mailbox_id = %u\n", vrec->mailbox_id); + printf(" : real_uid = %u\n", vrec->real_uid); + } else if (strcmp(ext[i].name, "map") == 0) { + const struct mdbox_mail_index_map_record *mrec = data; + printf(" : file_id = %u\n", mrec->file_id); + printf(" : offset = %u\n", mrec->offset); + printf(" : size = %u\n", mrec->size); + } else if (strcmp(ext[i].name, "mdbox") == 0) { + const struct mdbox_mail_index_record *drec = data; + printf(" : map_uid = %u\n", drec->map_uid); + printf(" : save_date = %u (%s)\n", drec->save_date, unixdate2str(drec->save_date)); + } else if (strcmp(ext[i].name, "obox") == 0) { + const struct obox_mail_index_record *orec = data; + printf(" : guid = %s\n", guid_128_to_string(orec->guid)); + printf(" : oid = %s\n", binary_to_hex(orec->oid, ext[i].record_size - sizeof(orec->guid))); + } else if (strcmp(ext[i].name, "mobox") == 0) { + const struct mobox_mail_index_record *orec = data; + printf(" : map_uid = %u\n", orec->map_uid); + printf(" : save_date = %u (%s)\n", orec->save_date, unixdate2str(orec->save_date)); + } else if (strcmp(ext[i].name, "mobox-map") == 0) { + const struct mobox_map_mail_index_record *orec = data; + printf(" : offset = %u\n", orec->offset); + printf(" : size = %u\n", orec->size); + printf(" : oid = %s\n", guid_128_to_string(orec->oid)); + } else if (strcmp(ext[i].name, "list") == 0) { + const struct mailbox_list_index_record *lrec = data; + printf(" : name_id = %u\n", lrec->name_id); + printf(" : parent_uid = %u\n", lrec->parent_uid); + printf(" : guid = %s\n", guid_128_to_string(lrec->guid)); + printf(" : uid_validity = %u\n", lrec->uid_validity); + } else if (strcmp(ext[i].name, "msgs") == 0) { + const struct mailbox_list_index_msgs_record *lrec = data; + printf(" : messages = %u\n", lrec->messages); + printf(" : unseen = %u\n", lrec->unseen); + printf(" : recent = %u\n", lrec->recent); + printf(" : uidnext = %u\n", lrec->uidnext); + } else if (strcmp(ext[i].name, "vsize") == 0 && + ext[i].record_size >= sizeof(struct mailbox_index_vsize)) { + /* this is "vsize" in dovecot.list.index, not the + 32bit "vsize" in dovecot.index */ + const struct mailbox_index_vsize *vrec = data; + printf(" : vsize = %"PRIu64"\n", vrec->vsize); + printf(" : highest_uid = %u\n", vrec->highest_uid); + printf(" : message_count = %u\n", vrec->message_count); + } + } +} + +static bool dir_has_index(const char *dir, const char *name) +{ + struct stat st; + + return stat(t_strconcat(dir, "/", name, NULL), &st) == 0 || + stat(t_strconcat(dir, "/", name, ".log", NULL), &st) == 0; +} + +static struct mail_index *path_open_index(const char *path) +{ + struct stat st; + const char *p; + + if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) { + if (dir_has_index(path, "dovecot.index")) + return mail_index_alloc(NULL, path, "dovecot.index"); + else if (dir_has_index(path, "dovecot.map.index")) + return mail_index_alloc(NULL, path, "dovecot.map.index"); + else + return NULL; + } else if ((p = strrchr(path, '/')) != NULL) + return mail_index_alloc(NULL, t_strdup_until(path, p), p + 1); + else + return mail_index_alloc(NULL, ".", path); +} + +static void +cmd_dump_index(const char *path, const char *const *args) +{ + struct mail_index *index; + struct mail_index_view *view; + struct mail_cache_view *cache_view; + unsigned int seq, uid = 0; + + index = path_open_index(path); + if (index == NULL || + mail_index_open(index, MAIL_INDEX_OPEN_FLAG_READONLY) <= 0) + i_fatal("Couldn't open index %s", path); + if (args[0] != NULL) { + if (str_to_uint(args[0], &uid) < 0) + i_fatal("Invalid uid number %s", args[0]); + } + + view = mail_index_view_open(index); + cache_view = mail_cache_view_open(index->cache, view); + + if (uid == 0) { + printf("-- INDEX: %s\n", index->filepath); + dump_hdr(index); + dump_extensions(index); + dump_keywords(index); + + printf("\n-- CACHE: %s\n", index->cache->filepath); + dump_cache_hdr(index->cache); + + printf("\n-- RECORDS: %u\n", index->map->hdr.messages_count); + } + for (seq = 1; seq <= index->map->hdr.messages_count; seq++) { + if (uid == 0 || mail_index_lookup(view, seq)->uid == uid) { + T_BEGIN { + dump_record(view, seq); + dump_cache(cache_view, seq); + printf("\n"); + } T_END; + } + } + mail_cache_view_close(&cache_view); + mail_index_view_close(&view); + mail_index_close(index); + mail_index_free(&index); +} + +static bool test_dump_index(const char *path) +{ + struct mail_index *index; + bool ret; + + index = path_open_index(path); + if (index == NULL) + return FALSE; + + ret = mail_index_open(index, MAIL_INDEX_OPEN_FLAG_READONLY) > 0; + if (ret) + mail_index_close(index); + mail_index_free(&index); + return ret; +} + +struct doveadm_cmd_dump doveadm_cmd_dump_index = { + "index", + test_dump_index, + cmd_dump_index +}; diff --git a/src/doveadm/doveadm-dump-log.c b/src/doveadm/doveadm-dump-log.c new file mode 100644 index 0000000..0725b28 --- /dev/null +++ b/src/doveadm/doveadm-dump-log.c @@ -0,0 +1,568 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "hex-binary.h" +#include "mail-index-private.h" +#include "mail-transaction-log-private.h" +#include "doveadm-dump.h" + +#include <stdio.h> + +static struct mail_transaction_ext_intro prev_intro; + +static void dump_hdr(struct istream *input, uint64_t *modseq_r, + unsigned int *version_r) +{ + struct mail_transaction_log_header hdr; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(input, &data, &size, sizeof(hdr)); + if (ret < 0 && input->stream_errno != 0) + i_fatal("read() failed: %s", i_stream_get_error(input)); + if (ret <= 0) { + i_fatal("file hdr read() %zu != %zu", + size, sizeof(hdr)); + } + memcpy(&hdr, data, sizeof(hdr)); + if (hdr.hdr_size < sizeof(hdr)) { + memset(PTR_OFFSET(&hdr, hdr.hdr_size), 0, + sizeof(hdr) - hdr.hdr_size); + } + i_stream_skip(input, hdr.hdr_size); + + printf("version = %u.%u\n", hdr.major_version, hdr.minor_version); + printf("hdr size = %u\n", hdr.hdr_size); + printf("index id = %u\n", hdr.indexid); + printf("file seq = %u\n", hdr.file_seq); + printf("prev file = %u/%u\n", hdr.prev_file_seq, hdr.prev_file_offset); + printf("create stamp = %u\n", hdr.create_stamp); + printf("initial modseq = %"PRIu64"\n", hdr.initial_modseq); + printf("compat flags = %x\n", hdr.compat_flags); + *modseq_r = hdr.initial_modseq; + *version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr); +} + +static const char *log_record_type(unsigned int type) +{ + const char *name; + + switch (type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: + name = "expunge"; + break; + case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: + name = "expunge-guid"; + break; + case MAIL_TRANSACTION_APPEND: + name = "append"; + break; + case MAIL_TRANSACTION_FLAG_UPDATE: + name = "flag-update"; + break; + case MAIL_TRANSACTION_HEADER_UPDATE: + name = "header-update"; + break; + case MAIL_TRANSACTION_EXT_INTRO: + name = "ext-intro"; + break; + case MAIL_TRANSACTION_EXT_RESET: + name = "ext-reset"; + break; + case MAIL_TRANSACTION_EXT_HDR_UPDATE: + name = "ext-hdr"; + break; + case MAIL_TRANSACTION_EXT_HDR_UPDATE32: + name = "ext-hdr32"; + break; + case MAIL_TRANSACTION_EXT_REC_UPDATE: + name = "ext-rec"; + break; + case MAIL_TRANSACTION_KEYWORD_UPDATE: + name = "keyword-update"; + break; + case MAIL_TRANSACTION_KEYWORD_RESET: + name = "keyword-reset"; + break; + case MAIL_TRANSACTION_EXT_ATOMIC_INC: + name = "ext-atomic-inc"; + break; + case MAIL_TRANSACTION_MODSEQ_UPDATE: + name = "modseq-update"; + break; + case MAIL_TRANSACTION_INDEX_DELETED: + name = "index-deleted"; + break; + case MAIL_TRANSACTION_INDEX_UNDELETED: + name = "index-undeleted"; + break; + case MAIL_TRANSACTION_BOUNDARY: + name = "boundary"; + break; + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + name = "attribute-update"; + break; + default: + name = t_strdup_printf("unknown: %x", type); + break; + } + + if ((type & MAIL_TRANSACTION_EXTERNAL) != 0) + name = t_strconcat(name, " (ext)", NULL); + if ((type & MAIL_TRANSACTION_SYNC) != 0) + name = t_strconcat(name, " (sync)", NULL); + return name; +} + +static void print_data(const void *data, size_t size) +{ + size_t i; + + for (i = 0; i < size; i++) + printf("%02x", ((const unsigned char *)data)[i]); + if (size == 4) { + const uint32_t *n = (const uint32_t *)data; + + printf(" (dec=%u)", *n); + } +} + +static void print_try_uint(const void *data, size_t size) +{ + size_t i; + + switch (size) { + case 1: { + const uint8_t *n = data; + printf("%u", *n); + break; + } + case 2: { + const uint16_t *n = data; + uint32_t n16; + + memcpy(&n16, n, sizeof(n16)); + printf("%u", n16); + break; + } + case 4: { + const uint32_t *n = data; + uint32_t n32; + + memcpy(&n32, n, sizeof(n32)); + printf("%u", n32); + break; + } + case 8: { + const uint64_t *n = data; + uint64_t n64; + + memcpy(&n64, n, sizeof(n64)); + printf("%"PRIu64, n64); + break; + } + default: + for (i = 0; i < size; i++) + printf("%02x", ((const unsigned char *)data)[i]); + } +} + +#define HDRF(field) { \ + #field, offsetof(struct mail_index_header, field), \ + sizeof(((struct mail_index_header *)0)->field) } + +static struct { + const char *name; + unsigned int offset, size; +} header_fields[] = { + HDRF(minor_version), + HDRF(base_header_size), + HDRF(header_size), + HDRF(record_size), + HDRF(compat_flags), + HDRF(indexid), + HDRF(flags), + HDRF(uid_validity), + HDRF(next_uid), + HDRF(messages_count), + HDRF(unused_old_recent_messages_count), + HDRF(seen_messages_count), + HDRF(deleted_messages_count), + HDRF(first_recent_uid), + HDRF(first_unseen_uid_lowwater), + HDRF(first_deleted_uid_lowwater), + HDRF(log_file_seq), + HDRF(log_file_tail_offset), + HDRF(log_file_head_offset), + HDRF(day_stamp) +}; + +static void log_header_update(const struct mail_transaction_header_update *u, + size_t data_size) +{ + const void *data = u + 1; + unsigned int offset = u->offset, size = u->size; + unsigned int i; + + if (sizeof(*u) + size > data_size) { + printf(" - offset = %u, size = %u (too large)\n", offset, size); + return; + } + + while (size > 0) { + /* don't bother trying to handle header updates that include + unknown/unexpected fields offsets/sizes */ + for (i = 0; i < N_ELEMENTS(header_fields); i++) { + if (header_fields[i].offset == offset && + header_fields[i].size <= size) + break; + } + + if (i == N_ELEMENTS(header_fields)) { + printf(" - offset = %u, size = %u: ", offset, size); + print_data(data, size); + printf("\n"); + break; + } + + printf(" - %s = ", header_fields[i].name); + print_try_uint(data, header_fields[i].size); + printf("\n"); + + data = CONST_PTR_OFFSET(data, header_fields[i].size); + offset += header_fields[i].size; + size -= header_fields[i].size; + } +} + +static void log_record_print(const struct mail_transaction_header *hdr, + const void *data, size_t data_size, + uint64_t *modseq) +{ + unsigned int size = mail_index_offset_to_uint32(hdr->size) - sizeof(*hdr); + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: { + const struct mail_transaction_expunge *exp = data; + + printf(" - uids="); + for (; size > 0; size -= sizeof(*exp), exp++) { + printf("%u-%u,", exp->uid1, exp->uid2); + } + printf("\n"); + break; + } + case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: { + const struct mail_transaction_expunge_guid *exp = data; + + for (; size > 0; size -= sizeof(*exp), exp++) { + printf(" - uid=%u (guid ", exp->uid); + print_data(exp->guid_128, sizeof(exp->guid_128)); + printf(")\n"); + } + break; + } + case MAIL_TRANSACTION_APPEND: { + const struct mail_index_record *rec = data; + + printf(" - uids="); + for (; size > 0; size -= sizeof(*rec), rec++) { + printf("%u", rec->uid); + if (rec->flags != 0) + printf(" (flags=%x)", rec->flags); + printf(","); + } + printf("\n"); + break; + } + case MAIL_TRANSACTION_FLAG_UPDATE: { + const struct mail_transaction_flag_update *u = data; + + for (; size > 0; size -= sizeof(*u), u++) { + printf(" - uids=%u-%u (flags +%x-%x, modseq_inc_flag=%d)\n", + u->uid1, u->uid2, u->add_flags, u->remove_flags, u->modseq_inc_flag); + } + break; + } + case MAIL_TRANSACTION_HEADER_UPDATE: { + const struct mail_transaction_header_update *u = data; + + log_header_update(u, data_size); + break; + } + case MAIL_TRANSACTION_EXT_INTRO: { + const struct mail_transaction_ext_intro *intro = data; + + prev_intro = *intro; + printf(" - ext_id = %u\n", intro->ext_id); + printf(" - reset_id = %u\n", intro->reset_id); + printf(" - hdr_size = %u\n", intro->hdr_size); + printf(" - record_size = %u\n", intro->record_size); + printf(" - record_align = %u\n", intro->record_align); + printf(" - flags = %u\n", intro->flags); + printf(" - name_size = %u\n", intro->name_size); + if (intro->name_size > 0) { + const char *name = (const char *)(intro+1); + + printf(" - name = '%.*s'\n", intro->name_size, name); + if (*modseq == 0 && intro->name_size == 6 && + memcmp(name, "modseq", 6) == 0) + *modseq = 1; + } + break; + } + case MAIL_TRANSACTION_EXT_RESET: { + const struct mail_transaction_ext_reset *reset = data; + + printf(" - new_reset_id = %u\n", reset->new_reset_id); + printf(" - preserve_data = %u\n", reset->preserve_data); + break; + } + case MAIL_TRANSACTION_EXT_HDR_UPDATE: { + const struct mail_transaction_ext_hdr_update *u = data; + + printf(" - offset = %u, size = %u", u->offset, u->size); + if (sizeof(*u) + u->size <= data_size) { + printf(": "); + print_data(u + 1, u->size); + } else { + printf(" (too large)"); + } + printf("\n"); + break; + } + case MAIL_TRANSACTION_EXT_HDR_UPDATE32: { + const struct mail_transaction_ext_hdr_update32 *u = data; + + printf(" - offset = %u, size = %u", u->offset, u->size); + if (sizeof(*u) + u->size <= data_size) { + printf(": "); + print_data(u + 1, u->size); + } else { + printf(" (too large)"); + } + printf("\n"); + break; + } + case MAIL_TRANSACTION_EXT_REC_UPDATE: { + const struct mail_transaction_ext_rec_update *rec = data, *end; + size_t record_size; + + end = CONST_PTR_OFFSET(data, size); + record_size = (sizeof(*rec) + prev_intro.record_size + 3) & ~3U; + while (rec < end) { + printf(" - uid=%u: ", rec->uid); + size_t bytes_left = (const char *)end - (const char *)(rec + 1); + if (prev_intro.record_size <= bytes_left) + print_data(rec + 1, prev_intro.record_size); + else + printf("(record_size too large)"); + printf("\n"); + rec = CONST_PTR_OFFSET(rec, record_size); + } + break; + } + case MAIL_TRANSACTION_EXT_ATOMIC_INC: { + const struct mail_transaction_ext_atomic_inc *rec = data, *end; + + end = CONST_PTR_OFFSET(data, size); + for (; rec < end; rec++) { + printf(" - uid=%u: ", rec->uid); + if (rec->diff > 0) + printf("+%d\n", rec->diff); + else + printf("%d\n", rec->diff); + } + break; + } + case MAIL_TRANSACTION_KEYWORD_UPDATE: { + const struct mail_transaction_keyword_update *u = data; + const uint32_t *uid; + unsigned int uid_offset; + + printf(" - modify=%d, name=%.*s, uids=", + u->modify_type, u->name_size, (const char *)(u+1)); + + uid_offset = sizeof(*u) + u->name_size + + ((u->name_size % 4) == 0 ? 0 : 4 - (u->name_size%4)); + uid = (const uint32_t *)((const char *)u + uid_offset); + size -= uid_offset; + + for (; size > 0; size -= sizeof(*uid)*2, uid += 2) { + printf("%u-%u,", uid[0], uid[1]); + } + printf("\n"); + break; + } + case MAIL_TRANSACTION_KEYWORD_RESET: { + const struct mail_transaction_keyword_reset *u = data; + + printf(" - uids="); + for (; size > 0; size -= sizeof(*u), u++) { + printf("%u-%u, ", u->uid1, u->uid2); + } + printf("\n"); + break; + } + case MAIL_TRANSACTION_MODSEQ_UPDATE: { + const struct mail_transaction_modseq_update *rec, *end; + + end = CONST_PTR_OFFSET(data, size); + for (rec = data; rec < end; rec++) { + printf(" - uid=%u modseq=%"PRIu64"\n", rec->uid, + ((uint64_t)rec->modseq_high32 << 32) | + rec->modseq_low32); + } + break; + } + case MAIL_TRANSACTION_INDEX_DELETED: + case MAIL_TRANSACTION_INDEX_UNDELETED: + break; + case MAIL_TRANSACTION_BOUNDARY: { + const struct mail_transaction_boundary *rec = data; + + printf(" - size=%u\n", rec->size); + break; + } + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: { + const char *keys = data; + const uint32_t *extra; + unsigned int i, extra_pos, extra_count = 0; + + for (i = 0; i < size && keys[i] != '\0'; ) { + if (keys[i] == '+') + extra_count++; + extra_count++; + i += strlen(keys+i) + 1; + } + if (i % sizeof(uint32_t) != 0) + i += sizeof(uint32_t) - i%sizeof(uint32_t); + extra = (const void *)(keys+i); + + if ((size-i) != extra_count*sizeof(uint32_t)) { + printf(" - broken entry\n"); + break; + } + + extra_pos = 0; + for (i = 0; i < size && keys[i] != '\0'; ) { + printf(" - %s: %s/%s : timestamp=%s", + keys[i] == '+' ? "add" : keys[i] == '-' ? "remove" : "?", + keys[i+1] == 'p' ? "private" : + keys[i+1] == 's' ? "shared" : "?error?", + keys+i+2, unixdate2str(extra[extra_pos++])); + if (keys[i] == '+') + printf(" value_len=%u", extra[extra_pos++]); + printf("\n"); + i += strlen(keys+i) + 1; + } + + break; + } + default: + break; + } +} + +static int dump_record(struct istream *input, uint64_t *modseq, + unsigned int version) +{ + struct mail_transaction_header hdr; + unsigned int hdr_size; + const unsigned char *data; + size_t size; + int ret; + + ret = i_stream_read_bytes(input, &data, &size, sizeof(hdr)); + if (ret < 0 && input->stream_errno != 0) + i_fatal("read() failed: %s", i_stream_get_error(input)); + if (ret <= 0) { + if (size == 0) + return 0; + i_fatal("rec hdr read() %zu != %zu", + size, sizeof(hdr)); + } + memcpy(&hdr, data, sizeof(hdr)); + + hdr_size = mail_index_offset_to_uint32(hdr.size); + if (hdr_size < sizeof(hdr)) { + printf("record: offset=%"PRIuUOFF_T", " + "type=%s, size=broken (%x)\n", + input->v_offset, log_record_type(hdr.type), hdr.size); + return 0; + } + + printf("record: offset=%"PRIuUOFF_T", type=%s, size=%u", + input->v_offset, log_record_type(hdr.type), hdr_size); + + i_stream_skip(input, sizeof(hdr)); + size_t data_size = hdr_size - sizeof(hdr); + ret = i_stream_read_bytes(input, &data, &size, data_size); + if (ret < 0 && input->stream_errno != 0) + i_fatal("read() failed: %s", i_stream_get_error(input)); + if (ret <= 0) { + i_fatal("rec data read() %zu != %zu", + size, data_size); + } + + uint64_t prev_modseq = *modseq; + mail_transaction_update_modseq(&hdr, data, modseq, version); + if (*modseq > prev_modseq) + printf(", modseq=%"PRIu64, *modseq); + printf("\n"); + + log_record_print(&hdr, data, data_size, modseq); + i_stream_skip(input, data_size); + return 1; +} + +static void cmd_dump_log(const char *path, const char *const *args ATTR_UNUSED) +{ + struct istream *input; + uint64_t modseq; + unsigned int version; + int ret; + + input = i_stream_create_file(path, SIZE_MAX); + dump_hdr(input, &modseq, &version); + do { + T_BEGIN { + ret = dump_record(input, &modseq, version); + } T_END; + } while (ret > 0); + i_stream_unref(&input); +} + +static bool test_dump_log(const char *path) +{ + struct mail_transaction_log_header hdr; + const char *p; + bool ret = FALSE; + int fd; + + p = strrchr(path, '/'); + if (p == NULL) + p = path; + p = strstr(p, ".log"); + if (p == NULL || !(p[4] == '\0' || p[4] == '.')) + return FALSE; + + fd = open(path, O_RDONLY); + if (fd == -1) + return FALSE; + + if (read(fd, &hdr, sizeof(hdr)) >= MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE && + hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION && + hdr.hdr_size >= MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE) + ret = TRUE; + i_close_fd(&fd); + return ret; +} + +struct doveadm_cmd_dump doveadm_cmd_dump_log = { + "log", + test_dump_log, + cmd_dump_log +}; diff --git a/src/doveadm/doveadm-dump-mailboxlog.c b/src/doveadm/doveadm-dump-mailboxlog.c new file mode 100644 index 0000000..8ff5a54 --- /dev/null +++ b/src/doveadm/doveadm-dump-mailboxlog.c @@ -0,0 +1,114 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hex-binary.h" +#include "mailbox-log.h" +#include "doveadm-dump.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <time.h> + +static int dump_record(int fd) +{ + off_t offset; + ssize_t ret; + struct mailbox_log_record rec; + time_t timestamp; + + offset = lseek(fd, 0, SEEK_CUR); + + ret = read(fd, &rec, sizeof(rec)); + if (ret == 0) + return 0; + + if (ret != sizeof(rec)) { + i_fatal("rec read() %zu != %zu", + ret, sizeof(rec)); + } + + printf("#%"PRIuUOFF_T": ", offset); + switch (rec.type) { + case MAILBOX_LOG_RECORD_DELETE_MAILBOX: + printf("delete-mailbox"); + break; + case MAILBOX_LOG_RECORD_DELETE_DIR: + printf("delete-dir"); + break; + case MAILBOX_LOG_RECORD_RENAME: + printf("rename"); + break; + case MAILBOX_LOG_RECORD_SUBSCRIBE: + printf("subscribe"); + break; + case MAILBOX_LOG_RECORD_UNSUBSCRIBE: + printf("unsubscribe"); + break; + case MAILBOX_LOG_RECORD_CREATE_DIR: + printf("create-dir"); + break; + } + printf(" %s", binary_to_hex(rec.mailbox_guid, + sizeof(rec.mailbox_guid))); + + timestamp = be32_to_cpu_unaligned(rec.timestamp); + printf(" (%s)\n", unixdate2str(timestamp)); + return 1; +} + +static void +cmd_dump_mailboxlog(const char *path, const char *const *args ATTR_UNUSED) +{ + int fd, ret; + + fd = open(path, O_RDONLY); + if (fd < 0) + i_fatal("open(%s) failed: %m", path); + + do { + T_BEGIN { + ret = dump_record(fd); + } T_END; + } while (ret > 0); + i_close_fd(&fd); +} + +static bool test_dump_mailboxlog(const char *path) +{ + const char *p; + int fd; + struct mailbox_log_record rec; + bool ret = FALSE; + + p = strrchr(path, '.'); + if (p == NULL || strcmp(p, ".log") != 0) + return FALSE; + + fd = open(path, O_RDONLY); + if (fd == -1) + return FALSE; + + if (read(fd, &rec, sizeof(rec)) == sizeof(rec) && + rec.padding[0] == 0 && rec.padding[1] == 0 && rec.padding[2] == 0) { + enum mailbox_log_record_type type = rec.type; + switch (type) { + case MAILBOX_LOG_RECORD_DELETE_MAILBOX: + case MAILBOX_LOG_RECORD_DELETE_DIR: + case MAILBOX_LOG_RECORD_RENAME: + case MAILBOX_LOG_RECORD_SUBSCRIBE: + case MAILBOX_LOG_RECORD_UNSUBSCRIBE: + case MAILBOX_LOG_RECORD_CREATE_DIR: + ret = TRUE; + break; + } + } + i_close_fd(&fd); + return ret; +} + +struct doveadm_cmd_dump doveadm_cmd_dump_mailboxlog = { + "mailboxlog", + test_dump_mailboxlog, + cmd_dump_mailboxlog +}; diff --git a/src/doveadm/doveadm-dump-thread.c b/src/doveadm/doveadm-dump-thread.c new file mode 100644 index 0000000..026b07f --- /dev/null +++ b/src/doveadm/doveadm-dump-thread.c @@ -0,0 +1,139 @@ +/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mmap-util.h" +#include "mail-index-private.h" +#include "mail-index-strmap.h" +#include "doveadm-dump.h" + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/stat.h> + +static uint32_t max_likely_index; + +static size_t dump_hdr(const struct mail_index_strmap_header *hdr) +{ + printf("version = %u\n", hdr->version); + printf("uid validity = %u\n", hdr->uid_validity); + return sizeof(*hdr); +} + +static int dump_record(const uint8_t **p, const uint8_t *end, uint32_t *uid) +{ + uint32_t uid_diff, n, i, count, crc32, idx; + size_t size; + + /* <uid diff> <n> <crc32>*count <str_idx>*count */ + if (mail_index_unpack_num(p, end, &uid_diff) < 0) + return -1; + *uid += uid_diff; + + if (mail_index_unpack_num(p, end, &n) < 0) + return -1; + printf(" - uid %u: n=%u\n", *uid, n); + + count = n < 2 ? n + 1 : n; + size = sizeof(crc32)*count + sizeof(idx)*count; + if (*p + size > end) + return -1; + for (i = 0; i < count; i++) { + if (i == 0) + printf(" - message-id: "); + else if (i == 1) { + if (n == 1) + printf(" - in-reply-to: "); + else + printf(" - references[1]: "); + } else { + printf(" - references[%u]: ", i); + } + memcpy(&crc32, *p + sizeof(crc32)*i, sizeof(crc32)); + memcpy(&idx, *p + sizeof(crc32)*count + sizeof(idx)*i, sizeof(idx)); + printf("crc32=%08x index=%u\n", crc32, idx); + if (idx > max_likely_index) + printf(" - index probably broken\n"); + } + *p += size; + return 0; +} + +static int dump_block(const uint8_t *data, const uint8_t *end, uint32_t *uid) +{ + const uint8_t *p; + uint32_t block_size; + + if (data + 4 >= end) + return -1; + + memcpy(&block_size, data, sizeof(block_size)); + block_size = mail_index_offset_to_uint32(block_size) >> 2; + printf(" - block_size=%u\n", block_size); + if (block_size == 0) { + /* finished */ + return -1; + } + if (data + sizeof(block_size) + block_size > end) { + printf(" - broken!\n"); + return -1; + } + p = data + sizeof(block_size); + end = p + block_size; + + *uid += 1; + while (p != end) { + if (dump_record(&p, end, uid) < 0) { + printf(" - broken\n"); + return -1; + } + } + return p - data; +} + +static void +cmd_dump_thread(const char *path, const char *const *args ATTR_UNUSED) +{ + unsigned int pos; + const void *map, *end; + struct stat st; + uint32_t uid; + int fd, ret; + + fd = open(path, O_RDONLY); + if (fd < 0) + i_fatal("open(%s) failed: %m", path); + + if (fstat(fd, &st) < 0) + i_fatal("fstat(%s) failed: %m", path); + max_likely_index = (st.st_size / 8) * 2; + + map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); + if (map == MAP_FAILED) + i_fatal("mmap() failed: %m"); + end = CONST_PTR_OFFSET(map, st.st_size); + pos = dump_hdr(map); + uid = 0; + do { + printf("block at offset %u:\n", pos); + T_BEGIN { + ret = dump_block(CONST_PTR_OFFSET(map, pos), end, &uid); + pos += ret; + } T_END; + } while (ret > 0); + i_close_fd(&fd); +} + +static bool test_dump_thread(const char *path) +{ + const char *p; + + p = strrchr(path, '.'); + return p != NULL && strcmp(p, ".thread") == 0; +} + +struct doveadm_cmd_dump doveadm_cmd_dump_thread = { + "thread", + test_dump_thread, + cmd_dump_thread +}; diff --git a/src/doveadm/doveadm-dump.c b/src/doveadm/doveadm-dump.c new file mode 100644 index 0000000..eb08e96 --- /dev/null +++ b/src/doveadm/doveadm-dump.c @@ -0,0 +1,162 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "istream-multiplex.h" +#include "doveadm.h" +#include "doveadm-dump.h" + +#include <stdio.h> +#include <unistd.h> + +static ARRAY(const struct doveadm_cmd_dump *) dumps; + +void doveadm_dump_register(const struct doveadm_cmd_dump *dump) +{ + array_push_back(&dumps, &dump); +} + +static const struct doveadm_cmd_dump * +dump_find_name(const char *name) +{ + const struct doveadm_cmd_dump *dump; + + array_foreach_elem(&dumps, dump) { + if (strcmp(dump->name, name) == 0) + return dump; + } + return NULL; +} + +static const struct doveadm_cmd_dump * +dump_find_test(const char *path) +{ + const struct doveadm_cmd_dump *dump; + + array_foreach_elem(&dumps, dump) { + if (dump->test != NULL && dump->test(path)) + return dump; + } + return NULL; +} + +static void cmd_dump(struct doveadm_cmd_context *cctx) +{ + const struct doveadm_cmd_dump *dump; + const char *path, *type = NULL, *const *args = NULL; + const char *no_args = NULL; + + if (!doveadm_cmd_param_str(cctx, "path", &path)) + help_ver2(&doveadm_cmd_dump); + (void)doveadm_cmd_param_str(cctx, "type", &type); + (void)doveadm_cmd_param_array(cctx, "args", &args); + + dump = type != NULL ? dump_find_name(type) : dump_find_test(path); + if (dump == NULL) { + if (type != NULL) { + print_dump_types(); + i_fatal_status(EX_USAGE, "Unknown type: %s", type); + } else { + i_fatal_status(EX_DATAERR, + "Can't autodetect file type: %s", path); + } + } else { + if (type == NULL) + printf("Detected file type: %s\n", dump->name); + } + dump->cmd(path, args != NULL ? args : &no_args); +} + +struct doveadm_cmd_ver2 doveadm_cmd_dump = { + .name = "dump", + .cmd = cmd_dump, + .usage = "[-t <type>] <path> [<type-specific args>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('t', "type", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +static void +cmd_dump_multiplex(const char *path, const char *const *args ATTR_UNUSED) +{ + const unsigned int channels_count = 256; + struct istream *file_input, *channels[channels_count]; + const unsigned char *data; + size_t size; + unsigned int i; + + file_input = i_stream_create_file(path, IO_BLOCK_SIZE); + /* A bit kludgy: istream-multiplex returns 0 if a wrong channel is + being read from. This causes a panic with blocking istreams. + Work around this by assuming that the file istream isn't blocking. */ + file_input->blocking = FALSE; + channels[0] = i_stream_create_multiplex(file_input, IO_BLOCK_SIZE); + i_stream_unref(&file_input); + + for (i = 1; i < channels_count; i++) + channels[i] = i_stream_multiplex_add_channel(channels[0], i); + + bool have_input; + do { + have_input = FALSE; + for (i = 0; i < channels_count; i++) { + if (i_stream_read_more(channels[i], &data, &size) > 0) { + printf("CHANNEL %u: %zu bytes:\n", i, size); + fwrite(data, 1, size, stdout); + printf("\n"); + have_input = TRUE; + i_stream_skip(channels[i], size); + } + } + } while (have_input); + + if (channels[0]->stream_errno != 0) + i_error("read() failed: %s", i_stream_get_error(channels[0])); + for (i = 0; i < channels_count; i++) + i_stream_unref(&channels[i]); +} + +struct doveadm_cmd_dump doveadm_cmd_dump_multiplex = { + "multiplex", + NULL, + cmd_dump_multiplex +}; + +static const struct doveadm_cmd_dump *dumps_builtin[] = { + &doveadm_cmd_dump_dbox, + &doveadm_cmd_dump_index, + &doveadm_cmd_dump_log, + &doveadm_cmd_dump_mailboxlog, + &doveadm_cmd_dump_thread, + &doveadm_cmd_dump_zlib, + &doveadm_cmd_dump_dcrypt_file, + &doveadm_cmd_dump_dcrypt_key, + &doveadm_cmd_dump_multiplex, +}; + +void print_dump_types(void) +{ + unsigned int i; + + fprintf(stderr, "Available dump types: %s", dumps_builtin[0]->name); + for (i = 1; i < N_ELEMENTS(dumps_builtin); i++) + fprintf(stderr, " %s", dumps_builtin[i]->name); + fprintf(stderr, "\n"); +} + +void doveadm_dump_init(void) +{ + unsigned int i; + + i_array_init(&dumps, N_ELEMENTS(dumps_builtin) + 8); + for (i = 0; i < N_ELEMENTS(dumps_builtin); i++) + doveadm_dump_register(dumps_builtin[i]); +} + +void doveadm_dump_deinit(void) +{ + array_free(&dumps); +} diff --git a/src/doveadm/doveadm-dump.h b/src/doveadm/doveadm-dump.h new file mode 100644 index 0000000..5776260 --- /dev/null +++ b/src/doveadm/doveadm-dump.h @@ -0,0 +1,27 @@ +#ifndef DOVEADM_DUMP_H +#define DOVEADM_DUMP_H + +#include "doveadm.h" + +struct doveadm_cmd_dump { + const char *name; + bool (*test)(const char *path); + void (*cmd)(const char *path, const char *const *args); +}; + +extern struct doveadm_cmd_dump doveadm_cmd_dump_dbox; +extern struct doveadm_cmd_dump doveadm_cmd_dump_index; +extern struct doveadm_cmd_dump doveadm_cmd_dump_log; +extern struct doveadm_cmd_dump doveadm_cmd_dump_mailboxlog; +extern struct doveadm_cmd_dump doveadm_cmd_dump_thread; +extern struct doveadm_cmd_dump doveadm_cmd_dump_zlib; +extern struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_file; +extern struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_key; + +void doveadm_dump_register(const struct doveadm_cmd_dump *dump); + +void print_dump_types(void); +void doveadm_dump_init(void); +void doveadm_dump_deinit(void); + +#endif diff --git a/src/doveadm/doveadm-fs.c b/src/doveadm/doveadm-fs.c new file mode 100644 index 0000000..17f6129 --- /dev/null +++ b/src/doveadm/doveadm-fs.c @@ -0,0 +1,608 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "md5.h" +#include "sha2.h" +#include "hash-method.h" +#include "hex-binary.h" +#include "istream.h" +#include "ostream.h" +#include "iostream-ssl.h" +#include "fs-api.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> + +static void fs_cmd_help(struct doveadm_cmd_context *cctx) ATTR_NORETURN; +static void cmd_fs_delete(struct doveadm_cmd_context *cctx); + +static struct fs * +cmd_fs_init(struct doveadm_cmd_context *cctx) +{ + struct ssl_iostream_settings ssl_set; + struct fs_settings fs_set; + struct fs *fs; + const char *fs_driver, *fs_args, *error; + + if (!doveadm_cmd_param_str(cctx, "fs-driver", &fs_driver) || + !doveadm_cmd_param_str(cctx, "fs-args", &fs_args)) + fs_cmd_help(cctx); + + doveadm_get_ssl_settings(&ssl_set, pool_datastack_create()); + ssl_set.verbose = doveadm_debug; + i_zero(&fs_set); + fs_set.ssl_client_set = &ssl_set; + fs_set.temp_dir = doveadm_settings->mail_temp_dir; + fs_set.base_dir = doveadm_settings->base_dir; + fs_set.debug = doveadm_debug; + + if (fs_init(fs_driver, fs_args, &fs_set, &fs, &error) < 0) + i_fatal("fs_init() failed: %s", error); + return fs; +} + +static void cmd_fs_get(struct doveadm_cmd_context *cctx) +{ + struct fs *fs; + struct fs_file *file; + struct istream *input; + const char *path; + const unsigned char *data; + size_t size; + ssize_t ret; + + doveadm_print_init(DOVEADM_PRINT_TYPE_PAGER); + doveadm_print_header("content", "content", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_str(cctx, "path", &path)) + fs_cmd_help(cctx); + + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + input = fs_read_stream(file, IO_BLOCK_SIZE); + while ((ret = i_stream_read_more(input, &data, &size)) > 0) { + doveadm_print_stream(data, size); + i_stream_skip(input, size); + } + doveadm_print_stream("", 0); + i_assert(ret == -1); + if (input->stream_errno == ENOENT) { + i_error("%s doesn't exist: %s", fs_file_path(file), + i_stream_get_error(input)); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else if (input->stream_errno != 0) { + i_error("read(%s) failed: %s", fs_file_path(file), + i_stream_get_error(input)); + doveadm_exit_code = EX_TEMPFAIL; + } + i_stream_unref(&input); + fs_file_deinit(&file); + fs_deinit(&fs); +} + +static void cmd_fs_put(struct doveadm_cmd_context *cctx) +{ + struct fs *fs; + enum fs_properties props; + const char *hash_str, *src_path, *dest_path; + struct fs_file *file; + struct istream *input; + struct ostream *output; + buffer_t *hash = NULL; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_str(cctx, "input-path", &src_path) || + !doveadm_cmd_param_str(cctx, "path", &dest_path)) + fs_cmd_help(cctx); + if (doveadm_cmd_param_str(cctx, "hash", &hash_str)) { + hash = t_buffer_create(32); + if (hex_to_binary(optarg, hash) < 0) + i_fatal("Invalid -h parameter: Hash not in hex"); + } + + file = fs_file_init(fs, dest_path, FS_OPEN_MODE_REPLACE); + props = fs_get_properties(fs); + if (hash == NULL) + ; + else if (hash->used == hash_method_md5.digest_size) { + if ((props & FS_PROPERTY_WRITE_HASH_MD5) == 0) + i_fatal("fs backend doesn't support MD5 hashes"); + fs_write_set_hash(file, + hash_method_lookup(hash_method_md5.name), hash->data); + } else if (hash->used == hash_method_sha256.digest_size) { + if ((props & FS_PROPERTY_WRITE_HASH_SHA256) == 0) + i_fatal("fs backend doesn't support SHA256 hashes"); + fs_write_set_hash(file, + hash_method_lookup(hash_method_sha256.name), hash->data); + } + + output = fs_write_stream(file); + input = i_stream_create_file(src_path, IO_BLOCK_SIZE); + o_stream_nsend_istream(output, input); + i_stream_destroy(&input); + if (fs_write_stream_finish(file, &output) < 0) { + i_error("fs_write_stream_finish() failed: %s", + fs_file_last_error(file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&file); + fs_deinit(&fs); +} + +static void cmd_fs_copy(struct doveadm_cmd_context *cctx) +{ + struct fs *fs; + struct fs_file *src_file, *dest_file; + const char *src_path, *dest_path; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_str(cctx, "source-path", &src_path) || + !doveadm_cmd_param_str(cctx, "destination-path", &dest_path)) + fs_cmd_help(cctx); + + src_file = fs_file_init(fs, src_path, FS_OPEN_MODE_READONLY); + dest_file = fs_file_init(fs, dest_path, FS_OPEN_MODE_REPLACE); + if (fs_copy(src_file, dest_file) == 0) ; + else if (errno == ENOENT) { + i_error("%s doesn't exist: %s", src_path, + fs_file_last_error(dest_file)); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else { + i_error("fs_copy(%s, %s) failed: %s", + src_path, dest_path, fs_file_last_error(dest_file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&src_file); + fs_file_deinit(&dest_file); + fs_deinit(&fs); +} + +static void cmd_fs_stat(struct doveadm_cmd_context *cctx) +{ + struct fs *fs; + struct fs_file *file; + struct stat st; + const char *path; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_str(cctx, "path", &path)) + fs_cmd_help(cctx); + + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + doveadm_print_formatted_set_format("%{path} size=%{size}"); + doveadm_print_header_simple("path"); + doveadm_print_header("size", "size", DOVEADM_PRINT_HEADER_FLAG_NUMBER); + + if (fs_stat(file, &st) == 0) { + doveadm_print(fs_file_path(file)); + doveadm_print(dec2str(st.st_size)); + } else if (errno == ENOENT) { + i_error("%s doesn't exist: %s", fs_file_path(file), + fs_file_last_error(file)); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else { + i_error("fs_stat(%s) failed: %s", + fs_file_path(file), fs_file_last_error(file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&file); + fs_deinit(&fs); +} + +static void cmd_fs_metadata(struct doveadm_cmd_context *cctx) +{ + struct fs *fs; + struct fs_file *file; + const struct fs_metadata *m; + const ARRAY_TYPE(fs_metadata) *metadata; + const char *path; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_str(cctx, "path", &path)) + fs_cmd_help(cctx); + + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + doveadm_print_formatted_set_format("%{key}=%{value}\n"); + doveadm_print_header_simple("key"); + doveadm_print_header_simple("value"); + + if (fs_get_metadata(file, &metadata) == 0) { + array_foreach(metadata, m) { + doveadm_print(m->key); + doveadm_print(m->value); + } + } else if (errno == ENOENT) { + i_error("%s doesn't exist: %s", fs_file_path(file), + fs_file_last_error(file)); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } else { + i_error("fs_stat(%s) failed: %s", + fs_file_path(file), fs_file_last_error(file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&file); + fs_deinit(&fs); +} + +struct fs_delete_ctx { + struct fs *fs; + const char *path_prefix; + + unsigned int files_count; + struct fs_file **files; +}; + +static int cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx) +{ + unsigned int i; + int ret = 0; + + for (i = 0; i < ctx->files_count; i++) { + if (ctx->files[i] == NULL) + ; + else if (fs_delete(ctx->files[i]) == 0) + fs_file_deinit(&ctx->files[i]); + else if (errno == EAGAIN) { + if (ret == 0) + ret = 1; + } else if (errno == ENOENT) { + i_error("%s doesn't exist: %s", fs_file_path(ctx->files[i]), + fs_file_last_error(ctx->files[i])); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + ret = -1; + } else { + i_error("fs_delete(%s) failed: %s", + fs_file_path(ctx->files[i]), + fs_file_last_error(ctx->files[i])); + doveadm_exit_code = EX_TEMPFAIL; + ret = -1; + } + } + return ret; +} + +static int doveadm_fs_delete_async_fname(struct fs_delete_ctx *ctx, + const char *fname) +{ + unsigned int i; + int ret; + + for (i = 0; i < ctx->files_count; i++) { + if (ctx->files[i] != NULL) + continue; + + ctx->files[i] = fs_file_init(ctx->fs, + t_strdup_printf("%s%s", ctx->path_prefix, fname), + FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC | + FS_OPEN_FLAG_ASYNC_NOQUEUE); + fname = NULL; + break; + } + if ((ret = cmd_fs_delete_ctx_run(ctx)) < 0) + return -1; + if (fname != NULL) { + if (ret > 0) + fs_wait_async(ctx->fs); + return doveadm_fs_delete_async_fname(ctx, fname); + } + return 0; +} + +static void doveadm_fs_delete_async_finish(struct fs_delete_ctx *ctx) +{ + unsigned int i; + + while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(ctx) > 0) { + fs_wait_async(ctx->fs); + } + for (i = 0; i < ctx->files_count; i++) { + fs_file_deinit(&ctx->files[i]); + } +} + +static void +cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count, + const char *path_prefix) +{ + struct fs_iter *iter; + ARRAY_TYPE(const_string) fnames; + struct fs_delete_ctx ctx; + const char *fname, *error; + int ret; + + i_zero(&ctx); + ctx.fs = fs; + ctx.path_prefix = path_prefix; + ctx.files_count = I_MAX(async_count, 1); + ctx.files = t_new(struct fs_file *, ctx.files_count); + + /* delete subdirs first. all fs backends can't handle recursive + lookups, so save the list first. */ + t_array_init(&fnames, 8); + iter = fs_iter_init(fs, path_prefix, FS_ITER_FLAG_DIRS); + while ((fname = fs_iter_next(iter)) != NULL) { + /* append "/" so that if FS_PROPERTY_DIRECTORIES is set, + we'll include the "/" suffix in the filename when deleting + it. */ + fname = t_strconcat(fname, "/", NULL); + array_push_back(&fnames, &fname); + } + if (fs_iter_deinit(&iter, &error) < 0) { + i_error("fs_iter_deinit(%s) failed: %s", + path_prefix, error); + doveadm_exit_code = EX_TEMPFAIL; + } + array_foreach_elem(&fnames, fname) T_BEGIN { + cmd_fs_delete_dir_recursive(fs, async_count, + t_strdup_printf("%s%s", path_prefix, fname)); + } T_END; + + /* delete files. again because we're doing this asynchronously finish + the iteration first. */ + if ((fs_get_properties(fs) & FS_PROPERTY_DIRECTORIES) != 0) { + /* we need to explicitly delete also the directories */ + } else { + array_clear(&fnames); + } + iter = fs_iter_init(fs, path_prefix, 0); + while ((fname = fs_iter_next(iter)) != NULL) { + fname = t_strdup(fname); + array_push_back(&fnames, &fname); + } + if (fs_iter_deinit(&iter, &error) < 0) { + i_error("fs_iter_deinit(%s) failed: %s", + path_prefix, error); + doveadm_exit_code = EX_TEMPFAIL; + } + + array_foreach_elem(&fnames, fname) { + T_BEGIN { + ret = doveadm_fs_delete_async_fname(&ctx, fname); + } T_END; + if (ret < 0) + break; + } + doveadm_fs_delete_async_finish(&ctx); +} + +static void cmd_fs_delete_recursive_path(struct fs *fs, const char *path, + unsigned int async_count) +{ + struct fs_file *file; + size_t path_len; + + path_len = strlen(path); + if (path_len > 0 && path[path_len-1] != '/') + path = t_strconcat(path, "/", NULL); + + cmd_fs_delete_dir_recursive(fs, async_count, path); + if ((fs_get_properties(fs) & FS_PROPERTY_DIRECTORIES) != 0) { + /* delete the root itself */ + file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY); + if (fs_delete(file) < 0) { + i_error("fs_delete(%s) failed: %s", + fs_file_path(file), fs_file_last_error(file)); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_file_deinit(&file); + } +} + +static void +cmd_fs_delete_recursive(struct doveadm_cmd_context *cctx, + unsigned int async_count) +{ + struct fs *fs; + const char *const *paths; + unsigned int i; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_array(cctx, "path", &paths)) + fs_cmd_help(cctx); + + for (i = 0; paths[i] != NULL; i++) + cmd_fs_delete_recursive_path(fs, paths[i], async_count); + fs_deinit(&fs); +} + +static void cmd_fs_delete_paths(struct doveadm_cmd_context *cctx, + unsigned int async_count) +{ + struct fs *fs; + struct fs_delete_ctx ctx; + const char *const *paths; + unsigned int i; + int ret; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_array(cctx, "path", &paths)) + fs_cmd_help(cctx); + + i_zero(&ctx); + ctx.fs = fs; + ctx.path_prefix = ""; + ctx.files_count = I_MAX(async_count, 1); + ctx.files = t_new(struct fs_file *, ctx.files_count); + + for (i = 0; paths[i] != NULL; i++) { + T_BEGIN { + ret = doveadm_fs_delete_async_fname(&ctx, paths[i]); + } T_END; + if (ret < 0) + break; + } + doveadm_fs_delete_async_finish(&ctx); + fs_deinit(&fs); +} + +static void cmd_fs_delete(struct doveadm_cmd_context *cctx) +{ + bool recursive = FALSE; + int64_t async_count = 0; + + (void)doveadm_cmd_param_bool(cctx, "recursive", &recursive); + (void)doveadm_cmd_param_int64(cctx, "max-parallel", &async_count); + + if (recursive) + cmd_fs_delete_recursive(cctx, async_count); + else + cmd_fs_delete_paths(cctx, async_count); +} + +static void cmd_fs_iter_full(struct doveadm_cmd_context *cctx, + enum fs_iter_flags flags) +{ + struct fs *fs; + struct fs_iter *iter; + const char *path, *fname, *error; + bool b; + + if (doveadm_cmd_param_bool(cctx, "no-cache", &b) && b) + flags |= FS_ITER_FLAG_NOCACHE; + if (doveadm_cmd_param_bool(cctx, "object-ids", &b) && b) + flags |= FS_ITER_FLAG_OBJECTIDS; + + fs = cmd_fs_init(cctx); + if (!doveadm_cmd_param_str(cctx, "path", &path)) + fs_cmd_help(cctx); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + doveadm_print_formatted_set_format("%{path}\n"); + doveadm_print_header_simple("path"); + + iter = fs_iter_init(fs, path, flags); + while ((fname = fs_iter_next(iter)) != NULL) { + doveadm_print(fname); + } + if (fs_iter_deinit(&iter, &error) < 0) { + i_error("fs_iter_deinit(%s) failed: %s", path, error); + doveadm_exit_code = EX_TEMPFAIL; + } + fs_deinit(&fs); +} + +static void cmd_fs_iter(struct doveadm_cmd_context *cctx) +{ + cmd_fs_iter_full(cctx, 0); +} + +static void cmd_fs_iter_dirs(struct doveadm_cmd_context *cctx) +{ + cmd_fs_iter_full(cctx, FS_ITER_FLAG_DIRS); +} + +struct doveadm_cmd_ver2 doveadm_cmd_fs[] = { +{ + .name = "fs get", + .cmd = cmd_fs_get, + .usage = "<fs-driver> <fs-args> <path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs put", + .cmd = cmd_fs_put, + .usage = "[-h <hash>] <fs-driver> <fs-args> <input path> <path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('h', "hash", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "input-path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs copy", + .cmd = cmd_fs_copy, + .usage = "<fs-driver> <fs-args> <source path> <dest path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "source-path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "destination-path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs stat", + .cmd = cmd_fs_stat, + .usage = "<fs-driver> <fs-args> <path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs metadata", + .cmd = cmd_fs_metadata, + .usage = "<fs-driver> <fs-args> <path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs delete", + .cmd = cmd_fs_delete, + .usage = "[-R] [-n <count>] <fs-driver> <fs-args> <path> [<path> ...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('R', "recursive", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('n', "max-parallel", CMD_PARAM_INT64, 0) +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs iter", + .cmd = cmd_fs_iter, + .usage = "[--no-cache] [--object-ids] <fs-driver> <fs-args> <path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('C', "no-cache", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('O', "object-ids", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "fs iter-dirs", + .cmd = cmd_fs_iter_dirs, + .usage = "<fs-driver> <fs-args> <path>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +static void fs_cmd_help(struct doveadm_cmd_context *cctx) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_fs); i++) { + if (doveadm_cmd_fs[i].cmd == cctx->cmd->cmd) + help_ver2(&doveadm_cmd_fs[i]); + } + i_unreached(); +} + +void doveadm_register_fs_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_fs); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_fs[i]); +} diff --git a/src/doveadm/doveadm-instance.c b/src/doveadm/doveadm-instance.c new file mode 100644 index 0000000..cd3d55e --- /dev/null +++ b/src/doveadm/doveadm-instance.c @@ -0,0 +1,155 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "master-instance.h" +#include "master-service-settings.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <signal.h> + +extern struct doveadm_cmd_ver2 doveadm_cmd_instance[]; + +static void instance_cmd_help(const struct doveadm_cmd_ver2 *cmd) ATTR_NORETURN; + +static bool pid_file_read(const char *path) +{ + char buf[32]; + int fd; + ssize_t ret; + pid_t pid; + bool found = FALSE; + + fd = open(path, O_RDONLY); + if (fd == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", path); + return FALSE; + } + + ret = read(fd, buf, sizeof(buf)); + if (ret < 0) + i_error("read(%s) failed: %m", path); + else if (ret > 0 && buf[ret-1] == '\n') { + buf[ret-1] = '\0'; + if (str_to_pid(buf, &pid) == 0) { + found = !(pid == getpid() || + (kill(pid, 0) < 0 && errno == ESRCH)); + } + } + i_close_fd(&fd); + return found; +} + +static void cmd_instance_list(struct doveadm_cmd_context *cctx) +{ + struct master_instance_list *list; + struct master_instance_list_iter *iter; + const struct master_instance *inst; + const char *instance_path, *pidfile_path; + bool show_config = FALSE; + const char *name = NULL; + + (void)doveadm_cmd_param_bool(cctx, "show-config", &show_config); + (void)doveadm_cmd_param_str(cctx, "name", &name); + + if (!show_config) { + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("path", "path", DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("name"); + doveadm_print_header_simple("last used"); + doveadm_print_header_simple("running"); + } + + instance_path = t_strconcat(service_set->state_dir, + "/"MASTER_INSTANCE_FNAME, NULL); + list = master_instance_list_init(instance_path); + iter = master_instance_list_iterate_init(list); + while ((inst = master_instance_iterate_list_next(iter)) != NULL) { + if (name != NULL && strcmp(name, inst->name) != 0) + continue; + + if (show_config) { + printf("%s\n", inst->config_path == NULL ? "" : + inst->config_path); + continue; + } + doveadm_print(inst->base_dir); + doveadm_print(inst->name); + doveadm_print(unixdate2str(inst->last_used)); + pidfile_path = t_strconcat(inst->base_dir, "/master.pid", NULL); + if (pid_file_read(pidfile_path)) + doveadm_print("yes"); + else + doveadm_print("no"); + } + master_instance_iterate_list_deinit(&iter); + master_instance_list_deinit(&list); +} + +static void cmd_instance_remove(struct doveadm_cmd_context *cctx) +{ + struct master_instance_list *list; + const struct master_instance *inst; + const char *base_dir, *instance_path, *name; + int ret; + + if (!doveadm_cmd_param_str(cctx, "name", &name)) + instance_cmd_help(cctx->cmd); + + instance_path = t_strconcat(service_set->state_dir, + "/"MASTER_INSTANCE_FNAME, NULL); + list = master_instance_list_init(instance_path); + inst = master_instance_list_find_by_name(list, name); + base_dir = inst != NULL ? inst->base_dir : name; + if ((ret = master_instance_list_remove(list, base_dir)) < 0) { + i_error("Failed to remove instance"); + doveadm_exit_code = EX_TEMPFAIL; + } else if (ret == 0) { + i_error("Instance already didn't exist"); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + } + master_instance_list_deinit(&list); +} + +struct doveadm_cmd_ver2 doveadm_cmd_instance[] = { +{ + .name = "instance list", + .cmd = cmd_instance_list, + .usage = "[-c] [<name>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('c', "show-config", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "instance remove", + .cmd = cmd_instance_remove, + .usage = "<name> | <base dir>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +static void instance_cmd_help(const struct doveadm_cmd_ver2 *cmd) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_instance); i++) { + if (doveadm_cmd_instance[i].cmd == cmd->cmd) + help_ver2(&doveadm_cmd_instance[i]); + } + i_unreached(); +} + +void doveadm_register_instance_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_instance); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_instance[i]); +} diff --git a/src/doveadm/doveadm-kick.c b/src/doveadm/doveadm-kick.c new file mode 100644 index 0000000..a0b06f3 --- /dev/null +++ b/src/doveadm/doveadm-kick.c @@ -0,0 +1,235 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "net.h" +#include "hash.h" +#include "doveadm.h" +#include "doveadm-who.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <sys/types.h> +#include <signal.h> + +struct kick_user { + const char *username; + bool kick_me; /* true if username and/or ip[/mask] matches. + ignored when the -f switch is given. */ +}; + +struct kick_pid { + pid_t pid; + ARRAY(struct kick_user) users; + bool kick; +}; + +struct kick_context { + struct who_context who; + HASH_TABLE(void *, struct kick_pid *) pids; + enum doveadm_client_type conn_type; + bool force_kick; + ARRAY(const char *) kicked_users; +}; + +static void +kick_aggregate_line(struct who_context *_ctx, const struct who_line *line) +{ + struct kick_context *ctx = (struct kick_context *)_ctx; + const bool user_match = who_line_filter_match(line, &ctx->who.filter); + struct kick_pid *k_pid; + struct kick_user new_user, *user; + + i_zero(&new_user); + + k_pid = hash_table_lookup(ctx->pids, POINTER_CAST(line->pid)); + if (k_pid == NULL) { + k_pid = p_new(ctx->who.pool, struct kick_pid, 1); + k_pid->pid = line->pid; + p_array_init(&k_pid->users, ctx->who.pool, 5); + hash_table_insert(ctx->pids, POINTER_CAST(line->pid), k_pid); + } + + array_foreach_modifiable(&k_pid->users, user) { + if (strcmp(line->username, user->username) == 0) { + if (user_match) + user->kick_me = TRUE; + return; + } + } + new_user.username = p_strdup(ctx->who.pool, line->username); + new_user.kick_me = user_match; + array_push_back(&k_pid->users, &new_user); +} + +static bool +kick_pid_want_kicked(struct kick_context *ctx, const struct kick_pid *k_pid, + bool *show_warning) +{ + unsigned int kick_count = 0; + const struct kick_user *user; + + if (array_count(&k_pid->users) == 1) { + user = array_front(&k_pid->users); + if (!user->kick_me) + return FALSE; + } else { + array_foreach(&k_pid->users, user) { + if (user->kick_me) + kick_count++; + } + if (kick_count == 0) + return FALSE; + if (kick_count < array_count(&k_pid->users) && + !ctx->force_kick) { + array_foreach(&k_pid->users, user) { + if (!user->kick_me) { + array_push_back(&ctx->kicked_users, + &user->username); + } + } + *show_warning = TRUE; + return FALSE; + } + } + return TRUE; +} + +static void +kick_print_kicked(struct kick_context *ctx, const bool show_warning) +{ + unsigned int i, count; + const char *const *users; + bool cli = (ctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI); + + if (array_count(&ctx->kicked_users) == 0) { + if (cli) + printf("no users kicked\n"); + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + return; + } + + if (cli) { + if (show_warning) { + printf("warning: other connections would also be " + "kicked from following users:\n"); + } else { + printf("kicked connections from the following users:\n"); + } + } + + array_sort(&ctx->kicked_users, i_strcmp_p); + users = array_get(&ctx->kicked_users, &count); + doveadm_print(users[0]); + for (i = 1; i < count; i++) { + if (strcmp(users[i-1], users[i]) != 0) + doveadm_print(users[i]); + } + + doveadm_print_flush(); + + if (cli) + printf("\n"); + + if (show_warning) + printf("Use the '-f' option to enforce the disconnect.\n"); +} + +static void kick_users(struct kick_context *ctx) +{ + bool show_enforce_warning = FALSE; + struct hash_iterate_context *iter; + void *key; + struct kick_pid *k_pid; + const struct kick_user *user; + + p_array_init(&ctx->kicked_users, ctx->who.pool, 10); + + iter = hash_table_iterate_init(ctx->pids); + while (hash_table_iterate(iter, ctx->pids, &key, &k_pid)) { + if (kick_pid_want_kicked(ctx, k_pid, &show_enforce_warning)) + k_pid->kick = TRUE; + } + hash_table_iterate_deinit(&iter); + + if (show_enforce_warning) { + kick_print_kicked(ctx, show_enforce_warning); + return; + } + + iter = hash_table_iterate_init(ctx->pids); + while (hash_table_iterate(iter, ctx->pids, &key, &k_pid)) { + if (!k_pid->kick) + continue; + + if (kill(k_pid->pid, SIGTERM) < 0 && errno != ESRCH) { + fprintf(stderr, "kill(%s, SIGTERM) failed: %m\n", + dec2str(k_pid->pid)); + } else { + array_foreach(&k_pid->users, user) { + array_push_back(&ctx->kicked_users, + &user->username); + } + } + } + hash_table_iterate_deinit(&iter); + + kick_print_kicked(ctx, show_enforce_warning); +} + +static void cmd_kick(struct doveadm_cmd_context *cctx) +{ + const char *const *masks; + struct kick_context ctx; + + i_zero(&ctx); + if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx.who.anvil_path))) + ctx.who.anvil_path = t_strconcat(doveadm_settings->base_dir, "/anvil", NULL); + (void)doveadm_cmd_param_bool(cctx, "force", &(ctx.force_kick)); + if (!doveadm_cmd_param_array(cctx, "mask", &masks)) { + doveadm_exit_code = EX_USAGE; + i_error("user and/or ip[/bits] must be specified."); + return; + } + ctx.conn_type = cctx->conn_type; + if (ctx.conn_type != DOVEADM_CONNECTION_TYPE_CLI) { + /* force-kick is a pretty ugly option. its output can't be + nicely translated to an API reply. it also wouldn't be very + useful in scripts, only for preventing a new admin from + accidentally kicking too many users. it's also useful only + in a non-recommended setup where processes are handling + multiple connections. so for now we'll preserve the option + for CLI, but always do a force-kick with non-CLI. */ + ctx.force_kick = TRUE; + } + ctx.who.pool = pool_alloconly_create("kick pids", 10240); + hash_table_create_direct(&ctx.pids, ctx.who.pool, 0); + + if (who_parse_args(&ctx.who, masks)!=0) { + hash_table_destroy(&ctx.pids); + pool_unref(&ctx.who.pool); + return; + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + doveadm_print_formatted_set_format("%{result} "); + doveadm_print_header_simple("result"); + + who_lookup(&ctx.who, kick_aggregate_line); + kick_users(&ctx); + + hash_table_destroy(&ctx.pids); + pool_unref(&ctx.who.pool); +} + +struct doveadm_cmd_ver2 doveadm_cmd_kick_ver2 = { + .name = "kick", + .cmd = cmd_kick, + .usage = "[-a <anvil socket path>] <user mask>[|]<ip/bits>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a',"socket-path",CMD_PARAM_STR,0) +DOVEADM_CMD_PARAM('f',"force",CMD_PARAM_BOOL,0) +DOVEADM_CMD_PARAM('\0',"mask",CMD_PARAM_ARRAY,CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-log.c b/src/doveadm/doveadm-log.c new file mode 100644 index 0000000..2654d9b --- /dev/null +++ b/src/doveadm/doveadm-log.c @@ -0,0 +1,406 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "istream.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "time-util.h" +#include "master-service-private.h" +#include "master-service-settings.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <dirent.h> +#include <signal.h> +#include <sys/stat.h> + +#define LAST_LOG_TYPE LOG_TYPE_PANIC +#define TEST_LOG_MSG_PREFIX "This is Dovecot's " +#define LOG_ERRORS_FNAME "log-errors" +#define LOG_TIMESTAMP_FORMAT "%b %d %H:%M:%S" + +static void ATTR_NULL(2) +cmd_log_test(struct doveadm_cmd_context *cctx ATTR_UNUSED) +{ + struct failure_context ctx; + unsigned int i; + + master_service->log_initialized = FALSE; + master_service->flags |= MASTER_SERVICE_FLAG_DONT_LOG_TO_STDERR; + master_service_init_log(master_service); + + i_zero(&ctx); + for (i = 0; i < LAST_LOG_TYPE; i++) { + const char *prefix = failure_log_type_prefixes[i]; + + /* add timestamp so that syslog won't just write + "repeated message" text */ + ctx.type = i; + i_log_type(&ctx, TEST_LOG_MSG_PREFIX"%s log (%u)", + t_str_lcase(t_strcut(prefix, ':')), + (unsigned int)ioloop_time); + } +} + +static void cmd_log_reopen(struct doveadm_cmd_context *cctx ATTR_UNUSED) +{ + doveadm_master_send_signal(SIGUSR1); +} + +struct log_find_file { + const char *path; + uoff_t size; + + /* 1 << enum log_type */ + unsigned int mask; +}; + +struct log_find_context { + pool_t pool; + HASH_TABLE(char *, struct log_find_file *) files; +}; + +static void cmd_log_find_add(struct log_find_context *ctx, + const char *path, enum log_type type) +{ + struct log_find_file *file; + char *key; + + file = hash_table_lookup(ctx->files, path); + if (file == NULL) { + file = p_new(ctx->pool, struct log_find_file, 1); + file->path = key = p_strdup(ctx->pool, path); + hash_table_insert(ctx->files, key, file); + } + + file->mask |= 1 << type; +} + +static void +cmd_log_find_syslog_files(struct log_find_context *ctx, const char *path) +{ + struct log_find_file *file; + DIR *dir; + struct dirent *d; + struct stat st; + char *key; + string_t *full_path; + size_t dir_len; + + dir = opendir(path); + if (dir == NULL) { + i_error("opendir(%s) failed: %m", path); + return; + } + + full_path = t_str_new(256); + str_append(full_path, path); + str_append_c(full_path, '/'); + dir_len = str_len(full_path); + + while ((d = readdir(dir)) != NULL) { + if (d->d_name[0] == '.') + continue; + + str_truncate(full_path, dir_len); + str_append(full_path, d->d_name); + if (stat(str_c(full_path), &st) < 0) + continue; + + if (S_ISDIR(st.st_mode)) { + /* recursively go through all subdirectories */ + cmd_log_find_syslog_files(ctx, str_c(full_path)); + } else if (hash_table_lookup(ctx->files, + str_c(full_path)) == NULL) { + file = p_new(ctx->pool, struct log_find_file, 1); + file->size = st.st_size; + file->path = key = + p_strdup(ctx->pool, str_c(full_path)); + hash_table_insert(ctx->files, key, file); + } + } + + (void)closedir(dir); +} + +static bool log_type_find(const char *str, enum log_type *type_r) +{ + unsigned int i; + size_t len = strlen(str); + + for (i = 0; i < LAST_LOG_TYPE; i++) { + if (strncasecmp(str, failure_log_type_prefixes[i], len) == 0 && + failure_log_type_prefixes[i][len] == ':') { + *type_r = i; + return TRUE; + } + } + return FALSE; +} + +static void cmd_log_find_syslog_file_messages(struct log_find_file *file) +{ + struct istream *input; + const char *line, *p; + enum log_type type; + int fd; + + fd = open(file->path, O_RDONLY); + if (fd == -1) + return; + + input = i_stream_create_fd_autoclose(&fd, 1024); + i_stream_seek(input, file->size); + while ((line = i_stream_read_next_line(input)) != NULL) { + p = strstr(line, TEST_LOG_MSG_PREFIX); + if (p == NULL) + continue; + p += strlen(TEST_LOG_MSG_PREFIX); + + /* <type> log */ + T_BEGIN { + if (log_type_find(t_strcut(p, ' '), &type)) + file->mask |= 1 << type; + } T_END; + } + i_stream_destroy(&input); +} + +static void cmd_log_find_syslog_messages(struct log_find_context *ctx) +{ + struct hash_iterate_context *iter; + struct stat st; + char *key; + struct log_find_file *file; + + iter = hash_table_iterate_init(ctx->files); + while (hash_table_iterate(iter, ctx->files, &key, &file)) { + if (stat(file->path, &st) < 0 || + (uoff_t)st.st_size <= file->size) + continue; + + cmd_log_find_syslog_file_messages(file); + } + hash_table_iterate_deinit(&iter); +} + +static void +cmd_log_find_syslog(struct log_find_context *ctx, + struct doveadm_cmd_context *cctx) +{ + const char *log_dir; + struct stat st; + + if (doveadm_cmd_param_str(cctx, "log-dir", &log_dir)) + ; + else if (stat("/var/log", &st) == 0 && S_ISDIR(st.st_mode)) + log_dir = "/var/log"; + else if (stat("/var/adm", &st) == 0 && S_ISDIR(st.st_mode)) + log_dir = "/var/adm"; + else + return; + + printf("Looking for log files from %s\n", log_dir); + cmd_log_find_syslog_files(ctx, log_dir); + cmd_log_test(cctx); + + /* give syslog some time to write the messages to files */ + sleep(1); + cmd_log_find_syslog_messages(ctx); +} + +static void cmd_log_find(struct doveadm_cmd_context *cctx) +{ + const struct master_service_settings *set; + const char *log_file_path; + struct log_find_context ctx; + unsigned int i; + + i_zero(&ctx); + ctx.pool = pool_alloconly_create("log file", 1024*32); + hash_table_create(&ctx.files, ctx.pool, 0, str_hash, strcmp); + + /* first get the paths that we know are used */ + set = master_service_settings_get(master_service); + log_file_path = set->log_path; + if (strcmp(log_file_path, "syslog") == 0) + log_file_path = ""; + if (*log_file_path != '\0') { + cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_WARNING); + cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_ERROR); + cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_FATAL); + } + + if (strcmp(set->info_log_path, "syslog") != 0) { + if (*set->info_log_path != '\0') + log_file_path = set->info_log_path; + if (*log_file_path != '\0') + cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_INFO); + } + + if (strcmp(set->debug_log_path, "syslog") != 0) { + if (*set->debug_log_path != '\0') + log_file_path = set->debug_log_path; + if (*log_file_path != '\0') + cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_DEBUG); + } + + if (*set->log_path == '\0' || + strcmp(set->log_path, "syslog") == 0 || + strcmp(set->info_log_path, "syslog") == 0 || + strcmp(set->debug_log_path, "syslog") == 0) { + /* at least some logs were logged via syslog */ + cmd_log_find_syslog(&ctx, cctx); + } + + /* print them */ + for (i = 0; i < LAST_LOG_TYPE; i++) { + struct hash_iterate_context *iter; + char *key; + struct log_find_file *file; + bool found = FALSE; + + iter = hash_table_iterate_init(ctx.files); + while (hash_table_iterate(iter, ctx.files, &key, &file)) { + if ((file->mask & (1 << i)) != 0) { + printf("%s%s\n", failure_log_type_prefixes[i], + file->path); + found = TRUE; + } + } + hash_table_iterate_deinit(&iter); + + if (!found) + printf("%sNot found\n", failure_log_type_prefixes[i]); + } + hash_table_destroy(&ctx.files); + pool_unref(&ctx.pool); +} + +static const char *t_cmd_log_error_trim(const char *orig) +{ + size_t pos; + + /* Trim whitespace from suffix and remove ':' if it exists */ + for (pos = strlen(orig); pos > 0; pos--) { + if (orig[pos-1] != ' ') { + if (orig[pos-1] == ':') + pos--; + break; + } + } + return orig[pos] == '\0' ? orig : t_strndup(orig, pos); +} + +static void cmd_log_error_write(const char *const *args, time_t min_timestamp) +{ + /* <type> <timestamp> <prefix> <text> */ + const char *type_prefix = "?"; + unsigned int type; + time_t t; + + /* find type's prefix */ + for (type = 0; type < LOG_TYPE_COUNT; type++) { + if (strcmp(args[0], failure_log_type_names[type]) == 0) { + type_prefix = failure_log_type_prefixes[type]; + break; + } + } + + if (str_to_time(args[1], &t) < 0) { + i_error("Invalid timestamp: %s", args[1]); + t = 0; + } + if (t >= min_timestamp) { + doveadm_print(t_strflocaltime(LOG_TIMESTAMP_FORMAT, t)); + doveadm_print(t_cmd_log_error_trim(args[2])); + doveadm_print(t_cmd_log_error_trim(type_prefix)); + doveadm_print(args[3]); + } +} + +static void cmd_log_errors(struct doveadm_cmd_context *cctx) +{ + struct istream *input; + const char *path, *line, *const *args; + time_t min_timestamp = 0; + int64_t since_int64; + int fd; + + if (doveadm_cmd_param_int64(cctx, "since", &since_int64)) + min_timestamp = since_int64; + + path = t_strconcat(doveadm_settings->base_dir, + "/"LOG_ERRORS_FNAME, NULL); + fd = net_connect_unix(path); + if (fd == -1) + i_fatal("net_connect_unix(%s) failed: %m", path); + net_set_nonblock(fd, FALSE); + + input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + doveadm_print_formatted_set_format("%{timestamp} %{type}: %{prefix}: %{text}\n"); + + doveadm_print_header_simple("timestamp"); + doveadm_print_header_simple("prefix"); + doveadm_print_header_simple("type"); + doveadm_print_header_simple("text"); + + while ((line = i_stream_read_next_line(input)) != NULL) T_BEGIN { + args = t_strsplit_tabescaped(line); + if (str_array_length(args) == 4) + cmd_log_error_write(args, min_timestamp); + else { + i_error("Invalid input from log: %s", line); + doveadm_exit_code = EX_PROTOCOL; + } + } T_END; + i_stream_destroy(&input); +} + +struct doveadm_cmd_ver2 doveadm_cmd_log[] = { +{ + .name = "log test", + .cmd = cmd_log_test, + .usage = "", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "log reopen", + .cmd = cmd_log_reopen, + .usage = "", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "log find", + .cmd = cmd_log_find, + .usage = "[<dir>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "log-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "log errors", + .usage = "[-s <min_timestamp>]", + .cmd = cmd_log_errors, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('s', "since", CMD_PARAM_INT64, 0) +DOVEADM_CMD_PARAMS_END +} +}; + +void doveadm_register_log_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_log); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_log[i]); +} diff --git a/src/doveadm/doveadm-mail-altmove.c b/src/doveadm/doveadm-mail-altmove.c new file mode 100644 index 0000000..7bcf7b0 --- /dev/null +++ b/src/doveadm/doveadm-mail-altmove.c @@ -0,0 +1,162 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index.h" +#include "mail-storage.h" +#include "mail-namespace.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +struct altmove_cmd_context { + struct doveadm_mail_cmd_context ctx; + bool reverse; +}; + +static int +cmd_altmove_box(struct doveadm_mail_cmd_context *ctx, + const struct mailbox_info *info, + struct mail_search_args *search_args, bool reverse) +{ + struct doveadm_mail_iter *iter; + struct mail *mail; + enum modify_type modify_type = + !reverse ? MODIFY_ADD : MODIFY_REMOVE; + + if (doveadm_mail_iter_init(ctx, info, search_args, 0, NULL, 0, + &iter) < 0) + return -1; + + while (doveadm_mail_iter_next(iter, &mail)) { + if (doveadm_debug) { + i_debug("altmove: box=%s uid=%u", + info->vname, mail->uid); + } + mail_update_flags(mail, modify_type, + (enum mail_flags)MAIL_INDEX_MAIL_FLAG_BACKEND); + } + return doveadm_mail_iter_deinit_sync(&iter); +} + +static int +ns_purge(struct doveadm_mail_cmd_context *ctx, struct mail_namespace *ns, + struct mail_storage *storage) +{ + if (mail_storage_purge(storage) < 0) { + i_error("Purging namespace '%s' failed: %s", ns->prefix, + mail_storage_get_last_internal_error(storage, NULL)); + doveadm_mail_failed_storage(ctx, storage); + return -1; + } + return 0; +} + +static int +cmd_altmove_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct altmove_cmd_context *ctx = (struct altmove_cmd_context *)_ctx; + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + struct mail_namespace *ns, *prev_ns = NULL; + ARRAY(struct mail_storage *) purged_storages; + struct mail_storage *const *storages, *ns_storage, *prev_storage = NULL; + unsigned int i, count; + int ret = 0; + + t_array_init(&purged_storages, 8); + iter = doveadm_mailbox_list_iter_init(_ctx, user, _ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + ns_storage = mail_namespace_get_default_storage(info->ns); + if (ns_storage != prev_storage) { + if (prev_storage != NULL) { + if (ns_purge(_ctx, prev_ns, prev_storage) < 0) + ret = -1; + array_push_back(&purged_storages, + &prev_storage); + } + prev_storage = ns_storage; + prev_ns = info->ns; + } + if (cmd_altmove_box(_ctx, info, _ctx->search_args, ctx->reverse) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + + if (prev_storage != NULL) { + if (ns_purge(_ctx, prev_ns, prev_storage) < 0) + ret = -1; + array_push_back(&purged_storages, &prev_storage); + } + + /* make sure all private storages have been purged */ + storages = array_get(&purged_storages, &count); + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if (ns->type != MAIL_NAMESPACE_TYPE_PRIVATE) + continue; + + ns_storage = mail_namespace_get_default_storage(ns); + for (i = 0; i < count; i++) { + if (ns_storage == storages[i]) + break; + } + if (i == count) { + if (ns_purge(_ctx, ns, ns_storage) < 0) + ret = -1; + array_push_back(&purged_storages, &ns_storage); + storages = array_get(&purged_storages, &count); + } + } + return ret; +} + +static void cmd_altmove_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("altmove"); + ctx->search_args = doveadm_mail_build_search_args(args); +} + +static bool +cmd_mailbox_altmove_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct altmove_cmd_context *ctx = (struct altmove_cmd_context *)_ctx; + + switch (c) { + case 'r': + ctx->reverse = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_altmove_alloc(void) +{ + struct altmove_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct altmove_cmd_context); + ctx->ctx.getopt_args = "r"; + ctx->ctx.v.parse_arg = cmd_mailbox_altmove_parse_arg; + ctx->ctx.v.init = cmd_altmove_init; + ctx->ctx.v.run = cmd_altmove_run; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_altmove_ver2 = { + .name = "altmove", + .mail_cmd = cmd_altmove_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-r] <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('r', "reverse", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-batch.c b/src/doveadm/doveadm-mail-batch.c new file mode 100644 index 0000000..2dbbb89 --- /dev/null +++ b/src/doveadm/doveadm-mail-batch.c @@ -0,0 +1,186 @@ +/* Copyright (c) 2012-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "doveadm-mail.h" + +#include <unistd.h> + +struct batch_cmd_context { + struct doveadm_mail_cmd_context ctx; + ARRAY(struct doveadm_mail_cmd_context *) commands; +}; + +static int cmd_batch_prerun(struct doveadm_mail_cmd_context *_ctx, + struct mail_storage_service_user *service_user, + const char **error_r) +{ + struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx; + struct doveadm_mail_cmd_context *cmd; + int ret = 0; + + array_foreach_elem(&ctx->commands, cmd) { + if (cmd->v.prerun != NULL && + cmd->v.prerun(cmd, service_user, error_r) < 0) { + ret = -1; + break; + } + } + return ret; +} + +static int cmd_batch_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx; + struct doveadm_mail_cmd_context *cmd; + int ret = 0; + + array_foreach_elem(&ctx->commands, cmd) { + cmd->cur_mail_user = user; + const char *reason_code = + event_reason_code_prefix("doveadm", "cmd_", + cmd->cmd->name); + struct event_reason *reason = event_reason_begin(reason_code); + ret = cmd->v.run(cmd, user); + event_reason_end(&reason); + if (ret < 0) { + i_assert(cmd->exit_code != 0); + _ctx->exit_code = cmd->exit_code; + break; + } + cmd->cur_mail_user = NULL; + } + return ret; +} + +static void +cmd_batch_add(struct batch_cmd_context *batchctx, + int argc, const char *const *argv) +{ + struct doveadm_mail_cmd_context *subctx; + const struct doveadm_cmd_ver2 *cmd_ver2; + const struct doveadm_mail_cmd *cmd; + const char *getopt_args; + int c; + + cmd_ver2 = doveadm_cmd_find_with_args_ver2(argv[0], &argc, &argv); + if (cmd_ver2 == NULL) + i_fatal_status(EX_USAGE, "doveadm batch: '%s' mail command doesn't exist", argv[0]); + + struct doveadm_mail_cmd *dyncmd = + p_new(batchctx->ctx.pool, struct doveadm_mail_cmd, 1); + dyncmd->usage_args = cmd_ver2->usage; + dyncmd->name = cmd_ver2->name; + dyncmd->alloc = cmd_ver2->mail_cmd; + cmd = dyncmd; + + subctx = doveadm_mail_cmd_init(cmd, doveadm_settings); + subctx->full_args = argv + 1; + subctx->service_flags |= batchctx->ctx.service_flags; + + i_getopt_reset(); + getopt_args = subctx->getopt_args != NULL ? subctx->getopt_args : ""; + while ((c = getopt(argc, (void *)argv, getopt_args)) > 0) { + if (subctx->v.parse_arg == NULL || + !subctx->v.parse_arg(subctx, c)) + doveadm_mail_help(cmd); + } + argv += optind; + if (argv[0] != NULL && cmd->usage_args == NULL) { + i_fatal_status(EX_USAGE, "doveadm %s: Unknown parameter: %s", + cmd->name, argv[0]); + } + subctx->args = argv; + if (subctx->v.preinit != NULL) + subctx->v.preinit(subctx); + array_push_back(&batchctx->commands, &subctx); +} + +static void +cmd_batch_preinit(struct doveadm_mail_cmd_context *_ctx) +{ + const char *const *args = _ctx->args; + struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx; + ARRAY_TYPE(const_string) sep_args; + const char *sep = args[0]; + unsigned int i, start; + int argc; + const char *const *argv; + + if (sep == NULL || args[1] == NULL) + doveadm_mail_help_name("batch"); + args++; + + p_array_init(&ctx->commands, _ctx->pool, 8); + p_array_init(&sep_args, _ctx->pool, 16); + for (i = start = 0;; i++) { + if (args[i] != NULL && strcmp(args[i], sep) != 0) { + array_push_back(&sep_args, &args[i]); + continue; + } + if (i > start) { + (void)array_append_space(&sep_args); + argc = i - start; + argv = array_idx(&sep_args, start); + cmd_batch_add(ctx, argc, argv); + start = i+1; + } + if (args[i] == NULL) + break; + } + (void)array_append_space(&sep_args); +} + +static void +cmd_batch_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[] ATTR_UNUSED) +{ + struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx; + struct doveadm_mail_cmd_context *cmd; + struct batch_cmd_context *subctx; + + array_foreach_elem(&ctx->commands, cmd) { + subctx = (struct batch_cmd_context *)cmd; + subctx->ctx.storage_service = _ctx->storage_service; + if (subctx->ctx.v.init != NULL) + subctx->ctx.v.init(&subctx->ctx, subctx->ctx.args); + } +} + +static void cmd_batch_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx; + struct doveadm_mail_cmd_context *cmd; + + array_foreach_elem(&ctx->commands, cmd) { + doveadm_mail_cmd_deinit(cmd); + doveadm_mail_cmd_free(cmd); + } +} + +static struct doveadm_mail_cmd_context *cmd_batch_alloc(void) +{ + struct batch_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct batch_cmd_context); + ctx->ctx.getopt_args = "+"; /* disable processing -args in the middle */ + ctx->ctx.v.preinit = cmd_batch_preinit; + ctx->ctx.v.init = cmd_batch_init; + ctx->ctx.v.prerun = cmd_batch_prerun; + ctx->ctx.v.run = cmd_batch_run; + ctx->ctx.v.deinit = cmd_batch_deinit; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_batch = { + .name = "batch", + .mail_cmd = cmd_batch_alloc, + .usage = "<sep> <cmd1> [<sep> <cmd2> [..]]", + .flags = CMD_FLAG_NO_UNORDERED_OPTIONS, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "separator", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-copymove.c b/src/doveadm/doveadm-mail-copymove.c new file mode 100644 index 0000000..7e26ed8 --- /dev/null +++ b/src/doveadm/doveadm-mail-copymove.c @@ -0,0 +1,224 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage.h" +#include "mail-namespace.h" +#include "doveadm-print.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +#include <stdio.h> + +struct copy_cmd_context { + struct doveadm_mail_cmd_context ctx; + + const char *source_username; + struct mail_storage_service_user *source_service_user; + struct mail_user *source_user; + + const char *destname; + bool move; +}; + +static int +cmd_copy_box(struct copy_cmd_context *ctx, struct mailbox *destbox, + const struct mailbox_info *info) +{ + struct doveadm_mail_iter *iter; + struct mailbox_transaction_context *desttrans; + struct mail_save_context *save_ctx; + struct mail *mail; + int ret = 0, ret2; + + if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args, 0, + NULL, 0, &iter) < 0) + return -1; + + /* use a separately committed transaction for each mailbox. + this guarantees that mails aren't expunged without actually having + been copied. */ + desttrans = mailbox_transaction_begin(destbox, + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + ctx->ctx.transaction_flags, __func__); + + while (doveadm_mail_iter_next(iter, &mail)) { + save_ctx = mailbox_save_alloc(desttrans); + mailbox_save_copy_flags(save_ctx, mail); + if (ctx->move) + ret2 = mailbox_move(&save_ctx, mail); + else + ret2 = mailbox_copy(&save_ctx, mail); + if (ret2 < 0) { + i_error("%s message UID %u from '%s' failed: %s", + ctx->move ? "Moving" : "Copying", + mail->uid, info->vname, + mailbox_get_last_internal_error(destbox, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, destbox); + ret = -1; + } + } + + if (mailbox_transaction_commit(&desttrans) < 0) { + i_error("Committing %s mails failed: %s", + ctx->move ? "moved" : "copied", + mailbox_get_last_internal_error(destbox, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, destbox); + /* rollback expunges */ + doveadm_mail_iter_deinit_rollback(&iter); + ret = -1; + } else { + if (doveadm_mail_iter_deinit_sync(&iter) < 0) + ret = -1; + } + return ret; +} + +static void +cmd_copy_alloc_source_user(struct copy_cmd_context *ctx) +{ + struct mail_storage_service_input input; + const char *error; + + input = ctx->ctx.storage_service_input; + input.username = ctx->source_username; + + mail_storage_service_io_deactivate_user(ctx->ctx.cur_service_user); + if (mail_storage_service_lookup_next(ctx->ctx.storage_service, &input, + &ctx->source_service_user, + &ctx->source_user, + &error) < 0) + i_fatal("Couldn't lookup user %s: %s", input.username, error); + mail_storage_service_io_deactivate_user(ctx->source_service_user); + mail_storage_service_io_activate_user(ctx->ctx.cur_service_user); +} + +static int +cmd_copy_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct copy_cmd_context *ctx = (struct copy_cmd_context *)_ctx; + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + struct mail_user *src_user; + struct mail_namespace *ns; + struct mailbox *destbox; + const struct mailbox_info *info; + int ret = 0; + + if (ctx->source_username != NULL && ctx->source_user == NULL) + cmd_copy_alloc_source_user(ctx); + + ns = mail_namespace_find(user->namespaces, ctx->destname); + destbox = mailbox_alloc(ns->list, ctx->destname, MAILBOX_FLAG_SAVEONLY); + if (mailbox_open(destbox) < 0) { + i_error("Can't open mailbox '%s': %s", ctx->destname, + mailbox_get_last_internal_error(destbox, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, destbox); + mailbox_free(&destbox); + return -1; + } + + src_user = ctx->source_user != NULL ? ctx->source_user : user; + iter = doveadm_mailbox_list_iter_init(_ctx, src_user, _ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_copy_box(ctx, destbox, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + + if (mailbox_sync(destbox, 0) < 0) { + i_error("Syncing mailbox '%s' failed: %s", ctx->destname, + mailbox_get_last_internal_error(destbox, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, destbox); + ret = -1; + } + mailbox_free(&destbox); + return ret; +} + +static void cmd_copy_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct copy_cmd_context *ctx = (struct copy_cmd_context *)_ctx; + const char *destname = args[0], *cmdname = ctx->move ? "move" : "copy"; + + if (destname == NULL || args[1] == NULL) + doveadm_mail_help_name(cmdname); + args++; + + if (args[0] != NULL && args[1] != NULL && + strcasecmp(args[0], "user") == 0) { + if ((_ctx->service_flags & + MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) == 0) + i_fatal("Use -u parameter to specify destination user"); + + ctx->source_username = p_strdup(_ctx->pool, args[1]); + args += 2; + } + + ctx->destname = p_strdup(ctx->ctx.pool, destname); + _ctx->search_args = doveadm_mail_build_search_args(args); + if (ctx->move) + expunge_search_args_check(ctx->ctx.search_args, cmdname); +} + +static void cmd_copy_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct copy_cmd_context *ctx = (struct copy_cmd_context *)_ctx; + + if (ctx->source_user != NULL) { + mail_storage_service_user_unref(&ctx->source_service_user); + mail_user_deinit(&ctx->source_user); + } +} + +static struct doveadm_mail_cmd_context *cmd_copy_alloc(void) +{ + struct copy_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct copy_cmd_context); + ctx->ctx.v.init = cmd_copy_init; + ctx->ctx.v.deinit = cmd_copy_deinit; + ctx->ctx.v.run = cmd_copy_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return &ctx->ctx; +} + +static struct doveadm_mail_cmd_context *cmd_move_alloc(void) +{ + struct copy_cmd_context *ctx; + + ctx = (struct copy_cmd_context *)cmd_copy_alloc(); + ctx->move = TRUE; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_copy_ver2 = { + .name = "copy", + .mail_cmd = cmd_copy_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<destination> [user <source user>] <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "destination-mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "source-type", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "source-user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_move_ver2 = { + .name = "move", + .mail_cmd = cmd_move_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<destination> [user <source user>] <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "destination-mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "source-type", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "source-user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-deduplicate.c b/src/doveadm/doveadm-mail-deduplicate.c new file mode 100644 index 0000000..f990287 --- /dev/null +++ b/src/doveadm/doveadm-mail-deduplicate.c @@ -0,0 +1,146 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash.h" +#include "mail-storage.h" +#include "mail-search-build.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +struct deduplicate_cmd_context { + struct doveadm_mail_cmd_context ctx; + bool by_msgid; +}; + +static int +cmd_deduplicate_box(struct doveadm_mail_cmd_context *_ctx, + const struct mailbox_info *info, + struct mail_search_args *search_args) +{ + struct deduplicate_cmd_context *ctx = + (struct deduplicate_cmd_context *)_ctx; + struct doveadm_mail_iter *iter; + struct mail *mail; + enum mail_error error; + pool_t pool; + HASH_TABLE(const char *, void *) hash; + const char *key, *errstr; + int ret = 0; + + if (doveadm_mail_iter_init(_ctx, info, search_args, 0, NULL, 0, + &iter) < 0) + return -1; + + pool = pool_alloconly_create("deduplicate", 10240); + hash_table_create(&hash, pool, 0, str_hash, strcmp); + while (doveadm_mail_iter_next(iter, &mail)) { + if (ctx->by_msgid) { + if (mail_get_first_header(mail, "Message-ID", &key) < 0) { + errstr = mailbox_get_last_internal_error(mail->box, &error); + if (error == MAIL_ERROR_NOTFOUND) + continue; + i_error("Couldn't lookup Message-ID: for UID=%u: %s", + mail->uid, errstr); + doveadm_mail_failed_error(_ctx, error); + ret = -1; + break; + } + } else { + if (mail_get_special(mail, MAIL_FETCH_GUID, &key) < 0) { + errstr = mailbox_get_last_internal_error(mail->box, &error); + if (error == MAIL_ERROR_NOTFOUND) + continue; + i_error("Couldn't lookup GUID: for UID=%u: %s", + mail->uid, errstr); + doveadm_mail_failed_error(_ctx, error); + ret = -1; + break; + } + } + if (key != NULL && *key != '\0') { + if (hash_table_lookup(hash, key) != NULL) + mail_expunge(mail); + else { + key = p_strdup(pool, key); + hash_table_insert(hash, key, POINTER_CAST(1)); + } + } + } + + if (doveadm_mail_iter_deinit_sync(&iter) < 0) + ret = -1; + + hash_table_destroy(&hash); + pool_unref(&pool); + return ret; +} + +static int +cmd_deduplicate_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) +{ + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_deduplicate_box(ctx, info, ctx->search_args) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_deduplicate_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("deduplicate"); + + ctx->search_args = doveadm_mail_build_search_args(args); +} + +static bool +cmd_deduplicate_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct deduplicate_cmd_context *ctx = + (struct deduplicate_cmd_context *)_ctx; + + switch (c) { + case 'm': + ctx->by_msgid = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_deduplicate_alloc(void) +{ + struct deduplicate_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct deduplicate_cmd_context); + ctx->ctx.getopt_args = "m"; + ctx->ctx.v.parse_arg = cmd_deduplicate_parse_arg; + ctx->ctx.v.init = cmd_deduplicate_init; + ctx->ctx.v.run = cmd_deduplicate_run; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_deduplicate_ver2 = { + .name = "deduplicate", + .mail_cmd = cmd_deduplicate_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-m] <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('m', "by-msgid", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-expunge.c b/src/doveadm/doveadm-mail-expunge.c new file mode 100644 index 0000000..4ab7e3d --- /dev/null +++ b/src/doveadm/doveadm-mail-expunge.c @@ -0,0 +1,286 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "mail-index.h" +#include "mail-storage.h" +#include "mail-search.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +struct expunge_cmd_context { + struct doveadm_mail_cmd_context ctx; + bool delete_empty_mailbox; +}; + +static int +cmd_expunge_box(struct doveadm_mail_cmd_context *_ctx, + const struct mailbox_info *info, + struct mail_search_args *search_args) +{ + struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx; + struct doveadm_mail_iter *iter; + struct mailbox *box; + struct mail *mail; + enum mail_error error; + int ret = 0; + + if (doveadm_mail_iter_init(_ctx, info, search_args, 0, NULL, 0, + &iter) < 0) + return -1; + + while (doveadm_mail_iter_next(iter, &mail)) { + if (doveadm_debug) { + i_debug("expunge: box=%s uid=%u", + info->vname, mail->uid); + } + mail_expunge(mail); + } + + if (doveadm_mail_iter_deinit_keep_box(&iter, &box) < 0) + ret = -1; + else if (mailbox_sync(box, 0) < 0) { + i_error("Syncing mailbox '%s' failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + + if (ctx->delete_empty_mailbox && ret == 0) { + if (mailbox_delete_empty(box) < 0) { + error = mailbox_get_last_mail_error(box); + if (error != MAIL_ERROR_EXISTS) { + i_error("Deleting mailbox '%s' failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } else { + if (mailbox_set_subscribed(box, FALSE) < 0) { + i_error("Unsubscribing mailbox '%s' failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } + } + mailbox_free(&box); + return ret; +} + +static bool +expunge_search_args_is_mailbox_ok(struct mail_search_arg *args); + +static bool +expunge_search_args_is_mailbox_or_ok(struct mail_search_arg *args) +{ + struct mail_search_arg *arg; + + for (arg = args; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_OR: + if (!expunge_search_args_is_mailbox_or_ok(arg->value.subargs)) + return FALSE; + break; + case SEARCH_SUB: + case SEARCH_INTHREAD: + if (!expunge_search_args_is_mailbox_ok(arg->value.subargs)) + return FALSE; + break; + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + break; + default: + return FALSE; + } + } + return TRUE; +} + +static bool +expunge_search_args_is_mailbox_ok(struct mail_search_arg *args) +{ + struct mail_search_arg *arg; + bool have_or = FALSE; + + /* a) we find one mailbox here in the SUB block */ + for (arg = args; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + return TRUE; + case SEARCH_OR: + have_or = TRUE; + break; + case SEARCH_SUB: + case SEARCH_INTHREAD: + if (expunge_search_args_is_mailbox_ok(arg->value.subargs)) + return TRUE; + break; + default: + break; + } + } + + /* b) there is at least one OR block, and all of the ORs must have + mailbox */ + if (!have_or) + return FALSE; + + for (arg = args; arg != NULL; arg = arg->next) { + if (arg->type == SEARCH_OR && + !expunge_search_args_is_mailbox_or_ok(arg->value.subargs)) + return FALSE; + } + return TRUE; +} + +static bool +expunge_search_args_is_msgset_ok(struct mail_search_arg *args); + +static bool +expunge_search_args_is_msgset_or_ok(struct mail_search_arg *args) +{ + struct mail_search_arg *arg; + + /* we're done if all OR branches contain something else besides + MAILBOXes */ + for (arg = args; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + return FALSE; + case SEARCH_OR: + if (!expunge_search_args_is_msgset_or_ok(arg->value.subargs)) + return FALSE; + break; + case SEARCH_SUB: + if (!expunge_search_args_is_msgset_ok(arg->value.subargs)) + return FALSE; + break; + default: + break; + } + } + return TRUE; +} + +static bool +expunge_search_args_is_msgset_ok(struct mail_search_arg *args) +{ + struct mail_search_arg *arg; + + /* all args can't be just MAILBOXes */ + for (arg = args; arg != NULL; arg = arg->next) { + switch (arg->type) { + case SEARCH_MAILBOX: + case SEARCH_MAILBOX_GUID: + case SEARCH_MAILBOX_GLOB: + break; + case SEARCH_OR: + /* if each OR branch has something else than just + MAILBOXes, we're ok */ + if (expunge_search_args_is_msgset_or_ok(arg->value.subargs)) + return TRUE; + break; + case SEARCH_SUB: + if (expunge_search_args_is_msgset_ok(arg->value.subargs)) + return TRUE; + break; + default: + return TRUE; + } + } + return FALSE; +} + +static int +cmd_expunge_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) +{ + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_expunge_box(ctx, info, ctx->search_args) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +void expunge_search_args_check(struct mail_search_args *args, const char *cmd) +{ + if (!expunge_search_args_is_mailbox_ok(args->args)) { + i_fatal_status(EX_USAGE, + "%s: To avoid accidents, search query " + "must contain MAILBOX in all search branches", cmd); + } + if (!expunge_search_args_is_msgset_ok(args->args)) { + i_fatal_status(EX_USAGE, + "%s: To avoid accidents, each branch in search query " + "must contain something else besides MAILBOX " + "(e.g. just add \"all\" if you want everything)", cmd); + } + mail_search_args_simplify(args); +} + +static void cmd_expunge_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("expunge"); + + ctx->search_args = doveadm_mail_build_search_args(args); + expunge_search_args_check(ctx->search_args, "expunge"); +} + +static bool cmd_expunge_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx; + + switch (c) { + case 'd': + ctx->delete_empty_mailbox = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_expunge_alloc(void) +{ + struct expunge_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct expunge_cmd_context); + ctx->ctx.getopt_args = "d"; + ctx->ctx.v.parse_arg = cmd_expunge_parse_arg; + ctx->ctx.v.init = cmd_expunge_init; + ctx->ctx.v.run = cmd_expunge_run; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_expunge_ver2 = { + .name = "expunge", + .mail_cmd = cmd_expunge_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-m] <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('d', "delete-empty-mailbox", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-fetch.c b/src/doveadm/doveadm-mail-fetch.c new file mode 100644 index 0000000..932a54d --- /dev/null +++ b/src/doveadm/doveadm-mail-fetch.c @@ -0,0 +1,683 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "str.h" +#include "message-address.h" +#include "message-size.h" +#include "message-parser.h" +#include "message-header-decode.h" +#include "message-decoder.h" +#include "imap-util.h" +#include "mail-user.h" +#include "mail-storage.h" +#include "mail-search.h" +#include "mail-namespace.h" +#include "imap-msgpart.h" +#include "doveadm-print.h" +#include "doveadm-mail.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" + +#include <stdio.h> + +struct fetch_cmd_context { + struct doveadm_mail_cmd_context ctx; + + struct mail *mail; + + ARRAY(struct fetch_field) fields; + ARRAY_TYPE(const_string) header_fields; + enum mail_fetch_field wanted_fields; + + const struct fetch_field *cur_field; + /* if print() returns -1, log this error if non-NULL. otherwise log + the storage error. */ + const char *print_error; +}; + +struct fetch_field { + const char *name; + enum mail_fetch_field wanted_fields; + int (*print)(struct fetch_cmd_context *ctx); +}; + +static int fetch_user(struct fetch_cmd_context *ctx) +{ + doveadm_print(ctx->ctx.cur_mail_user->username); + return 0; +} + +static int fetch_mailbox(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME, &value) < 0) + return -1; + + doveadm_print(value); + return 0; +} + +static int fetch_mailbox_guid(struct fetch_cmd_context *ctx) +{ + struct mailbox_metadata metadata; + + if (mailbox_get_metadata(ctx->mail->box, MAILBOX_METADATA_GUID, + &metadata) < 0) + return -1; + doveadm_print(guid_128_to_string(metadata.guid)); + return 0; +} + +static int fetch_seq(struct fetch_cmd_context *ctx) +{ + doveadm_print_num(ctx->mail->seq); + return 0; +} + +static int fetch_uid(struct fetch_cmd_context *ctx) +{ + doveadm_print_num(ctx->mail->uid); + return 0; +} + +static int fetch_guid(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_GUID, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static int fetch_flags(struct fetch_cmd_context *ctx) +{ + string_t *str = t_str_new(64); + + imap_write_flags(str, mail_get_flags(ctx->mail), + mail_get_keywords(ctx->mail)); + doveadm_print(str_c(str)); + return 0; +} + +static int fetch_modseq(struct fetch_cmd_context *ctx) +{ + doveadm_print_num(mail_get_modseq(ctx->mail)); + return 0; +} + +static void +fetch_set_istream_error(struct fetch_cmd_context *ctx, struct istream *input) +{ + ctx->print_error = t_strdup_printf("read(%s) failed: %s", + i_stream_get_name(input), i_stream_get_error(input)); +} + +static int fetch_hdr(struct fetch_cmd_context *ctx) +{ + struct istream *input; + struct message_size hdr_size; + int ret; + + if (mail_get_hdr_stream(ctx->mail, &hdr_size, &input) < 0) + return -1; + + input = i_stream_create_limit(input, hdr_size.physical_size); + if ((ret = doveadm_print_istream(input)) < 0) + fetch_set_istream_error(ctx, input); + i_stream_unref(&input); + return ret; +} + +static int fetch_hdr_field(struct fetch_cmd_context *ctx) +{ + const char *const *value, *filter, *name = ctx->cur_field->name; + string_t *str = t_str_new(256); + bool add_lf = FALSE; + + filter = strchr(name, '.'); + if (filter != NULL) + name = t_strdup_until(name, filter++); + + if (filter != NULL && strcmp(filter, "utf8") == 0) { + if (mail_get_headers_utf8(ctx->mail, name, &value) < 0) + return -1; + } else { + if (mail_get_headers(ctx->mail, name, &value) < 0) + return -1; + } + + for (; *value != NULL; value++) { + if (add_lf) + str_append_c(str, '\n'); + str_append(str, *value); + add_lf = TRUE; + } + + if (filter == NULL || strcmp(filter, "utf8") == 0) { + /* print the header as-is */ + } else if (strcmp(filter, "address") == 0 || + strcmp(filter, "address_name") == 0 || + strcmp(filter, "address_name.utf8") == 0) { + struct message_address *addr; + + addr = message_address_parse(pool_datastack_create(), + str_data(str), str_len(str), + UINT_MAX, 0); + str_truncate(str, 0); + add_lf = FALSE; + for (; addr != NULL; addr = addr->next) { + if (add_lf) + str_append_c(str, '\n'); + if (strcmp(filter, "address") == 0) { + if (addr->mailbox != NULL) + str_append(str, addr->mailbox); + if (addr->domain != NULL) { + str_append_c(str, '@'); + str_append(str, addr->domain); + } + } else if (addr->name != NULL) { + if (strcmp(filter, "address_name") == 0) + str_append(str, addr->name); + else { + message_header_decode_utf8( + (const void *)addr->name, + strlen(addr->name), str, NULL); + } + } + add_lf = TRUE; + } + } else { + i_fatal("Unknown header filter: %s", filter); + } + doveadm_print(str_c(str)); + return 0; +} + +static int fetch_body_field(struct fetch_cmd_context *ctx) +{ + const char *name = ctx->cur_field->name; + struct imap_msgpart *msgpart; + struct imap_msgpart_open_result result; + bool binary; + int ret; + + binary = str_begins(name, "binary."); + name += binary ? 7 : 5; + if (imap_msgpart_parse(name, &msgpart) < 0) + i_unreached(); /* we already verified this was ok */ + if (binary) + imap_msgpart_set_decode_to_binary(msgpart); + + if (imap_msgpart_open(ctx->mail, msgpart, &result) < 0) { + imap_msgpart_free(&msgpart); + return -1; + } + if ((ret = doveadm_print_istream(result.input)) < 0) + fetch_set_istream_error(ctx, result.input); + i_stream_unref(&result.input); + imap_msgpart_free(&msgpart); + return ret; +} + +static int fetch_body(struct fetch_cmd_context *ctx) +{ + struct istream *input; + struct message_size hdr_size; + int ret; + + if (mail_get_stream(ctx->mail, &hdr_size, NULL, &input) < 0) + return -1; + + i_stream_skip(input, hdr_size.physical_size); + if ((ret = doveadm_print_istream(input)) < 0) + fetch_set_istream_error(ctx, input); + return ret; +} + +static int fetch_body_snippet(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_BODY_SNIPPET, &value) < 0) + return -1; + /* [0] contains the snippet algorithm, skip over it */ + i_assert(value[0] != '\0'); + doveadm_print(value + 1); + return 0; +} + +static int fetch_text(struct fetch_cmd_context *ctx) +{ + struct istream *input; + int ret; + + if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0) + return -1; + if ((ret = doveadm_print_istream(input)) < 0) + fetch_set_istream_error(ctx, input); + return ret; +} + +static int fetch_text_utf8(struct fetch_cmd_context *ctx) +{ + const struct message_parser_settings parser_set = { + .hdr_flags = MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE, + }; + struct istream *input; + struct message_parser_ctx *parser; + struct message_decoder_context *decoder; + struct message_block raw_block, block; + struct message_part *parts; + int ret = 0; + + if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0) + return -1; + + parser = message_parser_init(pool_datastack_create(), input, &parser_set); + decoder = message_decoder_init(NULL, 0); + + while ((ret = message_parser_parse_next_block(parser, &raw_block)) > 0) { + if (!message_decoder_decode_next_block(decoder, &raw_block, + &block)) + continue; + + if (block.hdr == NULL) { + if (block.size > 0) + doveadm_print_stream(block.data, block.size); + } else if (block.hdr->eoh) + doveadm_print_stream("\n", 1); + else { + i_assert(block.hdr->name_len > 0); + doveadm_print_stream(block.hdr->name, + block.hdr->name_len); + doveadm_print_stream(": ", 2); + if (block.hdr->full_value_len > 0) { + doveadm_print_stream(block.hdr->full_value, + block.hdr->full_value_len); + } + doveadm_print_stream("\n", 1); + } + } + i_assert(ret != 0); + message_decoder_deinit(&decoder); + message_parser_deinit(&parser, &parts); + + doveadm_print_stream("", 0); + if (input->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(input), + i_stream_get_error(input)); + return -1; + } + return 0; +} + +static int fetch_size_physical(struct fetch_cmd_context *ctx) +{ + uoff_t size; + + if (mail_get_physical_size(ctx->mail, &size) < 0) + return -1; + doveadm_print_num(size); + return 0; +} + +static int fetch_size_virtual(struct fetch_cmd_context *ctx) +{ + uoff_t size; + + if (mail_get_virtual_size(ctx->mail, &size) < 0) + return -1; + doveadm_print_num(size); + return 0; +} + +static int fetch_date_received(struct fetch_cmd_context *ctx) +{ + time_t t; + + if (mail_get_received_date(ctx->mail, &t) < 0) + return -1; + doveadm_print(unixdate2str(t)); + return 0; +} + +static int fetch_date_sent(struct fetch_cmd_context *ctx) +{ + time_t t; + int tz; + char chr; + + if (mail_get_date(ctx->mail, &t, &tz) < 0) + return -1; + + chr = tz < 0 ? '-' : '+'; + if (tz < 0) tz = -tz; + doveadm_print(t_strdup_printf("%s (%c%02u%02u)", unixdate2str(t), + chr, tz/60, tz%60)); + return 0; +} + +static int fetch_date_saved(struct fetch_cmd_context *ctx) +{ + time_t t; + + if (mail_get_save_date(ctx->mail, &t) < 0) + return -1; + doveadm_print(unixdate2str(t)); + return 0; +} + +static int fetch_date_received_unixtime(struct fetch_cmd_context *ctx) +{ + time_t t; + + if (mail_get_received_date(ctx->mail, &t) < 0) + return -1; + doveadm_print(dec2str(t)); + return 0; +} + +static int fetch_date_sent_unixtime(struct fetch_cmd_context *ctx) +{ + time_t t; + int tz; + + if (mail_get_date(ctx->mail, &t, &tz) < 0) + return -1; + + doveadm_print(dec2str(t)); + return 0; +} + +static int fetch_date_saved_unixtime(struct fetch_cmd_context *ctx) +{ + time_t t; + + if (mail_get_save_date(ctx->mail, &t) < 0) + return -1; + doveadm_print(dec2str(t)); + return 0; +} + +static int fetch_imap_envelope(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_IMAP_ENVELOPE, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static int fetch_imap_body(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_IMAP_BODY, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static int fetch_imap_bodystructure(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_IMAP_BODYSTRUCTURE, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} +static int fetch_pop3_uidl(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_UIDL_BACKEND, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static int fetch_pop3_order(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_POP3_ORDER, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static int fetch_refcount(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_REFCOUNT, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static int fetch_storageid(struct fetch_cmd_context *ctx) +{ + const char *value; + + if (mail_get_special(ctx->mail, MAIL_FETCH_STORAGE_ID, &value) < 0) + return -1; + doveadm_print(value); + return 0; +} + +static const struct fetch_field fetch_fields[] = { + { "user", 0, fetch_user }, + { "mailbox", 0, fetch_mailbox }, + { "mailbox-guid", 0, fetch_mailbox_guid }, + { "seq", 0, fetch_seq }, + { "uid", 0, fetch_uid }, + { "guid", 0, fetch_guid }, + { "flags", MAIL_FETCH_FLAGS, fetch_flags }, + { "modseq", 0, fetch_modseq }, + { "hdr", MAIL_FETCH_STREAM_HEADER, fetch_hdr }, + { "body", MAIL_FETCH_STREAM_BODY, fetch_body }, + { "body.preview", MAIL_FETCH_BODY_SNIPPET, fetch_body_snippet }, + { "body.snippet", MAIL_FETCH_BODY_SNIPPET, fetch_body_snippet }, + { "text", MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY, fetch_text }, + { "text.utf8", MAIL_FETCH_STREAM_HEADER | + MAIL_FETCH_STREAM_BODY, fetch_text_utf8 }, + { "size.physical", MAIL_FETCH_PHYSICAL_SIZE, fetch_size_physical }, + { "size.virtual", MAIL_FETCH_VIRTUAL_SIZE, fetch_size_virtual }, + { "date.received", MAIL_FETCH_RECEIVED_DATE, fetch_date_received }, + { "date.sent", MAIL_FETCH_DATE, fetch_date_sent }, + { "date.saved", MAIL_FETCH_SAVE_DATE, fetch_date_saved }, + { "date.received.unixtime", MAIL_FETCH_RECEIVED_DATE, fetch_date_received_unixtime }, + { "date.sent.unixtime", MAIL_FETCH_DATE, fetch_date_sent_unixtime }, + { "date.saved.unixtime", MAIL_FETCH_SAVE_DATE, fetch_date_saved_unixtime }, + { "imap.envelope", MAIL_FETCH_IMAP_ENVELOPE, fetch_imap_envelope }, + { "imap.body", MAIL_FETCH_IMAP_BODY, fetch_imap_body }, + { "imap.bodystructure", MAIL_FETCH_IMAP_BODYSTRUCTURE, fetch_imap_bodystructure }, + { "pop3.uidl", MAIL_FETCH_UIDL_BACKEND, fetch_pop3_uidl }, + { "pop3.order", MAIL_FETCH_POP3_ORDER, fetch_pop3_order }, + { "refcount", MAIL_FETCH_REFCOUNT, fetch_refcount }, + { "storageid", MAIL_FETCH_STORAGE_ID, fetch_storageid } +}; + +static const struct fetch_field *fetch_field_find(const char *name) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(fetch_fields); i++) { + if (strcmp(fetch_fields[i].name, name) == 0) + return &fetch_fields[i]; + } + return NULL; +} + +static void print_fetch_fields(void) +{ + unsigned int i; + + fprintf(stderr, "Available fetch fields: hdr.<name> body.<section> binary.<section> %s", fetch_fields[0].name); + for (i = 1; i < N_ELEMENTS(fetch_fields); i++) + fprintf(stderr, " %s", fetch_fields[i].name); + fprintf(stderr, "\n"); +} + +static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str) +{ + const char *const *fields, *name; + const struct fetch_field *field; + struct fetch_field hdr_field, body_field; + struct imap_msgpart *msgpart; + + i_zero(&hdr_field); + hdr_field.print = fetch_hdr_field; + + i_zero(&body_field); + body_field.print = fetch_body_field; + + t_array_init(&ctx->fields, 32); + t_array_init(&ctx->header_fields, 32); + fields = t_strsplit_spaces(str, " "); + for (; *fields != NULL; fields++) { + name = t_str_lcase(*fields); + + doveadm_print_header_simple(name); + if ((field = fetch_field_find(name)) != NULL) { + ctx->wanted_fields |= field->wanted_fields; + array_push_back(&ctx->fields, field); + } else if (str_begins(name, "hdr.")) { + name += 4; + hdr_field.name = name; + array_push_back(&ctx->fields, &hdr_field); + name = t_strcut(name, '.'); + array_push_back(&ctx->header_fields, &name); + } else if (str_begins(name, "body.") || + str_begins(name, "binary.")) { + bool binary = str_begins(name, "binary."); + body_field.name = t_strarray_join(t_strsplit(name, ","), " "); + + name += binary ? 7 : 5; + if (imap_msgpart_parse(name, &msgpart) < 0) { + print_fetch_fields(); + i_fatal("Unknown fetch section: %s", name); + } + array_push_back(&ctx->fields, &body_field); + ctx->wanted_fields |= imap_msgpart_get_fetch_data(msgpart); + imap_msgpart_free(&msgpart); + } else { + print_fetch_fields(); + i_fatal("Unknown fetch field: %s", name); + } + } + array_append_zero(&ctx->header_fields); +} + +static int cmd_fetch_mail(struct fetch_cmd_context *ctx) +{ + const struct fetch_field *field; + struct mail *mail = ctx->mail; + int ret = 0; + + array_foreach(&ctx->fields, field) { + ctx->cur_field = field; + if (field->print(ctx) < 0) { + i_error("fetch(%s) failed for box=%s uid=%u: %s", + field->name, mailbox_get_vname(mail->box), + mail->uid, + ctx->print_error != NULL ? ctx->print_error : + mailbox_get_last_internal_error(mail->box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, mail->box); + ctx->print_error = NULL; + ret = -1; + } + } + return ret; +} + +static int +cmd_fetch_box(struct fetch_cmd_context *ctx, const struct mailbox_info *info) +{ + struct doveadm_mail_iter *iter; + int ret = 0; + + if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args, + ctx->wanted_fields, + array_front(&ctx->header_fields), + DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT, + &iter) < 0) + return -1; + + while (doveadm_mail_iter_next(iter, &ctx->mail)) { + T_BEGIN { + if (cmd_fetch_mail(ctx) < 0) + ret = -1; + } T_END; + } + if (doveadm_mail_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int +cmd_fetch_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct fetch_cmd_context *ctx = (struct fetch_cmd_context *)_ctx; + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(_ctx, user, _ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_fetch_box(ctx, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_fetch_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct fetch_cmd_context *ctx = (struct fetch_cmd_context *)_ctx; + const char *fetch_fields = args[0]; + + if (fetch_fields == NULL || args[1] == NULL) + doveadm_mail_help_name("fetch"); + + parse_fetch_fields(ctx, fetch_fields); + _ctx->search_args = doveadm_mail_build_search_args(args + 1); +} + +static struct doveadm_mail_cmd_context *cmd_fetch_alloc(void) +{ + struct fetch_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct fetch_cmd_context); + ctx->ctx.v.init = cmd_fetch_init; + ctx->ctx.v.run = cmd_fetch_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_PAGER); + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_fetch_ver2 = { + .name = "fetch", + .mail_cmd = cmd_fetch_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<fields> <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "field", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('\0', "fieldstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL | CMD_PARAM_FLAG_DO_NOT_EXPOSE) /* FIXME: horrible hack, remove me when possible */ +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-flags.c b/src/doveadm/doveadm-mail-flags.c new file mode 100644 index 0000000..7c252a7 --- /dev/null +++ b/src/doveadm/doveadm-mail-flags.c @@ -0,0 +1,178 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "imap-util.h" +#include "mail-storage.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +#include <stdio.h> + +struct flags_cmd_context { + struct doveadm_mail_cmd_context ctx; + enum modify_type modify_type; + enum mail_flags flags; + const char *const *keywords; +}; + +static int +cmd_flags_run_box(struct flags_cmd_context *ctx, + const struct mailbox_info *info) +{ + struct doveadm_mail_iter *iter; + struct mailbox *box; + struct mail *mail; + struct mail_keywords *kw = NULL; + + if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args, + 0, NULL, 0, &iter) < 0) + return -1; + box = doveadm_mail_iter_get_mailbox(iter); + + if (ctx->keywords != NULL) { + if (mailbox_keywords_create(box, ctx->keywords, &kw) < 0) { + i_error("Invalid keywords: %s", + mailbox_get_last_internal_error(box, NULL)); + (void)doveadm_mail_iter_deinit(&iter); + ctx->ctx.exit_code = DOVEADM_EX_NOTPOSSIBLE; + return -1; + } + } + + while (doveadm_mail_iter_next(iter, &mail)) { + mail_update_flags(mail, ctx->modify_type, ctx->flags); + if (kw != NULL) + mail_update_keywords(mail, ctx->modify_type, kw); + } + if (kw != NULL) + mailbox_keywords_unref(&kw); + return doveadm_mail_iter_deinit_sync(&iter); +} + +static int +cmd_flags_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct flags_cmd_context *ctx = (struct flags_cmd_context *)_ctx; + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(_ctx, user, _ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_flags_run_box(ctx, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_flags_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct flags_cmd_context *ctx = (struct flags_cmd_context *)_ctx; + const char *const *tmp; + enum mail_flags flag; + ARRAY_TYPE(const_string) keywords; + + if (args[0] == NULL || args[1] == NULL) { + switch (ctx->modify_type) { + case MODIFY_ADD: + doveadm_mail_help_name("flags add"); + case MODIFY_REMOVE: + doveadm_mail_help_name("flags remove"); + case MODIFY_REPLACE: + doveadm_mail_help_name("flags replace"); + } + i_unreached(); + } + + p_array_init(&keywords, _ctx->pool, 8); + for (tmp = t_strsplit(args[0], " "); *tmp != NULL; tmp++) { + const char *str = *tmp; + + if (str[0] == '\\') { + flag = imap_parse_system_flag(str); + if (flag == 0) + i_fatal("Invalid system flag: %s", str); + ctx->flags |= flag; + } else { + str = p_strdup(_ctx->pool, str); + array_push_back(&keywords, &str); + } + } + if (array_count(&keywords) > 0 || ctx->modify_type == MODIFY_REPLACE) { + array_append_zero(&keywords); + ctx->keywords = array_front(&keywords); + } + + _ctx->search_args = doveadm_mail_build_search_args(args+1); +} + +static struct doveadm_mail_cmd_context * +cmd_flag_alloc(enum modify_type modify_type) +{ + struct flags_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct flags_cmd_context); + ctx->modify_type = modify_type; + ctx->ctx.v.init = cmd_flags_init; + ctx->ctx.v.run = cmd_flags_run; + return &ctx->ctx; +} + +static struct doveadm_mail_cmd_context *cmd_flags_add_alloc(void) +{ + return cmd_flag_alloc(MODIFY_ADD); +} + +static struct doveadm_mail_cmd_context *cmd_flags_remove_alloc(void) +{ + return cmd_flag_alloc(MODIFY_REMOVE); +} + +static struct doveadm_mail_cmd_context *cmd_flags_replace_alloc(void) +{ + return cmd_flag_alloc(MODIFY_REPLACE); +} + +struct doveadm_cmd_ver2 doveadm_cmd_flags_add_ver2 = { + .name = "flags add", + .mail_cmd = cmd_flags_add_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<flags> <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "flag", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('\0', "flagstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL|CMD_PARAM_FLAG_DO_NOT_EXPOSE) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_flags_remove_ver2 = { + .name = "flags remove", + .mail_cmd = cmd_flags_remove_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<flags> <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "flag", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('\0', "flagstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL|CMD_PARAM_FLAG_DO_NOT_EXPOSE) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_flags_replace_ver2 = { + .name = "flags replace", + .mail_cmd = cmd_flags_replace_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<flags> <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "flag", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('\0', "flagstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL|CMD_PARAM_FLAG_DO_NOT_EXPOSE) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-import.c b/src/doveadm/doveadm-mail-import.c new file mode 100644 index 0000000..bd6ef6d --- /dev/null +++ b/src/doveadm/doveadm-mail-import.c @@ -0,0 +1,276 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "mail-storage.h" +#include "mail-storage-service.h" +#include "mail-namespace.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +struct import_cmd_context { + struct doveadm_mail_cmd_context ctx; + + const char *src_location; + const char *src_username; + struct mail_user *src_user; + const char *dest_parent; + bool subscribe; +}; + +static const char * +convert_vname_separators(const char *vname, char src_sep, char dest_sep) +{ + string_t *str = t_str_new(128); + + for (; *vname != '\0'; vname++) { + if (*vname == src_sep) + str_append_c(str, dest_sep); + else if (*vname == dest_sep) + str_append_c(str, '_'); + else + str_append_c(str, *vname); + } + return str_c(str); +} + +static int +dest_mailbox_open_or_create(struct import_cmd_context *ctx, + struct mail_user *user, + const struct mailbox_info *info, + struct mailbox **box_r) +{ + struct mail_namespace *ns; + struct mailbox *box; + enum mail_error error; + const char *name, *errstr; + + if (*ctx->dest_parent != '\0') { + /* prefix destination mailbox name with given parent mailbox */ + ns = mail_namespace_find(user->namespaces, ctx->dest_parent); + } else { + ns = mail_namespace_find(user->namespaces, info->vname); + } + name = convert_vname_separators(info->vname, + mail_namespace_get_sep(info->ns), + mail_namespace_get_sep(ns)); + + if (*ctx->dest_parent != '\0') { + name = t_strdup_printf("%s%c%s", ctx->dest_parent, + mail_namespace_get_sep(ns), name); + } + + box = mailbox_alloc(ns->list, name, MAILBOX_FLAG_SAVEONLY); + if (mailbox_create(box, NULL, FALSE) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error != MAIL_ERROR_EXISTS) { + i_error("Couldn't create mailbox %s: %s", name, errstr); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + mailbox_free(&box); + return -1; + } + } + if (ctx->subscribe) { + if (mailbox_set_subscribed(box, TRUE) < 0) { + i_error("Couldn't subscribe to mailbox %s: %s", + name, mailbox_get_last_internal_error(box, NULL)); + } + } + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Syncing mailbox %s failed: %s", name, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + mailbox_free(&box); + return -1; + } + *box_r = box; + return 0; +} + +static int +cmd_import_box_contents(struct doveadm_mail_cmd_context *ctx, + struct doveadm_mail_iter *iter, struct mail *src_mail, + struct mailbox *dest_box) +{ + struct mail_save_context *save_ctx; + struct mailbox_transaction_context *dest_trans; + const char *mailbox = mailbox_get_vname(dest_box); + int ret = 0; + + dest_trans = mailbox_transaction_begin(dest_box, + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + ctx->transaction_flags, __func__); + do { + if (doveadm_debug) { + i_debug("import: box=%s uid=%u", + mailbox, src_mail->uid); + } + save_ctx = mailbox_save_alloc(dest_trans); + mailbox_save_copy_flags(save_ctx, src_mail); + if (mailbox_copy(&save_ctx, src_mail) < 0) { + i_error("Copying box=%s uid=%u failed: %s", + mailbox, src_mail->uid, + mailbox_get_last_internal_error(dest_box, NULL)); + ret = -1; + } + } while (doveadm_mail_iter_next(iter, &src_mail)); + + if (mailbox_transaction_commit(&dest_trans) < 0) { + i_error("Committing copied mails to %s failed: %s", mailbox, + mailbox_get_last_internal_error(dest_box, NULL)); + ret = -1; + } + return ret; +} + +static int +cmd_import_box(struct import_cmd_context *ctx, struct mail_user *dest_user, + const struct mailbox_info *info, + struct mail_search_args *search_args) +{ + struct doveadm_mail_iter *iter; + struct mailbox *box; + struct mail *mail; + int ret = 0; + + if (doveadm_mail_iter_init(&ctx->ctx, info, search_args, 0, NULL, + DOVEADM_MAIL_ITER_FLAG_READONLY, + &iter) < 0) + return -1; + + if (doveadm_mail_iter_next(iter, &mail)) { + /* at least one mail matches in this mailbox */ + if (dest_mailbox_open_or_create(ctx, dest_user, info, &box) < 0) + ret = -1; + else { + if (cmd_import_box_contents(&ctx->ctx, iter, mail, box) < 0) { + doveadm_mail_failed_mailbox(&ctx->ctx, mail->box); + ret = -1; + } + mailbox_free(&box); + } + } + if (doveadm_mail_iter_deinit_sync(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_import_init_source_user(struct import_cmd_context *ctx, struct mail_user *dest_user) +{ + struct mail_storage_service_input input; + struct mail_storage_service_user *service_user; + struct mail_user *user; + const char *error; + + /* create a user for accessing the source storage */ + i_zero(&input); + input.module = "mail"; + input.username = ctx->src_username != NULL ? + ctx->src_username : + dest_user->username; + + mail_storage_service_io_deactivate_user(ctx->ctx.cur_service_user); + input.flags_override_add = MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES | + MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS; + if (mail_storage_service_lookup_next(ctx->ctx.storage_service, &input, + &service_user, &user, &error) < 0) + i_fatal("Import user initialization failed: %s", error); + if (mail_namespaces_init_location(user, ctx->src_location, &error) < 0) + i_fatal("Import namespace initialization failed: %s", error); + + ctx->src_user = user; + mail_storage_service_io_deactivate_user(service_user); + mail_storage_service_user_unref(&service_user); + mail_storage_service_io_activate_user(ctx->ctx.cur_service_user); +} + +static int +cmd_import_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx; + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + if (ctx->src_user == NULL) + cmd_import_init_source_user(ctx, user); + + iter = doveadm_mailbox_list_iter_init(_ctx, ctx->src_user, + _ctx->search_args, iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_import_box(ctx, user, info, _ctx->search_args) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_import_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx; + + if (str_array_length(args) < 3) + doveadm_mail_help_name("import"); + ctx->src_location = p_strdup(_ctx->pool, args[0]); + ctx->dest_parent = p_strdup(_ctx->pool, args[1]); + ctx->ctx.search_args = doveadm_mail_build_search_args(args+2); +} + +static void cmd_import_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx; + + if (ctx->src_user != NULL) + mail_user_deinit(&ctx->src_user); +} + +static bool cmd_import_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx; + + switch (c) { + case 'U': + ctx->src_username = p_strdup(_ctx->pool, optarg); + break; + case 's': + ctx->subscribe = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_import_alloc(void) +{ + struct import_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct import_cmd_context); + ctx->ctx.getopt_args = "s"; + ctx->ctx.v.parse_arg = cmd_import_parse_arg; + ctx->ctx.v.init = cmd_import_init; + ctx->ctx.v.deinit = cmd_import_deinit; + ctx->ctx.v.run = cmd_import_run; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_import_ver2 = { + .name = "import", + .mail_cmd = cmd_import_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-U source-user] [-s] <source mail location> <dest parent mailbox> <search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('U', "source-user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('s', "subscribe", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "source-location", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "dest-parent-mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-index.c b/src/doveadm/doveadm-mail-index.c new file mode 100644 index 0000000..524f373 --- /dev/null +++ b/src/doveadm/doveadm-mail-index.c @@ -0,0 +1,300 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strescape.h" +#include "net.h" +#include "write-full.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mail-search-build.h" +#include "mailbox-list-iter.h" +#include "doveadm-settings.h" +#include "doveadm-mail.h" + +#include <stdio.h> + +#define INDEXER_SOCKET_NAME "indexer" +#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n" + +struct index_cmd_context { + struct doveadm_mail_cmd_context ctx; + + int queue_fd; + unsigned int max_recent_msgs; + bool queue:1; + bool have_wildcards:1; +}; + +static int cmd_index_box_precache(struct doveadm_mail_cmd_context *dctx, + struct mailbox *box) +{ + struct mailbox_status status; + struct mailbox_transaction_context *trans; + struct mail_search_args *search_args; + struct mail_search_context *ctx; + struct mail *mail; + struct mailbox_metadata metadata; + uint32_t seq; + unsigned int counter = 0, max; + int ret = 0; + + if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS, + &metadata) < 0) { + i_error("Mailbox %s: Precache-fields lookup failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + } + if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ, + &status) < 0) { + i_error("Mailbox %s: Status lookup failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + return -1; + } + + seq = status.last_cached_seq + 1; + if (seq > status.messages) { + if (doveadm_verbose) { + i_info("%s: Cache is already up to date", + mailbox_get_vname(box)); + } + return 0; + } + if (doveadm_verbose) { + i_info("%s: Caching mails seq=%u..%u", + mailbox_get_vname(box), seq, status.messages); + } + + trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC | + dctx->transaction_flags, __func__); + search_args = mail_search_build_init(); + mail_search_build_add_seqset(search_args, seq, status.messages); + ctx = mailbox_search_init(trans, search_args, NULL, + metadata.precache_fields, NULL); + mail_search_args_unref(&search_args); + + max = status.messages - seq + 1; + while (mailbox_search_next(ctx, &mail)) { + if (mail_precache(mail) < 0) { + i_error("Mailbox %s: Precache for UID=%u failed: %s", + mailbox_get_vname(box), mail->uid, + mailbox_get_last_internal_error(box, NULL)); + ret = -1; + break; + } + if (doveadm_verbose && ++counter % 100 == 0) { + printf("\r%u/%u", counter, max); + fflush(stdout); + } + } + if (doveadm_verbose) + printf("\r%u/%u\n", counter, max); + if (mailbox_search_deinit(&ctx) < 0) { + i_error("Mailbox %s: Mail search failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + ret = -1; + } + if (mailbox_transaction_commit(&trans) < 0) { + i_error("Mailbox %s: Transaction commit failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + ret = -1; + } + return ret; +} + +static int +cmd_index_box(struct index_cmd_context *ctx, const struct mailbox_info *info) +{ + struct mailbox *box; + struct mailbox_status status; + int ret = 0; + + box = mailbox_alloc(info->ns->list, info->vname, + MAILBOX_FLAG_IGNORE_ACLS); + if (ctx->max_recent_msgs != 0) { + /* index only if there aren't too many recent messages. + don't bother syncing the mailbox, that alone can take a + while with large maildirs. */ + if (mailbox_open(box) < 0) { + i_error("Opening mailbox %s failed: %s", info->vname, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + mailbox_free(&box); + return -1; + } + + mailbox_get_open_status(box, STATUS_RECENT, &status); + if (status.recent > ctx->max_recent_msgs) { + mailbox_free(&box); + return 0; + } + } + + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Syncing mailbox %s failed: %s", info->vname, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + ret = -1; + } else { + if (cmd_index_box_precache(&ctx->ctx, box) < 0) { + doveadm_mail_failed_mailbox(&ctx->ctx, box); + ret = -1; + } + } + mailbox_free(&box); + return ret; +} + +static void index_queue_connect(struct index_cmd_context *ctx) +{ + const char *path; + + path = t_strconcat(doveadm_settings->base_dir, + "/"INDEXER_SOCKET_NAME, NULL); + ctx->queue_fd = net_connect_unix(path); + if (ctx->queue_fd == -1) + i_fatal("net_connect_unix(%s) failed: %m", path); + if (write_full(ctx->queue_fd, INDEXER_HANDSHAKE, + strlen(INDEXER_HANDSHAKE)) < 0) + i_fatal("write(indexer) failed: %m"); +} + +static void cmd_index_queue(struct index_cmd_context *ctx, + struct mail_user *user, const char *mailbox) +{ + if (ctx->queue_fd == -1) + index_queue_connect(ctx); + i_assert(ctx->queue_fd != -1); + + T_BEGIN { + string_t *str = t_str_new(256); + + str_append(str, "APPEND\t0\t"); + str_append_tabescaped(str, user->username); + str_append_c(str, '\t'); + str_append_tabescaped(str, mailbox); + str_printfa(str, "\t%u\n", ctx->max_recent_msgs); + if (write_full(ctx->queue_fd, str_data(str), str_len(str)) < 0) + i_fatal("write(indexer) failed: %m"); + } T_END; +} + +static int +cmd_index_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx; + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS | + MAILBOX_LIST_ITER_STAR_WITHIN_NS; + const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL; + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + unsigned int i; + int ret = 0; + + if (ctx->queue && !ctx->have_wildcards) { + /* we can do this quickly without going through the mailboxes */ + for (i = 0; _ctx->args[i] != NULL; i++) + cmd_index_queue(ctx, user, _ctx->args[i]); + return 0; + } + + iter = mailbox_list_iter_init_namespaces(user->namespaces, _ctx->args, + ns_mask, iter_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & (MAILBOX_NOSELECT | + MAILBOX_NONEXISTENT)) == 0) T_BEGIN { + if (ctx->queue) + cmd_index_queue(ctx, user, info->vname); + else { + if (cmd_index_box(ctx, info) < 0) + ret = -1; + } + } T_END; + } + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Listing mailboxes failed: %s", + mailbox_list_get_last_internal_error(user->namespaces->list, NULL)); + doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP); + ret = -1; + } + return ret; +} + +static void cmd_index_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx; + unsigned int i; + + if (args[0] == NULL) + doveadm_mail_help_name("index"); + for (i = 0; args[i] != NULL; i++) { + if (strchr(args[i], '*') != NULL || + strchr(args[i], '%') != NULL) { + ctx->have_wildcards = TRUE; + break; + } + } +} + +static void cmd_index_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx; + + if (ctx->queue_fd != -1) { + net_disconnect(ctx->queue_fd); + ctx->queue_fd = -1; + } +} + +static bool +cmd_index_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx; + + switch (c) { + case 'q': + ctx->queue = TRUE; + break; + case 'n': + if (str_to_uint(optarg, &ctx->max_recent_msgs) < 0) { + i_fatal_status(EX_USAGE, + "Invalid -n parameter number: %s", optarg); + } + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_index_alloc(void) +{ + struct index_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct index_cmd_context); + ctx->queue_fd = -1; + ctx->ctx.getopt_args = "qn:"; + ctx->ctx.v.parse_arg = cmd_index_parse_arg; + ctx->ctx.v.init = cmd_index_init; + ctx->ctx.v.deinit = cmd_index_deinit; + ctx->ctx.v.run = cmd_index_run; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_index_ver2 = { + .name = "index", + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-q] [-n <max recent>] <mailbox mask>", + .mail_cmd = cmd_index_alloc, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('q',"queue",CMD_PARAM_BOOL,0) +DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_STR,0) +DOVEADM_CMD_PARAM('\0',"mailbox-mask",CMD_PARAM_STR,CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-iter.c b/src/doveadm/doveadm-mail-iter.c new file mode 100644 index 0000000..eca5421 --- /dev/null +++ b/src/doveadm/doveadm-mail-iter.c @@ -0,0 +1,176 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ostream.h" +#include "mail-storage.h" +#include "mail-namespace.h" +#include "mail-search.h" +#include "doveadm-print.h" +#include "doveadm-mail.h" +#include "doveadm-mail-iter.h" + +struct doveadm_mail_iter { + struct doveadm_mail_cmd_context *ctx; + struct mail_search_args *search_args; + enum doveadm_mail_iter_flags flags; + + struct mailbox *box; + struct mailbox_transaction_context *t; + struct mail_search_context *search_ctx; + bool killed; +}; + +int doveadm_mail_iter_init(struct doveadm_mail_cmd_context *ctx, + const struct mailbox_info *info, + struct mail_search_args *search_args, + enum mail_fetch_field wanted_fields, + const char *const *wanted_headers, + enum doveadm_mail_iter_flags flags, + struct doveadm_mail_iter **iter_r) +{ + struct doveadm_mail_iter *iter; + struct mailbox_header_lookup_ctx *headers_ctx; + const char *errstr; + enum mail_error error; + + enum mailbox_flags readonly_flag = + (flags & DOVEADM_MAIL_ITER_FLAG_READONLY) != 0 ? + MAILBOX_FLAG_READONLY : 0; + + iter = i_new(struct doveadm_mail_iter, 1); + iter->ctx = ctx; + iter->flags = flags; + iter->box = mailbox_alloc(info->ns->list, info->vname, + MAILBOX_FLAG_IGNORE_ACLS | readonly_flag); + iter->search_args = search_args; + + if (mailbox_sync(iter->box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + errstr = mailbox_get_last_internal_error(iter->box, &error); + if (error == MAIL_ERROR_NOTFOUND) { + /* just ignore this mailbox */ + *iter_r = iter; + return 0; + } + i_error("Syncing mailbox %s failed: %s", info->vname, errstr); + doveadm_mail_failed_mailbox(ctx, iter->box); + mailbox_free(&iter->box); + i_free(iter); + return -1; + } + + headers_ctx = wanted_headers == NULL || wanted_headers[0] == NULL ? + NULL : mailbox_header_lookup_init(iter->box, wanted_headers); + + mail_search_args_init(search_args, iter->box, FALSE, NULL); + iter->t = mailbox_transaction_begin(iter->box, ctx->transaction_flags, + ctx->cmd->name); + iter->search_ctx = mailbox_search_init(iter->t, search_args, NULL, + wanted_fields, headers_ctx); + mailbox_header_lookup_unref(&headers_ctx); + *iter_r = iter; + return 0; +} + +static int +doveadm_mail_iter_deinit_transaction(struct doveadm_mail_iter *iter, + bool commit) +{ + int ret = 0; + + if (iter->search_ctx != NULL) { + if (mailbox_search_deinit(&iter->search_ctx) < 0) { + i_error("Searching mailbox %s failed: %s", + mailbox_get_vname(iter->box), + mailbox_get_last_internal_error(iter->box, NULL)); + ret = -1; + } + } + if (iter->t == NULL) + ; + else if (commit) { + if (mailbox_transaction_commit(&iter->t) < 0) { + i_error("Committing mailbox %s failed: %s", + mailbox_get_vname(iter->box), + mailbox_get_last_internal_error(iter->box, NULL)); + ret = -1; + } + } else { + mailbox_transaction_rollback(&iter->t); + } + mail_search_args_deinit(iter->search_args); + return ret; +} + +static int +doveadm_mail_iter_deinit_full(struct doveadm_mail_iter **_iter, + bool sync, bool commit, bool keep_box) +{ + struct doveadm_mail_iter *iter = *_iter; + int ret; + + *_iter = NULL; + + ret = doveadm_mail_iter_deinit_transaction(iter, commit); + if (ret == 0 && sync) { + ret = mailbox_sync(iter->box, 0); + if (ret < 0) { + i_error("Mailbox %s: Mailbox sync failed: %s", + mailbox_get_vname(iter->box), + mailbox_get_last_internal_error(iter->box, NULL)); + } + } + if (ret < 0) + doveadm_mail_failed_mailbox(iter->ctx, iter->box); + else if (iter->killed) { + iter->ctx->exit_code = EX_TEMPFAIL; + ret = -1; + } + if (!keep_box) + mailbox_free(&iter->box); + i_free(iter); + return ret; +} + +int doveadm_mail_iter_deinit(struct doveadm_mail_iter **_iter) +{ + return doveadm_mail_iter_deinit_full(_iter, FALSE, TRUE, FALSE); +} + +int doveadm_mail_iter_deinit_sync(struct doveadm_mail_iter **_iter) +{ + return doveadm_mail_iter_deinit_full(_iter, TRUE, TRUE, FALSE); +} + +int doveadm_mail_iter_deinit_keep_box(struct doveadm_mail_iter **iter, + struct mailbox **box_r) +{ + *box_r = (*iter)->box; + return doveadm_mail_iter_deinit_full(iter, FALSE, TRUE, TRUE); +} + +void doveadm_mail_iter_deinit_rollback(struct doveadm_mail_iter **_iter) +{ + (void)doveadm_mail_iter_deinit_full(_iter, FALSE, FALSE, FALSE); +} + +bool doveadm_mail_iter_next(struct doveadm_mail_iter *iter, + struct mail **mail_r) +{ + if (iter->search_ctx == NULL) + return FALSE; + if (doveadm_is_killed()) { + iter->killed = TRUE; + return FALSE; + } + if ((iter->flags & DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT) != 0 && + doveadm_print_ostream->stream_errno != 0) { + iter->killed = TRUE; + return FALSE; + } + return mailbox_search_next(iter->search_ctx, mail_r); +} + +struct mailbox *doveadm_mail_iter_get_mailbox(struct doveadm_mail_iter *iter) +{ + return iter->box; +} diff --git a/src/doveadm/doveadm-mail-iter.h b/src/doveadm/doveadm-mail-iter.h new file mode 100644 index 0000000..32cc232 --- /dev/null +++ b/src/doveadm/doveadm-mail-iter.h @@ -0,0 +1,34 @@ +#ifndef DOVEADM_MAIL_ITER_H +#define DOVEADM_MAIL_ITER_H + +#include "mailbox-list-iter.h" + +enum doveadm_mail_iter_flags { + /* Open the mailbox with MAILBOX_FLAG_READONLY */ + DOVEADM_MAIL_ITER_FLAG_READONLY = BIT(0), + /* Stop the iteration if client is detected to be disconnected. */ + DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT = BIT(1), +}; + +struct doveadm_mail_iter; +struct doveadm_mail_cmd_context; + +int doveadm_mail_iter_init(struct doveadm_mail_cmd_context *ctx, + const struct mailbox_info *info, + struct mail_search_args *search_args, + enum mail_fetch_field wanted_fields, + const char *const *wanted_headers, + enum doveadm_mail_iter_flags flags, + struct doveadm_mail_iter **iter_r) ATTR_NULL(6); +int doveadm_mail_iter_deinit(struct doveadm_mail_iter **iter); +int doveadm_mail_iter_deinit_sync(struct doveadm_mail_iter **iter); +int doveadm_mail_iter_deinit_keep_box(struct doveadm_mail_iter **iter, + struct mailbox **box_r); +void doveadm_mail_iter_deinit_rollback(struct doveadm_mail_iter **iter); +struct mailbox *doveadm_mail_iter_get_mailbox(struct doveadm_mail_iter *iter); + +bool doveadm_mail_iter_next(struct doveadm_mail_iter *iter, + struct mail **mail_r); + +#endif + diff --git a/src/doveadm/doveadm-mail-mailbox-cache.c b/src/doveadm/doveadm-mail-mailbox-cache.c new file mode 100644 index 0000000..718fca4 --- /dev/null +++ b/src/doveadm/doveadm-mail-mailbox-cache.c @@ -0,0 +1,440 @@ +/* Copyright (c) 2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "time-util.h" +#include "mail-index-private.h" +#include "mail-cache-private.h" +#include "mail-namespace.h" +#include "mail-storage-private.h" +#include "doveadm-print.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail.h" + +struct mailbox_cache_cmd_context { + struct doveadm_mail_cmd_context ctx; + + const char *const *boxes; + const char *const *fields; + uint64_t last_used; + enum mail_cache_decision_type decision; + bool all_fields; + bool set_decision; + bool set_last_used; + bool remove; +}; + +static int cmd_mailbox_cache_open_box(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user, + const char *boxname, + struct mailbox **box_r) +{ + struct mailbox *box = doveadm_mailbox_find(user, boxname); + + if (mailbox_open(box) < 0 || mailbox_sync(box, 0) < 0) { + i_error("Cannot open mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(ctx, box); + mailbox_free(&box); + return -1; + } + + *box_r = box; + + return 0; +} + +static void cmd_mailbox_cache_decision_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + const char *fields; + + doveadm_print_header("mailbox", "mailbox", DOVEADM_PRINT_HEADER_FLAG_STICKY); + doveadm_print_header_simple("field"); + doveadm_print_header_simple("decision"); + doveadm_print_header_simple("last-used"); + + if (!ctx->all_fields && + !doveadm_cmd_param_str(_ctx->cctx, "fieldstr", &fields)) { + i_fatal("Missing fields parameter"); + } else if (!ctx->all_fields) { + ctx->fields = t_strsplit_spaces(fields, ", "); + } + + ctx->boxes = args; +} + +static bool +cmd_mailbox_cache_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + + switch(c) { + case 'a': + ctx->all_fields = TRUE; + return TRUE; + /* this is handled in doveadm-mail as 'fieldstr' field */ + case 'f': + return TRUE; + case 'l': + if (str_to_uint64(optarg, &ctx->last_used) < 0) { + i_error("Invalid last-used '%s': not a number", optarg); + return FALSE; + } + ctx->set_last_used = TRUE; + return TRUE; + case 'd': + if (ctx->set_decision) { + i_error("Only one decision flag allowed"); + return FALSE; + } + if (strcmp(optarg, "no") == 0) { + ctx->decision = MAIL_CACHE_DECISION_NO; + } else if (strcmp(optarg, "temp") == 0) { + ctx->decision = MAIL_CACHE_DECISION_TEMP; + } else if (strcmp(optarg, "yes") == 0) { + ctx->decision = MAIL_CACHE_DECISION_YES; + } else { + i_error("Invalid decision '%s': " \ + "must be one of yes, temp, no", + optarg); + return FALSE; + } + ctx->set_decision = TRUE; + return TRUE; + } + return FALSE; +} + +static const char * +cmd_mailbox_cache_decision_to_str(enum mail_cache_decision_type decision) +{ + string_t *ret = t_str_new(10); + switch((decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED))) { + case MAIL_CACHE_DECISION_NO: + str_append(ret, "no"); + break; + case MAIL_CACHE_DECISION_TEMP: + str_append(ret, "temp"); + break; + case MAIL_CACHE_DECISION_YES: + str_append(ret, "yes"); + break; + } + return str_c(ret); +} + +static void +cmd_mailbox_cache_decision_process_field(struct mailbox_cache_cmd_context *ctx, + struct mail_cache_field_private *field) +{ + if (ctx->set_decision) { + field->field.decision = ctx->decision; + field->decision_dirty = TRUE; + } + + if (ctx->set_last_used) { + field->field.last_used = (time_t)ctx->last_used; + field->decision_dirty = TRUE; + } + + doveadm_print(cmd_mailbox_cache_decision_to_str(field->field.decision)); + doveadm_print(t_strflocaltime("%F %T %Z", field->field.last_used)); +} + +static void +cmd_mailbox_cache_decision_run_per_field(struct mailbox_cache_cmd_context *ctx, + struct mail_cache *cache) +{ + const char *const *field_name; + for(field_name = ctx->fields; *field_name != NULL; field_name++) { + doveadm_print(*field_name); + /* see if the field exists */ + unsigned int idx = mail_cache_register_lookup(cache, + *field_name); + if (idx == UINT_MAX) { + doveadm_print("<not found>"); + doveadm_print(""); + continue; + } + + cmd_mailbox_cache_decision_process_field(ctx, &cache->fields[idx]); + } +} + +static void +cmd_mailbox_cache_decision_run_all_fields(struct mailbox_cache_cmd_context *ctx, + struct mail_cache *cache) +{ + /* get all fields */ + for(unsigned int i = 0; i < cache->fields_count; i++) { + doveadm_print(cache->fields[i].field.name); + cmd_mailbox_cache_decision_process_field(ctx, &cache->fields[i]); + } +} + +static int cmd_mailbox_cache_decision_run_box(struct mailbox_cache_cmd_context *ctx, + struct mailbox *box) +{ + struct mailbox_transaction_context *t = + mailbox_transaction_begin(box, ctx->ctx.transaction_flags, + "mailbox cache decision"); + struct mail_cache *cache = t->box->cache; + struct mail_cache_view *view; + + if (mail_cache_open_and_verify(cache) < 0 || + MAIL_CACHE_IS_UNUSABLE(cache)) { + mailbox_transaction_rollback(&t); + i_error("Cache is unusable"); + ctx->ctx.exit_code = EX_TEMPFAIL; + return -1; + } + + view = mail_cache_view_open(cache, t->box->view); + + if (ctx->all_fields) + cmd_mailbox_cache_decision_run_all_fields(ctx, cache); + else + cmd_mailbox_cache_decision_run_per_field(ctx, cache); + + /* update headers */ + if (ctx->set_decision || ctx->set_last_used) + mail_cache_header_fields_update(cache); + + mail_cache_view_close(&view); + + if (mailbox_transaction_commit(&t) < 0) { + i_error("mailbox_transaction_commit() failed: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + return -1; + } + return 0; +} + +static int cmd_mailbox_cache_decision_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + const char *const *boxname; + int ret = 0; + + if (_ctx->exit_code != 0) + return -1; + + for(boxname = ctx->boxes; ret == 0 && *boxname != NULL; boxname++) { + struct mailbox *box; + if ((ret = cmd_mailbox_cache_open_box(_ctx, user, *boxname, &box)) < 0) + break; + doveadm_print_sticky("mailbox", mailbox_get_vname(box)); + ret = cmd_mailbox_cache_decision_run_box(ctx, box); + mailbox_free(&box); + } + + return ret; +} + +static int cmd_mailbox_cache_remove_box(struct mailbox_cache_cmd_context *ctx, + const struct mailbox_info *info) +{ + struct doveadm_mail_iter *iter; + struct mailbox *box; + struct mail *mail; + void *empty = NULL; + int ret = 0, count = 0; + + if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args, + 0, NULL, 0, &iter) < 0) + return -1; + + box = doveadm_mail_iter_get_mailbox(iter); + + struct mail_index_transaction *t = + mail_index_transaction_begin(box->view, MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL); + struct mail_cache_view *view = + mail_cache_view_open(box->cache, box->view); + + while (doveadm_mail_iter_next(iter, &mail)) { + count++; + doveadm_print(mailbox_get_vname(box)); + doveadm_print(dec2str(mail->uid)); + /* drop cache pointer */ + mail_index_update_ext(t, mail->seq, view->cache->ext_id, &empty, NULL); + doveadm_print("ok"); + } + + if (mail_index_transaction_commit(&t) < 0) { + i_error("mail_index_transaction_commit() failed: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + ret = -1; + } else { + mail_cache_expunge_count(view->cache, count); + } + + mail_cache_view_close(&view); + + if (doveadm_mail_iter_deinit(&iter) < 0) + ret = -1; + + return ret; +} + +static int cmd_mailbox_cache_remove_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(&ctx->ctx, user, ctx->ctx.search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_mailbox_cache_remove_box(ctx, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_mailbox_cache_remove_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + + if (args[0] == NULL) + doveadm_mail_help_name("mailbox cache remove"); + + doveadm_print_header_simple("mailbox"); + doveadm_print_header_simple("uid"); + doveadm_print_header_simple("result"); + + ctx->ctx.search_args = doveadm_mail_build_search_args(args); +} + +static int cmd_mailbox_cache_purge_run_box(struct mailbox_cache_cmd_context *ctx, + struct mailbox *box) +{ + if (mail_cache_purge(box->cache, (uint32_t)-1, + "doveadm mailbox cache purge") < 0) { + mailbox_set_index_error(box); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + return -1; + } + return 0; +} + +static int cmd_mailbox_cache_purge_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + const char *const *boxname; + int ret = 0; + + if (_ctx->exit_code != 0) + return -1; + + for(boxname = ctx->boxes; ret == 0 && *boxname != NULL; boxname++) { + struct mailbox *box; + if ((ret = cmd_mailbox_cache_open_box(_ctx, user, *boxname, &box)) < 0) + break; + ret = cmd_mailbox_cache_purge_run_box(ctx, box); + mailbox_free(&box); + } + + return ret; +} + +static void cmd_mailbox_cache_purge_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct mailbox_cache_cmd_context *ctx = + container_of(_ctx, struct mailbox_cache_cmd_context, ctx); + + ctx->boxes = args; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_cache_decision_alloc(void) +{ + struct mailbox_cache_cmd_context *ctx = + doveadm_mail_cmd_alloc(struct mailbox_cache_cmd_context); + ctx->ctx.v.init = cmd_mailbox_cache_decision_init; + ctx->ctx.v.parse_arg = cmd_mailbox_cache_parse_arg; + ctx->ctx.v.run = cmd_mailbox_cache_decision_run; + ctx->ctx.getopt_args = "al:f:d:"; + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + return &ctx->ctx; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_cache_remove_alloc(void) +{ + struct mailbox_cache_cmd_context *ctx = + doveadm_mail_cmd_alloc(struct mailbox_cache_cmd_context); + ctx->ctx.v.init = cmd_mailbox_cache_remove_init; + ctx->ctx.v.parse_arg = cmd_mailbox_cache_parse_arg; + ctx->ctx.v.run = cmd_mailbox_cache_remove_run; + ctx->ctx.getopt_args = ""; + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + return &ctx->ctx; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_cache_purge_alloc(void) +{ + struct mailbox_cache_cmd_context *ctx = + doveadm_mail_cmd_alloc(struct mailbox_cache_cmd_context); + ctx->ctx.v.init = cmd_mailbox_cache_purge_init; + ctx->ctx.v.run = cmd_mailbox_cache_purge_run; + ctx->ctx.getopt_args = ""; + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_decision = { + .name = "mailbox cache decision", + .mail_cmd = cmd_mailbox_cache_decision_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"--all --fields <fields> " \ + "--last-used <timestamp> --decision <decision> " \ + "<mailbox> [<mailbox> ... ]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('a', "all", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('f', "fieldstr", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('l', "last-used", CMD_PARAM_INT64, 0) +DOVEADM_CMD_PARAM('d', "decision", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_remove = { + .name = "mailbox cache remove", + .mail_cmd = cmd_mailbox_cache_remove_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<search string>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_purge = { + .name = "mailbox cache purge", + .mail_cmd = cmd_mailbox_cache_purge_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<mailbox> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-mailbox-metadata.c b/src/doveadm/doveadm-mail-mailbox-metadata.c new file mode 100644 index 0000000..593555b --- /dev/null +++ b/src/doveadm/doveadm-mail-mailbox-metadata.c @@ -0,0 +1,425 @@ +/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "istream.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "doveadm-print.h" +#include "doveadm-mail.h" +#include "doveadm-mailbox-list-iter.h" +#include "imap-metadata.h" + +enum doveadm_metadata_op { + DOVEADM_METADATA_OP_SET = 0, + DOVEADM_METADATA_OP_GET, + DOVEADM_METADATA_OP_LIST, +}; + +const char *doveadm_metadata_op_names[] = { + "set attribute", + "get attribute", + "list attribute", +}; + +struct metadata_cmd_context { + struct doveadm_mail_cmd_context ctx; + const char *mailbox; + enum mail_attribute_type key_type; + const char *key; + struct mail_attribute_value value; + bool empty_mailbox_name; + bool allow_empty_mailbox_name; + bool prepend_prefix; +}; + +static int +cmd_mailbox_metadata_get_mailbox(struct metadata_cmd_context *mctx, + struct mail_user *user, + enum doveadm_metadata_op op, + struct mail_namespace **ns_r, + struct mailbox **box_r) +{ + mctx->empty_mailbox_name = mctx->mailbox[0] == '\0'; + + if (mctx->empty_mailbox_name) { + if (!mctx->allow_empty_mailbox_name) { + const char *op_str = doveadm_metadata_op_names[op]; + i_error("Failed to %s: %s", op_str, + "mailbox name cannot be empty"); + mctx->ctx.exit_code = EX_USAGE; + return -1; + } + + /* Server attribute. It shouldn't depend on INBOX's ACLs, + so ignore them. */ + *ns_r = mail_namespace_find_inbox(user->namespaces); + *box_r = mailbox_alloc((*ns_r)->list, "INBOX", + MAILBOX_FLAG_IGNORE_ACLS | + MAILBOX_FLAG_ATTRIBUTE_SESSION); + + mctx->key = t_strconcat(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER, + mctx->key, NULL); + } else { + /* mailbox attributes */ + *ns_r = mail_namespace_find(user->namespaces, mctx->mailbox); + *box_r = mailbox_alloc((*ns_r)->list, mctx->mailbox, + MAILBOX_FLAG_ATTRIBUTE_SESSION); + } + + if (op == DOVEADM_METADATA_OP_SET && + mailbox_open(*box_r) < 0) { + i_error("Failed to open mailbox: %s", + mailbox_get_last_internal_error(*box_r, NULL)); + doveadm_mail_failed_mailbox(&mctx->ctx, *box_r); + mailbox_free(box_r); + return -1; + } + + return 0; +} + +static int +cmd_mailbox_metadata_set_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + struct mailbox_transaction_context *trans; + int ret; + + ret = cmd_mailbox_metadata_get_mailbox(ctx, user, DOVEADM_METADATA_OP_SET, + &ns, &box); + if (ret != 0) + return ret; + + trans = mailbox_transaction_begin(box, (ctx->empty_mailbox_name ? + MAILBOX_TRANSACTION_FLAG_EXTERNAL : 0) | + ctx->ctx.transaction_flags, __func__); + + ret = ctx->value.value == NULL ? + mailbox_attribute_unset(trans, ctx->key_type, ctx->key) : + mailbox_attribute_set(trans, ctx->key_type, ctx->key, &ctx->value); + if (ret < 0) { + i_error("Failed to set attribute: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + mailbox_transaction_rollback(&trans); + } else if (mailbox_transaction_commit(&trans) < 0) { + i_error("Failed to commit transaction: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + + mailbox_free(&box); + return ret; +} + +static void +cmd_mailbox_metadata_parse_key(const char *arg, + enum mail_attribute_type *type_r, + const char **key_r) +{ + arg = t_str_lcase(arg); + + if (str_begins(arg, "/private/")) { + *type_r = MAIL_ATTRIBUTE_TYPE_PRIVATE; + *key_r = arg + 9; + } else if (str_begins(arg, "/shared/")) { + *type_r = MAIL_ATTRIBUTE_TYPE_SHARED; + *key_r = arg + 8; + } else if (strcmp(arg, "/private") == 0) { + *type_r = MAIL_ATTRIBUTE_TYPE_PRIVATE; + *key_r = ""; + } else if (strcmp(arg, "/shared") == 0) { + *type_r = MAIL_ATTRIBUTE_TYPE_SHARED; + *key_r = ""; + } else { + i_fatal_status(EX_USAGE, "Invalid metadata key '%s': " + "Must begin with /private or /shared", arg); + } +} + +static void +cmd_mailbox_metadata_set_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + const char *key; + + if (str_array_length(args) != 3) + doveadm_mail_help_name("mailbox metadata set"); + cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key); + + ctx->mailbox = p_strdup(_ctx->pool, args[0]); + ctx->key = p_strdup(_ctx->pool, key); + ctx->value.value = p_strdup(_ctx->pool, args[2]); +} + +static bool +cmd_mailbox_metadata_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct metadata_cmd_context *ctx = + (struct metadata_cmd_context *)_ctx; + + switch (c) { + case 's': + ctx->allow_empty_mailbox_name = TRUE; + break; + case 'p': + ctx->prepend_prefix = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_set_alloc(void) +{ + struct metadata_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context); + ctx->ctx.v.init = cmd_mailbox_metadata_set_init; + ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg; + ctx->ctx.v.run = cmd_mailbox_metadata_set_run; + return &ctx->ctx; +} + +static void +cmd_mailbox_metadata_unset_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + const char *key; + + if (str_array_length(args) != 2) + doveadm_mail_help_name("mailbox metadata unset"); + cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key); + + ctx->mailbox = p_strdup(_ctx->pool, args[0]); + ctx->key = p_strdup(_ctx->pool, key); +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_unset_alloc(void) +{ + struct metadata_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context); + ctx->ctx.v.init = cmd_mailbox_metadata_unset_init; + ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg; + ctx->ctx.v.run = cmd_mailbox_metadata_set_run; + return &ctx->ctx; +} + +static int +cmd_mailbox_metadata_get_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + struct mail_attribute_value value; + int ret; + + ret = cmd_mailbox_metadata_get_mailbox(ctx, user, DOVEADM_METADATA_OP_GET, + &ns, &box); + if (ret != 0) + return ret; + + ret = mailbox_attribute_get_stream(box, ctx->key_type, ctx->key, &value); + if (ret < 0) { + i_error("Failed to get attribute: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + } else if (ret == 0) { + /* not found, print as empty */ + doveadm_print(""); + } else if (value.value_stream != NULL) { + if (doveadm_print_istream(value.value_stream) < 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(value.value_stream), + i_stream_get_error(value.value_stream)); + ret = -1; + } + } else { + doveadm_print(value.value); + } + + mailbox_free(&box); + return ret; +} + +static void +cmd_mailbox_metadata_get_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + const char *key; + + if (str_array_length(args) != 2) + doveadm_mail_help_name("mailbox metadata get"); + cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key); + + ctx->mailbox = p_strdup(_ctx->pool, args[0]); + ctx->key = p_strdup(_ctx->pool, key); + doveadm_print_header("value", "value", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_get_alloc(void) +{ + struct metadata_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context); + ctx->ctx.v.init = cmd_mailbox_metadata_get_init; + ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg; + ctx->ctx.v.run = cmd_mailbox_metadata_get_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return &ctx->ctx; +} + +static int +cmd_mailbox_metadata_list_run_iter(struct metadata_cmd_context *ctx, + struct mailbox *box, + enum mail_attribute_type type) +{ + struct mailbox_attribute_iter *iter; + const char *key; + string_t *outp = t_str_new(64); + + iter = mailbox_attribute_iter_init(box, type, ctx->key); + while ((key = mailbox_attribute_iter_next(iter)) != NULL) { + if (ctx->prepend_prefix) { + if (type == MAIL_ATTRIBUTE_TYPE_PRIVATE) + str_append(outp, IMAP_METADATA_PRIVATE_PREFIX"/"); + else if (type == MAIL_ATTRIBUTE_TYPE_SHARED) + str_append(outp, IMAP_METADATA_SHARED_PREFIX"/"); + else + i_unreached(); + str_append(outp, key); + doveadm_print(str_c(outp)); + str_truncate(outp, 0); + } else { + doveadm_print(key); + } + } + if (mailbox_attribute_iter_deinit(&iter) < 0) { + i_error("Mailbox %s: Failed to iterate mailbox attributes: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + return -1; + } + return 0; +} + +static int +cmd_mailbox_metadata_list_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + int ret = 0; + + ret = cmd_mailbox_metadata_get_mailbox(ctx, user, DOVEADM_METADATA_OP_LIST, + &ns, &box); + if (ret != 0) + return ret; + + if (ctx->key[0] == '\0' || ctx->key_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) { + if (cmd_mailbox_metadata_list_run_iter(ctx, box, MAIL_ATTRIBUTE_TYPE_PRIVATE) < 0) { + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } + if (ctx->key[0] == '\0' || ctx->key_type == MAIL_ATTRIBUTE_TYPE_SHARED) { + if (cmd_mailbox_metadata_list_run_iter(ctx, box, MAIL_ATTRIBUTE_TYPE_SHARED) < 0) { + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } + mailbox_free(&box); + return ret; +} + +static void +cmd_mailbox_metadata_list_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx; + const char *key = NULL; + + if (args[0] == NULL) + doveadm_mail_help_name("mailbox metadata list"); + if (args[1] != NULL) + cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key); + ctx->mailbox = p_strdup(_ctx->pool, args[0]); + ctx->key = key == NULL ? "" : p_strdup(_ctx->pool, key); + doveadm_print_header("key", "key", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_list_alloc(void) +{ + struct metadata_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context); + ctx->ctx.v.init = cmd_mailbox_metadata_list_init; + ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg; + ctx->ctx.v.run = cmd_mailbox_metadata_list_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_set_ver2 = { + .name = "mailbox metadata set", + .mail_cmd = cmd_mailbox_metadata_set_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> <key> <value>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "value", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_unset_ver2 = { + .name = "mailbox metadata unset", + .mail_cmd = cmd_mailbox_metadata_unset_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> <key>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_get_ver2 = { + .name = "mailbox metadata get", + .mail_cmd = cmd_mailbox_metadata_get_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> <key>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_list_ver2 = { + .name = "mailbox metadata list", + .mail_cmd = cmd_mailbox_metadata_list_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] [-p] <mailbox> [<key prefix>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('p', "prepend-prefix", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "key-prefix", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-mailbox-status.c b/src/doveadm/doveadm-mail-mailbox-status.c new file mode 100644 index 0000000..9f47095 --- /dev/null +++ b/src/doveadm/doveadm-mail-mailbox-status.c @@ -0,0 +1,275 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mail-search.h" +#include "doveadm-print.h" +#include "doveadm-mail.h" +#include "doveadm-mailbox-list-iter.h" + +#define ALL_STATUS_ITEMS \ + (STATUS_MESSAGES | STATUS_RECENT | \ + STATUS_UIDNEXT | STATUS_UIDVALIDITY | \ + STATUS_UNSEEN | STATUS_HIGHESTMODSEQ) +#define ALL_METADATA_ITEMS \ + (MAILBOX_METADATA_VIRTUAL_SIZE | MAILBOX_METADATA_GUID | \ + MAILBOX_METADATA_FIRST_SAVE_DATE) + +#define TOTAL_STATUS_ITEMS \ + (STATUS_MESSAGES | STATUS_RECENT | STATUS_UNSEEN) +#define TOTAL_METADATA_ITEMS \ + (MAILBOX_METADATA_VIRTUAL_SIZE) + +struct status_cmd_context { + struct doveadm_mail_cmd_context ctx; + struct mail_search_args *search_args; + + enum mailbox_status_items status_items; + enum mailbox_metadata_items metadata_items; + struct mailbox_status total_status; + struct mailbox_metadata total_metadata; + + bool total_sum:1; +}; + +static void status_parse_fields(struct status_cmd_context *ctx, + const char *const *fields) +{ + if (*fields == NULL) + i_fatal_status(EX_USAGE, "No status fields"); + + for (; *fields != NULL; fields++) { + const char *field = *fields; + + if (strcmp(field, "all") == 0) { + if (ctx->total_sum) { + ctx->status_items |= TOTAL_STATUS_ITEMS; + ctx->metadata_items |= TOTAL_METADATA_ITEMS; + } else { + ctx->status_items |= ALL_STATUS_ITEMS; + ctx->metadata_items |= ALL_METADATA_ITEMS; + } + } else if (strcmp(field, "messages") == 0) + ctx->status_items |= STATUS_MESSAGES; + else if (strcmp(field, "recent") == 0) + ctx->status_items |= STATUS_RECENT; + else if (strcmp(field, "uidnext") == 0) + ctx->status_items |= STATUS_UIDNEXT; + else if (strcmp(field, "uidvalidity") == 0) + ctx->status_items |= STATUS_UIDVALIDITY; + else if (strcmp(field, "unseen") == 0) + ctx->status_items |= STATUS_UNSEEN; + else if (strcmp(field, "highestmodseq") == 0) + ctx->status_items |= STATUS_HIGHESTMODSEQ; + else if (strcmp(field, "vsize") == 0) + ctx->metadata_items |= MAILBOX_METADATA_VIRTUAL_SIZE; + else if (strcmp(field, "guid") == 0) + ctx->metadata_items |= MAILBOX_METADATA_GUID; + else if (strcmp(field, "firstsaved") == 0) + ctx->metadata_items |= MAILBOX_METADATA_FIRST_SAVE_DATE; + else { + i_fatal_status(EX_USAGE, + "Unknown status field: %s", field); + } + + if (ctx->total_sum && + ((ctx->status_items & ENUM_NEGATE(TOTAL_STATUS_ITEMS)) != 0 || + (ctx->metadata_items & ENUM_NEGATE(TOTAL_METADATA_ITEMS)) != 0)) { + i_fatal_status(EX_USAGE, + "Status field %s can't be used with -t", field); + } + } +} + +static void ATTR_NULL(2) +status_output(struct status_cmd_context *ctx, struct mailbox *box, + const struct mailbox_status *status, + const struct mailbox_metadata *metadata) +{ + if (box != NULL) + doveadm_print(mailbox_get_vname(box)); + + if ((ctx->status_items & STATUS_MESSAGES) != 0) + doveadm_print_num(status->messages); + if ((ctx->status_items & STATUS_RECENT) != 0) + doveadm_print_num(status->recent); + if ((ctx->status_items & STATUS_UIDNEXT) != 0) + doveadm_print_num(status->uidnext); + if ((ctx->status_items & STATUS_UIDVALIDITY) != 0) + doveadm_print_num(status->uidvalidity); + if ((ctx->status_items & STATUS_UNSEEN) != 0) + doveadm_print_num(status->unseen); + if ((ctx->status_items & STATUS_HIGHESTMODSEQ) != 0) + doveadm_print_num(status->highest_modseq); + if ((ctx->metadata_items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) + doveadm_print_num(metadata->virtual_size); + if ((ctx->metadata_items & MAILBOX_METADATA_GUID) != 0) + doveadm_print(guid_128_to_string(metadata->guid)); + if ((ctx->metadata_items & MAILBOX_METADATA_FIRST_SAVE_DATE) > 0) { + if (metadata->first_save_date > -1) + doveadm_print_num(metadata->first_save_date); + else + doveadm_print("never"); + } +} + +static void +status_sum(struct status_cmd_context *ctx, + const struct mailbox_status *status, + const struct mailbox_metadata *metadata) +{ + struct mailbox_status *dest = &ctx->total_status; + + dest->messages += status->messages; + dest->recent += status->recent; + dest->unseen += status->unseen; + ctx->total_metadata.virtual_size += metadata->virtual_size; +} + +static int +status_mailbox(struct status_cmd_context *ctx, const struct mailbox_info *info) +{ + struct mailbox *box; + struct mailbox_status status; + struct mailbox_metadata metadata; + + box = doveadm_mailbox_find(ctx->ctx.cur_mail_user, info->vname); + if (mailbox_get_status(box, ctx->status_items, &status) < 0 || + mailbox_get_metadata(box, ctx->metadata_items, &metadata) < 0) { + i_error("Mailbox %s: Failed to lookup mailbox status: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(&ctx->ctx, box); + mailbox_free(&box); + return -1; + } + if (!ctx->total_sum) + status_output(ctx, box, &status, &metadata); + else + status_sum(ctx, &status, &metadata); + mailbox_free(&box); + return 0; +} + +static int +cmd_mailbox_status_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx; + enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + i_zero(&ctx->total_status); + i_zero(&ctx->total_metadata); + + iter = doveadm_mailbox_list_iter_init(_ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) { + T_BEGIN { + if (status_mailbox(ctx, info) < 0) + ret = -1; + } T_END; + } + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + + if (ctx->total_sum) { + status_output(ctx, NULL, &ctx->total_status, + &ctx->total_metadata); + } + return ret; +} + +static void cmd_mailbox_status_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx; + const char *fields = args[0]; + + if (fields == NULL || args[1] == NULL) + doveadm_mail_help_name("mailbox status"); + + status_parse_fields(ctx, t_strsplit_spaces(fields, " ")); + ctx->search_args = doveadm_mail_mailbox_search_args_build(args+1); + + if (!ctx->total_sum) { + doveadm_print_header("mailbox", "mailbox", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + } + if ((ctx->status_items & STATUS_MESSAGES) != 0) + doveadm_print_header_simple("messages"); + if ((ctx->status_items & STATUS_RECENT) != 0) + doveadm_print_header_simple("recent"); + if ((ctx->status_items & STATUS_UIDNEXT) != 0) + doveadm_print_header_simple("uidnext"); + if ((ctx->status_items & STATUS_UIDVALIDITY) != 0) + doveadm_print_header_simple("uidvalidity"); + if ((ctx->status_items & STATUS_UNSEEN) != 0) + doveadm_print_header_simple("unseen"); + if ((ctx->status_items & STATUS_HIGHESTMODSEQ) != 0) + doveadm_print_header_simple("highestmodseq"); + if ((ctx->metadata_items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0) + doveadm_print_header_simple("vsize"); + if ((ctx->metadata_items & MAILBOX_METADATA_GUID) != 0) + doveadm_print_header_simple("guid"); + if ((ctx->metadata_items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0) + doveadm_print_header_simple("firstsaved"); +} + +static void cmd_mailbox_status_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx; + + if (ctx->search_args != NULL) + mail_search_args_unref(&ctx->search_args); +} + +static bool +cmd_mailbox_status_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx; + + switch (c) { + case 't': + ctx->total_sum = TRUE; + break; + case 'f': + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_status_alloc(void) +{ + struct status_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct status_cmd_context); + ctx->ctx.getopt_args = "t"; + ctx->ctx.v.parse_arg = cmd_mailbox_status_parse_arg; + ctx->ctx.v.init = cmd_mailbox_status_init; + ctx->ctx.v.deinit = cmd_mailbox_status_deinit; + ctx->ctx.v.run = cmd_mailbox_status_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_status_ver2 = { + .name = "mailbox status", + .mail_cmd = cmd_mailbox_status_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<fields> <mailbox> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('t', "total-sum", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_ARRAY, 0) +DOVEADM_CMD_PARAM('\0', "fieldstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL | CMD_PARAM_FLAG_DO_NOT_EXPOSE) /* FIXME: horrible hack, remove me when possible */ +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c new file mode 100644 index 0000000..2b2c453 --- /dev/null +++ b/src/doveadm/doveadm-mail-mailbox.c @@ -0,0 +1,857 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "unichar.h" +#include "imap-utf7.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mail-search-build.h" +#include "doveadm-print.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail.h" + +#include <stdio.h> + +struct doveadm_mailbox_cmd_context { + struct doveadm_mail_cmd_context ctx; + bool subscriptions; +}; + +struct mailbox_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + ARRAY_TYPE(const_string) mailboxes; +}; + +struct create_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + ARRAY_TYPE(const_string) mailboxes; + struct mailbox_update update; +}; + +struct delete_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + ARRAY_TYPE(const_string) mailboxes; + bool recursive; + bool require_empty; + bool unsafe; +}; + +struct rename_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + const char *oldname, *newname; +}; + +struct list_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + struct mail_search_args *search_args; + bool mutf7; +}; + +struct update_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + const char *mailbox; + struct mailbox_update update; +}; + +struct path_cmd_context { + struct doveadm_mailbox_cmd_context ctx; + const char *mailbox; + enum mailbox_list_path_type path_type; +}; + +static const char *mailbox_list_path_type_names[] = { + "dir", "alt-dir", "mailbox", "alt-mailbox", + "control", "index", "index-private" +}; + +void doveadm_mailbox_args_check(const char *const args[]) +{ + unsigned int i; + + for (i = 0; args[i] != NULL; i++) { + if (!uni_utf8_str_is_valid(args[i])) { + i_fatal_status(EX_DATAERR, + "Mailbox name not valid UTF-8: %s", args[i]); + } + } +} + +static bool cmd_mailbox_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct doveadm_mailbox_cmd_context *ctx = + (struct doveadm_mailbox_cmd_context *)_ctx; + + switch (c) { + case 's': + ctx->subscriptions = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +#define doveadm_mailbox_cmd_alloc(type) \ + (type *)doveadm_mailbox_cmd_alloc_size(sizeof(type)) +static struct doveadm_mail_cmd_context * +doveadm_mailbox_cmd_alloc_size(size_t size) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc_size(size); + ctx->getopt_args = "s"; + ctx->v.parse_arg = cmd_mailbox_parse_arg; + return ctx; +} + +static bool +cmd_mailbox_list_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx; + + switch (c) { + case '7': + ctx->mutf7 = TRUE; + break; + case '8': + ctx->mutf7 = FALSE; + break; + case 's': + ctx->ctx.subscriptions = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static int +cmd_mailbox_list_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx; + enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + string_t *str = t_str_new(256); + + if (ctx->ctx.subscriptions) + iter_flags |= MAILBOX_LIST_ITER_SELECT_SUBSCRIBED; + + iter = doveadm_mailbox_list_iter_full_init(_ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) { + if (!ctx->mutf7) + doveadm_print(info->vname); + else { + str_truncate(str, 0); + if (imap_utf8_to_utf7(info->vname, str) < 0) + i_unreached(); + doveadm_print(str_c(str)); + } + } + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + return -1; + return 0; +} + +struct mail_search_args * +doveadm_mail_mailbox_search_args_build(const char *const args[]) +{ + struct mail_search_args *search_args; + struct mail_search_arg *arg; + enum mail_search_arg_type type; + unsigned int i; + + doveadm_mailbox_args_check(args); + search_args = mail_search_build_init(); + for (i = 0; args[i] != NULL; i++) { + if (strchr(args[i], '*') != NULL || + strchr(args[i], '%') != NULL) + type = SEARCH_MAILBOX_GLOB; + else + type = SEARCH_MAILBOX; + arg = mail_search_build_add(search_args, type); + arg->value.str = p_strdup(search_args->pool, args[i]); + } + if (i > 1) { + struct mail_search_arg *subargs = search_args->args; + + search_args->args = NULL; + arg = mail_search_build_add(search_args, SEARCH_OR); + arg->value.subargs = subargs; + } + return search_args; +} + +static void cmd_mailbox_list_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx; + + doveadm_print_header("mailbox", "mailbox", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + ctx->search_args = doveadm_mail_mailbox_search_args_build(args); +} + +static void cmd_mailbox_list_deinit(struct doveadm_mail_cmd_context *_ctx) +{ + struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx; + + if (ctx->search_args != NULL) + mail_search_args_unref(&ctx->search_args); +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_list_alloc(void) +{ + struct list_cmd_context *ctx; + + ctx = doveadm_mailbox_cmd_alloc(struct list_cmd_context); + ctx->ctx.ctx.v.init = cmd_mailbox_list_init; + ctx->ctx.ctx.v.deinit = cmd_mailbox_list_deinit; + ctx->ctx.ctx.v.run = cmd_mailbox_list_run; + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_list_parse_arg; + ctx->ctx.ctx.getopt_args = "78s"; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return &ctx->ctx.ctx; +} + +static int +cmd_mailbox_create_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct create_cmd_context *ctx = (struct create_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + const char *name; + int ret = 0; + + array_foreach_elem(&ctx->mailboxes, name) { + size_t len; + bool directory = FALSE; + + ns = mail_namespace_find(user->namespaces, name); + len = strlen(name); + if (len > 0 && name[len-1] == mail_namespace_get_sep(ns)) { + name = t_strndup(name, len-1); + directory = TRUE; + } + + box = mailbox_alloc(ns->list, name, 0); + if (mailbox_create(box, &ctx->update, directory) < 0) { + i_error("Can't create mailbox %s: %s", name, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + if (ctx->ctx.subscriptions) { + if (mailbox_set_subscribed(box, TRUE) < 0) { + i_error("Can't subscribe to mailbox %s: %s", name, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } + mailbox_free(&box); + } + return ret; +} + +static void cmd_mailbox_create_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx; + const char *name; + unsigned int i; + + if (args[0] == NULL) + doveadm_mail_help_name("mailbox create"); + doveadm_mailbox_args_check(args); + + for (i = 0; args[i] != NULL; i++) { + name = p_strdup(ctx->ctx.ctx.pool, args[i]); + array_push_back(&ctx->mailboxes, &name); + } +} + +static bool +cmd_mailbox_create_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct create_cmd_context *ctx = (struct create_cmd_context *)_ctx; + + switch (c) { + case 'g': + if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0) + doveadm_mail_help_name("mailbox create"); + break; + case 's': + ctx->ctx.subscriptions = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_create_alloc(void) +{ + struct create_cmd_context *ctx; + + ctx = doveadm_mailbox_cmd_alloc(struct create_cmd_context); + ctx->ctx.ctx.v.init = cmd_mailbox_create_init; + ctx->ctx.ctx.v.run = cmd_mailbox_create_run; + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_create_parse_arg; + ctx->ctx.ctx.getopt_args = "g:s"; + p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); + return &ctx->ctx.ctx; +} + +static int i_strcmp_reverse_p(const char *const *s1, const char *const *s2) +{ + return -strcmp(*s1, *s2); +} + +static int +get_child_mailboxes(struct mail_user *user, ARRAY_TYPE(const_string) *mailboxes, + const char *name) +{ + struct mailbox_list_iterate_context *iter; + struct mail_namespace *ns; + const struct mailbox_info *info; + const char *pattern, *child_name; + + ns = mail_namespace_find(user->namespaces, name); + pattern = name[0] == '\0' ? "*" : + t_strdup_printf("%s%c*", name, mail_namespace_get_sep(ns)); + iter = mailbox_list_iter_init(ns->list, pattern, + MAILBOX_LIST_ITER_RETURN_NO_FLAGS); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + child_name = t_strdup(info->vname); + array_push_back(mailboxes, &child_name); + } + return mailbox_list_iter_deinit(&iter); +} + +static int +cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct delete_cmd_context *ctx = (struct delete_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + struct mail_storage *storage; + const char *name; + ARRAY_TYPE(const_string) recursive_mailboxes; + const ARRAY_TYPE(const_string) *mailboxes = &ctx->mailboxes; + enum mailbox_flags mailbox_flags = 0; + int ret = 0, ret2; + + if (ctx->unsafe) + mailbox_flags |= MAILBOX_FLAG_DELETE_UNSAFE; + if (ctx->recursive) { + t_array_init(&recursive_mailboxes, 32); + array_foreach_elem(&ctx->mailboxes, name) { + if (get_child_mailboxes(user, &recursive_mailboxes, + name) < 0) { + doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP); + ret = -1; + } + if (name[0] != '\0') + array_push_back(&recursive_mailboxes, &name); + } + array_sort(&recursive_mailboxes, i_strcmp_reverse_p); + mailboxes = &recursive_mailboxes; + } + + array_foreach_elem(mailboxes, name) { + ns = mail_namespace_find(user->namespaces, name); + box = mailbox_alloc(ns->list, name, mailbox_flags); + storage = mailbox_get_storage(box); + ret2 = ctx->require_empty ? mailbox_delete_empty(box) : + mailbox_delete(box); + if (ret2 < 0) { + i_error("Can't delete mailbox %s: %s", name, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + if (ctx->ctx.subscriptions) { + if (mailbox_set_subscribed(box, FALSE) < 0) { + i_error("Can't unsubscribe mailbox %s: %s", name, + mail_storage_get_last_internal_error(storage, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + } + mailbox_free(&box); + } + return ret; +} + +static void cmd_mailbox_delete_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct delete_cmd_context *ctx = (struct delete_cmd_context *)_ctx; + const char *name; + unsigned int i; + + if (args[0] == NULL) + doveadm_mail_help_name("mailbox delete"); + doveadm_mailbox_args_check(args); + + for (i = 0; args[i] != NULL; i++) { + name = p_strdup(ctx->ctx.ctx.pool, args[i]); + array_push_back(&ctx->mailboxes, &name); + } + array_sort(&ctx->mailboxes, i_strcmp_reverse_p); +} + +static bool +cmd_mailbox_delete_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct delete_cmd_context *ctx = (struct delete_cmd_context *)_ctx; + + switch (c) { + case 'r': + ctx->recursive = TRUE; + break; + case 's': + ctx->ctx.subscriptions = TRUE; + break; + case 'e': + ctx->require_empty = TRUE; + break; + case 'Z': + ctx->unsafe = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_delete_alloc(void) +{ + struct delete_cmd_context *ctx; + + ctx = doveadm_mailbox_cmd_alloc(struct delete_cmd_context); + ctx->ctx.ctx.v.init = cmd_mailbox_delete_init; + ctx->ctx.ctx.v.run = cmd_mailbox_delete_run; + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_delete_parse_arg; + ctx->ctx.ctx.getopt_args = "ersZ"; + p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); + return &ctx->ctx.ctx; +} + +static int +cmd_mailbox_rename_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct rename_cmd_context *ctx = (struct rename_cmd_context *)_ctx; + struct mail_namespace *oldns, *newns; + struct mailbox *oldbox, *newbox; + const char *oldname = ctx->oldname; + const char *newname = ctx->newname; + int ret = 0; + + oldns = mail_namespace_find(user->namespaces, oldname); + newns = mail_namespace_find(user->namespaces, newname); + oldbox = mailbox_alloc(oldns->list, oldname, 0); + newbox = mailbox_alloc(newns->list, newname, 0); + if (mailbox_rename(oldbox, newbox) < 0) { + i_error("Can't rename mailbox %s to %s: %s", oldname, newname, + mailbox_get_last_internal_error(oldbox, NULL)); + doveadm_mail_failed_mailbox(_ctx, oldbox); + ret = -1; + } + if (ctx->ctx.subscriptions) { + if (mailbox_set_subscribed(oldbox, FALSE) < 0) { + i_error("Can't unsubscribe mailbox %s: %s", ctx->oldname, + mailbox_get_last_internal_error(oldbox, NULL)); + doveadm_mail_failed_mailbox(_ctx, oldbox); + ret = -1; + } + if (mailbox_set_subscribed(newbox, TRUE) < 0) { + i_error("Can't subscribe to mailbox %s: %s", ctx->newname, + mailbox_get_last_internal_error(newbox, NULL)); + doveadm_mail_failed_mailbox(_ctx, newbox); + ret = -1; + } + } + + mailbox_free(&oldbox); + mailbox_free(&newbox); + return ret; +} + +static void cmd_mailbox_rename_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct rename_cmd_context *ctx = (struct rename_cmd_context *)_ctx; + + if (str_array_length(args) != 2) + doveadm_mail_help_name("mailbox rename"); + doveadm_mailbox_args_check(args); + + ctx->oldname = p_strdup(ctx->ctx.ctx.pool, args[0]); + ctx->newname = p_strdup(ctx->ctx.ctx.pool, args[1]); +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_rename_alloc(void) +{ + struct rename_cmd_context *ctx; + + ctx = doveadm_mailbox_cmd_alloc(struct rename_cmd_context); + ctx->ctx.ctx.v.init = cmd_mailbox_rename_init; + ctx->ctx.ctx.v.run = cmd_mailbox_rename_run; + return &ctx->ctx.ctx; +} + +static int +cmd_mailbox_subscribe_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + const char *name; + int ret = 0; + + array_foreach_elem(&ctx->mailboxes, name) { + ns = mail_namespace_find(user->namespaces, name); + box = mailbox_alloc(ns->list, name, 0); + if (mailbox_set_subscribed(box, ctx->ctx.subscriptions) < 0) { + i_error("Can't %s mailbox %s: %s", name, + ctx->ctx.subscriptions ? "subscribe to" : + "unsubscribe", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + mailbox_free(&box); + } + return ret; +} + +static void cmd_mailbox_subscribe_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx; + const char *name; + unsigned int i; + + if (args[0] == NULL) { + doveadm_mail_help_name(ctx->ctx.subscriptions ? + "mailbox subscribe" : + "mailbox unsubscribe"); + } + doveadm_mailbox_args_check(args); + + for (i = 0; args[i] != NULL; i++) { + name = p_strdup(ctx->ctx.ctx.pool, args[i]); + array_push_back(&ctx->mailboxes, &name); + } +} + +static struct doveadm_mail_cmd_context * +cmd_mailbox_subscriptions_alloc(bool subscriptions) +{ + struct mailbox_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct mailbox_cmd_context); + ctx->ctx.subscriptions = subscriptions; + + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_parse_arg; + ctx->ctx.ctx.v.init = cmd_mailbox_subscribe_init; + ctx->ctx.ctx.v.run = cmd_mailbox_subscribe_run; + p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16); + return &ctx->ctx.ctx; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_subscribe_alloc(void) +{ + return cmd_mailbox_subscriptions_alloc(TRUE); +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_unsubscribe_alloc(void) +{ + return cmd_mailbox_subscriptions_alloc(FALSE); +} + +static +void cmd_mailbox_update_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + + if (str_array_length(args) != 1) + doveadm_mail_help_name("mailbox update"); + + doveadm_mailbox_args_check(args); + + ctx->mailbox = args[0]; + + if ((ctx->update.min_first_recent_uid != 0 || + ctx->update.min_next_uid != 0) && + ctx->update.min_first_recent_uid > ctx->update.min_next_uid) { + i_fatal_status(EX_DATAERR, + "min_first_recent_uid > min_next_uid"); + } +} + +static +bool cmd_mailbox_update_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + + switch (c) { + case 'g': + if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'V': + if (str_to_uint32(optarg, &(ctx->update.uid_validity)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'N': + if (str_to_uint32(optarg, &(ctx->update.min_next_uid)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'R': + if (str_to_uint32(optarg, &(ctx->update.min_first_recent_uid)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'H': + if (str_to_uint64(optarg, &(ctx->update.min_highest_modseq)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + case 'P': + if (str_to_uint64(optarg, &(ctx->update.min_highest_pvt_modseq)) < 0) + doveadm_mail_help_name("mailbox update"); + break; + default: + return FALSE; + } + return TRUE; +} + +static +int cmd_mailbox_update_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + enum mail_error mail_error; + int ret = 0; + + ns = mail_namespace_find(user->namespaces, ctx->mailbox); + box = mailbox_alloc(ns->list, ctx->mailbox, 0); + + if ((ret = mailbox_update(box, &(ctx->update))) != 0) { + i_error("Cannot update %s: %s", + ctx->mailbox, + mailbox_get_last_internal_error(box, &mail_error)); + doveadm_mail_failed_error(_ctx, mail_error); + } + + mailbox_free(&box); + + return ret; +} + +static +struct doveadm_mail_cmd_context *cmd_mailbox_update_alloc(void) +{ + struct update_cmd_context *ctx; + ctx = doveadm_mail_cmd_alloc(struct update_cmd_context); + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_update_parse_arg; + ctx->ctx.ctx.v.init = cmd_mailbox_update_init; + ctx->ctx.ctx.v.run = cmd_mailbox_update_run; + return &ctx->ctx.ctx; +} + +static void +cmd_mailbox_path_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[]) +{ + struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx; + + if (str_array_length(args) != 1) + doveadm_mail_help_name("mailbox path"); + + doveadm_mailbox_args_check(args); + + ctx->mailbox = args[0]; + doveadm_print_header("path", "path", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); +} + +static bool +mailbox_list_path_type_name_parse(const char *name, + enum mailbox_list_path_type *type_r) +{ + enum mailbox_list_path_type type; + + for (type = 0; type < N_ELEMENTS(mailbox_list_path_type_names); type++) { + if (strcmp(mailbox_list_path_type_names[type], name) == 0) { + *type_r = type; + return TRUE; + } + } + return FALSE; +} + +static bool +cmd_mailbox_path_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct path_cmd_context *ctx = (struct path_cmd_context *)_ctx; + + switch (c) { + case 't': + if (!mailbox_list_path_type_name_parse(optarg, &ctx->path_type)) + doveadm_mail_help_name("mailbox path"); + break; + default: + return FALSE; + } + return TRUE; +} + +static int +cmd_mailbox_path_run(struct doveadm_mail_cmd_context *_ctx, + struct mail_user *user) +{ + struct path_cmd_context *ctx = (struct path_cmd_context *)_ctx; + struct mail_namespace *ns; + enum mail_error mail_error; + const char *storage_name, *path; + int ret; + + ns = mail_namespace_find(user->namespaces, ctx->mailbox); + storage_name = mailbox_list_get_storage_name(ns->list, ctx->mailbox); + ret = mailbox_list_get_path(ns->list, storage_name, ctx->path_type, &path); + if (ret < 0) { + i_error("Failed to lookup mailbox %s path: %s", + ctx->mailbox, + mailbox_list_get_last_internal_error(ns->list, &mail_error)); + doveadm_mail_failed_error(_ctx, mail_error); + } else if (ret > 0) { + doveadm_print(path); + } + return ret; +} + +static struct doveadm_mail_cmd_context *cmd_mailbox_path_alloc(void) +{ + struct path_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct path_cmd_context); + ctx->path_type = MAILBOX_LIST_PATH_TYPE_INDEX; + ctx->ctx.ctx.v.parse_arg = cmd_mailbox_path_parse_arg; + ctx->ctx.ctx.v.init = cmd_mailbox_path_init; + ctx->ctx.ctx.v.run = cmd_mailbox_path_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return &ctx->ctx.ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_list_ver2 = { + .name = "mailbox list", + .mail_cmd = cmd_mailbox_list_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-7|-8] [-s] [<mailbox mask> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('7', "mutf7", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('8', "utf8", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_create_ver2 = { + .name = "mailbox create", + .mail_cmd = cmd_mailbox_create_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] [-g <guid>] <mailbox> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('g', "guid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2 = { + .name = "mailbox delete", + .mail_cmd = cmd_mailbox_delete_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-e] [-r] [-s] [-Z] <mailbox> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('e', "require-empty", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('r', "recursive", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('Z', "unsafe", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_rename_ver2 = { + .name = "mailbox rename", + .mail_cmd = cmd_mailbox_rename_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <old name> <new name>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "new-name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_subscribe_ver2 = { + .name = "mailbox subscribe", + .mail_cmd = cmd_mailbox_subscribe_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<mailbox> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_unsubscribe_ver2 = { + .name = "mailbox unsubscribe", + .mail_cmd = cmd_mailbox_unsubscribe_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<mailbox> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2 = { + .name = "mailbox update", + .mail_cmd = cmd_mailbox_update_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[--mailbox-guid guid] [--uid-validity uid] [--min-next-uid uid] [--min-first-recent-uid uid] [--min-highest-modseq seq] [--min-highest-pvt-modseq seq] <mailbox>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('V', "uid-validity", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('N', "min-next-uid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('R', "min-first-recent-uid", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('H', "min-highest-modseq", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('P', "min-highest-pvt-modseq", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_path_ver2 = { + .name = "mailbox path", + .mail_cmd = cmd_mailbox_path_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-t <type>] <mailbox>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('t', "type", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}; + diff --git a/src/doveadm/doveadm-mail-rebuild.c b/src/doveadm/doveadm-mail-rebuild.c new file mode 100644 index 0000000..211c3c3 --- /dev/null +++ b/src/doveadm/doveadm-mail-rebuild.c @@ -0,0 +1,101 @@ +/* Copyright (c) 2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage.h" +#include "doveadm-print.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" +#include "mail-storage-private.h" + +static int +cmd_rebuild_attachment_box(struct doveadm_mail_cmd_context *ctx, + const struct mailbox_info *info) +{ + struct doveadm_mail_iter *iter; + struct mail *mail; + int ret = 0; + + if (doveadm_mail_iter_init(ctx, info, ctx->search_args, + MAIL_FETCH_IMAP_BODYSTRUCTURE| + MAIL_FETCH_MESSAGE_PARTS, NULL, 0, + &iter) < 0) + return -1; + + while (doveadm_mail_iter_next(iter, &mail) && ret >= 0) { + T_BEGIN { + doveadm_print(dec2str(mail->uid)); + switch(mail_set_attachment_keywords(mail)) { + case -1: + doveadm_print("error"); + doveadm_mail_failed_mailbox(ctx, mail->box); + ret = -1; + break; + case 0: + doveadm_print("no"); + break; + case 1: + doveadm_print("yes"); + break; + default: + i_unreached(); + } + } T_END; + } + + if (doveadm_mail_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int +cmd_rebuild_attachment_run(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user) +{ + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_rebuild_attachment_box(ctx, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_rebuild_attachment_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + doveadm_print_header_simple("uid"); + doveadm_print_header_simple("attachment"); + ctx->search_args = doveadm_mail_build_search_args(args); +} + + +static struct doveadm_mail_cmd_context *cmd_rebuild_attachment_alloc(void) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.init = cmd_rebuild_attachment_init; + ctx->v.run = cmd_rebuild_attachment_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + return ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_rebuild_attachments = { + .name = "rebuild attachments", + .mail_cmd = cmd_rebuild_attachment_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-save.c b/src/doveadm/doveadm-mail-save.c new file mode 100644 index 0000000..d8312c8 --- /dev/null +++ b/src/doveadm/doveadm-mail-save.c @@ -0,0 +1,142 @@ +/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "mail-storage.h" +#include "doveadm-mail.h" + +struct save_cmd_context { + struct doveadm_mail_cmd_context ctx; + const char *mailbox; +}; + +static int +cmd_save_to_mailbox(struct save_cmd_context *ctx, struct mailbox *box, + struct istream *input) +{ + struct mail_storage *storage = mailbox_get_storage(box); + struct mailbox_transaction_context *trans; + struct mail_save_context *save_ctx; + ssize_t ret; + bool save_failed = FALSE; + + if (input->stream_errno != 0) { + i_error("open(%s) failed: %s", + i_stream_get_name(input), + i_stream_get_error(input)); + ctx->ctx.exit_code = EX_TEMPFAIL; + return -1; + } + + if (mailbox_open(box) < 0) { + i_error("Failed to open mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_storage(&ctx->ctx, storage); + return -1; + } + + trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL | + ctx->ctx.transaction_flags, __func__); + save_ctx = mailbox_save_alloc(trans); + if (mailbox_save_begin(&save_ctx, input) < 0) { + i_error("Saving failed: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_storage(&ctx->ctx, storage); + mailbox_transaction_rollback(&trans); + return -1; + } + do { + if (mailbox_save_continue(save_ctx) < 0) { + save_failed = TRUE; + ret = -1; + break; + } + } while ((ret = i_stream_read(input)) > 0); + i_assert(ret == -1); + + if (input->stream_errno != 0) { + i_error("read(msg input) failed: %s", i_stream_get_error(input)); + doveadm_mail_failed_error(&ctx->ctx, MAIL_ERROR_TEMP); + } else if (save_failed) { + i_error("Saving failed: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_storage(&ctx->ctx, storage); + } else if (mailbox_save_finish(&save_ctx) < 0) { + i_error("Saving failed: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_storage(&ctx->ctx, storage); + } else if (mailbox_transaction_commit(&trans) < 0) { + i_error("Save transaction commit failed: %s", + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_storage(&ctx->ctx, storage); + } else { + ret = 0; + } + if (save_ctx != NULL) + mailbox_save_cancel(&save_ctx); + if (trans != NULL) + mailbox_transaction_rollback(&trans); + i_assert(input->eof); + return ret < 0 ? -1 : 0; +} + +static int +cmd_save_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user) +{ + struct save_cmd_context *ctx = (struct save_cmd_context *)_ctx; + struct mail_namespace *ns; + struct mailbox *box; + int ret; + + ns = mail_namespace_find(user->namespaces, ctx->mailbox); + box = mailbox_alloc(ns->list, ctx->mailbox, MAILBOX_FLAG_SAVEONLY); + ret = cmd_save_to_mailbox(ctx, box, _ctx->cmd_input); + mailbox_free(&box); + return ret; +} + +static void cmd_save_init(struct doveadm_mail_cmd_context *_ctx, + const char *const args[] ATTR_UNUSED) +{ + doveadm_mail_get_input(_ctx); +} + +static bool +cmd_mailbox_save_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct save_cmd_context *ctx = (struct save_cmd_context *)_ctx; + + switch (c) { + case 'm': + ctx->mailbox = optarg; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_save_alloc(void) +{ + struct save_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct save_cmd_context); + ctx->ctx.getopt_args = "m:"; + ctx->ctx.v.parse_arg = cmd_mailbox_save_parse_arg; + ctx->ctx.v.init = cmd_save_init; + ctx->ctx.v.run = cmd_save_run; + ctx->mailbox = "INBOX"; + return &ctx->ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_save_ver2 = { + .name = "save", + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-m mailbox]", + .mail_cmd = cmd_save_alloc, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('m', "mailbox", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "file", CMD_PARAM_ISTREAM, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-search.c b/src/doveadm/doveadm-mail-search.c new file mode 100644 index 0000000..37c47e3 --- /dev/null +++ b/src/doveadm/doveadm-mail-search.c @@ -0,0 +1,103 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-storage.h" +#include "doveadm-print.h" +#include "doveadm-mailbox-list-iter.h" +#include "doveadm-mail-iter.h" +#include "doveadm-mail.h" + +#include <stdio.h> + +static int +cmd_search_box(struct doveadm_mail_cmd_context *ctx, + const struct mailbox_info *info) +{ + struct doveadm_mail_iter *iter; + struct mailbox *box; + struct mail *mail; + struct mailbox_metadata metadata; + const char *guid_str; + int ret = 0; + + if (doveadm_mail_iter_init(ctx, info, ctx->search_args, 0, NULL, + DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT, + &iter) < 0) + return -1; + box = doveadm_mail_iter_get_mailbox(iter); + + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { + i_error("Couldn't get mailbox '%s' GUID: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, NULL)); + ret = -1; + doveadm_mail_failed_mailbox(ctx, box); + } else { + guid_str = guid_128_to_string(metadata.guid); + while (doveadm_mail_iter_next(iter, &mail)) { + doveadm_print(guid_str); + T_BEGIN { + doveadm_print(dec2str(mail->uid)); + } T_END; + } + } + if (doveadm_mail_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static int +cmd_search_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) +{ + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct doveadm_mailbox_list_iter *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args, + iter_flags); + while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (cmd_search_box(ctx, info) < 0) + ret = -1; + } T_END; + if (doveadm_mailbox_list_iter_deinit(&iter) < 0) + ret = -1; + return ret; +} + +static void cmd_search_init(struct doveadm_mail_cmd_context *ctx, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("search"); + + doveadm_print_header("mailbox-guid", "mailbox-guid", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + doveadm_print_header("uid", "uid", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + + ctx->search_args = doveadm_mail_build_search_args(args); +} + +static struct doveadm_mail_cmd_context *cmd_search_alloc(void) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.init = cmd_search_init; + ctx->v.run = cmd_search_run; + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + return ctx; +} + +struct doveadm_cmd_ver2 doveadm_cmd_search_ver2 = { + .name = "search", + .mail_cmd = cmd_search_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mail-server.c b/src/doveadm/doveadm-mail-server.c new file mode 100644 index 0000000..a829a24 --- /dev/null +++ b/src/doveadm/doveadm-mail-server.c @@ -0,0 +1,404 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "ioloop.h" +#include "master-service.h" +#include "iostream-ssl.h" +#include "auth-master.h" +#include "mail-storage.h" +#include "mail-storage-service.h" +#include "server-connection.h" +#include "doveadm-settings.h" +#include "doveadm-print.h" +#include "doveadm-server.h" +#include "doveadm-mail.h" + +#define DOVEADM_SERVER_CONNECTIONS_MAX 4 +#define DOVEADM_SERVER_QUEUE_MAX 16 + +#define DOVEADM_MAIL_SERVER_FAILED() \ + (internal_failure || master_service_is_killed(master_service)) + +struct doveadm_mail_server_cmd { + struct server_connection *conn; + char *username; +}; + +static HASH_TABLE(char *, struct doveadm_server *) servers; +static pool_t server_pool; +static struct doveadm_mail_cmd_context *cmd_ctx; +static bool internal_failure = FALSE; + +static void doveadm_mail_server_handle(struct server_connection *conn, + const char *username); + +static struct doveadm_server * +doveadm_server_get(struct doveadm_mail_cmd_context *ctx, const char *name) +{ + struct doveadm_server *server; + const char *p; + char *dup_name; + + if (!hash_table_is_created(servers)) { + server_pool = pool_alloconly_create("doveadm servers", 1024*16); + hash_table_create(&servers, server_pool, 0, str_hash, strcmp); + } + server = hash_table_lookup(servers, name); + if (server == NULL) { + server = p_new(server_pool, struct doveadm_server, 1); + server->name = dup_name = p_strdup(server_pool, name); + p = strrchr(server->name, ':'); + server->hostname = p == NULL ? server->name : + p_strdup_until(server_pool, server->name, p); + + p_array_init(&server->connections, server_pool, + ctx->set->doveadm_worker_count); + p_array_init(&server->queue, server_pool, + DOVEADM_SERVER_QUEUE_MAX); + hash_table_insert(servers, dup_name, server); + } + return server; +} + +static struct server_connection * +doveadm_server_find_unused_conn(struct doveadm_server *server) +{ + struct server_connection *conn; + + array_foreach_elem(&server->connections, conn) { + if (server_connection_is_idle(conn)) + return conn; + } + return NULL; +} + +static bool doveadm_server_have_used_connections(struct doveadm_server *server) +{ + struct server_connection *conn; + + array_foreach_elem(&server->connections, conn) { + if (!server_connection_is_idle(conn)) + return TRUE; + } + return FALSE; +} + +static void doveadm_cmd_callback(int exit_code, const char *error, + void *context) +{ + struct doveadm_mail_server_cmd *servercmd = context; + struct doveadm_server *server = + server_connection_get_server(servercmd->conn); + const char *username = t_strdup(servercmd->username); + + i_free(servercmd->username); + i_free(servercmd); + + switch (exit_code) { + case 0: + break; + case SERVER_EXIT_CODE_DISCONNECTED: + i_error("%s: Command %s failed for %s: %s", + server->name, cmd_ctx->cmd->name, username, error); + internal_failure = TRUE; + io_loop_stop(current_ioloop); + return; + case EX_NOUSER: + i_error("%s: No such user: %s", server->name, username); + if (cmd_ctx->exit_code == 0) + cmd_ctx->exit_code = EX_NOUSER; + break; + default: + if (cmd_ctx->exit_code == 0 || exit_code == EX_TEMPFAIL) + cmd_ctx->exit_code = exit_code; + break; + } + + if (array_count(&server->queue) > 0) { + struct server_connection *conn; + char *const *usernamep = array_front(&server->queue); + char *username = *usernamep; + + conn = doveadm_server_find_unused_conn(server); + if (conn != NULL) { + array_pop_front(&server->queue); + doveadm_mail_server_handle(conn, username); + i_free(username); + } + } + + io_loop_stop(current_ioloop); +} + +static void doveadm_mail_server_handle(struct server_connection *conn, + const char *username) +{ + struct doveadm_mail_server_cmd *servercmd; + string_t *cmd; + unsigned int i; + + /* <flags> <username> <command> [<args>] */ + cmd = t_str_new(256); + if (doveadm_debug) + str_append_c(cmd, 'D'); + else if (doveadm_verbose) + str_append_c(cmd, 'v'); + str_append_c(cmd, '\t'); + + str_append_tabescaped(cmd, username); + str_append_c(cmd, '\t'); + str_append_tabescaped(cmd, cmd_ctx->cmd->name); + for (i = 0; cmd_ctx->full_args[i] != NULL; i++) { + str_append_c(cmd, '\t'); + str_append_tabescaped(cmd, cmd_ctx->full_args[i]); + } + str_append_c(cmd, '\n'); + + servercmd = i_new(struct doveadm_mail_server_cmd, 1); + servercmd->conn = conn; + servercmd->username = i_strdup(username); + server_connection_cmd(conn, str_c(cmd), cmd_ctx->cmd_input, + doveadm_cmd_callback, servercmd); +} + +static void doveadm_server_flush_one(struct doveadm_server *server) +{ + unsigned int count = array_count(&server->queue); + + do { + io_loop_run(current_ioloop); + } while (array_count(&server->queue) == count && + doveadm_server_have_used_connections(server) && + !DOVEADM_MAIL_SERVER_FAILED()); +} + +static int +doveadm_mail_server_user_get_host(struct doveadm_mail_cmd_context *ctx, + const struct mail_storage_service_input *input, + const char **user_r, const char **host_r, + struct ip_addr *hostip_r, in_port_t *port_r, + enum doveadm_proxy_ssl_flags *ssl_flags_r, + const char **error_r) +{ + struct auth_master_connection *auth_conn; + struct auth_user_info info; + pool_t pool; + const char *auth_socket_path, *proxy_host, *proxy_hostip, *const *fields; + unsigned int i; + in_port_t proxy_port; + bool proxying; + int ret; + + *user_r = input->username; + *host_r = ctx->set->doveadm_socket_path; + *port_r = ctx->set->doveadm_port; + + if (ctx->set->doveadm_port == 0) + return 0; + + if (strcmp(ctx->set->doveadm_ssl, "ssl") == 0) + *ssl_flags_r |= PROXY_SSL_FLAG_YES; + else if (strcmp(ctx->set->doveadm_ssl, "starttls") == 0) + *ssl_flags_r |= PROXY_SSL_FLAG_YES | PROXY_SSL_FLAG_STARTTLS; + + /* make sure we have an auth connection */ + mail_storage_service_init_settings(ctx->storage_service, input); + + i_zero(&info); + info.service = master_service_get_name(master_service); + info.local_ip = input->local_ip; + info.remote_ip = input->remote_ip; + info.local_port = input->local_port; + info.remote_port = input->remote_port; + + pool = pool_alloconly_create("auth lookup", 1024); + auth_conn = mail_storage_service_get_auth_conn(ctx->storage_service); + auth_socket_path = auth_master_get_socket_path(auth_conn); + ret = auth_master_pass_lookup(auth_conn, input->username, &info, + pool, &fields); + if (ret < 0) { + *error_r = fields[0] != NULL ? + t_strdup(fields[0]) : "passdb lookup failed"; + *error_r = t_strdup_printf("%s: %s (to see if user is proxied, " + "because doveadm_port is set)", + auth_socket_path, *error_r); + } else if (ret == 0) { + /* user not found from passdb. it could be in userdb though, + so just continue with the default host */ + } else { + proxy_host = NULL; proxy_hostip = NULL; proxying = FALSE; + proxy_port = ctx->set->doveadm_port; + for (i = 0; fields[i] != NULL; i++) { + if (str_begins(fields[i], "proxy") && + (fields[i][5] == '\0' || fields[i][5] == '=')) + proxying = TRUE; + else if (str_begins(fields[i], "host=")) + proxy_host = fields[i]+5; + else if (str_begins(fields[i], "hostip=")) + proxy_hostip = fields[i]+7; + else if (str_begins(fields[i], "user=")) + *user_r = t_strdup(fields[i]+5); + else if (str_begins(fields[i], "destuser=")) + *user_r = t_strdup(fields[i]+9); + else if (str_begins(fields[i], "port=")) { + if (net_str2port(fields[i]+5, &proxy_port) < 0) + proxy_port = 0; + } else if (str_begins(fields[i], "ssl=")) { + *ssl_flags_r |= PROXY_SSL_FLAG_YES; + if (strcmp(fields[i]+4, "any-cert") == 0) + *ssl_flags_r |= PROXY_SSL_FLAG_ANY_CERT; + } else if (str_begins(fields[i], "starttls=")) { + *ssl_flags_r |= PROXY_SSL_FLAG_YES | + PROXY_SSL_FLAG_STARTTLS; + if (strcmp(fields[i]+9, "any-cert") == 0) + *ssl_flags_r |= PROXY_SSL_FLAG_ANY_CERT; + } + } + if (proxy_hostip != NULL && + net_addr2ip(proxy_hostip, hostip_r) < 0) { + *error_r = t_strdup_printf("%s Invalid hostip value '%s'", + auth_socket_path, proxy_hostip); + ret = -1; + } + if (!proxying) + ret = 0; + else if (proxy_host == NULL) { + *error_r = t_strdup_printf("%s: Proxy is missing destination host", + auth_socket_path); + if (strstr(auth_socket_path, "/auth-userdb") != NULL) { + *error_r = t_strdup_printf( + "%s (maybe set auth_socket_path=director-userdb)", + *error_r); + } + ret = -1; + } else { + *port_r = proxy_port; + *host_r = t_strdup_printf("%s:%u", proxy_host, proxy_port); + } + } + pool_unref(&pool); + return ret; +} + +int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx, + const struct mail_storage_service_input *input, + const char **error_r) +{ + struct doveadm_server *server; + struct server_connection *conn; + const char *user, *host; + struct ip_addr hostip; + enum doveadm_proxy_ssl_flags ssl_flags = 0; + char *username_dup; + int ret; + in_port_t port; + + i_assert(cmd_ctx == ctx || cmd_ctx == NULL); + cmd_ctx = ctx; + + i_zero(&hostip); + ret = doveadm_mail_server_user_get_host(ctx, input, &user, &host, &hostip, + &port, &ssl_flags, error_r); + if (ret < 0) + return ret; + if (ret == 0 && + (ctx->set->doveadm_worker_count == 0 || doveadm_server)) { + /* run it ourself */ + return 0; + } + + /* server sends the sticky headers for each row as well, + so undo any sticks we might have added already */ + doveadm_print_unstick_headers(); + + server = doveadm_server_get(ctx, host); + server->ip = hostip; + server->ssl_flags = ssl_flags; + server->port = port; + conn = doveadm_server_find_unused_conn(server); + if (conn != NULL) + doveadm_mail_server_handle(conn, user); + else if (array_count(&server->connections) < + I_MAX(ctx->set->doveadm_worker_count, 1)) { + if (server_connection_create(server, &conn, error_r) < 0) { + internal_failure = TRUE; + return -1; + } else { + doveadm_mail_server_handle(conn, user); + } + } else { + if (array_count(&server->queue) >= DOVEADM_SERVER_QUEUE_MAX) + doveadm_server_flush_one(server); + + username_dup = i_strdup(user); + array_push_back(&server->queue, &username_dup); + } + *error_r = "doveadm server failure"; + return DOVEADM_MAIL_SERVER_FAILED() ? -1 : 1; +} + +static struct doveadm_server *doveadm_server_find_used(void) +{ + struct hash_iterate_context *iter; + struct doveadm_server *ret = NULL; + char *key; + struct doveadm_server *server; + + iter = hash_table_iterate_init(servers); + while (hash_table_iterate(iter, servers, &key, &server)) { + if (doveadm_server_have_used_connections(server)) { + ret = server; + break; + } + } + hash_table_iterate_deinit(&iter); + return ret; +} + +static void doveadm_servers_destroy_all_connections(void) +{ + struct hash_iterate_context *iter; + char *key; + struct doveadm_server *server; + + iter = hash_table_iterate_init(servers); + while (hash_table_iterate(iter, servers, &key, &server)) { + while (array_count(&server->connections) > 0) { + struct server_connection *const *connp, *conn; + + connp = array_front(&server->connections); + conn = *connp; + server_connection_destroy(&conn); + } + ssl_iostream_context_unref(&server->ssl_ctx); + } + hash_table_iterate_deinit(&iter); +} + +void doveadm_mail_server_flush(void) +{ + struct doveadm_server *server; + + if (!hash_table_is_created(servers)) { + cmd_ctx = NULL; + return; + } + + while ((server = doveadm_server_find_used()) != NULL && + !DOVEADM_MAIL_SERVER_FAILED()) + doveadm_server_flush_one(server); + + doveadm_servers_destroy_all_connections(); + if (master_service_is_killed(master_service)) + i_error("Aborted"); + if (DOVEADM_MAIL_SERVER_FAILED()) + doveadm_mail_failed_error(cmd_ctx, MAIL_ERROR_TEMP); + + hash_table_destroy(&servers); + pool_unref(&server_pool); + cmd_ctx = NULL; +} diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c new file mode 100644 index 0000000..574c3b7 --- /dev/null +++ b/src/doveadm/doveadm-mail.c @@ -0,0 +1,990 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "lib-signals.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-dot.h" +#include "istream-seekable.h" +#include "str.h" +#include "unichar.h" +#include "module-dir.h" +#include "wildcard-match.h" +#include "master-service.h" +#include "mail-user.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mail-storage-settings.h" +#include "mail-storage-service.h" +#include "mail-storage-hooks.h" +#include "mail-search-build.h" +#include "mail-search-parser.h" +#include "mailbox-list-iter.h" +#include "doveadm.h" +#include "client-connection.h" +#include "doveadm-settings.h" +#include "doveadm-print.h" +#include "doveadm-dsync.h" +#include "doveadm-mail.h" + +#include <stdio.h> + +#define DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS (5*60*1000) + +struct force_resync_cmd_context { + struct doveadm_mail_cmd_context ctx; + bool fsck; +}; + +void (*hook_doveadm_mail_init)(struct doveadm_mail_cmd_context *ctx); +struct doveadm_mail_cmd_module_register + doveadm_mail_cmd_module_register = { 0 }; +char doveadm_mail_cmd_hide = '\0'; + +static int killed_signo = 0; + +bool doveadm_is_killed(void) +{ + return killed_signo != 0; +} + +int doveadm_killed_signo(void) +{ + return killed_signo; +} + +void doveadm_mail_failed_error(struct doveadm_mail_cmd_context *ctx, + enum mail_error error) +{ + int exit_code = EX_TEMPFAIL; + + switch (error) { + case MAIL_ERROR_NONE: + i_unreached(); + case MAIL_ERROR_TEMP: + case MAIL_ERROR_UNAVAILABLE: + break; + case MAIL_ERROR_NOTPOSSIBLE: + case MAIL_ERROR_EXISTS: + case MAIL_ERROR_CONVERSION: + case MAIL_ERROR_INVALIDDATA: + exit_code = DOVEADM_EX_NOTPOSSIBLE; + break; + case MAIL_ERROR_PARAMS: + exit_code = EX_USAGE; + break; + case MAIL_ERROR_PERM: + exit_code = EX_NOPERM; + break; + case MAIL_ERROR_NOQUOTA: + exit_code = EX_CANTCREAT; + break; + case MAIL_ERROR_NOTFOUND: + exit_code = DOVEADM_EX_NOTFOUND; + break; + case MAIL_ERROR_EXPUNGED: + break; + case MAIL_ERROR_INUSE: + case MAIL_ERROR_LIMIT: + exit_code = DOVEADM_EX_NOTPOSSIBLE; + break; + case MAIL_ERROR_LOOKUP_ABORTED: + break; + } + /* tempfail overrides all other exit codes, otherwise use whatever + error happened first */ + if (ctx->exit_code == 0 || exit_code == EX_TEMPFAIL) + ctx->exit_code = exit_code; +} + +void doveadm_mail_failed_storage(struct doveadm_mail_cmd_context *ctx, + struct mail_storage *storage) +{ + enum mail_error error; + + mail_storage_get_last_error(storage, &error); + doveadm_mail_failed_error(ctx, error); +} + +void doveadm_mail_failed_mailbox(struct doveadm_mail_cmd_context *ctx, + struct mailbox *box) +{ + doveadm_mail_failed_storage(ctx, mailbox_get_storage(box)); +} + +void doveadm_mail_failed_list(struct doveadm_mail_cmd_context *ctx, + struct mailbox_list *list) +{ + enum mail_error error; + + mailbox_list_get_last_error(list, &error); + doveadm_mail_failed_error(ctx, error); +} + +struct doveadm_mail_cmd_context * +doveadm_mail_cmd_alloc_size(size_t size) +{ + struct doveadm_mail_cmd_context *ctx; + pool_t pool; + + i_assert(size >= sizeof(struct doveadm_mail_cmd_context)); + + pool = pool_alloconly_create("doveadm mail cmd", 1024); + ctx = p_malloc(pool, size); + ctx->pool = pool; + ctx->cmd_input_fd = -1; + return ctx; +} + +static int +cmd_purge_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user) +{ + struct mail_namespace *ns; + struct mail_storage *storage; + int ret = 0; + + for (ns = user->namespaces; ns != NULL; ns = ns->next) { + if (ns->type != MAIL_NAMESPACE_TYPE_PRIVATE || + ns->alias_for != NULL) + continue; + + storage = mail_namespace_get_default_storage(ns); + if (mail_storage_purge(storage) < 0) { + i_error("Purging namespace '%s' failed: %s", ns->prefix, + mail_storage_get_last_internal_error(storage, NULL)); + doveadm_mail_failed_storage(ctx, storage); + ret = -1; + } + } + return ret; +} + +static struct doveadm_mail_cmd_context *cmd_purge_alloc(void) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context); + ctx->v.run = cmd_purge_run; + return ctx; +} + +static void doveadm_mail_cmd_input_input(struct doveadm_mail_cmd_context *ctx) +{ + const unsigned char *data; + size_t size; + + while (i_stream_read_more(ctx->cmd_input, &data, &size) > 0) + i_stream_skip(ctx->cmd_input, size); + if (!ctx->cmd_input->eof) + return; + + if (ctx->cmd_input->stream_errno != 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(ctx->cmd_input), + i_stream_get_error(ctx->cmd_input)); + } + io_loop_stop(current_ioloop); +} + +static void doveadm_mail_cmd_input_timeout(struct doveadm_mail_cmd_context *ctx) +{ + struct istream *input; + + input = i_stream_create_error_str(ETIMEDOUT, "Timed out in %u secs", + DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS/1000); + i_stream_set_name(input, i_stream_get_name(ctx->cmd_input)); + i_stream_destroy(&ctx->cmd_input); + ctx->cmd_input = input; + ctx->exit_code = EX_TEMPFAIL; + io_loop_stop(current_ioloop); +} + +static void doveadm_mail_cmd_input_read(struct doveadm_mail_cmd_context *ctx) +{ + struct ioloop *ioloop; + struct io *io; + struct timeout *to; + + ioloop = io_loop_create(); + /* Read the pending input from stream. Delay adding the IO in case + we're reading from a file. That would cause a panic with epoll. */ + io_loop_set_running(ioloop); + doveadm_mail_cmd_input_input(ctx); + if (io_loop_is_running(ioloop)) { + io = io_add(ctx->cmd_input_fd, IO_READ, + doveadm_mail_cmd_input_input, ctx); + to = timeout_add(DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS, + doveadm_mail_cmd_input_timeout, ctx); + io_loop_run(ioloop); + io_remove(&io); + timeout_remove(&to); + } + io_loop_destroy(&ioloop); + + i_assert(ctx->cmd_input->eof); + i_stream_seek(ctx->cmd_input, 0); +} + +void doveadm_mail_get_input(struct doveadm_mail_cmd_context *ctx) +{ + const struct doveadm_cmd_context *cctx = ctx->cctx; + bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI); + struct istream *inputs[2]; + + if (ctx->cmd_input != NULL) + return; + + if (!cli && cctx->input == NULL) { + ctx->cmd_input = i_stream_create_error_str(EINVAL, "Input stream missing (provide with file parameter)"); + return; + } + + if (!cli) + inputs[0] = i_stream_create_dot(cctx->input, FALSE); + else { + inputs[0] = i_stream_create_fd(STDIN_FILENO, 1024*1024); + i_stream_set_name(inputs[0], "stdin"); + } + inputs[1] = NULL; + ctx->cmd_input_fd = i_stream_get_fd(inputs[0]); + ctx->cmd_input = i_stream_create_seekable_path(inputs, 1024*256, + "/tmp/doveadm."); + i_stream_set_name(ctx->cmd_input, i_stream_get_name(inputs[0])); + i_stream_unref(&inputs[0]); + + doveadm_mail_cmd_input_read(ctx); +} + +struct mailbox * +doveadm_mailbox_find(struct mail_user *user, const char *mailbox) +{ + struct mail_namespace *ns; + + if (!uni_utf8_str_is_valid(mailbox)) { + i_fatal_status(EX_DATAERR, + "Mailbox name not valid UTF-8: %s", mailbox); + } + + ns = mail_namespace_find(user->namespaces, mailbox); + return mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_IGNORE_ACLS); +} + +struct mail_search_args * +doveadm_mail_build_search_args(const char *const args[]) +{ + struct mail_search_parser *parser; + struct mail_search_args *sargs; + const char *error, *charset = "UTF-8"; + + parser = mail_search_parser_init_cmdline(args); + if (mail_search_build(mail_search_register_get_human(), + parser, &charset, &sargs, &error) < 0) + i_fatal("%s", error); + mail_search_parser_deinit(&parser); + return sargs; +} + +static int cmd_force_resync_box(struct doveadm_mail_cmd_context *_ctx, + const struct mailbox_info *info) +{ + struct force_resync_cmd_context *ctx = + (struct force_resync_cmd_context *)_ctx; + enum mailbox_flags flags = MAILBOX_FLAG_IGNORE_ACLS; + struct mailbox *box; + int ret = 0; + + if (ctx->fsck) + flags |= MAILBOX_FLAG_FSCK; + + box = mailbox_alloc(info->ns->list, info->vname, flags); + if (mailbox_open(box) < 0) { + i_error("Opening mailbox %s failed: %s", info->vname, + mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } else if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FORCE_RESYNC | + MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) < 0) { + i_error("Forcing a resync on mailbox %s failed: %s", + info->vname, mailbox_get_last_internal_error(box, NULL)); + doveadm_mail_failed_mailbox(_ctx, box); + ret = -1; + } + mailbox_free(&box); + return ret; +} + +static int cmd_force_resync_prerun(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED, + struct mail_storage_service_user *service_user, + const char **error_r) +{ + if (mail_storage_service_user_set_setting(service_user, + "mailbox_list_index_very_dirty_syncs", + "no", + error_r) <= 0) + i_unreached(); + return 0; +} + +static int cmd_force_resync_run(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user) +{ + const enum mailbox_list_iter_flags iter_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS | + MAILBOX_LIST_ITER_STAR_WITHIN_NS; + const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL; + struct mailbox_list_iterate_context *iter; + const struct mailbox_info *info; + int ret = 0; + + iter = mailbox_list_iter_init_namespaces(user->namespaces, ctx->args, + ns_mask, iter_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if ((info->flags & (MAILBOX_NOSELECT | + MAILBOX_NONEXISTENT)) == 0) T_BEGIN { + if (cmd_force_resync_box(ctx, info) < 0) + ret = -1; + } T_END; + } + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Listing mailboxes failed: %s", + mailbox_list_get_last_internal_error(user->namespaces->list, NULL)); + doveadm_mail_failed_list(ctx, user->namespaces->list); + ret = -1; + } + return ret; +} + +static void +cmd_force_resync_init(struct doveadm_mail_cmd_context *_ctx ATTR_UNUSED, + const char *const args[]) +{ + if (args[0] == NULL) + doveadm_mail_help_name("force-resync"); +} + +static bool +cmd_force_resync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c) +{ + struct force_resync_cmd_context *ctx = + (struct force_resync_cmd_context *)_ctx; + + switch (c) { + case 'f': + ctx->fsck = TRUE; + break; + default: + return FALSE; + } + return TRUE; +} + +static struct doveadm_mail_cmd_context *cmd_force_resync_alloc(void) +{ + struct force_resync_cmd_context *ctx; + + ctx = doveadm_mail_cmd_alloc(struct force_resync_cmd_context); + ctx->ctx.getopt_args = "f"; + ctx->ctx.v.parse_arg = cmd_force_resync_parse_arg; + ctx->ctx.v.init = cmd_force_resync_init; + ctx->ctx.v.run = cmd_force_resync_run; + ctx->ctx.v.prerun = cmd_force_resync_prerun; + return &ctx->ctx; +} + +static void +doveadm_cctx_to_storage_service_input(const struct doveadm_cmd_context *cctx, + struct mail_storage_service_input *input_r) +{ + i_zero(input_r); + input_r->service = "doveadm"; + input_r->remote_ip = cctx->remote_ip; + input_r->remote_port = cctx->remote_port; + input_r->local_ip = cctx->local_ip; + input_r->local_port = cctx->local_port; + input_r->username = cctx->username; +} + +static int +doveadm_mail_next_user(struct doveadm_mail_cmd_context *ctx, + const char **error_r) +{ + const struct doveadm_cmd_context *cctx = ctx->cctx; + struct mail_storage_service_input input; + const char *error, *ip; + int ret; + + i_assert(cctx != NULL); + + ip = net_ip2addr(&cctx->remote_ip); + if (ip[0] == '\0') + i_set_failure_prefix("doveadm(%s): ", cctx->username); + else + i_set_failure_prefix("doveadm(%s,%s): ", ip, cctx->username); + doveadm_cctx_to_storage_service_input(cctx, &input); + if (ctx->cmd_input != NULL) + i_stream_seek(ctx->cmd_input, 0); + + /* see if we want to execute this command via (another) + doveadm server */ + ret = doveadm_mail_server_user(ctx, &input, error_r); + if (ret != 0) + return ret; + + ret = mail_storage_service_lookup(ctx->storage_service, &input, + &ctx->cur_service_user, &error); + if (ret <= 0) { + if (ret < 0) { + *error_r = t_strdup_printf("User lookup failed: %s", + error); + } + return ret; + } + + if (ctx->v.prerun != NULL) { + if (ctx->v.prerun(ctx, ctx->cur_service_user, error_r) < 0) { + mail_storage_service_user_unref(&ctx->cur_service_user); + return -1; + } + } + + ret = mail_storage_service_next(ctx->storage_service, + ctx->cur_service_user, + &ctx->cur_mail_user, error_r); + if (ret < 0) { + mail_storage_service_user_unref(&ctx->cur_service_user); + return ret; + } + + struct event_reason *reason = + event_reason_begin(event_reason_code_prefix("doveadm", "cmd_", + ctx->cmd->name)); + T_BEGIN { + if (ctx->v.run(ctx, ctx->cur_mail_user) < 0) { + i_assert(ctx->exit_code != 0); + } + } T_END; + mail_user_deinit(&ctx->cur_mail_user); + /* user deinit may still do some work, so finish the reason after it */ + event_reason_end(&reason); + + mail_storage_service_user_unref(&ctx->cur_service_user); + return 1; +} + +static void sig_die(const siginfo_t *si, void *context ATTR_UNUSED) +{ + killed_signo = si->si_signo; +} + +int doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, + const char **error_r) +{ + const struct doveadm_cmd_context *cctx = ctx->cctx; + + i_assert(cctx->username != NULL); + + doveadm_cctx_to_storage_service_input(cctx, &ctx->storage_service_input); + ctx->storage_service = mail_storage_service_init(master_service, NULL, + ctx->service_flags); + ctx->v.init(ctx, ctx->args); + if (hook_doveadm_mail_init != NULL) + hook_doveadm_mail_init(ctx); + + lib_signals_set_handler(SIGINT, 0, sig_die, NULL); + lib_signals_set_handler(SIGTERM, 0, sig_die, NULL); + + return doveadm_mail_next_user(ctx, error_r); +} + +static void +doveadm_mail_all_users(struct doveadm_mail_cmd_context *ctx, + const char *wildcard_user) +{ + struct doveadm_cmd_context *cctx = ctx->cctx; + unsigned int user_idx; + const char *ip, *user, *error; + int ret; + + ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; + + doveadm_cctx_to_storage_service_input(cctx, &ctx->storage_service_input); + ctx->storage_service = mail_storage_service_init(master_service, NULL, + ctx->service_flags); + lib_signals_set_handler(SIGINT, 0, sig_die, NULL); + lib_signals_set_handler(SIGTERM, 0, sig_die, NULL); + + ctx->v.init(ctx, ctx->args); + + mail_storage_service_all_init_mask(ctx->storage_service, + wildcard_user != NULL ? wildcard_user : ""); + + if (hook_doveadm_mail_init != NULL) + hook_doveadm_mail_init(ctx); + + user_idx = 0; + while ((ret = ctx->v.get_next_user(ctx, &user)) > 0) { + if (wildcard_user != NULL) { + if (!wildcard_match_icase(user, wildcard_user)) + continue; + } + cctx->username = user; + doveadm_print_sticky("username", user); + T_BEGIN { + ret = doveadm_mail_next_user(ctx, &error); + if (ret < 0) + i_error("%s", error); + else if (ret == 0) + i_info("User no longer exists, skipping"); + } T_END; + if (ret == -1) + break; + if (doveadm_verbose) { + if (++user_idx % 100 == 0) { + printf("\r%d", user_idx); + fflush(stdout); + } + } + if (killed_signo != 0) { + i_warning("Killed with signal %d", killed_signo); + ret = -1; + break; + } + } + if (doveadm_verbose) + printf("\n"); + ip = net_ip2addr(&cctx->remote_ip); + if (ip[0] == '\0') + i_set_failure_prefix("doveadm: "); + else + i_set_failure_prefix("doveadm(%s): ", ip); + if (ret < 0) { + i_error("Failed to iterate through some users"); + ctx->exit_code = EX_TEMPFAIL; + } +} + +static void +doveadm_mail_cmd_init_noop(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED, + const char *const args[] ATTR_UNUSED) +{ +} + +static int +doveadm_mail_cmd_get_next_user(struct doveadm_mail_cmd_context *ctx, + const char **username_r) +{ + if (ctx->users_list_input == NULL) + return mail_storage_service_all_next(ctx->storage_service, username_r); + + *username_r = i_stream_read_next_line(ctx->users_list_input); + if (ctx->users_list_input->stream_errno != 0) { + i_error("read(%s) failed: %s", + i_stream_get_name(ctx->users_list_input), + i_stream_get_error(ctx->users_list_input)); + return -1; + } + return *username_r != NULL ? 1 : 0; +} + +static void +doveadm_mail_cmd_deinit_noop(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED) +{ +} + +struct doveadm_mail_cmd_context * +doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd, + const struct doveadm_settings *set) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = cmd->alloc(); + ctx->set = set; + ctx->cmd = cmd; + if (ctx->v.init == NULL) + ctx->v.init = doveadm_mail_cmd_init_noop; + if (ctx->v.get_next_user == NULL) + ctx->v.get_next_user = doveadm_mail_cmd_get_next_user; + if (ctx->v.deinit == NULL) + ctx->v.deinit = doveadm_mail_cmd_deinit_noop; + + p_array_init(&ctx->module_contexts, ctx->pool, 5); + return ctx; +} + +static struct doveadm_mail_cmd_context * +doveadm_mail_cmdline_init(const struct doveadm_mail_cmd *cmd) +{ + struct doveadm_mail_cmd_context *ctx; + + ctx = doveadm_mail_cmd_init(cmd, doveadm_settings); + ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT; + if (doveadm_debug) + ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG; + return ctx; +} + +static void +doveadm_mail_cmd_exec(struct doveadm_mail_cmd_context *ctx, + const char *wildcard_user) +{ + const struct doveadm_cmd_context *cctx = ctx->cctx; + bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI); + int ret; + const char *error; + + if (ctx->v.preinit != NULL) + ctx->v.preinit(ctx); + + ctx->iterate_single_user = + !ctx->iterate_all_users && wildcard_user == NULL; + if (doveadm_print_is_initialized() && + (!ctx->iterate_single_user || ctx->add_username_header)) { + doveadm_print_header("username", "Username", + DOVEADM_PRINT_HEADER_FLAG_STICKY | + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + } + + if (ctx->iterate_single_user) { + if (cctx->username == NULL) + i_fatal_status(EX_USAGE, "USER environment is missing and -u option not used"); + if (!cli) { + /* we may access multiple users */ + ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP; + } + + if (ctx->add_username_header) + doveadm_print_sticky("username", cctx->username); + ret = doveadm_mail_single_user(ctx, &error); + if (ret < 0) { + /* user lookup/init failed somehow */ + doveadm_exit_code = EX_TEMPFAIL; + i_error("%s", error); + } else if (ret == 0) { + doveadm_exit_code = EX_NOUSER; + i_error("User doesn't exist"); + } + } else { + ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP; + doveadm_mail_all_users(ctx, wildcard_user); + } + doveadm_mail_server_flush(); + doveadm_mail_cmd_deinit(ctx); + doveadm_print_flush(); + + /* service deinit unloads mail plugins, so do it late */ + mail_storage_service_deinit(&ctx->storage_service); + + if (ctx->exit_code != 0) + doveadm_exit_code = ctx->exit_code; +} + +void doveadm_mail_cmd_deinit(struct doveadm_mail_cmd_context *ctx) +{ + ctx->v.deinit(ctx); + if (ctx->search_args != NULL) + mail_search_args_unref(&ctx->search_args); +} + +void doveadm_mail_cmd_free(struct doveadm_mail_cmd_context *ctx) +{ + i_stream_unref(&ctx->users_list_input); + i_stream_unref(&ctx->cmd_input); + pool_unref(&ctx->pool); +} + +void doveadm_mail_help(const struct doveadm_mail_cmd *cmd) +{ + fprintf(stderr, "doveadm %s "DOVEADM_CMD_MAIL_USAGE_PREFIX" %s\n", + cmd->name, cmd->usage_args == NULL ? "" : cmd->usage_args); + lib_exit(EX_USAGE); +} + +void doveadm_mail_try_help_name(const char *cmd_name) +{ + const struct doveadm_cmd_ver2 *cmd2; + + cmd2 = doveadm_cmd_find_ver2(cmd_name); + if (cmd2 != NULL) + help_ver2(cmd2); +} + +void doveadm_mail_help_name(const char *cmd_name) +{ + doveadm_mail_try_help_name(cmd_name); + i_fatal("Missing help for command %s", cmd_name); +} + +static struct doveadm_cmd_ver2 doveadm_cmd_force_resync_ver2 = { + .name = "force-resync", + .mail_cmd = cmd_force_resync_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-f] <mailbox mask>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAM('f', "fsck", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +static struct doveadm_cmd_ver2 doveadm_cmd_purge_ver2 = { + .name = "purge", + .mail_cmd = cmd_purge_alloc, + .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_MAIL_COMMON +DOVEADM_CMD_PARAMS_END +}; + +static struct doveadm_cmd_ver2 *mail_commands_ver2[] = { + &doveadm_cmd_batch, + &doveadm_cmd_dsync_backup, + &doveadm_cmd_dsync_mirror, + &doveadm_cmd_dsync_server, + &doveadm_cmd_mailbox_metadata_set_ver2, + &doveadm_cmd_mailbox_metadata_unset_ver2, + &doveadm_cmd_mailbox_metadata_get_ver2, + &doveadm_cmd_mailbox_metadata_list_ver2, + &doveadm_cmd_mailbox_status_ver2, + &doveadm_cmd_mailbox_list_ver2, + &doveadm_cmd_mailbox_create_ver2, + &doveadm_cmd_mailbox_delete_ver2, + &doveadm_cmd_mailbox_rename_ver2, + &doveadm_cmd_mailbox_subscribe_ver2, + &doveadm_cmd_mailbox_unsubscribe_ver2, + &doveadm_cmd_mailbox_update_ver2, + &doveadm_cmd_mailbox_path_ver2, + &doveadm_cmd_fetch_ver2, + &doveadm_cmd_save_ver2, + &doveadm_cmd_index_ver2, + &doveadm_cmd_altmove_ver2, + &doveadm_cmd_deduplicate_ver2, + &doveadm_cmd_expunge_ver2, + &doveadm_cmd_flags_add_ver2, + &doveadm_cmd_flags_remove_ver2, + &doveadm_cmd_flags_replace_ver2, + &doveadm_cmd_import_ver2, + &doveadm_cmd_force_resync_ver2, + &doveadm_cmd_purge_ver2, + &doveadm_cmd_search_ver2, + &doveadm_cmd_copy_ver2, + &doveadm_cmd_move_ver2, + &doveadm_cmd_mailbox_cache_decision, + &doveadm_cmd_mailbox_cache_remove, + &doveadm_cmd_mailbox_cache_purge, + &doveadm_cmd_rebuild_attachments, +}; + +void doveadm_mail_init(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(mail_commands_ver2); i++) + doveadm_cmd_register_ver2(mail_commands_ver2[i]); +} + +void doveadm_mail_init_finish(void) +{ + struct module_dir_load_settings mod_set; + + i_zero(&mod_set); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.require_init_funcs = TRUE; + mod_set.debug = doveadm_debug; + mod_set.binary_name = "doveadm"; + + /* load all configured mail plugins */ + mail_storage_service_modules = + module_dir_load_missing(mail_storage_service_modules, + doveadm_settings->mail_plugin_dir, + doveadm_settings->mail_plugins, + &mod_set); + /* keep mail_storage_init() referenced so that its _deinit() doesn't + try to free doveadm plugins' hooks too early. */ + mail_storage_init(); +} + +void doveadm_mail_deinit(void) +{ + mail_storage_deinit(); + module_dir_unload(&mail_storage_service_modules); +} + +static int doveadm_cmd_parse_arg(struct doveadm_mail_cmd_context *mctx, + const struct doveadm_cmd_param *arg, + ARRAY_TYPE(const_string) *full_args) +{ + const char *short_opt_str = + p_strdup_printf(mctx->pool, "-%c", arg->short_opt); + const char *arg_value = NULL; + + switch (arg->type) { + case CMD_PARAM_BOOL: + break; + case CMD_PARAM_INT64: + arg_value = dec2str(arg->value.v_int64); + break; + case CMD_PARAM_IP: + arg_value = net_ip2addr(&arg->value.v_ip); + break; + case CMD_PARAM_STR: + arg_value = arg->value.v_string; + break; + case CMD_PARAM_ARRAY: { + const char *str; + + array_foreach_elem(&arg->value.v_array, str) { + optarg = (char *)str; + if (!mctx->v.parse_arg(mctx, arg->short_opt)) + return -1; + array_push_back(full_args, &short_opt_str); + array_push_back(full_args, &str); + } + return 0; + } + default: + i_panic("Cannot convert parameter %s to short opt", arg->name); + } + + optarg = (char *)arg_value; + if (!mctx->v.parse_arg(mctx, arg->short_opt)) + return -1; + + array_push_back(full_args, &short_opt_str); + if (arg_value != NULL) + array_push_back(full_args, &arg_value); + return 0; +} + +void +doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx) +{ + struct doveadm_mail_cmd_context *mctx; + const char *wildcard_user; + const char *fieldstr; + ARRAY_TYPE(const_string) pargv, full_args; + int i; + bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI); + bool tcp_server = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_TCP); + struct doveadm_mail_cmd mail_cmd = { + cctx->cmd->mail_cmd, cctx->cmd->name, cctx->cmd->usage + }; + + if (!cli) { + mctx = doveadm_mail_cmd_init(&mail_cmd, doveadm_settings); + /* doveadm-server always does userdb lookups */ + mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; + } else { + mctx = doveadm_mail_cmdline_init(&mail_cmd); + } + mctx->cctx = cctx; + mctx->iterate_all_users = FALSE; + wildcard_user = NULL; + p_array_init(&full_args, mctx->pool, 8); + p_array_init(&pargv, mctx->pool, 8); + + for(i=0;i<cctx->argc;i++) { + const struct doveadm_cmd_param *arg = &cctx->argv[i]; + + if (!arg->value_set) + continue; + + if (strcmp(arg->name, "all-users") == 0) { + if (tcp_server) + mctx->add_username_header = TRUE; + else + mctx->iterate_all_users = arg->value.v_bool; + fieldstr = "-A"; + array_push_back(&full_args, &fieldstr); + } else if (strcmp(arg->name, "socket-path") == 0) { + doveadm_settings->doveadm_socket_path = arg->value.v_string; + if (doveadm_settings->doveadm_worker_count == 0) + doveadm_settings->doveadm_worker_count = 1; + } else if (strcmp(arg->name, "user") == 0) { + mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; + if (!tcp_server) + cctx->username = arg->value.v_string; + + fieldstr = "-u"; + array_push_back(&full_args, &fieldstr); + array_push_back(&full_args, &arg->value.v_string); + if (strchr(arg->value.v_string, '*') != NULL || + strchr(arg->value.v_string, '?') != NULL) { + if (tcp_server) + mctx->add_username_header = TRUE; + else { + wildcard_user = arg->value.v_string; + cctx->username = NULL; + } + } + } else if (strcmp(arg->name, "user-file") == 0) { + mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP; + wildcard_user = "*"; + mctx->users_list_input = arg->value.v_istream; + fieldstr = "-F"; + array_push_back(&full_args, &fieldstr); + fieldstr = ""; /* value doesn't really matter */ + array_push_back(&full_args, &fieldstr); + i_stream_ref(mctx->users_list_input); + } else if (strcmp(arg->name, "field") == 0 || + strcmp(arg->name, "flag") == 0) { + /* mailbox status, fetch, flags: convert an array into a + single space-separated parameter (alternative to + fieldstr) */ + fieldstr = p_array_const_string_join(mctx->pool, + &arg->value.v_array, " "); + array_push_back(&pargv, &fieldstr); + } else if (strcmp(arg->name, "file") == 0) { + /* input for doveadm_mail_get_input(), + used by e.g. save */ + if (mctx->cmd_input != NULL) { + i_error("Only one file input allowed: %s", arg->name); + doveadm_mail_cmd_free(mctx); + doveadm_exit_code = EX_USAGE; + return; + } + mctx->cmd_input = arg->value.v_istream; + i_stream_ref(mctx->cmd_input); + + } else if (strcmp(arg->name, "trans-flags") == 0) { + /* This parameter allows to set additional + * mailbox transaction flags. */ + mctx->transaction_flags = arg->value.v_int64; + + /* Keep all named special parameters above this line */ + + } else if (mctx->v.parse_arg != NULL && arg->short_opt != '\0') { + if (doveadm_cmd_parse_arg(mctx, arg, &full_args) < 0) { + i_error("Invalid parameter %c", arg->short_opt); + doveadm_mail_cmd_free(mctx); + doveadm_exit_code = EX_USAGE; + } + } else if ((arg->flags & CMD_PARAM_FLAG_POSITIONAL) != 0) { + /* feed this into pargv */ + if (arg->type == CMD_PARAM_ARRAY) + array_append_array(&pargv, &arg->value.v_array); + else if (arg->type == CMD_PARAM_STR) + array_push_back(&pargv, &arg->value.v_string); + } else { + doveadm_exit_code = EX_USAGE; + i_error("invalid parameter: %s", arg->name); + doveadm_mail_cmd_free(mctx); + return; + } + } + + const char *dashdash = "--"; + array_push_back(&full_args, &dashdash); + + array_append_zero(&pargv); + /* All the -parameters need to be included in full_args so that + they're sent to doveadm-server. */ + unsigned int args_pos = array_count(&full_args); + array_append_array(&full_args, &pargv); + + mctx->args = array_idx(&full_args, args_pos); + mctx->full_args = array_front(&full_args); + + doveadm_mail_cmd_exec(mctx, wildcard_user); + doveadm_mail_cmd_free(mctx); +} diff --git a/src/doveadm/doveadm-mail.h b/src/doveadm/doveadm-mail.h new file mode 100644 index 0000000..af5d0fa --- /dev/null +++ b/src/doveadm/doveadm-mail.h @@ -0,0 +1,206 @@ +#ifndef DOVEADM_MAIL_H +#define DOVEADM_MAIL_H + +#include <stdio.h> +#include "doveadm.h" +#include "doveadm-util.h" +#include "module-context.h" +#include "mail-error.h" +#include "mail-storage.h" +#include "mail-storage-service.h" + +struct mailbox; +struct mailbox_list; +struct mail_storage; +struct mail_user; +struct doveadm_mail_cmd_context; + +struct doveadm_mail_cmd_vfuncs { + /* Parse one getopt() parameter. This is called for each parameter. */ + bool (*parse_arg)(struct doveadm_mail_cmd_context *ctx, int c); + /* Usually not needed. The preinit() is called just after parsing all + parameters, but before any userdb lookups are done. This allows the + preinit() to alter the userdb lookup behavior (especially + service_flags). */ + void (*preinit)(struct doveadm_mail_cmd_context *ctx); + /* Initialize the command. Most importantly if the function prints + anything, this should initialize the headers. It shouldn't however + do any actual work. The init() is called also when doveadm is + performing the work via doveadm-server, which could be running + remotely with completely different Dovecot configuration. */ + void (*init)(struct doveadm_mail_cmd_context *ctx, + const char *const args[]); + /* Usually not needed. When iterating through multiple users, use this + function to get the next username. Overriding this is usually done + only when there's a known username filter, such as the expire + plugin. */ + int (*get_next_user)(struct doveadm_mail_cmd_context *ctx, + const char **username_r); + /* Usually not needed. This is called between + mail_storage_service_lookup() and mail_storage_service_next() for + each user. */ + int (*prerun)(struct doveadm_mail_cmd_context *ctx, + struct mail_storage_service_user *service_user, + const char **error_r); + /* This is the main function which performs all the work for the + command. This is called once per each user. */ + int (*run)(struct doveadm_mail_cmd_context *ctx, + struct mail_user *mail_user); + /* Deinitialize the command. Called once at the end - even if + preinit() or init() was never called. */ + void (*deinit)(struct doveadm_mail_cmd_context *ctx); +}; + +struct doveadm_mail_cmd_module_register { + unsigned int id; +}; + +union doveadm_mail_cmd_module_context { + struct doveadm_mail_cmd_vfuncs super; + struct doveadm_mail_cmd_module_register *reg; +}; + +struct doveadm_mail_cmd_context { + pool_t pool; + struct doveadm_cmd_context *cctx; + const struct doveadm_mail_cmd *cmd; + const char *const *args; + /* args including -options */ + const char *const *full_args; + + const char *getopt_args; + const struct doveadm_settings *set; + enum mail_storage_service_flags service_flags; + enum mailbox_transaction_flags transaction_flags; + struct mail_storage_service_ctx *storage_service; + struct mail_storage_service_input storage_service_input; + /* search args aren't set for all mail commands */ + struct mail_search_args *search_args; + struct istream *users_list_input; + + struct mail_storage_service_user *cur_service_user; + struct mail_user *cur_mail_user; + struct doveadm_mail_cmd_vfuncs v; + + struct istream *cmd_input; + int cmd_input_fd; + + ARRAY(union doveadm_mail_cmd_module_context *) module_contexts; + + /* if non-zero, exit with this code */ + int exit_code; + + /* This command is being called by a remote doveadm client. */ + bool proxying:1; + /* We're handling only a single user */ + bool iterate_single_user:1; + /* We're going through all users (not set for wildcard usernames) */ + bool iterate_all_users:1; + /* Add username header to all replies */ + bool add_username_header:1; +}; + +struct doveadm_mail_cmd { + struct doveadm_mail_cmd_context *(*alloc)(void); + const char *name; + const char *usage_args; +}; + +extern void (*hook_doveadm_mail_init)(struct doveadm_mail_cmd_context *ctx); +extern struct doveadm_mail_cmd_module_register doveadm_mail_cmd_module_register; +extern char doveadm_mail_cmd_hide; + +bool doveadm_is_killed(void); +int doveadm_killed_signo(void); + +void doveadm_mail_help(const struct doveadm_mail_cmd *cmd) ATTR_NORETURN; +void doveadm_mail_help_name(const char *cmd_name) ATTR_NORETURN; +void doveadm_mail_try_help_name(const char *cmd_name); + +void doveadm_mail_init(void); +void doveadm_mail_init_finish(void); +void doveadm_mail_deinit(void); + +struct doveadm_mail_cmd_context * +doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd, + const struct doveadm_settings *set); +void doveadm_mail_cmd_deinit(struct doveadm_mail_cmd_context *ctx); +void doveadm_mail_cmd_free(struct doveadm_mail_cmd_context *ctx); +int doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx, + const char **error_r); +int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx, + const struct mail_storage_service_input *input, + const char **error_r); +void doveadm_mail_server_flush(void); + +/* Request input stream to be read (from stdin). This must be called from + the command's init() function. */ +void doveadm_mail_get_input(struct doveadm_mail_cmd_context *ctx); + +struct mailbox * +doveadm_mailbox_find(struct mail_user *user, const char *mailbox); +struct mail_search_args * +doveadm_mail_build_search_args(const char *const args[]); +void doveadm_mailbox_args_check(const char *const args[]); +struct mail_search_args * +doveadm_mail_mailbox_search_args_build(const char *const args[]); + +void expunge_search_args_check(struct mail_search_args *args, const char *cmd); + +struct doveadm_mail_cmd_context * +doveadm_mail_cmd_alloc_size(size_t size); +#define doveadm_mail_cmd_alloc(type) \ + (type *)doveadm_mail_cmd_alloc_size(sizeof(type)) + +void doveadm_mail_failed_error(struct doveadm_mail_cmd_context *ctx, + enum mail_error error); +void doveadm_mail_failed_storage(struct doveadm_mail_cmd_context *ctx, + struct mail_storage *storage); +void doveadm_mail_failed_mailbox(struct doveadm_mail_cmd_context *ctx, + struct mailbox *box); +void doveadm_mail_failed_list(struct doveadm_mail_cmd_context *ctx, + struct mailbox_list *list); + +extern struct doveadm_cmd_ver2 doveadm_cmd_batch; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_set_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_unset_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_get_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_list_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_status_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_list_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_create_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_rename_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_subscribe_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_unsubscribe_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_fetch_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_save_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_index_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_altmove_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_deduplicate_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_expunge_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_flags_add_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_flags_remove_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_flags_replace_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_import_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_search_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_copy_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_move_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_path_ver2; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_decision; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_remove; +extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_purge; +extern struct doveadm_cmd_ver2 doveadm_cmd_rebuild_attachments; + +#define DOVEADM_CMD_MAIL_COMMON \ +DOVEADM_CMD_PARAM('A', "all-users", CMD_PARAM_BOOL, 0) \ +DOVEADM_CMD_PARAM('S', "socket-path", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) \ +DOVEADM_CMD_PARAM('\0', "trans-flags", CMD_PARAM_INT64, 0) \ +DOVEADM_CMD_PARAM('F', "user-file", CMD_PARAM_ISTREAM, 0) + +#define DOVEADM_CMD_MAIL_USAGE_PREFIX \ + "[-u <user>|-A] [-S <socket_path>] " + +#endif diff --git a/src/doveadm/doveadm-mailbox-list-iter.c b/src/doveadm/doveadm-mailbox-list-iter.c new file mode 100644 index 0000000..c77398c --- /dev/null +++ b/src/doveadm/doveadm-mailbox-list-iter.c @@ -0,0 +1,195 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "mail-search.h" +#include "mail-namespace.h" +#include "mailbox-list.h" +#include "doveadm-mail.h" +#include "doveadm-mailbox-list-iter.h" + +struct doveadm_mailbox_list_iter { + struct mail_user *user; + struct doveadm_mail_cmd_context *ctx; + struct mail_search_args *search_args; + enum mailbox_list_iter_flags iter_flags; + + struct mailbox_list_iterate_context *iter; + + struct mailbox_info info; + ARRAY_TYPE(const_string) patterns; + unsigned int pattern_idx; + + bool only_selectable; +}; + +static bool +search_args_get_mailbox_patterns(const struct mail_search_arg *args, + ARRAY_TYPE(const_string) *patterns, + bool *have_guid, bool *have_wildcards) +{ + const struct mail_search_arg *subargs; + + for (; args != NULL; args = args->next) { + switch (args->type) { + case SEARCH_OR: + /* we don't currently try to optimize OR. */ + break; + case SEARCH_SUB: + case SEARCH_INTHREAD: + subargs = args->value.subargs; + for (; subargs != NULL; subargs = subargs->next) { + if (!search_args_get_mailbox_patterns(subargs, + patterns, have_guid, + have_wildcards)) + return FALSE; + } + break; + case SEARCH_MAILBOX_GLOB: + *have_wildcards = TRUE; + /* fall through */ + case SEARCH_MAILBOX: + if (args->match_not) { + array_clear(patterns); + return FALSE; + } + array_push_back(patterns, &args->value.str); + break; + case SEARCH_MAILBOX_GUID: + *have_guid = TRUE; + break; + default: + break; + } + } + return TRUE; +} + +static struct doveadm_mailbox_list_iter * +doveadm_mailbox_list_iter_init_nsmask(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user, + struct mail_search_args *search_args, + enum mailbox_list_iter_flags iter_flags, + enum mail_namespace_type ns_mask) +{ + static const char *all_pattern = "*"; + struct doveadm_mailbox_list_iter *iter; + bool have_guid = FALSE, have_wildcards = FALSE; + + iter = i_new(struct doveadm_mailbox_list_iter, 1); + iter->ctx = ctx; + iter->search_args = search_args; + iter->user = user; + i_array_init(&iter->patterns, 16); + (void)search_args_get_mailbox_patterns(search_args->args, + &iter->patterns, + &have_guid, &have_wildcards); + + if (array_count(&iter->patterns) == 0) { + iter_flags |= MAILBOX_LIST_ITER_SKIP_ALIASES; + if (have_guid) { + ns_mask |= MAIL_NAMESPACE_TYPE_SHARED | + MAIL_NAMESPACE_TYPE_PUBLIC; + } + array_push_back(&iter->patterns, &all_pattern); + } else if (have_wildcards) { + iter_flags |= MAILBOX_LIST_ITER_STAR_WITHIN_NS; + ns_mask |= MAIL_NAMESPACE_TYPE_SHARED | + MAIL_NAMESPACE_TYPE_PUBLIC; + } else { + /* just return the listed mailboxes without actually + iterating through. this also allows accessing mailboxes + without lookup ACL right */ + return iter; + } + array_append_zero(&iter->patterns); + + iter->only_selectable = TRUE; + iter->iter_flags = iter_flags; + iter->iter = mailbox_list_iter_init_namespaces(user->namespaces, + array_front(&iter->patterns), + ns_mask, iter_flags); + return iter; +} + +struct doveadm_mailbox_list_iter * +doveadm_mailbox_list_iter_init(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user, + struct mail_search_args *search_args, + enum mailbox_list_iter_flags iter_flags) +{ + enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_PRIVATE; + + return doveadm_mailbox_list_iter_init_nsmask(ctx, user, search_args, + iter_flags, ns_mask); +} + +struct doveadm_mailbox_list_iter * +doveadm_mailbox_list_iter_full_init(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user, + struct mail_search_args *search_args, + enum mailbox_list_iter_flags iter_flags) +{ + enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL; + struct doveadm_mailbox_list_iter *iter; + + iter = doveadm_mailbox_list_iter_init_nsmask(ctx, user, search_args, + iter_flags, ns_mask); + iter->only_selectable = FALSE; + return iter; +} + +int doveadm_mailbox_list_iter_deinit(struct doveadm_mailbox_list_iter **_iter) +{ + struct doveadm_mailbox_list_iter *iter = *_iter; + enum mail_error error; + int ret; + + *_iter = NULL; + + if (iter->iter == NULL) + ret = 0; + else if ((ret = mailbox_list_iter_deinit(&iter->iter)) < 0) { + i_error("Listing mailboxes failed: %s", + mailbox_list_get_last_internal_error(iter->user->namespaces->list, &error)); + doveadm_mail_failed_error(iter->ctx, error); + } + array_free(&iter->patterns); + i_free(iter); + return ret; +} + +const struct mailbox_info * +doveadm_mailbox_list_iter_next(struct doveadm_mailbox_list_iter *iter) +{ + const struct mailbox_info *info; + const char *const *patterns; + unsigned int count; + + while (iter->iter == NULL) { + patterns = array_get(&iter->patterns, &count); + if (iter->pattern_idx == count) + return NULL; + + iter->info.vname = patterns[iter->pattern_idx++]; + iter->info.ns = mail_namespace_find(iter->user->namespaces, + iter->info.vname); + return &iter->info; + } + + while ((info = mailbox_list_iter_next(iter->iter)) != NULL) { + char sep = mail_namespace_get_sep(info->ns); + + if ((info->flags & (MAILBOX_NOSELECT | + MAILBOX_NONEXISTENT)) != 0) { + if (iter->only_selectable) + continue; + } + + if (mail_search_args_match_mailbox(iter->search_args, + info->vname, sep)) + break; + } + return info; +} diff --git a/src/doveadm/doveadm-mailbox-list-iter.h b/src/doveadm/doveadm-mailbox-list-iter.h new file mode 100644 index 0000000..9b205e2 --- /dev/null +++ b/src/doveadm/doveadm-mailbox-list-iter.h @@ -0,0 +1,25 @@ +#ifndef DOVEADM_MAILBOX_LIST_ITER_H +#define DOVEADM_MAILBOX_LIST_ITER_H + +#include "mailbox-list-iter.h" + +struct doveadm_mail_cmd_context; + +/* List only selectable mailboxes */ +struct doveadm_mailbox_list_iter * +doveadm_mailbox_list_iter_init(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user, + struct mail_search_args *search_args, + enum mailbox_list_iter_flags iter_flags); +/* List all mailboxes */ +struct doveadm_mailbox_list_iter * +doveadm_mailbox_list_iter_full_init(struct doveadm_mail_cmd_context *ctx, + struct mail_user *user, + struct mail_search_args *search_args, + enum mailbox_list_iter_flags iter_flags); +int doveadm_mailbox_list_iter_deinit(struct doveadm_mailbox_list_iter **iter); + +const struct mailbox_info * +doveadm_mailbox_list_iter_next(struct doveadm_mailbox_list_iter *iter); + +#endif diff --git a/src/doveadm/doveadm-master.c b/src/doveadm/doveadm-master.c new file mode 100644 index 0000000..2f161d7 --- /dev/null +++ b/src/doveadm/doveadm-master.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strescape.h" +#include "istream.h" +#include "write-full.h" +#include "master-service.h" +#include "sleep.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <unistd.h> +#include <fcntl.h> +#include <signal.h> + +#define MASTER_PID_FILE_NAME "master.pid" + +static bool pid_file_read(const char *path, pid_t *pid_r) +{ + char buf[32]; + int fd; + ssize_t ret; + bool found; + + fd = open(path, O_RDONLY); + if (fd == -1) { + if (errno == ENOENT) + return FALSE; + i_fatal("open(%s) failed: %m", path); + } + + ret = read(fd, buf, sizeof(buf)-1); + if (ret <= 0) { + if (ret == 0) + i_error("Empty PID file in %s", path); + else + i_fatal("read(%s) failed: %m", path); + found = FALSE; + } else { + if (buf[ret-1] == '\n') + ret--; + buf[ret] = '\0'; + if (str_to_pid(buf, pid_r) < 0) + found = FALSE; + else { + found = !(*pid_r == getpid() || + (kill(*pid_r, 0) < 0 && errno == ESRCH)); + } + } + i_close_fd(&fd); + return found; +} + +void doveadm_master_send_signal(int signo) +{ + const char *pidfile_path; + unsigned int i; + pid_t pid; + + pidfile_path = t_strconcat(doveadm_settings->base_dir, + "/"MASTER_PID_FILE_NAME, NULL); + + if (!pid_file_read(pidfile_path, &pid)) + i_fatal("Dovecot is not running (read from %s)", pidfile_path); + + if (kill(pid, signo) < 0) + i_fatal("kill(%s, %d) failed: %m", dec2str(pid), signo); + + if (signo == SIGTERM) { + /* wait for a while for the process to die */ + i_sleep_msecs(1); + for (i = 0; i < 30; i++) { + if (kill(pid, 0) < 0) { + if (errno != ESRCH) + i_error("kill() failed: %m"); + break; + } + i_sleep_msecs(100); + } + } +} + +static void cmd_stop(struct doveadm_cmd_context *cctx ATTR_UNUSED) +{ + doveadm_master_send_signal(SIGTERM); +} + +static void cmd_reload(struct doveadm_cmd_context *cctx ATTR_UNUSED) +{ + doveadm_master_send_signal(SIGHUP); +} + +static struct istream *master_service_send_cmd(const char *cmd) +{ + struct istream *input; + const char *path, *line; + + path = t_strconcat(doveadm_settings->base_dir, "/master", NULL); + int fd = net_connect_unix(path); + if (fd == -1) + i_fatal("net_connect_unix(%s) failed: %m", path); + net_set_nonblock(fd, FALSE); + + const char *str = + t_strdup_printf("VERSION\tmaster-client\t1\t0\n%s\n", cmd); + if (write_full(fd, str, strlen(str)) < 0) + i_fatal("write(%s) failed: %m", path); + + input = i_stream_create_fd_autoclose(&fd, IO_BLOCK_SIZE); + alarm(5); + if ((line = i_stream_read_next_line(input)) == NULL) + i_fatal("read(%s) failed: %m", path); + if (!version_string_verify(line, "master-server", 1)) { + i_fatal_status(EX_PROTOCOL, + "%s is not a compatible master socket", path); + } + alarm(0); + return input; +} + +static struct istream * +master_service_send_cmd_with_args(const char *cmd, const char *const *args) +{ + string_t *str = t_str_new(128); + + str_append(str, cmd); + if (args != NULL) { + for (unsigned int i = 0; args[i] != NULL; i++) { + str_append_c(str, '\t'); + str_append_tabescaped(str, args[i]); + } + } + return master_service_send_cmd(str_c(str)); +} + +static void cmd_service_stop(struct doveadm_cmd_context *cctx) +{ + const char *line, *const *services; + + if (!doveadm_cmd_param_array(cctx, "service", &services)) + i_fatal("service parameter missing"); + + struct istream *input = + master_service_send_cmd_with_args("STOP", services); + + alarm(5); + if ((line = i_stream_read_next_line(input)) == NULL) { + i_error("read(%s) failed: %s", i_stream_get_name(input), + i_stream_get_error(input)); + doveadm_exit_code = EX_TEMPFAIL; + } else if (line[0] == '-') { + doveadm_exit_code = DOVEADM_EX_NOTFOUND; + i_error("%s", line+1); + } else if (line[0] != '+') { + i_error("Unexpected input from %s: %s", + i_stream_get_name(input), line); + doveadm_exit_code = EX_TEMPFAIL; + } + alarm(0); + i_stream_destroy(&input); +} + +static void cmd_service_status(struct doveadm_cmd_context *cctx) +{ + const char *line, *const *services; + unsigned int fields_count; + + if (!doveadm_cmd_param_array(cctx, "service", &services)) + services = NULL; + + struct istream *input = master_service_send_cmd("SERVICE-STATUS"); + + doveadm_print_init(DOVEADM_PRINT_TYPE_PAGER); + doveadm_print_header_simple("name"); + doveadm_print_header_simple("process_count"); + doveadm_print_header_simple("process_avail"); + doveadm_print_header_simple("process_limit"); + doveadm_print_header_simple("client_limit"); + doveadm_print_header_simple("throttle_secs"); + doveadm_print_header_simple("exit_failure_last"); + doveadm_print_header_simple("exit_failures_in_sec"); + doveadm_print_header_simple("last_drop_warning"); + doveadm_print_header_simple("listen_pending"); + doveadm_print_header_simple("listening"); + doveadm_print_header_simple("doveadm_stop"); + doveadm_print_header_simple("process_total"); + fields_count = doveadm_print_get_headers_count(); + + alarm(5); + while ((line = i_stream_read_next_line(input)) != NULL) { + if (line[0] == '\0') + break; + T_BEGIN { + const char *const *args = t_strsplit_tabescaped(line); + if (args[0] != NULL && + (services == NULL || + str_array_find(services, args[0]))) { + unsigned int i; + for (i = 0; i < fields_count && args[i] != NULL; i++) + doveadm_print(args[i]); + for (; i < fields_count; i++) + doveadm_print(""); + } + } T_END; + } + if (line == NULL) { + i_error("read(%s) failed: %s", i_stream_get_name(input), + i_stream_get_error(input)); + doveadm_exit_code = EX_TEMPFAIL; + } + alarm(0); + i_stream_destroy(&input); +} + +static void cmd_process_status(struct doveadm_cmd_context *cctx) +{ + const char *line, *const *services; + + if (!doveadm_cmd_param_array(cctx, "service", &services)) + services = NULL; + + struct istream *input = + master_service_send_cmd_with_args("PROCESS-STATUS", services); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header_simple("name"); + doveadm_print_header_simple("pid"); + doveadm_print_header_simple("available_count"); + doveadm_print_header_simple("total_count"); + doveadm_print_header_simple("idle_start"); + doveadm_print_header_simple("last_status_update"); + doveadm_print_header_simple("last_kill_sent"); + + alarm(5); + while ((line = i_stream_read_next_line(input)) != NULL) { + if (line[0] == '\0') + break; + T_BEGIN { + const char *const *args = t_strsplit_tabescaped(line); + if (str_array_length(args) >= 7) { + for (unsigned int i = 0; i < 7; i++) + doveadm_print(args[i]); + } + } T_END; + } + if (line == NULL) { + i_error("read(%s) failed: %s", i_stream_get_name(input), + i_stream_get_error(input)); + doveadm_exit_code = EX_TEMPFAIL; + } + alarm(0); + i_stream_destroy(&input); +} + +struct doveadm_cmd_ver2 doveadm_cmd_stop_ver2 = { + .cmd = cmd_stop, + .name = "stop", + .usage = "", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_reload_ver2 = { + .cmd = cmd_reload, + .name = "reload", + .usage = "", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2 = { + .cmd = cmd_service_stop, + .name = "service stop", + .usage = "<service> [<service> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_service_status_ver2 = { + .cmd = cmd_service_status, + .name = "service status", + .usage = "[<service> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_process_status_ver2 = { + .cmd = cmd_process_status, + .name = "process status", + .usage = "[<service> [...]]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-mutf7.c b/src/doveadm/doveadm-mutf7.c new file mode 100644 index 0000000..3f788f5 --- /dev/null +++ b/src/doveadm/doveadm-mutf7.c @@ -0,0 +1,56 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "imap-utf7.h" +#include "doveadm.h" + +#include <stdio.h> +#include <unistd.h> + +static void cmd_mailbox_mutf7(struct doveadm_cmd_context *cctx) +{ + string_t *str; + const char *const *names; + bool from_utf8, to_utf8; + unsigned int i; + + if (!doveadm_cmd_param_array(cctx, "name", &names)) + help_ver2(&doveadm_cmd_mailbox_mutf7); + if (!doveadm_cmd_param_bool(cctx, "from-utf8", &from_utf8)) { + if (!doveadm_cmd_param_bool(cctx, "to-utf8", &to_utf8)) + from_utf8 = TRUE; + else + from_utf8 = !to_utf8; + } + + str = t_str_new(128); + for (i = 0; names[i] != NULL; i++) { + str_truncate(str, 0); + if (from_utf8) { + if (imap_utf8_to_utf7(names[i], str) < 0) { + i_error("Mailbox name not valid UTF-8: %s", + names[i]); + doveadm_exit_code = EX_DATAERR; + } + } else { + if (imap_utf7_to_utf8(names[i], str) < 0) { + i_error("Mailbox name not valid mUTF-7: %s", + names[i]); + doveadm_exit_code = EX_DATAERR; + } + } + printf("%s\n", str_c(str)); + } +} + +struct doveadm_cmd_ver2 doveadm_cmd_mailbox_mutf7 = { + .name = "mailbox mutf7", + .cmd = cmd_mailbox_mutf7, + .usage = "[-7|-8] <name> [...]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('7', "to-utf8", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('8', "from-utf8", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-oldstats.c b/src/doveadm/doveadm-oldstats.c new file mode 100644 index 0000000..4be575e --- /dev/null +++ b/src/doveadm/doveadm-oldstats.c @@ -0,0 +1,604 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "net.h" +#include "ioloop.h" +#include "istream.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "strescape.h" +#include "write-full.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <termios.h> + +struct top_line { + char *id; + /* [headers_count] */ + const char **prev_values, **cur_values; + + bool flip:1; +}; + +struct top_context { + const char *path; + int fd; + struct istream *input; + const char *sort_type; + + char **headers; + unsigned int headers_count; + + pool_t prev_pool, cur_pool; + /* id => struct top_line. */ + HASH_TABLE(char *, struct top_line *) sessions; + ARRAY(struct top_line *) lines; + int (*lines_sort)(struct top_line *const *, struct top_line *const *); + + unsigned int last_update_idx, user_idx; + unsigned int sort_idx1, sort_idx2; + + bool flip:1; +}; + +static struct top_context *sort_ctx = NULL; +static const char *disk_input_field = "disk_input"; +static const char *disk_output_field = "disk_output"; + +static char ** +p_read_next_line(pool_t pool, struct istream *input) +{ + const char *line; + + line = i_stream_read_next_line(input); + if (line == NULL) + return NULL; + + return p_strsplit_tabescaped(pool, line); +} + +static const char *const* +read_next_line(struct istream *input) +{ + return (const void *)p_read_next_line(pool_datastack_create(), input); +} + +static void stats_dump(const char *path, const char *cmd) +{ + struct istream *input; + const char *const *args; + unsigned int i; + int fd; + + fd = doveadm_connect(path); + net_set_nonblock(fd, FALSE); + if (write_full(fd, cmd, strlen(cmd)) < 0) + i_fatal("write(%s) failed: %m", path); + + input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + + /* read header */ + args = read_next_line(input); + if (args == NULL) + i_fatal("read(%s) unexpectedly disconnected", path); + if (*args == NULL) + i_info("no statistics available"); + else { + for (; *args != NULL; args++) + doveadm_print_header_simple(*args); + + /* read lines */ + do { + T_BEGIN { + args = read_next_line(input); + if (args != NULL && args[0] == NULL) + args = NULL; + if (args != NULL) { + for (i = 0; args[i] != NULL; i++) + doveadm_print(args[i]); + } + } T_END; + } while (args != NULL); + } + if (input->stream_errno != 0) + i_fatal("read(%s) failed: %s", path, i_stream_get_error(input)); + i_stream_destroy(&input); +} + +static void +doveadm_cmd_stats_dump(struct doveadm_cmd_context* cctx) +{ + const char *path, *cmd; + const char *args[3] = {0}; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &path)) + path = t_strconcat(doveadm_settings->base_dir, "/old-stats", NULL); + + if (!doveadm_cmd_param_str(cctx, "type", &args[0])) { + i_error("Missing type parameter"); + doveadm_exit_code = EX_USAGE; + return; + } + + /* purely optional */ + if (!doveadm_cmd_param_str(cctx, "filter", &args[1])) + args[1] = NULL; + + cmd = t_strdup_printf("EXPORT\t%s\n", t_strarray_join(args, "\t")); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); + stats_dump(path, cmd); + return; +} + +static void +stats_line_set_prev_values(struct top_context *ctx, + const struct top_line *old_line, + struct top_line *line) +{ + const char **values; + unsigned int i; + + if (old_line->prev_values == NULL || + strcmp(old_line->cur_values[ctx->last_update_idx], + line->cur_values[ctx->last_update_idx]) != 0) { + values = old_line->cur_values; + } else { + values = old_line->prev_values; + } + + /* line hasn't been updated, keep old values */ + line->prev_values = + p_new(ctx->cur_pool, const char *, ctx->headers_count); + for (i = 0; i < ctx->headers_count; i++) + line->prev_values[i] = p_strdup(ctx->cur_pool, values[i]); +} + +static void stats_read(struct top_context *ctx) +{ + struct top_line *old_line, *line; + unsigned int i; + char **args; + + /* read lines */ + while ((args = p_read_next_line(ctx->cur_pool, ctx->input)) != NULL) { + if (args[0] == NULL) { + /* end of stats */ + return; + } + if (str_array_length((void *)args) != ctx->headers_count) + i_fatal("read(%s): invalid stats line", ctx->path); + + line = p_new(ctx->cur_pool, struct top_line, 1); + line->id = args[0]; + line->flip = ctx->flip; + line->cur_values = p_new(ctx->cur_pool, const char *, ctx->headers_count); + for (i = 0; i < ctx->headers_count; i++) + line->cur_values[i] = args[i]; + + old_line = hash_table_lookup(ctx->sessions, line->id); + if (old_line != NULL) { + stats_line_set_prev_values(ctx, old_line, line); + array_push_back(&ctx->lines, &line); + } + hash_table_update(ctx->sessions, line->id, line); + } + + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->path, + i_stream_get_error(ctx->input)); + } + i_fatal("read(%s): unexpected EOF", ctx->path); +} + +static void stats_drop_stale(struct top_context *ctx) +{ + struct hash_iterate_context *iter; + char *id; + struct top_line *line; + + iter = hash_table_iterate_init(ctx->sessions); + while (hash_table_iterate(iter, ctx->sessions, &id, &line)) { + if (line->flip != ctx->flip) + hash_table_remove(ctx->sessions, id); + } + hash_table_iterate_deinit(&iter); +} + +static int get_double(const char *str, double *num_r) +{ + char *p; + + *num_r = strtod(str, &p); + return *p == '\0' ? 0 : -1; +} + +static double sort_cpu_diff(const struct top_line *line) +{ + double prev, cur, diff, prev_time, cur_time, time_multiplier; + + if (get_double(line->prev_values[sort_ctx->last_update_idx], &prev_time) < 0 || + get_double(line->cur_values[sort_ctx->last_update_idx], &cur_time) < 0) + i_fatal("sorting: invalid last_update value"); + time_multiplier = (cur_time - prev_time) * 100; + + if (get_double(line->prev_values[sort_ctx->sort_idx1], &prev) < 0 || + get_double(line->cur_values[sort_ctx->sort_idx1], &cur) < 0) + i_fatal("sorting: not a double"); + + diff = cur - prev; + + if (sort_ctx->sort_idx2 != 0) { + if (get_double(line->prev_values[sort_ctx->sort_idx2], &prev) < 0 || + get_double(line->cur_values[sort_ctx->sort_idx2], &cur) < 0) + i_fatal("sorting: not a double"); + diff += cur - prev; + } + return diff * time_multiplier; +} + +static int sort_cpu(struct top_line *const *l1, struct top_line *const *l2) +{ + double d1, d2; + + d1 = sort_cpu_diff(*l1); + d2 = sort_cpu_diff(*l2); + if (d1 < d2) + return -1; + if (d1 > d2) + return 1; + return strcmp((*l1)->cur_values[sort_ctx->user_idx], + (*l2)->cur_values[sort_ctx->user_idx]); +} + +static double sort_num_diff(const struct top_line *line) +{ + uint64_t prev, cur, diff; + + if (str_to_uint64(line->prev_values[sort_ctx->sort_idx1], &prev) < 0 || + str_to_uint64(line->cur_values[sort_ctx->sort_idx1], &cur) < 0) + i_fatal("sorting: not a number"); + diff = cur - prev; + + if (sort_ctx->sort_idx2 != 0) { + if (str_to_uint64(line->prev_values[sort_ctx->sort_idx2], &prev) < 0 || + str_to_uint64(line->cur_values[sort_ctx->sort_idx2], &cur) < 0) + i_fatal("sorting: not a number"); + diff += cur - prev; + } + return diff; +} + +static int sort_num(struct top_line *const *l1, struct top_line *const *l2) +{ + uint64_t n1, n2; + + n1 = sort_num_diff(*l1); + n2 = sort_num_diff(*l2); + if (n1 < n2) + return -1; + if (n1 > n2) + return 1; + return strcmp((*l1)->cur_values[sort_ctx->user_idx], + (*l2)->cur_values[sort_ctx->user_idx]); +} + +static bool +stats_header_find(struct top_context *ctx, const char *name, + unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; ctx->headers[i] != NULL; i++) { + if (strcmp(ctx->headers[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +static void stats_top_get_sorting(struct top_context *ctx) +{ + if (stats_header_find(ctx, ctx->sort_type, &ctx->sort_idx1)) + return; + + if (strcmp(ctx->sort_type, "cpu") == 0) { + if (!stats_header_find(ctx, "user_cpu", &ctx->sort_idx1) || + !stats_header_find(ctx, "sys_cpu", &ctx->sort_idx2)) + i_fatal("cpu sort type is missing fields"); + return; + } + if (strcmp(ctx->sort_type, "disk") == 0) { + if (!stats_header_find(ctx, disk_input_field, &ctx->sort_idx1) || + !stats_header_find(ctx, disk_output_field, &ctx->sort_idx2)) + i_fatal("disk sort type is missing fields"); + return; + } + i_fatal("unknown sort type: %s", ctx->sort_type); +} + +static bool stats_top_round(struct top_context *ctx) +{ +#define TOP_CMD "EXPORT\tsession\tconnected\n" + const char *const *args; + pool_t tmp_pool; + + if (write_full(ctx->fd, TOP_CMD, strlen(TOP_CMD)) < 0) + i_fatal("write(%s) failed: %m", ctx->path); + + /* read header */ + if (ctx->headers != NULL) { + args = read_next_line(ctx->input); + if (args == NULL) + i_fatal("read(%s) unexpectedly disconnected", ctx->path); + if (*args == NULL) + return TRUE; + if (str_array_length(args) != ctx->headers_count) + i_fatal("headers changed"); + } else { + ctx->headers = p_read_next_line(default_pool, ctx->input); + if (ctx->headers == NULL) + i_fatal("read(%s) unexpectedly disconnected", ctx->path); + if (*ctx->headers == NULL) { + i_free_and_null(ctx->headers); + return FALSE; + } + ctx->headers_count = str_array_length((void *)ctx->headers); + if (!stats_header_find(ctx, "last_update", &ctx->last_update_idx)) + i_fatal("last_update header missing"); + if (!stats_header_find(ctx, "user", &ctx->user_idx)) + i_fatal("user header missing"); + stats_top_get_sorting(ctx); + } + + array_clear(&ctx->lines); + p_clear(ctx->prev_pool); + tmp_pool = ctx->prev_pool; + ctx->prev_pool = ctx->cur_pool; + ctx->cur_pool = tmp_pool; + + ctx->flip = !ctx->flip; + stats_read(ctx); + stats_drop_stale(ctx); + + sort_ctx = ctx; + array_sort(&ctx->lines, *ctx->lines_sort); + sort_ctx = NULL; + return TRUE; +} + +static void +stats_top_output_diff(struct top_context *ctx, + const struct top_line *line, unsigned int i) +{ + uint64_t prev_num, cur_num; + double prev_double, cur_double, prev_time, cur_time; + char numstr[MAX_INT_STRLEN]; + + if (str_to_uint64(line->prev_values[i], &prev_num) == 0 && + str_to_uint64(line->cur_values[i], &cur_num) == 0) { + if (i_snprintf(numstr, sizeof(numstr), "%"PRIu64, + (cur_num - prev_num)) < 0) + i_unreached(); + doveadm_print(numstr); + } else if (get_double(line->prev_values[i], &prev_double) == 0 && + get_double(line->cur_values[i], &cur_double) == 0 && + get_double(line->prev_values[ctx->last_update_idx], &prev_time) == 0 && + get_double(line->cur_values[ctx->last_update_idx], &cur_time) == 0) { + /* %CPU */ + if (i_snprintf(numstr, sizeof(numstr), "%d", + (int)((cur_double - prev_double) * + (cur_time - prev_time) * 100)) < 0) + i_unreached(); + doveadm_print(numstr); + } else { + doveadm_print(line->cur_values[i]); + } +} + +static void stats_top_output(struct top_context *ctx) +{ + static const char *names[] = { + "user", "service", "user_cpu", "sys_cpu", + "", "" + }; + struct winsize ws; + struct top_line *const *lines; + unsigned int i, j, row, maxrow, count, indexes[N_ELEMENTS(names)]; + + names[4] = disk_input_field; + names[5] = disk_output_field; + + /* ANSI clear screen and move cursor to top of screen */ + printf("\x1b[2J\x1b[1;1H"); fflush(stdout); + doveadm_print_deinit(); + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + + doveadm_print_header("USER", "USER", DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("SERVICE"); + doveadm_print_header_simple("%CPU"); + doveadm_print_header_simple("%SYS"); + doveadm_print_header_simple("DISKIN"); + doveadm_print_header_simple("DISKOUT"); + + if (!stats_top_round(ctx)) { + /* no connections yet */ + return; + } + + for (i = 0; i < N_ELEMENTS(names); i++) { + if (!stats_header_find(ctx, names[i], &indexes[i])) + indexes[i] = UINT_MAX; + } + + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0) + ws.ws_row = 24; + maxrow = ws.ws_row-1; + + lines = array_get(&ctx->lines, &count); + for (i = 0, row = 1; row < maxrow && i < count; i++, row++) { + for (j = 0; j < N_ELEMENTS(names); j++) { + if (indexes[j] == UINT_MAX) + doveadm_print("?"); + else + stats_top_output_diff(ctx, lines[i], indexes[j]); + } + } +} + +static void stats_top_start(struct top_context *ctx) +{ + struct timeout *to; + + stats_top_output(ctx); + to = timeout_add(1000, stats_top_output, ctx); + io_loop_run(current_ioloop); + timeout_remove(&to); +} + +static void stats_top(const char *path, const char *sort_type) +{ + struct top_context ctx; + + i_zero(&ctx); + ctx.path = path; + ctx.fd = doveadm_connect(path); + ctx.prev_pool = pool_alloconly_create("stats top", 1024*16); + ctx.cur_pool = pool_alloconly_create("stats top", 1024*16); + i_array_init(&ctx.lines, 128); + hash_table_create(&ctx.sessions, default_pool, 0, str_hash, strcmp); + net_set_nonblock(ctx.fd, FALSE); + + ctx.input = i_stream_create_fd(ctx.fd, SIZE_MAX); + + if (strstr(sort_type, "cpu") != NULL) + ctx.lines_sort = sort_cpu; + else + ctx.lines_sort = sort_num; + ctx.sort_type = sort_type; + + stats_top_start(&ctx); + i_stream_destroy(&ctx.input); + hash_table_destroy(&ctx.sessions); + array_free(&ctx.lines); + pool_unref(&ctx.prev_pool); + pool_unref(&ctx.cur_pool); + i_close_fd(&ctx.fd); +} + +static void stats_reset(const char *path) +{ + const char **ptr ATTR_UNUSED; + int fd,ret; + string_t *cmd; + struct istream *input; + const char *line; + + fd = doveadm_connect(path); + net_set_nonblock(fd, FALSE); + input = i_stream_create_fd(fd, SIZE_MAX); + + cmd = t_str_new(10); + str_append(cmd, "RESET"); +/* XXX: Not supported yet. + for(ptr = items; *ptr; ptr++) + { + str_append_c(cmd, '\t'); + str_append(cmd, *ptr); + } +*/ + str_append_c(cmd, '\n'); + + /* send command */ + ret = write_full(fd, str_c(cmd), str_len(cmd)); + + if (ret < 0) { + i_close_fd(&fd); + i_error("write(%s) failed: %m", path); + return; + } + + line = i_stream_read_next_line(input); + + if (line == NULL) { + i_error("read(%s) failed: %s", path, i_stream_get_error(input)); + } else if (!str_begins(line, "OK")) { + i_error("%s",line); + } else { + i_info("Stats reset"); + } + + i_stream_destroy(&input); + i_close_fd(&fd); +} + +static void cmd_stats_top(struct doveadm_cmd_context *cctx) +{ + const char *path, *sort_type; + bool b; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &path)) { + path = t_strconcat(doveadm_settings->base_dir, + "/old-stats", NULL); + } + if (!doveadm_cmd_param_bool(cctx, "show-disk-io", &b) && b) { + disk_input_field = "read_bytes"; + disk_output_field = "write_bytes"; + } + if (!doveadm_cmd_param_str(cctx, "sort-field", &sort_type)) + sort_type = "disk"; + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + stats_top(path, sort_type); +} + +static void cmd_stats_reset(struct doveadm_cmd_context *cctx) +{ + const char *path; + + if (!doveadm_cmd_param_str(cctx, "socket-path", &path)) { + path = t_strconcat(doveadm_settings->base_dir, + "/old-stats", NULL); + } + + stats_reset(path); +} + +struct doveadm_cmd_ver2 doveadm_cmd_oldstats_dump_ver2 = { + .cmd = doveadm_cmd_stats_dump, + .name = "oldstats dump", + .usage = "[-s <stats socket path>] <type> [<filter>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "type", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "filter", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_oldstats_top_ver2 = { + .cmd = cmd_stats_top, + .name = "oldstats top", + .usage = "[-s <stats socket path>] [-b] [<sort field>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('b', "show-disk-io", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0', "sort-field", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + + +struct doveadm_cmd_ver2 doveadm_cmd_oldstats_reset_ver2 = { + .cmd = cmd_stats_reset, + .name = "oldstats reset", + .usage = "[-s <stats socket path>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-penalty.c b/src/doveadm/doveadm-penalty.c new file mode 100644 index 0000000..2f6d2d5 --- /dev/null +++ b/src/doveadm/doveadm-penalty.c @@ -0,0 +1,126 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "net.h" +#include "istream.h" +#include "hash.h" +#include "strescape.h" +#include "time-util.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <unistd.h> + +struct penalty_line { + struct ip_addr ip; + unsigned int penalty; + time_t last_penalty, last_update; +}; + +struct penalty_context { + const char *anvil_path; + + struct ip_addr net_ip; + unsigned int net_bits; +}; + +static void penalty_parse_line(const char *line, struct penalty_line *line_r) +{ + const char *const *args = t_strsplit_tabescaped(line); + const char *ident = args[0]; + const char *penalty_str = args[1]; + const char *last_penalty_str = args[2]; + const char *last_update_str = args[3]; + + i_zero(line_r); + + (void)net_addr2ip(ident, &line_r->ip); + if (str_to_uint(penalty_str, &line_r->penalty) < 0 || + str_to_time(last_penalty_str, &line_r->last_penalty) < 0 || + str_to_time(last_update_str, &line_r->last_update) < 0) + i_fatal("Read invalid penalty line: %s", line); +} + +static void +penalty_print_line(struct penalty_context *ctx, + const struct penalty_line *line) +{ + if (ctx->net_bits > 0) { + if (!net_is_in_network(&line->ip, &ctx->net_ip, ctx->net_bits)) + return; + } + + doveadm_print(net_ip2addr(&line->ip)); + doveadm_print(dec2str(line->penalty)); + doveadm_print(unixdate2str(line->last_penalty)); + doveadm_print(t_strflocaltime("%H:%M:%S", line->last_update)); +} + +static void penalty_lookup(struct penalty_context *ctx) +{ +#define ANVIL_HANDSHAKE "VERSION\tanvil\t1\t0\n" +#define ANVIL_CMD ANVIL_HANDSHAKE"PENALTY-DUMP\n" + struct istream *input; + const char *line; + int fd; + + fd = doveadm_connect(ctx->anvil_path); + net_set_nonblock(fd, FALSE); + if (write(fd, ANVIL_CMD, strlen(ANVIL_CMD)) < 0) + i_fatal("write(%s) failed: %m", ctx->anvil_path); + + input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + while ((line = i_stream_read_next_line(input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + struct penalty_line penalty_line; + + penalty_parse_line(line, &penalty_line); + penalty_print_line(ctx, &penalty_line); + } T_END; + } + if (input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->anvil_path, + i_stream_get_error(input)); + } + + i_stream_destroy(&input); +} + +static void cmd_penalty(struct doveadm_cmd_context *cctx) +{ + struct penalty_context ctx; + const char *netmask; + + i_zero(&ctx); + if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx.anvil_path))) + ctx.anvil_path = t_strconcat(doveadm_settings->base_dir, "/anvil", NULL); + + if (doveadm_cmd_param_str(cctx, "netmask", &netmask)) { + if (net_parse_range(netmask, &ctx.net_ip, &ctx.net_bits) != 0) { + doveadm_exit_code = EX_USAGE; + i_error("Invalid netmask '%s' given", netmask); + return; + } + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header_simple("IP"); + doveadm_print_header_simple("penalty"); + doveadm_print_header_simple("last_penalty"); + doveadm_print_header_simple("last_update"); + + penalty_lookup(&ctx); +} + +struct doveadm_cmd_ver2 doveadm_cmd_penalty_ver2 = { + .name = "penalty", + .cmd = cmd_penalty, + .usage = "[-a <anvil socket path>] [<ip/bits>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a',"socket-path", CMD_PARAM_STR,0) +DOVEADM_CMD_PARAM('\0',"netmask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-print-flow.c b/src/doveadm/doveadm-print-flow.c new file mode 100644 index 0000000..7f4d01e --- /dev/null +++ b/src/doveadm/doveadm-print-flow.c @@ -0,0 +1,111 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ostream.h" +#include "doveadm-print-private.h" + +struct doveadm_print_flow_header { + const char *title; + enum doveadm_print_header_flags flags; +}; + +struct doveadm_print_flow_context { + pool_t pool; + ARRAY(struct doveadm_print_flow_header) headers; + unsigned int header_idx; + + bool streaming:1; +}; + +static struct doveadm_print_flow_context *ctx; + +static void +doveadm_print_flow_header(const struct doveadm_print_header *hdr) +{ + struct doveadm_print_flow_header *fhdr; + + fhdr = array_append_space(&ctx->headers); + fhdr->title = p_strdup(ctx->pool, hdr->title); + fhdr->flags = hdr->flags; +} + +static void flow_next_hdr(void) +{ + if (++ctx->header_idx < array_count(&ctx->headers)) + o_stream_nsend(doveadm_print_ostream, " ", 1); + else { + ctx->header_idx = 0; + o_stream_nsend(doveadm_print_ostream, "\n", 1); + } +} + +static void doveadm_print_flow_print_header(const struct doveadm_print_flow_header *hdr) +{ + if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) { + o_stream_nsend_str(doveadm_print_ostream, hdr->title); + o_stream_nsend(doveadm_print_ostream, "=", 1); + } +} + +static void doveadm_print_flow_print(const char *value) +{ + const struct doveadm_print_flow_header *hdr = + array_idx(&ctx->headers, ctx->header_idx); + + doveadm_print_flow_print_header(hdr); + o_stream_nsend_str(doveadm_print_ostream, value); + flow_next_hdr(); +} + +static void +doveadm_print_flow_print_stream(const unsigned char *value, size_t size) +{ + const struct doveadm_print_flow_header *hdr = + array_idx(&ctx->headers, ctx->header_idx); + + if (!ctx->streaming) { + ctx->streaming = TRUE; + doveadm_print_flow_print_header(hdr); + } + o_stream_nsend(doveadm_print_ostream, value, size); + if (size == 0) { + flow_next_hdr(); + ctx->streaming = FALSE; + } +} + +static void doveadm_print_flow_init(void) +{ + pool_t pool; + + pool = pool_alloconly_create("doveadm print flow", 1024); + ctx = p_new(pool, struct doveadm_print_flow_context, 1); + ctx->pool = pool; + p_array_init(&ctx->headers, pool, 16); +} + +static void doveadm_print_flow_flush(void) +{ + if (ctx->header_idx != 0) { + o_stream_nsend(doveadm_print_ostream, "\n", 1); + ctx->header_idx = 0; + } +} + +static void doveadm_print_flow_deinit(void) +{ + pool_unref(&ctx->pool); + ctx = NULL; +} + +struct doveadm_print_vfuncs doveadm_print_flow_vfuncs = { + "flow", + + doveadm_print_flow_init, + doveadm_print_flow_deinit, + doveadm_print_flow_header, + doveadm_print_flow_print, + doveadm_print_flow_print_stream, + doveadm_print_flow_flush +}; diff --git a/src/doveadm/doveadm-print-formatted.c b/src/doveadm/doveadm-print-formatted.c new file mode 100644 index 0000000..d0518b0 --- /dev/null +++ b/src/doveadm/doveadm-print-formatted.c @@ -0,0 +1,92 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "ostream.h" +#include "doveadm.h" +#include "doveadm-server.h" +#include "doveadm-print.h" +#include "doveadm-print-private.h" +#include "client-connection.h" +#include "var-expand.h" + +struct doveadm_print_formatted_context { + pool_t pool; + const char *format; + ARRAY(struct var_expand_table) headers; + string_t *buf; + string_t *vbuf; + unsigned int idx; +}; + +static struct doveadm_print_formatted_context ctx; + +void doveadm_print_formatted_set_format(const char *format) +{ + ctx.format = format; +} + +static void doveadm_print_formatted_init(void) +{ + i_zero(&ctx); + ctx.pool = pool_alloconly_create("doveadm formatted print", 1024); + ctx.buf = str_new(ctx.pool, 256); + p_array_init(&ctx.headers, ctx.pool, 8); + ctx.idx = 0; +} + +static void +doveadm_print_formatted_header(const struct doveadm_print_header *hdr) +{ + struct var_expand_table entry; + i_zero(&entry); + entry.key = '\0'; + entry.long_key = p_strdup(ctx.pool, hdr->key); + entry.value = NULL; + array_push_back(&ctx.headers, &entry); +} + + +static void doveadm_print_formatted_flush(void) +{ + o_stream_nsend(doveadm_print_ostream, str_data(ctx.buf), str_len(ctx.buf)); + str_truncate(ctx.buf, 0); +} + +static void doveadm_print_formatted_print(const char *value) +{ + if (ctx.format == NULL) { + i_fatal("formatted formatter cannot be used without a format."); + } + const char *error; + struct var_expand_table *entry = array_idx_modifiable(&ctx.headers, ctx.idx++); + entry->value = value; + + if (ctx.idx >= array_count(&ctx.headers)) { + if (var_expand(ctx.buf, ctx.format, array_front(&ctx.headers), &error) <= 0) { + i_error("Failed to expand print format '%s': %s", + ctx.format, error); + } + doveadm_print_formatted_flush(); + ctx.idx = 0; + } + +} + +static void doveadm_print_formatted_deinit(void) +{ + pool_unref(&ctx.pool); +} + +struct doveadm_print_vfuncs doveadm_print_formatted_vfuncs = { + "formatted", + + doveadm_print_formatted_init, + doveadm_print_formatted_deinit, + doveadm_print_formatted_header, + doveadm_print_formatted_print, + NULL, + doveadm_print_formatted_flush +}; + diff --git a/src/doveadm/doveadm-print-json.c b/src/doveadm/doveadm-print-json.c new file mode 100644 index 0000000..540ecbb --- /dev/null +++ b/src/doveadm/doveadm-print-json.c @@ -0,0 +1,161 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "ostream.h" +#include "json-parser.h" +#include "doveadm.h" +#include "doveadm-server.h" +#include "doveadm-print.h" +#include "doveadm-print-private.h" +#include "client-connection.h" + +struct doveadm_print_json_context { + unsigned int header_idx, header_count; + bool first_row; + bool in_stream; + bool flushed; + ARRAY(struct doveadm_print_header) headers; + pool_t pool; + string_t *str; +}; + +static struct doveadm_print_json_context ctx; + +static void doveadm_print_json_flush_internal(void); + +static void doveadm_print_json_init(void) +{ + i_zero(&ctx); + ctx.pool = pool_alloconly_create("doveadm json print", 1024); + ctx.str = str_new(ctx.pool, 256); + p_array_init(&ctx.headers, ctx.pool, 1); + ctx.first_row = TRUE; + ctx.in_stream = FALSE; +} + +static void +doveadm_print_json_header(const struct doveadm_print_header *hdr) +{ + struct doveadm_print_header *lhdr; + lhdr = array_append_space(&ctx.headers); + lhdr->key = p_strdup(ctx.pool, hdr->key); + lhdr->flags = hdr->flags; + ctx.header_count++; +} + +static void +doveadm_print_json_value_header(const struct doveadm_print_header *hdr) +{ + // get header name + if (ctx.header_idx == 0) { + if (ctx.first_row == TRUE) { + ctx.first_row = FALSE; + str_append_c(ctx.str, '['); + } else { + str_append_c(ctx.str, ','); + } + str_append_c(ctx.str, '{'); + } else { + str_append_c(ctx.str, ','); + } + + str_append_c(ctx.str, '"'); + json_append_escaped(ctx.str, hdr->key); + str_append_c(ctx.str, '"'); + str_append_c(ctx.str, ':'); +} + +static void +doveadm_print_json_value_footer(void) { + if (++ctx.header_idx == ctx.header_count) { + ctx.header_idx = 0; + str_append_c(ctx.str, '}'); + doveadm_print_json_flush_internal(); + } +} + +static void doveadm_print_json_print(const char *value) +{ + const struct doveadm_print_header *hdr = array_idx(&ctx.headers, ctx.header_idx); + + doveadm_print_json_value_header(hdr); + + if (value == NULL) { + str_append(ctx.str, "null"); + } else if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_NUMBER) != 0) { + i_assert(str_is_float(value, '\0')); + str_append(ctx.str, value); + } else { + str_append_c(ctx.str, '"'); + json_append_escaped(ctx.str, value); + str_append_c(ctx.str, '"'); + } + + doveadm_print_json_value_footer(); +} + +static void +doveadm_print_json_print_stream(const unsigned char *value, size_t size) +{ + if (!ctx.in_stream) { + const struct doveadm_print_header *hdr = + array_idx(&ctx.headers, ctx.header_idx); + doveadm_print_json_value_header(hdr); + i_assert((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_NUMBER) == 0); + str_append_c(ctx.str, '"'); + ctx.in_stream = TRUE; + } + + if (size == 0) { + str_append_c(ctx.str, '"'); + doveadm_print_json_value_footer(); + ctx.in_stream = FALSE; + return; + } + + json_append_escaped_data(ctx.str, value, size); + + if (str_len(ctx.str) >= IO_BLOCK_SIZE) + doveadm_print_json_flush_internal(); +} + +static void doveadm_print_json_flush_internal(void) +{ + o_stream_nsend(doveadm_print_ostream, str_data(ctx.str), str_len(ctx.str)); + str_truncate(ctx.str, 0); +} + +static void doveadm_print_json_flush(void) +{ + if (ctx.flushed) + return; + ctx.flushed = TRUE; + + if (ctx.first_row == FALSE) + str_append_c(ctx.str,']'); + else { + str_append_c(ctx.str,'['); + str_append_c(ctx.str,']'); + } + doveadm_print_json_flush_internal(); +} + +static void doveadm_print_json_deinit(void) +{ + pool_unref(&ctx.pool); +} + +struct doveadm_print_vfuncs doveadm_print_json_vfuncs = { + "json", + + doveadm_print_json_init, + doveadm_print_json_deinit, + doveadm_print_json_header, + doveadm_print_json_print, + doveadm_print_json_print_stream, + doveadm_print_json_flush +}; + diff --git a/src/doveadm/doveadm-print-pager.c b/src/doveadm/doveadm-print-pager.c new file mode 100644 index 0000000..3e14383 --- /dev/null +++ b/src/doveadm/doveadm-print-pager.c @@ -0,0 +1,113 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ostream.h" +#include "doveadm-print-private.h" + +struct doveadm_print_pager_header { + const char *title; + enum doveadm_print_header_flags flags; +}; + +struct doveadm_print_pager_context { + pool_t pool; + ARRAY(struct doveadm_print_pager_header) headers; + unsigned int header_idx; + + bool streaming:1; + bool first_page:1; +}; + +static struct doveadm_print_pager_context *ctx; + +static void +doveadm_print_pager_header(const struct doveadm_print_header *hdr) +{ + struct doveadm_print_pager_header *fhdr; + + fhdr = array_append_space(&ctx->headers); + fhdr->flags = hdr->flags; + fhdr->title = p_strdup(ctx->pool, hdr->title); +} + +static void pager_next_hdr(void) +{ + if (++ctx->header_idx == array_count(&ctx->headers)) { + ctx->header_idx = 0; + } +} + +static void doveadm_print_pager_print(const char *value) +{ + const struct doveadm_print_pager_header *hdr = + array_idx(&ctx->headers, ctx->header_idx); + + if (ctx->header_idx == 0 && !ctx->first_page) { + o_stream_nsend(doveadm_print_ostream, "\f\n", 2); + } + ctx->first_page = FALSE; + if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) { + o_stream_nsend_str(doveadm_print_ostream, hdr->title); + o_stream_nsend(doveadm_print_ostream, ": ", 2); + } + o_stream_nsend_str(doveadm_print_ostream, value); + o_stream_nsend(doveadm_print_ostream, "\n", 1); + pager_next_hdr(); +} + +static void +doveadm_print_pager_print_stream(const unsigned char *value, size_t size) +{ + const struct doveadm_print_pager_header *hdr = + array_idx(&ctx->headers, ctx->header_idx); + + if (!ctx->streaming) { + ctx->streaming = TRUE; + if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) { + o_stream_nsend_str(doveadm_print_ostream, hdr->title); + o_stream_nsend(doveadm_print_ostream, ":\n", 2); + } + } + o_stream_nsend(doveadm_print_ostream, value, size); + if (size == 0) { + pager_next_hdr(); + ctx->streaming = FALSE; + } +} + +static void doveadm_print_pager_init(void) +{ + pool_t pool; + + pool = pool_alloconly_create("doveadm print pager", 1024); + ctx = p_new(pool, struct doveadm_print_pager_context, 1); + ctx->pool = pool; + ctx->first_page = TRUE; + p_array_init(&ctx->headers, pool, 16); +} + +static void doveadm_print_pager_flush(void) +{ + if (ctx->header_idx != 0) { + o_stream_nsend(doveadm_print_ostream, "\n", 1); + ctx->header_idx = 0; + } +} + +static void doveadm_print_pager_deinit(void) +{ + pool_unref(&ctx->pool); + ctx = NULL; +} + +struct doveadm_print_vfuncs doveadm_print_pager_vfuncs = { + DOVEADM_PRINT_TYPE_PAGER, + + doveadm_print_pager_init, + doveadm_print_pager_deinit, + doveadm_print_pager_header, + doveadm_print_pager_print, + doveadm_print_pager_print_stream, + doveadm_print_pager_flush +}; diff --git a/src/doveadm/doveadm-print-private.h b/src/doveadm/doveadm-print-private.h new file mode 100644 index 0000000..fe85560 --- /dev/null +++ b/src/doveadm/doveadm-print-private.h @@ -0,0 +1,31 @@ +#ifndef DOVEADM_PRINT_PRIVATE_H +#define DOVEADM_PRINT_PRIVATE_H + +#include "doveadm-print.h" + +struct doveadm_print_header { + const char *key; + const char *title; + enum doveadm_print_header_flags flags; +}; + +struct doveadm_print_vfuncs { + const char *name; + + void (*init)(void); + void (*deinit)(void); + + void (*header)(const struct doveadm_print_header *hdr); + void (*print)(const char *value); + void (*print_stream)(const unsigned char *value, size_t size); + void (*flush)(void); +}; + +extern struct doveadm_print_vfuncs doveadm_print_flow_vfuncs; +extern struct doveadm_print_vfuncs doveadm_print_tab_vfuncs; +extern struct doveadm_print_vfuncs doveadm_print_table_vfuncs; +extern struct doveadm_print_vfuncs doveadm_print_pager_vfuncs; +extern struct doveadm_print_vfuncs doveadm_print_json_vfuncs; +extern struct doveadm_print_vfuncs doveadm_print_formatted_vfuncs; + +#endif diff --git a/src/doveadm/doveadm-print-server.c b/src/doveadm/doveadm-print-server.c new file mode 100644 index 0000000..93c6ed9 --- /dev/null +++ b/src/doveadm/doveadm-print-server.c @@ -0,0 +1,119 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "ostream.h" +#include "doveadm.h" +#include "doveadm-print-private.h" +#include "client-connection.h" + +#define DOVEADM_PRINT_FLUSH_TIMEOUT_SECS 60 + +struct doveadm_print_server_context { + unsigned int header_idx, header_count; + + string_t *str; +}; + +static struct doveadm_print_server_context ctx; + +static void doveadm_print_server_flush(void); + +static void doveadm_print_server_init(void) +{ + ctx.str = str_new(default_pool, 256); +} + +static void doveadm_print_server_deinit(void) +{ + str_free(&ctx.str); +} + +static void +doveadm_print_server_header(const struct doveadm_print_header *hdr ATTR_UNUSED) +{ + /* no need to transfer these. the client should already know what + it's getting */ + ctx.header_count++; +} + +static void doveadm_print_server_print(const char *value) +{ + str_append_tabescaped(ctx.str, value); + str_append_c(ctx.str, '\t'); + + if (++ctx.header_idx == ctx.header_count) { + ctx.header_idx = 0; + doveadm_print_server_flush(); + } +} + +static void +doveadm_print_server_print_stream(const unsigned char *value, size_t size) +{ + if (size == 0) { + doveadm_print_server_print(""); + return; + } + str_append_tabescaped_n(ctx.str, value, size); + + if (str_len(ctx.str) >= IO_BLOCK_SIZE) + doveadm_print_server_flush(); +} + +static int flush_callback(struct doveadm_print_server_context *ctx ATTR_UNUSED) +{ + + int ret; + /* Keep flushing until everything is sent */ + if ((ret = o_stream_flush(doveadm_print_ostream)) != 0) + io_loop_stop(current_ioloop); + return ret; +} + +static void handle_flush_timeout(struct doveadm_print_server_context *ctx ATTR_UNUSED) +{ + io_loop_stop(current_ioloop); + o_stream_close(doveadm_print_ostream); + i_error("write(%s) failed: Timed out after %u seconds", + o_stream_get_name(doveadm_print_ostream), + DOVEADM_PRINT_FLUSH_TIMEOUT_SECS); +} + +static void doveadm_print_server_flush(void) +{ + o_stream_nsend(doveadm_print_ostream, + str_data(ctx.str), str_len(ctx.str)); + str_truncate(ctx.str, 0); + o_stream_uncork(doveadm_print_ostream); + + if (o_stream_get_buffer_used_size(doveadm_print_ostream) < IO_BLOCK_SIZE || + doveadm_print_ostream->stream_errno != 0) + return; + /* Wait until buffer is flushed to avoid it growing too large */ + struct ioloop *prev_loop = current_ioloop; + struct ioloop *loop = io_loop_create(); + /* Ensure we don't get stuck here forever */ + struct timeout *to = + timeout_add(DOVEADM_PRINT_FLUSH_TIMEOUT_SECS*1000, handle_flush_timeout, &ctx); + o_stream_switch_ioloop_to(doveadm_print_ostream, loop); + o_stream_set_flush_callback(doveadm_print_ostream, flush_callback, &ctx); + io_loop_run(loop); + timeout_remove(&to); + o_stream_unset_flush_callback(doveadm_print_ostream); + o_stream_switch_ioloop_to(doveadm_print_ostream, prev_loop); + io_loop_destroy(&loop); +} + +struct doveadm_print_vfuncs doveadm_print_server_vfuncs = { + DOVEADM_PRINT_TYPE_SERVER, + + doveadm_print_server_init, + doveadm_print_server_deinit, + doveadm_print_server_header, + doveadm_print_server_print, + doveadm_print_server_print_stream, + doveadm_print_server_flush +}; diff --git a/src/doveadm/doveadm-print-tab.c b/src/doveadm/doveadm-print-tab.c new file mode 100644 index 0000000..c5871ec --- /dev/null +++ b/src/doveadm/doveadm-print-tab.c @@ -0,0 +1,76 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "ostream.h" +#include "doveadm-print-private.h" + +struct doveadm_print_tab_context { + unsigned int header_idx, header_count; + + bool header_written:1; +}; + +static struct doveadm_print_tab_context ctx; + +static void doveadm_print_tab_flush_header(void) +{ + if (!ctx.header_written) { + if (!doveadm_print_hide_titles) + o_stream_nsend(doveadm_print_ostream, "\n", 1); + ctx.header_written = TRUE; + } +} + +static void +doveadm_print_tab_header(const struct doveadm_print_header *hdr) +{ + ctx.header_count++; + if (!doveadm_print_hide_titles) { + if (ctx.header_count > 1) + o_stream_nsend(doveadm_print_ostream, "\t", 1); + o_stream_nsend_str(doveadm_print_ostream, hdr->title); + } +} + +static void doveadm_print_tab_print(const char *value) +{ + doveadm_print_tab_flush_header(); + if (ctx.header_idx > 0) + o_stream_nsend(doveadm_print_ostream, "\t", 1); + o_stream_nsend_str(doveadm_print_ostream, value); + + if (++ctx.header_idx == ctx.header_count) { + ctx.header_idx = 0; + o_stream_nsend(doveadm_print_ostream, "\n", 1); + } +} + +static void +doveadm_print_tab_print_stream(const unsigned char *value, size_t size) +{ + if (size == 0) { + doveadm_print_tab_print(""); + return; + } + doveadm_print_tab_flush_header(); + if (ctx.header_idx > 0) + o_stream_nsend(doveadm_print_ostream, "\t", 1); + o_stream_nsend(doveadm_print_ostream, value, size); +} + +static void doveadm_print_tab_flush(void) +{ + doveadm_print_tab_flush_header(); +} + +struct doveadm_print_vfuncs doveadm_print_tab_vfuncs = { + "tab", + + NULL, + NULL, + doveadm_print_tab_header, + doveadm_print_tab_print, + doveadm_print_tab_print_stream, + doveadm_print_tab_flush +}; diff --git a/src/doveadm/doveadm-print-table.c b/src/doveadm/doveadm-print-table.c new file mode 100644 index 0000000..25a4441 --- /dev/null +++ b/src/doveadm/doveadm-print-table.c @@ -0,0 +1,268 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "unichar.h" +#include "doveadm-print-private.h" + +#include <stdio.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <termios.h> + +#define DEFAULT_COLUMNS 80 +#define MIN_COLUMNS 30 +#define MAX_BUFFER_LINES 100 + +struct doveadm_print_table_header { + const char *key; + const char *title; + enum doveadm_print_header_flags flags; + size_t min_length, max_length, length; +}; + +struct doveadm_print_table_context { + pool_t pool; + ARRAY(struct doveadm_print_table_header) headers; + ARRAY_TYPE(const_string) buffered_values; + string_t *stream; + unsigned int hdr_idx; + unsigned int columns; + + bool lengths_set:1; +}; + +static struct doveadm_print_table_context *ctx; + +static void +doveadm_print_table_header(const struct doveadm_print_header *hdr) +{ + struct doveadm_print_table_header *thdr; + + thdr = array_append_space(&ctx->headers); + thdr->key = p_strdup(ctx->pool, hdr->key); + thdr->title = p_strdup(ctx->pool, hdr->title); + thdr->length = thdr->max_length = thdr->min_length = strlen(hdr->title); + thdr->flags = hdr->flags; +} + +static void doveadm_calc_header_length(void) +{ + struct doveadm_print_table_header *headers; + const char *value, *const *values; + unsigned int i, line, hdr_count, value_count, line_count; + size_t len, max_length, orig_length, diff; + + ctx->lengths_set = TRUE; + + headers = array_get_modifiable(&ctx->headers, &hdr_count); + values = array_get(&ctx->buffered_values, &value_count); + i_assert((value_count % hdr_count) == 0); + line_count = value_count / hdr_count; + + /* find min and max lengths of fields */ + for (line = 0; line < line_count; line++) { + for (i = 0; i < hdr_count; i++) { + value = values[line*hdr_count + i]; + len = value == NULL ? 0 : uni_utf8_strlen(value); + if (headers[i].min_length > len) + headers[i].min_length = len; + if (headers[i].max_length < len) { + headers[i].max_length = len; + headers[i].length = len; + } + } + } + + /* +1 for space between fields */ + max_length = 0; + for (i = 0; i < hdr_count; i++) + max_length += headers[i].max_length + 1; + max_length--; + + while (max_length > ctx->columns) { + /* shrink something so we'll fit */ + orig_length = max_length; + for (i = hdr_count - 1;; i--) { + diff = headers[i].length - headers[i].min_length; + if (max_length - diff <= ctx->columns) { + /* we can finish with this */ + diff = max_length - ctx->columns; + headers[i].length -= diff; + max_length -= diff; + break; + } + if (diff > 0) { + /* take a bit off from it */ + headers[i].length -= diff == 1 ? 1 : diff/2; + } + + if (i == 0) + break; + } + if (max_length == orig_length) { + /* can't shrink it any more */ + break; + } + } + if (max_length < ctx->columns) { + for (i = 0; i < hdr_count; i++) { + if ((headers[i].flags & DOVEADM_PRINT_HEADER_FLAG_EXPAND) != 0) { + i++; + break; + } + } + headers[i-1].length += (ctx->columns - max_length) / 2; + } +} + +static size_t utf8_correction(const char *str) +{ + size_t i, len = 0; + + for (i = 0; str[i] != '\0'; i++) { + if ((str[i] & 0xc0) == 0x80) + len++; + } + return len; +} + +static void doveadm_print_next(const char *value) +{ + const struct doveadm_print_table_header *hdr; + int value_padded_len; + + hdr = array_idx(&ctx->headers, ctx->hdr_idx); + + value_padded_len = hdr->length + utf8_correction(value); + if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY) == 0) + printf("%-*s", value_padded_len, value); + else + printf("%*s", value_padded_len, value); + + if (++ctx->hdr_idx == array_count(&ctx->headers)) { + ctx->hdr_idx = 0; + printf("\n"); + } else { + printf(" "); + } +} + +static void doveadm_print_headers(void) +{ + const struct doveadm_print_table_header *headers; + unsigned int i, count; + + if (doveadm_print_hide_titles) + return; + + headers = array_get(&ctx->headers, &count); + /* if all headers are hidden, don't print any of them */ + for (i = 0; i < count; i++) { + if ((headers[i].flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) + break; + } + if (i == count) + return; + for (i = 0; i < count; i++) { + if (i > 0) printf(" "); + + if ((headers[i].flags & + DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY) == 0) { + printf("%-*s", (int)headers[i].length, + headers[i].title); + } else { + printf("%*s", (int)headers[i].length, + headers[i].title); + } + } + printf("\n"); +} + +static void doveadm_buffer_flush(void) +{ + const char *value; + + doveadm_calc_header_length(); + doveadm_print_headers(); + + array_foreach_elem(&ctx->buffered_values, value) + doveadm_print_next(value); + array_clear(&ctx->buffered_values); +} + +static void doveadm_print_table_print(const char *value) +{ + unsigned int line_count; + + if (!ctx->lengths_set) { + line_count = array_count(&ctx->buffered_values) / + array_count(&ctx->headers); + if (line_count < MAX_BUFFER_LINES) { + value = p_strdup(ctx->pool, value); + array_push_back(&ctx->buffered_values, &value); + return; + } + doveadm_buffer_flush(); + } + doveadm_print_next(value); +} + +static void +doveadm_print_table_print_stream(const unsigned char *value, size_t size) +{ + if (memchr(value, '\n', size) != NULL) + i_fatal("table formatter doesn't support multi-line values"); + + if (size != 0) + str_append_data(ctx->stream, value, size); + else { + doveadm_print_table_print(str_c(ctx->stream)); + str_truncate(ctx->stream, 0); + } +} + +static void doveadm_print_table_flush(void) +{ + if (!ctx->lengths_set && array_count(&ctx->headers) > 0) + doveadm_buffer_flush(); +} + +static void doveadm_print_table_init(void) +{ + pool_t pool; + struct winsize ws; + + pool = pool_alloconly_create("doveadm print table", 2048); + ctx = p_new(pool, struct doveadm_print_table_context, 1); + ctx->pool = pool; + ctx->stream = str_new(default_pool, 128); + p_array_init(&ctx->headers, pool, 16); + i_array_init(&ctx->buffered_values, 64); + ctx->columns = DEFAULT_COLUMNS; + + if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0) { + ctx->columns = ws.ws_col < MIN_COLUMNS ? + MIN_COLUMNS : ws.ws_col; + } +} + +static void doveadm_print_table_deinit(void) +{ + str_free(&ctx->stream); + array_free(&ctx->buffered_values); + pool_unref(&ctx->pool); + ctx = NULL; +} + +struct doveadm_print_vfuncs doveadm_print_table_vfuncs = { + "table", + + doveadm_print_table_init, + doveadm_print_table_deinit, + doveadm_print_table_header, + doveadm_print_table_print, + doveadm_print_table_print_stream, + doveadm_print_table_flush +}; diff --git a/src/doveadm/doveadm-print.c b/src/doveadm/doveadm-print.c new file mode 100644 index 0000000..c1c5fac --- /dev/null +++ b/src/doveadm/doveadm-print.c @@ -0,0 +1,211 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "ostream.h" +#include "doveadm-print-private.h" + +struct doveadm_print_header_context { + const char *key; + char *sticky_value; + bool sticky; +}; + +struct doveadm_print_context { + pool_t pool; + ARRAY(struct doveadm_print_header_context) headers; + const struct doveadm_print_vfuncs *v; + + unsigned int header_idx; + bool print_stream_open; +}; + +bool doveadm_print_hide_titles = FALSE; +struct ostream *doveadm_print_ostream = NULL; + +static struct doveadm_print_context *ctx; + +bool doveadm_print_is_initialized(void) +{ + return ctx != NULL; +} + +void doveadm_print_header(const char *key, const char *title, + enum doveadm_print_header_flags flags) +{ + struct doveadm_print_header hdr; + struct doveadm_print_header_context *hdr_ctx; + + i_assert(title != NULL); + + i_zero(&hdr); + hdr.key = key; + hdr.title = title; + hdr.flags = flags; + ctx->v->header(&hdr); + + hdr_ctx = array_append_space(&ctx->headers); + hdr_ctx->key = p_strdup(ctx->pool, key); + hdr_ctx->sticky = (flags & DOVEADM_PRINT_HEADER_FLAG_STICKY) != 0; +} + +void doveadm_print_header_simple(const char *key_title) +{ + doveadm_print_header(key_title, key_title, 0); +} + +unsigned int doveadm_print_get_headers_count(void) +{ + return array_count(&ctx->headers); +} + +static void doveadm_print_sticky_headers(void) +{ + const struct doveadm_print_header_context *headers; + unsigned int count; + + headers = array_get(&ctx->headers, &count); + i_assert(count > 0); + for (;;) { + if (ctx->header_idx == count) + ctx->header_idx = 0; + else if (headers[ctx->header_idx].sticky) { + ctx->v->print(headers[ctx->header_idx].sticky_value); + ctx->header_idx++; + } else { + break; + } + } +} + +void doveadm_print(const char *value) +{ + i_assert(!ctx->print_stream_open); + + doveadm_print_sticky_headers(); + ctx->v->print(value); + ctx->header_idx++; +} + +void doveadm_print_num(uintmax_t value) +{ + T_BEGIN { + doveadm_print(dec2str(value)); + } T_END; +} + +void doveadm_print_stream(const void *value, size_t size) +{ + if (!ctx->print_stream_open) { + doveadm_print_sticky_headers(); + ctx->print_stream_open = TRUE; + } + ctx->v->print_stream(value, size); + if (size == 0) { + ctx->header_idx++; + ctx->print_stream_open = FALSE; + } +} + +int doveadm_print_istream(struct istream *input) +{ + const unsigned char *data; + size_t size; + ssize_t ret; + + while ((ret = i_stream_read_more(input, &data, &size)) > 0) { + doveadm_print_stream(data, size); + i_stream_skip(input, size); + } + i_assert(ret == -1); + doveadm_print_stream("", 0); + if (input->stream_errno != 0) { + /* caller will log the error */ + return -1; + } + return 0; +} + +void doveadm_print_sticky(const char *key, const char *value) +{ + struct doveadm_print_header_context *hdr; + + if (ctx == NULL) { + /* command doesn't really print anything */ + return; + } + + array_foreach_modifiable(&ctx->headers, hdr) { + if (strcmp(hdr->key, key) == 0) { + i_free(hdr->sticky_value); + hdr->sticky_value = i_strdup(value); + return; + } + } + i_unreached(); +} + +void doveadm_print_flush(void) +{ + if (ctx != NULL && ctx->v->flush != NULL) + ctx->v->flush(); + o_stream_uncork(doveadm_print_ostream); + o_stream_cork(doveadm_print_ostream); +} + +void doveadm_print_unstick_headers(void) +{ + struct doveadm_print_header_context *hdr; + + if (ctx != NULL) { + array_foreach_modifiable(&ctx->headers, hdr) + hdr->sticky = FALSE; + } +} + +void doveadm_print_init(const char *name) +{ + pool_t pool; + unsigned int i; + + if (ctx != NULL) { + /* already forced the type */ + return; + } + + pool = pool_alloconly_create("doveadm print", 1024); + ctx = p_new(pool, struct doveadm_print_context, 1); + ctx->pool = pool; + p_array_init(&ctx->headers, pool, 16); + + for (i = 0; doveadm_print_vfuncs_all[i] != NULL; i++) { + if (strcmp(doveadm_print_vfuncs_all[i]->name, name) == 0) { + ctx->v = doveadm_print_vfuncs_all[i]; + break; + } + } + if (ctx->v == NULL) + i_fatal("Unknown print formatter: %s", name); + if (ctx->v->init != NULL) + ctx->v->init(); +} + +void doveadm_print_deinit(void) +{ + struct doveadm_print_header_context *hdr; + + if (ctx == NULL) + return; + + if (ctx->v->flush != NULL && doveadm_print_ostream != NULL) { + ctx->v->flush(); + o_stream_uncork(doveadm_print_ostream); + } + if (ctx->v->deinit != NULL) + ctx->v->deinit(); + array_foreach_modifiable(&ctx->headers, hdr) + i_free(hdr->sticky_value); + pool_unref(&ctx->pool); + ctx = NULL; +} diff --git a/src/doveadm/doveadm-print.h b/src/doveadm/doveadm-print.h new file mode 100644 index 0000000..2d610e8 --- /dev/null +++ b/src/doveadm/doveadm-print.h @@ -0,0 +1,48 @@ +#ifndef DOVEADM_PRINT_H +#define DOVEADM_PRINT_H + +#define DOVEADM_PRINT_TYPE_TAB "tab" +#define DOVEADM_PRINT_TYPE_FLOW "flow" +#define DOVEADM_PRINT_TYPE_PAGER "pager" +#define DOVEADM_PRINT_TYPE_TABLE "table" +#define DOVEADM_PRINT_TYPE_SERVER "server" +#define DOVEADM_PRINT_TYPE_JSON "json" +#define DOVEADM_PRINT_TYPE_FORMATTED "formatted" + +enum doveadm_print_header_flags { + DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY = 0x01, + DOVEADM_PRINT_HEADER_FLAG_STICKY = 0x02, + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE = 0x04, + DOVEADM_PRINT_HEADER_FLAG_EXPAND = 0x08, + DOVEADM_PRINT_HEADER_FLAG_NUMBER = 0x10 +}; + +extern const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[]; +extern bool doveadm_print_hide_titles; +/* points to either stdout or to doveadm-server's TCP connection */ +extern struct ostream *doveadm_print_ostream; + +bool doveadm_print_is_initialized(void); + +void doveadm_print_header(const char *key, const char *title, + enum doveadm_print_header_flags flags); +void doveadm_print_header_simple(const char *key_title); +unsigned int doveadm_print_get_headers_count(void); + +void doveadm_print(const char *value); +void doveadm_print_num(uintmax_t value); +/* Stream for same field continues until len=0 */ +void doveadm_print_stream(const void *value, size_t size); +/* Print the whole input stream. Returns 0 if ok, -1 if stream read() failed. + The caller must log the error. */ +int doveadm_print_istream(struct istream *input); +void doveadm_print_sticky(const char *key, const char *value); +void doveadm_print_flush(void); +void doveadm_print_unstick_headers(void); + +void doveadm_print_init(const char *name); +void doveadm_print_deinit(void); + +void doveadm_print_formatted_set_format(const char *format); + +#endif diff --git a/src/doveadm/doveadm-proxy.c b/src/doveadm/doveadm-proxy.c new file mode 100644 index 0000000..04fde52 --- /dev/null +++ b/src/doveadm/doveadm-proxy.c @@ -0,0 +1,214 @@ +/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "strescape.h" +#include "ipc-client.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> + +struct proxy_context { + struct ipc_client *ipc; + const char *username_field; + const char *kick_hosts; +}; + +extern struct doveadm_cmd_ver2 doveadm_cmd_proxy[]; + +static void proxy_cmd_help(struct doveadm_cmd_context *cctx) ATTR_NORETURN; + +static struct proxy_context * +cmd_proxy_init(struct doveadm_cmd_context *cctx) +{ + struct proxy_context *ctx; + const char *socket_path; + + ctx = t_new(struct proxy_context, 1); + if (!doveadm_cmd_param_str(cctx, "socket-path", &socket_path)) { + socket_path = t_strconcat(doveadm_settings->base_dir, + "/ipc", NULL); + } + (void)doveadm_cmd_param_str(cctx, "passdb-field", &ctx->username_field); + (void)doveadm_cmd_param_str(cctx, "host", &ctx->kick_hosts); + ctx->ipc = ipc_client_init(socket_path); + return ctx; +} + +static void cmd_proxy_list_header(const char *const *args) +{ + struct { + const char *key; + const char *title; + } header_map[] = { + { "service", "proto" }, + { "src-ip", "src ip" }, + { "dest-ip", "dest ip" }, + { "dest-port", "port" }, + }; + for (unsigned int i = 0; args[i] != NULL; i++) { + const char *arg = args[i]; + + if (strcmp(arg, "username") == 0 || + str_begins(arg, "user_")) { + doveadm_print_header(arg, arg, + DOVEADM_PRINT_HEADER_FLAG_EXPAND); + continue; + } + const char *title = arg; + for (unsigned int j = 0; j < N_ELEMENTS(header_map); j++) { + if (strcmp(header_map[j].key, arg) == 0) { + title = header_map[j].title; + break; + } + } + doveadm_print_header(arg, title, 0); + } +} + +static void cmd_proxy_list_callback(enum ipc_client_cmd_state state, + const char *data, void *context) +{ + bool *seen_header = context; + + switch (state) { + case IPC_CLIENT_CMD_STATE_REPLY: { + const char *const *args = t_strsplit_tabescaped(data); + + if (!*seen_header) { + cmd_proxy_list_header(args); + *seen_header = TRUE; + } else { + for (; *args != NULL; args++) + doveadm_print(*args); + } + return; + } + case IPC_CLIENT_CMD_STATE_OK: + break; + case IPC_CLIENT_CMD_STATE_ERROR: + i_error("LIST-FULL failed: %s", data); + break; + } + io_loop_stop(current_ioloop); +} + +static void cmd_proxy_list(struct doveadm_cmd_context *cctx) +{ + struct proxy_context *ctx; + bool seen_header = FALSE; + + ctx = cmd_proxy_init(cctx); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + + io_loop_set_running(current_ioloop); + ipc_client_cmd(ctx->ipc, "proxy\t*\tLIST-FULL", + cmd_proxy_list_callback, &seen_header); + if (io_loop_is_running(current_ioloop)) + io_loop_run(current_ioloop); + ipc_client_deinit(&ctx->ipc); +} + +static void cmd_proxy_kick_callback(enum ipc_client_cmd_state state, + const char *data, void *context ATTR_UNUSED) +{ + switch (state) { + case IPC_CLIENT_CMD_STATE_REPLY: + return; + case IPC_CLIENT_CMD_STATE_OK: + if (data[0] == '\0') + data = "0"; + doveadm_print(data); + break; + case IPC_CLIENT_CMD_STATE_ERROR: + i_error("KICK failed: %s", data); + doveadm_exit_code = EX_TEMPFAIL; + break; + } + io_loop_stop(current_ioloop); +} + +static void cmd_proxy_kick(struct doveadm_cmd_context *cctx) +{ + struct proxy_context *ctx; + const char *const *users = NULL; + string_t *cmd; + + ctx = cmd_proxy_init(cctx); + (void)doveadm_cmd_param_array(cctx, "user", &users); + if (users == NULL && ctx->kick_hosts == NULL) { + proxy_cmd_help(cctx); + return; + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED); + doveadm_print_formatted_set_format("%{count} connections kicked\n"); + doveadm_print_header_simple("count"); + + cmd = t_str_new(128); + str_append(cmd, "proxy\t*\t"); + if (ctx->kick_hosts != NULL) { + str_append(cmd, "KICK-HOST\t"); + str_append(cmd, ctx->kick_hosts); + } + else if (ctx->username_field == NULL) + str_append(cmd, "KICK"); + else { + str_append(cmd, "KICK-ALT\t"); + str_append_tabescaped(cmd, ctx->username_field); + } + if (users != NULL) { + for (unsigned int i = 0; users[i] != NULL; i++) { + str_append_c(cmd, '\t'); + str_append_tabescaped(cmd, users[i]); + } + } + ipc_client_cmd(ctx->ipc, str_c(cmd), cmd_proxy_kick_callback, NULL); + io_loop_run(current_ioloop); + ipc_client_deinit(&ctx->ipc); +} + +struct doveadm_cmd_ver2 doveadm_cmd_proxy[] = { +{ + .name = "proxy list", + .usage = "[-a <ipc socket path>]", + .cmd = cmd_proxy_list, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "proxy kick", + .usage = "[-a <ipc socket path>] [-f <passdb field>] [-h <host> [...] | <user> [...]]", + .cmd = cmd_proxy_kick, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('f', "passdb-field", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('h', "host", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +} +}; + +static void proxy_cmd_help(struct doveadm_cmd_context *cctx) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_proxy); i++) { + if (doveadm_cmd_proxy[i].cmd == cctx->cmd->cmd) + help_ver2(&doveadm_cmd_proxy[i]); + } + i_unreached(); +} + +void doveadm_register_proxy_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_proxy); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_proxy[i]); +} diff --git a/src/doveadm/doveadm-pw.c b/src/doveadm/doveadm-pw.c new file mode 100644 index 0000000..ed85006 --- /dev/null +++ b/src/doveadm/doveadm-pw.c @@ -0,0 +1,135 @@ +/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "password-scheme.h" +#include "randgen.h" +#include "doveadm.h" +#include "askpass.h" +#include "module-dir.h" + +#include <ctype.h> +#include <fcntl.h> +#include <stdio.h> +#include <unistd.h> + +#define DEFAULT_SCHEME "CRYPT" + +static struct module *modules = NULL; + +static void cmd_pw(struct doveadm_cmd_context *cctx) +{ + const char *hash = NULL; + const char *scheme = NULL; + const char *plaintext = NULL; + const char *test_hash = NULL; + bool list_schemes = FALSE, reverse_verify = FALSE; + int64_t rounds_int64; + struct module_dir_load_settings mod_set; + struct password_generate_params gen_params; + i_zero(&gen_params); + + password_schemes_init(); + + i_zero(&mod_set); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.require_init_funcs = TRUE; + mod_set.ignore_dlopen_errors = TRUE; + mod_set.debug = doveadm_debug; + + modules = module_dir_load_missing(modules, AUTH_MODULE_DIR, NULL, &mod_set); + module_dir_init(modules); + + (void)doveadm_cmd_param_bool(cctx, "list", &list_schemes); + (void)doveadm_cmd_param_str(cctx, "plaintext", &plaintext); + if (doveadm_cmd_param_int64(cctx, "rounds", &rounds_int64)) { + if (rounds_int64 > UINT_MAX) + i_fatal("Invalid number of rounds: %"PRId64, rounds_int64); + gen_params.rounds = rounds_int64; + } + (void)doveadm_cmd_param_str(cctx, "scheme", &scheme); + if (doveadm_cmd_param_str(cctx, "test-hash", &test_hash)) + reverse_verify = TRUE; + (void)doveadm_cmd_param_str(cctx, "user", &gen_params.user); + (void)doveadm_cmd_param_bool(cctx, "reverse-verify", &reverse_verify); + + if (list_schemes) { + ARRAY_TYPE(password_scheme_p) arr; + const struct password_scheme *const *schemes; + unsigned int i, count; + t_array_init(&arr, 30); + password_schemes_get(&arr); + schemes = array_get(&arr, &count); + for (i = 0; i < count; i++) + printf("%s ", schemes[i]->name); + printf("\n"); + module_dir_unload(&modules); + password_schemes_deinit(); + return; + } + + scheme = scheme == NULL ? DEFAULT_SCHEME : t_str_ucase(scheme); + + if (test_hash != NULL && plaintext == NULL) + plaintext = t_askpass("Enter password to verify: "); + while (plaintext == NULL) { + const char *check; + static int lives = 3; + + plaintext = t_askpass("Enter new password: "); + check = t_askpass("Retype new password: "); + if (strcmp(plaintext, check) != 0) { + i_error("Passwords don't match!"); + if (--lives == 0) + lib_exit(1); + plaintext = NULL; + } + } + + if (!password_generate_encoded(plaintext, &gen_params, scheme, &hash)) + i_fatal("Unknown scheme: %s", scheme); + if (reverse_verify) { + const unsigned char *raw_password; + size_t size; + const char *error; + + if (test_hash != NULL) { + scheme = password_get_scheme(&test_hash); + if (scheme == NULL) + i_fatal("Missing {scheme} prefix from hash"); + hash = test_hash; + } + + if (password_decode(hash, scheme, &raw_password, &size, + &error) <= 0) + i_fatal("reverse decode check failed: %s", error); + + if (password_verify(plaintext, &gen_params, scheme, + raw_password, size, &error) <= 0) { + i_fatal("reverse password verification check failed: %s", + error); + } + + printf("{%s}%s (verified)\n", scheme, hash); + } else { + printf("{%s}%s\n", scheme, hash); + } + + module_dir_unload(&modules); + password_schemes_deinit(); +} + +struct doveadm_cmd_ver2 doveadm_cmd_pw = { + .name = "pw", + .cmd = cmd_pw, + .usage = "[-l] [-p plaintext] [-r rounds] [-s scheme] [-t hash] [-u user] [-V]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('l', "list", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('p', "plaintext", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('r', "rounds", CMD_PARAM_INT64, 0) +DOVEADM_CMD_PARAM('s', "scheme", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('t', "test-hash", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('V', "reverse-verify", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-replicator.c b/src/doveadm/doveadm-replicator.c new file mode 100644 index 0000000..17af8b9 --- /dev/null +++ b/src/doveadm/doveadm-replicator.c @@ -0,0 +1,372 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "ioloop.h" +#include "str.h" +#include "strescape.h" +#include "istream.h" +#include "write-full.h" +#include "master-service.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> + +struct replicator_context { + const char *socket_path; + const char *priority; + const char *user_mask, *username; + struct istream *input; + bool full_sync; +}; + +extern struct doveadm_cmd_ver2 doveadm_cmd_replicator[]; + +static void replicator_cmd_help(const struct doveadm_cmd_ver2 *cmd) ATTR_NORETURN; + +static void +replicator_send(struct replicator_context *ctx, const char *data) +{ + if (write_full(i_stream_get_fd(ctx->input), data, strlen(data)) < 0) + i_fatal("write(%s) failed: %m", ctx->socket_path); +} + +static void replicator_connect(struct replicator_context *ctx) +{ +#define REPLICATOR_HANDSHAKE "VERSION\treplicator-doveadm-client\t1\t0\n" + const char *line; + int fd; + + fd = doveadm_connect(ctx->socket_path); + net_set_nonblock(fd, FALSE); + + ctx->input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + replicator_send(ctx, REPLICATOR_HANDSHAKE); + + alarm(5); + line = i_stream_read_next_line(ctx->input); + alarm(0); + if (line == NULL) { + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } else if (ctx->input->eof) + i_fatal("%s disconnected", ctx->socket_path); + else + i_fatal("read(%s) timed out", ctx->socket_path); + } + if (!version_string_verify(line, "replicator-doveadm-server", 1)) { + i_fatal_status(EX_PROTOCOL, + "%s not a compatible replicator-doveadm socket", + ctx->socket_path); + } +} + +static void replicator_disconnect(struct replicator_context *ctx) +{ + if (ctx->input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->socket_path, + i_stream_get_error(ctx->input)); + } + i_stream_destroy(&ctx->input); +} + +static struct replicator_context * +cmd_replicator_init(struct doveadm_cmd_context *cctx) +{ + struct replicator_context *ctx; + + ctx = t_new(struct replicator_context, 1); + ctx->socket_path = t_strconcat(doveadm_settings->base_dir, + "/replicator-doveadm", NULL); + + (void)doveadm_cmd_param_str(cctx, "socket-path", &ctx->socket_path); + (void)doveadm_cmd_param_bool(cctx, "full-sync", &ctx->full_sync); + (void)doveadm_cmd_param_str(cctx, "priority", &ctx->priority); + (void)doveadm_cmd_param_str(cctx, "user-mask", &ctx->user_mask); + (void)doveadm_cmd_param_str(cctx, "user", &ctx->username); + + replicator_connect(ctx); + return ctx; +} + +static const char *time_ago(time_t t) +{ + int diff = ioloop_time - t; + + if (t == 0) + return "-"; + return t_strdup_printf("%02d:%02d:%02d", diff/3600, + (diff/60)%60, diff%60); +} + +static void cmd_replicator_status_overview(struct replicator_context *ctx) +{ + char *line, *value; + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("field", "field", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + doveadm_print_header("value", "value", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + + replicator_send(ctx, "STATUS\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + value = strchr(line, '\t'); + if (value != NULL) + *value++ = '\0'; + else + value = ""; + doveadm_print(line); + doveadm_print(value); + } + replicator_disconnect(ctx); +} + +static void cmd_replicator_status(struct doveadm_cmd_context *cctx) +{ + struct replicator_context *ctx; + const char *line, *const *args; + time_t last_fast, last_full, last_success; + + ctx = cmd_replicator_init(cctx); + if (ctx->user_mask == NULL) { + cmd_replicator_status_overview(ctx); + return; + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("username", "username", + DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("priority"); + doveadm_print_header_simple("fast sync"); + doveadm_print_header_simple("full sync"); + doveadm_print_header_simple("success sync"); + doveadm_print_header_simple("failed"); + + replicator_send(ctx, t_strdup_printf("STATUS\t%s\n", + str_tabescape(ctx->user_mask))); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + args = t_strsplit_tabescaped(line); + if (str_array_length(args) >= 5 && + str_to_time(args[2], &last_fast) == 0 && + str_to_time(args[3], &last_full) == 0 && + str_to_time(args[5], &last_success) == 0) { + doveadm_print(args[0]); + doveadm_print(args[1]); + doveadm_print(time_ago(last_fast)); + doveadm_print(time_ago(last_full)); + doveadm_print(time_ago(last_success)); + doveadm_print(args[4][0] == '0' ? "-" : "y"); + } + } T_END; + } + if (line == NULL) { + i_error("Replicator disconnected unexpectedly"); + doveadm_exit_code = EX_TEMPFAIL; + } + replicator_disconnect(ctx); +} + +static void cmd_replicator_dsync_status(struct doveadm_cmd_context *cctx) +{ + struct replicator_context *ctx; + const char *line; + unsigned int i; + + ctx = cmd_replicator_init(cctx); + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + doveadm_print_header("username", "username", + DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header_simple("type"); + doveadm_print_header_simple("status"); + + replicator_send(ctx, "STATUS-DSYNC\n"); + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + const char *const *args = t_strsplit_tabescaped(line); + + for (i = 0; i < 3; i++) { + if (args[i] == NULL) + break; + doveadm_print(args[i]); + } + for (; i < 3; i++) + doveadm_print(""); + } T_END; + } + replicator_disconnect(ctx); +} + +static void cmd_replicator_replicate(struct doveadm_cmd_context *cctx) +{ + struct replicator_context *ctx; + string_t *str; + const char *line; + + ctx = cmd_replicator_init(cctx); + if (ctx->user_mask == NULL) + replicator_cmd_help(cctx->cmd); + + str = t_str_new(128); + str_append(str, "REPLICATE\t"); + if (ctx->priority == NULL) + str_append_tabescaped(str, "low"); + else + str_append_tabescaped(str, ctx->priority); + str_append_c(str, '\t'); + if (ctx->full_sync) + str_append_c(str, 'f'); + str_append_c(str, '\t'); + str_append_tabescaped(str, ctx->user_mask); + str_append_c(str, '\n'); + replicator_send(ctx, str_c(str)); + + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + doveadm_print_header("result", "result", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + i_error("Replicator disconnected unexpectedly"); + doveadm_exit_code = EX_TEMPFAIL; + } else if (line[0] != '+') { + i_error("Replicator failed: %s", line+1); + doveadm_exit_code = EX_USAGE; + } else { + doveadm_print(t_strdup_printf("%s users updated", line+1)); + } + replicator_disconnect(ctx); +} + +static void cmd_replicator_add(struct doveadm_cmd_context *cctx) +{ + struct replicator_context *ctx; + string_t *str; + const char *line; + + ctx = cmd_replicator_init(cctx); + if (ctx->user_mask == NULL) + replicator_cmd_help(cctx->cmd); + + str = t_str_new(128); + str_append(str, "ADD\t"); + str_append_tabescaped(str, ctx->user_mask); + str_append_c(str, '\n'); + replicator_send(ctx, str_c(str)); + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + i_error("Replicator disconnected unexpectedly"); + doveadm_exit_code = EX_TEMPFAIL; + } else if (line[0] != '+') { + i_error("Replicator failed: %s", line+1); + doveadm_exit_code = EX_USAGE; + } + replicator_disconnect(ctx); +} + +static void cmd_replicator_remove(struct doveadm_cmd_context *cctx) +{ + struct replicator_context *ctx; + string_t *str; + const char *line; + + ctx = cmd_replicator_init(cctx); + if (ctx->username == NULL) + replicator_cmd_help(cctx->cmd); + + str = t_str_new(128); + str_append(str, "REMOVE\t"); + str_append_tabescaped(str, ctx->username); + str_append_c(str, '\n'); + replicator_send(ctx, str_c(str)); + + line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + i_error("Replicator disconnected unexpectedly"); + doveadm_exit_code = EX_TEMPFAIL; + } else if (line[0] != '+') { + i_error("Replicator failed: %s", line+1); + doveadm_exit_code = EX_USAGE; + } + replicator_disconnect(ctx); +} + +struct doveadm_cmd_ver2 doveadm_cmd_replicator[] = { +{ + .name = "replicator status", + .cmd = cmd_replicator_status, + .usage = "[-a <replicator socket path>] [<user mask>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "replicator dsync-status", + .cmd = cmd_replicator_dsync_status, + .usage = "[-a <replicator socket path>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "replicator replicate", + .cmd = cmd_replicator_replicate, + .usage = "[-a <replicator socket path>] [-f] [-p <priority>] <user mask>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('f', "full-sync", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('p', "priority", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "replicator add", + .cmd = cmd_replicator_add, + .usage = "[-a <replicator socket path>] <user mask>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +{ + .name = "replicator remove", + .cmd = cmd_replicator_remove, + .usage = "[-a <replicator socket path>] <username>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}, +}; + +static void replicator_cmd_help(const struct doveadm_cmd_ver2 *cmd) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_replicator); i++) { + if (doveadm_cmd_replicator[i].cmd == cmd->cmd) + help_ver2(&doveadm_cmd_replicator[i]); + } + i_unreached(); +} + +void doveadm_register_replicator_commands(void) +{ + unsigned int i; + + for (i = 0; i < N_ELEMENTS(doveadm_cmd_replicator); i++) + doveadm_cmd_register_ver2(&doveadm_cmd_replicator[i]); +} diff --git a/src/doveadm/doveadm-server.h b/src/doveadm/doveadm-server.h new file mode 100644 index 0000000..e111b4a --- /dev/null +++ b/src/doveadm/doveadm-server.h @@ -0,0 +1,34 @@ +#ifndef DOVEADM_SERVER_H +#define DOVEADM_SERVER_H + +extern struct client_connection *doveadm_client; +extern struct doveadm_print_vfuncs doveadm_print_server_vfuncs; + +enum doveadm_proxy_ssl_flags { + /* Use SSL/TLS enabled */ + PROXY_SSL_FLAG_YES = 0x01, + /* Don't do SSL handshake immediately after connected */ + PROXY_SSL_FLAG_STARTTLS = 0x02, + /* Don't require that the received certificate is valid */ + PROXY_SSL_FLAG_ANY_CERT = 0x04 +}; + +struct doveadm_server { + /* hostname:port or socket name for logging */ + const char *name; + /* hostname without port */ + const char *hostname; + /* host ip to use */ + struct ip_addr ip; + /* port to use */ + in_port_t port; + + /* ssl related settings */ + enum doveadm_proxy_ssl_flags ssl_flags; + struct ssl_iostream_context *ssl_ctx; + + ARRAY(struct server_connection *) connections; + ARRAY_TYPE(string) queue; +}; + +#endif diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c new file mode 100644 index 0000000..54d6631 --- /dev/null +++ b/src/doveadm/doveadm-settings.c @@ -0,0 +1,319 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "var-expand.h" +#include "settings-parser.h" +#include "service-settings.h" +#include "mail-storage-settings.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "master-service-ssl-settings.h" +#include "iostream-ssl.h" +#include "doveadm-settings.h" + +ARRAY_TYPE(doveadm_setting_root) doveadm_setting_roots; +bool doveadm_verbose_proctitle; + +static pool_t doveadm_settings_pool = NULL; + +static bool doveadm_settings_check(void *_set, pool_t pool, const char **error_r); + +/* <settings checks> */ +static struct file_listener_settings doveadm_unix_listeners_array[] = { + { "doveadm-server", 0600, "", "" } +}; +static struct file_listener_settings *doveadm_unix_listeners[] = { + &doveadm_unix_listeners_array[0] +}; +static buffer_t doveadm_unix_listeners_buf = { + { { doveadm_unix_listeners, sizeof(doveadm_unix_listeners) } } +}; +/* </settings checks> */ + +struct service_settings doveadm_service_settings = { + .name = "doveadm", + .protocol = "", + .type = "", + .executable = "doveadm-server", + .user = "", + .group = "", + .privileged_group = "", + .extra_groups = "$default_internal_group", + .chroot = "", + + .drop_priv_before_exec = FALSE, + + .process_min_avail = 0, + .process_limit = 0, + .client_limit = 1, + .service_count = 1, + .idle_kill = 0, + .vsz_limit = UOFF_T_MAX, + + .unix_listeners = { { &doveadm_unix_listeners_buf, + sizeof(doveadm_unix_listeners[0]) } }, + .fifo_listeners = ARRAY_INIT, + .inet_listeners = ARRAY_INIT +}; + +#undef DEF +#define DEF(type, name) \ + SETTING_DEFINE_STRUCT_##type(#name, name, struct doveadm_settings) + +static const struct setting_define doveadm_setting_defines[] = { + DEF(STR, base_dir), + DEF(STR, libexec_dir), + DEF(STR, mail_plugins), + DEF(STR, mail_plugin_dir), + DEF(STR_VARS, mail_temp_dir), + DEF(BOOL, auth_debug), + DEF(STR, auth_socket_path), + DEF(STR, doveadm_socket_path), + DEF(UINT, doveadm_worker_count), + DEF(IN_PORT, doveadm_port), + { .type = SET_ALIAS, .key = "doveadm_proxy_port" }, + DEF(ENUM, doveadm_ssl), + DEF(STR, doveadm_username), + DEF(STR, doveadm_password), + DEF(STR, doveadm_allowed_commands), + DEF(STR, dsync_alt_char), + DEF(STR, dsync_remote_cmd), + DEF(STR, director_username_hash), + DEF(STR, doveadm_api_key), + DEF(STR, dsync_features), + DEF(UINT, dsync_commit_msgs_interval), + DEF(STR, doveadm_http_rawlog_dir), + DEF(STR, dsync_hashed_headers), + + { .type = SET_STRLIST, .key = "plugin", + .offset = offsetof(struct doveadm_settings, plugin_envs) }, + + SETTING_DEFINE_LIST_END +}; + +const struct doveadm_settings doveadm_default_settings = { + .base_dir = PKG_RUNDIR, + .libexec_dir = PKG_LIBEXECDIR, + .mail_plugins = "", + .mail_plugin_dir = MODULEDIR, + .mail_temp_dir = "/tmp", + .auth_debug = FALSE, + .auth_socket_path = "auth-userdb", + .doveadm_socket_path = "doveadm-server", + .doveadm_worker_count = 0, + .doveadm_port = 0, + .doveadm_ssl = "no:ssl:starttls", + .doveadm_username = "doveadm", + .doveadm_password = "", + .doveadm_allowed_commands = "", + .dsync_alt_char = "_", + .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U", + .dsync_features = "", + .dsync_hashed_headers = "Date Message-ID", + .dsync_commit_msgs_interval = 100, + .director_username_hash = "%Lu", + .doveadm_api_key = "", + .doveadm_http_rawlog_dir = "", + + .plugin_envs = ARRAY_INIT +}; + +static const struct setting_parser_info *doveadm_setting_dependencies[] = { + &mail_user_setting_parser_info, + NULL +}; + +const struct setting_parser_info doveadm_setting_parser_info = { + .module_name = "doveadm", + .defines = doveadm_setting_defines, + .defaults = &doveadm_default_settings, + + .type_offset = SIZE_MAX, + .struct_size = sizeof(struct doveadm_settings), + + .parent_offset = SIZE_MAX, + .check_func = doveadm_settings_check, + .dependencies = doveadm_setting_dependencies +}; + +struct doveadm_settings *doveadm_settings; +const struct master_service_settings *service_set; + +static void +fix_base_path(struct doveadm_settings *set, pool_t pool, const char **str) +{ + if (*str != NULL && **str != '\0' && **str != '/') + *str = p_strconcat(pool, set->base_dir, "/", *str, NULL); +} + +/* <settings checks> */ +struct dsync_feature_list { + const char *name; + enum dsync_features num; +}; + +static const struct dsync_feature_list dsync_feature_list[] = { + { "empty-header-workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND }, + { NULL, 0 } +}; + +static int +dsync_settings_parse_features(struct doveadm_settings *set, + const char **error_r) +{ + enum dsync_features features = 0; + const struct dsync_feature_list *list; + const char *const *str; + + str = t_strsplit_spaces(set->dsync_features, " ,"); + for (; *str != NULL; str++) { + list = dsync_feature_list; + for (; list->name != NULL; list++) { + if (strcasecmp(*str, list->name) == 0) { + features |= list->num; + break; + } + } + if (list->name == NULL) { + *error_r = t_strdup_printf("dsync_features: " + "Unknown feature: %s", *str); + return -1; + } + } + set->parsed_features = features; + return 0; +} + +static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED, + const char **error_r) +{ + struct doveadm_settings *set = _set; + +#ifndef CONFIG_BINARY + fix_base_path(set, pool, &set->auth_socket_path); + fix_base_path(set, pool, &set->doveadm_socket_path); +#endif + if (*set->dsync_hashed_headers == '\0') { + *error_r = "dsync_hashed_headers must not be empty"; + return FALSE; + } + if (*set->dsync_alt_char == '\0') { + *error_r = "dsync_alt_char must not be empty"; + return FALSE; + } + if (dsync_settings_parse_features(set, error_r) != 0) + return FALSE; + return TRUE; +} +/* </settings checks> */ + +const struct master_service_ssl_settings *doveadm_ssl_set = NULL; + +void doveadm_get_ssl_settings(struct ssl_iostream_settings *set_r, pool_t pool) +{ + master_service_ssl_client_settings_to_iostream_set(doveadm_ssl_set, + pool, set_r); +} + +void doveadm_settings_expand(struct doveadm_settings *set, pool_t pool) +{ + struct var_expand_table tab[] = { { '\0', NULL, NULL } }; + const char *error; + + if (settings_var_expand(&doveadm_setting_parser_info, set, + pool, tab, &error) <= 0) + i_fatal("Failed to expand settings: %s", error); +} + +void doveadm_read_settings(void) +{ + static const struct setting_parser_info *default_set_roots[] = { + &master_service_ssl_setting_parser_info, + &doveadm_setting_parser_info, + }; + struct master_service_settings_input input; + struct master_service_settings_output output; + const struct doveadm_settings *set; + struct doveadm_setting_root *root; + ARRAY(const struct setting_parser_info *) set_roots; + ARRAY_TYPE(const_string) module_names; + void **sets; + const char *error; + + t_array_init(&set_roots, N_ELEMENTS(default_set_roots) + + array_count(&doveadm_setting_roots) + 1); + array_append(&set_roots, default_set_roots, + N_ELEMENTS(default_set_roots)); + t_array_init(&module_names, 4); + array_foreach_modifiable(&doveadm_setting_roots, root) { + array_push_back(&module_names, &root->info->module_name); + array_push_back(&set_roots, &root->info); + } + array_append_zero(&module_names); + array_append_zero(&set_roots); + + i_zero(&input); + input.roots = array_front(&set_roots); + input.module = "doveadm"; + input.extra_modules = array_front(&module_names); + input.service = "doveadm"; + input.preserve_user = TRUE; + input.preserve_home = TRUE; + if (master_service_settings_read(master_service, &input, + &output, &error) < 0) + i_fatal("Error reading configuration: %s", error); + + doveadm_settings_pool = pool_alloconly_create("doveadm settings", 1024); + service_set = master_service_settings_get(master_service); + service_set = settings_dup(&master_service_setting_parser_info, + service_set, doveadm_settings_pool); + doveadm_verbose_proctitle = service_set->verbose_proctitle; + + sets = master_service_settings_get_others(master_service); + set = sets[1]; + doveadm_settings = settings_dup(&doveadm_setting_parser_info, set, + doveadm_settings_pool); + doveadm_ssl_set = settings_dup(&master_service_ssl_setting_parser_info, + master_service_ssl_settings_get(master_service), + doveadm_settings_pool); + doveadm_settings_expand(doveadm_settings, doveadm_settings_pool); + doveadm_settings->parsed_features = set->parsed_features; /* copy this value by hand */ + + array_foreach_modifiable(&doveadm_setting_roots, root) { + unsigned int idx = + array_foreach_idx(&doveadm_setting_roots, root); + root->settings = settings_dup(root->info, sets[2+idx], + doveadm_settings_pool); + } +} + +void doveadm_setting_roots_add(const struct setting_parser_info *info) +{ + struct doveadm_setting_root *root; + + root = array_append_space(&doveadm_setting_roots); + root->info = info; +} + +void *doveadm_setting_roots_get_settings(const struct setting_parser_info *info) +{ + const struct doveadm_setting_root *root; + + array_foreach(&doveadm_setting_roots, root) { + if (root->info == info) + return root->settings; + } + i_panic("Failed to find settings for module %s", info->module_name); +} + +void doveadm_settings_init(void) +{ + i_array_init(&doveadm_setting_roots, 8); +} + +void doveadm_settings_deinit(void) +{ + array_free(&doveadm_setting_roots); + pool_unref(&doveadm_settings_pool); +} diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h new file mode 100644 index 0000000..b3c2184 --- /dev/null +++ b/src/doveadm/doveadm-settings.h @@ -0,0 +1,65 @@ +#ifndef DOVEADM_SETTINGS_H +#define DOVEADM_SETTINGS_H + +#include "net.h" + +struct ssl_iostream_settings; + +/* <settings checks> */ +enum dsync_features { + DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1, +}; +/* </settings checks> */ + +struct doveadm_settings { + const char *base_dir; + const char *libexec_dir; + const char *mail_plugins; + const char *mail_plugin_dir; + const char *mail_temp_dir; + bool auth_debug; + const char *auth_socket_path; + const char *doveadm_socket_path; + unsigned int doveadm_worker_count; + in_port_t doveadm_port; + const char *doveadm_ssl; + const char *doveadm_username; + const char *doveadm_password; + const char *doveadm_allowed_commands; + const char *dsync_alt_char; + const char *dsync_remote_cmd; + const char *director_username_hash; + const char *doveadm_api_key; + const char *dsync_features; + const char *dsync_hashed_headers; + unsigned int dsync_commit_msgs_interval; + const char *doveadm_http_rawlog_dir; + enum dsync_features parsed_features; + ARRAY(const char *) plugin_envs; +}; + +struct doveadm_setting_root { + const struct setting_parser_info *info; + void *settings; +}; +ARRAY_DEFINE_TYPE(doveadm_setting_root, struct doveadm_setting_root); + +extern const struct setting_parser_info doveadm_setting_parser_info; +extern struct doveadm_settings *doveadm_settings; +extern const struct master_service_settings *service_set; +extern const struct master_service_ssl_settings *doveadm_ssl_set; +extern ARRAY_TYPE(doveadm_setting_root) doveadm_setting_roots; +extern bool doveadm_verbose_proctitle; + +void doveadm_get_ssl_settings(struct ssl_iostream_settings *set_r, pool_t pool); +void doveadm_settings_expand(struct doveadm_settings *set, pool_t pool); + +void doveadm_setting_roots_add(const struct setting_parser_info *info); +void *doveadm_setting_roots_get_settings(const struct setting_parser_info *info); + +void doveadm_read_settings(void); + +void doveadm_settings_init(void); +void doveadm_settings_deinit(void); + +#endif diff --git a/src/doveadm/doveadm-sis.c b/src/doveadm/doveadm-sis.c new file mode 100644 index 0000000..85e82e9 --- /dev/null +++ b/src/doveadm/doveadm-sis.c @@ -0,0 +1,330 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "hex-binary.h" +#include "hostpid.h" +#include "randgen.h" +#include "read-full.h" +#include "fs-sis-common.h" +#include "doveadm.h" +#include "doveadm-print.h" + +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <dirent.h> +#include <sys/stat.h> + +/* Files are in <rootdir>/ha/sh/<hash>-<guid> + They may be hard linked to hashes/<hash> +*/ + +static const char *sis_get_dir(const char *rootdir, const char *hash) +{ + if (strlen(hash) < 4 || strchr(hash, '/') != NULL) + i_fatal("Invalid hash in filename: %s", hash); + return t_strdup_printf("%s/%c%c/%c%c", rootdir, + hash[0], hash[1], hash[2], hash[3]); +} + +static int +file_contents_equal(const char *path1, const char *path2, ino_t *path2_inode_r) +{ + struct stat st1, st2; + int fd1, fd2, ret = -1; + + *path2_inode_r = 0; + + /* do a byte-by-byte comparison for the files to find out if they're + the same or if this is a hash collision */ + fd1 = open(path1, O_RDONLY); + if (fd1 == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", path1); + return -1; + } + fd2 = open(path2, O_RDONLY); + if (fd2 == -1) { + if (errno != ENOENT) + i_error("open(%s) failed: %m", path2); + i_close_fd(&fd1); + return -1; + } + + if (fstat(fd1, &st1) < 0) + i_error("fstat(%s) failed: %m", path1); + else if (fstat(fd2, &st2) < 0) + i_error("fstat(%s) failed: %m", path1); + else if (st1.st_size != st2.st_size) + ret = 0; + else { + /* @UNSAFE: sizes match. compare. */ + unsigned char buf1[IO_BLOCK_SIZE], buf2[IO_BLOCK_SIZE]; + ssize_t ret1; + int ret2; + + while ((ret1 = read(fd1, buf1, sizeof(buf1))) > 0) { + i_assert((size_t)ret1 <= sizeof(buf2)); + if ((ret2 = read_full(fd2, buf2, ret1)) <= 0) { + if (ret2 < 0) + i_error("read(%s) failed: %m", path2); + else + ret = 0; + break; + } + if (memcmp(buf1, buf2, ret1) != 0) { + ret = 0; + break; + } + } + if (ret1 < 0) + i_error("read(%s) failed: %m", path1); + else if (ret1 == 0) + ret = 1; + *path2_inode_r = st2.st_ino; + } + + if (close(fd1) < 0) + i_error("close(%s) failed: %m", path1); + if (close(fd2) < 0) + i_error("close(%s) failed: %m", path2); + + return ret; +} + +static int +hardlink_replace(const char *src, const char *dest, ino_t src_inode) +{ + const char *p, *destdir, *tmppath; + unsigned char randbuf[8]; + struct stat st; + + p = strrchr(dest, '/'); + i_assert(p != NULL); + destdir = t_strdup_until(dest, p); + + random_fill(randbuf, sizeof(randbuf)); + tmppath = t_strdup_printf("%s/temp.%s.%s.%s", + destdir, my_hostname, my_pid, + binary_to_hex(randbuf, sizeof(randbuf))); + if (link(src, tmppath) < 0) { + if (errno == EMLINK) + return 0; + i_error("link(%s, %s) failed: %m", src, tmppath); + return -1; + } + if (stat(tmppath, &st) < 0) { + i_error("stat(%s) failed: %m", tmppath); + return -1; + } + if (st.st_ino != src_inode) { + i_unlink(tmppath); + return 0; + } + if (rename(tmppath, dest) < 0) { + i_error("rename(%s, %s) failed: %m", src, tmppath); + i_unlink(tmppath); + return -1; + } + return 1; +} + +static int sis_try_deduplicate(const char *rootdir, const char *fname) +{ + const char *p, *hash, *hashdir, *path, *hashes_dir, *hashes_path; + struct stat st; + ino_t inode; + int ret; + + /* fname should be in <hash>-<guid> format */ + p = strchr(fname, '-'); + i_assert(p != NULL); + + hash = t_strdup_until(fname, p); + hashdir = sis_get_dir(rootdir, hash); + path = t_strdup_printf("%s/%s", hashdir, fname); + + hashes_dir = t_strconcat(hashdir, "/", HASH_DIR_NAME, NULL); + hashes_path = t_strconcat(hashes_dir, "/", hash, NULL); + if (link(path, hashes_path) == 0) { + /* first file with this hash. we're done */ + return 0; + } + if (errno == ENOENT) { + /* either path was already deleted or hashes dir + doesn't exist */ + if (mkdir(hashes_dir, 0700) < 0) { + if (errno == EEXIST) + return 0; + i_error("mkdir(%s) failed: %m", hashes_dir); + return -1; + } + /* try again */ + if (link(path, hashes_path) == 0 || errno == ENOENT) + return 0; + } + if (errno != EEXIST) { + i_error("link(%s, %s) failed: %m", path, hashes_path); + return -1; + } + + /* need to do a byte-by-byte comparison. but check first if someone + else already had deduplicated the file. */ + if (stat(path, &st) < 0) { + if (errno == ENOENT) { + /* just got deleted */ + return 0; + } + i_error("stat(%s) failed: %m", path); + return -1; + } + if (st.st_nlink > 1) { + /* already deduplicated */ + return 0; + } + + ret = file_contents_equal(path, hashes_path, &inode); + if (ret < 0) { + if (errno == ENOENT) { + /* either path or hashes_path was deleted. */ + return sis_try_deduplicate(rootdir, fname); + } + return -1; + } + if (ret > 0) { + /* equal, replace with hard link */ + ret = hardlink_replace(hashes_path, path, inode); + if (ret > 0) + return 0; + else if (ret < 0) + return -1; + /* too many hard links or inode changed */ + } + + /* replace hashes link with this */ + return hardlink_replace(path, hashes_path, st.st_ino) < 0 ? -1 : 0; +} + +static void cmd_sis_deduplicate(struct doveadm_cmd_context *cctx) +{ + const char *rootdir, *queuedir; + DIR *dir; + struct dirent *d; + struct stat st, first_st; + string_t *path; + size_t dir_len; + int ret; + + if (!doveadm_cmd_param_str(cctx, "root-dir", &rootdir) || + !doveadm_cmd_param_str(cctx, "queue-dir", &queuedir)) + help_ver2(&doveadm_cmd_sis_deduplicate); + + /* go through the filenames in the queue dir and see if + we can deduplicate them. */ + if (stat(rootdir, &st) < 0) + i_fatal("stat(%s) failed: %m", rootdir); + + path = t_str_new(256); + str_append(path, queuedir); + str_append_c(path, '/'); + dir_len = str_len(path); + + dir = opendir(queuedir); + if (dir == NULL) + i_fatal("opendir(%s) failed: %m", queuedir); + + first_st.st_size = -1; + while ((d = readdir(dir)) != NULL) { + if (d->d_name[0] == '.') + continue; + + str_truncate(path, dir_len); + str_append(path, d->d_name); + + if (first_st.st_size < 0) { + if (stat(str_c(path), &first_st) < 0) + i_fatal("stat(%s) failed: %m", str_c(path)); + } + if (strchr(d->d_name, '-') == NULL || first_st.st_size != 0) { + i_fatal("%s is not a valid sis-queue file, " + "is the queue directory correct?", + str_c(path)); + } + + T_BEGIN { + ret = sis_try_deduplicate(rootdir, d->d_name); + } T_END; + if (ret == 0) + i_unlink(str_c(path)); + } + if (closedir(dir) < 0) + i_error("closedir(%s) failed: %m", queuedir); +} + +static void cmd_sis_find(struct doveadm_cmd_context *cctx) +{ + const char *rootdir, *path, *hash; + DIR *dir; + struct dirent *d; + struct stat st; + string_t *str; + size_t dir_len, hash_len; + + if (!doveadm_cmd_param_str(cctx, "root-dir", &rootdir) || + !doveadm_cmd_param_str(cctx, "hash", &hash) || + strlen(hash) < 4) + help_ver2(&doveadm_cmd_sis_find); + + if (stat(rootdir, &st) < 0) { + if (errno == ENOENT) + i_fatal("Attachment dir doesn't exist: %s", rootdir); + i_fatal("stat(%s) failed: %m", rootdir); + } + hash_len = strlen(hash); + + path = sis_get_dir(rootdir, hash); + str = t_str_new(256); + str_append(str, path); + str_append_c(str, '/'); + dir_len = str_len(str); + + dir = opendir(path); + if (dir == NULL) { + if (errno == ENOENT) + return; + i_fatal("opendir(%s) failed: %m", path); + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW); + doveadm_print_header("path", "path", + DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE); + while ((d = readdir(dir)) != NULL) { + if (strncmp(d->d_name, hash, hash_len) == 0) { + str_truncate(str, dir_len); + str_append(str, d->d_name); + doveadm_print(str_c(str)); + } + } + if (closedir(dir) < 0) + i_error("closedir(%s) failed: %m", path); +} + +struct doveadm_cmd_ver2 doveadm_cmd_sis_deduplicate = { + .name = "sis deduplicate", + .cmd = cmd_sis_deduplicate, + .usage = "<root dir> <queue dir>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "root-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "queue-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; +struct doveadm_cmd_ver2 doveadm_cmd_sis_find = { + .name = "sis find", + .cmd = cmd_sis_find, + .usage = "<root dir> <hash>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "root-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "hash", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-stats.c b/src/doveadm/doveadm-stats.c new file mode 100644 index 0000000..ddb198b --- /dev/null +++ b/src/doveadm/doveadm-stats.c @@ -0,0 +1,343 @@ +/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "istream.h" +#include "str.h" +#include "strescape.h" +#include "write-full.h" +#include "master-service.h" +#include "doveadm.h" +#include "doveadm-print.h" +#include "stats-settings.h" + +#include <math.h> + +#define DOVEADM_DUMP_DEFAULT_FIELDS \ + "count sum min max avg median stddev %95" + +#define ADD_NAME_PARAM "name" +#define ADD_DESCR_PARAM "description" +#define ADD_FIELDS_PARAM "fields" +#define ADD_GROUPBY_PARAM "group-by" +#define ADD_FILTER_PARAM "filter" +#define ADD_EXPORTER_PARAM "exporter" +#define ADD_EXPORTERINCL_PARAM "exporter-include" + +enum doveadm_dump_field_type { + DOVEADM_DUMP_FIELD_TYPE_PASSTHROUGH = 0, + DOVEADM_DUMP_FIELD_TYPE_STDDEV, +}; + +struct stats_cmd_context { + string_t *cmd; + struct doveadm_cmd_context *cctx; + struct istream *input; + const char *path; + void *data; +}; + +struct dump_data { + const char **fields; + unsigned int field_count; + enum doveadm_dump_field_type *field_types; +}; + +struct stats_cmd_vfuncs { + int (*build_cmd)(struct stats_cmd_context *ctx, const char **error_r); + void (*process_response)(struct stats_cmd_context *ctx); +}; + +static int build_stats_dump_cmd(struct stats_cmd_context *ctx, const char **error_r); +static int build_stats_add_cmd(struct stats_cmd_context *ctx, const char **error_r); +static int build_stats_remove_cmd(struct stats_cmd_context *ctx, const char **error_r); + +static void stats_dump_process_response(struct stats_cmd_context *ctx); +static void stats_modify_process_response(struct stats_cmd_context *ctx); + +static void stats_send_cmd(struct stats_cmd_context *ctx); + +static struct stats_cmd_vfuncs dump_vfuncs = { + .build_cmd = build_stats_dump_cmd, + .process_response = stats_dump_process_response +}; + +static struct stats_cmd_vfuncs add_vfuncs = { + .build_cmd = build_stats_add_cmd, + .process_response = stats_modify_process_response +}; + +static struct stats_cmd_vfuncs remove_vfuncs = { + .build_cmd = build_stats_remove_cmd, + .process_response = stats_modify_process_response +}; + +static string_t *init_stats_cmd(void) +{ + string_t *cmd = t_str_new(128); + str_append(cmd, "VERSION\tstats-reader-client\t2\t0\n"); + return cmd; +} + +static void stats_exec_cmd(struct doveadm_cmd_context *cctx, + struct stats_cmd_vfuncs *vfuncs) +{ + struct stats_cmd_context ctx; + const char *build_cmd_error; + ctx.cctx = cctx; + if (vfuncs->build_cmd(&ctx, &build_cmd_error) < 0) { + i_error("%s", build_cmd_error); + return; + } + stats_send_cmd(&ctx); + vfuncs->process_response(&ctx); + i_stream_destroy(&ctx.input); +} + +static void handle_disconnection(struct stats_cmd_context *ctx) +{ + i_error("read(%s) failed: %s", ctx->path, + i_stream_get_disconnect_reason(ctx->input)); +} + +static void stats_send_cmd(struct stats_cmd_context *ctx) +{ + int fd; + const char *line; + if (!doveadm_cmd_param_str(ctx->cctx, "socket-path", &ctx->path)) + ctx->path = t_strconcat(doveadm_settings->base_dir, + "/stats-reader", NULL); + + fd = doveadm_connect(ctx->path); + net_set_nonblock(fd, FALSE); + if (write_full(fd, str_data(ctx->cmd), str_len(ctx->cmd)) < 0) + i_fatal("write(%s) failed %m", ctx->path); + ctx->input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + + if ((line = i_stream_read_next_line(ctx->input)) == NULL) + i_fatal("%s: Failed to read VERSION line", ctx->path); + else if (!version_string_verify(line, "stats-reader-server", 2)) { + i_fatal_status(EX_PROTOCOL, + "%s is not a compatible stats-reader socket", ctx->path); + } +} + +static void dump_timing(const char *const **args, + const enum doveadm_dump_field_type field_types[], + unsigned int fields_count) +{ + unsigned int i, args_count = str_array_length(*args); + + if (args_count > fields_count) + args_count = fields_count; + for (i = 0; i < args_count; i++) { + const char *value = (*args)[i]; + + switch (field_types[i]) { + case DOVEADM_DUMP_FIELD_TYPE_PASSTHROUGH: + break; + case DOVEADM_DUMP_FIELD_TYPE_STDDEV: { + double variance = strtod(value, NULL); + value = t_strdup_printf("%.02f", sqrt(variance)); + break; + } + } + doveadm_print(value); + } + *args += args_count; +} + +static int build_stats_dump_cmd(struct stats_cmd_context *ctx, + const char **error_r ATTR_UNUSED) +{ + bool reset; + struct dump_data *data = t_new(struct dump_data, 1); + const char *fields_raw; + const char **fields; + if (!doveadm_cmd_param_bool(ctx->cctx, "reset", &reset)) + reset = FALSE; + if (!doveadm_cmd_param_str(ctx->cctx, "fields", &fields_raw)) + fields_raw = DOVEADM_DUMP_DEFAULT_FIELDS; + + fields = t_strsplit_spaces(fields_raw, ", "); + data->fields = fields; + data->field_count = str_array_length(fields); + data->field_types = + t_malloc0(sizeof(enum doveadm_dump_field_type) * data->field_count); + ctx->data = data; + ctx->cmd = init_stats_cmd(); + str_append(ctx->cmd, reset ? "DUMP-RESET" : "DUMP"); + unsigned int i; + for (i = 0; i < data->field_count; i++) { + str_append_c(ctx->cmd, '\t'); + if (strcmp(fields[i], "stddev") == 0) { + data->field_types[i] = DOVEADM_DUMP_FIELD_TYPE_STDDEV; + str_append(ctx->cmd, "variance"); + } else { + str_append_tabescaped(ctx->cmd, fields[i]); + } + } + str_append_c(ctx->cmd, '\n'); + return 0; +} + +static void stats_dump_process_response(struct stats_cmd_context *ctx) +{ + unsigned int i; + char *line; + struct dump_data *data = ctx->data; + + doveadm_print_init(DOVEADM_PRINT_TYPE_TAB); + doveadm_print_header_simple("metric_name"); + doveadm_print_header_simple("field"); + for (i = 0; i < data->field_count; i++) + doveadm_print_header(data->fields[i], data->fields[i], + DOVEADM_PRINT_HEADER_FLAG_NUMBER); + + while ((line = i_stream_read_next_line(ctx->input)) != NULL) { + if (line[0] == '\0') + break; + T_BEGIN { + const char *const *args = + t_strsplit_tabescaped_inplace(line); + + const char *metric_name = args[0]; + doveadm_print(metric_name); args++; + doveadm_print("duration"); + dump_timing(&args, data->field_types, data->field_count); + while (*args != NULL) { + doveadm_print(metric_name); + doveadm_print(*args); args++; + dump_timing(&args, data->field_types, data->field_count); + } + } T_END; + } + if (line == NULL) + handle_disconnection(ctx); +} + +static int build_stats_add_cmd(struct stats_cmd_context *ctx, + const char **error_r) +{ + unsigned int i; + const char *parameter; + struct { + const char *name; + const char *default_val; + } params[] = { + { ADD_NAME_PARAM, "" }, + { ADD_DESCR_PARAM, "" }, + { ADD_FIELDS_PARAM, "" }, + { ADD_GROUPBY_PARAM, "" }, + { ADD_FILTER_PARAM, "" }, + { ADD_EXPORTER_PARAM, "" }, + /* Default exporter-include is to be modified + together with stats-settings */ + { ADD_EXPORTERINCL_PARAM, + STATS_METRIC_SETTINGS_DEFAULT_EXPORTER_INCLUDE }, + }; + + ctx->cmd = init_stats_cmd(); + str_append(ctx->cmd, "METRICS-ADD"); + + for (i = 0; i < N_ELEMENTS(params); i++) { + if (!doveadm_cmd_param_str(ctx->cctx, params[i].name, ¶meter)) + parameter = params[i].default_val; + if (parameter[0] == '\0' && + (strcmp(params[i].name, "name") == 0 || + strcmp(params[i].name, "filter") == 0)) { + *error_r = + t_strdup_printf("stats add: missing %s parameter", + params[i].name); + return -1; + } + str_append_c(ctx->cmd, '\t'); + str_append_tabescaped(ctx->cmd, parameter); + } + + str_append_c(ctx->cmd, '\n'); + return 0; +} + +static void stats_modify_process_response(struct stats_cmd_context *ctx) +{ + const char *line = i_stream_read_next_line(ctx->input); + if (line == NULL) { + handle_disconnection(ctx); + return; + } + if (line[0] == '-') + i_error("%s", ++line); + else if (line[0] != '+') + i_error("Invalid response: %s", line); +} + +static int build_stats_remove_cmd(struct stats_cmd_context *ctx, + const char **error_r) +{ + const char *name; + + ctx->cmd = init_stats_cmd(); + str_append(ctx->cmd, "METRICS-REMOVE\t"); + + if (!doveadm_cmd_param_str(ctx->cctx, "name", &name)) { + *error_r = "stats remove: missing name parameter"; + return -1; + } + str_append_tabescaped(ctx->cmd, name); + str_append_c(ctx->cmd, '\n'); + return 0; +} + +static void doveadm_cmd_stats_dump(struct doveadm_cmd_context *cctx) +{ + stats_exec_cmd(cctx, &dump_vfuncs); +} + +static void doveadm_cmd_stats_add(struct doveadm_cmd_context *cctx) +{ + stats_exec_cmd(cctx, &add_vfuncs); +} + +static void doveadm_cmd_stats_remove(struct doveadm_cmd_context *cctx) +{ + stats_exec_cmd(cctx, &remove_vfuncs); +} + +struct doveadm_cmd_ver2 doveadm_cmd_stats_dump_ver2 = { + .cmd = doveadm_cmd_stats_dump, + .name = "stats dump", + .usage = "[-s <stats socket path>] [-r] [-f <fields>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('r', "reset", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('f', "fields", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_stats_add_ver2 = { + .cmd = doveadm_cmd_stats_add, + .name = "stats add", + .usage = "[--"ADD_DESCR_PARAM" <string>] " + "[--"ADD_EXPORTER_PARAM" <name> [--"ADD_EXPORTERINCL_PARAM" <fields>]] " + "[--"ADD_FIELDS_PARAM" <fields>] " + "[--"ADD_GROUPBY_PARAM" <fields>] <name> <filter>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', ADD_NAME_PARAM, CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', ADD_FILTER_PARAM, CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', ADD_EXPORTER_PARAM, CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', ADD_EXPORTERINCL_PARAM, CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', ADD_DESCR_PARAM, CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', ADD_FIELDS_PARAM, CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('\0', ADD_GROUPBY_PARAM, CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAMS_END +}; + +struct doveadm_cmd_ver2 doveadm_cmd_stats_remove_ver2 = { + .cmd = doveadm_cmd_stats_remove, + .name = "stats remove", + .usage = "<name>", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-util.c b/src/doveadm/doveadm-util.c new file mode 100644 index 0000000..a65ef7f --- /dev/null +++ b/src/doveadm/doveadm-util.c @@ -0,0 +1,221 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "net.h" +#include "time-util.h" +#include "master-service.h" +#include "module-dir.h" +#include "doveadm-settings.h" +#include "doveadm-mail.h" +#include "doveadm-util.h" + +#include <time.h> +#include <dirent.h> +#include <sys/stat.h> +#include <ctype.h> + +#define DOVEADM_TCP_CONNECT_TIMEOUT_SECS 30 + +bool doveadm_verbose = FALSE, doveadm_debug = FALSE, doveadm_server = FALSE; +static struct module *modules = NULL; + +void doveadm_load_modules(void) +{ + struct module_dir_load_settings mod_set; + + /* some doveadm plugins have dependencies to mail plugins. we can load + only those whose dependencies have been loaded earlier, the rest are + ignored. */ + i_zero(&mod_set); + mod_set.abi_version = DOVECOT_ABI_VERSION; + mod_set.require_init_funcs = TRUE; + mod_set.debug = doveadm_debug; + mod_set.ignore_dlopen_errors = TRUE; + + modules = module_dir_load_missing(modules, DOVEADM_MODULEDIR, + NULL, &mod_set); + module_dir_init(modules); +} + +void doveadm_unload_modules(void) +{ + module_dir_unload(&modules); +} + +bool doveadm_has_unloaded_plugin(const char *name) +{ + struct module *module; + DIR *dir; + struct dirent *d; + const char *plugin_name; + size_t name_len = strlen(name); + bool found = FALSE; + + /* first check that it's not actually loaded */ + for (module = modules; module != NULL; module = module->next) { + if (strcmp(module_get_plugin_name(module), name) == 0) + return FALSE; + } + + dir = opendir(DOVEADM_MODULEDIR); + if (dir == NULL) + return FALSE; + + while ((d = readdir(dir)) != NULL) { + plugin_name = module_file_get_name(d->d_name); + if (str_begins(plugin_name, "doveadm_")) + plugin_name += 8; + + if (strncmp(plugin_name, name, name_len) == 0 && + (plugin_name[name_len] == '\0' || + strcmp(plugin_name + name_len, "_plugin") == 0)) { + found = TRUE; + break; + } + } + (void)closedir(dir); + return found; +} + +const char *unixdate2str(time_t timestamp) +{ + return t_strflocaltime("%Y-%m-%d %H:%M:%S", timestamp); +} + +const char *doveadm_plugin_getenv(const char *name) +{ + const char *const *envs; + unsigned int i, count; + + if (!array_is_created(&doveadm_settings->plugin_envs)) + return NULL; + + envs = array_get(&doveadm_settings->plugin_envs, &count); + for (i = 0; i < count; i += 2) { + if (strcmp(envs[i], name) == 0) + return envs[i+1]; + } + return NULL; +} + +static int +doveadm_tcp_connect_port(const char *host, in_port_t port) +{ + struct ip_addr *ips; + unsigned int ips_count; + int ret, fd; + + alarm(DOVEADM_TCP_CONNECT_TIMEOUT_SECS); + ret = net_gethostbyname(host, &ips, &ips_count); + if (ret != 0) { + i_fatal("Lookup of host %s failed: %s", + host, net_gethosterror(ret)); + } + fd = net_connect_ip_blocking(&ips[0], port, NULL); + if (fd == -1) { + i_fatal("connect(%s:%u) failed: %m", + net_ip2addr(&ips[0]), port); + } + alarm(0); + return fd; +} + +int doveadm_tcp_connect(const char *target, in_port_t default_port) +{ + const char *host; + in_port_t port; + + if (net_str2hostport(target, default_port, &host, &port) < 0) { + i_fatal("Port not known for %s. Either set proxy_port " + "or use %s:port", target, target); + } + return doveadm_tcp_connect_port(host, port); +} + +int doveadm_connect_with_default_port(const char *path, + in_port_t default_port) +{ + int fd; + + /* we'll assume UNIX sockets typically have an absolute path, + or at the very least '/' somewhere. */ + if (strchr(path, '/') == NULL) + fd = doveadm_tcp_connect(path, default_port); + else { + fd = net_connect_unix(path); + if (fd == -1) + i_fatal("net_connect_unix(%s) failed: %m", path); + } + return fd; +} + +int doveadm_connect(const char *path) +{ + return doveadm_connect_with_default_port(path, 0); +} + +int i_strccdascmp(const char *a, const char *b) +{ + while(*a != '\0' && *b != '\0') { + if ((*a == ' ' || *a == '-') && *a != *b && *b != ' ' && *b != '-') { + if (i_toupper(*(a+1)) == *(b)) a++; + else break; + } else if ((*b == ' ' || *b == '-') && *a != *b && *a != ' ' && *a != '-') { + if (*a == i_toupper(*(b+1))) b++; + else break; + } else if (!((*a == ' ' || *a == '-') && + (*b == ' ' || *b == '-')) && + (*a != *b)) break; + a++; b++; + } + return *a-*b; +} + +char doveadm_log_type_to_char(enum log_type type) +{ + switch(type) { + case LOG_TYPE_DEBUG: + return '\x01'; + case LOG_TYPE_INFO: + return '\x02'; + case LOG_TYPE_WARNING: + return '\x03'; + case LOG_TYPE_ERROR: + return '\x04'; + case LOG_TYPE_FATAL: + return '\x05'; + case LOG_TYPE_PANIC: + return '\x06'; + default: + i_unreached(); + } +} + +bool doveadm_log_type_from_char(char c, enum log_type *type_r) +{ + switch(c) { + case '\x01': + *type_r = LOG_TYPE_DEBUG; + break; + case '\x02': + *type_r = LOG_TYPE_INFO; + break; + case '\x03': + *type_r = LOG_TYPE_WARNING; + break; + case '\x04': + *type_r = LOG_TYPE_ERROR; + break; + case '\x05': + *type_r = LOG_TYPE_FATAL; + break; + case '\x06': + *type_r = LOG_TYPE_PANIC; + break; + default: + *type_r = LOG_TYPE_WARNING; + return FALSE; + } + return TRUE; +} diff --git a/src/doveadm/doveadm-util.h b/src/doveadm/doveadm-util.h new file mode 100644 index 0000000..8aefd92 --- /dev/null +++ b/src/doveadm/doveadm-util.h @@ -0,0 +1,31 @@ +#ifndef DOVEADM_UTIL_H +#define DOVEADM_UTIL_H + +#include "net.h" + +#define DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR 1 +#define DOVEADM_SERVER_PROTOCOL_VERSION_MINOR 2 +#define DOVEADM_SERVER_PROTOCOL_VERSION_LINE "VERSION\tdoveadm-server\t1\t2" +#define DOVEADM_CLIENT_PROTOCOL_VERSION_LINE "VERSION\tdoveadm-client\t1\t2" + +extern bool doveadm_verbose, doveadm_debug, doveadm_server; + +const char *unixdate2str(time_t timestamp); +const char *doveadm_plugin_getenv(const char *name); +int doveadm_connect(const char *path); +int doveadm_tcp_connect(const char *target, in_port_t default_port); +int doveadm_connect_with_default_port(const char *path, + in_port_t default_port); + +void doveadm_load_modules(void); +void doveadm_unload_modules(void); +bool doveadm_has_unloaded_plugin(const char *name); + +char doveadm_log_type_to_char(enum log_type type) ATTR_PURE; +bool doveadm_log_type_from_char(char c, enum log_type *type_r); + +/* Similar to strcmp(), except "camel case" == "camel-case" == "camelCase". + Otherwise the comparison is case-sensitive. */ +int i_strccdascmp(const char *a, const char *b) ATTR_PURE; + +#endif diff --git a/src/doveadm/doveadm-who.c b/src/doveadm/doveadm-who.c new file mode 100644 index 0000000..b60e515 --- /dev/null +++ b/src/doveadm/doveadm-who.c @@ -0,0 +1,364 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "net.h" +#include "istream.h" +#include "wildcard-match.h" +#include "hash.h" +#include "str.h" +#include "strescape.h" +#include "doveadm.h" +#include "doveadm-print.h" +#include "doveadm-who.h" + +#include <stdio.h> +#include <unistd.h> + +struct who_user { + const char *username; + const char *service; + ARRAY(struct ip_addr) ips; + ARRAY(pid_t) pids; + unsigned int connection_count; +}; + +static void who_user_ip(const struct who_user *user, struct ip_addr *ip_r) +{ + if (array_count(&user->ips) == 0) + i_zero(ip_r); + else { + const struct ip_addr *ip = array_front(&user->ips); + *ip_r = *ip; + } +} + +static unsigned int who_user_hash(const struct who_user *user) +{ + struct ip_addr ip; + unsigned int hash = str_hash(user->service); + + if (user->username[0] != '\0') + hash += str_hash(user->username); + else { + who_user_ip(user, &ip); + hash += net_ip_hash(&ip); + } + return hash; +} + +static int who_user_cmp(const struct who_user *user1, + const struct who_user *user2) +{ + if (strcmp(user1->username, user2->username) != 0) + return 1; + if (strcmp(user1->service, user2->service) != 0) + return 1; + + if (user1->username[0] == '\0') { + /* tracking only IP addresses, not usernames */ + struct ip_addr ip1, ip2; + + who_user_ip(user1, &ip1); + who_user_ip(user2, &ip2); + return net_ip_cmp(&ip1, &ip2); + } + return 0; +} + +static bool +who_user_has_ip(const struct who_user *user, const struct ip_addr *ip) +{ + const struct ip_addr *ex_ip; + + array_foreach(&user->ips, ex_ip) { + if (net_ip_compare(ex_ip, ip)) + return TRUE; + } + return FALSE; +} + +static int who_parse_line(const char *line, struct who_line *line_r) +{ + const char *const *args = t_strsplit_tabescaped(line); + const char *ident = args[0]; + const char *pid_str = args[1]; + const char *refcount_str = args[2]; + const char *p, *ip_str; + + i_zero(line_r); + + /* ident = service/ip/username (imap, pop3) + or service/username (lmtp) */ + p = strchr(ident, '/'); + if (p == NULL) + return -1; + if (str_to_pid(pid_str, &line_r->pid) < 0) + return -1; + line_r->service = t_strdup_until(ident, p++); + line_r->username = strchr(p, '/'); + if (line_r->username == NULL) { + /* no IP */ + line_r->username = p; + } else { + ip_str = t_strdup_until(p, line_r->username++); + (void)net_addr2ip(ip_str, &line_r->ip); + } + if (str_to_uint(refcount_str, &line_r->refcount) < 0) + return -1; + return 0; +} + +static bool who_user_has_pid(struct who_user *user, pid_t pid) +{ + pid_t ex_pid; + + array_foreach_elem(&user->pids, ex_pid) { + if (ex_pid == pid) + return TRUE; + } + return FALSE; +} + +static void who_aggregate_line(struct who_context *ctx, + const struct who_line *line) +{ + struct who_user *user, lookup_user; + + lookup_user.username = line->username; + lookup_user.service = line->service; + + user = hash_table_lookup(ctx->users, &lookup_user); + if (user == NULL) { + user = p_new(ctx->pool, struct who_user, 1); + user->username = p_strdup(ctx->pool, line->username); + user->service = p_strdup(ctx->pool, line->service); + p_array_init(&user->ips, ctx->pool, 3); + p_array_init(&user->pids, ctx->pool, 8); + hash_table_insert(ctx->users, user, user); + } + user->connection_count += line->refcount; + + if (line->ip.family != 0 && !who_user_has_ip(user, &line->ip)) + array_push_back(&user->ips, &line->ip); + + if (!who_user_has_pid(user, line->pid)) + array_push_back(&user->pids, &line->pid); +} + +int who_parse_args(struct who_context *ctx, const char *const *masks) +{ + struct ip_addr net_ip; + unsigned int i, net_bits; + + for (i = 0; masks[i] != NULL; i++) { + if (!str_is_numeric(masks[i], '\0') && + net_parse_range(masks[i], &net_ip, &net_bits) == 0) { + if (ctx->filter.net_bits != 0) { + i_error("Multiple network masks not supported"); + doveadm_exit_code = EX_USAGE; + return -1; + } + ctx->filter.net_ip = net_ip; + ctx->filter.net_bits = net_bits; + } else { + if (ctx->filter.username != NULL) { + i_error("Multiple username masks not supported"); + doveadm_exit_code = EX_USAGE; + return -1; + } + ctx->filter.username = masks[i]; + } + } + return 0; +} + +void who_lookup(struct who_context *ctx, who_callback_t *callback) +{ +#define ANVIL_HANDSHAKE "VERSION\tanvil\t1\t0\n" +#define ANVIL_CMD ANVIL_HANDSHAKE"CONNECT-DUMP\n" + struct istream *input; + const char *line; + int fd; + + fd = doveadm_connect(ctx->anvil_path); + net_set_nonblock(fd, FALSE); + if (write(fd, ANVIL_CMD, strlen(ANVIL_CMD)) < 0) + i_fatal("write(%s) failed: %m", ctx->anvil_path); + + input = i_stream_create_fd_autoclose(&fd, SIZE_MAX); + while ((line = i_stream_read_next_line(input)) != NULL) { + if (*line == '\0') + break; + T_BEGIN { + struct who_line who_line; + + if (who_parse_line(line, &who_line) < 0) + i_error("Invalid input: %s", line); + else + callback(ctx, &who_line); + } T_END; + } + if (input->stream_errno != 0) { + i_fatal("read(%s) failed: %s", ctx->anvil_path, + i_stream_get_error(input)); + } + + i_stream_destroy(&input); +} + +static bool who_user_filter_match(const struct who_user *user, + const struct who_filter *filter) +{ + if (filter->username != NULL) { + if (!wildcard_match_icase(user->username, filter->username)) + return FALSE; + } + if (filter->net_bits > 0) { + const struct ip_addr *ip; + bool ret = FALSE; + + array_foreach(&user->ips, ip) { + if (net_is_in_network(ip, &filter->net_ip, + filter->net_bits)) { + ret = TRUE; + break; + } + } + if (!ret) + return FALSE; + } + return TRUE; +} + +static void who_print_user(const struct who_user *user) +{ + const struct ip_addr *ip; + pid_t pid; + string_t *str = t_str_new(256); + + doveadm_print(user->username); + doveadm_print(dec2str(user->connection_count)); + doveadm_print(user->service); + + str_append_c(str, '('); + array_foreach_elem(&user->pids, pid) + str_printfa(str, "%ld ", (long)pid); + if (str_len(str) > 1) + str_truncate(str, str_len(str)-1); + str_append_c(str, ')'); + doveadm_print(str_c(str)); + + str_truncate(str, 0); + str_append_c(str, '('); + array_foreach(&user->ips, ip) + str_printfa(str, "%s ", net_ip2addr(ip)); + if (str_len(str) > 1) + str_truncate(str, str_len(str)-1); + str_append_c(str, ')'); + doveadm_print(str_c(str)); + + doveadm_print_flush(); +} + +static void who_print(struct who_context *ctx) +{ + struct hash_iterate_context *iter; + struct who_user *user; + + doveadm_print_header("username", "username", 0); + doveadm_print_header("connections", "#", + DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY); + doveadm_print_header("service", "proto", 0); + doveadm_print_header("pids", "(pids)", 0); + doveadm_print_header("ips", "(ips)", 0); + + iter = hash_table_iterate_init(ctx->users); + while (hash_table_iterate(iter, ctx->users, &user, &user)) { + if (who_user_filter_match(user, &ctx->filter)) T_BEGIN { + who_print_user(user); + } T_END; + } + hash_table_iterate_deinit(&iter); +} + +bool who_line_filter_match(const struct who_line *line, + const struct who_filter *filter) +{ + if (filter->username != NULL) { + if (!wildcard_match_icase(line->username, filter->username)) + return FALSE; + } + if (filter->net_bits > 0) { + if (!net_is_in_network(&line->ip, &filter->net_ip, + filter->net_bits)) + return FALSE; + } + return TRUE; +} + +static void who_print_line(struct who_context *ctx, + const struct who_line *line) +{ + unsigned int i; + + if (!who_line_filter_match(line, &ctx->filter)) + return; + + for (i = 0; i < line->refcount; i++) T_BEGIN { + doveadm_print(line->username); + doveadm_print(line->service); + doveadm_print(dec2str(line->pid)); + doveadm_print(net_ip2addr(&line->ip)); + } T_END; +} + +static void cmd_who(struct doveadm_cmd_context *cctx) +{ + const char *const *masks; + struct who_context ctx; + bool separate_connections = FALSE; + + i_zero(&ctx); + if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx.anvil_path))) + ctx.anvil_path = t_strconcat(doveadm_settings->base_dir, "/anvil", NULL); + (void)doveadm_cmd_param_bool(cctx, "separate-connections", &separate_connections); + + ctx.pool = pool_alloconly_create("who users", 10240); + hash_table_create(&ctx.users, ctx.pool, 0, who_user_hash, who_user_cmp); + + if (doveadm_cmd_param_array(cctx, "mask", &masks)) { + if (who_parse_args(&ctx, masks) != 0) { + hash_table_destroy(&ctx.users); + pool_unref(&ctx.pool); + return; + } + } + + doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE); + if (!separate_connections) { + who_lookup(&ctx, who_aggregate_line); + who_print(&ctx); + } else { + doveadm_print_header("username", "username", + DOVEADM_PRINT_HEADER_FLAG_EXPAND); + doveadm_print_header("service", "proto", 0); + doveadm_print_header_simple("pid"); + doveadm_print_header_simple("ip"); + who_lookup(&ctx, who_print_line); + } + + hash_table_destroy(&ctx.users); + pool_unref(&ctx.pool); +} + +struct doveadm_cmd_ver2 doveadm_cmd_who_ver2 = { + .name = "who", + .cmd = cmd_who, + .usage = "[-a <anvil socket path>] [-1] [<user mask>] [<ip/bits>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('a',"socket-path", CMD_PARAM_STR, 0) +DOVEADM_CMD_PARAM('1',"separate-connections", CMD_PARAM_BOOL, 0) +DOVEADM_CMD_PARAM('\0',"mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm-who.h b/src/doveadm/doveadm-who.h new file mode 100644 index 0000000..57c663a --- /dev/null +++ b/src/doveadm/doveadm-who.h @@ -0,0 +1,37 @@ +#ifndef DOVEADM_WHO_H +#define DOVEADM_WHO_H + +struct who_line { + const char *username; + const char *service; + struct ip_addr ip; + pid_t pid; + unsigned int refcount; +}; + + +struct who_filter { + const char *username; + struct ip_addr net_ip; + unsigned int net_bits; +}; + +struct who_context { + const char *anvil_path; + struct who_filter filter; + + pool_t pool; + HASH_TABLE(struct who_user *, struct who_user *) users; +}; + +typedef void who_callback_t(struct who_context *ctx, + const struct who_line *line); + +int who_parse_args(struct who_context *ctx, const char *const *masks); + +void who_lookup(struct who_context *ctx, who_callback_t *callback); + +bool who_line_filter_match(const struct who_line *line, + const struct who_filter *filter); + +#endif /* DOVEADM_WHO_H */ diff --git a/src/doveadm/doveadm-zlib.c b/src/doveadm/doveadm-zlib.c new file mode 100644 index 0000000..f9bb233 --- /dev/null +++ b/src/doveadm/doveadm-zlib.c @@ -0,0 +1,297 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "net.h" +#include "istream.h" +#include "ostream.h" +#include "istream-zlib.h" +#include "ostream-zlib.h" +#include "module-dir.h" +#include "master-service.h" +#include "compression.h" +#include "doveadm-dump.h" + +#include <stdio.h> +#include <fcntl.h> +#include <unistd.h> + +static bool test_dump_imapzlib(const char *path) +{ + const char *p; + char buf[4096]; + int fd, ret; + bool match = FALSE; + + p = strrchr(path, '.'); + if (p == NULL || (strcmp(p, ".in") != 0 && strcmp(p, ".out") != 0)) + return FALSE; + + fd = open(path, O_RDONLY); + if (fd == -1) + return FALSE; + + ret = read(fd, buf, sizeof(buf)-1); + if (ret > 0) { + buf[ret] = '\0'; + (void)str_lcase(buf); + match = strstr(buf, " ok begin compression.") != NULL || + strstr(buf, " compress deflate") != NULL; + } + i_close_fd(&fd); + return match; +} + +#ifdef HAVE_ZLIB +static void +cmd_dump_imapzlib(const char *path, const char *const *args ATTR_UNUSED) +{ + struct istream *input, *input2; + const unsigned char *data; + size_t size; + const char *line; + int fd; + + fd = open(path, O_RDONLY); + if (fd < 0) + i_fatal("open(%s) failed: %m", path); + input = i_stream_create_fd_autoclose(&fd, 1024*32); + while ((line = i_stream_read_next_line(input)) != NULL) { + /* skip tag */ + printf("%s\r\n", line); + while (*line != ' ' && *line != '\0') line++; + if (*line == '\0') + continue; + line++; + + if (str_begins(line, "OK Begin compression") || + strcasecmp(line, "COMPRESS DEFLATE") == 0) + break; + } + + input2 = i_stream_create_deflate(input); + i_stream_unref(&input); + + while (i_stream_read_more(input2, &data, &size) != -1) { + if (fwrite(data, 1, size, stdout) != size) + break; + i_stream_skip(input2, size); + } + if (input2->stream_errno != 0) + i_error("read(%s) failed: %s", path, i_stream_get_error(input)); + i_stream_unref(&input2); + fflush(stdout); +} + +struct client { + int fd; + struct io *io_client, *io_server; + struct istream *input, *stdin_input; + struct ostream *output; + const struct compression_handler *handler; + char *algorithm; + bool compressed; + bool compress_waiting; +}; + +static bool +client_input_get_compress_algorithm(struct client *client, const char *line) +{ + /* skip tag */ + while (*line != ' ' && *line != '\0') + line++; + if (strncasecmp(line, " COMPRESS ", 10) != 0) + return FALSE; + + if (compression_lookup_handler(t_str_lcase(line+10), + &client->handler) <= 0) + i_fatal("Unsupported compression mechanism: %s", line+10); + return TRUE; +} + +static bool client_input_uncompressed(struct client *client) +{ + const char *line; + + if (client->compress_waiting) { + /* just read all the pipelined input for now */ + (void)i_stream_read(client->stdin_input); + return TRUE; + } + + while ((line = i_stream_read_next_line(client->stdin_input)) != NULL) { + o_stream_nsend_str(client->output, line); + o_stream_nsend(client->output, "\n", 1); + if (client_input_get_compress_algorithm(client, line)) + return TRUE; + } + return FALSE; +} + +static void client_input(struct client *client) +{ + const unsigned char *data; + size_t size; + + if (!client->compressed && + client_input_uncompressed(client)) { + /* stop until server has sent reply to COMPRESS command. */ + client->compress_waiting = TRUE; + return; + } + if (client->compressed) { + if (i_stream_read_more(client->stdin_input, &data, &size) > 0) { + o_stream_nsend(client->output, data, size); + i_stream_skip(client->stdin_input, size); + } + if (o_stream_flush(client->output) < 0) { + i_fatal("write() failed: %s", + o_stream_get_error(client->output)); + } + } + if (client->stdin_input->eof) { + if (client->stdin_input->stream_errno != 0) { + i_fatal("read(stdin) failed: %s", + i_stream_get_error(client->stdin_input)); + } + master_service_stop(master_service); + } +} + +static bool server_input_is_compress_reply(const char *line) +{ + /* skip tag */ + while (*line != ' ' && *line != '\0') + line++; + return str_begins(line, " OK Begin compression"); +} + +static bool server_input_uncompressed(struct client *client) +{ + const char *line; + + while ((line = i_stream_read_next_line(client->input)) != NULL) { + if (write(STDOUT_FILENO, line, strlen(line)) < 0) + i_fatal("write(stdout) failed: %m"); + if (write(STDOUT_FILENO, "\n", 1) < 0) + i_fatal("write(stdout) failed: %m"); + if (server_input_is_compress_reply(line)) + return TRUE; + } + return FALSE; +} + +static void server_input(struct client *client) +{ + const unsigned char *data; + size_t size; + + if (i_stream_read(client->input) == -1) { + if (client->input->stream_errno != 0) { + i_fatal("read(server) failed: %s", + i_stream_get_error(client->input)); + } + + i_info("Server disconnected"); + master_service_stop(master_service); + return; + } + + if (!client->compressed && server_input_uncompressed(client)) { + /* start compression */ + struct istream *input; + struct ostream *output; + + i_info("<Compression started>"); + input = client->handler->create_istream(client->input); + output = client->handler->create_ostream(client->output, 6); + i_stream_unref(&client->input); + o_stream_unref(&client->output); + client->input = input; + client->output = output; + client->compressed = TRUE; + client->compress_waiting = FALSE; + i_stream_set_input_pending(client->stdin_input, TRUE); + } + + data = i_stream_get_data(client->input, &size); + if (write(STDOUT_FILENO, data, size) < 0) + i_fatal("write(stdout) failed: %m"); + i_stream_skip(client->input, size); +} + +static void cmd_zlibconnect(struct doveadm_cmd_context *cctx) +{ + struct client client; + const char *host; + struct ip_addr *ips; + unsigned int ips_count; + int64_t port_int64; + in_port_t port = 143; + int fd, ret; + + if (!doveadm_cmd_param_str(cctx, "host", &host)) + help_ver2(&doveadm_cmd_zlibconnect); + if (doveadm_cmd_param_int64(cctx, "port", &port_int64)) { + if (port_int64 == 0 || port_int64 > 65535) + i_fatal("Invalid port: %"PRId64, port_int64); + port = (in_port_t)port_int64; + } + + ret = net_gethostbyname(host, &ips, &ips_count); + if (ret != 0) { + i_fatal("Host %s lookup failed: %s", host, + net_gethosterror(ret)); + } + + if ((fd = net_connect_ip(&ips[0], port, NULL)) == -1) + i_fatal("connect(%s, %u) failed: %m", host, port); + + i_info("Connected to %s port %u.", net_ip2addr(&ips[0]), port); + + i_zero(&client); + client.fd = fd; + fd_set_nonblock(STDIN_FILENO, TRUE); + client.stdin_input = i_stream_create_fd(STDIN_FILENO, SIZE_MAX); + client.input = i_stream_create_fd(fd, SIZE_MAX); + client.output = o_stream_create_fd(fd, 0); + o_stream_set_no_error_handling(client.output, TRUE); + client.io_client = io_add_istream(client.stdin_input, client_input, &client); + client.io_server = io_add_istream(client.input, server_input, &client); + master_service_run(master_service, NULL); + io_remove(&client.io_client); + io_remove(&client.io_server); + i_stream_unref(&client.stdin_input); + i_stream_unref(&client.input); + o_stream_unref(&client.output); + if (close(fd) < 0) + i_fatal("close() failed: %m"); +} +#else +static void +cmd_dump_imapzlib(const char *path ATTR_UNUSED, + const char *const *args ATTR_UNUSED) +{ + i_fatal("Dovecot compiled without zlib support"); +} + +static void cmd_zlibconnect(struct doveadm_cmd_context *cctx ATTR_UNUSED) +{ + i_fatal("Dovecot compiled without zlib support"); +} +#endif + +struct doveadm_cmd_dump doveadm_cmd_dump_zlib = { + "imapzlib", + test_dump_imapzlib, + cmd_dump_imapzlib +}; + +struct doveadm_cmd_ver2 doveadm_cmd_zlibconnect = { + .name = "zlibconnect", + .cmd = cmd_zlibconnect, + .usage = "<host> [<port>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_INT64, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; diff --git a/src/doveadm/doveadm.c b/src/doveadm/doveadm.c new file mode 100644 index 0000000..f175f2c --- /dev/null +++ b/src/doveadm/doveadm.c @@ -0,0 +1,384 @@ +/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "sort.h" +#include "ostream.h" +#include "env-util.h" +#include "execv-const.h" +#include "dict.h" +#include "master-service-private.h" +#include "master-service-settings.h" +#include "settings-parser.h" +#include "doveadm-print-private.h" +#include "doveadm-dump.h" +#include "doveadm-mail.h" +#include "doveadm-settings.h" +#include "doveadm-dsync.h" +#include "doveadm.h" + +#include <unistd.h> + +const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[] = { + &doveadm_print_flow_vfuncs, + &doveadm_print_tab_vfuncs, + &doveadm_print_table_vfuncs, + &doveadm_print_pager_vfuncs, + &doveadm_print_json_vfuncs, + &doveadm_print_formatted_vfuncs, + NULL +}; + +int doveadm_exit_code = 0; + +static void failure_exit_callback(int *status) +{ + enum fatal_exit_status fatal_status = *status; + + switch (fatal_status) { + case FATAL_LOGWRITE: + case FATAL_LOGERROR: + case FATAL_LOGOPEN: + case FATAL_OUTOFMEM: + case FATAL_EXEC: + case FATAL_DEFAULT: + *status = EX_TEMPFAIL; + break; + } +} + +static void +doveadm_usage_compress_lines(FILE *out, const char *str, const char *prefix) +{ + const char *cmd, *args, *p, *short_name, *sub_name; + const char *prev_name = "", *prev_sub_name = ""; + const char **lines; + unsigned int i, count; + size_t prefix_len = strlen(prefix); + + /* split lines */ + lines = (void *)p_strsplit(pool_datastack_create(), str, "\n"); + for (count = 0; lines[count] != NULL; count++) ; + + /* sort lines */ + i_qsort(lines, count, sizeof(*lines), i_strcmp_p); + + /* print lines, compress subcommands into a single line */ + for (i = 0; i < count; i++) { + args = strchr(lines[i], '\t'); + if (args == NULL) { + cmd = lines[i]; + args = ""; + } else { + cmd = t_strdup_until(lines[i], args); + args++; + } + if (*prefix != '\0') { + if (strncmp(cmd, prefix, prefix_len) != 0 || + cmd[prefix_len] != ' ') + continue; + cmd += prefix_len + 1; + } + + p = strchr(cmd, ' '); + if (p == NULL) { + if (*prev_name != '\0') { + fprintf(out, "\n"); + prev_name = ""; + } + fprintf(out, USAGE_CMDNAME_FMT" %s\n", cmd, args); + } else { + short_name = t_strdup_until(cmd, p); + if (strcmp(prev_name, short_name) != 0) { + if (*prev_name != '\0') + fprintf(out, "\n"); + fprintf(out, USAGE_CMDNAME_FMT" %s", + short_name, t_strcut(p + 1, ' ')); + prev_name = short_name; + prev_sub_name = ""; + } else { + sub_name = t_strcut(p + 1, ' '); + if (strcmp(prev_sub_name, sub_name) != 0) { + fprintf(out, "|%s", sub_name); + prev_sub_name = sub_name; + } + } + } + } + if (*prev_name != '\0') + fprintf(out, "\n"); +} + +static void ATTR_NORETURN +usage_prefix(const char *prefix) +{ + const struct doveadm_cmd_ver2 *cmd2; + string_t *str = t_str_new(1024); + + fprintf(stderr, "usage: doveadm [-Dv] [-f <formatter>] "); + if (*prefix != '\0') + fprintf(stderr, "%s ", prefix); + fprintf(stderr, "<command> [<args>]\n"); + + array_foreach(&doveadm_cmds_ver2, cmd2) + str_printfa(str, "%s\t%s\n", cmd2->name, cmd2->usage); + + doveadm_usage_compress_lines(stderr, str_c(str), prefix); + + lib_exit(EX_USAGE); +} + +void usage(void) +{ + usage_prefix(""); +} + +void help_ver2(const struct doveadm_cmd_ver2 *cmd) +{ + fprintf(stderr, "doveadm %s %s\n", cmd->name, cmd->usage); + lib_exit(EX_USAGE); +} + +static void cmd_help(struct doveadm_cmd_context *cctx) +{ + const char *cmd, *man_argv[3]; + + if (!doveadm_cmd_param_str(cctx, "cmd", &cmd)) + usage_prefix(""); + + env_put("MANPATH", MANDIR); + man_argv[0] = "man"; + man_argv[1] = t_strconcat("doveadm-", cmd, NULL); + man_argv[2] = NULL; + execvp_const(man_argv[0], man_argv); +} + +static struct doveadm_cmd_ver2 doveadm_cmd_help = { + .name = "help", + .cmd = cmd_help, + .usage = "[<cmd>]", +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "cmd", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +static void cmd_config(struct doveadm_cmd_context *cctx) +{ + const char *const *args, **argv; + + if (!doveadm_cmd_param_array(cctx, "args", &args)) + args = NULL; + + env_put(MASTER_CONFIG_FILE_ENV, + master_service_get_config_path(master_service)); + + unsigned int len = str_array_length(args); + argv = t_new(const char *, len + 2); + argv[0] = BINDIR"/doveconf"; + if (len > 0) { + i_assert(args != NULL); + memcpy(argv+1, args, len * sizeof(args[0])); + } + execv_const(argv[0], argv); +} + +static struct doveadm_cmd_ver2 doveadm_cmd_config = { + .name = "config", + .cmd = cmd_config, + .usage = "[doveconf parameters]", + .flags = CMD_FLAG_NO_OPTIONS, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +static void cmd_exec(struct doveadm_cmd_context *cctx); +static struct doveadm_cmd_ver2 doveadm_cmd_exec = { + .name = "exec", + .cmd = cmd_exec, + .usage = "<binary> [binary parameters]", + .flags = CMD_FLAG_NO_OPTIONS, +DOVEADM_CMD_PARAMS_START +DOVEADM_CMD_PARAM('\0', "binary", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL) +DOVEADM_CMD_PARAMS_END +}; + +static void cmd_exec(struct doveadm_cmd_context *cctx) +{ + const char *path, *binary, *const *args, **argv; + + if (!doveadm_cmd_param_str(cctx, "binary", &binary)) + help_ver2(&doveadm_cmd_exec); + if (!doveadm_cmd_param_array(cctx, "args", &args)) + args = NULL; + + path = t_strdup_printf("%s/%s", doveadm_settings->libexec_dir, binary); + + unsigned int len = str_array_length(args); + argv = t_new(const char *, len + 2); + argv[0] = path; + if (len > 0) { + i_assert(args != NULL); + memcpy(argv+1, args, len * sizeof(args[0])); + } + execv_const(argv[0], argv); +} + +static bool doveadm_has_subcommands(const char *cmd_name) +{ + const struct doveadm_cmd_ver2 *cmd2; + size_t len = strlen(cmd_name); + + array_foreach(&doveadm_cmds_ver2, cmd2) { + if (strncmp(cmd2->name, cmd_name, len) == 0 && + cmd2->name[len] == ' ') + return TRUE; + } + return FALSE; +} + +static struct doveadm_cmd_ver2 *doveadm_cmdline_commands_ver2[] = { + &doveadm_cmd_config, + &doveadm_cmd_dump, + &doveadm_cmd_exec, + &doveadm_cmd_help, + &doveadm_cmd_oldstats_top_ver2, + &doveadm_cmd_pw, + &doveadm_cmd_zlibconnect, +}; + +int main(int argc, char *argv[]) +{ + enum master_service_flags service_flags = + MASTER_SERVICE_FLAG_STANDALONE | + MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN | + MASTER_SERVICE_FLAG_NO_SSL_INIT | + MASTER_SERVICE_FLAG_NO_INIT_DATASTACK_FRAME; + struct doveadm_cmd_context cctx; + const char *cmd_name; + unsigned int i; + bool quick_init = FALSE; + int c; + + i_zero(&cctx); + cctx.conn_type = DOVEADM_CONNECTION_TYPE_CLI; + + i_set_failure_exit_callback(failure_exit_callback); + doveadm_dsync_main(&argc, &argv); + + /* "+" is GNU extension to stop at the first non-option. + others just accept -+ option. */ + master_service = master_service_init("doveadm", service_flags, + &argc, &argv, "+Df:hv"); + while ((c = master_getopt(master_service)) > 0) { + switch (c) { + case 'D': + doveadm_debug = TRUE; + doveadm_verbose = TRUE; + break; + case 'f': + doveadm_print_init(optarg); + break; + case 'h': + doveadm_print_hide_titles = TRUE; + break; + case 'v': + doveadm_verbose = TRUE; + break; + default: + return FATAL_DEFAULT; + } + } + cmd_name = argv[optind]; + + if (cmd_name != NULL && strcmp(cmd_name, "help") == 0 && + argv[optind+1] != NULL) { + /* "help cmd" doesn't need any configuration */ + quick_init = TRUE; + } + master_service_init_log(master_service); + + doveadm_settings_init(); + doveadm_cmds_init(); + for (i = 0; i < N_ELEMENTS(doveadm_cmdline_commands_ver2); i++) + doveadm_cmd_register_ver2(doveadm_cmdline_commands_ver2[i]); + doveadm_register_auth_commands(); + + if (cmd_name != NULL && (quick_init || + strcmp(cmd_name, "config") == 0 || + strcmp(cmd_name, "stop") == 0 || + strcmp(cmd_name, "reload") == 0)) { + /* special case commands: even if there is something wrong + with the config (e.g. mail_plugins), don't fail these + commands */ + if (strcmp(cmd_name, "help") != 0) + doveadm_read_settings(); + quick_init = TRUE; + } else { + quick_init = FALSE; + doveadm_print_ostream = o_stream_create_fd(STDOUT_FILENO, 0); + o_stream_set_no_error_handling(doveadm_print_ostream, TRUE); + o_stream_cork(doveadm_print_ostream); + doveadm_dump_init(); + doveadm_mail_init(); + dict_drivers_register_builtin(); + doveadm_load_modules(); + + /* read settings only after loading doveadm plugins, which + may modify what settings are read */ + doveadm_read_settings(); + if (doveadm_debug && getenv("LOG_STDERR_TIMESTAMP") == NULL) + i_set_failure_timestamp_format(master_service->set->log_timestamp); + master_service_init_stats_client(master_service, TRUE); + /* Load mail_plugins */ + doveadm_mail_init_finish(); + /* kludgy: Load the rest of the doveadm plugins after + mail_plugins have been loaded. */ + doveadm_load_modules(); + + if (cmd_name == NULL) { + /* show usage after registering all plugins */ + usage_prefix(""); + } + } + + argc -= optind; + argv += optind; + i_getopt_reset(); + + master_service_init_finish(master_service); + if (!doveadm_debug) { + /* disable debugging unless -D is given */ + i_set_debug_file("/dev/null"); + } + + /* this has to be done here because proctitle hack can break + the env pointer */ + cctx.username = getenv("USER"); + + if (!doveadm_cmd_try_run_ver2(cmd_name, argc, (const char**)argv, &cctx)) { + if (doveadm_has_subcommands(cmd_name)) + usage_prefix(cmd_name); + if (doveadm_has_unloaded_plugin(cmd_name)) { + i_fatal("Unknown command '%s', but plugin %s exists. " + "Try to set mail_plugins=%s", + cmd_name, cmd_name, cmd_name); + } + usage(); + } + + if (!quick_init) { + doveadm_mail_deinit(); + doveadm_dump_deinit(); + doveadm_unload_modules(); + dict_drivers_unregister_builtin(); + doveadm_print_deinit(); + o_stream_unref(&doveadm_print_ostream); + } + doveadm_cmds_deinit(); + doveadm_settings_deinit(); + master_service_deinit(&master_service); + return doveadm_exit_code; +} diff --git a/src/doveadm/doveadm.h b/src/doveadm/doveadm.h new file mode 100644 index 0000000..d4f74e5 --- /dev/null +++ b/src/doveadm/doveadm.h @@ -0,0 +1,34 @@ +#ifndef DOVEADM_H +#define DOVEADM_H + +#include <sysexits.h> +#include "doveadm-util.h" +#include "doveadm-settings.h" + +#define USAGE_CMDNAME_FMT " %-12s" + +#define DOVEADM_EX_NOTFOUND EX_NOHOST +#define DOVEADM_EX_NOTPOSSIBLE EX_DATAERR +#define DOVEADM_EX_UNKNOWN -1 + +#define DOVEADM_EX_NOREPLICATE 1001 + +enum doveadm_client_type { + DOVEADM_CONNECTION_TYPE_CLI = 0, + DOVEADM_CONNECTION_TYPE_TCP, + DOVEADM_CONNECTION_TYPE_HTTP, +}; + +#include "doveadm-cmd.h" + +extern bool doveadm_verbose_proctitle; +extern int doveadm_exit_code; + +void usage(void) ATTR_NORETURN; +void help_ver2(const struct doveadm_cmd_ver2 *cmd) ATTR_NORETURN; +void doveadm_master_send_signal(int signo); + +const char *doveadm_exit_code_to_str(int code); +int doveadm_str_to_exit_code(const char *reason); + +#endif diff --git a/src/doveadm/dsync/Makefile.am b/src/doveadm/dsync/Makefile.am new file mode 100644 index 0000000..20cbfc7 --- /dev/null +++ b/src/doveadm/dsync/Makefile.am @@ -0,0 +1,76 @@ +pkglib_LTLIBRARIES = libdovecot-dsync.la +noinst_LTLIBRARIES = libdsync.la + +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +libdsync_la_SOURCES = \ + dsync-brain.c \ + dsync-brain-mailbox.c \ + dsync-brain-mailbox-tree.c \ + dsync-brain-mailbox-tree-sync.c \ + dsync-brain-mails.c \ + dsync-deserializer.c \ + dsync-mail.c \ + dsync-mailbox.c \ + dsync-mailbox-import.c \ + dsync-mailbox-export.c \ + dsync-mailbox-state.c \ + dsync-mailbox-tree.c \ + dsync-mailbox-tree-fill.c \ + dsync-mailbox-tree-sync.c \ + dsync-serializer.c \ + dsync-ibc.c \ + dsync-ibc-stream.c \ + dsync-ibc-pipe.c \ + dsync-transaction-log-scan.c + +libdovecot_dsync_la_SOURCES = +libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la +libdovecot_dsync_la_DEPENDENCIES = libdsync.la +libdovecot_dsync_la_LDFLAGS = -export-dynamic + +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + dsync-brain.h \ + dsync-ibc.h + +noinst_HEADERS = \ + dsync-brain-private.h \ + dsync-mail.h \ + dsync-mailbox.h \ + dsync-mailbox-import.h \ + dsync-mailbox-export.h \ + dsync-mailbox-state.h \ + dsync-mailbox-tree.h \ + dsync-mailbox-tree-private.h \ + dsync-serializer.h \ + dsync-deserializer.h \ + dsync-ibc-private.h \ + dsync-transaction-log-scan.h + +test_programs = \ + test-dsync-mailbox-tree-sync + +noinst_PROGRAMS = $(test_programs) + +test_libs = \ + ../../lib-test/libtest.la \ + ../../lib/liblib.la + +test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c +test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs) +test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs) + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done diff --git a/src/doveadm/dsync/Makefile.in b/src/doveadm/dsync/Makefile.in new file mode 100644 index 0000000..ba39895 --- /dev/null +++ b/src/doveadm/dsync/Makefile.in @@ -0,0 +1,1019 @@ +# Makefile.in generated by automake 1.16.1 from Makefile.am. +# @configure_input@ + +# Copyright (C) 1994-2018 Free Software Foundation, Inc. + +# This Makefile.in is free software; the Free Software Foundation +# gives unlimited permission to copy and/or distribute it, +# with or without modifications, as long as this notice is preserved. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY, to the extent permitted by law; without +# even the implied warranty of MERCHANTABILITY or FITNESS FOR A +# PARTICULAR PURPOSE. + +@SET_MAKE@ + + + +VPATH = @srcdir@ +am__is_gnu_make = { \ + if test -z '$(MAKELEVEL)'; then \ + false; \ + elif test -n '$(MAKE_HOST)'; then \ + true; \ + elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ + true; \ + else \ + false; \ + fi; \ +} +am__make_running_with_option = \ + case $${target_option-} in \ + ?) ;; \ + *) echo "am__make_running_with_option: internal error: invalid" \ + "target option '$${target_option-}' specified" >&2; \ + exit 1;; \ + esac; \ + has_opt=no; \ + sane_makeflags=$$MAKEFLAGS; \ + if $(am__is_gnu_make); then \ + sane_makeflags=$$MFLAGS; \ + else \ + case $$MAKEFLAGS in \ + *\\[\ \ ]*) \ + bs=\\; \ + sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ + | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ + esac; \ + fi; \ + skip_next=no; \ + strip_trailopt () \ + { \ + flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ + }; \ + for flg in $$sane_makeflags; do \ + test $$skip_next = yes && { skip_next=no; continue; }; \ + case $$flg in \ + *=*|--*) continue;; \ + -*I) strip_trailopt 'I'; skip_next=yes;; \ + -*I?*) strip_trailopt 'I';; \ + -*O) strip_trailopt 'O'; skip_next=yes;; \ + -*O?*) strip_trailopt 'O';; \ + -*l) strip_trailopt 'l'; skip_next=yes;; \ + -*l?*) strip_trailopt 'l';; \ + -[dEDm]) skip_next=yes;; \ + -[JT]) skip_next=yes;; \ + esac; \ + case $$flg in \ + *$$target_option*) has_opt=yes; break;; \ + esac; \ + done; \ + test $$has_opt = yes +am__make_dryrun = (target_option=n; $(am__make_running_with_option)) +am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) +pkgdatadir = $(datadir)/@PACKAGE@ +pkgincludedir = $(includedir)/@PACKAGE@ +pkglibdir = $(libdir)/@PACKAGE@ +pkglibexecdir = $(libexecdir)/@PACKAGE@ +am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd +install_sh_DATA = $(install_sh) -c -m 644 +install_sh_PROGRAM = $(install_sh) -c +install_sh_SCRIPT = $(install_sh) -c +INSTALL_HEADER = $(INSTALL_DATA) +transform = $(program_transform_name) +NORMAL_INSTALL = : +PRE_INSTALL = : +POST_INSTALL = : +NORMAL_UNINSTALL = : +PRE_UNINSTALL = : +POST_UNINSTALL = : +build_triplet = @build@ +host_triplet = @host@ +noinst_PROGRAMS = $(am__EXEEXT_1) +subdir = src/doveadm/dsync +ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 +am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \ + $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \ + $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \ + $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \ + $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \ + $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \ + $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \ + $(top_srcdir)/m4/flexible_array_member.m4 \ + $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \ + $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \ + $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \ + $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \ + $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \ + $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \ + $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ + $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \ + $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \ + $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \ + $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \ + $(top_srcdir)/m4/pr_set_dumpable.m4 \ + $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \ + $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \ + $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \ + $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \ + $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \ + $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \ + $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \ + $(top_srcdir)/m4/typeof_dev_t.m4 \ + $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \ + $(top_srcdir)/m4/want_apparmor.m4 \ + $(top_srcdir)/m4/want_bsdauth.m4 \ + $(top_srcdir)/m4/want_bzlib.m4 \ + $(top_srcdir)/m4/want_cassandra.m4 \ + $(top_srcdir)/m4/want_cdb.m4 \ + $(top_srcdir)/m4/want_checkpassword.m4 \ + $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \ + $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \ + $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \ + $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \ + $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \ + $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \ + $(top_srcdir)/m4/want_prefetch.m4 \ + $(top_srcdir)/m4/want_shadow.m4 \ + $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \ + $(top_srcdir)/m4/want_sqlite.m4 \ + $(top_srcdir)/m4/want_stemmer.m4 \ + $(top_srcdir)/m4/want_systemd.m4 \ + $(top_srcdir)/m4/want_textcat.m4 \ + $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \ + $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac +am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ + $(ACLOCAL_M4) +DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \ + $(pkginc_lib_HEADERS) $(am__DIST_COMMON) +mkinstalldirs = $(install_sh) -d +CONFIG_HEADER = $(top_builddir)/config.h +CONFIG_CLEAN_FILES = +CONFIG_CLEAN_VPATH_FILES = +am__EXEEXT_1 = test-dsync-mailbox-tree-sync$(EXEEXT) +PROGRAMS = $(noinst_PROGRAMS) +am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; +am__vpath_adj = case $$p in \ + $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ + *) f=$$p;; \ + esac; +am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; +am__install_max = 40 +am__nobase_strip_setup = \ + srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` +am__nobase_strip = \ + for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" +am__nobase_list = $(am__nobase_strip_setup); \ + for p in $$list; do echo "$$p $$p"; done | \ + sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ + $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ + if (++n[$$2] == $(am__install_max)) \ + { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ + END { for (dir in files) print dir, files[dir] }' +am__base_list = \ + sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ + sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' +am__uninstall_files_from_dir = { \ + test -z "$$files" \ + || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ + || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ + $(am__cd) "$$dir" && rm -f $$files; }; \ + } +am__installdirs = "$(DESTDIR)$(pkglibdir)" \ + "$(DESTDIR)$(pkginc_libdir)" +LTLIBRARIES = $(noinst_LTLIBRARIES) $(pkglib_LTLIBRARIES) +am_libdovecot_dsync_la_OBJECTS = +libdovecot_dsync_la_OBJECTS = $(am_libdovecot_dsync_la_OBJECTS) +AM_V_lt = $(am__v_lt_@AM_V@) +am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) +am__v_lt_0 = --silent +am__v_lt_1 = +libdovecot_dsync_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \ + $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \ + $(AM_CFLAGS) $(CFLAGS) $(libdovecot_dsync_la_LDFLAGS) \ + $(LDFLAGS) -o $@ +libdsync_la_LIBADD = +am_libdsync_la_OBJECTS = dsync-brain.lo dsync-brain-mailbox.lo \ + dsync-brain-mailbox-tree.lo dsync-brain-mailbox-tree-sync.lo \ + dsync-brain-mails.lo dsync-deserializer.lo dsync-mail.lo \ + dsync-mailbox.lo dsync-mailbox-import.lo \ + dsync-mailbox-export.lo dsync-mailbox-state.lo \ + dsync-mailbox-tree.lo dsync-mailbox-tree-fill.lo \ + dsync-mailbox-tree-sync.lo dsync-serializer.lo dsync-ibc.lo \ + dsync-ibc-stream.lo dsync-ibc-pipe.lo \ + dsync-transaction-log-scan.lo +libdsync_la_OBJECTS = $(am_libdsync_la_OBJECTS) +am_test_dsync_mailbox_tree_sync_OBJECTS = \ + test-dsync-mailbox-tree-sync.$(OBJEXT) +test_dsync_mailbox_tree_sync_OBJECTS = \ + $(am_test_dsync_mailbox_tree_sync_OBJECTS) +AM_V_P = $(am__v_P_@AM_V@) +am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) +am__v_P_0 = false +am__v_P_1 = : +AM_V_GEN = $(am__v_GEN_@AM_V@) +am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) +am__v_GEN_0 = @echo " GEN " $@; +am__v_GEN_1 = +AM_V_at = $(am__v_at_@AM_V@) +am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) +am__v_at_0 = @ +am__v_at_1 = +DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) +depcomp = $(SHELL) $(top_srcdir)/depcomp +am__maybe_remake_depfiles = depfiles +am__depfiles_remade = ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo \ + ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo \ + ./$(DEPDIR)/dsync-brain-mailbox.Plo \ + ./$(DEPDIR)/dsync-brain-mails.Plo ./$(DEPDIR)/dsync-brain.Plo \ + ./$(DEPDIR)/dsync-deserializer.Plo \ + ./$(DEPDIR)/dsync-ibc-pipe.Plo \ + ./$(DEPDIR)/dsync-ibc-stream.Plo ./$(DEPDIR)/dsync-ibc.Plo \ + ./$(DEPDIR)/dsync-mail.Plo \ + ./$(DEPDIR)/dsync-mailbox-export.Plo \ + ./$(DEPDIR)/dsync-mailbox-import.Plo \ + ./$(DEPDIR)/dsync-mailbox-state.Plo \ + ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo \ + ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo \ + ./$(DEPDIR)/dsync-mailbox-tree.Plo \ + ./$(DEPDIR)/dsync-mailbox.Plo ./$(DEPDIR)/dsync-serializer.Plo \ + ./$(DEPDIR)/dsync-transaction-log-scan.Plo \ + ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po +am__mv = mv -f +COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \ + $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) +LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \ + $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ + $(AM_CFLAGS) $(CFLAGS) +AM_V_CC = $(am__v_CC_@AM_V@) +am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@) +am__v_CC_0 = @echo " CC " $@; +am__v_CC_1 = +CCLD = $(CC) +LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \ + $(AM_LDFLAGS) $(LDFLAGS) -o $@ +AM_V_CCLD = $(am__v_CCLD_@AM_V@) +am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@) +am__v_CCLD_0 = @echo " CCLD " $@; +am__v_CCLD_1 = +SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \ + $(test_dsync_mailbox_tree_sync_SOURCES) +DIST_SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \ + $(test_dsync_mailbox_tree_sync_SOURCES) +am__can_run_installinfo = \ + case $$AM_UPDATE_INFO_DIR in \ + n|no|NO) false;; \ + *) (install-info --version) >/dev/null 2>&1;; \ + esac +HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS) +am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) +# Read a list of newline-separated strings from the standard input, +# and print each of them once, without duplicates. Input order is +# *not* preserved. +am__uniquify_input = $(AWK) '\ + BEGIN { nonempty = 0; } \ + { items[$$0] = 1; nonempty = 1; } \ + END { if (nonempty) { for (i in items) print i; }; } \ +' +# Make sure the list of sources is unique. This is necessary because, +# e.g., the same source file might be shared among _SOURCES variables +# for different programs/libraries. +am__define_uniq_tagged_files = \ + list='$(am__tagged_files)'; \ + unique=`for i in $$list; do \ + if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ + done | $(am__uniquify_input)` +ETAGS = etags +CTAGS = ctags +am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp +DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) +ACLOCAL = @ACLOCAL@ +ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@ +AMTAR = @AMTAR@ +AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ +APPARMOR_LIBS = @APPARMOR_LIBS@ +AR = @AR@ +AUTH_CFLAGS = @AUTH_CFLAGS@ +AUTH_LIBS = @AUTH_LIBS@ +AUTOCONF = @AUTOCONF@ +AUTOHEADER = @AUTOHEADER@ +AUTOMAKE = @AUTOMAKE@ +AWK = @AWK@ +BINARY_CFLAGS = @BINARY_CFLAGS@ +BINARY_LDFLAGS = @BINARY_LDFLAGS@ +BISON = @BISON@ +CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@ +CASSANDRA_LIBS = @CASSANDRA_LIBS@ +CC = @CC@ +CCDEPMODE = @CCDEPMODE@ +CDB_LIBS = @CDB_LIBS@ +CFLAGS = @CFLAGS@ +CLUCENE_CFLAGS = @CLUCENE_CFLAGS@ +CLUCENE_LIBS = @CLUCENE_LIBS@ +COMPRESS_LIBS = @COMPRESS_LIBS@ +CPP = @CPP@ +CPPFLAGS = @CPPFLAGS@ +CRYPT_LIBS = @CRYPT_LIBS@ +CXX = @CXX@ +CXXCPP = @CXXCPP@ +CXXDEPMODE = @CXXDEPMODE@ +CXXFLAGS = @CXXFLAGS@ +CYGPATH_W = @CYGPATH_W@ +DEFS = @DEFS@ +DEPDIR = @DEPDIR@ +DICT_LIBS = @DICT_LIBS@ +DLLIB = @DLLIB@ +DLLTOOL = @DLLTOOL@ +DSYMUTIL = @DSYMUTIL@ +DUMPBIN = @DUMPBIN@ +ECHO_C = @ECHO_C@ +ECHO_N = @ECHO_N@ +ECHO_T = @ECHO_T@ +EGREP = @EGREP@ +EXEEXT = @EXEEXT@ +FGREP = @FGREP@ +FLEX = @FLEX@ +FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@ +FUZZER_LDFLAGS = @FUZZER_LDFLAGS@ +GREP = @GREP@ +INSTALL = @INSTALL@ +INSTALL_DATA = @INSTALL_DATA@ +INSTALL_PROGRAM = @INSTALL_PROGRAM@ +INSTALL_SCRIPT = @INSTALL_SCRIPT@ +INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ +KRB5CONFIG = @KRB5CONFIG@ +KRB5_CFLAGS = @KRB5_CFLAGS@ +KRB5_LIBS = @KRB5_LIBS@ +LD = @LD@ +LDAP_LIBS = @LDAP_LIBS@ +LDFLAGS = @LDFLAGS@ +LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@ +LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@ +LIBCAP = @LIBCAP@ +LIBDOVECOT = @LIBDOVECOT@ +LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@ +LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@ +LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@ +LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@ +LIBDOVECOT_LDA = @LIBDOVECOT_LDA@ +LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@ +LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@ +LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@ +LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@ +LIBDOVECOT_LUA = @LIBDOVECOT_LUA@ +LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@ +LIBDOVECOT_SQL = @LIBDOVECOT_SQL@ +LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@ +LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@ +LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@ +LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@ +LIBICONV = @LIBICONV@ +LIBICU_CFLAGS = @LIBICU_CFLAGS@ +LIBICU_LIBS = @LIBICU_LIBS@ +LIBOBJS = @LIBOBJS@ +LIBS = @LIBS@ +LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@ +LIBSODIUM_LIBS = @LIBSODIUM_LIBS@ +LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@ +LIBTIRPC_LIBS = @LIBTIRPC_LIBS@ +LIBTOOL = @LIBTOOL@ +LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@ +LIBUNWIND_LIBS = @LIBUNWIND_LIBS@ +LIBWRAP_LIBS = @LIBWRAP_LIBS@ +LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@ +LIPO = @LIPO@ +LN_S = @LN_S@ +LTLIBICONV = @LTLIBICONV@ +LTLIBOBJS = @LTLIBOBJS@ +LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ +LUA_CFLAGS = @LUA_CFLAGS@ +LUA_LIBS = @LUA_LIBS@ +MAINT = @MAINT@ +MAKEINFO = @MAKEINFO@ +MANIFEST_TOOL = @MANIFEST_TOOL@ +MKDIR_P = @MKDIR_P@ +MODULE_LIBS = @MODULE_LIBS@ +MODULE_SUFFIX = @MODULE_SUFFIX@ +MYSQL_CFLAGS = @MYSQL_CFLAGS@ +MYSQL_CONFIG = @MYSQL_CONFIG@ +MYSQL_LIBS = @MYSQL_LIBS@ +NM = @NM@ +NMEDIT = @NMEDIT@ +NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@ +OBJDUMP = @OBJDUMP@ +OBJEXT = @OBJEXT@ +OTOOL = @OTOOL@ +OTOOL64 = @OTOOL64@ +PACKAGE = @PACKAGE@ +PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ +PACKAGE_NAME = @PACKAGE_NAME@ +PACKAGE_STRING = @PACKAGE_STRING@ +PACKAGE_TARNAME = @PACKAGE_TARNAME@ +PACKAGE_URL = @PACKAGE_URL@ +PACKAGE_VERSION = @PACKAGE_VERSION@ +PANDOC = @PANDOC@ +PATH_SEPARATOR = @PATH_SEPARATOR@ +PGSQL_CFLAGS = @PGSQL_CFLAGS@ +PGSQL_LIBS = @PGSQL_LIBS@ +PG_CONFIG = @PG_CONFIG@ +PIE_CFLAGS = @PIE_CFLAGS@ +PIE_LDFLAGS = @PIE_LDFLAGS@ +PKG_CONFIG = @PKG_CONFIG@ +PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@ +PKG_CONFIG_PATH = @PKG_CONFIG_PATH@ +QUOTA_LIBS = @QUOTA_LIBS@ +RANLIB = @RANLIB@ +RELRO_LDFLAGS = @RELRO_LDFLAGS@ +RPCGEN = @RPCGEN@ +RUN_TEST = @RUN_TEST@ +SED = @SED@ +SETTING_FILES = @SETTING_FILES@ +SET_MAKE = @SET_MAKE@ +SHELL = @SHELL@ +SQLITE_CFLAGS = @SQLITE_CFLAGS@ +SQLITE_LIBS = @SQLITE_LIBS@ +SQL_CFLAGS = @SQL_CFLAGS@ +SQL_LIBS = @SQL_LIBS@ +SSL_CFLAGS = @SSL_CFLAGS@ +SSL_LIBS = @SSL_LIBS@ +STRIP = @STRIP@ +SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@ +SYSTEMD_LIBS = @SYSTEMD_LIBS@ +VALGRIND = @VALGRIND@ +VERSION = @VERSION@ +ZSTD_CFLAGS = @ZSTD_CFLAGS@ +ZSTD_LIBS = @ZSTD_LIBS@ +abs_builddir = @abs_builddir@ +abs_srcdir = @abs_srcdir@ +abs_top_builddir = @abs_top_builddir@ +abs_top_srcdir = @abs_top_srcdir@ +ac_ct_AR = @ac_ct_AR@ +ac_ct_CC = @ac_ct_CC@ +ac_ct_CXX = @ac_ct_CXX@ +ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ +am__include = @am__include@ +am__leading_dot = @am__leading_dot@ +am__quote = @am__quote@ +am__tar = @am__tar@ +am__untar = @am__untar@ +bindir = @bindir@ +build = @build@ +build_alias = @build_alias@ +build_cpu = @build_cpu@ +build_os = @build_os@ +build_vendor = @build_vendor@ +builddir = @builddir@ +datadir = @datadir@ +datarootdir = @datarootdir@ +dict_drivers = @dict_drivers@ +docdir = @docdir@ +dvidir = @dvidir@ +exec_prefix = @exec_prefix@ +host = @host@ +host_alias = @host_alias@ +host_cpu = @host_cpu@ +host_os = @host_os@ +host_vendor = @host_vendor@ +htmldir = @htmldir@ +includedir = @includedir@ +infodir = @infodir@ +install_sh = @install_sh@ +libdir = @libdir@ +libexecdir = @libexecdir@ +localedir = @localedir@ +localstatedir = @localstatedir@ +mandir = @mandir@ +mkdir_p = @mkdir_p@ +moduledir = @moduledir@ +oldincludedir = @oldincludedir@ +pdfdir = @pdfdir@ +prefix = @prefix@ +program_transform_name = @program_transform_name@ +psdir = @psdir@ +rundir = @rundir@ +runstatedir = @runstatedir@ +sbindir = @sbindir@ +sharedstatedir = @sharedstatedir@ +sql_drivers = @sql_drivers@ +srcdir = @srcdir@ +ssldir = @ssldir@ +statedir = @statedir@ +sysconfdir = @sysconfdir@ +systemdservicetype = @systemdservicetype@ +systemdsystemunitdir = @systemdsystemunitdir@ +target_alias = @target_alias@ +top_build_prefix = @top_build_prefix@ +top_builddir = @top_builddir@ +top_srcdir = @top_srcdir@ +pkglib_LTLIBRARIES = libdovecot-dsync.la +noinst_LTLIBRARIES = libdsync.la +AM_CPPFLAGS = \ + -I$(top_srcdir)/src/lib \ + -I$(top_srcdir)/src/lib-test \ + -I$(top_srcdir)/src/lib-settings \ + -I$(top_srcdir)/src/lib-ssl-iostream \ + -I$(top_srcdir)/src/lib-master \ + -I$(top_srcdir)/src/lib-mail \ + -I$(top_srcdir)/src/lib-imap \ + -I$(top_srcdir)/src/lib-index \ + -I$(top_srcdir)/src/lib-storage + +libdsync_la_SOURCES = \ + dsync-brain.c \ + dsync-brain-mailbox.c \ + dsync-brain-mailbox-tree.c \ + dsync-brain-mailbox-tree-sync.c \ + dsync-brain-mails.c \ + dsync-deserializer.c \ + dsync-mail.c \ + dsync-mailbox.c \ + dsync-mailbox-import.c \ + dsync-mailbox-export.c \ + dsync-mailbox-state.c \ + dsync-mailbox-tree.c \ + dsync-mailbox-tree-fill.c \ + dsync-mailbox-tree-sync.c \ + dsync-serializer.c \ + dsync-ibc.c \ + dsync-ibc-stream.c \ + dsync-ibc-pipe.c \ + dsync-transaction-log-scan.c + +libdovecot_dsync_la_SOURCES = +libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la +libdovecot_dsync_la_DEPENDENCIES = libdsync.la +libdovecot_dsync_la_LDFLAGS = -export-dynamic +pkginc_libdir = $(pkgincludedir) +pkginc_lib_HEADERS = \ + dsync-brain.h \ + dsync-ibc.h + +noinst_HEADERS = \ + dsync-brain-private.h \ + dsync-mail.h \ + dsync-mailbox.h \ + dsync-mailbox-import.h \ + dsync-mailbox-export.h \ + dsync-mailbox-state.h \ + dsync-mailbox-tree.h \ + dsync-mailbox-tree-private.h \ + dsync-serializer.h \ + dsync-deserializer.h \ + dsync-ibc-private.h \ + dsync-transaction-log-scan.h + +test_programs = \ + test-dsync-mailbox-tree-sync + +test_libs = \ + ../../lib-test/libtest.la \ + ../../lib/liblib.la + +test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c +test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs) +test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs) +all: all-am + +.SUFFIXES: +.SUFFIXES: .c .lo .o .obj +$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps) + @for dep in $?; do \ + case '$(am__configure_deps)' in \ + *$$dep*) \ + ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ + && { if test -f $@; then exit 0; else break; fi; }; \ + exit 1;; \ + esac; \ + done; \ + echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile'; \ + $(am__cd) $(top_srcdir) && \ + $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile +Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status + @case '$?' in \ + *config.status*) \ + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ + *) \ + echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ + cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ + esac; + +$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh + +$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps) + cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh +$(am__aclocal_m4_deps): + +clean-noinstPROGRAMS: + @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ + echo " rm -f" $$list; \ + rm -f $$list || exit $$?; \ + test -n "$(EXEEXT)" || exit 0; \ + list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ + echo " rm -f" $$list; \ + rm -f $$list + +clean-noinstLTLIBRARIES: + -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES) + @list='$(noinst_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES) + @$(NORMAL_INSTALL) + @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \ + list2=; for p in $$list; do \ + if test -f $$p; then \ + list2="$$list2 $$p"; \ + else :; fi; \ + done; \ + test -z "$$list2" || { \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkglibdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkglibdir)" || exit 1; \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(pkglibdir)'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(pkglibdir)"; \ + } + +uninstall-pkglibLTLIBRARIES: + @$(NORMAL_UNINSTALL) + @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \ + for p in $$list; do \ + $(am__strip_dir) \ + echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(pkglibdir)/$$f'"; \ + $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(pkglibdir)/$$f"; \ + done + +clean-pkglibLTLIBRARIES: + -test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES) + @list='$(pkglib_LTLIBRARIES)'; \ + locs=`for p in $$list; do echo $$p; done | \ + sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \ + sort -u`; \ + test -z "$$locs" || { \ + echo rm -f $${locs}; \ + rm -f $${locs}; \ + } + +libdovecot-dsync.la: $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_DEPENDENCIES) $(EXTRA_libdovecot_dsync_la_DEPENDENCIES) + $(AM_V_CCLD)$(libdovecot_dsync_la_LINK) -rpath $(pkglibdir) $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_LIBADD) $(LIBS) + +libdsync.la: $(libdsync_la_OBJECTS) $(libdsync_la_DEPENDENCIES) $(EXTRA_libdsync_la_DEPENDENCIES) + $(AM_V_CCLD)$(LINK) $(libdsync_la_OBJECTS) $(libdsync_la_LIBADD) $(LIBS) + +test-dsync-mailbox-tree-sync$(EXEEXT): $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_DEPENDENCIES) $(EXTRA_test_dsync_mailbox_tree_sync_DEPENDENCIES) + @rm -f test-dsync-mailbox-tree-sync$(EXEEXT) + $(AM_V_CCLD)$(LINK) $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_LDADD) $(LIBS) + +mostlyclean-compile: + -rm -f *.$(OBJEXT) + +distclean-compile: + -rm -f *.tab.c + +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mails.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-deserializer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-pipe.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-stream.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mail.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-export.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-import.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-state.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-fill.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-sync.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-serializer.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-transaction-log-scan.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po@am__quote@ # am--include-marker + +$(am__depfiles_remade): + @$(MKDIR_P) $(@D) + @echo '# dummy' >$@-t && $(am__mv) $@-t $@ + +am--depfiles: $(am__depfiles_remade) + +.c.o: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $< + +.c.obj: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'` + +.c.lo: +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $< +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $< + +mostlyclean-libtool: + -rm -f *.lo + +clean-libtool: + -rm -rf .libs _libs +install-pkginc_libHEADERS: $(pkginc_lib_HEADERS) + @$(NORMAL_INSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + if test -n "$$list"; then \ + echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \ + fi; \ + for p in $$list; do \ + if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ + echo "$$d$$p"; \ + done | $(am__base_list) | \ + while read files; do \ + echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \ + $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \ + done + +uninstall-pkginc_libHEADERS: + @$(NORMAL_UNINSTALL) + @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \ + files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \ + dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir) + +ID: $(am__tagged_files) + $(am__define_uniq_tagged_files); mkid -fID $$unique +tags: tags-am +TAGS: tags + +tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + set x; \ + here=`pwd`; \ + $(am__define_uniq_tagged_files); \ + shift; \ + if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ + test -n "$$unique" || unique=$$empty_fix; \ + if test $$# -gt 0; then \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + "$$@" $$unique; \ + else \ + $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ + $$unique; \ + fi; \ + fi +ctags: ctags-am + +CTAGS: ctags +ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) + $(am__define_uniq_tagged_files); \ + test -z "$(CTAGS_ARGS)$$unique" \ + || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ + $$unique + +GTAGS: + here=`$(am__cd) $(top_builddir) && pwd` \ + && $(am__cd) $(top_srcdir) \ + && gtags -i $(GTAGS_ARGS) "$$here" +cscopelist: cscopelist-am + +cscopelist-am: $(am__tagged_files) + list='$(am__tagged_files)'; \ + case "$(srcdir)" in \ + [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ + *) sdir=$(subdir)/$(srcdir) ;; \ + esac; \ + for i in $$list; do \ + if test -f "$$i"; then \ + echo "$(subdir)/$$i"; \ + else \ + echo "$$sdir/$$i"; \ + fi; \ + done >> $(top_builddir)/cscope.files + +distclean-tags: + -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags + +distdir: $(BUILT_SOURCES) + $(MAKE) $(AM_MAKEFLAGS) distdir-am + +distdir-am: $(DISTFILES) + @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ + list='$(DISTFILES)'; \ + dist_files=`for file in $$list; do echo $$file; done | \ + sed -e "s|^$$srcdirstrip/||;t" \ + -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ + case $$dist_files in \ + */*) $(MKDIR_P) `echo "$$dist_files" | \ + sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ + sort -u` ;; \ + esac; \ + for file in $$dist_files; do \ + if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ + if test -d $$d/$$file; then \ + dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ + if test -d "$(distdir)/$$file"; then \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ + cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ + find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ + fi; \ + cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ + else \ + test -f "$(distdir)/$$file" \ + || cp -p $$d/$$file "$(distdir)/$$file" \ + || exit 1; \ + fi; \ + done +check-am: all-am + $(MAKE) $(AM_MAKEFLAGS) check-local +check: check-am +all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS) +installdirs: + for dir in "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(pkginc_libdir)"; do \ + test -z "$$dir" || $(MKDIR_P) "$$dir"; \ + done +install: install-am +install-exec: install-exec-am +install-data: install-data-am +uninstall: uninstall-am + +install-am: all-am + @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am + +installcheck: installcheck-am +install-strip: + if test -z '$(STRIP)'; then \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + install; \ + else \ + $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ + install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ + "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ + fi +mostlyclean-generic: + +clean-generic: + +distclean-generic: + -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) + -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) + +maintainer-clean-generic: + @echo "This command is intended for maintainers to use" + @echo "it deletes files that may require special tools to rebuild." +clean: clean-am + +clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \ + clean-noinstPROGRAMS clean-pkglibLTLIBRARIES mostlyclean-am + +distclean: distclean-am + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mails.Plo + -rm -f ./$(DEPDIR)/dsync-brain.Plo + -rm -f ./$(DEPDIR)/dsync-deserializer.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo + -rm -f ./$(DEPDIR)/dsync-ibc.Plo + -rm -f ./$(DEPDIR)/dsync-mail.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-serializer.Plo + -rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo + -rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po + -rm -f Makefile +distclean-am: clean-am distclean-compile distclean-generic \ + distclean-tags + +dvi: dvi-am + +dvi-am: + +html: html-am + +html-am: + +info: info-am + +info-am: + +install-data-am: install-pkginc_libHEADERS + +install-dvi: install-dvi-am + +install-dvi-am: + +install-exec-am: install-pkglibLTLIBRARIES + +install-html: install-html-am + +install-html-am: + +install-info: install-info-am + +install-info-am: + +install-man: + +install-pdf: install-pdf-am + +install-pdf-am: + +install-ps: install-ps-am + +install-ps-am: + +installcheck-am: + +maintainer-clean: maintainer-clean-am + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-brain-mails.Plo + -rm -f ./$(DEPDIR)/dsync-brain.Plo + -rm -f ./$(DEPDIR)/dsync-deserializer.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo + -rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo + -rm -f ./$(DEPDIR)/dsync-ibc.Plo + -rm -f ./$(DEPDIR)/dsync-mail.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo + -rm -f ./$(DEPDIR)/dsync-mailbox.Plo + -rm -f ./$(DEPDIR)/dsync-serializer.Plo + -rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo + -rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po + -rm -f Makefile +maintainer-clean-am: distclean-am maintainer-clean-generic + +mostlyclean: mostlyclean-am + +mostlyclean-am: mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool + +pdf: pdf-am + +pdf-am: + +ps: ps-am + +ps-am: + +uninstall-am: uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES + +.MAKE: check-am install-am install-strip + +.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \ + check-local clean clean-generic clean-libtool \ + clean-noinstLTLIBRARIES clean-noinstPROGRAMS \ + clean-pkglibLTLIBRARIES cscopelist-am ctags ctags-am distclean \ + distclean-compile distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-pdf install-pdf-am install-pkginc_libHEADERS \ + install-pkglibLTLIBRARIES install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ + pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ + uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES + +.PRECIOUS: Makefile + + +check-local: + for bin in $(test_programs); do \ + if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \ + done + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c new file mode 100644 index 0000000..2030e04 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c @@ -0,0 +1,229 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "dsync-mailbox-tree.h" +#include "dsync-brain-private.h" + +static int +sync_create_box(struct dsync_brain *brain, struct mailbox *box, + const guid_128_t mailbox_guid, uint32_t uid_validity, + enum mail_error *error_r) +{ + struct mailbox_metadata metadata; + struct mailbox_update update; + enum mail_error error; + const char *errstr; + int ret; + + i_zero(&update); + memcpy(update.mailbox_guid, mailbox_guid, sizeof(update.mailbox_guid)); + update.uid_validity = uid_validity; + + if (mailbox_create(box, &update, FALSE) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error != MAIL_ERROR_EXISTS) { + i_error("Can't create mailbox %s: %s", + mailbox_get_vname(box), errstr); + *error_r = error; + return -1; + } + } + if (brain->no_mail_sync) { + /* trust that create worked, we can't actually open it + and verify. */ + return 0; + } + /* sync the mailbox so we can look up its latest status */ + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + + /* verify that the GUID is what we wanted. if it's not, it probably + means that the mailbox had already been created. then we'll use the + GUID that is higher. + + mismatching UIDVALIDITY is handled later, because we choose it by + checking which mailbox has more messages */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) { + i_error("Can't get mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + + ret = memcmp(mailbox_guid, metadata.guid, sizeof(metadata.guid)); + + /* if THEIR guid is bigger than OUR guid, and we are not doing + backup in either direction, OR GUID did not match and we are + receiving backup, try change the mailbox GUID. + */ + + if ((ret > 0 && !brain->backup_recv && + !brain->backup_send) || (ret != 0 && brain->backup_recv)) { + if (brain->debug) { + i_debug("brain %c: Changing mailbox %s GUID %s -> %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), + guid_128_to_string(metadata.guid), + guid_128_to_string(mailbox_guid)); + } + i_zero(&update); + memcpy(update.mailbox_guid, mailbox_guid, + sizeof(update.mailbox_guid)); + if (mailbox_update(box, &update) < 0) { + i_error("Can't update mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + /* verify that the update worked */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, + &metadata) < 0) { + i_error("Can't get mailbox GUID %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, error_r)); + return -1; + } + if (memcmp(mailbox_guid, metadata.guid, + sizeof(metadata.guid)) != 0) { + i_error("Backend didn't update mailbox %s GUID", + mailbox_get_vname(box)); + *error_r = MAIL_ERROR_TEMP; + return -1; + } + } else if (ret < 0) { + if (brain->debug) { + i_debug("brain %c: Other brain should change mailbox " + "%s GUID %s -> %s", + brain->master_brain ? 'M' : 'S', + mailbox_get_vname(box), + guid_128_to_string(mailbox_guid), + guid_128_to_string(metadata.guid)); + } + } + return 0; +} + +int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, + const struct dsync_mailbox_tree_sync_change *change, + enum mail_error *error_r) +{ + struct mailbox *box = NULL, *destbox; + const char *errstr, *func_name = NULL, *storage_name; + enum mail_error error; + int ret = -1; + + if (brain->backup_send) { + i_assert(brain->no_backup_overwrite); + return 0; + } + + switch (change->type) { + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: + /* make sure we're deleting the correct mailbox */ + ret = dsync_brain_mailbox_alloc(brain, change->mailbox_guid, + &box, &errstr, error_r); + if (ret < 0) { + i_error("Mailbox sync: Couldn't allocate mailbox %s GUID %s: %s", + change->full_name, + guid_128_to_string(change->mailbox_guid), errstr); + return -1; + } + if (ret == 0) { + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox %s GUID %s deletion conflict: %s", + change->full_name, + guid_128_to_string(change->mailbox_guid), errstr)); + return 0; + } + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR: + storage_name = mailbox_list_get_storage_name(change->ns->list, + change->full_name); + if (mailbox_list_delete_dir(change->ns->list, storage_name) == 0) + return 0; + + errstr = mailbox_list_get_last_internal_error(change->ns->list, &error); + if (error == MAIL_ERROR_NOTFOUND || + error == MAIL_ERROR_EXISTS) { + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox %s mailbox_list_delete_dir conflict: %s", + change->full_name, errstr)); + return 0; + } else { + i_error("Mailbox sync: mailbox_list_delete_dir failed: %s", + errstr); + *error_r = error; + return -1; + } + default: + box = mailbox_alloc(change->ns->list, change->full_name, 0); + break; + } + mailbox_skip_create_name_restrictions(box, TRUE); + switch (change->type) { + case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX: + ret = sync_create_box(brain, box, change->mailbox_guid, + change->uid_validity, error_r); + mailbox_free(&box); + return ret; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR: + ret = mailbox_create(box, NULL, TRUE); + if (ret < 0 && + mailbox_get_last_mail_error(box) == MAIL_ERROR_EXISTS) { + /* it doesn't matter if somebody else created this + directory or we automatically did while creating its + child mailbox. it's there now anyway and we don't + gain anything by treating this failure any + differently from success. */ + ret = 0; + } + func_name = "mailbox_create"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX: + ret = mailbox_delete(box); + func_name = "mailbox_delete"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR: + i_unreached(); + case DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME: + destbox = mailbox_alloc(change->ns->list, + change->rename_dest_name, 0); + mailbox_skip_create_name_restrictions(destbox, TRUE); + ret = mailbox_rename(box, destbox); + func_name = "mailbox_rename"; + mailbox_free(&destbox); + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE: + ret = mailbox_set_subscribed(box, TRUE); + func_name = "mailbox_set_subscribed"; + break; + case DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE: + ret = mailbox_set_subscribed(box, FALSE); + func_name = "mailbox_set_subscribed"; + break; + } + if (ret < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error == MAIL_ERROR_EXISTS || + error == MAIL_ERROR_NOTFOUND) { + /* mailbox was already created or was already deleted. + let the next sync figure out what to do */ + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox %s %s conflict: %s", + mailbox_get_vname(box), func_name, errstr)); + ret = 0; + } else { + i_error("Mailbox %s sync: %s failed: %s", + mailbox_get_vname(box), func_name, errstr); + *error_r = error; + } + } + mailbox_free(&box); + return ret; +} diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree.c b/src/doveadm/dsync/dsync-brain-mailbox-tree.c new file mode 100644 index 0000000..44b76bf --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c @@ -0,0 +1,624 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "mail-namespace.h" +#include "mailbox-list-private.h" +#include "dsync-ibc.h" +#include "dsync-mailbox-tree.h" +#include "dsync-brain-private.h" + +#include <ctype.h> + +static void dsync_brain_check_namespaces(struct dsync_brain *brain) +{ + struct mail_namespace *ns, *first_ns = NULL; + char sep, escape_char; + + i_assert(brain->hierarchy_sep == '\0'); + i_assert(brain->escape_char == '\0'); + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + sep = mail_namespace_get_sep(ns); + escape_char = mailbox_list_get_settings(ns->list)->vname_escape_char; + if (first_ns == NULL) { + brain->hierarchy_sep = sep; + brain->escape_char = escape_char; + first_ns = ns; + } else if (brain->hierarchy_sep != sep) { + i_fatal("Synced namespaces have conflicting separators " + "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")", + brain->hierarchy_sep, first_ns->prefix, + sep, ns->prefix); + } else if (brain->escape_char != escape_char) { + i_fatal("Synced namespaces have conflicting escape chars " + "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")", + brain->escape_char, first_ns->prefix, + escape_char, ns->prefix); + } + } + if (brain->hierarchy_sep != '\0') + return; + + i_fatal("All your namespaces have a location setting. " + "Only namespaces with empty location settings are converted. " + "(One namespace should default to mail_location setting)"); +} + +void dsync_brain_mailbox_trees_init(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + + dsync_brain_check_namespaces(brain); + + brain->local_mailbox_tree = + dsync_mailbox_tree_init(brain->hierarchy_sep, + brain->escape_char, brain->alt_char); + /* we'll convert remote mailbox names to use our own separator */ + brain->remote_mailbox_tree = + dsync_mailbox_tree_init(brain->hierarchy_sep, + brain->escape_char, brain->alt_char); + + /* fill the local mailbox tree */ + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + if (brain->debug) + i_debug("brain %c: Namespace %s has location %s", + brain->master_brain ? 'M' : 'S', + ns->prefix, ns->set->location); + if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, ns, + brain->sync_box, + brain->sync_box_guid, + brain->exclude_mailboxes, + &brain->mail_error) < 0) { + brain->failed = TRUE; + break; + } + } + + brain->local_tree_iter = + dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); +} + +static const char *const * +dsync_brain_mailbox_to_parts(struct dsync_brain *brain, const char *name) +{ + char sep[] = { brain->hierarchy_sep, '\0' }; + char **parts = p_strsplit(unsafe_data_stack_pool, name, sep); + for (unsigned int i = 0; parts[i] != NULL; i++) { + mailbox_list_name_unescape((const char **)&parts[i], + brain->escape_char); + } + return (const char *const *)parts; +} + +void dsync_brain_send_mailbox_tree(struct dsync_brain *brain) +{ + struct dsync_mailbox_node *node; + enum dsync_ibc_send_ret ret; + const char *full_name; + + while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, + &full_name, &node)) { + if (node->ns == NULL) { + /* This node was created when adding a namespace prefix + to the tree that has multiple hierarchical names, + but the parent names don't belong to any synced + namespace. For example when syncing "-n Shared/user/" + so "Shared/" is skipped. Or if there is e.g. + "Public/files/" namespace prefix, but no "Public/" + namespace at all. */ + continue; + } + + T_BEGIN { + const char *const *parts; + + if (brain->debug) { + i_debug("brain %c: Local mailbox tree: %s %s", + brain->master_brain ? 'M' : 'S', full_name, + dsync_mailbox_node_to_string(node)); + } + + parts = dsync_brain_mailbox_to_parts(brain, full_name); + ret = dsync_ibc_send_mailbox_tree_node(brain->ibc, + parts, node); + } T_END; + if (ret == DSYNC_IBC_SEND_RET_FULL) + return; + } + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_TREE); + + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE_DELETES; +} + +void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain) +{ + const struct dsync_mailbox_delete *deletes; + unsigned int count; + + deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree, + &count); + dsync_ibc_send_mailbox_deletes(brain->ibc, deletes, count, + brain->hierarchy_sep, + brain->escape_char); + + brain->state = DSYNC_STATE_RECV_MAILBOX_TREE; +} + +static bool +dsync_namespace_match_parts(struct mail_namespace *ns, + const char *const *name_parts) +{ + const char *part, *prefix = ns->prefix; + size_t part_len; + char ns_sep = mail_namespace_get_sep(ns); + + if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 && + strcmp(name_parts[0], "INBOX") == 0 && name_parts[1] == NULL) + return TRUE; + + for (; *name_parts != NULL && *prefix != '\0'; name_parts++) { + part = *name_parts; + part_len = strlen(part); + + if (!str_begins(prefix, part)) + return FALSE; + if (prefix[part_len] != ns_sep) + return FALSE; + prefix += part_len + 1; + } + if (*name_parts != NULL) { + /* namespace prefix found with a mailbox */ + return TRUE; + } + if (*prefix == '\0') { + /* namespace prefix itself matched */ + return TRUE; + } + return FALSE; +} + +static struct mail_namespace * +dsync_find_namespace(struct dsync_brain *brain, const char *const *name_parts) +{ + struct mail_namespace *ns, *best_ns = NULL; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + if (ns->prefix_len == 0) { + /* prefix="" is the fallback namespace */ + if (best_ns == NULL) + best_ns = ns; + } else if (dsync_namespace_match_parts(ns, name_parts)) { + if (best_ns == NULL || + best_ns->prefix_len < ns->prefix_len) + best_ns = ns; + } + } + return best_ns; +} + +static bool +dsync_is_valid_name(struct mail_namespace *ns, const char *vname) +{ + struct mailbox *box; + bool ret; + + box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); + ret = mailbox_verify_create_name(box) == 0; + mailbox_free(&box); + return ret; +} + +static bool +dsync_is_valid_name_until(struct mail_namespace *ns, string_t *vname_full, + unsigned int end_pos) +{ + const char *vname; + if (end_pos == str_len(vname_full)) + vname = str_c(vname_full); + else + vname = t_strndup(str_c(vname_full), end_pos); + return dsync_is_valid_name(ns, vname); +} + +static bool +dsync_fix_mailbox_name_until(struct mail_namespace *ns, string_t *vname_full, + char alt_char, unsigned int start_pos, + unsigned int *_end_pos) +{ + unsigned int end_pos = *_end_pos; + unsigned int i; + + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + + /* 1) change any real separators to alt separators (this + wouldn't be necessary with listescape, but don't bother + detecting it) */ + char list_sep = mailbox_list_get_hierarchy_sep(ns->list); + char ns_sep = mail_namespace_get_sep(ns); + if (list_sep != ns_sep) { + char *v = str_c_modifiable(vname_full); + for (i = start_pos; i < end_pos; i++) { + if (v[i] == list_sep) + v[i] = alt_char; + } + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + } + + /* 2) '/' characters aren't valid without listescape */ + if (ns_sep != '/' && list_sep != '/') { + char *v = str_c_modifiable(vname_full); + for (i = start_pos; i < end_pos; i++) { + if (v[i] == '/') + v[i] = alt_char; + } + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + } + + /* 3) probably some reserved name (e.g. dbox-Mails or ..) */ + str_insert(vname_full, start_pos, "_"); end_pos++; *_end_pos += 1; + if (dsync_is_valid_name_until(ns, vname_full, end_pos)) + return TRUE; + + return FALSE; +} + +static void +dsync_fix_mailbox_name(struct mail_namespace *ns, string_t *vname_str, + char alt_char) +{ + const char *old_vname; + char *vname; + char ns_sep = mail_namespace_get_sep(ns); + guid_128_t guid; + unsigned int i, start_pos; + + vname = str_c_modifiable(vname_str); + if (strncmp(vname, ns->prefix, ns->prefix_len) == 0) + start_pos = ns->prefix_len; + else + start_pos = 0; + + /* replace control chars */ + for (i = start_pos; vname[i] != '\0'; i++) { + if ((unsigned char)vname[i] < ' ') + vname[i] = alt_char; + } + /* make it valid UTF8 */ + if (!uni_utf8_str_is_valid(vname)) { + old_vname = t_strdup(vname + start_pos); + str_truncate(vname_str, start_pos); + if (uni_utf8_get_valid_data((const void *)old_vname, + strlen(old_vname), vname_str)) + i_unreached(); + vname = str_c_modifiable(vname_str); + } + if (dsync_is_valid_name(ns, vname)) + return; + + /* Check/fix each hierarchical name separately */ + const char *p; + do { + i_assert(start_pos <= str_len(vname_str)); + p = strchr(str_c(vname_str) + start_pos, ns_sep); + unsigned int end_pos; + if (p == NULL) + end_pos = str_len(vname_str); + else + end_pos = p - str_c(vname_str); + + if (!dsync_fix_mailbox_name_until(ns, vname_str, alt_char, + start_pos, &end_pos)) { + /* Couldn't fix it. Name is too long? Just give up and + generate a unique name. */ + guid_128_generate(guid); + str_truncate(vname_str, 0); + str_append(vname_str, ns->prefix); + str_append(vname_str, guid_128_to_string(guid)); + i_assert(dsync_is_valid_name(ns, str_c(vname_str))); + break; + } + start_pos = end_pos + 1; + } while (p != NULL); +} + +static int +dsync_get_mailbox_name(struct dsync_brain *brain, const char *const *name_parts, + const char **name_r, struct mail_namespace **ns_r) +{ + struct mail_namespace *ns; + const char *p; + string_t *vname; + char ns_sep; + + i_assert(*name_parts != NULL); + + ns = dsync_find_namespace(brain, name_parts); + if (ns == NULL) + return -1; + ns_sep = mail_namespace_get_sep(ns); + + /* build the mailbox name */ + char escape_chars[] = { + brain->escape_char, + ns_sep, + '\0' + }; + struct dsync_mailbox_list *dlist = DSYNC_LIST_CONTEXT(ns->list); + if (dlist != NULL && !dlist->have_orig_escape_char) { + /* The escape character was added only for dsync internally. + Normally there is no escape character configured. Change + the mailbox names so that it doesn't rely on it. */ + escape_chars[0] = '\0'; + } + vname = t_str_new(128); + for (; *name_parts != NULL; name_parts++) { + if (escape_chars[0] != '\0') { + mailbox_list_name_escape(*name_parts, escape_chars, + vname); + } else { + for (p = *name_parts; *p != '\0'; p++) { + if (*p != ns_sep) + str_append_c(vname, *p); + else + str_append_c(vname, brain->alt_char); + } + } + str_append_c(vname, ns_sep); + } + str_truncate(vname, str_len(vname)-1); + + dsync_fix_mailbox_name(ns, vname, brain->alt_char); + *name_r = str_c(vname); + *ns_r = ns; + return 0; +} + +static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain) +{ + struct dsync_mailbox_tree_sync_ctx *ctx; + const struct dsync_mailbox_tree_sync_change *change; + enum dsync_mailbox_trees_sync_type sync_type; + enum dsync_mailbox_trees_sync_flags sync_flags = + (brain->debug ? DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG : 0) | + (brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0); + int ret; + + if (brain->no_backup_overwrite) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; + else if (brain->backup_send) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL; + else if (brain->backup_recv) + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE; + else + sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY; + + ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree, + brain->remote_mailbox_tree, + sync_type, sync_flags); + while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) { + T_BEGIN { + ret = dsync_brain_mailbox_tree_sync_change( + brain, change, &brain->mail_error); + } T_END; + if (ret < 0) { + brain->failed = TRUE; + break; + } + } + if (dsync_mailbox_trees_sync_deinit(&ctx) < 0) + brain->failed = TRUE; +} + +static int +dsync_brain_recv_mailbox_tree_add(struct dsync_brain *brain, + const char *const *parts, + const struct dsync_mailbox_node *remote_node, + const char *sep) +{ + struct dsync_mailbox_node *node; + struct mail_namespace *ns; + const char *name; + + if (dsync_get_mailbox_name(brain, parts, &name, &ns) < 0) + return -1; + if (brain->debug) { + i_debug("brain %c: Remote mailbox tree: %s %s", + brain->master_brain ? 'M' : 'S', + t_strarray_join(parts, sep), + dsync_mailbox_node_to_string(remote_node)); + } + node = dsync_mailbox_tree_get(brain->remote_mailbox_tree, name); + node->ns = ns; + dsync_mailbox_node_copy_data(node, remote_node); + return 0; +} + +bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain) +{ + const struct dsync_mailbox_node *remote_node; + struct dsync_mailbox_node *dup_node1, *dup_node2; + const char *const *parts; + enum dsync_ibc_recv_ret ret; + int ret2; + char sep[2]; + bool changed = FALSE; + + sep[0] = brain->hierarchy_sep; sep[1] = '\0'; + while ((ret = dsync_ibc_recv_mailbox_tree_node(brain->ibc, &parts, + &remote_node)) > 0) { + T_BEGIN { + ret2 = dsync_brain_recv_mailbox_tree_add( + brain, parts, remote_node, sep); + } T_END; + if (ret2 < 0) { + i_error("Couldn't find namespace for mailbox %s", + t_strarray_join(parts, sep)); + brain->failed = TRUE; + return TRUE; + } + } + if (ret != DSYNC_IBC_RECV_RET_FINISHED) + return changed; + + if (dsync_mailbox_tree_build_guid_hash(brain->remote_mailbox_tree, + &dup_node1, &dup_node2) < 0) { + i_error("Remote sent duplicate mailbox GUID %s for mailboxes %s and %s", + guid_128_to_string(dup_node1->mailbox_guid), + dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree, + dup_node1), + dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree, + dup_node2)); + brain->failed = TRUE; + } + + brain->state = DSYNC_STATE_RECV_MAILBOX_TREE_DELETES; + return TRUE; +} + +static void +dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + const struct dsync_mailbox_delete *other_del, + const struct dsync_mailbox_node **node_r, + const char **status_r) +{ + const struct dsync_mailbox_node *node; + struct dsync_mailbox_node *other_node, *old_node; + const char *name; + + /* see if we can find the deletion based on mailbox tree that should + still have the mailbox */ + node = *node_r = dsync_mailbox_tree_find_delete(tree, other_del); + if (node == NULL) { + *status_r = "not found"; + return; + } + + switch (other_del->type) { + case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX: + /* mailbox is always deleted */ + break; + case DSYNC_MAILBOX_DELETE_TYPE_DIR: + if (other_del->timestamp <= node->last_renamed_or_created) { + /* we don't want to delete this directory, we already + have a newer timestamp for it */ + *status_r = "keep directory, we have a newer timestamp"; + return; + } + break; + case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE: + if (other_del->timestamp <= node->last_subscription_change) { + /* we don't want to unsubscribe, since we already have + a newer subscription timestamp */ + *status_r = "keep subscription, we have a newer timestamp"; + return; + } + break; + } + + /* make a node for it in the other mailbox tree */ + name = dsync_mailbox_node_get_full_name(tree, node); + other_node = dsync_mailbox_tree_get(other_tree, name); + + if (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + (!guid_128_is_empty(other_node->mailbox_guid) || + other_del->type != DSYNC_MAILBOX_DELETE_TYPE_MAILBOX)) { + /* other side has already created a new mailbox or + directory with this name, we can't delete it */ + *status_r = "name has already been recreated"; + return; + } + + /* ok, mark the other node deleted */ + if (other_del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) { + memcpy(other_node->mailbox_guid, node->mailbox_guid, + sizeof(other_node->mailbox_guid)); + } + if (other_node->ns != node->ns && other_node->ns != NULL) { + /* namespace mismatch for this node. this shouldn't happen + normally, but especially during some misconfigurations it's + possible that one side has created mailboxes that conflict + with another namespace's prefix. since we're here because + one of the mailboxes was deleted, we'll just ignore this. */ + *status_r = "namespace mismatch"; + return; + } + other_node->ns = node->ns; + if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) { + other_node->existence = DSYNC_MAILBOX_NODE_DELETED; + *status_r = "marked as deleted"; + } else { + other_node->last_subscription_change = other_del->timestamp; + other_node->subscribed = FALSE; + *status_r = "marked as unsubscribed"; + } + + if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node, + &old_node) < 0) + i_unreached(); +} + +bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain) +{ + const struct dsync_mailbox_node *node; + const char *status; + const struct dsync_mailbox_delete *deletes; + unsigned int i, count; + char sep, escape_char; + + if (dsync_ibc_recv_mailbox_deletes(brain->ibc, &deletes, &count, + &sep, &escape_char) == 0) + return FALSE; + + /* apply remote's mailbox deletions based on our local tree */ + dsync_mailbox_tree_set_remote_chars(brain->local_mailbox_tree, sep, + escape_char); + for (i = 0; i < count; i++) { + dsync_brain_mailbox_tree_add_delete(brain->local_mailbox_tree, + brain->remote_mailbox_tree, + &deletes[i], &node, &status); + if (brain->debug) { + const char *node_name = node == NULL ? "" : + dsync_mailbox_node_get_full_name(brain->local_mailbox_tree, node); + i_debug("brain %c: Remote mailbox tree deletion: guid=%s type=%s timestamp=%ld name=%s local update=%s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(deletes[i].guid), + dsync_mailbox_delete_type_to_string(deletes[i].type), + deletes[i].timestamp, node_name, status); + } + } + + /* apply local mailbox deletions based on remote tree */ + deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree, + &count); + dsync_mailbox_tree_set_remote_chars(brain->remote_mailbox_tree, + brain->hierarchy_sep, + brain->escape_char); + for (i = 0; i < count; i++) { + dsync_brain_mailbox_tree_add_delete(brain->remote_mailbox_tree, + brain->local_mailbox_tree, + &deletes[i], &node, &status); + } + + dsync_brain_mailbox_trees_sync(brain); + brain->state = brain->master_brain ? + DSYNC_STATE_MASTER_SEND_MAILBOX : + DSYNC_STATE_SLAVE_RECV_MAILBOX; + i_assert(brain->local_tree_iter == NULL); + brain->local_tree_iter = + dsync_mailbox_tree_iter_init(brain->local_mailbox_tree); + return TRUE; +} diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c new file mode 100644 index 0000000..759018f --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mailbox.c @@ -0,0 +1,923 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "mail-cache-private.h" +#include "mail-namespace.h" +#include "mail-storage-private.h" +#include "dsync-ibc.h" +#include "dsync-mailbox-tree.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-brain-private.h" + +static int +ns_mailbox_try_alloc(struct dsync_brain *brain, struct mail_namespace *ns, + const guid_128_t guid, struct mailbox **box_r, + const char **errstr_r, enum mail_error *error_r) +{ + enum mailbox_flags flags = 0; + struct mailbox *box; + enum mailbox_existence existence; + int ret; + + if (brain->backup_send) { + /* make sure mailbox isn't modified */ + flags |= MAILBOX_FLAG_READONLY; + } + + box = mailbox_alloc_guid(ns->list, guid, flags); + ret = mailbox_exists(box, FALSE, &existence); + if (ret < 0) { + *errstr_r = mailbox_get_last_internal_error(box, error_r); + mailbox_free(&box); + return -1; + } + if (existence != MAILBOX_EXISTENCE_SELECT) { + mailbox_free(&box); + *errstr_r = existence == MAILBOX_EXISTENCE_NONE ? + "Mailbox was already deleted" : + "Mailbox is no longer selectable"; + return 0; + } + *box_r = box; + return 1; +} + +int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, + struct mailbox **box_r, const char **errstr_r, + enum mail_error *error_r) +{ + struct mail_namespace *ns; + int ret; + + *box_r = NULL; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + if ((ret = ns_mailbox_try_alloc(brain, ns, guid, box_r, + errstr_r, error_r)) != 0) + return ret; + } + return 0; +} + +static void +dsync_mailbox_cache_field_dup(ARRAY_TYPE(mailbox_cache_field) *dest, + const ARRAY_TYPE(mailbox_cache_field) *src, + pool_t pool) +{ + const struct mailbox_cache_field *src_field; + struct mailbox_cache_field *dest_field; + + p_array_init(dest, pool, array_count(src)); + array_foreach(src, src_field) { + dest_field = array_append_space(dest); + dest_field->name = p_strdup(pool, src_field->name); + dest_field->decision = src_field->decision; + dest_field->last_used = src_field->last_used; + } +} + + +static const struct dsync_mailbox_state * +dsync_mailbox_state_find(struct dsync_brain *brain, + const guid_128_t mailbox_guid) +{ + const uint8_t *guid_p; + + guid_p = mailbox_guid; + return hash_table_lookup(brain->mailbox_states, guid_p); +} + +static void +dsync_mailbox_state_remove(struct dsync_brain *brain, + const guid_128_t mailbox_guid) +{ + const uint8_t *guid_p; + + guid_p = mailbox_guid; + if (hash_table_lookup(brain->mailbox_states, guid_p) != NULL) + hash_table_remove(brain->mailbox_states, guid_p); +} + +void dsync_brain_sync_init_box_states(struct dsync_brain *brain) +{ + if (brain->backup_send) { + /* we have an exporter, but no importer. */ + brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES; + brain->box_recv_state = brain->mail_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : + DSYNC_BOX_STATE_RECV_LAST_COMMON; + } else if (brain->backup_recv) { + /* we have an importer, but no exporter */ + brain->box_send_state = brain->mail_requests ? + DSYNC_BOX_STATE_MAIL_REQUESTS : + DSYNC_BOX_STATE_DONE; + brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES; + } else { + brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES; + brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES; + } +} + +static void +dsync_brain_sync_mailbox_init(struct dsync_brain *brain, + struct mailbox *box, + struct file_lock *lock, + const struct dsync_mailbox *local_dsync_box, + bool wait_for_remote_box) +{ + const struct dsync_mailbox_state *state; + + i_assert(brain->box_importer == NULL); + i_assert(brain->box_exporter == NULL); + i_assert(box->synced); + + brain->box = box; + brain->box_lock = lock; + brain->pre_box_state = brain->state; + if (wait_for_remote_box) { + brain->box_send_state = DSYNC_BOX_STATE_MAILBOX; + brain->box_recv_state = DSYNC_BOX_STATE_MAILBOX; + } else { + dsync_brain_sync_init_box_states(brain); + } + brain->local_dsync_box = *local_dsync_box; + if (brain->dsync_box_pool != NULL) + p_clear(brain->dsync_box_pool); + else { + brain->dsync_box_pool = + pool_alloconly_create(MEMPOOL_GROWING"dsync brain box pool", 2048); + } + dsync_mailbox_cache_field_dup(&brain->local_dsync_box.cache_fields, + &local_dsync_box->cache_fields, + brain->dsync_box_pool); + i_zero(&brain->remote_dsync_box); + + state = dsync_mailbox_state_find(brain, local_dsync_box->mailbox_guid); + if (state != NULL) + brain->mailbox_state = *state; + else { + i_zero(&brain->mailbox_state); + memcpy(brain->mailbox_state.mailbox_guid, + local_dsync_box->mailbox_guid, + sizeof(brain->mailbox_state.mailbox_guid)); + brain->mailbox_state.last_uidvalidity = + local_dsync_box->uid_validity; + } +} + +static void +dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box) +{ + enum dsync_mailbox_import_flags import_flags = 0; + const struct dsync_mailbox_state *state; + uint32_t last_common_uid; + uint64_t last_common_modseq, last_common_pvt_modseq; + + i_assert(brain->box_importer == NULL); + i_assert(brain->log_scan != NULL); + + i_assert(memcmp(brain->local_dsync_box.mailbox_guid, + remote_dsync_box->mailbox_guid, + sizeof(remote_dsync_box->mailbox_guid)) == 0); + + brain->remote_dsync_box = *remote_dsync_box; + dsync_mailbox_cache_field_dup(&brain->remote_dsync_box.cache_fields, + &remote_dsync_box->cache_fields, + brain->dsync_box_pool); + + state = dsync_mailbox_state_find(brain, remote_dsync_box->mailbox_guid); + if (state != NULL) { + last_common_uid = state->last_common_uid; + last_common_modseq = state->last_common_modseq; + last_common_pvt_modseq = state->last_common_pvt_modseq; + } else { + last_common_uid = 0; + last_common_modseq = 0; + last_common_pvt_modseq = 0; + } + + if (brain->mail_requests) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS; + if (brain->master_brain) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN; + if (brain->backup_recv && !brain->no_backup_overwrite) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES; + if (brain->debug) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_DEBUG; + if (brain->local_dsync_box.have_save_guids && + (remote_dsync_box->have_save_guids || + (brain->backup_recv && remote_dsync_box->have_guids))) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS; + if (brain->local_dsync_box.have_only_guid128 || + remote_dsync_box->have_only_guid128) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128; + if (brain->no_notify) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY; + if (brain->empty_hdr_workaround) + import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND; + + brain->box_importer = brain->backup_send ? NULL : + dsync_mailbox_import_init(brain->box, brain->virtual_all_box, + brain->log_scan, + last_common_uid, last_common_modseq, + last_common_pvt_modseq, + remote_dsync_box->uid_next, + remote_dsync_box->first_recent_uid, + remote_dsync_box->highest_modseq, + remote_dsync_box->highest_pvt_modseq, + brain->sync_since_timestamp, + brain->sync_until_timestamp, + brain->sync_max_size, + brain->sync_flag, + brain->import_commit_msgs_interval, + import_flags, brain->hdr_hash_version, + brain->hashed_headers); +} + +int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box) +{ + struct mailbox_status status; + enum dsync_mailbox_exporter_flags exporter_flags = 0; + uint32_t last_common_uid, highest_wanted_uid; + uint64_t last_common_modseq, last_common_pvt_modseq; + const char *desync_reason = ""; + bool pvt_too_old; + int ret; + + i_assert(brain->log_scan == NULL); + i_assert(brain->box_exporter == NULL); + + last_common_uid = brain->mailbox_state.last_common_uid; + last_common_modseq = brain->mailbox_state.last_common_modseq; + last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq; + highest_wanted_uid = last_common_uid == 0 ? + (uint32_t)-1 : last_common_uid; + ret = dsync_transaction_log_scan_init(brain->box->view, + brain->box->view_pvt, + highest_wanted_uid, + last_common_modseq, + last_common_pvt_modseq, + &brain->log_scan, &pvt_too_old); + if (ret < 0) { + i_error("Failed to read transaction log for mailbox %s", + mailbox_get_vname(brain->box)); + brain->failed = TRUE; + return -1; + } + + mailbox_get_open_status(brain->box, STATUS_UIDNEXT | + STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ, &status); + if (status.nonpermanent_modseqs) + status.highest_modseq = 0; + if (ret == 0) { + if (pvt_too_old) { + desync_reason = t_strdup_printf( + "Private modseq %"PRIu64" no longer in transaction log " + "(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)", + last_common_pvt_modseq, + status.highest_pvt_modseq, last_common_uid, + status.uidnext); + } else { + desync_reason = t_strdup_printf( + "Modseq %"PRIu64" no longer in transaction log " + "(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)", + last_common_modseq, + status.highest_modseq, last_common_uid, + status.uidnext); + } + } + + if (last_common_uid != 0) { + /* if last_common_* is higher than our current ones it means + that the incremental sync state is stale and we need to do + a full resync */ + if (status.uidnext < last_common_uid) { + desync_reason = t_strdup_printf("uidnext %u < %u", + status.uidnext, last_common_uid); + ret = 0; + } else if (status.highest_modseq < last_common_modseq) { + desync_reason = t_strdup_printf("highest_modseq %"PRIu64" < %"PRIu64, + status.highest_modseq, last_common_modseq); + ret = 0; + } else if (status.highest_pvt_modseq < last_common_pvt_modseq) { + desync_reason = t_strdup_printf("highest_pvt_modseq %"PRIu64" < %"PRIu64, + status.highest_pvt_modseq, last_common_pvt_modseq); + ret = 0; + } + } + if (ret == 0) { + i_warning("Failed to do incremental sync for mailbox %s, " + "retry with a full sync (%s)", + mailbox_get_vname(brain->box), desync_reason); + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Incremental sync failed: %s", desync_reason)); + brain->require_full_resync = TRUE; + return 0; + } + + if (!brain->mail_requests) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS; + if (remote_dsync_box->have_save_guids && + (brain->local_dsync_box.have_save_guids || + (brain->backup_send && brain->local_dsync_box.have_guids))) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS; + if (brain->no_mail_prefetch) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL; + if (brain->sync_since_timestamp > 0 || + brain->sync_until_timestamp > 0) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS; + if (brain->sync_max_size > 0) + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES; + if (remote_dsync_box->messages_count == 0) { + /* remote mailbox is empty - we don't really need to export + header hashes since they're not going to match anything + anyway. */ + exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES; + } + + brain->box_exporter = brain->backup_recv ? NULL : + dsync_mailbox_export_init(brain->box, brain->log_scan, + last_common_uid, + exporter_flags, + brain->hdr_hash_version, + brain->hashed_headers); + dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box); + return 1; +} + +void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain) +{ + enum mail_error error; + + i_assert(brain->box != NULL); + + array_push_back(&brain->remote_mailbox_states, &brain->mailbox_state); + if (brain->box_exporter != NULL) { + const char *errstr; + + i_assert(brain->failed || brain->require_full_resync || + brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED); + if (dsync_mailbox_export_deinit(&brain->box_exporter, + &errstr, &error) < 0) + i_error("Mailbox export failed: %s", errstr); + } + if (brain->box_importer != NULL) { + uint32_t last_common_uid, last_messages_count; + uint64_t last_common_modseq, last_common_pvt_modseq; + const char *changes_during_sync; + bool require_full_resync; + + i_assert(brain->failed); + (void)dsync_mailbox_import_deinit(&brain->box_importer, + FALSE, + &last_common_uid, + &last_common_modseq, + &last_common_pvt_modseq, + &last_messages_count, + &changes_during_sync, + &require_full_resync, + &brain->mail_error); + if (require_full_resync) + brain->require_full_resync = TRUE; + } + if (brain->log_scan != NULL) + dsync_transaction_log_scan_deinit(&brain->log_scan); + file_lock_free(&brain->box_lock); + mailbox_free(&brain->box); + + brain->state = brain->pre_box_state; +} + +static int dsync_box_get(struct mailbox *box, struct dsync_mailbox *dsync_box_r, + enum mail_error *error_r) +{ + const enum mailbox_status_items status_items = + STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES | + STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ | + STATUS_HIGHESTPVTMODSEQ; + const enum mailbox_metadata_items metadata_items = + MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID; + struct mailbox_status status; + struct mailbox_metadata metadata; + const char *errstr; + enum mail_error error; + + /* get metadata first, since it may autocreate the mailbox */ + if (mailbox_get_metadata(box, metadata_items, &metadata) < 0 || + mailbox_get_status(box, status_items, &status) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + if (error == MAIL_ERROR_NOTFOUND || + error == MAIL_ERROR_NOTPOSSIBLE) { + /* Mailbox isn't selectable, try the next one. We + should have already caught \Noselect mailboxes, but + check them anyway here. The NOTPOSSIBLE check is + mainly for invalid mbox files. */ + return 0; + } + i_error("Failed to access mailbox %s: %s", + mailbox_get_vname(box), errstr); + *error_r = error; + return -1; + } + if (status.nonpermanent_modseqs) + status.highest_modseq = 0; + + i_assert(status.uidvalidity != 0 || status.messages == 0); + + i_zero(dsync_box_r); + memcpy(dsync_box_r->mailbox_guid, metadata.guid, + sizeof(dsync_box_r->mailbox_guid)); + dsync_box_r->uid_validity = status.uidvalidity; + dsync_box_r->uid_next = status.uidnext; + dsync_box_r->messages_count = status.messages; + dsync_box_r->first_recent_uid = status.first_recent_uid; + dsync_box_r->highest_modseq = status.highest_modseq; + dsync_box_r->highest_pvt_modseq = status.highest_pvt_modseq; + dsync_mailbox_cache_field_dup(&dsync_box_r->cache_fields, + metadata.cache_fields, + pool_datastack_create()); + dsync_box_r->have_guids = status.have_guids; + dsync_box_r->have_save_guids = status.have_save_guids; + dsync_box_r->have_only_guid128 = status.have_only_guid128; + return 1; +} + +static bool +dsync_brain_has_mailbox_state_changed(struct dsync_brain *brain, + const struct dsync_mailbox *dsync_box) +{ + const struct dsync_mailbox_state *state; + + if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + return TRUE; + + state = dsync_mailbox_state_find(brain, dsync_box->mailbox_guid); + return state == NULL || + state->last_uidvalidity != dsync_box->uid_validity || + state->last_common_uid+1 != dsync_box->uid_next || + state->last_common_modseq != dsync_box->highest_modseq || + state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq || + state->last_messages_count != dsync_box->messages_count; +} + +static int +dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, + struct file_lock **lock_r, + struct dsync_mailbox *dsync_box_r) +{ + enum mailbox_flags flags = 0; + struct dsync_mailbox dsync_box; + struct mailbox *box; + struct file_lock *lock = NULL; + struct dsync_mailbox_node *node; + const char *vname = NULL; + enum mail_error error; + bool synced = FALSE; + int ret; + + *box_r = NULL; + + while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !guid_128_is_empty(node->mailbox_guid)) + break; + vname = NULL; + } + if (vname == NULL) { + /* no more mailboxes */ + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + return -1; + } + + if (brain->backup_send) { + /* make sure mailbox isn't modified */ + flags |= MAILBOX_FLAG_READONLY; + } + box = mailbox_alloc(node->ns->list, vname, flags); + for (;;) { + if ((ret = dsync_box_get(box, &dsync_box, &error)) <= 0) { + if (ret < 0) { + brain->mail_error = error; + brain->failed = TRUE; + } + mailbox_free(&box); + file_lock_free(&lock); + return ret; + } + + /* if mailbox's last_common_* state equals the current state, + we can skip the mailbox */ + if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) { + if (brain->debug) { + i_debug("brain %c: Skipping mailbox %s with unchanged state " + "uidvalidity=%u uidnext=%u highestmodseq=%"PRIu64" highestpvtmodseq=%"PRIu64" messages=%u", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box.mailbox_guid), + dsync_box.uid_validity, + dsync_box.uid_next, + dsync_box.highest_modseq, + dsync_box.highest_pvt_modseq, + dsync_box.messages_count); + } + mailbox_free(&box); + file_lock_free(&lock); + return 0; + } + if (synced) { + /* ok, the mailbox really changed */ + break; + } + + /* mailbox appears to have changed. do a full sync here and get the + state again. Lock before syncing. */ + if (dsync_mailbox_lock(brain, box, &lock) < 0) { + brain->failed = TRUE; + mailbox_free(&box); + return -1; + } + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + brain->failed = TRUE; + mailbox_free(&box); + file_lock_free(&lock); + return -1; + } + synced = TRUE; + } + + *box_r = box; + *lock_r = lock; + *dsync_box_r = dsync_box; + return 1; +} + +static bool +dsync_brain_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r, + struct file_lock **lock_r, + struct dsync_mailbox *dsync_box_r) +{ + int ret; + + if (brain->no_mail_sync) + return FALSE; + + while ((ret = dsync_brain_try_next_mailbox(brain, box_r, lock_r, dsync_box_r)) == 0) + ; + return ret > 0; +} + +void dsync_brain_master_send_mailbox(struct dsync_brain *brain) +{ + struct dsync_mailbox dsync_box; + struct mailbox *box; + struct file_lock *lock; + + i_assert(brain->master_brain); + i_assert(brain->box == NULL); + + if (!dsync_brain_next_mailbox(brain, &box, &lock, &dsync_box)) { + brain->state = DSYNC_STATE_FINISH; + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX); + return; + } + + /* start exporting this mailbox (wait for remote to start importing) */ + dsync_ibc_send_mailbox(brain->ibc, &dsync_box); + dsync_brain_sync_mailbox_init(brain, box, lock, &dsync_box, TRUE); + brain->state = DSYNC_STATE_SYNC_MAILS; +} + +bool dsync_boxes_need_sync(struct dsync_brain *brain, + const struct dsync_mailbox *box1, + const struct dsync_mailbox *box2, + const char **reason_r) +{ + if (brain->no_mail_sync) + return FALSE; + if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_CHANGED) { + *reason_r = "Full sync"; + return TRUE; + } + if (box1->uid_validity != box2->uid_validity) + *reason_r = t_strdup_printf("UIDVALIDITY changed: %u -> %u", + box1->uid_validity, box2->uid_validity); + else if (box1->uid_next != box2->uid_next) + *reason_r = t_strdup_printf("UIDNEXT changed: %u -> %u", + box1->uid_next, box2->uid_next); + else if (box1->messages_count != box2->messages_count) + *reason_r = t_strdup_printf("Message count changed: %u -> %u", + box1->messages_count, box2->messages_count); + else if (box1->highest_modseq != box2->highest_modseq) { + *reason_r = t_strdup_printf("HIGHESTMODSEQ changed %" + PRIu64" -> %"PRIu64, + box1->highest_modseq, + box2->highest_modseq); + if (box1->highest_modseq == 0 || + box2->highest_modseq == 0) { + *reason_r = t_strdup_printf( + "%s (Permanent MODSEQs aren't supported)", + *reason_r); + } + } else if (box1->highest_pvt_modseq != box2->highest_pvt_modseq) + *reason_r = t_strdup_printf("Private HIGHESTMODSEQ changed %" + PRIu64" -> %"PRIu64, + box1->highest_pvt_modseq, + box2->highest_pvt_modseq); + else if (box1->first_recent_uid != box2->first_recent_uid) + *reason_r = t_strdup_printf("First RECENT UID changed: %u -> %u", + box1->first_recent_uid, box2->first_recent_uid); + else + return FALSE; + return TRUE; +} + +static int +mailbox_cache_field_name_cmp(const struct mailbox_cache_field *f1, + const struct mailbox_cache_field *f2) +{ + return strcmp(f1->name, f2->name); +} + +static void +dsync_cache_fields_update(const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + struct mailbox *box, + struct mailbox_update *update) +{ + ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes; + const struct mailbox_cache_field *local_fields, *remote_fields; + unsigned int li, ri, local_count, remote_count; + time_t drop_older_timestamp; + int ret; + + if (array_count(&remote_box->cache_fields) == 0) { + /* remote has no cached fields. there's nothing to update. */ + return; + } + + t_array_init(&local_sorted, array_count(&local_box->cache_fields)); + t_array_init(&remote_sorted, array_count(&remote_box->cache_fields)); + array_append_array(&local_sorted, &local_box->cache_fields); + array_append_array(&remote_sorted, &remote_box->cache_fields); + array_sort(&local_sorted, mailbox_cache_field_name_cmp); + array_sort(&remote_sorted, mailbox_cache_field_name_cmp); + + if (array_count(&local_sorted) == 0) { + /* local has no cached fields. set them to same as remote. */ + array_append_zero(&remote_sorted); + update->cache_updates = array_front(&remote_sorted); + return; + } + + /* figure out what to change */ + local_fields = array_get(&local_sorted, &local_count); + remote_fields = array_get(&remote_sorted, &remote_count); + t_array_init(&changes, local_count + remote_count); + drop_older_timestamp = ioloop_time - + box->index->optimization_set.cache.unaccessed_field_drop_secs; + + for (li = ri = 0; li < local_count || ri < remote_count; ) { + ret = li == local_count ? 1 : + ri == remote_count ? -1 : + strcmp(local_fields[li].name, remote_fields[ri].name); + if (ret == 0) { + /* field exists in both local and remote */ + const struct mailbox_cache_field *lf = &local_fields[li]; + const struct mailbox_cache_field *rf = &remote_fields[ri]; + + if (lf->last_used > rf->last_used || + (lf->last_used == rf->last_used && + lf->decision > rf->decision)) { + /* use local decision and timestamp */ + } else { + array_push_back(&changes, rf); + } + li++; ri++; + } else if (ret < 0) { + /* remote field doesn't exist */ + li++; + } else { + /* local field doesn't exist */ + if (remote_fields[ri].last_used < drop_older_timestamp) { + /* field hasn't be used for a long time, remote + will probably drop this soon as well */ + } else { + array_push_back(&changes, &remote_fields[ri]); + } + ri++; + } + } + i_assert(li == local_count && ri == remote_count); + if (array_count(&changes) > 0) { + array_append_zero(&changes); + update->cache_updates = array_front(&changes); + } +} + +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + const char **reason_r) +{ + struct mailbox_update update; + const struct dsync_mailbox_state *state; + bool ret = TRUE; + + *reason_r = NULL; + i_zero(&update); + + if (local_box->uid_validity != remote_box->uid_validity) { + /* Keep the UIDVALIDITY for the mailbox that has more + messages. If they equal, use the higher UIDVALIDITY. */ + if (remote_box->messages_count > local_box->messages_count || + (remote_box->messages_count == local_box->messages_count && + remote_box->uid_validity > local_box->uid_validity)) + update.uid_validity = remote_box->uid_validity; + + state = dsync_mailbox_state_find(brain, local_box->mailbox_guid); + if (state != NULL && state->last_common_uid > 0) { + /* we can't continue syncing this mailbox in this + session, because the other side already started + sending mailbox changes, but not for all mails. */ + dsync_mailbox_state_remove(brain, local_box->mailbox_guid); + *reason_r = "UIDVALIDITY changed during a stateful sync, need to restart"; + brain->failed = TRUE; + ret = FALSE; + } + } + + dsync_cache_fields_update(local_box, remote_box, box, &update); + + if (update.uid_validity == 0 && + update.cache_updates == NULL) { + /* no changes */ + return ret; + } + + if (mailbox_update(box, &update) < 0) { + i_error("Couldn't update mailbox %s metadata: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + brain->failed = TRUE; + } + return ret; +} + +static void +dsync_brain_slave_send_mailbox_lost(struct dsync_brain *brain, + const struct dsync_mailbox *dsync_box, + bool ignore) +{ + struct dsync_mailbox delete_box; + + if (brain->debug) { + i_debug("brain %c: We don't have mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + i_zero(&delete_box); + memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid, + sizeof(delete_box.mailbox_guid)); + t_array_init(&delete_box.cache_fields, 0); + if (ignore) + delete_box.mailbox_ignore = TRUE; + else + delete_box.mailbox_lost = TRUE; + dsync_ibc_send_mailbox(brain->ibc, &delete_box); +} + +bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain) +{ + const struct dsync_mailbox *dsync_box; + struct dsync_mailbox local_dsync_box; + struct mailbox *box; + struct file_lock *lock; + const char *errstr, *resync_reason, *reason; + enum mail_error error; + int ret; + bool resync; + + i_assert(!brain->master_brain); + i_assert(brain->box == NULL); + + if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0) + return FALSE; + if (ret < 0) { + brain->state = DSYNC_STATE_FINISH; + return TRUE; + } + + if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid, + &box, &errstr, &error) < 0) { + i_error("Couldn't allocate mailbox GUID %s: %s", + guid_128_to_string(dsync_box->mailbox_guid), errstr); + brain->mail_error = error; + brain->failed = TRUE; + return TRUE; + } + if (box == NULL) { + /* mailbox was probably deleted/renamed during sync */ + if (brain->backup_send && brain->no_backup_overwrite) { + if (brain->debug) { + i_debug("brain %c: Ignore nonexistent " + "mailbox GUID %s with -1 sync", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, TRUE); + return TRUE; + } + //FIXME: verify this from log, and if not log an error. + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Mailbox GUID %s was lost", + guid_128_to_string(dsync_box->mailbox_guid))); + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE); + return TRUE; + } + /* Lock before syncing */ + if (dsync_mailbox_lock(brain, box, &lock) < 0) { + mailbox_free(&box); + brain->failed = TRUE; + return TRUE; + } + if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) { + i_error("Can't sync mailbox %s: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + file_lock_free(&lock); + mailbox_free(&box); + brain->failed = TRUE; + return TRUE; + } + + if ((ret = dsync_box_get(box, &local_dsync_box, &error)) <= 0) { + file_lock_free(&lock); + mailbox_free(&box); + if (ret < 0) { + brain->mail_error = error; + brain->failed = TRUE; + return TRUE; + } + /* another process just deleted this mailbox? */ + if (brain->debug) { + i_debug("brain %c: Skipping lost mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE); + return TRUE; + } + i_assert(local_dsync_box.uid_validity != 0); + i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid, + sizeof(dsync_box->mailbox_guid)) == 0); + + resync = !dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box, + dsync_box, &resync_reason); + + if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box, &reason)) { + /* no fields appear to have changed, skip this mailbox */ + if (brain->debug) { + i_debug("brain %c: Skipping unchanged mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + } + dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box); + file_lock_free(&lock); + mailbox_free(&box); + return TRUE; + } + if (brain->debug) { + i_debug("brain %c: Syncing mailbox %s: %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid), reason); + } + + /* start export/import */ + dsync_brain_sync_mailbox_init(brain, box, lock, &local_dsync_box, FALSE); + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) + return TRUE; + if (resync) + dsync_brain_set_changes_during_sync(brain, resync_reason); + if (ret == 0 || resync) { + brain->require_full_resync = TRUE; + dsync_brain_sync_mailbox_deinit(brain); + dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE); + return TRUE; + } + + dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box); + brain->state = DSYNC_STATE_SYNC_MAILS; + return TRUE; +} diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c new file mode 100644 index 0000000..3feea20 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-mails.c @@ -0,0 +1,438 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "dsync-ibc.h" +#include "dsync-mail.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" +#include "dsync-brain-private.h" + +const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1] = { + "mailbox", + "changes", + "attributes", + "mail_requests", + "mails", + "recv_last_common", + "done" +}; + +static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain) +{ + const struct dsync_mailbox *dsync_box; + const char *resync_reason, *reason; + enum dsync_ibc_recv_ret ret; + bool resync; + + i_assert(brain->master_brain); + + if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + i_error("Remote sent end-of-list instead of a mailbox"); + brain->failed = TRUE; + return TRUE; + } + if (memcmp(dsync_box->mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(dsync_box->mailbox_guid)) != 0) { + i_error("Remote sent mailbox with a wrong GUID"); + brain->failed = TRUE; + return TRUE; + } + + if (dsync_box->mailbox_ignore) { + /* ignore this box */ + if (brain->debug) + i_debug("brain %c: Ignoring missing remote box GUID %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid)); + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + if (dsync_box->mailbox_lost) { + /* remote lost the mailbox. it's probably already deleted, but + verify it on next sync just to be sure */ + dsync_brain_set_changes_during_sync(brain, t_strdup_printf( + "Remote lost mailbox GUID %s (maybe it was just deleted?)", + guid_128_to_string(dsync_box->mailbox_guid))); + brain->require_full_resync = TRUE; + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + resync = !dsync_brain_mailbox_update_pre(brain, brain->box, + &brain->local_dsync_box, + dsync_box, &resync_reason); + + if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box, + &reason)) { + /* no fields appear to have changed, skip this mailbox */ + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + if (brain->debug) { + i_debug("brain %c: Syncing mailbox %s: %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(dsync_box->mailbox_guid), reason); + } + if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0) + return TRUE; + if (resync) + dsync_brain_set_changes_during_sync(brain, resync_reason); + if (ret == 0 || resync) { + brain->require_full_resync = TRUE; + brain->failed = TRUE; + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; + } + dsync_brain_sync_init_box_states(brain); + return TRUE; +} + +static bool dsync_brain_recv_mailbox_attribute(struct dsync_brain *brain) +{ + const struct dsync_mailbox_attribute *attr; + struct istream *input; + enum dsync_ibc_recv_ret ret; + + if ((ret = dsync_ibc_recv_mailbox_attribute(brain->ibc, &attr)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->box_recv_state = DSYNC_BOX_STATE_CHANGES; + return TRUE; + } + if (dsync_mailbox_import_attribute(brain->box_importer, attr) < 0) + brain->failed = TRUE; + input = attr->value_stream; + i_stream_unref(&input); + return TRUE; +} + +static void dsync_brain_send_end_of_list(struct dsync_brain *brain, + enum dsync_ibc_eol_type type) +{ + i_assert(!brain->failed); + dsync_ibc_send_end_of_list(brain->ibc, type); +} + +static int dsync_brain_export_deinit(struct dsync_brain *brain) +{ + const char *errstr; + enum mail_error error; + + if (dsync_mailbox_export_deinit(&brain->box_exporter, + &errstr, &error) < 0) { + i_error("Exporting mailbox %s failed: %s", + mailbox_get_vname(brain->box), errstr); + brain->mail_error = error; + brain->failed = TRUE; + return -1; + } + return 0; +} + +static void dsync_brain_send_mailbox_attribute(struct dsync_brain *brain) +{ + const struct dsync_mailbox_attribute *attr; + int ret; + + while ((ret = dsync_mailbox_export_next_attr(brain->box_exporter, &attr)) > 0) { + if (dsync_ibc_send_mailbox_attribute(brain->ibc, attr) == 0) + return; + } + if (ret < 0) { + if (dsync_brain_export_deinit(brain) == 0) + i_unreached(); + return; + } + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE); + brain->box_send_state = DSYNC_BOX_STATE_CHANGES; +} + +static bool dsync_brain_recv_mail_change(struct dsync_brain *brain) +{ + const struct dsync_mail_change *change; + enum dsync_ibc_recv_ret ret; + + if ((ret = dsync_ibc_recv_change(brain->ibc, &change)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + if (dsync_mailbox_import_changes_finish(brain->box_importer) < 0) + brain->failed = TRUE; + if (brain->mail_requests && brain->box_exporter != NULL) + brain->box_recv_state = DSYNC_BOX_STATE_MAIL_REQUESTS; + else + brain->box_recv_state = DSYNC_BOX_STATE_MAILS; + return TRUE; + } + if (dsync_mailbox_import_change(brain->box_importer, change) < 0) + brain->failed = TRUE; + return TRUE; +} + +static void dsync_brain_send_mail_change(struct dsync_brain *brain) +{ + const struct dsync_mail_change *change; + int ret; + + while ((ret = dsync_mailbox_export_next(brain->box_exporter, &change)) > 0) { + if (dsync_ibc_send_change(brain->ibc, change) == 0) + return; + } + if (ret < 0) { + if (dsync_brain_export_deinit(brain) == 0) + i_unreached(); + return; + } + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_CHANGES); + if (brain->mail_requests && brain->box_importer != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAIL_REQUESTS; + else + brain->box_send_state = DSYNC_BOX_STATE_MAILS; +} + +static bool dsync_brain_recv_mail_request(struct dsync_brain *brain) +{ + const struct dsync_mail_request *request; + enum dsync_ibc_recv_ret ret; + + i_assert(brain->mail_requests); + i_assert(brain->box_exporter != NULL); + + if ((ret = dsync_ibc_recv_mail_request(brain->ibc, &request)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->box_recv_state = brain->box_importer != NULL ? + DSYNC_BOX_STATE_MAILS : + DSYNC_BOX_STATE_RECV_LAST_COMMON; + return TRUE; + } + dsync_mailbox_export_want_mail(brain->box_exporter, request); + return TRUE; +} + +static bool dsync_brain_send_mail_request(struct dsync_brain *brain) +{ + const struct dsync_mail_request *request; + + i_assert(brain->mail_requests); + + while ((request = dsync_mailbox_import_next_request(brain->box_importer)) != NULL) { + if (dsync_ibc_send_mail_request(brain->ibc, request) == 0) + return TRUE; + } + if (brain->box_recv_state < DSYNC_BOX_STATE_MAIL_REQUESTS) + return FALSE; + + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_REQUESTS); + if (brain->box_exporter != NULL) + brain->box_send_state = DSYNC_BOX_STATE_MAILS; + else { + i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE); + brain->box_send_state = DSYNC_BOX_STATE_DONE; + } + return TRUE; +} + +static void dsync_brain_sync_half_finished(struct dsync_brain *brain) +{ + struct dsync_mailbox_state state; + const char *changes_during_sync; + bool require_full_resync; + + if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON || + brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON) + return; + + /* finished with this mailbox */ + i_zero(&state); + memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(state.mailbox_guid)); + state.last_uidvalidity = brain->local_dsync_box.uid_validity; + if (brain->box_importer == NULL) { + /* this mailbox didn't exist on remote */ + state.last_common_uid = brain->local_dsync_box.uid_next-1; + state.last_common_modseq = + brain->local_dsync_box.highest_modseq; + state.last_common_pvt_modseq = + brain->local_dsync_box.highest_pvt_modseq; + state.last_messages_count = + brain->local_dsync_box.messages_count; + } else { + if (dsync_mailbox_import_deinit(&brain->box_importer, + !brain->failed, + &state.last_common_uid, + &state.last_common_modseq, + &state.last_common_pvt_modseq, + &state.last_messages_count, + &changes_during_sync, + &require_full_resync, + &brain->mail_error) < 0) { + if (require_full_resync) { + /* don't treat this as brain failure or the + state won't be sent to the other brain. + this also means we'll continue syncing the + following mailboxes. */ + brain->require_full_resync = TRUE; + } else { + brain->failed = TRUE; + } + } + if (changes_during_sync != NULL) { + state.changes_during_sync = TRUE; + dsync_brain_set_changes_during_sync(brain, changes_during_sync); + } + } + if (brain->require_full_resync) { + state.last_uidvalidity = 0; + state.changes_during_sync = TRUE; + } + brain->mailbox_state = state; + dsync_ibc_send_mailbox_state(brain->ibc, &state); +} + +static bool dsync_brain_recv_mail(struct dsync_brain *brain) +{ + struct dsync_mail *mail; + enum dsync_ibc_recv_ret ret; + + if ((ret = dsync_ibc_recv_mail(brain->ibc, &mail)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON; + if (brain->box_exporter != NULL && + brain->box_send_state >= DSYNC_BOX_STATE_RECV_LAST_COMMON) { + if (dsync_brain_export_deinit(brain) < 0) + return TRUE; + } + dsync_brain_sync_half_finished(brain); + return TRUE; + } + if (brain->debug) { + i_debug("brain %c: import mail uid %u guid %s", + brain->master_brain ? 'M' : 'S', mail->uid, mail->guid); + } + if (dsync_mailbox_import_mail(brain->box_importer, mail) < 0) + brain->failed = TRUE; + i_stream_unref(&mail->input); + return TRUE; +} + +static bool dsync_brain_send_mail(struct dsync_brain *brain) +{ + const struct dsync_mail *mail; + + if (brain->mail_requests && + brain->box_recv_state < DSYNC_BOX_STATE_MAILS) { + /* wait for mail requests to finish. we could already start + exporting, but then we're going to do quite a lot of + separate searches. especially with pipe backend we'd do + a separate search for each mail. */ + return FALSE; + } + + while (dsync_mailbox_export_next_mail(brain->box_exporter, &mail) > 0) { + if (dsync_ibc_send_mail(brain->ibc, mail) == 0) + return TRUE; + } + + if (dsync_brain_export_deinit(brain) < 0) + return TRUE; + + brain->box_send_state = DSYNC_BOX_STATE_DONE; + dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS); + + dsync_brain_sync_half_finished(brain); + return TRUE; +} + +static bool dsync_brain_recv_last_common(struct dsync_brain *brain) +{ + enum dsync_ibc_recv_ret ret; + struct dsync_mailbox_state state; + + if ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) == 0) + return FALSE; + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + i_error("Remote sent end-of-list instead of a mailbox state"); + brain->failed = TRUE; + return TRUE; + } + i_assert(brain->box_send_state == DSYNC_BOX_STATE_DONE); + i_assert(memcmp(state.mailbox_guid, brain->local_dsync_box.mailbox_guid, + sizeof(state.mailbox_guid)) == 0); + + /* normally the last_common_* values should be the same in local and + remote, but during unexpected changes they may differ. use the + values that are lower as the final state. */ + if (brain->mailbox_state.last_common_uid > state.last_common_uid) + brain->mailbox_state.last_common_uid = state.last_common_uid; + if (brain->mailbox_state.last_common_modseq > state.last_common_modseq) + brain->mailbox_state.last_common_modseq = state.last_common_modseq; + if (brain->mailbox_state.last_common_pvt_modseq > state.last_common_pvt_modseq) + brain->mailbox_state.last_common_pvt_modseq = state.last_common_pvt_modseq; + if (state.changes_during_sync) + brain->changes_during_remote_sync = TRUE; + + dsync_brain_sync_mailbox_deinit(brain); + return TRUE; +} + +bool dsync_brain_sync_mails(struct dsync_brain *brain) +{ + bool changed = FALSE; + + i_assert(brain->box != NULL); + + switch (brain->box_recv_state) { + case DSYNC_BOX_STATE_MAILBOX: + changed = dsync_brain_master_sync_recv_mailbox(brain); + break; + case DSYNC_BOX_STATE_ATTRIBUTES: + changed = dsync_brain_recv_mailbox_attribute(brain); + break; + case DSYNC_BOX_STATE_CHANGES: + changed = dsync_brain_recv_mail_change(brain); + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + changed = dsync_brain_recv_mail_request(brain); + break; + case DSYNC_BOX_STATE_MAILS: + changed = dsync_brain_recv_mail(brain); + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + changed = dsync_brain_recv_last_common(brain); + break; + case DSYNC_BOX_STATE_DONE: + break; + } + + if (!dsync_ibc_is_send_queue_full(brain->ibc) && !brain->failed) { + switch (brain->box_send_state) { + case DSYNC_BOX_STATE_MAILBOX: + /* wait for mailbox to be received first */ + break; + case DSYNC_BOX_STATE_ATTRIBUTES: + dsync_brain_send_mailbox_attribute(brain); + changed = TRUE; + break; + case DSYNC_BOX_STATE_CHANGES: + dsync_brain_send_mail_change(brain); + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAIL_REQUESTS: + if (dsync_brain_send_mail_request(brain)) + changed = TRUE; + break; + case DSYNC_BOX_STATE_MAILS: + if (dsync_brain_send_mail(brain)) + changed = TRUE; + break; + case DSYNC_BOX_STATE_RECV_LAST_COMMON: + i_unreached(); + case DSYNC_BOX_STATE_DONE: + break; + } + } + return changed; +} diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h new file mode 100644 index 0000000..9849703 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain-private.h @@ -0,0 +1,163 @@ +#ifndef DSYNC_BRAIN_PRIVATE_H +#define DSYNC_BRAIN_PRIVATE_H + +#include "hash.h" +#include "dsync-brain.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" + +#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock" +#define DSYNC_MAILBOX_LOCK_FILENAME ".dovecot-box-sync.lock" +#define DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS 30 + +struct dsync_mailbox_tree_sync_change; + +enum dsync_state { + DSYNC_STATE_MASTER_RECV_HANDSHAKE, + DSYNC_STATE_SLAVE_RECV_HANDSHAKE, + /* if sync_type=STATE, the master brain knows the saved "last common + mailbox state". this state is sent to the slave. */ + DSYNC_STATE_MASTER_SEND_LAST_COMMON, + DSYNC_STATE_SLAVE_RECV_LAST_COMMON, + + /* both sides send their mailbox trees */ + DSYNC_STATE_SEND_MAILBOX_TREE, + DSYNC_STATE_SEND_MAILBOX_TREE_DELETES, + DSYNC_STATE_RECV_MAILBOX_TREE, + DSYNC_STATE_RECV_MAILBOX_TREE_DELETES, + + /* master decides in which order mailboxes are synced (it knows the + slave's mailboxes by looking at the received mailbox tree) */ + DSYNC_STATE_MASTER_SEND_MAILBOX, + DSYNC_STATE_SLAVE_RECV_MAILBOX, + /* once mailbox is selected, the mails inside it are synced. + after the mails are synced, another mailbox is synced. */ + DSYNC_STATE_SYNC_MAILS, + + DSYNC_STATE_FINISH, + DSYNC_STATE_DONE +}; + +enum dsync_box_state { + DSYNC_BOX_STATE_MAILBOX, + DSYNC_BOX_STATE_CHANGES, + DSYNC_BOX_STATE_ATTRIBUTES, + DSYNC_BOX_STATE_MAIL_REQUESTS, + DSYNC_BOX_STATE_MAILS, + DSYNC_BOX_STATE_RECV_LAST_COMMON, + DSYNC_BOX_STATE_DONE +}; + +struct dsync_brain { + pool_t pool; + struct mail_user *user; + struct dsync_ibc *ibc; + const char *process_title_prefix; + ARRAY(struct mail_namespace *) sync_namespaces; + const char *sync_box; + struct mailbox *virtual_all_box; + guid_128_t sync_box_guid; + const char *const *exclude_mailboxes; + enum dsync_brain_sync_type sync_type; + time_t sync_since_timestamp; + time_t sync_until_timestamp; + uoff_t sync_max_size; + const char *sync_flag; + char alt_char; + unsigned int import_commit_msgs_interval; + unsigned int hdr_hash_version; + + unsigned int lock_timeout; + int lock_fd; + const char *lock_path; + struct file_lock *lock; + + char hierarchy_sep, escape_char; + struct dsync_mailbox_tree *local_mailbox_tree; + struct dsync_mailbox_tree *remote_mailbox_tree; + struct dsync_mailbox_tree_iter *local_tree_iter; + + enum dsync_state state, pre_box_state; + enum dsync_box_state box_recv_state; + enum dsync_box_state box_send_state; + unsigned int proctitle_update_counter; + + struct dsync_transaction_log_scan *log_scan; + struct dsync_mailbox_importer *box_importer; + struct dsync_mailbox_exporter *box_exporter; + + struct mailbox *box; + struct file_lock *box_lock; + unsigned int mailbox_lock_timeout_secs; + struct dsync_mailbox local_dsync_box, remote_dsync_box; + pool_t dsync_box_pool; + /* list of mailbox states + for master brain: given to brain at init and + for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */ + HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states; + /* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */ + struct hash_iterate_context *mailbox_states_iter; + /* state of the mailbox we're currently syncing, changed at + init and deinit */ + struct dsync_mailbox_state mailbox_state; + /* new states for synced mailboxes */ + ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states; + + const char *changes_during_sync; + enum mail_error mail_error; + + const char *const *hashed_headers; + + bool master_brain:1; + bool mail_requests:1; + bool backup_send:1; + bool backup_recv:1; + bool purge:1; + bool debug:1; + bool sync_visible_namespaces:1; + bool no_mail_sync:1; + bool no_backup_overwrite:1; + bool no_mail_prefetch:1; + bool changes_during_remote_sync:1; + bool require_full_resync:1; + bool verbose_proctitle:1; + bool no_notify:1; + bool failed:1; + bool empty_hdr_workaround:1; +}; + +extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1]; + +void dsync_brain_mailbox_trees_init(struct dsync_brain *brain); +void dsync_brain_send_mailbox_tree(struct dsync_brain *brain); +void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain); +bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain); +bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain); +int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain, + const struct dsync_mailbox_tree_sync_change *change, + enum mail_error *error_r); + +void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain); +int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid, + struct mailbox **box_r, const char **errstr_r, + enum mail_error *error_r); +bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain, + struct mailbox *box, + const struct dsync_mailbox *local_box, + const struct dsync_mailbox *remote_box, + const char **reason_r); +bool dsync_boxes_need_sync(struct dsync_brain *brain, + const struct dsync_mailbox *box1, + const struct dsync_mailbox *box2, + const char **reason_r); +void dsync_brain_sync_init_box_states(struct dsync_brain *brain); +void dsync_brain_set_changes_during_sync(struct dsync_brain *brain, + const char *reason); + +void dsync_brain_master_send_mailbox(struct dsync_brain *brain); +bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain); +int dsync_brain_sync_mailbox_open(struct dsync_brain *brain, + const struct dsync_mailbox *remote_dsync_box); +bool dsync_brain_sync_mails(struct dsync_brain *brain); + +#endif diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c new file mode 100644 index 0000000..8ff7247 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain.c @@ -0,0 +1,901 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "hostpid.h" +#include "str.h" +#include "file-create-locked.h" +#include "process-title.h" +#include "settings-parser.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "mail-namespace.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox-import.h" +#include "dsync-mailbox-export.h" + +#include <sys/stat.h> + +enum dsync_brain_title { + DSYNC_BRAIN_TITLE_NONE = 0, + DSYNC_BRAIN_TITLE_LOCKING, +}; + +static const char *dsync_state_names[] = { + "master_recv_handshake", + "slave_recv_handshake", + "master_send_last_common", + "slave_recv_last_common", + "send_mailbox_tree", + "send_mailbox_tree_deletes", + "recv_mailbox_tree", + "recv_mailbox_tree_deletes", + "master_send_mailbox", + "slave_recv_mailbox", + "sync_mails", + "finish", + "done" +}; + +struct dsync_mailbox_list_module dsync_mailbox_list_module = + MODULE_CONTEXT_INIT(&mailbox_list_module_register); + +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain); + +static const char * +dsync_brain_get_proctitle_full(struct dsync_brain *brain, + enum dsync_brain_title title) +{ + string_t *str = t_str_new(128); + const char *import_title, *export_title; + + str_append_c(str, '['); + if (brain->process_title_prefix != NULL) + str_append(str, brain->process_title_prefix); + str_append(str, brain->user->username); + if (brain->box == NULL) { + str_append_c(str, ' '); + str_append(str, dsync_state_names[brain->state]); + } else { + str_append_c(str, ' '); + str_append(str, mailbox_get_vname(brain->box)); + import_title = brain->box_importer == NULL ? "" : + dsync_mailbox_import_get_proctitle(brain->box_importer); + export_title = brain->box_exporter == NULL ? "" : + dsync_mailbox_export_get_proctitle(brain->box_exporter); + if (import_title[0] == '\0' && export_title[0] == '\0') { + str_printfa(str, " send:%s recv:%s", + dsync_box_state_names[brain->box_send_state], + dsync_box_state_names[brain->box_recv_state]); + } else { + if (import_title[0] != '\0') { + str_append(str, " import:"); + str_append(str, import_title); + } + if (export_title[0] != '\0') { + str_append(str, " export:"); + str_append(str, export_title); + } + } + } + switch (title) { + case DSYNC_BRAIN_TITLE_NONE: + break; + case DSYNC_BRAIN_TITLE_LOCKING: + str_append(str, " locking "DSYNC_LOCK_FILENAME); + break; + } + str_append_c(str, ']'); + return str_c(str); +} + +static const char *dsync_brain_get_proctitle(struct dsync_brain *brain) +{ + return dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_NONE); +} + +static void dsync_brain_run_io(void *context) +{ + struct dsync_brain *brain = context; + bool changed, try_pending; + + if (dsync_ibc_has_failed(brain->ibc)) { + io_loop_stop(current_ioloop); + brain->failed = TRUE; + return; + } + + try_pending = TRUE; + do { + if (!dsync_brain_run(brain, &changed)) { + io_loop_stop(current_ioloop); + break; + } + if (changed) + try_pending = TRUE; + else if (try_pending) { + if (dsync_ibc_has_pending_data(brain->ibc)) + changed = TRUE; + try_pending = FALSE; + } + } while (changed); +} + +static struct dsync_brain * +dsync_brain_common_init(struct mail_user *user, struct dsync_ibc *ibc) +{ + struct dsync_brain *brain; + const struct master_service_settings *service_set; + pool_t pool; + + service_set = master_service_settings_get(master_service); + mail_user_ref(user); + + pool = pool_alloconly_create("dsync brain", 10240); + brain = p_new(pool, struct dsync_brain, 1); + brain->pool = pool; + brain->user = user; + brain->ibc = ibc; + brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN; + brain->lock_fd = -1; + brain->verbose_proctitle = service_set->verbose_proctitle; + hash_table_create(&brain->mailbox_states, pool, 0, + guid_128_hash, guid_128_cmp); + p_array_init(&brain->remote_mailbox_states, pool, 64); + return brain; +} + +static void +dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags) +{ + brain->mail_requests = + (flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0; + brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0; + brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0; + brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0; + brain->sync_visible_namespaces = + (flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0; + brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0; + brain->no_backup_overwrite = + (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0; + brain->no_mail_prefetch = + (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0; + brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0; + brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0; +} + +static void +dsync_brain_open_virtual_all_box(struct dsync_brain *brain, + const char *vname) +{ + struct mail_namespace *ns; + + ns = mail_namespace_find(brain->user->namespaces, vname); + brain->virtual_all_box = + mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY); +} + +struct dsync_brain * +dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const struct dsync_brain_settings *set) +{ + struct dsync_ibc_settings ibc_set; + struct dsync_brain *brain; + struct mail_namespace *ns; + string_t *sync_ns_str = NULL; + const char *error; + + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE || + (set->state != NULL && *set->state != '\0')); + i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1); + + brain = dsync_brain_common_init(user, ibc); + brain->process_title_prefix = + p_strdup(brain->pool, set->process_title_prefix); + brain->sync_type = sync_type; + if (array_count(&set->sync_namespaces) > 0) { + sync_ns_str = t_str_new(128); + p_array_init(&brain->sync_namespaces, brain->pool, + array_count(&set->sync_namespaces)); + array_foreach_elem(&set->sync_namespaces, ns) { + str_append(sync_ns_str, ns->prefix); + str_append_c(sync_ns_str, '\n'); + array_push_back(&brain->sync_namespaces, &ns); + } + str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1); + } + brain->alt_char = set->mailbox_alt_char == '\0' ? '_' : + set->mailbox_alt_char; + brain->sync_since_timestamp = set->sync_since_timestamp; + brain->sync_until_timestamp = set->sync_until_timestamp; + brain->sync_max_size = set->sync_max_size; + brain->sync_flag = p_strdup(brain->pool, set->sync_flag); + brain->sync_box = p_strdup(brain->pool, set->sync_box); + brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(brain->pool, set->exclude_mailboxes); + memcpy(brain->sync_box_guid, set->sync_box_guid, + sizeof(brain->sync_box_guid)); + brain->lock_timeout = set->lock_timeout_secs; + if (brain->lock_timeout != 0) + brain->mailbox_lock_timeout_secs = brain->lock_timeout; + else + brain->mailbox_lock_timeout_secs = + DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS; + brain->import_commit_msgs_interval = set->import_commit_msgs_interval; + brain->master_brain = TRUE; + brain->hashed_headers = + (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers); + dsync_brain_set_flags(brain, flags); + + if (set->virtual_all_box != NULL) + dsync_brain_open_virtual_all_box(brain, set->virtual_all_box); + + if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE) + ; + else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool, + set->state, &error) < 0) { + hash_table_clear(brain->mailbox_states, FALSE); + i_error("Saved sync state is invalid, " + "falling back to full sync: %s", error); + brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + } else { + if (brain->debug) { + i_debug("brain %c: Imported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } + } + dsync_brain_mailbox_trees_init(brain); + + i_zero(&ibc_set); + ibc_set.hostname = my_hostdomain(); + ibc_set.sync_ns_prefixes = sync_ns_str == NULL ? + NULL : str_c(sync_ns_str); + ibc_set.sync_box = set->sync_box; + ibc_set.virtual_all_box = set->virtual_all_box; + ibc_set.exclude_mailboxes = set->exclude_mailboxes; + ibc_set.sync_since_timestamp = set->sync_since_timestamp; + ibc_set.sync_until_timestamp = set->sync_until_timestamp; + ibc_set.sync_max_size = set->sync_max_size; + ibc_set.sync_flags = set->sync_flag; + memcpy(ibc_set.sync_box_guid, set->sync_box_guid, + sizeof(ibc_set.sync_box_guid)); + ibc_set.alt_char = brain->alt_char; + ibc_set.sync_type = sync_type; + ibc_set.hdr_hash_v2 = TRUE; + ibc_set.lock_timeout = set->lock_timeout_secs; + ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval; + ibc_set.hashed_headers = set->hashed_headers; + /* reverse the backup direction for the slave */ + ibc_set.brain_flags = flags & ENUM_NEGATE(DSYNC_BRAIN_FLAG_BACKUP_SEND | + DSYNC_BRAIN_FLAG_BACKUP_RECV); + if ((flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0) + ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + else if ((flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0) + ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + dsync_ibc_send_handshake(ibc, &ibc_set); + + dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain); + brain->state = DSYNC_STATE_MASTER_RECV_HANDSHAKE; + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + return brain; +} + +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, + bool local, const char *process_title_prefix, + char default_alt_char) +{ + struct dsync_ibc_settings ibc_set; + struct dsync_brain *brain; + + i_assert(default_alt_char != '\0'); + + brain = dsync_brain_common_init(user, ibc); + brain->alt_char = default_alt_char; + brain->process_title_prefix = + p_strdup(brain->pool, process_title_prefix); + brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE; + + if (local) { + /* both master and slave are running within the same process, + update the proctitle only for master. */ + brain->verbose_proctitle = FALSE; + } + + i_zero(&ibc_set); + ibc_set.hdr_hash_v2 = TRUE; + ibc_set.hostname = my_hostdomain(); + dsync_ibc_send_handshake(ibc, &ibc_set); + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain); + return brain; +} + +static void dsync_brain_purge(struct dsync_brain *brain) +{ + struct mail_namespace *ns; + struct mail_storage *storage; + + for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) { + if (!dsync_brain_want_namespace(brain, ns)) + continue; + + storage = mail_namespace_get_default_storage(ns); + if (mail_storage_purge(storage) < 0) { + i_error("Purging namespace '%s' failed: %s", ns->prefix, + mail_storage_get_last_internal_error(storage, NULL)); + } + } +} + +int dsync_brain_deinit(struct dsync_brain **_brain, enum mail_error *error_r) +{ + struct dsync_brain *brain = *_brain; + int ret; + + *_brain = NULL; + + if (dsync_ibc_has_timed_out(brain->ibc)) { + i_error("Timeout during state=%s%s", + dsync_state_names[brain->state], + brain->state != DSYNC_STATE_SYNC_MAILS ? "" : + t_strdup_printf(" (send=%s recv=%s)", + dsync_box_state_names[brain->box_send_state], + dsync_box_state_names[brain->box_recv_state])); + } + if (dsync_ibc_has_failed(brain->ibc) || + brain->state != DSYNC_STATE_DONE) + brain->failed = TRUE; + dsync_ibc_close_mail_streams(brain->ibc); + + if (brain->purge && !brain->failed) + dsync_brain_purge(brain); + + if (brain->box != NULL) + dsync_brain_sync_mailbox_deinit(brain); + if (brain->virtual_all_box != NULL) + mailbox_free(&brain->virtual_all_box); + if (brain->local_tree_iter != NULL) + dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter); + if (brain->local_mailbox_tree != NULL) + dsync_mailbox_tree_deinit(&brain->local_mailbox_tree); + if (brain->remote_mailbox_tree != NULL) + dsync_mailbox_tree_deinit(&brain->remote_mailbox_tree); + hash_table_iterate_deinit(&brain->mailbox_states_iter); + hash_table_destroy(&brain->mailbox_states); + + pool_unref(&brain->dsync_box_pool); + + if (brain->lock_fd != -1) { + /* unlink the lock file before it gets unlocked */ + i_unlink(brain->lock_path); + if (brain->debug) { + i_debug("brain %c: Unlocked %s", + brain->master_brain ? 'M' : 'S', + brain->lock_path); + } + file_lock_free(&brain->lock); + i_close_fd(&brain->lock_fd); + } + + ret = brain->failed ? -1 : 0; + mail_user_unref(&brain->user); + + *error_r = !brain->failed ? 0 : + (brain->mail_error == 0 ? MAIL_ERROR_TEMP : brain->mail_error); + pool_unref(&brain->pool); + return ret; +} + +static int +dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname) +{ + const struct file_create_settings lock_set = { + .lock_timeout_secs = brain->lock_timeout, + .lock_settings = { + .lock_method = FILE_LOCK_METHOD_FCNTL, + }, + }; + const char *home, *error, *local_hostname = my_hostdomain(); + bool created; + int ret; + + if ((ret = strcmp(remote_hostname, local_hostname)) < 0) { + /* locking done by remote */ + if (brain->debug) { + i_debug("brain %c: Locking done by remote " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + local_hostname, remote_hostname); + } + return 0; + } + if (ret == 0 && !brain->master_brain) { + /* running dsync within the same server. + locking done by master brain. */ + if (brain->debug) { + i_debug("brain %c: Locking done by local master-brain " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + local_hostname, remote_hostname); + } + return 0; + } + + if ((ret = mail_user_get_home(brain->user, &home)) < 0) { + i_error("Couldn't look up user's home dir"); + return -1; + } + if (ret == 0) { + i_error("User has no home directory"); + return -1; + } + + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING)); + brain->lock_path = p_strconcat(brain->pool, home, + "/"DSYNC_LOCK_FILENAME, NULL); + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + if (brain->lock_fd == -1 && errno == ENOENT) { + /* home directory not created */ + if (mail_user_home_mkdir(brain->user) < 0) + return -1; + brain->lock_fd = file_create_locked(brain->lock_path, &lock_set, + &brain->lock, &created, &error); + } + if (brain->lock_fd == -1) + i_error("Couldn't lock %s: %s", brain->lock_path, error); + else if (brain->debug) { + i_debug("brain %c: Locking done locally in %s " + "(local hostname=%s, remote hostname=%s)", + brain->master_brain ? 'M' : 'S', + brain->lock_path, local_hostname, remote_hostname); + } + if (brain->verbose_proctitle) + process_title_set(dsync_brain_get_proctitle(brain)); + return brain->lock_fd == -1 ? -1 : 0; +} + +static void +dsync_brain_set_hdr_hash_version(struct dsync_brain *brain, + const struct dsync_ibc_settings *ibc_set) +{ + if (ibc_set->hdr_hash_v3) + brain->hdr_hash_version = 3; + else if (ibc_set->hdr_hash_v2) + brain->hdr_hash_version = 3; + else + brain->hdr_hash_version = 1; +} + +static bool dsync_brain_master_recv_handshake(struct dsync_brain *brain) +{ + const struct dsync_ibc_settings *ibc_set; + + i_assert(brain->master_brain); + + if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0) + return FALSE; + + if (brain->lock_timeout > 0) { + if (dsync_brain_lock(brain, ibc_set->hostname) < 0) { + brain->failed = TRUE; + return FALSE; + } + } + dsync_brain_set_hdr_hash_version(brain, ibc_set); + + brain->state = brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE ? + DSYNC_STATE_MASTER_SEND_LAST_COMMON : + DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; +} + +static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain) +{ + const struct dsync_ibc_settings *ibc_set; + struct mail_namespace *ns; + const char *const *prefixes; + + i_assert(!brain->master_brain); + + if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0) + return FALSE; + dsync_brain_set_hdr_hash_version(brain, ibc_set); + + if (ibc_set->lock_timeout > 0) { + brain->lock_timeout = ibc_set->lock_timeout; + brain->mailbox_lock_timeout_secs = brain->lock_timeout; + if (dsync_brain_lock(brain, ibc_set->hostname) < 0) { + brain->failed = TRUE; + return FALSE; + } + } else { + brain->mailbox_lock_timeout_secs = + DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS; + } + + if (ibc_set->sync_ns_prefixes != NULL) { + p_array_init(&brain->sync_namespaces, brain->pool, 4); + prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n"); + if (prefixes[0] == NULL) { + /* ugly workaround for strsplit API: there was one + prefix="" entry */ + static const char *empty_prefix[] = { "", NULL }; + prefixes = empty_prefix; + } + for (; *prefixes != NULL; prefixes++) { + ns = mail_namespace_find(brain->user->namespaces, + *prefixes); + if (ns == NULL) { + i_error("Namespace not found: '%s'", *prefixes); + brain->failed = TRUE; + return FALSE; + } + array_push_back(&brain->sync_namespaces, &ns); + } + } + brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box); + brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes); + brain->sync_since_timestamp = ibc_set->sync_since_timestamp; + brain->sync_until_timestamp = ibc_set->sync_until_timestamp; + brain->sync_max_size = ibc_set->sync_max_size; + brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags); + memcpy(brain->sync_box_guid, ibc_set->sync_box_guid, + sizeof(brain->sync_box_guid)); + if (ibc_set->alt_char != '\0') + brain->alt_char = ibc_set->alt_char; + i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN); + brain->sync_type = ibc_set->sync_type; + + dsync_brain_set_flags(brain, ibc_set->brain_flags); + if (ibc_set->hashed_headers != NULL) + brain->hashed_headers = + p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers); + /* this flag is only set on the remote slave brain */ + brain->purge = (ibc_set->brain_flags & + DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0; + + if (ibc_set->virtual_all_box != NULL) + dsync_brain_open_virtual_all_box(brain, ibc_set->virtual_all_box); + dsync_brain_mailbox_trees_init(brain); + + if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE) + brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON; + else + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + return TRUE; +} + +static void dsync_brain_master_send_last_common(struct dsync_brain *brain) +{ + struct dsync_mailbox_state *state; + uint8_t *guid; + enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK; + + i_assert(brain->master_brain); + + if (brain->mailbox_states_iter == NULL) { + brain->mailbox_states_iter = + hash_table_iterate_init(brain->mailbox_states); + } + + for (;;) { + if (ret == DSYNC_IBC_SEND_RET_FULL) + return; + if (!hash_table_iterate(brain->mailbox_states_iter, + brain->mailbox_states, &guid, &state)) + break; + ret = dsync_ibc_send_mailbox_state(brain->ibc, state); + } + hash_table_iterate_deinit(&brain->mailbox_states_iter); + + dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_STATE); + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; +} + +static void dsync_mailbox_state_add(struct dsync_brain *brain, + const struct dsync_mailbox_state *state) +{ + struct dsync_mailbox_state *dupstate; + uint8_t *guid_p; + + dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1); + *dupstate = *state; + guid_p = dupstate->mailbox_guid; + hash_table_insert(brain->mailbox_states, guid_p, dupstate); +} + +static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain) +{ + struct dsync_mailbox_state state; + enum dsync_ibc_recv_ret ret; + bool changed = FALSE; + + i_assert(!brain->master_brain); + + while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) { + dsync_mailbox_state_add(brain, &state); + changed = TRUE; + } + if (ret == DSYNC_IBC_RECV_RET_FINISHED) { + brain->state = DSYNC_STATE_SEND_MAILBOX_TREE; + changed = TRUE; + } + return changed; +} + +static bool dsync_brain_finish(struct dsync_brain *brain) +{ + const char *error; + enum mail_error mail_error; + bool require_full_resync; + enum dsync_ibc_recv_ret ret; + + if (!brain->master_brain) { + dsync_ibc_send_finish(brain->ibc, + brain->failed ? "dsync failed" : NULL, + brain->mail_error, + brain->require_full_resync); + brain->state = DSYNC_STATE_DONE; + return TRUE; + } + ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error, + &require_full_resync); + if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN) + return FALSE; + if (error != NULL) { + i_error("Remote dsync failed: %s", error); + brain->failed = TRUE; + if (mail_error != 0 && + (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP)) + brain->mail_error = mail_error; + } + if (require_full_resync) + brain->require_full_resync = TRUE; + brain->state = DSYNC_STATE_DONE; + return TRUE; +} + +static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r) +{ + enum dsync_state orig_state = brain->state; + enum dsync_box_state orig_box_recv_state = brain->box_recv_state; + enum dsync_box_state orig_box_send_state = brain->box_send_state; + bool changed = FALSE, ret = TRUE; + + if (brain->failed) + return FALSE; + + switch (brain->state) { + case DSYNC_STATE_MASTER_RECV_HANDSHAKE: + changed = dsync_brain_master_recv_handshake(brain); + break; + case DSYNC_STATE_SLAVE_RECV_HANDSHAKE: + changed = dsync_brain_slave_recv_handshake(brain); + break; + case DSYNC_STATE_MASTER_SEND_LAST_COMMON: + dsync_brain_master_send_last_common(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_LAST_COMMON: + changed = dsync_brain_slave_recv_last_common(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE: + dsync_brain_send_mailbox_tree(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE: + changed = dsync_brain_recv_mailbox_tree(brain); + break; + case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES: + dsync_brain_send_mailbox_tree_deletes(brain); + changed = TRUE; + break; + case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES: + changed = dsync_brain_recv_mailbox_tree_deletes(brain); + break; + case DSYNC_STATE_MASTER_SEND_MAILBOX: + dsync_brain_master_send_mailbox(brain); + changed = TRUE; + break; + case DSYNC_STATE_SLAVE_RECV_MAILBOX: + changed = dsync_brain_slave_recv_mailbox(brain); + break; + case DSYNC_STATE_SYNC_MAILS: + changed = dsync_brain_sync_mails(brain); + break; + case DSYNC_STATE_FINISH: + changed = dsync_brain_finish(brain); + break; + case DSYNC_STATE_DONE: + changed = TRUE; + ret = FALSE; + break; + } + if (brain->verbose_proctitle) { + if (orig_state != brain->state || + orig_box_recv_state != brain->box_recv_state || + orig_box_send_state != brain->box_send_state || + ++brain->proctitle_update_counter % 100 == 0) + process_title_set(dsync_brain_get_proctitle(brain)); + } + *changed_r = changed; + return brain->failed ? FALSE : ret; +} + +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r) +{ + bool ret; + + *changed_r = FALSE; + + if (dsync_ibc_has_failed(brain->ibc)) { + brain->failed = TRUE; + return FALSE; + } + + T_BEGIN { + ret = dsync_brain_run_real(brain, changed_r); + } T_END; + return ret; +} + +static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_state *state; + uint8_t *guid; + + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%"PRIu64" pvt_modseq=%"PRIu64" messages=%u changes_during_sync=%d", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid), + state->last_uidvalidity, + state->last_common_uid, + state->last_common_modseq, + state->last_common_pvt_modseq, + state->last_messages_count, + state->changes_during_sync ? 1 : 0); + } + hash_table_iterate_deinit(&iter); +} + +void dsync_brain_get_state(struct dsync_brain *brain, string_t *output) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_node *node; + const struct dsync_mailbox_state *new_state; + struct dsync_mailbox_state *state; + const uint8_t *guid_p; + uint8_t *guid; + + if (brain->require_full_resync) + return; + + /* update mailbox states */ + array_foreach(&brain->remote_mailbox_states, new_state) { + guid_p = new_state->mailbox_guid; + state = hash_table_lookup(brain->mailbox_states, guid_p); + if (state != NULL) + *state = *new_state; + else + dsync_mailbox_state_add(brain, new_state); + } + + /* remove nonexistent mailboxes */ + iter = hash_table_iterate_init(brain->mailbox_states); + while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) { + node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree, + guid); + if (node == NULL || + node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + if (brain->debug) { + i_debug("brain %c: Removed state for deleted mailbox %s", + brain->master_brain ? 'M' : 'S', + guid_128_to_string(guid)); + } + hash_table_remove(brain->mailbox_states, guid); + } + } + hash_table_iterate_deinit(&iter); + + if (brain->debug) { + i_debug("brain %c: Exported mailbox states:", + brain->master_brain ? 'M' : 'S'); + dsync_brain_mailbox_states_dump(brain); + } + dsync_mailbox_states_export(brain->mailbox_states, output); +} + +enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain) +{ + return brain->sync_type; +} + +bool dsync_brain_has_failed(struct dsync_brain *brain) +{ + return brain->failed; +} + +const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain, + bool *remote_only_r) +{ + if (brain->changes_during_sync == NULL && + brain->changes_during_remote_sync) { + *remote_only_r = TRUE; + return "Remote notified that changes happened during sync"; + } + *remote_only_r = FALSE; + return brain->changes_during_sync; +} + +static bool dsync_brain_want_shared_namespace(const struct mail_namespace *ns, + const struct mail_namespace *sync_ns) +{ + /* Include shared namespaces and all its + children in the sync (e.g. "Shared/example.com" + will be synced to "Shared/"). + This also allows "dsync -n Shared/example.com/" + with "Shared/example.com/username/" style + shared namespace config. */ + return (ns->type == MAIL_NAMESPACE_TYPE_SHARED) && + (sync_ns->type == MAIL_NAMESPACE_TYPE_SHARED) && + str_begins(ns->prefix, sync_ns->prefix); +} + +bool dsync_brain_want_namespace(struct dsync_brain *brain, + struct mail_namespace *ns) +{ + struct mail_namespace *sync_ns; + + if (array_is_created(&brain->sync_namespaces)) { + array_foreach_elem(&brain->sync_namespaces, sync_ns) { + if (ns == sync_ns) + return TRUE; + if (dsync_brain_want_shared_namespace(ns, sync_ns)) + return TRUE; + } + return FALSE; + } + if (ns->alias_for != NULL) { + /* always skip aliases */ + return FALSE; + } + if (brain->sync_visible_namespaces) { + if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0) + return TRUE; + if ((ns->flags & (NAMESPACE_FLAG_LIST_PREFIX | + NAMESPACE_FLAG_LIST_CHILDREN)) != 0) + return TRUE; + return FALSE; + } else { + return strcmp(ns->unexpanded_set->location, + SETTING_STRVAR_UNEXPANDED) == 0; + } +} + +void dsync_brain_set_changes_during_sync(struct dsync_brain *brain, + const char *reason) +{ + if (brain->debug) { + i_debug("brain %c: Change during sync: %s", + brain->master_brain ? 'M' : 'S', reason); + } + if (brain->changes_during_sync == NULL) + brain->changes_during_sync = p_strdup(brain->pool, reason); +} diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h new file mode 100644 index 0000000..5813148 --- /dev/null +++ b/src/doveadm/dsync/dsync-brain.h @@ -0,0 +1,130 @@ +#ifndef DSYNC_BRAIN_H +#define DSYNC_BRAIN_H + +#include "module-context.h" +#include "guid.h" +#include "mail-error.h" +#include "mailbox-list-private.h" + +struct mail_namespace; +struct mail_user; +struct dsync_ibc; + +enum dsync_brain_flags { + DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS = 0x01, + DSYNC_BRAIN_FLAG_BACKUP_SEND = 0x02, + DSYNC_BRAIN_FLAG_BACKUP_RECV = 0x04, + DSYNC_BRAIN_FLAG_DEBUG = 0x08, + DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES= 0x10, + /* Sync everything but the actual mails (e.g. mailbox creates, + deletes) */ + DSYNC_BRAIN_FLAG_NO_MAIL_SYNC = 0x20, + /* Used with BACKUP_SEND/RECV: Don't force the + Use the two-way syncing algorithm, but don't actually modify + anything locally. (Useful during migration.) */ + DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40, + /* Run storage purge on the remote after syncing. + Useful with e.g. a nightly doveadm backup. */ + DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80, + /* Don't prefetch mail bodies until they're actually needed. This works + only with pipe ibc. It's useful if most of the mails can be copied + directly within filesystem without having to read them. */ + DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100, + /* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */ + DSYNC_BRAIN_FLAG_NO_NOTIFY = 0x400, + /* Workaround missing Date/Message-ID headers */ + DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND = 0x800, +}; + +enum dsync_brain_sync_type { + DSYNC_BRAIN_SYNC_TYPE_UNKNOWN, + /* Go through all mailboxes to make sure everything is synced */ + DSYNC_BRAIN_SYNC_TYPE_FULL, + /* Go through all mailboxes that have changed (based on UIDVALIDITY, + UIDNEXT, HIGHESTMODSEQ). If both sides have had equal amount of + changes in some mailbox, it may get incorrectly skipped. */ + DSYNC_BRAIN_SYNC_TYPE_CHANGED, + /* Use saved state to find out what has changed. */ + DSYNC_BRAIN_SYNC_TYPE_STATE +}; + +struct dsync_brain_settings { + const char *process_title_prefix; + /* Sync only these namespaces */ + ARRAY(struct mail_namespace *) sync_namespaces; + /* Sync only this mailbox name */ + const char *sync_box; + /* Use this virtual \All mailbox to be able to copy mails with the same + GUID instead of saving them twice. With most storages this results + in less disk space usage. */ + const char *virtual_all_box; + /* Sync only this mailbox GUID */ + guid_128_t sync_box_guid; + /* Exclude these mailboxes from the sync. They can contain '*' + wildcards and be \special-use flags. */ + const char *const *exclude_mailboxes; + /* Alternative character to use in mailbox names where the original + character cannot be used. */ + char mailbox_alt_char; + /* Sync only mails with received timestamp at least this high. */ + time_t sync_since_timestamp; + /* Sync only mails with received timestamp less or equal than this */ + time_t sync_until_timestamp; + /* Don't sync mails larger than this. */ + uoff_t sync_max_size; + /* Sync only mails which contains / doesn't contain this flag. + '-' at the beginning means this flag must not exist. */ + const char *sync_flag; + /* Headers to hash (defaults to Date, Message-ID) */ + const char *const *hashed_headers; + + /* If non-zero, use dsync lock file for this user */ + unsigned int lock_timeout_secs; + /* If non-zero, importing will attempt to commit transaction after + saving this many messages. */ + unsigned int import_commit_msgs_interval; + /* Input state for DSYNC_BRAIN_SYNC_TYPE_STATE */ + const char *state; +}; + +#define DSYNC_LIST_CONTEXT(obj) \ + MODULE_CONTEXT(obj, dsync_mailbox_list_module) +struct dsync_mailbox_list { + union mailbox_list_module_context module_ctx; + bool have_orig_escape_char; +}; +extern MODULE_CONTEXT_DEFINE(dsync_mailbox_list_module, + &mailbox_list_module_register); + +struct dsync_brain * +dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc, + enum dsync_brain_sync_type sync_type, + enum dsync_brain_flags flags, + const struct dsync_brain_settings *set); +struct dsync_brain * +dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc, + bool local, const char *process_title_prefix, + char default_alt_char); +/* Returns 0 if everything was successful, -1 if syncing failed in some way */ +int dsync_brain_deinit(struct dsync_brain **brain, enum mail_error *error_r); + +/* Returns TRUE if brain needs to run more, FALSE if it's finished. + changed_r is TRUE if anything happened during this run. */ +bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r); +/* Returns TRUE if brain has failed, and there's no point in continuing. */ +bool dsync_brain_has_failed(struct dsync_brain *brain); +/* Returns the current sync state string, which can be given as parameter to + dsync_brain_master_init() to quickly sync only the new changes. */ +void dsync_brain_get_state(struct dsync_brain *brain, string_t *output); +/* Returns the sync type that was used. Mainly useful with slave brain. */ +enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain); +/* If there were any unexpected changes during the sync, return the reason + for them. Otherwise return NULL. If remote_only_r=TRUE, this brain itself + didn't see any changes, but the remote brain did. */ +const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain, + bool *remote_only_r); +/* Returns TRUE if we want to sync this namespace. */ +bool dsync_brain_want_namespace(struct dsync_brain *brain, + struct mail_namespace *ns); + +#endif diff --git a/src/doveadm/dsync/dsync-deserializer.c b/src/doveadm/dsync/dsync-deserializer.c new file mode 100644 index 0000000..2a90c47 --- /dev/null +++ b/src/doveadm/dsync/dsync-deserializer.c @@ -0,0 +1,193 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "str.h" +#include "strescape.h" +#include "dsync-serializer.h" +#include "dsync-deserializer.h" + +struct dsync_deserializer { + pool_t pool; + const char *name; + const char *const *required_fields; + const char *const *keys; + unsigned int *required_field_indexes; + unsigned int required_field_count; +}; + +struct dsync_deserializer_decoder { + pool_t pool; + struct dsync_deserializer *deserializer; + const char *const *values; + unsigned int values_count; +}; + +static bool field_find(const char *const *names, const char *name, + unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; names[i] != NULL; i++) { + if (strcmp(names[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +int dsync_deserializer_init(const char *name, const char *const *required_fields, + const char *header_line, + struct dsync_deserializer **deserializer_r, + const char **error_r) +{ + struct dsync_deserializer *deserializer; + const char **dup_required_fields; + unsigned int i, required_count; + pool_t pool; + + *deserializer_r = NULL; + + pool = pool_alloconly_create("dsync deserializer", 1024); + deserializer = p_new(pool, struct dsync_deserializer, 1); + deserializer->pool = pool; + deserializer->name = p_strdup(pool, name); + deserializer->keys = (void *)p_strsplit_tabescaped(pool, header_line); + + deserializer->required_field_count = required_count = + required_fields == NULL ? 0 : + str_array_length(required_fields); + dup_required_fields = p_new(pool, const char *, required_count + 1); + deserializer->required_field_indexes = + p_new(pool, unsigned int, required_count + 1); + for (i = 0; i < required_count; i++) { + dup_required_fields[i] = + p_strdup(pool, required_fields[i]); + if (!field_find(deserializer->keys, required_fields[i], + &deserializer->required_field_indexes[i])) { + *error_r = t_strdup_printf( + "Header missing required field %s", + required_fields[i]); + pool_unref(&pool); + return -1; + } + } + deserializer->required_fields = dup_required_fields; + + *deserializer_r = deserializer; + return 0; +} + +void dsync_deserializer_deinit(struct dsync_deserializer **_deserializer) +{ + struct dsync_deserializer *deserializer = *_deserializer; + + *_deserializer = NULL; + + pool_unref(&deserializer->pool); +} + +int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer, + const char *input, + struct dsync_deserializer_decoder **decoder_r, + const char **error_r) +{ + struct dsync_deserializer_decoder *decoder; + unsigned int i; + char **values; + pool_t pool; + + *decoder_r = NULL; + + pool = pool_alloconly_create("dsync deserializer decode", 1024); + decoder = p_new(pool, struct dsync_deserializer_decoder, 1); + decoder->pool = pool; + decoder->deserializer = deserializer; + values = p_strsplit_tabescaped(pool, input); + + /* fix NULLs */ + for (i = 0; values[i] != NULL; i++) { + if (values[i][0] == NULL_CHR) { + /* NULL? */ + if (values[i][1] == '\0') + values[i] = NULL; + else + values[i] += 1; + } + } + decoder->values_count = i; + + /* see if all required fields exist */ + for (i = 0; i < deserializer->required_field_count; i++) { + unsigned int ridx = deserializer->required_field_indexes[i]; + + if (ridx >= decoder->values_count || values[ridx] == NULL) { + *error_r = t_strdup_printf("Missing required field %s", + deserializer->required_fields[i]); + pool_unref(&pool); + return -1; + } + } + decoder->values = (void *)values; + + *decoder_r = decoder; + return 0; +} + +static bool +dsync_deserializer_find_field(struct dsync_deserializer *deserializer, + const char *key, unsigned int *idx_r) +{ + unsigned int i; + + for (i = 0; deserializer->keys[i] != NULL; i++) { + if (strcmp(deserializer->keys[i], key) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder, + const char *key, const char **value_r) +{ + unsigned int idx; + + if (!dsync_deserializer_find_field(decoder->deserializer, key, &idx) || + idx >= decoder->values_count) { + *value_r = NULL; + return FALSE; + } else { + *value_r = decoder->values[idx]; + return *value_r != NULL; + } +} + +const char * +dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder, + const char *key) +{ + const char *value; + + if (!dsync_deserializer_decode_try(decoder, key, &value)) { + i_panic("dsync_deserializer_decode_get() " + "used for non-required key %s", key); + } + return value; +} + +const char * +dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder) +{ + return decoder->deserializer->name; +} + +void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **_decoder) +{ + struct dsync_deserializer_decoder *decoder = *_decoder; + + *_decoder = NULL; + + pool_unref(&decoder->pool); +} diff --git a/src/doveadm/dsync/dsync-deserializer.h b/src/doveadm/dsync/dsync-deserializer.h new file mode 100644 index 0000000..45b19cc --- /dev/null +++ b/src/doveadm/dsync/dsync-deserializer.h @@ -0,0 +1,27 @@ +#ifndef DSYNC_DESERIALIZER_H +#define DSYNC_DESERIALIZER_H + +struct dsync_deserializer; +struct dsync_deserializer_decoder; + +int dsync_deserializer_init(const char *name, const char *const *required_fields, + const char *header_line, + struct dsync_deserializer **deserializer_r, + const char **error_r); +void dsync_deserializer_deinit(struct dsync_deserializer **deserializer); + +int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer, + const char *input, + struct dsync_deserializer_decoder **decoder_r, + const char **error_r); +bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder, + const char *key, const char **value_r); +/* key must be in required fields. The return value is never NULL. */ +const char * +dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder, + const char *key); +const char * +dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder); +void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **decoder); + +#endif diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c new file mode 100644 index 0000000..1b8886e --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc-pipe.c @@ -0,0 +1,599 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "istream.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc-private.h" + +enum item_type { + ITEM_END_OF_LIST, + ITEM_HANDSHAKE, + ITEM_MAILBOX_STATE, + ITEM_MAILBOX_TREE_NODE, + ITEM_MAILBOX_DELETE, + ITEM_MAILBOX, + ITEM_MAILBOX_ATTRIBUTE, + ITEM_MAIL_CHANGE, + ITEM_MAIL_REQUEST, + ITEM_MAIL, + ITEM_FINISH +}; + +struct item { + enum item_type type; + pool_t pool; + + union { + struct dsync_ibc_settings set; + struct dsync_mailbox_state state; + struct dsync_mailbox_node node; + guid_128_t mailbox_guid; + struct dsync_mailbox dsync_box; + struct dsync_mailbox_attribute attr; + struct dsync_mail_change change; + struct dsync_mail_request request; + struct dsync_mail mail; + struct { + const struct dsync_mailbox_delete *deletes; + unsigned int count; + char hierarchy_sep; + char escape_char; + } mailbox_delete; + struct { + const char *error; + enum mail_error mail_error; + bool require_full_resync; + } finish; + } u; +}; + +struct dsync_ibc_pipe { + struct dsync_ibc ibc; + + ARRAY(pool_t) pools; + ARRAY(struct item) item_queue; + struct dsync_ibc_pipe *remote; + + pool_t pop_pool; + struct item pop_item; +}; + +static pool_t dsync_ibc_pipe_get_pool(struct dsync_ibc_pipe *pipe) +{ + pool_t *pools, ret; + unsigned int count; + + pools = array_get_modifiable(&pipe->pools, &count); + if (count == 0) + return pool_alloconly_create(MEMPOOL_GROWING"pipe item pool", 1024); + + ret = pools[count-1]; + array_delete(&pipe->pools, count-1, 1); + p_clear(ret); + return ret; +} + +static struct item * ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_pipe_push_item(struct dsync_ibc_pipe *pipe, enum item_type type) +{ + struct item *item; + + item = array_append_space(&pipe->item_queue); + item->type = type; + + switch (type) { + case ITEM_END_OF_LIST: + case ITEM_MAILBOX_STATE: + case ITEM_MAILBOX_DELETE: + break; + case ITEM_HANDSHAKE: + case ITEM_MAILBOX: + case ITEM_MAILBOX_TREE_NODE: + case ITEM_MAILBOX_ATTRIBUTE: + case ITEM_MAIL_CHANGE: + case ITEM_MAIL_REQUEST: + case ITEM_MAIL: + case ITEM_FINISH: + item->pool = dsync_ibc_pipe_get_pool(pipe); + break; + } + return item; +} + +static struct item * +dsync_ibc_pipe_pop_item(struct dsync_ibc_pipe *pipe, enum item_type type) +{ + struct item *item; + + if (array_count(&pipe->item_queue) == 0) + return NULL; + + item = array_front_modifiable(&pipe->item_queue); + i_assert(item->type == type); + pipe->pop_item = *item; + array_pop_front(&pipe->item_queue); + item = NULL; + + pool_unref(&pipe->pop_pool); + pipe->pop_pool = pipe->pop_item.pool; + return &pipe->pop_item; +} + +static bool dsync_ibc_pipe_try_pop_eol(struct dsync_ibc_pipe *pipe) +{ + const struct item *item; + + if (array_count(&pipe->item_queue) == 0) + return FALSE; + + item = array_front(&pipe->item_queue); + if (item->type != ITEM_END_OF_LIST) + return FALSE; + + array_pop_front(&pipe->item_queue); + return TRUE; +} + +static void dsync_ibc_pipe_deinit(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + pool_t pool; + + if (pipe->remote != NULL) { + i_assert(pipe->remote->remote == pipe); + pipe->remote->remote = NULL; + } + + pool_unref(&pipe->pop_pool); + array_foreach_modifiable(&pipe->item_queue, item) { + pool_unref(&item->pool); + } + array_foreach_elem(&pipe->pools, pool) + pool_unref(&pool); + array_free(&pipe->pools); + array_free(&pipe->item_queue); + i_free(pipe); +} + +static void +dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_HANDSHAKE); + item->u.set = *set; + item->u.set.sync_ns_prefixes = + p_strdup(item->pool, set->sync_ns_prefixes); + item->u.set.sync_box = p_strdup(item->pool, set->sync_box); + item->u.set.virtual_all_box = p_strdup(item->pool, set->virtual_all_box); + item->u.set.exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL : + p_strarray_dup(item->pool, set->exclude_mailboxes); + memcpy(item->u.set.sync_box_guid, set->sync_box_guid, + sizeof(item->u.set.sync_box_guid)); + item->u.set.sync_since_timestamp = set->sync_since_timestamp; + item->u.set.sync_until_timestamp = set->sync_until_timestamp; + item->u.set.sync_max_size = set->sync_max_size; + item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_HANDSHAKE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *set_r = &item->u.set; + return DSYNC_IBC_RECV_RET_OK; +} + +static bool dsync_ibc_pipe_is_send_queue_full(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + return array_count(&pipe->remote->item_queue) > 0; +} + +static bool dsync_ibc_pipe_has_pending_data(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + return array_count(&pipe->item_queue) > 0; +} + +static void +dsync_ibc_pipe_send_end_of_list(struct dsync_ibc *ibc, + enum dsync_ibc_eol_type type ATTR_UNUSED) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + dsync_ibc_pipe_push_item(pipe->remote, ITEM_END_OF_LIST); +} + +static void +dsync_ibc_pipe_send_mailbox_state(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_STATE); + item->u.state = *state; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_state(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_STATE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *state_r = item->u.state; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_TREE_NODE); + + /* a little bit kludgy way to send it */ + item->u.node.name = (void *)p_strarray_dup(item->pool, name); + dsync_mailbox_node_copy_data(&item->u.node, node); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_TREE_NODE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *name_r = (const void *)item->u.node.name; + item->u.node.name = NULL; + + *node_r = &item->u.node; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_DELETE); + + /* we'll assume that the deletes are permanent. this works for now.. */ + /* a little bit kludgy way to send it */ + item->u.mailbox_delete.deletes = deletes; + item->u.mailbox_delete.count = count; + item->u.mailbox_delete.hierarchy_sep = hierarchy_sep; + item->u.mailbox_delete.escape_char = escape_char; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, + char *hierarchy_sep_r, + char *escape_char_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_DELETE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *deletes_r = item->u.mailbox_delete.deletes; + *count_r = item->u.mailbox_delete.count; + *hierarchy_sep_r = item->u.mailbox_delete.hierarchy_sep; + *escape_char_r = item->u.mailbox_delete.escape_char; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + const struct mailbox_cache_field *cf; + struct mailbox_cache_field *ncf; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX); + item->u.dsync_box = *dsync_box; + p_array_init(&item->u.dsync_box.cache_fields, item->pool, + array_count(&dsync_box->cache_fields)); + array_foreach(&dsync_box->cache_fields, cf) { + ncf = array_append_space(&item->u.dsync_box.cache_fields); + ncf->name = p_strdup(item->pool, cf->name); + ncf->decision = cf->decision; + ncf->last_used = cf->last_used; + } +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *dsync_box_r = &item->u.dsync_box; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_ATTRIBUTE); + dsync_mailbox_attribute_dup(item->pool, attr, &item->u.attr); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_ATTRIBUTE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *attr_r = &item->u.attr; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_change(struct dsync_ibc *ibc, + const struct dsync_mail_change *change) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_CHANGE); + dsync_mail_change_dup(item->pool, change, &item->u.change); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_change(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_CHANGE); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *change_r = &item->u.change; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request *request) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_REQUEST); + item->u.request.guid = p_strdup(item->pool, request->guid); + item->u.request.uid = request->uid; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_REQUEST); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *request_r = &item->u.request; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL); + item->u.mail.guid = p_strdup(item->pool, mail->guid); + item->u.mail.uid = mail->uid; + item->u.mail.pop3_uidl = p_strdup(item->pool, mail->pop3_uidl); + item->u.mail.pop3_order = mail->pop3_order; + item->u.mail.received_date = mail->received_date; + if (mail->input != NULL) { + item->u.mail.input = mail->input; + i_stream_ref(mail->input); + } + item->u.mail.input_mail = mail->input_mail; + item->u.mail.input_mail_uid = mail->input_mail_uid; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + if (dsync_ibc_pipe_try_pop_eol(pipe)) + return DSYNC_IBC_RECV_RET_FINISHED; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *mail_r = &item->u.mail; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_FINISH); + item->u.finish.error = p_strdup(item->pool, error); + item->u.finish.mail_error = mail_error; + item->u.finish.require_full_resync = require_full_resync; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + struct item *item; + + item = dsync_ibc_pipe_pop_item(pipe, ITEM_FINISH); + if (item == NULL) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + *error_r = item->u.finish.error; + *mail_error_r = item->u.finish.mail_error; + *require_full_resync_r = item->u.finish.require_full_resync; + return DSYNC_IBC_RECV_RET_OK; +} + +static void pipe_close_mail_streams(struct dsync_ibc_pipe *pipe) +{ + struct item *item; + + if (array_count(&pipe->item_queue) > 0) { + item = array_front_modifiable(&pipe->item_queue); + if (item->type == ITEM_MAIL && + item->u.mail.input != NULL) + i_stream_unref(&item->u.mail.input); + } +} + +static void dsync_ibc_pipe_close_mail_streams(struct dsync_ibc *ibc) +{ + struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc; + + pipe_close_mail_streams(pipe); + pipe_close_mail_streams(pipe->remote); +} + +static const struct dsync_ibc_vfuncs dsync_ibc_pipe_vfuncs = { + dsync_ibc_pipe_deinit, + dsync_ibc_pipe_send_handshake, + dsync_ibc_pipe_recv_handshake, + dsync_ibc_pipe_send_end_of_list, + dsync_ibc_pipe_send_mailbox_state, + dsync_ibc_pipe_recv_mailbox_state, + dsync_ibc_pipe_send_mailbox_tree_node, + dsync_ibc_pipe_recv_mailbox_tree_node, + dsync_ibc_pipe_send_mailbox_deletes, + dsync_ibc_pipe_recv_mailbox_deletes, + dsync_ibc_pipe_send_mailbox, + dsync_ibc_pipe_recv_mailbox, + dsync_ibc_pipe_send_mailbox_attribute, + dsync_ibc_pipe_recv_mailbox_attribute, + dsync_ibc_pipe_send_change, + dsync_ibc_pipe_recv_change, + dsync_ibc_pipe_send_mail_request, + dsync_ibc_pipe_recv_mail_request, + dsync_ibc_pipe_send_mail, + dsync_ibc_pipe_recv_mail, + dsync_ibc_pipe_send_finish, + dsync_ibc_pipe_recv_finish, + dsync_ibc_pipe_close_mail_streams, + dsync_ibc_pipe_is_send_queue_full, + dsync_ibc_pipe_has_pending_data +}; + +static struct dsync_ibc_pipe * +dsync_ibc_pipe_alloc(void) +{ + struct dsync_ibc_pipe *pipe; + + pipe = i_new(struct dsync_ibc_pipe, 1); + pipe->ibc.v = dsync_ibc_pipe_vfuncs; + i_array_init(&pipe->pools, 4); + i_array_init(&pipe->item_queue, 4); + return pipe; +} + +void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r, struct dsync_ibc **ibc2_r) +{ + struct dsync_ibc_pipe *pipe1, *pipe2; + + pipe1 = dsync_ibc_pipe_alloc(); + pipe2 = dsync_ibc_pipe_alloc(); + pipe1->remote = pipe2; + pipe2->remote = pipe1; + *ibc1_r = &pipe1->ibc; + *ibc2_r = &pipe2->ibc; +} diff --git a/src/doveadm/dsync/dsync-ibc-private.h b/src/doveadm/dsync/dsync-ibc-private.h new file mode 100644 index 0000000..c055aca --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc-private.h @@ -0,0 +1,96 @@ +#ifndef DSYNC_IBC_PRIVATE_H +#define DSYNC_IBC_PRIVATE_H + +#include "dsync-ibc.h" + +struct dsync_ibc_vfuncs { + void (*deinit)(struct dsync_ibc *ibc); + + void (*send_handshake)(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set); + enum dsync_ibc_recv_ret + (*recv_handshake)(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r); + + void (*send_end_of_list)(struct dsync_ibc *ibc, + enum dsync_ibc_eol_type type); + + void (*send_mailbox_state)(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state); + enum dsync_ibc_recv_ret + (*recv_mailbox_state)(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r); + + void (*send_mailbox_tree_node)(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node); + enum dsync_ibc_recv_ret + (*recv_mailbox_tree_node)(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r); + + void (*send_mailbox_deletes)(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char); + enum dsync_ibc_recv_ret + (*recv_mailbox_deletes)(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, + char *hierarchy_sep_r, + char *escape_char_r); + + void (*send_mailbox)(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box); + enum dsync_ibc_recv_ret + (*recv_mailbox)(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r); + + void (*send_mailbox_attribute)(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr); + enum dsync_ibc_recv_ret + (*recv_mailbox_attribute)(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r); + + void (*send_change)(struct dsync_ibc *ibc, + const struct dsync_mail_change *change); + enum dsync_ibc_recv_ret + (*recv_change)(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r); + + void (*send_mail_request)(struct dsync_ibc *ibc, + const struct dsync_mail_request *request); + enum dsync_ibc_recv_ret + (*recv_mail_request)(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r); + + void (*send_mail)(struct dsync_ibc *ibc, + const struct dsync_mail *mail); + enum dsync_ibc_recv_ret + (*recv_mail)(struct dsync_ibc *ibc, + struct dsync_mail **mail_r); + + void (*send_finish)(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync); + enum dsync_ibc_recv_ret + (*recv_finish)(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r); + + void (*close_mail_streams)(struct dsync_ibc *ibc); + bool (*is_send_queue_full)(struct dsync_ibc *ibc); + bool (*has_pending_data)(struct dsync_ibc *ibc); +}; + +struct dsync_ibc { + struct dsync_ibc_vfuncs v; + + io_callback_t *io_callback; + void *io_context; + + bool failed:1; + bool timeout:1; +}; + +#endif diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c new file mode 100644 index 0000000..17115b0 --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc-stream.c @@ -0,0 +1,2138 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "safe-mkstemp.h" +#include "ioloop.h" +#include "istream.h" +#include "istream-seekable.h" +#include "istream-dot.h" +#include "ostream.h" +#include "str.h" +#include "strescape.h" +#include "master-service.h" +#include "mail-cache.h" +#include "mail-storage-private.h" +#include "dsync-serializer.h" +#include "dsync-deserializer.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-state.h" +#include "dsync-mailbox-tree.h" +#include "dsync-ibc-private.h" + + +#define DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE (1024*128) + +#define DSYNC_PROTOCOL_VERSION_MAJOR 3 +#define DSYNC_PROTOCOL_VERSION_MINOR 5 +#define DSYNC_HANDSHAKE_VERSION "VERSION\tdsync\t3\t5\n" + +#define DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES 1 +#define DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID 2 +#define DSYNC_PROTOCOL_MINOR_HAVE_FINISH 3 +#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2 4 +#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3 5 + +enum item_type { + ITEM_NONE, + ITEM_DONE, + + ITEM_HANDSHAKE, + ITEM_MAILBOX_STATE, + ITEM_MAILBOX_TREE_NODE, + ITEM_MAILBOX_DELETE, + ITEM_MAILBOX, + + ITEM_MAILBOX_ATTRIBUTE, + ITEM_MAIL_CHANGE, + ITEM_MAIL_REQUEST, + ITEM_MAIL, + ITEM_FINISH, + + ITEM_MAILBOX_CACHE_FIELD, + + ITEM_END_OF_LIST +}; + +#define END_OF_LIST_LINE "." +static const struct { + /* full human readable name of the item */ + const char *name; + /* unique character identifying the item */ + char chr; + const char *required_keys; + const char *optional_keys; + unsigned int min_minor_version; +} items[ITEM_END_OF_LIST+1] = { + { NULL, '\0', NULL, NULL, 0 }, + { .name = "done", + .chr = 'X', + .optional_keys = "" + }, + { .name = "handshake", + .chr = 'H', + .required_keys = "hostname", + .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type " + "debug sync_visible_namespaces exclude_mailboxes " + "send_mail_requests backup_send backup_recv lock_timeout " + "no_mail_sync no_backup_overwrite purge_remote " + "no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp " + "virtual_all_box empty_hdr_workaround import_commit_msgs_interval " + "hashed_headers alt_char" + }, + { .name = "mailbox_state", + .chr = 'S', + .required_keys = "mailbox_guid last_uidvalidity last_common_uid " + "last_common_modseq last_common_pvt_modseq", + .optional_keys = "last_messages_count changes_during_sync" + }, + { .name = "mailbox_tree_node", + .chr = 'N', + .required_keys = "name existence", + .optional_keys = "mailbox_guid uid_validity uid_next " + "last_renamed_or_created subscribed last_subscription_change" + }, + { .name = "mailbox_delete", + .chr = 'D', + .required_keys = "hierarchy_sep", + .optional_keys = "escape_char mailboxes dirs unsubscribes" + }, + { .name = "mailbox", + .chr = 'B', + .required_keys = "mailbox_guid uid_validity uid_next messages_count " + "first_recent_uid highest_modseq highest_pvt_modseq", + .optional_keys = "mailbox_lost mailbox_ignore " + "cache_fields have_guids have_save_guids have_only_guid128" + }, + { .name = "mailbox_attribute", + .chr = 'A', + .required_keys = "type key", + .optional_keys = "value stream deleted last_change modseq", + .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES + }, + { .name = "mail_change", + .chr = 'C', + .required_keys = "type uid", + .optional_keys = "guid hdr_hash modseq pvt_modseq " + "add_flags remove_flags final_flags " + "keywords_reset keyword_changes received_timestamp virtual_size" + }, + { .name = "mail_request", + .chr = 'R', + .optional_keys = "guid uid" + }, + { .name = "mail", + .chr = 'M', + .optional_keys = "guid uid pop3_uidl pop3_order received_date saved_date stream" + }, + { .name = "finish", + .chr = 'F', + .optional_keys = "error mail_error require_full_resync", + .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_FINISH + }, + { .name = "mailbox_cache_field", + .chr = 'c', + .required_keys = "name decision", + .optional_keys = "last_used" + }, + + { "end_of_list", '\0', NULL, NULL, 0 } +}; + +struct dsync_ibc_stream { + struct dsync_ibc ibc; + + char *name, *temp_path_prefix; + unsigned int timeout_secs; + struct istream *input; + struct ostream *output; + struct io *io; + struct timeout *to; + + unsigned int minor_version; + struct dsync_serializer *serializers[ITEM_END_OF_LIST]; + struct dsync_deserializer *deserializers[ITEM_END_OF_LIST]; + + pool_t ret_pool; + struct dsync_deserializer_decoder *cur_decoder; + + struct istream *value_output, *value_input; + struct dsync_mail *cur_mail; + struct dsync_mailbox_attribute *cur_attr; + char value_output_last; + + enum item_type last_recv_item, last_sent_item; + bool last_recv_item_eol:1; + bool last_sent_item_eol:1; + + bool version_received:1; + bool handshake_received:1; + bool has_pending_data:1; + bool finish_received:1; + bool done_received:1; + bool stopped:1; +}; + +static const char *dsync_ibc_stream_get_state(struct dsync_ibc_stream *ibc) +{ + if (!ibc->version_received) + return "version not received"; + else if (!ibc->handshake_received) + return "handshake not received"; + + return t_strdup_printf("last sent=%s%s, last recv=%s%s", + items[ibc->last_sent_item].name, + ibc->last_sent_item_eol ? " (EOL)" : "", + items[ibc->last_recv_item].name, + ibc->last_recv_item_eol ? " (EOL)" : ""); +} + +static void dsync_ibc_stream_stop(struct dsync_ibc_stream *ibc) +{ + ibc->stopped = TRUE; + i_stream_close(ibc->input); + o_stream_close(ibc->output); + io_loop_stop(current_ioloop); +} + +static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc) +{ + do { + i_stream_skip(ibc->value_input, + i_stream_get_data_size(ibc->value_input)); + } while (i_stream_read(ibc->value_input) > 0); + if (ibc->value_input->eof) { + if (ibc->value_input->stream_errno != 0) { + i_error("dsync(%s): read(%s) failed: %s (%s)", ibc->name, + i_stream_get_name(ibc->value_input), + i_stream_get_error(ibc->value_input), + dsync_ibc_stream_get_state(ibc)); + dsync_ibc_stream_stop(ibc); + return -1; + } + /* finished reading the mail stream */ + i_assert(ibc->value_input->eof); + i_stream_seek(ibc->value_input, 0); + ibc->has_pending_data = TRUE; + ibc->value_input = NULL; + return 1; + } + return 0; +} + +static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc) +{ + timeout_reset(ibc->to); + if (ibc->value_input != NULL) { + if (dsync_ibc_stream_read_mail_stream(ibc) == 0) + return; + } + o_stream_cork(ibc->output); + ibc->ibc.io_callback(ibc->ibc.io_context); + o_stream_uncork(ibc->output); +} + +static int dsync_ibc_stream_send_value_stream(struct dsync_ibc_stream *ibc) +{ + const unsigned char *data; + unsigned char add; + size_t i, size; + int ret; + + while ((ret = i_stream_read_more(ibc->value_output, &data, &size)) > 0) { + add = '\0'; + for (i = 0; i < size; i++) { + if (data[i] == '.' && + ((i == 0 && ibc->value_output_last == '\n') || + (i > 0 && data[i-1] == '\n'))) { + /* escape the dot */ + add = '.'; + break; + } + } + + if (i > 0) { + o_stream_nsend(ibc->output, data, i); + ibc->value_output_last = data[i-1]; + i_stream_skip(ibc->value_output, i); + } + + if (o_stream_get_buffer_used_size(ibc->output) >= 4096) { + if ((ret = o_stream_flush(ibc->output)) < 0) { + dsync_ibc_stream_stop(ibc); + return -1; + } + if (ret == 0) { + /* continue later */ + o_stream_set_flush_pending(ibc->output, TRUE); + return 0; + } + } + + if (add != '\0') { + o_stream_nsend(ibc->output, &add, 1); + ibc->value_output_last = add; + } + } + i_assert(ret == -1); + + if (ibc->value_output->stream_errno != 0) { + i_error("dsync(%s): read(%s) failed: %s (%s)", + ibc->name, i_stream_get_name(ibc->value_output), + i_stream_get_error(ibc->value_output), + dsync_ibc_stream_get_state(ibc)); + dsync_ibc_stream_stop(ibc); + return -1; + } + + /* finished sending the stream. use "CRLF." instead of "LF." just in + case we're sending binary data that ends with CR. */ + o_stream_nsend_str(ibc->output, "\r\n.\r\n"); + i_stream_unref(&ibc->value_output); + return 1; +} + +static int dsync_ibc_stream_output(struct dsync_ibc_stream *ibc) +{ + struct ostream *output = ibc->output; + int ret; + + if ((ret = o_stream_flush(output)) < 0) + ret = 1; + else if (ibc->value_output != NULL) { + if (dsync_ibc_stream_send_value_stream(ibc) < 0) + ret = 1; + } + timeout_reset(ibc->to); + + if (!dsync_ibc_is_send_queue_full(&ibc->ibc)) + ibc->ibc.io_callback(ibc->ibc.io_context); + return ret; +} + +static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc) +{ + i_error("dsync(%s): I/O has stalled, no activity for %u seconds (%s)", + ibc->name, ibc->timeout_secs, dsync_ibc_stream_get_state(ibc)); + ibc->ibc.timeout = TRUE; + dsync_ibc_stream_stop(ibc); +} + +static void dsync_ibc_stream_init(struct dsync_ibc_stream *ibc) +{ + unsigned int i; + + ibc->io = io_add_istream(ibc->input, dsync_ibc_stream_input, ibc); + io_set_pending(ibc->io); + o_stream_set_no_error_handling(ibc->output, TRUE); + o_stream_set_flush_callback(ibc->output, dsync_ibc_stream_output, ibc); + ibc->to = timeout_add(ibc->timeout_secs * 1000, + dsync_ibc_stream_timeout, ibc); + o_stream_cork(ibc->output); + o_stream_nsend_str(ibc->output, DSYNC_HANDSHAKE_VERSION); + + /* initialize serializers and send their headers to remote */ + for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) T_BEGIN { + const char *keys; + + keys = items[i].required_keys == NULL ? items[i].optional_keys : + t_strconcat(items[i].required_keys, " ", + items[i].optional_keys, NULL); + if (keys != NULL) { + i_assert(items[i].chr != '\0'); + + ibc->serializers[i] = + dsync_serializer_init(t_strsplit_spaces(keys, " ")); + o_stream_nsend(ibc->output, &items[i].chr, 1); + o_stream_nsend_str(ibc->output, + dsync_serializer_encode_header_line(ibc->serializers[i])); + } + } T_END; + o_stream_nsend_str(ibc->output, ".\n"); + o_stream_uncork(ibc->output); +} + +static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + unsigned int i; + + for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) { + if (ibc->serializers[i] != NULL) + dsync_serializer_deinit(&ibc->serializers[i]); + if (ibc->deserializers[i] != NULL) + dsync_deserializer_deinit(&ibc->deserializers[i]); + } + if (ibc->cur_decoder != NULL) + dsync_deserializer_decode_finish(&ibc->cur_decoder); + if (ibc->value_output != NULL) + i_stream_unref(&ibc->value_output); + else { + /* If the remote has not told us that they are closing we + notify remote that we're closing. this is mainly to avoid + "read() failed: EOF" errors on failing dsyncs. + + Avoid a race condition: + We do not tell the remote we are closing if they have + already told us because they close the + connection after sending ITEM_DONE and will + not be ever receive anything else from us unless + it just happens to get combined into the same packet + as a previous response and is already in the buffer. + */ + if (!ibc->done_received && !ibc->finish_received) { + o_stream_nsend_str(ibc->output, + t_strdup_printf("%c\n", items[ITEM_DONE].chr)); + } + (void)o_stream_finish(ibc->output); + } + + timeout_remove(&ibc->to); + io_remove(&ibc->io); + i_stream_destroy(&ibc->input); + o_stream_destroy(&ibc->output); + pool_unref(&ibc->ret_pool); + i_free(ibc->temp_path_prefix); + i_free(ibc->name); + i_free(ibc); +} + +static int dsync_ibc_stream_next_line(struct dsync_ibc_stream *ibc, + const char **line_r) +{ + string_t *error; + const char *line; + ssize_t ret; + + line = i_stream_next_line(ibc->input); + if (line != NULL) { + *line_r = line; + return 1; + } + /* try reading some */ + if ((ret = i_stream_read(ibc->input)) == -1) { + if (ibc->stopped) + return -1; + error = t_str_new(128); + if (ibc->input->stream_errno != 0) { + str_printfa(error, "read(%s) failed: %s", ibc->name, + i_stream_get_error(ibc->input)); + } else { + i_assert(ibc->input->eof); + str_printfa(error, "read(%s) failed: EOF", ibc->name); + } + str_printfa(error, " (%s)", dsync_ibc_stream_get_state(ibc)); + i_error("%s", str_c(error)); + dsync_ibc_stream_stop(ibc); + return -1; + } + i_assert(ret >= 0); + *line_r = i_stream_next_line(ibc->input); + if (*line_r == NULL) { + ibc->has_pending_data = FALSE; + return 0; + } + ibc->has_pending_data = TRUE; + return 1; +} + +static void ATTR_FORMAT(3, 4) ATTR_NULL(2) +dsync_ibc_input_error(struct dsync_ibc_stream *ibc, + struct dsync_deserializer_decoder *decoder, + const char *fmt, ...) +{ + va_list args; + const char *error; + + va_start(args, fmt); + error = t_strdup_vprintf(fmt, args); + if (decoder == NULL) + i_error("dsync(%s): %s", ibc->name, error); + else { + i_error("dsync(%s): %s: %s", ibc->name, + dsync_deserializer_decoder_get_name(decoder), error); + } + va_end(args); + + dsync_ibc_stream_stop(ibc); +} + +static void +dsync_ibc_stream_send_string(struct dsync_ibc_stream *ibc, + const string_t *str) +{ + i_assert(ibc->value_output == NULL); + o_stream_nsend(ibc->output, str_data(str), str_len(str)); +} + +static int seekable_fd_callback(const char **path_r, void *context) +{ + struct dsync_ibc_stream *ibc = context; + string_t *path; + int fd; + + path = t_str_new(128); + str_append(path, ibc->temp_path_prefix); + fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1); + if (fd == -1) { + i_error("safe_mkstemp(%s) failed: %m", str_c(path)); + return -1; + } + + /* we just want the fd, unlink it */ + if (i_unlink(str_c(path)) < 0) { + /* shouldn't happen.. */ + i_close_fd(&fd); + return -1; + } + + *path_r = str_c(path); + return fd; +} + +static struct istream * +dsync_ibc_stream_input_stream(struct dsync_ibc_stream *ibc) +{ + struct istream *inputs[2]; + + inputs[0] = i_stream_create_dot(ibc->input, FALSE); + inputs[1] = NULL; + ibc->value_input = i_stream_create_seekable(inputs, MAIL_READ_FULL_BLOCK_SIZE, + seekable_fd_callback, ibc); + i_stream_unref(&inputs[0]); + + return ibc->value_input; +} + +static int +dsync_ibc_check_missing_deserializers(struct dsync_ibc_stream *ibc) +{ + unsigned int i; + int ret = 0; + + for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) { + if (ibc->deserializers[i] == NULL && + ibc->minor_version >= items[i].min_minor_version && + (items[i].required_keys != NULL || + items[i].optional_keys != NULL)) { + dsync_ibc_input_error(ibc, NULL, + "Remote didn't handshake deserializer for %s", + items[i].name); + ret = -1; + } + } + return ret; +} + +static bool +dsync_ibc_stream_handshake(struct dsync_ibc_stream *ibc, const char *line) +{ + enum item_type item = ITEM_NONE; + const char *const *required_keys, *error; + unsigned int i; + + if (ibc->handshake_received) + return TRUE; + + if (!ibc->version_received) { + if (!version_string_verify_full(line, "dsync", + DSYNC_PROTOCOL_VERSION_MAJOR, + &ibc->minor_version)) { + dsync_ibc_input_error(ibc, NULL, + "Remote dsync doesn't use compatible protocol"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + ibc->version_received = TRUE; + return FALSE; + } + + if (strcmp(line, END_OF_LIST_LINE) == 0) { + /* finished handshaking */ + if (dsync_ibc_check_missing_deserializers(ibc) < 0) + return FALSE; + ibc->handshake_received = TRUE; + ibc->last_recv_item = ITEM_HANDSHAKE; + return FALSE; + } + + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (items[i].chr == line[0]) { + item = i; + break; + } + } + if (item == ITEM_NONE) { + /* unknown deserializer, ignore */ + return FALSE; + } + + required_keys = items[item].required_keys == NULL ? NULL : + t_strsplit(items[item].required_keys, " "); + if (dsync_deserializer_init(items[item].name, + required_keys, line + 1, + &ibc->deserializers[item], &error) < 0) { + dsync_ibc_input_error(ibc, NULL, + "Remote sent invalid handshake for %s: %s", + items[item].name, error); + } + return FALSE; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_input_next(struct dsync_ibc_stream *ibc, enum item_type item, + struct dsync_deserializer_decoder **decoder_r) +{ + enum item_type line_item = ITEM_NONE; + const char *line, *error; + unsigned int i; + + i_assert(ibc->value_input == NULL); + + timeout_reset(ibc->to); + + do { + if (dsync_ibc_stream_next_line(ibc, &line) <= 0) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } while (!dsync_ibc_stream_handshake(ibc, line)); + + ibc->last_recv_item = item; + ibc->last_recv_item_eol = FALSE; + + if (strcmp(line, END_OF_LIST_LINE) == 0) { + /* end of this list */ + ibc->last_recv_item_eol = TRUE; + return DSYNC_IBC_RECV_RET_FINISHED; + } + if (line[0] == items[ITEM_DONE].chr) { + /* remote cleanly closed the connection, possibly because of + some failure (which it should have logged). we don't want to + log any stream errors anyway after this. */ + ibc->done_received = TRUE; + dsync_ibc_stream_stop(ibc); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + + } + for (i = 1; i < ITEM_END_OF_LIST; i++) { + if (*line == items[i].chr) { + line_item = i; + break; + } + } + if (line_item != item) { + dsync_ibc_input_error(ibc, NULL, + "Received unexpected input %c != %c", + *line, items[item].chr); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (ibc->cur_decoder != NULL) + dsync_deserializer_decode_finish(&ibc->cur_decoder); + if (dsync_deserializer_decode_begin(ibc->deserializers[item], + line+1, &ibc->cur_decoder, + &error) < 0) { + dsync_ibc_input_error(ibc, NULL, "Invalid input to %s: %s", + items[item].name, error); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *decoder_r = ibc->cur_decoder; + return DSYNC_IBC_RECV_RET_OK; +} + +static struct dsync_serializer_encoder * +dsync_ibc_send_encode_begin(struct dsync_ibc_stream *ibc, enum item_type item) +{ + ibc->last_sent_item = item; + ibc->last_sent_item_eol = FALSE; + return dsync_serializer_encode_begin(ibc->serializers[item]); +} + +static void +dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc, + const struct dsync_ibc_settings *set) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char sync_type[2]; + + str_append_c(str, items[ITEM_HANDSHAKE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_HANDSHAKE); + dsync_serializer_encode_add(encoder, "hostname", set->hostname); + if (set->sync_ns_prefixes != NULL) { + dsync_serializer_encode_add(encoder, "sync_ns_prefix", + set->sync_ns_prefixes); + } + if (set->sync_box != NULL) + dsync_serializer_encode_add(encoder, "sync_box", set->sync_box); + if (set->virtual_all_box != NULL) { + dsync_serializer_encode_add(encoder, "virtual_all_box", + set->virtual_all_box); + } + if (set->exclude_mailboxes != NULL) { + string_t *substr = t_str_new(64); + unsigned int i; + + for (i = 0; set->exclude_mailboxes[i] != NULL; i++) { + if (i != 0) + str_append_c(substr, '\t'); + str_append_tabescaped(substr, set->exclude_mailboxes[i]); + } + dsync_serializer_encode_add(encoder, "exclude_mailboxes", + str_c(substr)); + } + if (!guid_128_is_empty(set->sync_box_guid)) { + dsync_serializer_encode_add(encoder, "sync_box_guid", + guid_128_to_string(set->sync_box_guid)); + } + + sync_type[0] = sync_type[1] = '\0'; + switch (set->sync_type) { + case DSYNC_BRAIN_SYNC_TYPE_UNKNOWN: + break; + case DSYNC_BRAIN_SYNC_TYPE_FULL: + sync_type[0] = 'f'; + break; + case DSYNC_BRAIN_SYNC_TYPE_CHANGED: + sync_type[0] = 'c'; + break; + case DSYNC_BRAIN_SYNC_TYPE_STATE: + sync_type[0] = 's'; + break; + } + if (sync_type[0] != '\0') + dsync_serializer_encode_add(encoder, "sync_type", sync_type); + if (set->lock_timeout > 0) { + dsync_serializer_encode_add(encoder, "lock_timeout", + t_strdup_printf("%u", set->lock_timeout)); + } + if (set->import_commit_msgs_interval > 0) { + dsync_serializer_encode_add(encoder, "import_commit_msgs_interval", + t_strdup_printf("%u", set->import_commit_msgs_interval)); + } + if (set->sync_since_timestamp > 0) { + dsync_serializer_encode_add(encoder, "sync_since_timestamp", + t_strdup_printf("%ld", (long)set->sync_since_timestamp)); + } + if (set->sync_until_timestamp > 0) { + dsync_serializer_encode_add(encoder, "sync_until_timestamp", + t_strdup_printf("%ld", (long)set->sync_since_timestamp)); + } + if (set->sync_max_size > 0) { + dsync_serializer_encode_add(encoder, "sync_max_size", + t_strdup_printf("%"PRIu64, set->sync_max_size)); + } + if (set->sync_flags != NULL) { + dsync_serializer_encode_add(encoder, "sync_flags", + set->sync_flags); + } + if (set->alt_char != '\0') { + dsync_serializer_encode_add(encoder, "alt_char", + t_strdup_printf("%c", set->alt_char)); + } + if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0) + dsync_serializer_encode_add(encoder, "send_mail_requests", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0) + dsync_serializer_encode_add(encoder, "backup_send", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0) + dsync_serializer_encode_add(encoder, "backup_recv", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_DEBUG) != 0) + dsync_serializer_encode_add(encoder, "debug", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0) + dsync_serializer_encode_add(encoder, "sync_visible_namespaces", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0) + dsync_serializer_encode_add(encoder, "no_mail_sync", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0) + dsync_serializer_encode_add(encoder, "no_backup_overwrite", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0) + dsync_serializer_encode_add(encoder, "purge_remote", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0) + dsync_serializer_encode_add(encoder, "no_notify", ""); + if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0) + dsync_serializer_encode_add(encoder, "empty_hdr_workaround", ""); + /* this can be NULL in slave */ + string_t *str2 = t_str_new(32); + if (set->hashed_headers != NULL) { + for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) { + str_append_tabescaped(str2, *ptr); + str_append_c(str2, '\t'); + } + } + dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2)); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc, + const struct dsync_ibc_settings **set_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + struct dsync_ibc_settings *set; + const char *value; + pool_t pool = ibc->ret_pool; + enum dsync_ibc_recv_ret ret; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_HANDSHAKE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) { + if (ret != DSYNC_IBC_RECV_RET_TRYAGAIN) { + i_error("dsync(%s): Unexpected input in handshake", + ibc->name); + dsync_ibc_stream_stop(ibc); + } + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + p_clear(pool); + set = p_new(pool, struct dsync_ibc_settings, 1); + + value = dsync_deserializer_decode_get(decoder, "hostname"); + set->hostname = p_strdup(pool, value); + /* now that we know the remote's hostname, use it for the + stream's name */ + i_free(ibc->name); + ibc->name = i_strdup(set->hostname); + + if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value)) + set->sync_ns_prefixes = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "sync_box", &value)) + set->sync_box = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "virtual_all_box", &value)) + set->virtual_all_box = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "sync_box_guid", &value) && + guid_128_from_string(value, set->sync_box_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_box_guid: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "exclude_mailboxes", &value) && + *value != '\0') { + char **boxes = p_strsplit_tabescaped(pool, value); + set->exclude_mailboxes = (const void *)boxes; + } + if (dsync_deserializer_decode_try(decoder, "sync_type", &value)) { + switch (value[0]) { + case 'f': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL; + break; + case 'c': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED; + break; + case 's': + set->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE; + break; + default: + dsync_ibc_input_error(ibc, decoder, + "Unknown sync_type: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "lock_timeout", &value)) { + if (str_to_uint(value, &set->lock_timeout) < 0 || + set->lock_timeout == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid lock_timeout: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "import_commit_msgs_interval", &value)) { + if (str_to_uint(value, &set->import_commit_msgs_interval) < 0 || + set->import_commit_msgs_interval == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid import_commit_msgs_interval: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) { + if (str_to_time(value, &set->sync_since_timestamp) < 0 || + set->sync_since_timestamp == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_since_timestamp: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_until_timestamp", &value)) { + if (str_to_time(value, &set->sync_until_timestamp) < 0 || + set->sync_until_timestamp == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_until_timestamp: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) { + if (str_to_uoff(value, &set->sync_max_size) < 0 || + set->sync_max_size == 0) { + dsync_ibc_input_error(ibc, decoder, + "Invalid sync_max_size: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + if (dsync_deserializer_decode_try(decoder, "sync_flags", &value)) + set->sync_flags = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "alt_char", &value)) + set->alt_char = value[0]; + if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS; + if (dsync_deserializer_decode_try(decoder, "backup_send", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND; + if (dsync_deserializer_decode_try(decoder, "backup_recv", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV; + if (dsync_deserializer_decode_try(decoder, "debug", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_DEBUG; + if (dsync_deserializer_decode_try(decoder, "sync_visible_namespaces", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES; + if (dsync_deserializer_decode_try(decoder, "no_mail_sync", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC; + if (dsync_deserializer_decode_try(decoder, "no_backup_overwrite", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE; + if (dsync_deserializer_decode_try(decoder, "purge_remote", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE; + if (dsync_deserializer_decode_try(decoder, "no_notify", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY; + if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value)) + set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND; + if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value)) + set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value); + set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2; + set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3; + + *set_r = set; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_end_of_list(struct dsync_ibc *_ibc, + enum dsync_ibc_eol_type type) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + i_assert(ibc->value_output == NULL); + + switch (type) { + case DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE: + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) + return; + break; + default: + break; + } + + ibc->last_sent_item_eol = TRUE; + o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n"); +} + +static void +dsync_ibc_stream_send_mailbox_state(struct dsync_ibc *_ibc, + const struct dsync_mailbox_state *state) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_MAILBOX_STATE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_STATE); + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(state->mailbox_guid)); + dsync_serializer_encode_add(encoder, "last_uidvalidity", + dec2str(state->last_uidvalidity)); + dsync_serializer_encode_add(encoder, "last_common_uid", + dec2str(state->last_common_uid)); + dsync_serializer_encode_add(encoder, "last_common_modseq", + dec2str(state->last_common_modseq)); + dsync_serializer_encode_add(encoder, "last_common_pvt_modseq", + dec2str(state->last_common_pvt_modseq)); + dsync_serializer_encode_add(encoder, "last_messages_count", + dec2str(state->last_messages_count)); + if (state->changes_during_sync) + dsync_serializer_encode_add(encoder, "changes_during_sync", ""); + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_state(struct dsync_ibc *_ibc, + struct dsync_mailbox_state *state_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + const char *value; + enum dsync_ibc_recv_ret ret; + + i_zero(state_r); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_STATE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "mailbox_guid"); + if (guid_128_from_string(value, state_r->mailbox_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_uidvalidity"); + if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_uidvalidity"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_uid"); + if (str_to_uint32(value, &state_r->last_common_uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_modseq"); + if (str_to_uint64(value, &state_r->last_common_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "last_common_pvt_modseq"); + if (str_to_uint64(value, &state_r->last_common_pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_messages_count", &value) && + str_to_uint32(value, &state_r->last_messages_count) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "changes_during_sync", &value)) + state_r->changes_during_sync = TRUE; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mailbox_tree_node(struct dsync_ibc *_ibc, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str, *namestr; + + i_assert(*name != NULL); + + str = t_str_new(128); + str_append_c(str, items[ITEM_MAILBOX_TREE_NODE].chr); + + /* convert all hierarchy separators to tabs. mailbox names really + aren't supposed to have any tabs, but escape them anyway if there + are. */ + namestr = t_str_new(128); + for (; *name != NULL; name++) { + str_append_tabescaped(namestr, *name); + str_append_c(namestr, '\t'); + } + str_truncate(namestr, str_len(namestr)-1); + + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_TREE_NODE); + dsync_serializer_encode_add(encoder, "name", str_c(namestr)); + switch (node->existence) { + case DSYNC_MAILBOX_NODE_NONEXISTENT: + dsync_serializer_encode_add(encoder, "existence", "n"); + break; + case DSYNC_MAILBOX_NODE_EXISTS: + dsync_serializer_encode_add(encoder, "existence", "y"); + break; + case DSYNC_MAILBOX_NODE_DELETED: + dsync_serializer_encode_add(encoder, "existence", "d"); + break; + } + + if (!guid_128_is_empty(node->mailbox_guid)) { + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(node->mailbox_guid)); + } + if (node->uid_validity != 0) { + dsync_serializer_encode_add(encoder, "uid_validity", + dec2str(node->uid_validity)); + } + if (node->uid_next != 0) { + dsync_serializer_encode_add(encoder, "uid_next", + dec2str(node->uid_next)); + } + if (node->last_renamed_or_created != 0) { + dsync_serializer_encode_add(encoder, "last_renamed_or_created", + dec2str(node->last_renamed_or_created)); + } + if (node->last_subscription_change != 0) { + dsync_serializer_encode_add(encoder, "last_subscription_change", + dec2str(node->last_subscription_change)); + } + if (node->subscribed) + dsync_serializer_encode_add(encoder, "subscribed", ""); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_tree_node(struct dsync_ibc *_ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox_node *node; + const char *value; + enum dsync_ibc_recv_ret ret; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_TREE_NODE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + p_clear(ibc->ret_pool); + node = p_new(ibc->ret_pool, struct dsync_mailbox_node, 1); + + value = dsync_deserializer_decode_get(decoder, "name"); + if (*value == '\0') { + dsync_ibc_input_error(ibc, decoder, "Empty name"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *name_r = (void *)p_strsplit_tabescaped(ibc->ret_pool, value); + + value = dsync_deserializer_decode_get(decoder, "existence"); + switch (*value) { + case 'n': + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + break; + case 'y': + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + break; + case 'd': + node->existence = DSYNC_MAILBOX_NODE_DELETED; + break; + } + + if (dsync_deserializer_decode_try(decoder, "mailbox_guid", &value) && + guid_128_from_string(value, node->mailbox_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "uid_validity", &value) && + str_to_uint32(value, &node->uid_validity) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "uid_next", &value) && + str_to_uint32(value, &node->uid_next) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_next"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_renamed_or_created", &value) && + str_to_time(value, &node->last_renamed_or_created) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_renamed_or_created"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "last_subscription_change", &value) && + str_to_time(value, &node->last_subscription_change) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_subscription_change"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "subscribed", &value)) + node->subscribed = TRUE; + + *node_r = node; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_encode_delete(string_t *str, + struct dsync_serializer_encoder *encoder, + const struct dsync_mailbox_delete *deletes, + unsigned int count, const char *key, + enum dsync_mailbox_delete_type type) +{ + unsigned int i; + + str_truncate(str, 0); + for (i = 0; i < count; i++) { + if (deletes[i].type == type) { + str_append(str, guid_128_to_string(deletes[i].guid)); + str_printfa(str, " %ld ", (long)deletes[i].timestamp); + } + } + if (str_len(str) > 0) { + str_truncate(str, str_len(str)-1); + dsync_serializer_encode_add(encoder, key, str_c(str)); + } +} + +static void +dsync_ibc_stream_send_mailbox_deletes(struct dsync_ibc *_ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str, *substr; + char sep[2]; + + str = t_str_new(128); + str_append_c(str, items[ITEM_MAILBOX_DELETE].chr); + + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_DELETE); + sep[0] = hierarchy_sep; sep[1] = '\0'; + dsync_serializer_encode_add(encoder, "hierarchy_sep", sep); + sep[0] = escape_char; sep[1] = '\0'; + dsync_serializer_encode_add(encoder, "escape_char", sep); + + substr = t_str_new(128); + dsync_ibc_stream_encode_delete(substr, encoder, deletes, count, + "mailboxes", + DSYNC_MAILBOX_DELETE_TYPE_MAILBOX); + dsync_ibc_stream_encode_delete(substr, encoder, deletes, count, + "dirs", + DSYNC_MAILBOX_DELETE_TYPE_DIR); + dsync_ibc_stream_encode_delete(substr, encoder, deletes, count, + "unsubscribes", + DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +ARRAY_DEFINE_TYPE(dsync_mailbox_delete, struct dsync_mailbox_delete); +static int +decode_mailbox_deletes(ARRAY_TYPE(dsync_mailbox_delete) *deletes, + const char *value, enum dsync_mailbox_delete_type type) +{ + struct dsync_mailbox_delete *del; + const char *const *tmp; + unsigned int i; + + tmp = t_strsplit(value, " "); + for (i = 0; tmp[i] != NULL; i += 2) { + del = array_append_space(deletes); + del->type = type; + if (guid_128_from_string(tmp[i], del->guid) < 0) + return -1; + if (tmp[i+1] == NULL || + str_to_time(tmp[i+1], &del->timestamp) < 0) + return -1; + } + return 0; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_deletes(struct dsync_ibc *_ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r, + char *escape_char_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + ARRAY_TYPE(dsync_mailbox_delete) deletes; + const char *value; + enum dsync_ibc_recv_ret ret; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_DELETE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + p_clear(ibc->ret_pool); + p_array_init(&deletes, ibc->ret_pool, 16); + + value = dsync_deserializer_decode_get(decoder, "hierarchy_sep"); + if (strlen(value) != 1) { + dsync_ibc_input_error(ibc, decoder, "Invalid hierarchy_sep"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *hierarchy_sep_r = value[0]; + + if (!dsync_deserializer_decode_try(decoder, "escape_char", &value)) + *escape_char_r = '\0'; + else { + if (strlen(value) > 1) { + dsync_ibc_input_error(ibc, decoder, "Invalid escape_char '%s'", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *escape_char_r = value[0]; + } + + if (dsync_deserializer_decode_try(decoder, "mailboxes", &value) && + decode_mailbox_deletes(&deletes, value, + DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailboxes"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "dirs", &value) && + decode_mailbox_deletes(&deletes, value, + DSYNC_MAILBOX_DELETE_TYPE_DIR) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid dirs"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "unsubscribes", &value) && + decode_mailbox_deletes(&deletes, value, + DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid dirs"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + *deletes_r = array_get(&deletes, count_r); + return DSYNC_IBC_RECV_RET_OK; +} + +static const char * +get_cache_fields(struct dsync_ibc_stream *ibc, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_serializer_encoder *encoder; + string_t *str; + const struct mailbox_cache_field *cache_fields; + unsigned int i, count; + char decision[3]; + + cache_fields = array_get(&dsync_box->cache_fields, &count); + if (count == 0) + return ""; + + str = t_str_new(128); + for (i = 0; i < count; i++) { + const struct mailbox_cache_field *field = &cache_fields[i]; + + encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_CACHE_FIELD]); + dsync_serializer_encode_add(encoder, "name", field->name); + + memset(decision, 0, sizeof(decision)); + switch (field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) { + case MAIL_CACHE_DECISION_NO: + decision[0] = 'n'; + break; + case MAIL_CACHE_DECISION_TEMP: + decision[0] = 't'; + break; + case MAIL_CACHE_DECISION_YES: + decision[0] = 'y'; + break; + } + i_assert(decision[0] != '\0'); + if ((field->decision & MAIL_CACHE_DECISION_FORCED) != 0) + decision[1] = 'F'; + dsync_serializer_encode_add(encoder, "decision", decision); + if (field->last_used != 0) { + dsync_serializer_encode_add(encoder, "last_used", + dec2str(field->last_used)); + } + dsync_serializer_encode_finish(&encoder, str); + } + if (i > 0) { + /* remove the trailing LF */ + str_truncate(str, str_len(str)-1); + } + return str_c(str); +} + +static void +dsync_ibc_stream_send_mailbox(struct dsync_ibc *_ibc, + const struct dsync_mailbox *dsync_box) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + const char *value; + + str_append_c(str, items[ITEM_MAILBOX].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX); + dsync_serializer_encode_add(encoder, "mailbox_guid", + guid_128_to_string(dsync_box->mailbox_guid)); + + if (dsync_box->mailbox_lost) + dsync_serializer_encode_add(encoder, "mailbox_lost", ""); + if (dsync_box->mailbox_ignore) + dsync_serializer_encode_add(encoder, "mailbox_ignore", ""); + if (dsync_box->have_guids) + dsync_serializer_encode_add(encoder, "have_guids", ""); + if (dsync_box->have_save_guids) + dsync_serializer_encode_add(encoder, "have_save_guids", ""); + if (dsync_box->have_only_guid128) + dsync_serializer_encode_add(encoder, "have_only_guid128", ""); + dsync_serializer_encode_add(encoder, "uid_validity", + dec2str(dsync_box->uid_validity)); + dsync_serializer_encode_add(encoder, "uid_next", + dec2str(dsync_box->uid_next)); + dsync_serializer_encode_add(encoder, "messages_count", + dec2str(dsync_box->messages_count)); + dsync_serializer_encode_add(encoder, "first_recent_uid", + dec2str(dsync_box->first_recent_uid)); + dsync_serializer_encode_add(encoder, "highest_modseq", + dec2str(dsync_box->highest_modseq)); + dsync_serializer_encode_add(encoder, "highest_pvt_modseq", + dec2str(dsync_box->highest_pvt_modseq)); + + value = get_cache_fields(ibc, dsync_box); + if (value != NULL) + dsync_serializer_encode_add(encoder, "cache_fields", value); + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static int +parse_cache_field(struct dsync_ibc_stream *ibc, struct dsync_mailbox *box, + const char *value) +{ + struct dsync_deserializer_decoder *decoder; + struct mailbox_cache_field field; + const char *error; + int ret = 0; + + if (dsync_deserializer_decode_begin(ibc->deserializers[ITEM_MAILBOX_CACHE_FIELD], + value, &decoder, &error) < 0) { + dsync_ibc_input_error(ibc, NULL, + "cache_field: Invalid input: %s", error); + return -1; + } + + i_zero(&field); + value = dsync_deserializer_decode_get(decoder, "name"); + field.name = p_strdup(ibc->ret_pool, value); + + value = dsync_deserializer_decode_get(decoder, "decision"); + switch (*value) { + case 'n': + field.decision = MAIL_CACHE_DECISION_NO; + break; + case 't': + field.decision = MAIL_CACHE_DECISION_TEMP; + break; + case 'y': + field.decision = MAIL_CACHE_DECISION_YES; + break; + default: + dsync_ibc_input_error(ibc, decoder, "Invalid decision: %s", + value); + ret = -1; + break; + } + if (value[1] == 'F') + field.decision |= MAIL_CACHE_DECISION_FORCED; + + if (dsync_deserializer_decode_try(decoder, "last_used", &value) && + str_to_time(value, &field.last_used) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_used"); + ret = -1; + } + array_push_back(&box->cache_fields, &field); + + dsync_deserializer_decode_finish(&decoder); + return ret; +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox(struct dsync_ibc *_ibc, + const struct dsync_mailbox **dsync_box_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox *box; + const char *value; + enum dsync_ibc_recv_ret ret; + + p_clear(pool); + box = p_new(pool, struct dsync_mailbox, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "mailbox_guid"); + if (guid_128_from_string(value, box->mailbox_guid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "mailbox_lost", &value)) + box->mailbox_lost = TRUE; + if (dsync_deserializer_decode_try(decoder, "mailbox_ignore", &value)) + box->mailbox_ignore = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_guids", &value)) + box->have_guids = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_save_guids", &value) || + (box->have_guids && ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID)) + box->have_save_guids = TRUE; + if (dsync_deserializer_decode_try(decoder, "have_only_guid128", &value)) + box->have_only_guid128 = TRUE; + value = dsync_deserializer_decode_get(decoder, "uid_validity"); + if (str_to_uint32(value, &box->uid_validity) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "uid_next"); + if (str_to_uint32(value, &box->uid_next) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid_next"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "messages_count"); + if (str_to_uint32(value, &box->messages_count) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid messages_count"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "first_recent_uid"); + if (str_to_uint32(value, &box->first_recent_uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid first_recent_uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "highest_modseq"); + if (str_to_uint64(value, &box->highest_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid highest_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + value = dsync_deserializer_decode_get(decoder, "highest_pvt_modseq"); + if (str_to_uint64(value, &box->highest_pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid highest_pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + p_array_init(&box->cache_fields, pool, 32); + if (dsync_deserializer_decode_try(decoder, "cache_fields", &value)) { + const char *const *fields = t_strsplit(value, "\n"); + for (; *fields != NULL; fields++) { + if (parse_cache_field(ibc, box, *fields) < 0) + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + } + + *dsync_box_r = box; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mailbox_attribute(struct dsync_ibc *_ibc, + const struct dsync_mailbox_attribute *attr) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char type[2]; + + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) + return; + + str_append_c(str, items[ITEM_MAILBOX_ATTRIBUTE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_ATTRIBUTE); + + type[0] = type[1] = '\0'; + switch (attr->type) { + case MAIL_ATTRIBUTE_TYPE_PRIVATE: + type[0] = 'p'; + break; + case MAIL_ATTRIBUTE_TYPE_SHARED: + type[0] = 's'; + break; + } + i_assert(type[0] != '\0'); + dsync_serializer_encode_add(encoder, "type", type); + dsync_serializer_encode_add(encoder, "key", attr->key); + if (attr->value != NULL) + dsync_serializer_encode_add(encoder, "value", attr->value); + else if (attr->value_stream != NULL) + dsync_serializer_encode_add(encoder, "stream", ""); + + if (attr->deleted) + dsync_serializer_encode_add(encoder, "deleted", ""); + if (attr->last_change != 0) { + dsync_serializer_encode_add(encoder, "last_change", + dec2str(attr->last_change)); + } + if (attr->modseq != 0) { + dsync_serializer_encode_add(encoder, "modseq", + dec2str(attr->modseq)); + } + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); + + if (attr->value_stream != NULL) { + ibc->value_output_last = '\0'; + ibc->value_output = attr->value_stream; + i_stream_ref(ibc->value_output); + (void)dsync_ibc_stream_send_value_stream(ibc); + } +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mailbox_attribute(struct dsync_ibc *_ibc, + const struct dsync_mailbox_attribute **attr_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mailbox_attribute *attr; + const char *value; + enum dsync_ibc_recv_ret ret; + + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES) + return DSYNC_IBC_RECV_RET_FINISHED; + + if (ibc->value_input != NULL) { + /* wait until the mail's stream has been read */ + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (ibc->cur_attr != NULL) { + /* finished reading the stream, return the mail now */ + *attr_r = ibc->cur_attr; + ibc->cur_attr = NULL; + return DSYNC_IBC_RECV_RET_OK; + } + + p_clear(pool); + attr = p_new(pool, struct dsync_mailbox_attribute, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_ATTRIBUTE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "type"); + switch (*value) { + case 'p': + attr->type = MAIL_ATTRIBUTE_TYPE_PRIVATE; + break; + case 's': + attr->type = MAIL_ATTRIBUTE_TYPE_SHARED; + break; + default: + dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + value = dsync_deserializer_decode_get(decoder, "key"); + attr->key = p_strdup(pool, value); + + if (dsync_deserializer_decode_try(decoder, "deleted", &value)) + attr->deleted = TRUE; + if (dsync_deserializer_decode_try(decoder, "last_change", &value) && + str_to_time(value, &attr->last_change) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid last_change"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "modseq", &value) && + str_to_uint64(value, &attr->modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + /* NOTE: stream reading must be the last here, because reading a large + stream will be finished later by return TRYAGAIN. We need to + deserialize all the other fields before that or they'll get lost. */ + if (dsync_deserializer_decode_try(decoder, "stream", &value)) { + attr->value_stream = dsync_ibc_stream_input_stream(ibc); + if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { + ibc->cur_attr = attr; + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + /* already finished reading the stream */ + i_assert(ibc->value_input == NULL); + } else if (dsync_deserializer_decode_try(decoder, "value", &value)) + attr->value = p_strdup(pool, value); + + *attr_r = attr; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_change(struct dsync_ibc *_ibc, + const struct dsync_mail_change *change) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + char type[2]; + + str_append_c(str, items[ITEM_MAIL_CHANGE].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_CHANGE); + + type[0] = type[1] = '\0'; + switch (change->type) { + case DSYNC_MAIL_CHANGE_TYPE_SAVE: + type[0] = 's'; + break; + case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE: + type[0] = 'e'; + break; + case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE: + type[0] = 'f'; + break; + } + i_assert(type[0] != '\0'); + dsync_serializer_encode_add(encoder, "type", type); + dsync_serializer_encode_add(encoder, "uid", dec2str(change->uid)); + if (change->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", change->guid); + if (change->hdr_hash != NULL) { + dsync_serializer_encode_add(encoder, "hdr_hash", + change->hdr_hash); + } + if (change->modseq != 0) { + dsync_serializer_encode_add(encoder, "modseq", + dec2str(change->modseq)); + } + if (change->pvt_modseq != 0) { + dsync_serializer_encode_add(encoder, "pvt_modseq", + dec2str(change->pvt_modseq)); + } + if (change->add_flags != 0) { + dsync_serializer_encode_add(encoder, "add_flags", + t_strdup_printf("%x", change->add_flags)); + } + if (change->remove_flags != 0) { + dsync_serializer_encode_add(encoder, "remove_flags", + t_strdup_printf("%x", change->remove_flags)); + } + if (change->final_flags != 0) { + dsync_serializer_encode_add(encoder, "final_flags", + t_strdup_printf("%x", change->final_flags)); + } + if (change->keywords_reset) + dsync_serializer_encode_add(encoder, "keywords_reset", ""); + + if (array_is_created(&change->keyword_changes) && + array_count(&change->keyword_changes) > 0) { + string_t *kw_str = t_str_new(128); + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + str_append_tabescaped(kw_str, changes[0]); + for (i = 1; i < count; i++) { + str_append_c(kw_str, '\t'); + str_append_tabescaped(kw_str, changes[i]); + } + dsync_serializer_encode_add(encoder, "keyword_changes", + str_c(kw_str)); + } + if (change->received_timestamp > 0) { + dsync_serializer_encode_add(encoder, "received_timestamp", + t_strdup_printf("%"PRIxTIME_T, change->received_timestamp)); + } + if (change->virtual_size > 0) { + dsync_serializer_encode_add(encoder, "virtual_size", + t_strdup_printf("%llx", (unsigned long long)change->virtual_size)); + } + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc, + const struct dsync_mail_change **change_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail_change *change; + const char *value; + unsigned int uintval; + unsigned long long ullongval; + enum dsync_ibc_recv_ret ret; + + p_clear(pool); + change = p_new(pool, struct dsync_mail_change, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_CHANGE, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + value = dsync_deserializer_decode_get(decoder, "type"); + switch (*value) { + case 's': + change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE; + break; + case 'e': + change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE; + break; + case 'f': + change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE; + break; + default: + dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + value = dsync_deserializer_decode_get(decoder, "uid"); + if (str_to_uint32(value, &change->uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + change->guid = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "hdr_hash", &value)) + change->hdr_hash = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "modseq", &value) && + str_to_uint64(value, &change->modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "pvt_modseq", &value) && + str_to_uint64(value, &change->pvt_modseq) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid pvt_modseq"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid add_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->add_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid remove_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->remove_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) { + if (str_to_uint_hex(value, &uintval) < 0 || + uintval > (uint8_t)-1) { + dsync_ibc_input_error(ibc, decoder, + "Invalid final_flags: %s", value); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->final_flags = uintval; + } + if (dsync_deserializer_decode_try(decoder, "keywords_reset", &value)) + change->keywords_reset = TRUE; + + if (dsync_deserializer_decode_try(decoder, "keyword_changes", &value) && + *value != '\0') { + const char *const *changes = t_strsplit_tabescaped(value); + unsigned int i, count = str_array_length(changes); + + p_array_init(&change->keyword_changes, pool, count); + for (i = 0; i < count; i++) { + value = p_strdup(pool, changes[i]); + array_push_back(&change->keyword_changes, &value); + } + } + if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value)) { + if (str_to_ullong_hex(value, &ullongval) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->received_timestamp = ullongval; + } + if (dsync_deserializer_decode_try(decoder, "virtual_size", &value)) { + if (str_to_ullong_hex(value, &ullongval) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + change->virtual_size = ullongval; + } + + *change_r = change; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mail_request(struct dsync_ibc *_ibc, + const struct dsync_mail_request *request) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_MAIL_REQUEST].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_REQUEST); + if (request->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", request->guid); + if (request->uid != 0) { + dsync_serializer_encode_add(encoder, "uid", + dec2str(request->uid)); + } + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mail_request(struct dsync_ibc *_ibc, + const struct dsync_mail_request **request_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail_request *request; + const char *value; + enum dsync_ibc_recv_ret ret; + + p_clear(ibc->ret_pool); + request = p_new(ibc->ret_pool, struct dsync_mail_request, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_REQUEST, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + request->guid = p_strdup(ibc->ret_pool, value); + if (dsync_deserializer_decode_try(decoder, "uid", &value) && + str_to_uint32(value, &request->uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + + *request_r = request; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_mail(struct dsync_ibc *_ibc, + const struct dsync_mail *mail) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + i_assert(!mail->minimal_fields); + i_assert(ibc->value_output == NULL); + + str_append_c(str, items[ITEM_MAIL].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL); + if (mail->guid != NULL) + dsync_serializer_encode_add(encoder, "guid", mail->guid); + if (mail->uid != 0) + dsync_serializer_encode_add(encoder, "uid", dec2str(mail->uid)); + if (mail->pop3_uidl != NULL) { + dsync_serializer_encode_add(encoder, "pop3_uidl", + mail->pop3_uidl); + } + if (mail->pop3_order > 0) { + dsync_serializer_encode_add(encoder, "pop3_order", + dec2str(mail->pop3_order)); + } + if (mail->received_date > 0) { + dsync_serializer_encode_add(encoder, "received_date", + dec2str(mail->received_date)); + } + if (mail->saved_date != 0) { + dsync_serializer_encode_add(encoder, "saved_date", + dec2str(mail->saved_date)); + } + if (mail->input != NULL) + dsync_serializer_encode_add(encoder, "stream", ""); + + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); + + if (mail->input != NULL) { + ibc->value_output_last = '\0'; + ibc->value_output = mail->input; + i_stream_ref(ibc->value_output); + (void)dsync_ibc_stream_send_value_stream(ibc); + } +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + pool_t pool = ibc->ret_pool; + struct dsync_deserializer_decoder *decoder; + struct dsync_mail *mail; + const char *value; + enum dsync_ibc_recv_ret ret; + + if (ibc->value_input != NULL) { + /* wait until the mail's stream has been read */ + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (ibc->cur_mail != NULL) { + /* finished reading the stream, return the mail now */ + *mail_r = ibc->cur_mail; + ibc->cur_mail = NULL; + return DSYNC_IBC_RECV_RET_OK; + } + + p_clear(pool); + mail = p_new(pool, struct dsync_mail, 1); + + ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "guid", &value)) + mail->guid = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "uid", &value) && + str_to_uint32(value, &mail->uid) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid uid"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "pop3_uidl", &value)) + mail->pop3_uidl = p_strdup(pool, value); + if (dsync_deserializer_decode_try(decoder, "pop3_order", &value) && + str_to_uint32(value, &mail->pop3_order) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid pop3_order"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "received_date", &value) && + str_to_time(value, &mail->received_date) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid received_date"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "saved_date", &value) && + str_to_time(value, &mail->saved_date) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid saved_date"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "stream", &value)) { + mail->input = dsync_ibc_stream_input_stream(ibc); + if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) { + ibc->cur_mail = mail; + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + /* already finished reading the stream */ + i_assert(ibc->value_input == NULL); + } + + *mail_r = mail; + return DSYNC_IBC_RECV_RET_OK; +} + +static void +dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_serializer_encoder *encoder; + string_t *str = t_str_new(128); + + str_append_c(str, items[ITEM_FINISH].chr); + encoder = dsync_ibc_send_encode_begin(ibc, ITEM_FINISH); + if (error != NULL) + dsync_serializer_encode_add(encoder, "error", error); + if (mail_error != 0) { + dsync_serializer_encode_add(encoder, "mail_error", + dec2str(mail_error)); + } + if (require_full_resync) + dsync_serializer_encode_add(encoder, "require_full_resync", ""); + dsync_serializer_encode_finish(&encoder, str); + dsync_ibc_stream_send_string(ibc, str); +} + +static enum dsync_ibc_recv_ret +dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + struct dsync_deserializer_decoder *decoder; + const char *value; + enum dsync_ibc_recv_ret ret; + int i = 0; + + *error_r = NULL; + *mail_error_r = 0; + *require_full_resync_r = FALSE; + + p_clear(ibc->ret_pool); + + if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_FINISH) + return DSYNC_IBC_RECV_RET_OK; + + ret = dsync_ibc_stream_input_next(ibc, ITEM_FINISH, &decoder); + if (ret != DSYNC_IBC_RECV_RET_OK) + return ret; + + if (dsync_deserializer_decode_try(decoder, "error", &value)) + *error_r = p_strdup(ibc->ret_pool, value); + if (dsync_deserializer_decode_try(decoder, "mail_error", &value) && + str_to_int(value, &i) < 0) { + dsync_ibc_input_error(ibc, decoder, "Invalid mail_error"); + return DSYNC_IBC_RECV_RET_TRYAGAIN; + } + if (dsync_deserializer_decode_try(decoder, "require_full_resync", &value)) + *require_full_resync_r = TRUE; + *mail_error_r = i; + + ibc->finish_received = TRUE; + return DSYNC_IBC_RECV_RET_OK; +} + +static void dsync_ibc_stream_close_mail_streams(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + if (ibc->value_output != NULL) { + i_stream_unref(&ibc->value_output); + dsync_ibc_stream_stop(ibc); + } +} + +static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + size_t bytes; + + if (ibc->value_output != NULL) + return TRUE; + + bytes = o_stream_get_buffer_used_size(ibc->output); + if (bytes < DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE) + return FALSE; + + o_stream_set_flush_pending(ibc->output, TRUE); + return TRUE; +} + +static bool dsync_ibc_stream_has_pending_data(struct dsync_ibc *_ibc) +{ + struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc; + + return ibc->has_pending_data; +} + +static const struct dsync_ibc_vfuncs dsync_ibc_stream_vfuncs = { + dsync_ibc_stream_deinit, + dsync_ibc_stream_send_handshake, + dsync_ibc_stream_recv_handshake, + dsync_ibc_stream_send_end_of_list, + dsync_ibc_stream_send_mailbox_state, + dsync_ibc_stream_recv_mailbox_state, + dsync_ibc_stream_send_mailbox_tree_node, + dsync_ibc_stream_recv_mailbox_tree_node, + dsync_ibc_stream_send_mailbox_deletes, + dsync_ibc_stream_recv_mailbox_deletes, + dsync_ibc_stream_send_mailbox, + dsync_ibc_stream_recv_mailbox, + dsync_ibc_stream_send_mailbox_attribute, + dsync_ibc_stream_recv_mailbox_attribute, + dsync_ibc_stream_send_change, + dsync_ibc_stream_recv_change, + dsync_ibc_stream_send_mail_request, + dsync_ibc_stream_recv_mail_request, + dsync_ibc_stream_send_mail, + dsync_ibc_stream_recv_mail, + dsync_ibc_stream_send_finish, + dsync_ibc_stream_recv_finish, + dsync_ibc_stream_close_mail_streams, + dsync_ibc_stream_is_send_queue_full, + dsync_ibc_stream_has_pending_data +}; + +struct dsync_ibc * +dsync_ibc_init_stream(struct istream *input, struct ostream *output, + const char *name, const char *temp_path_prefix, + unsigned int timeout_secs) +{ + struct dsync_ibc_stream *ibc; + + ibc = i_new(struct dsync_ibc_stream, 1); + ibc->ibc.v = dsync_ibc_stream_vfuncs; + ibc->input = input; + ibc->output = output; + i_stream_ref(ibc->input); + o_stream_ref(ibc->output); + ibc->name = i_strdup(name); + ibc->temp_path_prefix = i_strdup(temp_path_prefix); + ibc->timeout_secs = timeout_secs; + ibc->ret_pool = pool_alloconly_create("ibc stream data", 2048); + dsync_ibc_stream_init(ibc); + return &ibc->ibc; +} diff --git a/src/doveadm/dsync/dsync-ibc.c b/src/doveadm/dsync/dsync-ibc.c new file mode 100644 index 0000000..ede7b83 --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc.c @@ -0,0 +1,239 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "dsync-mail.h" +#include "dsync-ibc-private.h" + +void dsync_ibc_deinit(struct dsync_ibc **_ibc) +{ + struct dsync_ibc *ibc = *_ibc; + + *_ibc = NULL; + ibc->v.deinit(ibc); +} + +void dsync_ibc_set_io_callback(struct dsync_ibc *ibc, + io_callback_t *callback, void *context) +{ + ibc->io_callback = callback; + ibc->io_context = context; +} + +void dsync_ibc_send_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set) +{ + ibc->v.send_handshake(ibc, set); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r) +{ + return ibc->v.recv_handshake(ibc, set_r); +} + +static enum dsync_ibc_send_ret +dsync_ibc_send_ret(struct dsync_ibc *ibc) +{ + return ibc->v.is_send_queue_full(ibc) ? + DSYNC_IBC_SEND_RET_FULL : + DSYNC_IBC_SEND_RET_OK; +} + +enum dsync_ibc_send_ret +dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type) +{ + ibc->v.send_end_of_list(ibc, type); + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state) +{ + T_BEGIN { + ibc->v.send_mailbox_state(ibc, state); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r) +{ + return ibc->v.recv_mailbox_state(ibc, state_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node) +{ + i_assert(*name != NULL); + + T_BEGIN { + ibc->v.send_mailbox_tree_node(ibc, name, node); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r) +{ + return ibc->v.recv_mailbox_tree_node(ibc, name_r, node_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char) +{ + T_BEGIN { + ibc->v.send_mailbox_deletes(ibc, deletes, count, + hierarchy_sep, escape_char); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r, + char *escape_char_r) +{ + return ibc->v.recv_mailbox_deletes(ibc, deletes_r, count_r, + hierarchy_sep_r, escape_char_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box) +{ + T_BEGIN { + ibc->v.send_mailbox(ibc, dsync_box); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r) +{ + return ibc->v.recv_mailbox(ibc, dsync_box_r); +} + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr) +{ + T_BEGIN { + ibc->v.send_mailbox_attribute(ibc, attr); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r) +{ + return ibc->v.recv_mailbox_attribute(ibc, attr_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_change(struct dsync_ibc *ibc, + const struct dsync_mail_change *change) +{ + i_assert(change->uid > 0); + + T_BEGIN { + ibc->v.send_change(ibc, change); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_change(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r) +{ + return ibc->v.recv_change(ibc, change_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request *request) +{ + i_assert(request->guid != NULL || request->uid != 0); + + T_BEGIN { + ibc->v.send_mail_request(ibc, request); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r) +{ + return ibc->v.recv_mail_request(ibc, request_r); +} + +enum dsync_ibc_send_ret +dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail) +{ + i_assert(*mail->guid != '\0' || mail->uid != 0); + + T_BEGIN { + ibc->v.send_mail(ibc, mail); + } T_END; + return dsync_ibc_send_ret(ibc); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r) +{ + return ibc->v.recv_mail(ibc, mail_r); +} + +void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync) +{ + ibc->v.send_finish(ibc, error, mail_error, require_full_resync); +} + +enum dsync_ibc_recv_ret +dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r) +{ + return ibc->v.recv_finish(ibc, error_r, mail_error_r, + require_full_resync_r); +} + +void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc) +{ + ibc->v.close_mail_streams(ibc); +} + +bool dsync_ibc_has_failed(struct dsync_ibc *ibc) +{ + return ibc->failed; +} + +bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc) +{ + return ibc->timeout; +} + +bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc) +{ + return ibc->v.is_send_queue_full(ibc); +} + +bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc) +{ + return ibc->v.has_pending_data(ibc); +} diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h new file mode 100644 index 0000000..dc85560 --- /dev/null +++ b/src/doveadm/dsync/dsync-ibc.h @@ -0,0 +1,178 @@ +#ifndef DSYNC_IBC_H +#define DSYNC_IBC_H + +/* dsync inter-brain communicator */ + +#include "ioloop.h" +#include "guid.h" +#include "mail-error.h" +#include "dsync-brain.h" + +struct dsync_mailbox; +struct dsync_mailbox_state; +struct dsync_mailbox_node; +struct dsync_mailbox_delete; +struct dsync_mailbox_attribute; +struct dsync_mail; +struct dsync_mail_change; +struct dsync_mail_request; + +enum dsync_ibc_send_ret { + DSYNC_IBC_SEND_RET_OK = 1, + /* send queue is full, stop sending more */ + DSYNC_IBC_SEND_RET_FULL = 0 +}; + +enum dsync_ibc_recv_ret { + DSYNC_IBC_RECV_RET_FINISHED = -1, + /* try again / error (the error handling delayed until io callback) */ + DSYNC_IBC_RECV_RET_TRYAGAIN = 0, + DSYNC_IBC_RECV_RET_OK = 1 +}; + +enum dsync_ibc_eol_type { + DSYNC_IBC_EOL_MAILBOX_STATE, + DSYNC_IBC_EOL_MAILBOX_TREE, + DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE, + DSYNC_IBC_EOL_MAILBOX, + DSYNC_IBC_EOL_MAIL_CHANGES, + DSYNC_IBC_EOL_MAIL_REQUESTS, + DSYNC_IBC_EOL_MAILS +}; + +struct dsync_ibc_settings { + /* Server hostname. Used for determining which server does the + locking. */ + const char *hostname; + /* if non-NULL, sync only these namespaces (LF-separated) */ + const char *sync_ns_prefixes; + /* if non-NULL, sync only this mailbox name */ + const char *sync_box; + /* if non-NULL, use this mailbox for finding messages with GUIDs and + copying them instead of saving them again. */ + const char *virtual_all_box; + /* if non-empty, sync only this mailbox GUID */ + guid_128_t sync_box_guid; + /* Exclude these mailboxes from the sync. They can contain '*' + wildcards and be \special-use flags. */ + const char *const *exclude_mailboxes; + /* Sync only mails with received timestamp at least this high. */ + time_t sync_since_timestamp; + /* Sync only mails with received timestamp less or equal than this */ + time_t sync_until_timestamp; + /* Don't sync mails larger than this. */ + uoff_t sync_max_size; + /* Sync only mails with specified flags. */ + const char *sync_flags; + /* Hashed headers */ + const char *const *hashed_headers; + + char alt_char; + enum dsync_brain_sync_type sync_type; + enum dsync_brain_flags brain_flags; + bool hdr_hash_v2; + bool hdr_hash_v3; + unsigned int lock_timeout; + unsigned int import_commit_msgs_interval; +}; + +void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r, + struct dsync_ibc **ibc2_r); +struct dsync_ibc * +dsync_ibc_init_stream(struct istream *input, struct ostream *output, + const char *name, const char *temp_path_prefix, + unsigned int timeout_secs); +void dsync_ibc_deinit(struct dsync_ibc **ibc); + +/* I/O callback is called whenever new data is available. It's also called on + errors, so check first the error status. */ +void dsync_ibc_set_io_callback(struct dsync_ibc *ibc, + io_callback_t *callback, void *context); + +void dsync_ibc_send_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings *set); +enum dsync_ibc_recv_ret +dsync_ibc_recv_handshake(struct dsync_ibc *ibc, + const struct dsync_ibc_settings **set_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc, + const struct dsync_mailbox_state *state); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc, + struct dsync_mailbox_state *state_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const *name, + const struct dsync_mailbox_node *node); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc, + const char *const **name_r, + const struct dsync_mailbox_node **node_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete *deletes, + unsigned int count, char hierarchy_sep, + char escape_char); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc, + const struct dsync_mailbox_delete **deletes_r, + unsigned int *count_r, char *hierarchy_sep_r, + char *escape_char_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox *dsync_box); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox(struct dsync_ibc *ibc, + const struct dsync_mailbox **dsync_box_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute *attr); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc, + const struct dsync_mailbox_attribute **attr_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_change(struct dsync_ibc *ibc, + const struct dsync_mail_change *change); +enum dsync_ibc_recv_ret +dsync_ibc_recv_change(struct dsync_ibc *ibc, + const struct dsync_mail_change **change_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request *request); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail_request(struct dsync_ibc *ibc, + const struct dsync_mail_request **request_r); + +enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT +dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail); +enum dsync_ibc_recv_ret +dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r); + +void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error, + enum mail_error mail_error, + bool require_full_resync); +enum dsync_ibc_recv_ret +dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r, + enum mail_error *mail_error_r, + bool *require_full_resync_r); + +/* Close any mail input streams that are kept open. This needs to be called + before the mail is attempted to be freed (usually on error conditions). */ +void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc); + +bool dsync_ibc_has_failed(struct dsync_ibc *ibc); +bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc); +bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc); +bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc); + +#endif diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c new file mode 100644 index 0000000..b82818f --- /dev/null +++ b/src/doveadm/dsync/dsync-mail.c @@ -0,0 +1,156 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hex-binary.h" +#include "md5.h" +#include "istream.h" +#include "istream-crlf.h" +#include "message-header-hash.h" +#include "message-size.h" +#include "mail-storage.h" +#include "dsync-mail.h" + +struct mailbox_header_lookup_ctx * +dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers) +{ + return mailbox_header_lookup_init(box, hashed_headers); +} + +int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, + const char *const *hashed_headers, const char **hdr_hash_r) +{ + struct istream *hdr_input, *input; + struct mailbox_header_lookup_ctx *hdr_ctx; + struct message_header_hash_context hash_ctx; + struct md5_context md5_ctx; + unsigned char md5_result[MD5_RESULTLEN]; + const unsigned char *data; + size_t size; + ssize_t sret; + int ret = 0; + + hdr_ctx = mailbox_header_lookup_init(mail->box, hashed_headers); + ret = mail_get_header_stream(mail, hdr_ctx, &hdr_input); + mailbox_header_lookup_unref(&hdr_ctx); + if (ret < 0) + return -1; + + input = i_stream_create_lf(hdr_input); + + md5_init(&md5_ctx); + i_zero(&hash_ctx); + while ((sret = i_stream_read_more(input, &data, &size)) > 0) { + message_header_hash_more(&hash_ctx, &hash_method_md5, &md5_ctx, + version, data, size); + i_stream_skip(input, size); + } + i_assert(sret == -1); + if (input->stream_errno != 0) + ret = -1; + i_stream_unref(&input); + + md5_final(&md5_ctx, md5_result); + *hdr_hash_r = binary_to_hex(md5_result, sizeof(md5_result)); + return ret; +} + +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r) +{ + const char *guid; + + i_zero(dmail_r); + + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) { + *error_field_r = "GUID"; + return -1; + } + dmail_r->guid = guid; + dmail_r->uid = mail->uid; + + dmail_r->input_mail = mail; + dmail_r->input_mail_uid = mail->uid; + + if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) { + *error_field_r = "saved-date"; + return -1; + } + if (!minimal_fill) + return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r); + dmail_r->minimal_fields = TRUE; + return 0; +} + +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r) +{ + const char *str; + + if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) { + *error_field_r = "body"; + return -1; + } + + if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail_r->pop3_uidl) < 0) { + *error_field_r = "pop3-uidl"; + return -1; + } + if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) { + *error_field_r = "pop3-order"; + return -1; + } + if (*str != '\0') { + if (str_to_uint32(str, &dmail_r->pop3_order) < 0) + i_unreached(); + } + if (mail_get_received_date(mail, &dmail_r->received_date) < 0) { + *error_field_r = "received-date"; + return -1; + } + return 0; +} + +static void +const_string_array_dup(pool_t pool, const ARRAY_TYPE(const_string) *src, + ARRAY_TYPE(const_string) *dest) +{ + const char *const *strings, *str; + unsigned int i, count; + + if (!array_is_created(src)) + return; + + strings = array_get(src, &count); + if (count == 0) + return; + + p_array_init(dest, pool, count); + for (i = 0; i < count; i++) { + str = p_strdup(pool, strings[i]); + array_push_back(dest, &str); + } +} + +void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, + struct dsync_mail_change *dest_r) +{ + dest_r->type = src->type; + dest_r->uid = src->uid; + if (src->guid != NULL) { + dest_r->guid = *src->guid == '\0' ? "" : + p_strdup(pool, src->guid); + } + dest_r->hdr_hash = p_strdup(pool, src->hdr_hash); + dest_r->modseq = src->modseq; + dest_r->pvt_modseq = src->pvt_modseq; + + dest_r->add_flags = src->add_flags; + dest_r->remove_flags = src->remove_flags; + dest_r->final_flags = src->final_flags; + dest_r->keywords_reset = src->keywords_reset; + const_string_array_dup(pool, &src->keyword_changes, + &dest_r->keyword_changes); + dest_r->received_timestamp = src->received_timestamp; + dest_r->virtual_size = src->virtual_size; +} diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h new file mode 100644 index 0000000..fc00059 --- /dev/null +++ b/src/doveadm/dsync/dsync-mail.h @@ -0,0 +1,108 @@ +#ifndef DSYNC_MAIL_H +#define DSYNC_MAIL_H + +#include "mail-types.h" + +struct md5_context; +struct mail; +struct mailbox; + +struct dsync_mail { + /* either GUID="" or uid=0 */ + const char *guid; + uint32_t uid; + time_t saved_date; + + /* If non-NULL, we're syncing within the dsync process using ibc-pipe. + This mail can be used to mailbox_copy() the mail. */ + struct mail *input_mail; + /* Verify that this equals to input_mail->uid */ + uint32_t input_mail_uid; + + /* TRUE if the following fields aren't set, because minimal_fill=TRUE + parameter was used. */ + bool minimal_fields; + + const char *pop3_uidl; + uint32_t pop3_order; + time_t received_date; + /* Input stream containing the message text, or NULL if all instances + of the message were already expunged from this mailbox. */ + struct istream *input; +}; + +struct dsync_mail_request { + /* either GUID=NULL or uid=0 */ + const char *guid; + uint32_t uid; +}; + +enum dsync_mail_change_type { + DSYNC_MAIL_CHANGE_TYPE_SAVE, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE +}; + +#define KEYWORD_CHANGE_ADD '+' +#define KEYWORD_CHANGE_REMOVE '-' +#define KEYWORD_CHANGE_FINAL '=' +#define KEYWORD_CHANGE_ADD_AND_FINAL '&' + +struct dsync_mail_change { + enum dsync_mail_change_type type; + + uint32_t uid; + /* Message's GUID: + - for expunges either 128bit hex or NULL if unknown + - "" if backend doesn't support GUIDs */ + const char *guid; + /* If GUID is "", this contains hash of the message header, + otherwise NULL */ + const char *hdr_hash; + + /* Message's current modseq (saves, flag changes) */ + uint64_t modseq; + /* Message's current private modseq (for private flags in + shared mailboxes, otherwise 0) */ + uint64_t pvt_modseq; + + /* List of flag/keyword changes: (saves, flag changes) */ + + /* Flags added/removed since last sync, and final flags containing + flags that exist now but haven't changed */ + uint8_t add_flags, remove_flags, final_flags; + uint8_t add_pvt_flags, remove_pvt_flags; + /* Remove all keywords before applying changes. This is used only with + old transaction logs, new ones never reset keywords (just explicitly + remove unwanted keywords) */ + bool keywords_reset; + /* +add, -remove, =final, &add_and_final. */ + ARRAY_TYPE(const_string) keyword_changes; + + /* Received timestamp for saves, if brain.sync_since/until_timestamp is set */ + time_t received_timestamp; + /* Mail's size for saves if brain.sync_max_size is set, + UOFF_T_MAX otherwise. */ + uoff_t virtual_size; +}; + +struct mailbox_header_lookup_ctx * +dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers); + +int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version, + const char *const *hashed_headers, const char **hdr_hash_r); +static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash) +{ + /* md5(\n) */ + return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0; +} + +int dsync_mail_fill(struct mail *mail, bool minimal_fill, + struct dsync_mail *dmail_r, const char **error_field_r); +int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r, + const char **error_field_r); + +void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src, + struct dsync_mail_change *dest_r); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c new file mode 100644 index 0000000..a58fbec --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-export.c @@ -0,0 +1,961 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "istream.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-export.h" + +struct dsync_mail_guid_instances { + ARRAY_TYPE(seq_range) seqs; + bool requested; + bool searched; +}; + +struct dsync_mailbox_exporter { + pool_t pool; + struct mailbox *box; + struct dsync_transaction_log_scan *log_scan; + uint32_t last_common_uid; + + struct mailbox_header_lookup_ctx *wanted_headers; + struct mailbox_transaction_context *trans; + struct mail_search_context *search_ctx; + unsigned int search_pos, search_count; + unsigned int hdr_hash_version; + + const char *const *hashed_headers; + + /* GUID => instances */ + HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids; + ARRAY_TYPE(seq_range) requested_uids; + ARRAY_TYPE(seq_range) search_uids; + + ARRAY_TYPE(seq_range) expunged_seqs; + ARRAY_TYPE(const_string) expunged_guids; + unsigned int expunged_guid_idx; + + /* uint32_t UID => struct dsync_mail_change */ + HASH_TABLE(void *, struct dsync_mail_change *) changes; + /* changes sorted by UID */ + ARRAY(struct dsync_mail_change *) sorted_changes; + unsigned int change_idx; + uint32_t highest_changed_uid; + + struct mailbox_attribute_iter *attr_iter; + struct hash_iterate_context *attr_change_iter; + enum mail_attribute_type attr_type; + struct dsync_mailbox_attribute attr; + + struct dsync_mail_change change; + struct dsync_mail dsync_mail; + + const char *error; + enum mail_error mail_error; + + bool body_search_initialized:1; + bool auto_export_mails:1; + bool mails_have_guids:1; + bool minimal_dmail_fill:1; + bool return_all_mails:1; + bool export_received_timestamps:1; + bool export_virtual_sizes:1; + bool no_hdr_hashes:1; +}; + +static int dsync_mail_error(struct dsync_mailbox_exporter *exporter, + struct mail *mail, const char *field) +{ + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_internal_error(exporter->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return 0; + + exporter->mail_error = error; + exporter->error = p_strdup_printf(exporter->pool, + "Can't lookup %s for UID=%u: %s", + field, mail->uid, errstr); + return -1; +} + +static bool +final_keyword_check(struct dsync_mail_change *change, const char *name, + char *type_r) +{ + const char *const *changes; + unsigned int i, count; + + *type_r = KEYWORD_CHANGE_FINAL; + + changes = array_get(&change->keyword_changes, &count); + for (i = 0; i < count; i++) { + if (strcmp(changes[i]+1, name) != 0) + continue; + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + /* replace with ADD_AND_FINAL */ + array_delete(&change->keyword_changes, i, 1); + *type_r = KEYWORD_CHANGE_ADD_AND_FINAL; + return FALSE; + case KEYWORD_CHANGE_REMOVE: + /* a final keyword is marked as removed. + this shouldn't normally happen. */ + array_delete(&change->keyword_changes, i, 1); + return FALSE; + case KEYWORD_CHANGE_ADD_AND_FINAL: + case KEYWORD_CHANGE_FINAL: + /* no change */ + return TRUE; + } + } + return FALSE; +} + +static void +search_update_flag_changes(struct dsync_mailbox_exporter *exporter, + struct mail *mail, struct dsync_mail_change *change) +{ + const char *const *keywords; + unsigned int i; + char type; + + i_assert((change->add_flags & change->remove_flags) == 0); + + change->modseq = mail_get_modseq(mail); + change->pvt_modseq = mail_get_pvt_modseq(mail); + change->final_flags = mail_get_flags(mail); + + keywords = mail_get_keywords(mail); + if (!array_is_created(&change->keyword_changes) && + keywords[0] != NULL) { + p_array_init(&change->keyword_changes, exporter->pool, + str_array_length(keywords)); + } + for (i = 0; keywords[i] != NULL; i++) { + /* add the final keyword if it's not already there + as +keyword */ + if (!final_keyword_check(change, keywords[i], &type)) { + const char *keyword_change = + p_strdup_printf(exporter->pool, "%c%s", + type, keywords[i]); + array_push_back(&change->keyword_changes, + &keyword_change); + } + } +} + +static int +exporter_get_guids(struct dsync_mailbox_exporter *exporter, + struct mail *mail, const char **guid_r, + const char **hdr_hash_r) +{ + *guid_r = ""; + *hdr_hash_r = NULL; + + /* always try to get GUID, even if we're also getting header hash */ + if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0) + return dsync_mail_error(exporter, mail, "GUID"); + + if (!exporter->mails_have_guids) { + /* get header hash also */ + if (exporter->no_hdr_hashes) { + *hdr_hash_r = ""; + return 1; + } + if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version, + exporter->hashed_headers, hdr_hash_r) < 0) + return dsync_mail_error(exporter, mail, "hdr-stream"); + return 1; + } else if (**guid_r == '\0') { + exporter->mail_error = MAIL_ERROR_TEMP; + exporter->error = "Backend doesn't support GUIDs, " + "sync with header hashes instead"; + return -1; + } else { + /* GUIDs are required, we don't need header hash */ + return 1; + } +} + +static int +search_update_flag_change_guid(struct dsync_mailbox_exporter *exporter, + struct mail *mail) +{ + struct dsync_mail_change *change, *log_change; + const char *guid, *hdr_hash; + int ret; + + change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid)); + if (change != NULL) { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE); + } else { + i_assert(exporter->return_all_mails); + + change = p_new(exporter->pool, struct dsync_mail_change, 1); + change->uid = mail->uid; + change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE; + hash_table_insert(exporter->changes, + POINTER_CAST(mail->uid), change); + } + + if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0) + return -1; + if (ret == 0) { + /* the message was expunged during export */ + i_zero(change); + change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE; + change->uid = mail->uid; + + /* find its GUID from log if possible */ + log_change = dsync_transaction_log_scan_find_new_expunge( + exporter->log_scan, mail->uid); + if (log_change != NULL) + change->guid = log_change->guid; + } else { + change->guid = *guid == '\0' ? "" : + p_strdup(exporter->pool, guid); + change->hdr_hash = p_strdup(exporter->pool, hdr_hash); + search_update_flag_changes(exporter, mail, change); + } + return 0; +} + +static struct dsync_mail_change * +export_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid) +{ + struct dsync_mail_change *change; + + change = hash_table_lookup(exporter->changes, POINTER_CAST(uid)); + if (change == NULL) { + change = p_new(exporter->pool, struct dsync_mail_change, 1); + change->uid = uid; + hash_table_insert(exporter->changes, POINTER_CAST(uid), change); + } else { + /* move flag changes into a save. this happens only when + last_common_uid isn't known */ + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE); + i_assert(exporter->last_common_uid == 0); + } + + change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE; + return change; +} + +static void +export_add_mail_instance(struct dsync_mailbox_exporter *exporter, + struct dsync_mail_change *change, uint32_t seq) +{ + struct dsync_mail_guid_instances *instances; + + if (exporter->auto_export_mails && !exporter->mails_have_guids) { + /* GUIDs not supported, mail is requested by UIDs */ + seq_range_array_add(&exporter->requested_uids, change->uid); + return; + } + if (*change->guid == '\0') { + /* mail UIDs are manually requested */ + i_assert(!exporter->mails_have_guids); + return; + } + + instances = hash_table_lookup(exporter->export_guids, change->guid); + if (instances == NULL) { + instances = p_new(exporter->pool, + struct dsync_mail_guid_instances, 1); + p_array_init(&instances->seqs, exporter->pool, 2); + hash_table_insert(exporter->export_guids, + p_strdup(exporter->pool, change->guid), + instances); + if (exporter->auto_export_mails) + instances->requested = TRUE; + } + seq_range_array_add(&instances->seqs, seq); +} + +static int +search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail) +{ + struct dsync_mail_change *change; + const char *guid, *hdr_hash; + enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID; + time_t received_timestamp = 0; + uoff_t virtual_size = UOFF_T_MAX; + int ret; + + /* update wanted fields in case we didn't already set them for the + search */ + if (exporter->export_received_timestamps) + wanted_fields |= MAIL_FETCH_RECEIVED_DATE; + if (exporter->export_virtual_sizes) + wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE; + mail_add_temp_wanted_fields(mail, wanted_fields, + exporter->wanted_headers); + + /* If message is already expunged here, just skip it */ + if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0) + return ret; + + if (exporter->export_received_timestamps) { + if (mail_get_received_date(mail, &received_timestamp) < 0) + return dsync_mail_error(exporter, mail, "received-time"); + if (received_timestamp == 0) { + /* don't allow timestamps to be zero. we want to have + asserts verify that the timestamp is set properly. */ + received_timestamp = 1; + } + } + if (exporter->export_virtual_sizes) { + if (mail_get_virtual_size(mail, &virtual_size) < 0) + return dsync_mail_error(exporter, mail, "virtual-size"); + i_assert(virtual_size != UOFF_T_MAX); + } + + change = export_save_change_get(exporter, mail->uid); + change->guid = *guid == '\0' ? "" : + p_strdup(exporter->pool, guid); + change->hdr_hash = p_strdup(exporter->pool, hdr_hash); + change->received_timestamp = received_timestamp; + change->virtual_size = virtual_size; + search_update_flag_changes(exporter, mail, change); + + export_add_mail_instance(exporter, change, mail->seq); + return 1; +} + +static void +dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter, + ARRAY_TYPE(seq_range) *uids) +{ + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change; + + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, exporter->changes, &key, &change)) { + if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE) + seq_range_array_add(uids, change->uid); + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter) +{ + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change; + + /* any flag changes for UIDs above last_common_uid weren't found by + mailbox search, which means they were already expunged. for some + reason the log scanner found flag changes for the message, but not + the expunge. just remove these. */ + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, exporter->changes, &key, &change)) { + if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE && + change->uid > exporter->last_common_uid) + hash_table_remove(exporter->changes, key); + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter) +{ + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + struct mail *mail; + enum mail_fetch_field wanted_fields = 0; + struct mailbox_header_lookup_ctx *wanted_headers = NULL; + int ret = 0; + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&sarg->value.seqset, search_args->pool, 1); + + if (exporter->return_all_mails || exporter->last_common_uid == 0) { + /* we want to know about all mails */ + seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1); + } else { + /* lookup GUIDs for messages with flag changes */ + dsync_mailbox_export_add_flagchange_uids(exporter, + &sarg->value.seqset); + /* lookup new messages */ + seq_range_array_add_range(&sarg->value.seqset, + exporter->last_common_uid + 1, + (uint32_t)-1); + } + + if (exporter->last_common_uid == 0) { + /* we're syncing all mails, so we can request the wanted + fields for all the mails */ + wanted_fields = MAIL_FETCH_GUID; + wanted_headers = exporter->wanted_headers; + } + + exporter->trans = mailbox_transaction_begin(exporter->box, + MAILBOX_TRANSACTION_FLAG_SYNC, + __func__); + search_ctx = mailbox_search_init(exporter->trans, search_args, NULL, + wanted_fields, wanted_headers); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + T_BEGIN { + if (mail->uid <= exporter->last_common_uid) + ret = search_update_flag_change_guid(exporter, mail); + else + ret = search_add_save(exporter, mail); + } T_END; + if (ret < 0) + break; + } + i_assert(ret >= 0 || exporter->error != NULL); + + dsync_mailbox_export_drop_expunged_flag_changes(exporter); + + if (mailbox_search_deinit(&search_ctx) < 0 && + exporter->error == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Mail search failed: %s", + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + } +} + +static int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1, + struct dsync_mail_change *const *c2) +{ + if ((*c1)->uid < (*c2)->uid) + return -1; + if ((*c1)->uid > (*c2)->uid) + return 1; + return 0; +} + +static void +dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter) +{ + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change; + + p_array_init(&exporter->sorted_changes, exporter->pool, + hash_table_count(exporter->changes)); + + iter = hash_table_iterate_init(exporter->changes); + while (hash_table_iterate(iter, exporter->changes, &key, &change)) + array_push_back(&exporter->sorted_changes, &change); + hash_table_iterate_deinit(&iter); + array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp); +} + +static void +dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter, + enum mail_attribute_type type) +{ + exporter->attr_iter = + mailbox_attribute_iter_init(exporter->box, type, ""); + exporter->attr_type = type; +} + +static void +dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter, + struct dsync_transaction_log_scan *log_scan) +{ + HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes; + struct hash_iterate_context *iter; + void *key; + struct dsync_mail_change *change, *dup_change; + + log_changes = dsync_transaction_log_scan_get_hash(log_scan); + if (dsync_transaction_log_scan_has_all_changes(log_scan)) { + /* we tried to access too old/invalid modseqs. to make sure + no changes get lost, we need to send all of the messages */ + exporter->return_all_mails = TRUE; + } + + /* clone the hash table, since we're changing it. */ + hash_table_create_direct(&exporter->changes, exporter->pool, + hash_table_count(log_changes)); + iter = hash_table_iterate_init(log_changes); + while (hash_table_iterate(iter, log_changes, &key, &change)) { + dup_change = p_new(exporter->pool, struct dsync_mail_change, 1); + *dup_change = *change; + hash_table_insert(exporter->changes, key, dup_change); + if (exporter->highest_changed_uid < change->uid) + exporter->highest_changed_uid = change->uid; + } + hash_table_iterate_deinit(&iter); +} + +struct dsync_mailbox_exporter * +dsync_mailbox_export_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + enum dsync_mailbox_exporter_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers) +{ + struct dsync_mailbox_exporter *exporter; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export", + 4096); + exporter = p_new(pool, struct dsync_mailbox_exporter, 1); + exporter->pool = pool; + exporter->box = box; + exporter->log_scan = log_scan; + exporter->last_common_uid = last_common_uid; + exporter->auto_export_mails = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0; + exporter->mails_have_guids = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0; + exporter->minimal_dmail_fill = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0; + exporter->export_received_timestamps = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0; + exporter->export_virtual_sizes = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0; + exporter->hdr_hash_version = hdr_hash_version; + exporter->no_hdr_hashes = + (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0; + exporter->hashed_headers = hashed_headers; + + p_array_init(&exporter->requested_uids, pool, 16); + p_array_init(&exporter->search_uids, pool, 16); + hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp); + p_array_init(&exporter->expunged_seqs, pool, 16); + p_array_init(&exporter->expunged_guids, pool, 16); + + if (!exporter->mails_have_guids && !exporter->no_hdr_hashes) + exporter->wanted_headers = + dsync_mail_get_hash_headers(box, exporter->hashed_headers); + + /* first scan transaction log and save any expunges and flag changes */ + dsync_mailbox_export_log_scan(exporter, log_scan); + /* get saves and also find GUIDs for flag changes */ + dsync_mailbox_export_search(exporter); + /* get the changes sorted by UID */ + dsync_mailbox_export_sort_changes(exporter); + + dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE); + return exporter; +} + +static int +dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter) +{ + struct dsync_mailbox_attribute *attr; + struct mail_attribute_value value; + + while (hash_table_iterate(exporter->attr_change_iter, + dsync_transaction_log_scan_get_attr_hash(exporter->log_scan), + &attr, &attr)) { + if (attr->exported || !attr->deleted) + continue; + + /* lookup the value mainly to get its last_change value. */ + if (mailbox_attribute_get_stream(exporter->box, attr->type, + attr->key, &value) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute %s lookup failed: %s", attr->key, + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + break; + } + if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { + i_stream_unref(&value.value_stream); + continue; + } + + attr->last_change = value.last_change; + if (value.value != NULL || value.value_stream != NULL) { + attr->value = p_strdup(exporter->pool, value.value); + attr->value_stream = value.value_stream; + attr->deleted = FALSE; + } + attr->exported = TRUE; + exporter->attr = *attr; + return 1; + } + hash_table_iterate_deinit(&exporter->attr_change_iter); + return 0; +} + +static int +dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter) +{ + HASH_TABLE_TYPE(dsync_attr_change) attr_changes; + struct dsync_mailbox_attribute lookup_attr, *attr; + struct dsync_mailbox_attribute *attr_change; + const char *key; + struct mail_attribute_value value; + bool export_all_attrs; + + export_all_attrs = exporter->return_all_mails || + exporter->last_common_uid == 0; + attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan); + lookup_attr.type = exporter->attr_type; + + /* note that the order of processing may be important for some + attributes. for example sieve can't set a script active until it's + first been created */ + while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) { + lookup_attr.key = key; + attr_change = hash_table_lookup(attr_changes, &lookup_attr); + if (attr_change == NULL && !export_all_attrs) + continue; + + if (mailbox_attribute_get_stream(exporter->box, + exporter->attr_type, key, + &value) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute %s lookup failed: %s", key, + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + return -1; + } + if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) { + /* readonly attributes can't be changed, + no point in exporting them */ + if (value.value_stream != NULL) + i_stream_unref(&value.value_stream); + continue; + } + if (value.value == NULL && value.value_stream == NULL && + (attr_change == NULL || !attr_change->deleted)) { + /* the attribute was just deleted? + skip for this sync. */ + continue; + } + if (attr_change != NULL && attr_change->exported) { + /* duplicate attribute returned. + shouldn't normally happen, but don't crash. */ + i_warning("Ignoring duplicate attributes '%s'", key); + continue; + } + + attr = &exporter->attr; + i_zero(attr); + attr->type = exporter->attr_type; + attr->value = p_strdup(exporter->pool, value.value); + attr->value_stream = value.value_stream; + attr->last_change = value.last_change; + if (attr_change != NULL) { + attr_change->exported = TRUE; + attr->key = attr_change->key; + attr->deleted = attr_change->deleted && + !DSYNC_ATTR_HAS_VALUE(attr); + attr->modseq = attr_change->modseq; + } else { + attr->key = p_strdup(exporter->pool, key); + } + return 1; + } + if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) { + exporter->error = p_strdup_printf(exporter->pool, + "Mailbox attribute iteration failed: %s", + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + return -1; + } + if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) { + /* export shared attributes */ + dsync_mailbox_export_attr_init(exporter, + MAIL_ATTRIBUTE_TYPE_SHARED); + return dsync_mailbox_export_iter_next_attr(exporter); + } + exporter->attr_change_iter = hash_table_iterate_init(attr_changes); + return dsync_mailbox_export_iter_next_nonexistent_attr(exporter); +} + +int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter, + const struct dsync_mailbox_attribute **attr_r) +{ + int ret; + + if (exporter->error != NULL) + return -1; + + i_stream_unref(&exporter->attr.value_stream); + + if (exporter->attr_iter != NULL) { + ret = dsync_mailbox_export_iter_next_attr(exporter); + } else { + ret = dsync_mailbox_export_iter_next_nonexistent_attr(exporter); + } + if (ret > 0) + *attr_r = &exporter->attr; + return ret; +} + +int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_change **change_r) +{ + struct dsync_mail_change *const *changes; + unsigned int count; + + if (exporter->error != NULL) + return -1; + + changes = array_get(&exporter->sorted_changes, &count); + if (exporter->change_idx == count) + return 0; + *change_r = changes[exporter->change_idx++]; + return 1; +} + +static int +dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter) +{ + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + struct hash_iterate_context *iter; + const struct seq_range *uids; + char *guid; + const char *const_guid; + enum mail_fetch_field wanted_fields; + struct dsync_mail_guid_instances *instances; + const struct seq_range *range; + unsigned int i, count; + uint32_t seq, seq1, seq2; + + i_assert(exporter->search_ctx == NULL); + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_SEQSET); + p_array_init(&sarg->value.seqset, search_args->pool, 128); + + /* get a list of messages we want to fetch. if there are more than one + instance for a GUID, use the first one. */ + iter = hash_table_iterate_init(exporter->export_guids); + while (hash_table_iterate(iter, exporter->export_guids, + &guid, &instances)) { + if (!instances->requested || + array_count(&instances->seqs) == 0) + continue; + + uids = array_front(&instances->seqs); + seq = uids[0].seq1; + if (!instances->searched) { + instances->searched = TRUE; + seq_range_array_add(&sarg->value.seqset, seq); + } else if (seq_range_exists(&exporter->expunged_seqs, seq)) { + /* we're on a second round, refetching expunged + messages */ + seq_range_array_remove(&instances->seqs, seq); + seq_range_array_remove(&exporter->expunged_seqs, seq); + if (array_count(&instances->seqs) == 0) { + /* no instances left */ + const_guid = guid; + array_push_back(&exporter->expunged_guids, + &const_guid); + continue; + } + uids = array_front(&instances->seqs); + seq = uids[0].seq1; + seq_range_array_add(&sarg->value.seqset, seq); + } + } + hash_table_iterate_deinit(&iter); + + /* add requested UIDs */ + range = array_get(&exporter->requested_uids, &count); + for (i = 0; i < count; i++) { + mailbox_get_seq_range(exporter->box, + range[i].seq1, range[i].seq2, + &seq1, &seq2); + seq_range_array_add_range(&sarg->value.seqset, + seq1, seq2); + } + array_clear(&exporter->search_uids); + array_append_array(&exporter->search_uids, &exporter->requested_uids); + array_clear(&exporter->requested_uids); + + wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE; + if (!exporter->minimal_dmail_fill) { + wanted_fields |= MAIL_FETCH_RECEIVED_DATE | + MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER | + MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY; + } + exporter->search_count += seq_range_count(&sarg->value.seqset); + exporter->search_ctx = + mailbox_search_init(exporter->trans, search_args, NULL, + wanted_fields, NULL); + mail_search_args_unref(&search_args); + return array_count(&sarg->value.seqset) > 0 ? 1 : 0; +} + +static void +dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter) +{ + if (exporter->search_ctx == NULL) + return; + + if (mailbox_search_deinit(&exporter->search_ctx) < 0 && + exporter->error == NULL) { + exporter->error = p_strdup_printf(exporter->pool, + "Mail search failed: %s", + mailbox_get_last_internal_error(exporter->box, + &exporter->mail_error)); + } +} + +static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter, + struct mail *mail) +{ + struct dsync_mail_guid_instances *instances; + const char *error_field; + + if (dsync_mail_fill(mail, exporter->minimal_dmail_fill, + &exporter->dsync_mail, &error_field) < 0) + return dsync_mail_error(exporter, mail, error_field); + + instances = *exporter->dsync_mail.guid == '\0' ? NULL : + hash_table_lookup(exporter->export_guids, + exporter->dsync_mail.guid); + if (instances != NULL) { + /* GUID found */ + } else if (exporter->dsync_mail.uid != 0) { + /* mail requested by UID */ + } else { + exporter->mail_error = MAIL_ERROR_TEMP; + exporter->error = p_strdup_printf(exporter->pool, + "GUID unexpectedly changed for UID=%u GUID=%s", + mail->uid, exporter->dsync_mail.guid); + return -1; + } + + if (!seq_range_exists(&exporter->search_uids, mail->uid)) + exporter->dsync_mail.uid = 0; + else + exporter->dsync_mail.guid = ""; + + /* this message was successfully returned, don't try retrying it */ + if (instances != NULL) + array_clear(&instances->seqs); + return 1; +} + +void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_request *request) +{ + struct dsync_mail_guid_instances *instances; + + i_assert(!exporter->auto_export_mails); + + if (request->guid == NULL) { + i_assert(request->uid > 0); + seq_range_array_add(&exporter->requested_uids, request->uid); + return; + } + + instances = hash_table_lookup(exporter->export_guids, request->guid); + if (instances == NULL) { + exporter->mail_error = MAIL_ERROR_TEMP; + exporter->error = p_strdup_printf(exporter->pool, + "Remote requested unexpected GUID %s", request->guid); + return; + } + instances->requested = TRUE; +} + +int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail **mail_r) +{ + struct mail *mail; + const char *const *guids; + unsigned int count; + int ret; + + if (exporter->error != NULL) + return -1; + if (!exporter->body_search_initialized) { + exporter->body_search_initialized = TRUE; + if (dsync_mailbox_export_body_search_init(exporter) < 0) { + i_assert(exporter->error != NULL); + return -1; + } + } + + while (mailbox_search_next(exporter->search_ctx, &mail)) { + exporter->search_pos++; + if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) { + *mail_r = &exporter->dsync_mail; + return 1; + } + if (ret < 0) { + i_assert(exporter->error != NULL); + return -1; + } + /* the message was expunged. if the GUID has another instance, + try sending it later. */ + seq_range_array_add(&exporter->expunged_seqs, mail->seq); + } + /* if some instances of messages were expunged, retry fetching them + with other instances */ + dsync_mailbox_export_body_search_deinit(exporter); + if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) { + i_assert(exporter->error != NULL); + return -1; + } + if (ret > 0) { + /* not finished yet */ + return dsync_mailbox_export_next_mail(exporter, mail_r); + } + + /* finished with messages. if there are any expunged messages, + return them */ + guids = array_get(&exporter->expunged_guids, &count); + if (exporter->expunged_guid_idx < count) { + i_zero(&exporter->dsync_mail); + exporter->dsync_mail.guid = + guids[exporter->expunged_guid_idx++]; + *mail_r = &exporter->dsync_mail; + return 1; + } + return 0; +} + +int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter, + const char **errstr_r, enum mail_error *error_r) +{ + struct dsync_mailbox_exporter *exporter = *_exporter; + + *_exporter = NULL; + + if (exporter->attr_iter != NULL) + (void)mailbox_attribute_iter_deinit(&exporter->attr_iter); + dsync_mailbox_export_body_search_deinit(exporter); + (void)mailbox_transaction_commit(&exporter->trans); + mailbox_header_lookup_unref(&exporter->wanted_headers); + + i_stream_unref(&exporter->attr.value_stream); + hash_table_destroy(&exporter->export_guids); + hash_table_destroy(&exporter->changes); + + i_assert((exporter->error != NULL) == (exporter->mail_error != 0)); + + *error_r = exporter->mail_error; + *errstr_r = t_strdup(exporter->error); + pool_unref(&exporter->pool); + return *errstr_r != NULL ? -1 : 0; +} + +const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter) +{ + if (exporter->search_ctx == NULL) + return ""; + return t_strdup_printf("%u/%u", exporter->search_pos, + exporter->search_count); +} diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h new file mode 100644 index 0000000..67e3216 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-export.h @@ -0,0 +1,37 @@ +#ifndef DSYNC_MAILBOX_EXPORT_H +#define DSYNC_MAILBOX_EXPORT_H + +enum dsync_mailbox_exporter_flags { + DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01, + DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02, + DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04, + DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS = 0x08, + DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES = 0x20, + DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES = 0x40, +}; + +struct dsync_mailbox_exporter * +dsync_mailbox_export_init(struct mailbox *box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + enum dsync_mailbox_exporter_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers); +/* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */ +int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter, + const struct dsync_mailbox_attribute **attr_r); +/* Returns 1 if change was returned, 0 if no more changes, -1 on error */ +int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_change **change_r); + +void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail_request *request); +/* Returns 1 if mail was returned, 0 if no more mails, -1 on error */ +int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter, + const struct dsync_mail **mail_r); +int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **exporter, + const char **errstr_r, enum mail_error *error_r); + +const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c new file mode 100644 index 0000000..8a8fbe4 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-import.c @@ -0,0 +1,2997 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "hex-binary.h" +#include "istream.h" +#include "seq-range-array.h" +#include "imap-util.h" +#include "mail-storage-private.h" +#include "mail-search-build.h" +#include "dsync-transaction-log-scan.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-mailbox-import.h" + +struct importer_mail { + const char *guid; + uint32_t uid; +}; + +struct importer_new_mail { + /* linked list of mails for this GUID */ + struct importer_new_mail *next; + /* if non-NULL, this mail exists in both local and remote. this link + points to the other side. */ + struct importer_new_mail *link; + + const char *guid; + struct dsync_mail_change *change; + + /* the final UID for the message */ + uint32_t final_uid; + /* the original local UID, or 0 if exists only remotely */ + uint32_t local_uid; + /* the original remote UID, or 0 if exists only remotely */ + uint32_t remote_uid; + /* UID for the mail in the virtual \All mailbox */ + uint32_t virtual_all_uid; + + bool uid_in_local:1; + bool uid_is_usable:1; + bool skip:1; + bool expunged:1; + bool copy_failed:1; + bool saved:1; +}; + +/* for quickly testing that two-way sync doesn't actually do any unexpected + modifications. */ +#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/ + +HASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *); +HASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *); + +struct dsync_mailbox_importer { + pool_t pool; + struct mailbox *box; + uint32_t last_common_uid; + uint64_t last_common_modseq, last_common_pvt_modseq; + uint32_t remote_uid_next; + uint32_t remote_first_recent_uid; + uint64_t remote_highest_modseq, remote_highest_pvt_modseq; + time_t sync_since_timestamp; + time_t sync_until_timestamp; + uoff_t sync_max_size; + enum mailbox_transaction_flags transaction_flags; + unsigned int hdr_hash_version; + unsigned int commit_msgs_interval; + + const char *const *hashed_headers; + + enum mail_flags sync_flag; + const char *sync_keyword; + bool sync_flag_dontwant; + + struct mailbox_transaction_context *trans, *ext_trans; + struct mail_search_context *search_ctx; + struct mail *mail, *ext_mail; + + struct mailbox *virtual_all_box; + struct mailbox_transaction_context *virtual_trans; + struct mail *virtual_mail; + + struct mail *cur_mail; + const char *cur_guid; + const char *cur_hdr_hash; + + /* UID => struct dsync_mail_change */ + HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes; + HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes; + + ARRAY_TYPE(seq_range) maybe_expunge_uids; + ARRAY(struct dsync_mail_change *) maybe_saves; + + /* GUID => struct importer_new_mail */ + HASH_TABLE_TYPE(guid_new_mail) import_guids; + /* UID => struct importer_new_mail */ + HASH_TABLE_TYPE(uid_new_mail) import_uids; + + ARRAY(struct importer_new_mail *) newmails; + ARRAY_TYPE(uint32_t) wanted_uids; + ARRAY_TYPE(uint32_t) saved_uids; + uint32_t highest_wanted_uid; + + ARRAY(struct dsync_mail_request) mail_requests; + unsigned int mail_request_idx; + + uint32_t prev_uid, next_local_seq, local_uid_next; + uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq; + unsigned int import_pos, import_count; + unsigned int first_unsaved_idx, saves_since_commit; + + enum mail_error mail_error; + + bool failed:1; + bool require_full_resync:1; + bool debug:1; + bool stateful_import:1; + bool last_common_uid_found:1; + bool cur_uid_has_change:1; + bool cur_mail_skip:1; + bool local_expunged_guids_set:1; + bool new_uids_assigned:1; + bool want_mail_requests:1; + bool master_brain:1; + bool revert_local_changes:1; + bool mails_have_guids:1; + bool mails_use_guid128:1; + bool delete_mailbox:1; + bool empty_hdr_workaround:1; +}; + +static const char *dsync_mail_change_type_names[] = { + "save", "expunge", "flag-change" +}; + +static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *all_newmails, + bool remote_mail); +static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, + bool final); + +static void ATTR_FORMAT(2, 3) +imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...) +{ + va_list args; + + if (importer->debug) T_BEGIN { + va_start(args, fmt); + i_debug("brain %c: Import %s: %s", + importer->master_brain ? 'M' : 'S', + mailbox_get_vname(importer->box), + t_strdup_vprintf(fmt, args)); + va_end(args); + } T_END; +} + +static void +dsync_import_unexpected_state(struct dsync_mailbox_importer *importer, + const char *error) +{ + if (!importer->stateful_import) { + i_error("Mailbox %s: %s", mailbox_get_vname(importer->box), + error); + } else { + i_warning("Mailbox %s doesn't match previous state: %s " + "(dsync must be run again without the state)", + mailbox_get_vname(importer->box), error); + } + importer->require_full_resync = TRUE; +} + +static void +dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer) +{ + struct mail_search_args *search_args; + struct mail_search_arg *sarg; + + search_args = mail_search_build_init(); + sarg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&sarg->value.seqset, search_args->pool, 128); + seq_range_array_add_range(&sarg->value.seqset, + importer->last_common_uid+1, (uint32_t)-1); + + importer->search_ctx = + mailbox_search_init(importer->trans, search_args, NULL, + 0, NULL); + mail_search_args_unref(&search_args); + + if (mailbox_search_next(importer->search_ctx, &importer->cur_mail)) + importer->next_local_seq = importer->cur_mail->seq; + /* this flag causes cur_guid to be looked up later */ + importer->cur_mail_skip = TRUE; +} + +static void +dsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer) +{ + const enum mailbox_transaction_flags ext_trans_flags = + importer->transaction_flags | + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; + + importer->trans = mailbox_transaction_begin(importer->box, + importer->transaction_flags, + "dsync import"); + importer->ext_trans = mailbox_transaction_begin(importer->box, + ext_trans_flags, + "dsync ext import"); + importer->mail = mail_alloc(importer->trans, 0, NULL); + importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL); +} + +struct dsync_mailbox_importer * +dsync_mailbox_import_init(struct mailbox *box, + struct mailbox *virtual_all_box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + uint64_t last_common_pvt_modseq, + uint32_t remote_uid_next, + uint32_t remote_first_recent_uid, + uint64_t remote_highest_modseq, + uint64_t remote_highest_pvt_modseq, + time_t sync_since_timestamp, + time_t sync_until_timestamp, + uoff_t sync_max_size, + const char *sync_flag, + unsigned int commit_msgs_interval, + enum dsync_mailbox_import_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers) +{ + struct dsync_mailbox_importer *importer; + struct mailbox_status status; + pool_t pool; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer", + 10240); + importer = p_new(pool, struct dsync_mailbox_importer, 1); + importer->pool = pool; + importer->box = box; + importer->virtual_all_box = virtual_all_box; + importer->last_common_uid = last_common_uid; + importer->last_common_modseq = last_common_modseq; + importer->last_common_pvt_modseq = last_common_pvt_modseq; + importer->last_common_uid_found = + last_common_uid != 0 || last_common_modseq != 0; + importer->remote_uid_next = remote_uid_next; + importer->remote_first_recent_uid = remote_first_recent_uid; + importer->remote_highest_modseq = remote_highest_modseq; + importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq; + importer->sync_since_timestamp = sync_since_timestamp; + importer->sync_until_timestamp = sync_until_timestamp; + importer->sync_max_size = sync_max_size; + importer->stateful_import = importer->last_common_uid_found; + importer->hashed_headers = hashed_headers; + + if (sync_flag != NULL) { + if (sync_flag[0] == '-') { + importer->sync_flag_dontwant = TRUE; + sync_flag++; + } + if (sync_flag[0] == '\\') + importer->sync_flag = imap_parse_system_flag(sync_flag); + else + importer->sync_keyword = p_strdup(pool, sync_flag); + } + importer->commit_msgs_interval = commit_msgs_interval; + importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC; + if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0) + importer->transaction_flags |= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY; + + hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp); + hash_table_create_direct(&importer->import_uids, pool, 0); + i_array_init(&importer->maybe_expunge_uids, 16); + i_array_init(&importer->maybe_saves, 128); + i_array_init(&importer->newmails, 128); + i_array_init(&importer->wanted_uids, 128); + i_array_init(&importer->saved_uids, 128); + + dsync_mailbox_import_transaction_begin(importer); + + if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) { + i_array_init(&importer->mail_requests, 128); + importer->want_mail_requests = TRUE; + } + importer->master_brain = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0; + importer->revert_local_changes = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0; + importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0; + importer->mails_have_guids = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0; + importer->mails_use_guid128 = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0; + importer->hdr_hash_version = hdr_hash_version; + importer->empty_hdr_workaround = + (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0; + + mailbox_get_open_status(importer->box, STATUS_UIDNEXT | + STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ, + &status); + if (status.nonpermanent_modseqs) + status.highest_modseq = 0; + importer->local_uid_next = status.uidnext; + importer->local_initial_highestmodseq = status.highest_modseq; + importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq; + dsync_mailbox_import_search_init(importer); + + if (!importer->stateful_import) + ; + else if (importer->local_uid_next <= last_common_uid) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local UIDNEXT %u <= last common UID %u", + importer->local_uid_next, last_common_uid)); + } else if (importer->local_initial_highestmodseq < last_common_modseq) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64, + importer->local_initial_highestmodseq, + last_common_modseq)); + } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64, + importer->local_initial_highestpvtmodseq, + last_common_pvt_modseq)); + } + + importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan); + importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan); + return importer; +} + +static int +dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer, + enum mail_attribute_type type, const char *key, + struct dsync_mailbox_attribute **attr_r) +{ + struct dsync_mailbox_attribute lookup_attr, *attr; + const struct dsync_mailbox_attribute *attr_change; + struct mail_attribute_value value; + + *attr_r = NULL; + + if (mailbox_attribute_get_stream(importer->box, type, key, &value) < 0) { + i_error("Mailbox %s: Failed to get attribute %s: %s", + mailbox_get_vname(importer->box), key, + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + return -1; + } + + lookup_attr.type = type; + lookup_attr.key = key; + + attr_change = hash_table_lookup(importer->local_attr_changes, + &lookup_attr); + if (attr_change == NULL && + value.value == NULL && value.value_stream == NULL) { + /* we have no knowledge of this attribute */ + return 0; + } + attr = t_new(struct dsync_mailbox_attribute, 1); + attr->type = type; + attr->key = key; + attr->value = value.value; + attr->value_stream = value.value_stream; + attr->last_change = value.last_change; + if (attr_change != NULL) { + attr->deleted = attr_change->deleted && + !DSYNC_ATTR_HAS_VALUE(attr); + attr->modseq = attr_change->modseq; + } + *attr_r = attr; + return 0; +} + +static int +dsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r) +{ + const unsigned char *data1, *data2; + size_t size1, size2, size; + + *cmp_r = -1; /* quiet gcc */ + + for (;;) { + (void)i_stream_read_more(input1, &data1, &size1); + (void)i_stream_read_more(input2, &data2, &size2); + + if (size1 == 0 || size2 == 0) + break; + size = I_MIN(size1, size2); + *cmp_r = memcmp(data1, data2, size); + if (*cmp_r != 0) + return 0; + i_stream_skip(input1, size); + i_stream_skip(input2, size); + } + if (input1->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(input1), + i_stream_get_error(input1)); + return -1; + } + if (input2->stream_errno != 0) { + i_error("read(%s) failed: %s", i_stream_get_name(input2), + i_stream_get_error(input2)); + return -1; + } + if (size1 == 0 && size2 == 0) + *cmp_r = 0; + else + *cmp_r = size1 == 0 ? -1 : 1; + return 0; +} + +static int +dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1, + const struct dsync_mailbox_attribute *attr2, + int *cmp_r) +{ + struct istream *input1, *input2; + int ret; + + i_assert(attr1->value_stream != NULL || attr1->value != NULL); + i_assert(attr2->value_stream != NULL || attr2->value != NULL); + + if (attr1->value != NULL && attr2->value != NULL) { + *cmp_r = strcmp(attr1->value, attr2->value); + return 0; + } + /* at least one of them is a stream. make both of them streams. */ + input1 = attr1->value_stream != NULL ? attr1->value_stream : + i_stream_create_from_data(attr1->value, strlen(attr1->value)); + input2 = attr2->value_stream != NULL ? attr2->value_stream : + i_stream_create_from_data(attr2->value, strlen(attr2->value)); + i_stream_seek(input1, 0); + i_stream_seek(input2, 0); + ret = dsync_istreams_cmp(input1, input2, cmp_r); + if (attr1->value_stream == NULL) + i_stream_unref(&input1); + if (attr2->value_stream == NULL) + i_stream_unref(&input2); + return ret; +} + +static int +dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr, + const struct dsync_mailbox_attribute *local_attr, + int *cmp_r) +{ + if (DSYNC_ATTR_HAS_VALUE(attr) && + !DSYNC_ATTR_HAS_VALUE(local_attr)) { + /* remote has a value and local doesn't -> use it */ + *cmp_r = 1; + return 0; + } else if (!DSYNC_ATTR_HAS_VALUE(attr) && + DSYNC_ATTR_HAS_VALUE(local_attr)) { + /* remote doesn't have a value, bt local does -> skip */ + *cmp_r = -1; + return 0; + } + + return dsync_attributes_cmp_values(attr, local_attr, cmp_r); +} + +static int +dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer, + const struct dsync_mailbox_attribute *attr, + const struct dsync_mailbox_attribute *local_attr, + const char **result_r) +{ + struct mail_attribute_value value; + int cmp; + bool ignore = FALSE; + + i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted); + + if (attr->deleted && + (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) { + /* attribute doesn't exist on either side -> ignore */ + *result_r = "Nonexistent in both sides"; + return 0; + } + if (local_attr == NULL) { + /* we haven't seen this locally -> use whatever remote has */ + *result_r = "Nonexistent locally"; + } else if (local_attr->modseq <= importer->last_common_modseq && + attr->modseq > importer->last_common_modseq && + importer->last_common_modseq > 0) { + /* we're doing incremental syncing, and we can see that the + attribute was changed remotely, but not locally -> use it */ + *result_r = "Changed remotely"; + } else if (local_attr->modseq > importer->last_common_modseq && + attr->modseq <= importer->last_common_modseq && + importer->last_common_modseq > 0) { + /* we're doing incremental syncing, and we can see that the + attribute was changed locally, but not remotely -> ignore */ + *result_r = "Changed locally"; + ignore = TRUE; + } else if (attr->last_change > local_attr->last_change) { + /* remote has a newer timestamp -> use it */ + *result_r = "Remote has newer timestamp"; + } else if (attr->last_change < local_attr->last_change) { + /* remote has an older timestamp -> ignore */ + *result_r = "Local has newer timestamp"; + ignore = TRUE; + } else { + /* the timestamps are the same. now we're down to guessing + the right answer, unless the values are actually equal, + so check that first. next try to use modseqs, but if even + they are the same, fallback to just picking one based on the + value. */ + if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) { + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + return -1; + } + if (cmp == 0) { + /* identical scripts */ + *result_r = "Unchanged value"; + return 0; + } + + if (attr->modseq > local_attr->modseq) { + /* remote has a higher modseq -> use it */ + *result_r = "Remote has newer modseq"; + } else if (attr->modseq < local_attr->modseq) { + /* remote has an older modseq -> ignore */ + *result_r = "Local has newer modseq"; + ignore = TRUE; + } else if (cmp < 0) { + ignore = TRUE; + *result_r = "Value changed, but unknown which is newer - picking local"; + } else { + *result_r = "Value changed, but unknown which is newer - picking remote"; + } + } + if (ignore) + return 0; + + i_zero(&value); + value.value = attr->value; + value.value_stream = attr->value_stream; + value.last_change = attr->last_change; + if (mailbox_attribute_set(importer->trans, attr->type, + attr->key, &value) < 0) { + i_error("Mailbox %s: Failed to set attribute %s: %s", + mailbox_get_vname(importer->box), attr->key, + mailbox_get_last_internal_error(importer->box, NULL)); + /* the attributes aren't vital, don't fail everything just + because of them. */ + } + return 0; +} + +int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, + const struct dsync_mailbox_attribute *attr) +{ + struct dsync_mailbox_attribute *local_attr; + const char *result = ""; + int ret; + + if (dsync_mailbox_import_lookup_attr(importer, attr->type, + attr->key, &local_attr) < 0) + ret = -1; + else { + ret = dsync_mailbox_import_attribute_real(importer, attr, + local_attr, &result); + if (local_attr != NULL && local_attr->value_stream != NULL) + i_stream_unref(&local_attr->value_stream); + } + imp_debug(importer, "Import attribute %s: %s", attr->key, + ret < 0 ? "failed" : result); + return ret; +} + +static void dsync_mail_error(struct dsync_mailbox_importer *importer, + struct mail *mail, const char *field) +{ + const char *errstr; + enum mail_error error; + + errstr = mailbox_get_last_internal_error(importer->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return; + + i_error("Mailbox %s: Can't lookup %s for UID=%u: %s", + mailbox_get_vname(mail->box), field, mail->uid, errstr); + importer->mail_error = error; + importer->failed = TRUE; +} + +static bool +dsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char *guid, const char **cmp_guid_r) +{ + guid_128_t guid_128, change_guid_128; + + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + if (guid_128_from_string(change->guid, change_guid_128) < 0) + i_unreached(); + } else if (importer->mails_use_guid128) { + mail_generate_guid_128_hash(change->guid, change_guid_128); + } else { + if (cmp_guid_r != NULL) + *cmp_guid_r = change->guid; + return strcmp(change->guid, guid) == 0; + } + + mail_generate_guid_128_hash(guid, guid_128); + if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) { + if (cmp_guid_r != NULL) { + *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)", + binary_to_hex(change_guid_128, sizeof(change_guid_128)), + change->guid); + } + return FALSE; + } + return TRUE; +} + +static int +importer_try_next_mail(struct dsync_mailbox_importer *importer, + uint32_t wanted_uid) +{ + struct mail_private *pmail; + const char *hdr_hash; + + if (importer->cur_mail == NULL) { + /* end of search */ + return -1; + } + while (importer->cur_mail->seq < importer->next_local_seq || + importer->cur_mail->uid < wanted_uid) { + if (!importer->cur_uid_has_change && + !importer->last_common_uid_found) { + /* this message exists locally, but remote didn't send + expunge-change for it. if the message's + uid <= last-common-uid, it should be deleted */ + seq_range_array_add(&importer->maybe_expunge_uids, + importer->cur_mail->uid); + } + + importer->cur_mail_skip = FALSE; + if (!mailbox_search_next(importer->search_ctx, + &importer->cur_mail)) { + importer->cur_mail = NULL; + importer->cur_guid = NULL; + importer->cur_hdr_hash = NULL; + return -1; + } + importer->cur_uid_has_change = FALSE; + } + importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid; + if (importer->mails_have_guids) { + if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID, + &importer->cur_guid) < 0) { + dsync_mail_error(importer, importer->cur_mail, "GUID"); + return 0; + } + } else { + if (dsync_mail_get_hdr_hash(importer->cur_mail, + importer->hdr_hash_version, + importer->hashed_headers, + &hdr_hash) < 0) { + dsync_mail_error(importer, importer->cur_mail, + "header hash"); + return 0; + } + pmail = (struct mail_private *)importer->cur_mail; + importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash); + importer->cur_guid = ""; + } + /* make sure next_local_seq gets updated in case we came here + because of min_uid */ + importer->next_local_seq = importer->cur_mail->seq; + return 1; +} + +static bool +importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid) +{ + int ret; + + for (;;) { + T_BEGIN { + ret = importer_try_next_mail(importer, wanted_uid); + } T_END; + if (ret != 0 || importer->failed) + break; + importer->next_local_seq = importer->cur_mail->seq + 1; + } + return ret > 0; +} + +static int +importer_mail_cmp(const struct importer_mail *m1, + const struct importer_mail *m2) +{ + int ret; + + if (m1->guid == NULL) + return 1; + if (m2->guid == NULL) + return -1; + + ret = strcmp(m1->guid, m2->guid); + if (ret != 0) + return ret; + + if (m1->uid < m2->uid) + return -1; + if (m1->uid > m2->uid) + return 1; + return 0; +} + +static void newmail_link(struct dsync_mailbox_importer *importer, + struct importer_new_mail *newmail, uint32_t remote_uid) +{ + struct importer_new_mail *first_mail, **last, *mail, *link = NULL; + + if (*newmail->guid != '\0') { + first_mail = hash_table_lookup(importer->import_guids, + newmail->guid); + if (first_mail == NULL) { + /* first mail for this GUID */ + hash_table_insert(importer->import_guids, + newmail->guid, newmail); + return; + } + } else { + if (remote_uid == 0) { + /* mail exists locally. we don't want to request + it, and we'll assume it has no duplicate + instances. */ + return; + } + first_mail = hash_table_lookup(importer->import_uids, + POINTER_CAST(remote_uid)); + if (first_mail == NULL) { + /* first mail for this UID */ + hash_table_insert(importer->import_uids, + POINTER_CAST(remote_uid), newmail); + return; + } + } + /* 1) add the newmail to the end of the linked list + 2) find our link + + FIXME: this loop is slow if the same GUID has a ton of instances. + Could it be improved in some way? */ + last = &first_mail->next; + for (mail = first_mail; mail != NULL; mail = mail->next) { + if (mail->final_uid == newmail->final_uid) + mail->uid_is_usable = TRUE; + if (link == NULL && mail->link == NULL && + mail->uid_in_local != newmail->uid_in_local) + link = mail; + last = &mail->next; + } + *last = newmail; + if (link != NULL && newmail->link == NULL) { + link->link = newmail; + newmail->link = link; + } +} + +static void +dsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer, + uint32_t uid, const char *reason) +{ + i_assert(importer->revert_local_changes); + + /* UID either already exists or UIDNEXT is too high. we can't set the + wanted UID, so we'll need to delete the whole mailbox and resync */ + i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s", + mailbox_get_vname(importer->box), uid, reason); + importer->delete_mailbox = TRUE; + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; +} + +static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + struct importer_mail m1, m2; + struct importer_new_mail *newmail; + int diff; + bool remote_saved; + + i_zero(&m1); + if (importer->cur_mail != NULL) { + m1.guid = importer->mails_have_guids ? + importer->cur_guid : importer->cur_hdr_hash; + m1.uid = importer->cur_mail->uid; + } + i_zero(&m2); + if (save_change != NULL) { + m2.guid = importer->mails_have_guids ? + save_change->guid : save_change->hdr_hash; + m2.uid = save_change->uid; + i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + } + + if (importer->empty_hdr_workaround && !importer->mails_have_guids && + importer->cur_mail != NULL && save_change != NULL && + (dsync_mail_hdr_hash_is_empty(m1.guid) || + dsync_mail_hdr_hash_is_empty(m2.guid))) { + /* one of the headers is empty. assume it's broken and that + the header matches what we have currently. */ + diff = 0; + } else { + diff = importer_mail_cmp(&m1, &m2); + } + if (diff < 0) { + /* add a record for local mail */ + i_assert(importer->cur_mail != NULL); + if (importer->revert_local_changes) { + if (save_change == NULL && + importer->cur_mail->uid >= importer->remote_uid_next) { + dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid, + t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next)); + return TRUE; + } + mail_expunge(importer->cur_mail); + importer->cur_mail_skip = TRUE; + importer->next_local_seq++; + return FALSE; + } + newmail = p_new(importer->pool, struct importer_new_mail, 1); + newmail->guid = p_strdup(importer->pool, importer->cur_guid); + newmail->final_uid = importer->cur_mail->uid; + newmail->local_uid = importer->cur_mail->uid; + newmail->uid_in_local = TRUE; + newmail->uid_is_usable = + newmail->final_uid >= importer->remote_uid_next; + remote_saved = FALSE; + } else if (diff > 0) { + i_assert(save_change != NULL); + newmail = p_new(importer->pool, struct importer_new_mail, 1); + newmail->guid = save_change->guid; + newmail->final_uid = save_change->uid; + newmail->remote_uid = save_change->uid; + newmail->uid_in_local = FALSE; + newmail->uid_is_usable = + newmail->final_uid >= importer->local_uid_next; + if (!newmail->uid_is_usable && importer->revert_local_changes) { + dsync_mailbox_revert_existing_uid(importer, newmail->final_uid, + t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next)); + return TRUE; + } + remote_saved = TRUE; + } else { + /* identical */ + i_assert(importer->cur_mail != NULL); + i_assert(save_change != NULL); + newmail = p_new(importer->pool, struct importer_new_mail, 1); + newmail->guid = save_change->guid; + newmail->final_uid = importer->cur_mail->uid; + newmail->local_uid = importer->cur_mail->uid; + newmail->remote_uid = save_change->uid; + newmail->uid_in_local = TRUE; + newmail->uid_is_usable = TRUE; + newmail->link = newmail; + remote_saved = TRUE; + } + + if (newmail->uid_in_local) { + importer->cur_mail_skip = TRUE; + importer->next_local_seq++; + } + /* NOTE: assumes save_change is allocated from importer pool */ + newmail->change = save_change; + + array_push_back(&importer->newmails, &newmail); + if (newmail->uid_in_local) + newmail_link(importer, newmail, 0); + else { + i_assert(save_change != NULL); + newmail_link(importer, newmail, save_change->uid); + } + return remote_saved; +} + +static bool ATTR_NULL(2) +dsync_mailbox_try_save(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + if (importer->cur_mail_skip) { + if (!importer_next_mail(importer, 0) && save_change == NULL) + return FALSE; + } + return dsync_mailbox_try_save_cur(importer, save_change); +} + +static void dsync_mailbox_save(struct dsync_mailbox_importer *importer, + struct dsync_mail_change *save_change) +{ + while (!dsync_mailbox_try_save(importer, save_change)) ; +} + +static bool +dsync_import_set_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *guid, *cmp_guid; + + if (!mail_set_uid(importer->mail, change->uid)) + return FALSE; + if (change->guid == NULL) { + /* GUID is unknown */ + return TRUE; + } + if (*change->guid == '\0') { + /* backend doesn't support GUIDs. if hdr_hash is set, we could + verify it, but since this message really is supposed to + match, it's probably too much trouble. */ + return TRUE; + } + + /* verify that GUID matches, just in case */ + if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) { + dsync_mail_error(importer, importer->mail, "GUID"); + return FALSE; + } + if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "Unexpected GUID mismatch for UID=%u: %s != %s", + change->uid, guid, cmp_guid)); + return FALSE; + } + return TRUE; +} + +static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *cmp_guid; + + if (change->guid == NULL || change->guid[0] == '\0' || + importer->cur_guid[0] == '\0') + return TRUE; + if (!dsync_mail_change_guid_equals(importer, change, + importer->cur_guid, &cmp_guid)) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "Unexpected GUID mismatch (2) for UID=%u: %s != %s", + change->uid, importer->cur_guid, cmp_guid)); + return FALSE; + } + return TRUE; +} + +static void +merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove, + uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove, + uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote, + uint32_t *change_add_r, uint32_t *change_remove_r, + bool *remote_changed, bool *remote_pvt_changed) +{ + uint32_t combined_add, combined_remove, conflict_flags; + uint32_t local_wanted, remote_wanted, conflict_pvt_flags; + + /* resolve conflicts */ + conflict_flags = local_add & remote_remove; + if (conflict_flags != 0) { + conflict_pvt_flags = conflict_flags & pvt_mask; + conflict_flags &= ~pvt_mask; + if (prefer_remote) + local_add &= ~conflict_flags; + else + remote_remove &= ~conflict_flags; + if (prefer_pvt_remote) + local_add &= ~conflict_pvt_flags; + else + remote_remove &= ~conflict_pvt_flags; + } + conflict_flags = local_remove & remote_add; + if (conflict_flags != 0) { + conflict_pvt_flags = conflict_flags & pvt_mask; + conflict_flags &= ~pvt_mask; + if (prefer_remote) + local_remove &= ~conflict_flags; + else + remote_add &= ~conflict_flags; + if (prefer_pvt_remote) + local_remove &= ~conflict_pvt_flags; + else + remote_add &= ~conflict_pvt_flags; + } + + combined_add = local_add|remote_add; + combined_remove = local_remove|remote_remove; + i_assert((combined_add & combined_remove) == 0); + + /* don't change flags that are currently identical in both sides */ + conflict_flags = local_final ^ remote_final; + combined_add &= conflict_flags; + combined_remove &= conflict_flags; + + /* see if there are conflicting final flags */ + local_wanted = (local_final|combined_add) & ~combined_remove; + remote_wanted = (remote_final|combined_add) & ~combined_remove; + + conflict_flags = local_wanted ^ remote_wanted; + if (conflict_flags != 0) { + if (prefer_remote && prefer_pvt_remote) + local_wanted = remote_wanted; + else if (prefer_remote && !prefer_pvt_remote) { + local_wanted = (local_wanted & pvt_mask) | + (remote_wanted & ~pvt_mask); + } else if (!prefer_remote && prefer_pvt_remote) { + local_wanted = (local_wanted & ~pvt_mask) | + (remote_wanted & pvt_mask); + } + } + + *change_add_r = local_wanted & ~local_final; + *change_remove_r = local_final & ~local_wanted; + if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask)) + *remote_changed = TRUE; + if ((local_wanted & pvt_mask) != (remote_final & pvt_mask)) + *remote_pvt_changed = TRUE; +} + +static bool +keyword_find(ARRAY_TYPE(const_string) *keywords, const char *name, + unsigned int *idx_r) +{ + const char *const *names; + unsigned int i, count; + + names = array_get(keywords, &count); + for (i = 0; i < count; i++) { + if (strcmp(names[i], name) == 0) { + *idx_r = i; + return TRUE; + } + } + return FALSE; +} + +static void keywords_append(ARRAY_TYPE(const_string) *dest, + const ARRAY_TYPE(const_string) *keywords, + uint32_t bits, unsigned int start_idx) +{ + const char *name; + unsigned int i; + + for (i = 0; i < 32; i++) { + if ((bits & (1U << i)) == 0) + continue; + + name = array_idx_elem(keywords, start_idx+i); + array_push_back(dest, &name); + } +} + +static void +merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes, + const ARRAY_TYPE(const_string) *remote_changes, + bool prefer_remote, + bool *remote_changed, bool *remote_pvt_changed) +{ + /* local_changes and remote_changes are assumed to have no + duplicates names */ + uint32_t *local_add, *local_remove, *local_final; + uint32_t *remote_add, *remote_remove, *remote_final; + uint32_t *change_add, *change_remove; + ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords; + const char *const *changes, *name, *const *local_keywords; + struct mail_keywords *kw; + unsigned int i, count, name_idx, array_size; + + local_keywords = mail_get_keywords(mail); + + /* we'll assign a common index for each keyword name and place + the changes to separate bit arrays. */ + if (array_is_created(remote_changes)) + changes = array_get(remote_changes, &count); + else { + changes = NULL; + count = 0; + } + + array_size = str_array_length(local_keywords) + count; + if (array_is_created(local_changes)) + array_size += array_count(local_changes); + if (array_size == 0) { + /* this message has no keywords */ + return; + } + t_array_init(&all_keywords, array_size); + t_array_init(&add_keywords, array_size); + t_array_init(&remove_keywords, array_size); + + /* @UNSAFE: create large enough arrays to fit all keyword indexes. */ + array_size = (array_size+31)/32; + local_add = t_new(uint32_t, array_size); + local_remove = t_new(uint32_t, array_size); + local_final = t_new(uint32_t, array_size); + remote_add = t_new(uint32_t, array_size); + remote_remove = t_new(uint32_t, array_size); + remote_final = t_new(uint32_t, array_size); + change_add = t_new(uint32_t, array_size); + change_remove = t_new(uint32_t, array_size); + + /* get remote changes */ + for (i = 0; i < count; i++) { + name = changes[i]+1; + name_idx = array_count(&all_keywords); + array_push_back(&all_keywords, &name); + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + remote_add[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_REMOVE: + remote_remove[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_FINAL: + remote_final[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_ADD_AND_FINAL: + remote_add[name_idx/32] |= 1U << (name_idx%32); + remote_final[name_idx/32] |= 1U << (name_idx%32); + break; + } + } + + /* get local changes. use existing indexes for names when they exist. */ + if (array_is_created(local_changes)) + changes = array_get(local_changes, &count); + else { + changes = NULL; + count = 0; + } + for (i = 0; i < count; i++) { + name = changes[i]+1; + if (!keyword_find(&all_keywords, name, &name_idx)) { + name_idx = array_count(&all_keywords); + array_push_back(&all_keywords, &name); + } + + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + case KEYWORD_CHANGE_ADD_AND_FINAL: + local_add[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_REMOVE: + local_remove[name_idx/32] |= 1U << (name_idx%32); + break; + case KEYWORD_CHANGE_FINAL: + break; + } + } + for (i = 0; local_keywords[i] != NULL; i++) { + name = local_keywords[i]; + if (!keyword_find(&all_keywords, name, &name_idx)) { + name_idx = array_count(&all_keywords); + array_push_back(&all_keywords, &name); + } + local_final[name_idx/32] |= 1U << (name_idx%32); + } + i_assert(array_count(&all_keywords) <= array_size*32); + array_size = (array_count(&all_keywords)+31) / 32; + + /* merge keywords */ + for (i = 0; i < array_size; i++) { + merge_flags(local_final[i], local_add[i], local_remove[i], + remote_final[i], remote_add[i], remote_remove[i], + 0, prefer_remote, prefer_remote, + &change_add[i], &change_remove[i], + remote_changed, remote_pvt_changed); + if (change_add[i] != 0) { + keywords_append(&add_keywords, &all_keywords, + change_add[i], i*32); + } + if (change_remove[i] != 0) { + keywords_append(&remove_keywords, &all_keywords, + change_remove[i], i*32); + } + } + + /* apply changes */ + if (array_count(&add_keywords) > 0) { + array_append_zero(&add_keywords); + kw = mailbox_keywords_create_valid(mail->box, + array_front(&add_keywords)); + mail_update_keywords(mail, MODIFY_ADD, kw); + mailbox_keywords_unref(&kw); + } + if (array_count(&remove_keywords) > 0) { + array_append_zero(&remove_keywords); + kw = mailbox_keywords_create_valid(mail->box, + array_front(&remove_keywords)); + mail_update_keywords(mail, MODIFY_REMOVE, kw); + mailbox_keywords_unref(&kw); + } +} + +static void +dsync_mailbox_import_replace_flags(struct mail *mail, + const struct dsync_mail_change *change) +{ + ARRAY_TYPE(const_string) keywords; + struct mail_keywords *kw; + const char *const *changes, *name; + unsigned int i, count; + + if (array_is_created(&change->keyword_changes)) + changes = array_get(&change->keyword_changes, &count); + else { + changes = NULL; + count = 0; + } + t_array_init(&keywords, count+1); + for (i = 0; i < count; i++) { + switch (changes[i][0]) { + case KEYWORD_CHANGE_ADD: + case KEYWORD_CHANGE_FINAL: + case KEYWORD_CHANGE_ADD_AND_FINAL: + name = changes[i]+1; + array_push_back(&keywords, &name); + break; + case KEYWORD_CHANGE_REMOVE: + break; + } + } + array_append_zero(&keywords); + + kw = mailbox_keywords_create_valid(mail->box, array_front(&keywords)); + mail_update_keywords(mail, MODIFY_REPLACE, kw); + mailbox_keywords_unref(&kw); + + mail_update_flags(mail, MODIFY_REPLACE, + change->add_flags | change->final_flags); + if (mail_get_modseq(mail) < change->modseq) + mail_update_modseq(mail, change->modseq); + if (mail_get_pvt_modseq(mail) < change->pvt_modseq) + mail_update_pvt_modseq(mail, change->pvt_modseq); +} + +static void +dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const struct dsync_mail_change *local_change; + enum mail_flags local_add, local_remove; + uint32_t change_add, change_remove; + uint64_t new_modseq; + ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT; + struct mail *mail; + bool prefer_remote, prefer_pvt_remote; + bool remote_changed = FALSE, remote_pvt_changed = FALSE; + + i_assert((change->add_flags & change->remove_flags) == 0); + + if (importer->cur_mail != NULL && + importer->cur_mail->uid == change->uid) { + if (!dsync_check_cur_guid(importer, change)) + return; + mail = importer->cur_mail; + } else { + if (!dsync_import_set_mail(importer, change)) + return; + mail = importer->mail; + } + + if (importer->revert_local_changes) { + /* dsync backup: just make the local look like remote. */ + dsync_mailbox_import_replace_flags(mail, change); + return; + } + + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL) { + local_add = local_remove = 0; + } else { + local_add = local_change->add_flags; + local_remove = local_change->remove_flags; + local_keyword_changes = local_change->keyword_changes; + } + + if (mail_get_modseq(mail) < change->modseq) + prefer_remote = TRUE; + else if (mail_get_modseq(mail) > change->modseq) + prefer_remote = FALSE; + else { + /* identical modseq, we'll just have to pick one. + Note that both brains need to pick the same one, otherwise + they become unsynced. */ + prefer_remote = !importer->master_brain; + } + if (mail_get_pvt_modseq(mail) < change->pvt_modseq) + prefer_pvt_remote = TRUE; + else if (mail_get_pvt_modseq(mail) > change->pvt_modseq) + prefer_pvt_remote = FALSE; + else + prefer_pvt_remote = !importer->master_brain; + + /* merge flags */ + merge_flags(mail_get_flags(mail), local_add, local_remove, + change->final_flags, change->add_flags, change->remove_flags, + mailbox_get_private_flags_mask(mail->box), + prefer_remote, prefer_pvt_remote, + &change_add, &change_remove, + &remote_changed, &remote_pvt_changed); + + if (change_add != 0) + mail_update_flags(mail, MODIFY_ADD, change_add); + if (change_remove != 0) + mail_update_flags(mail, MODIFY_REMOVE, change_remove); + + /* merge keywords */ + merge_keywords(mail, &local_keyword_changes, &change->keyword_changes, + prefer_remote, &remote_changed, &remote_pvt_changed); + + /* update modseqs. try to anticipate when we have to increase modseq + to get it closer to what remote has (although we can't guess it + exactly correctly) */ + new_modseq = change->modseq; + if (remote_changed && new_modseq <= importer->remote_highest_modseq) + new_modseq = importer->remote_highest_modseq+1; + if (mail_get_modseq(mail) < new_modseq) + mail_update_modseq(mail, new_modseq); + + new_modseq = change->pvt_modseq; + if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq) + new_modseq = importer->remote_highest_pvt_modseq+1; + if (mail_get_pvt_modseq(mail) < new_modseq) + mail_update_pvt_modseq(mail, new_modseq); +} + +static bool +dsync_mail_change_have_keyword(const struct dsync_mail_change *change, + const char *keyword) +{ + const char *str; + + if (!array_is_created(&change->keyword_changes)) + return FALSE; + + array_foreach_elem(&change->keyword_changes, str) { + switch (str[0]) { + case KEYWORD_CHANGE_FINAL: + case KEYWORD_CHANGE_ADD_AND_FINAL: + if (strcasecmp(str+1, keyword) == 0) + return TRUE; + break; + default: + break; + } + } + return FALSE; +} + +static bool +dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + if (importer->sync_since_timestamp > 0) { + i_assert(change->received_timestamp > 0); + if (change->received_timestamp < importer->sync_since_timestamp) { + /* mail has too old timestamp - skip it */ + *result_r = "Ignoring missing local mail with too old timestamp"; + return FALSE; + } + } + if (importer->sync_until_timestamp > 0) { + i_assert(change->received_timestamp > 0); + if (change->received_timestamp > importer->sync_until_timestamp) { + /* mail has too new timestamp - skip it */ + *result_r = "Ignoring missing local mail with too new timestamp"; + return FALSE; + } + } + if (importer->sync_max_size > 0) { + i_assert(change->virtual_size != UOFF_T_MAX); + if (change->virtual_size > importer->sync_max_size) { + /* mail is too large - skip it */ + *result_r = "Ignoring missing local mail with too large size"; + return FALSE; + } + } + if (importer->sync_flag != 0) { + bool have_flag = (change->final_flags & importer->sync_flag) != 0; + + if (have_flag && importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that doesn't have wanted flags"; + return FALSE; + } + if (!have_flag && !importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that has unwanted flags"; + return FALSE; + } + } + if (importer->sync_keyword != NULL) { + bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword); + + if (have_kw && importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that doesn't have wanted keywords"; + return FALSE; + } + if (!have_kw && !importer->sync_flag_dontwant) { + *result_r = "Ignoring missing local mail that has unwanted keywords"; + return FALSE; + } + } + return TRUE; +} + +static void +dsync_mailbox_import_save(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + struct dsync_mail_change *save; + const char *result; + + i_assert(change->guid != NULL); + + if (change->uid == importer->last_common_uid) { + /* we've already verified that the GUID matches. + apply flag changes if there are any. */ + i_assert(!importer->last_common_uid_found); + dsync_mailbox_import_flag_change(importer, change); + return; + } + if (!dsync_mailbox_import_want_change(importer, change, &result)) + return; + + save = p_new(importer->pool, struct dsync_mail_change, 1); + dsync_mail_change_dup(importer->pool, change, save); + + if (importer->last_common_uid_found) { + /* this is a new mail. its UID may or may not conflict with + an existing local mail, we'll figure it out later. */ + i_assert(change->uid > importer->last_common_uid); + dsync_mailbox_save(importer, save); + } else { + /* the local mail is expunged. we'll decide later if we want + to save this mail locally or expunge it form remote. */ + i_assert(change->uid > importer->last_common_uid); + i_assert(importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid); + array_push_back(&importer->maybe_saves, &save); + } +} + +static void +dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + + if (importer->last_common_uid_found) { + /* expunge the message, unless its GUID unexpectedly doesn't + match */ + i_assert(change->uid <= importer->last_common_uid); + if (dsync_import_set_mail(importer, change)) + mail_expunge(importer->mail); + } else if (importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid) { + /* already expunged locally, we can ignore this. + uid=last_common_uid if we managed to verify from + transaction log that the GUIDs match */ + i_assert(change->uid >= importer->last_common_uid); + } else if (change->uid == importer->last_common_uid) { + /* already verified that the GUID matches */ + i_assert(importer->cur_mail->uid == change->uid); + if (dsync_check_cur_guid(importer, change)) + mail_expunge(importer->cur_mail); + } else { + /* we don't know yet if we should expunge this + message or not. queue it until we do. */ + i_assert(change->uid > importer->last_common_uid); + seq_range_array_add(&importer->maybe_expunge_uids, change->uid); + } +} + +static void +dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer) +{ + /* If there are local mails after last_common_uid which we skipped + while trying to match the next message, we need to now go back */ + if (importer->cur_mail != NULL && + importer->cur_mail->uid <= importer->last_common_uid+1) + return; + + importer->cur_mail = NULL; + importer->cur_guid = NULL; + importer->cur_hdr_hash = NULL; + importer->next_local_seq = 0; + + (void)mailbox_search_deinit(&importer->search_ctx); + dsync_mailbox_import_search_init(importer); +} + +static void +dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer) +{ + struct dsync_mail_change *const *saves; + struct seq_range_iter iter; + unsigned int n, i, count; + uint32_t uid; + + if (importer->debug) T_BEGIN { + string_t *expunges = t_str_new(64); + + imap_write_seq_range(expunges, &importer->maybe_expunge_uids); + imp_debug(importer, "Last common UID=%u. Delayed expunges=%s", + importer->last_common_uid, str_c(expunges)); + } T_END; + + importer->last_common_uid_found = TRUE; + dsync_mailbox_rewind_search(importer); + + /* expunge the messages whose expunge-decision we delayed previously */ + seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + if (uid > importer->last_common_uid) { + /* we expunge messages only up to last_common_uid, + ignore the rest */ + break; + } + + if (mail_set_uid(importer->mail, uid)) + mail_expunge(importer->mail); + } + + /* handle pending saves */ + saves = array_get(&importer->maybe_saves, &count); + for (i = 0; i < count; i++) { + if (saves[i]->uid > importer->last_common_uid) { + imp_debug(importer, "Delayed save UID=%u: Save", + saves[i]->uid); + dsync_mailbox_save(importer, saves[i]); + } else { + imp_debug(importer, "Delayed save UID=%u: Ignore", + saves[i]->uid); + } + } +} + +static int +dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + const char *hdr_hash, *cmp_guid; + + if (*change->guid != '\0' && *importer->cur_guid != '\0') { + /* we have GUIDs, verify them */ + if (dsync_mail_change_guid_equals(importer, change, + importer->cur_guid, &cmp_guid)) { + *result_r = "GUIDs match"; + return 1; + } else { + *result_r = t_strdup_printf("GUIDs don't match (%s vs %s)", + importer->cur_guid, cmp_guid); + return 0; + } + } + + /* verify hdr_hash if it exists */ + if (change->hdr_hash == NULL) { + i_assert(*importer->cur_guid == '\0'); + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* the message was already expunged, so we don't know + its header. return "unknown". */ + *result_r = "Unknown match for expunge"; + return -1; + } + i_error("Mailbox %s: GUIDs not supported, " + "sync with header hashes instead", + mailbox_get_vname(importer->box)); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + *result_r = "Error, invalid parameters"; + return -1; + } + + if (dsync_mail_get_hdr_hash(importer->cur_mail, + importer->hdr_hash_version, + importer->hashed_headers, &hdr_hash) < 0) { + dsync_mail_error(importer, importer->cur_mail, "hdr-stream"); + *result_r = "Error fetching header stream"; + return -1; + } + if (importer->empty_hdr_workaround && + (dsync_mail_hdr_hash_is_empty(change->hdr_hash) || + dsync_mail_hdr_hash_is_empty(hdr_hash))) { + *result_r = "Empty headers found with workaround enabled - assuming a match"; + return 1; + } else if (strcmp(change->hdr_hash, hdr_hash) == 0) { + *result_r = "Headers hashes match"; + return 1; + } else { + *result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)", + change->hdr_hash, hdr_hash); + return 0; + } +} + +static bool +dsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + const struct dsync_mail_change *local_change; + + if (*change->guid == '\0') { + /* remote doesn't support GUIDs, can't verify expunge */ + *result_r = "GUIDs not supported, can't verify expunge"; + return FALSE; + } + + /* local message is expunged. see if we can find its GUID from + transaction log and check if the GUIDs match. The GUID in + log is a 128bit GUID, so we may need to convert the remote's + GUID string to 128bit GUID first. */ + local_change = hash_table_lookup(importer->local_changes, + POINTER_CAST(change->uid)); + if (local_change == NULL || local_change->guid == NULL) { + *result_r = "Expunged local mail's GUID not found"; + return FALSE; + } + + i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + if (dsync_mail_change_guid_equals(importer, local_change, + change->guid, NULL)) { + importer->last_common_uid = change->uid; + *result_r = "Expunged local mail's GUID matches remote"; + } else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + dsync_mailbox_common_uid_found(importer); + *result_r = "Expunged local mail's GUID doesn't match remote GUID"; + } else { + /* GUID mismatch for two expunged mails. dsync can't update + GUIDs for already expunged messages, so we can't immediately + determine that the rest of the messages are a mismatch. so + for now we'll just skip over this pair. */ + *result_r = "Expunged mails' GUIDs don't match - delaying decision"; + /* NOTE: the return value here doesn't matter, because the only + caller that checks for it never reaches this code path */ + } + return TRUE; +} + +static void +dsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + i_assert(importer->revert_local_changes); + + /* mail exists on remote, but not locally. we'll need to + insert this mail back, which means deleting the whole + mailbox and resyncing. */ + i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally", + mailbox_get_vname(importer->box), + change->uid, change->guid); + importer->delete_mailbox = TRUE; + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; +} + +static void +dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change, + const char **result_r) +{ + int ret; + + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || + ((change->received_timestamp > 0 || + (importer->sync_since_timestamp == 0 && + importer->sync_until_timestamp == 0)) && + (change->virtual_size != UOFF_T_MAX || importer->sync_max_size == 0))); + + /* try to find the matching local mail */ + if (!importer_next_mail(importer, change->uid)) { + /* no more local mails. we can still try to match + expunged mails though. */ + if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* mail doesn't exist remotely either, don't bother + looking it up locally. */ + *result_r = "Expunged mail not found locally"; + return; + } + i_assert(change->guid != NULL); + if (!dsync_mailbox_import_want_change(importer, change, result_r)) + ; + else if (importer->local_uid_next <= change->uid) { + dsync_mailbox_common_uid_found(importer); + *result_r = "Mail's UID is above local UIDNEXT"; + } else if (importer->revert_local_changes) { + dsync_mailbox_revert_missing(importer, change); + *result_r = "Reverting local change by deleting mailbox"; + } else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) { + /* it's unknown if this mail existed locally and was + expunged. since we don't want to lose any mails, + assume that we need to preserve the mail. use the + last message with a matching GUID as the last common + UID. */ + dsync_mailbox_common_uid_found(importer); + } + *result_r = t_strdup_printf("%s - No more local mails found", *result_r); + return; + } + + if (change->guid == NULL) { + /* we can't know if this UID matches */ + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE); + *result_r = "Expunged mail has no GUID, can't verify it"; + return; + } + if (importer->cur_mail->uid == change->uid) { + /* we have a matching local UID. check GUID to see if it's + really the same mail or not */ + if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) { + /* unknown */ + return; + } + if (ret > 0) { + importer->last_common_uid = change->uid; + } else if (!importer->revert_local_changes) { + /* mismatch - found the first non-common UID */ + dsync_mailbox_common_uid_found(importer); + } else { + /* mismatch and we want to revert local changes - + need to delete the mailbox. */ + dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r); + } + return; + } + /* mail exists remotely, but doesn't exist locally. */ + if (!dsync_mailbox_import_want_change(importer, change, result_r)) + return; + if (importer->revert_local_changes && + change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + dsync_mailbox_revert_missing(importer, change); + *result_r = "Reverting local change by deleting mailbox"; + } else { + (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r); + } + *result_r = t_strdup_printf("%s (next local mail UID=%u)", + *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid); +} + +int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change) +{ + const char *result; + + i_assert(!importer->new_uids_assigned); + i_assert(importer->prev_uid < change->uid); + + importer->prev_uid = change->uid; + + if (importer->failed) + return -1; + if (importer->require_full_resync) + return 0; + + if (!importer->last_common_uid_found) { + result = NULL; + dsync_mailbox_find_common_uid(importer, change, &result); + i_assert(result != NULL); + } else { + result = "New mail"; + } + + imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s", + dsync_mail_change_type_names[change->type], + change->guid != NULL ? change->guid : "<unknown>", change->uid, + change->hdr_hash != NULL ? change->hdr_hash : "", result); + + if (importer->failed) + return -1; + if (importer->require_full_resync) + return 0; + + if (importer->last_common_uid_found) { + /* a) uid <= last_common_uid for flag changes and expunges. + this happens only when last_common_uid was originally given + as parameter to importer. + + when we're finding the last_common_uid ourself, + uid>last_common_uid always in here, because + last_common_uid_found=TRUE only after we find the first + mismatch. + + b) uid > last_common_uid for i) new messages, ii) expunges + that were sent "just in case" */ + if (change->uid <= importer->last_common_uid) { + i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE); + } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* ignore */ + return 0; + } else { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE); + } + } else { + /* a) uid < last_common_uid can never happen */ + i_assert(change->uid >= importer->last_common_uid); + /* b) uid = last_common_uid if we've verified that the + messages' GUIDs match so far. + + c) uid > last_common_uid: i) TYPE_EXPUNGE change has + GUID=NULL, so we couldn't verify yet if it matches our + local message, ii) local message is expunged and we couldn't + find its GUID */ + if (change->uid > importer->last_common_uid) { + i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE || + importer->cur_mail == NULL || + change->uid < importer->cur_mail->uid); + } + } + + switch (change->type) { + case DSYNC_MAIL_CHANGE_TYPE_SAVE: + dsync_mailbox_import_save(importer, change); + break; + case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE: + dsync_mailbox_import_expunge(importer, change); + break; + case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE: + i_assert(importer->last_common_uid_found); + dsync_mailbox_import_flag_change(importer, change); + break; + } + return importer->failed ? -1 : 0; +} + +static int +importer_new_mail_final_uid_cmp(struct importer_new_mail *const *newmail1, + struct importer_new_mail *const *newmail2) +{ + if ((*newmail1)->final_uid < (*newmail2)->final_uid) + return -1; + if ((*newmail1)->final_uid > (*newmail2)->final_uid) + return 1; + return 0; +} + +static void +dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer) +{ + struct importer_new_mail *newmail; + uint32_t common_uid_next, new_uid; + + common_uid_next = I_MAX(importer->local_uid_next, + importer->remote_uid_next); + array_foreach_elem(&importer->newmails, newmail) { + if (newmail->skip) { + /* already assigned */ + i_assert(newmail->final_uid != 0); + continue; + } + + /* figure out what UID to use for the mail */ + if (newmail->uid_is_usable) { + /* keep the UID */ + new_uid = newmail->final_uid; + } else if (newmail->link != NULL && + newmail->link->uid_is_usable) { + /* we can use the linked message's UID and expunge + this mail */ + new_uid = newmail->link->final_uid; + } else { + i_assert(!importer->revert_local_changes); + new_uid = common_uid_next++; + imp_debug(importer, "UID %u isn't usable, assigning new UID %u", + newmail->final_uid, new_uid); + } + + newmail->final_uid = new_uid; + if (newmail->link != NULL && newmail->link != newmail) { + /* skip processing the linked mail */ + newmail->link->skip = TRUE; + } + } + importer->last_common_uid = common_uid_next-1; + importer->new_uids_assigned = TRUE; + /* Sort the newmails by their final_uid. This is used for tracking + whether an intermediate commit is allowed. */ + array_sort(&importer->newmails, importer_new_mail_final_uid_cmp); +} + +static int +dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer, + struct mail *mail, uint32_t uid, const char *guid, + struct dsync_mail *dmail_r) +{ + const char *error_field, *errstr; + enum mail_error error; + + if (!mail_set_uid(mail, uid)) + return 0; + + /* NOTE: Errors are logged, but they don't cause the entire import + to fail. */ + if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) { + errstr = mailbox_get_last_internal_error(mail->box, &error); + if (error == MAIL_ERROR_EXPUNGED) + return 0; + + i_error("Mailbox %s: Can't lookup %s for UID=%u: %s", + mailbox_get_vname(importer->box), + error_field, uid, errstr); + return -1; + } + if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) { + dsync_import_unexpected_state(importer, t_strdup_printf( + "Unexpected GUID mismatch (3) for UID=%u: %s != %s", + uid, dmail_r->guid, guid)); + return -1; + } + return 1; +} + +static void +dsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer, + uint32_t uid) +{ + i_assert(importer->search_ctx == NULL); + + if (importer->highest_wanted_uid < uid) + importer->highest_wanted_uid = uid; + array_push_back(&importer->wanted_uids, &uid); +} + +static void +dsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer) +{ + struct importer_new_mail *const *newmails; + unsigned int count; + + newmails = array_get(&importer->newmails, &count); + while (importer->first_unsaved_idx < count) { + if (!newmails[importer->first_unsaved_idx]->saved) + break; + importer->first_unsaved_idx++; + } +} + +static void +dsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer, + struct importer_new_mail *newmail) +{ + dsync_mailbox_import_saved_uid(importer, newmail->final_uid); + newmail->saved = TRUE; + + dsync_mailbox_import_update_first_saved(importer); + importer->saves_since_commit++; + /* we can commit only if all the upcoming mails will have UIDs that + are larger than we're committing. + + Note that if any existing UIDs have been changed, the new UID is + usually higher than anything that is being saved so we can't do + an intermediate commit. It's too much extra work to try to handle + that situation. So here this never happens, because then + array_count(wanted_uids) is always higher than first_unsaved_idx. */ + if (importer->saves_since_commit >= importer->commit_msgs_interval && + importer->first_unsaved_idx == array_count(&importer->wanted_uids)) { + if (dsync_mailbox_import_commit(importer, FALSE) < 0) + importer->failed = TRUE; + importer->saves_since_commit = 0; + } +} + +static bool +dsync_msg_change_uid(struct dsync_mailbox_importer *importer, + uint32_t old_uid, uint32_t new_uid) +{ + struct mail_save_context *save_ctx; + + IMPORTER_DEBUG_CHANGE(importer); + + if (!mail_set_uid(importer->mail, old_uid)) + return FALSE; + + save_ctx = mailbox_save_alloc(importer->ext_trans); + mailbox_save_copy_flags(save_ctx, importer->mail); + mailbox_save_set_uid(save_ctx, new_uid); + if (mailbox_move(&save_ctx, importer->mail) < 0) + return FALSE; + dsync_mailbox_import_saved_uid(importer, new_uid); + return TRUE; +} + +static bool +dsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer, + ARRAY_TYPE(seq_range) *unwanted_uids, + uint32_t wanted_uid) +{ + const struct seq_range *range; + unsigned int count, n; + struct seq_range_iter iter; + uint32_t uid; + + /* optimize by first trying to use the latest UID */ + range = array_get(unwanted_uids, &count); + if (count == 0) + return FALSE; + if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) { + seq_range_array_remove(unwanted_uids, range[count-1].seq2); + return TRUE; + } + if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED) + seq_range_array_remove(unwanted_uids, range[count-1].seq2); + + /* now try to use any of them by iterating through them. (would be + easier&faster to just iterate backwards, but probably too much + trouble to add such API) */ + n = 0; seq_range_array_iter_init(&iter, unwanted_uids); + while (seq_range_array_iter_nth(&iter, n++, &uid)) { + if (dsync_msg_change_uid(importer, uid, wanted_uid)) { + seq_range_array_remove(unwanted_uids, uid); + return TRUE; + } + if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED) + seq_range_array_remove(unwanted_uids, uid); + } + return FALSE; +} + +static bool +dsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer, + struct importer_new_mail *all_newmails, + ARRAY_TYPE(seq_range) *local_uids, + ARRAY_TYPE(seq_range) *wanted_uids) +{ + ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids; + struct seq_range_iter local_iter, wanted_iter; + unsigned int local_n, wanted_n; + uint32_t local_uid, wanted_uid; + struct importer_new_mail *mail; + struct dsync_mail dmail; + + if (array_count(local_uids) == 0) + return FALSE; + + local_n = wanted_n = 0; + seq_range_array_iter_init(&local_iter, local_uids); + seq_range_array_iter_init(&wanted_iter, wanted_uids); + + /* wanted_uids contains UIDs that need to exist at the end. those that + don't already exist in local_uids have a higher UID than any + existing local UID */ + t_array_init(&assigned_uids, array_count(wanted_uids)); + t_array_init(&unwanted_uids, 8); + while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) { + if (seq_range_array_iter_nth(&wanted_iter, wanted_n, + &wanted_uid)) { + if (local_uid == wanted_uid) { + /* we have exactly the UID we want. keep it. */ + seq_range_array_add(&assigned_uids, wanted_uid); + wanted_n++; + continue; + } + i_assert(local_uid < wanted_uid); + } + /* we no longer want this local UID. */ + seq_range_array_add(&unwanted_uids, local_uid); + } + + /* reuse as many existing messages as possible by changing their UIDs */ + while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) { + if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids, + wanted_uid)) + break; + seq_range_array_add(&assigned_uids, wanted_uid); + wanted_n++; + } + + /* expunge all unwanted messages */ + local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids); + while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) { + IMPORTER_DEBUG_CHANGE(importer); + if (mail_set_uid(importer->mail, local_uid)) + mail_expunge(importer->mail); + } + + /* mark mails whose UIDs we got to be skipped over later */ + for (mail = all_newmails; mail != NULL; mail = mail->next) { + if (!mail->skip && + seq_range_exists(&assigned_uids, mail->final_uid)) + mail->skip = TRUE; + } + + if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) { + /* we've assigned all wanted UIDs */ + return TRUE; + } + + /* try to find one existing message that we can use to copy to the + other instances */ + local_n = 0; seq_range_array_iter_init(&local_iter, local_uids); + while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) { + if (dsync_mailbox_import_local_uid(importer, importer->mail, + local_uid, all_newmails->guid, + &dmail) > 0) { + if (dsync_mailbox_save_newmails(importer, &dmail, + all_newmails, FALSE)) + return TRUE; + } + } + return FALSE; +} + +static bool +dsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer, + struct importer_new_mail *all_newmails) +{ + struct dsync_mail dmail; + + if (all_newmails->virtual_all_uid == 0) + return FALSE; + + if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail, + all_newmails->virtual_all_uid, + all_newmails->guid, &dmail) > 0) { + if (dsync_mailbox_save_newmails(importer, &dmail, + all_newmails, FALSE)) + return TRUE; + } + return FALSE; +} + +static bool +dsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer, + struct importer_new_mail *all_newmails) +{ + ARRAY_TYPE(seq_range) local_uids, wanted_uids; + struct dsync_mail_request *request; + struct importer_new_mail *mail; + const char *request_guid = NULL; + uint32_t request_uid = 0; + + i_assert(all_newmails != NULL); + + /* get the list of the current local UIDs and the wanted UIDs. + find the first remote instance that we can request in case there are + no local instances */ + t_array_init(&local_uids, 8); + t_array_init(&wanted_uids, 8); + for (mail = all_newmails; mail != NULL; mail = mail->next) { + if (mail->uid_in_local) + seq_range_array_add(&local_uids, mail->local_uid); + else if (request_guid == NULL) { + if (*mail->guid != '\0') + request_guid = mail->guid; + request_uid = mail->remote_uid; + i_assert(request_uid != 0); + } + if (!mail->skip) + seq_range_array_add(&wanted_uids, mail->final_uid); + } + i_assert(array_count(&wanted_uids) > 0); + + if (!dsync_mailbox_import_try_local(importer, all_newmails, + &local_uids, &wanted_uids) && + !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) { + /* no local instance. request from remote */ + IMPORTER_DEBUG_CHANGE(importer); + if (importer->want_mail_requests) { + request = array_append_space(&importer->mail_requests); + request->guid = request_guid; + request->uid = request_uid; + } + return FALSE; + } + /* successfully handled all the mails locally */ + importer->import_pos++; + return TRUE; +} + +static void +dsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer) +{ + struct mail_search_context *search_ctx; + struct mail_search_args *search_args; + struct importer_new_mail *newmail; + struct mail *mail; + const char *guid; + + if (mailbox_sync(importer->virtual_all_box, 0) < 0) { + i_error("Couldn't sync \\All mailbox '%s': %s", + mailbox_get_vname(importer->virtual_all_box), + mailbox_get_last_internal_error(importer->virtual_all_box, NULL)); + return; + } + + search_args = mail_search_build_init(); + mail_search_build_add_all(search_args); + + importer->virtual_trans = + mailbox_transaction_begin(importer->virtual_all_box, + importer->transaction_flags, + __func__); + search_ctx = mailbox_search_init(importer->virtual_trans, search_args, + NULL, MAIL_FETCH_GUID, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) { + /* ignore errors */ + continue; + } + newmail = hash_table_lookup(importer->import_guids, guid); + if (newmail != NULL && newmail->virtual_all_uid == 0) + newmail->virtual_all_uid = mail->uid; + } + if (mailbox_search_deinit(&search_ctx) < 0) { + i_error("Couldn't search \\All mailbox '%s': %s", + mailbox_get_vname(importer->virtual_all_box), + mailbox_get_last_internal_error(importer->virtual_all_box, NULL)); + } + + importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL); +} + +static void +dsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer) +{ + struct hash_iterate_context *iter; + const char *key; + void *key2; + struct importer_new_mail *mail; + + if (importer->virtual_all_box != NULL && + hash_table_count(importer->import_guids) > 0) { + /* find UIDs in \All mailbox for all wanted GUIDs. */ + dsync_mailbox_import_find_virtual_uids(importer); + } + + iter = hash_table_iterate_init(importer->import_guids); + while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) { + T_BEGIN { + if (dsync_mailbox_import_handle_mail(importer, mail)) + hash_table_remove(importer->import_guids, key); + } T_END; + } + hash_table_iterate_deinit(&iter); + + iter = hash_table_iterate_init(importer->import_uids); + while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) { + T_BEGIN { + if (dsync_mailbox_import_handle_mail(importer, mail)) + hash_table_remove(importer->import_uids, key2); + } T_END; + } + hash_table_iterate_deinit(&iter); + if (!importer->mails_have_guids) { + array_foreach_elem(&importer->newmails, mail) { + if (mail->uid_in_local) + (void)dsync_mailbox_import_handle_mail(importer, mail); + } + } +} + +int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer) +{ + i_assert(!importer->new_uids_assigned); + + if (!importer->last_common_uid_found) { + /* handle pending expunges and flag updates */ + dsync_mailbox_common_uid_found(importer); + } + /* skip common local mails */ + (void)importer_next_mail(importer, importer->last_common_uid+1); + /* if there are any local mails left, add them to newmails list */ + while (importer->cur_mail != NULL && !importer->failed) + (void)dsync_mailbox_try_save(importer, NULL); + + if (importer->search_ctx != NULL) { + if (mailbox_search_deinit(&importer->search_ctx) < 0) { + i_error("Mailbox %s: Search failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } + } + importer->import_count = hash_table_count(importer->import_guids) + + hash_table_count(importer->import_uids); + + dsync_mailbox_import_assign_new_uids(importer); + /* save mails from local sources where possible, + request the rest from remote */ + if (!importer->failed) + dsync_mailbox_import_handle_local_mails(importer); + return importer->failed ? -1 : 0; +} + +const struct dsync_mail_request * +dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer) +{ + const struct dsync_mail_request *requests; + unsigned int count; + + requests = array_get(&importer->mail_requests, &count); + if (importer->mail_request_idx == count) + return NULL; + return &requests[importer->mail_request_idx++]; +} + +static const char *const * +dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change) +{ + ARRAY_TYPE(const_string) keywords; + const char *const *changes; + unsigned int i, count; + + if (!array_is_created(&change->keyword_changes)) + return NULL; + + changes = array_get(&change->keyword_changes, &count); + t_array_init(&keywords, count); + for (i = 0; i < count; i++) { + if (changes[i][0] == KEYWORD_CHANGE_ADD || + changes[i][0] == KEYWORD_CHANGE_FINAL || + changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) { + const char *name = changes[i]+1; + + array_push_back(&keywords, &name); + } + } + if (array_count(&keywords) == 0) + return NULL; + + array_append_zero(&keywords); + return array_front(&keywords); +} + +static void +dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer, + struct mail_save_context *save_ctx, + const struct dsync_mail_change *change) +{ + const char *const *keyword_names; + struct mail_keywords *keywords; + + keyword_names = dsync_mailbox_get_final_keywords(change); + keywords = keyword_names == NULL ? NULL : + mailbox_keywords_create_valid(importer->box, + keyword_names); + mailbox_save_set_flags(save_ctx, change->final_flags, keywords); + if (keywords != NULL) + mailbox_keywords_unref(&keywords); + + if (change->modseq > 1) { + (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE); + mailbox_save_set_min_modseq(save_ctx, change->modseq); + } + /* FIXME: if there already are private flags, they get lost because + saving can't handle updating private index. they get added on the + next sync though. if this is fixed here, set min_pvt_modseq also. */ +} + +static int +dsync_msg_try_copy(struct dsync_mailbox_importer *importer, + struct mail_save_context **save_ctx_p, + struct importer_new_mail **all_newmails_forcopy) +{ + struct importer_new_mail *inst; + + for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) { + if (inst->uid_in_local && !inst->copy_failed && + mail_set_uid(importer->mail, inst->local_uid)) { + if (mailbox_copy(save_ctx_p, importer->mail) < 0) { + enum mail_error error; + const char *errstr; + + errstr = mailbox_get_last_internal_error(importer->box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_warning("Failed to copy mail from UID=%u: " + "%s - falling back to other means", + inst->local_uid, errstr); + } + inst->copy_failed = TRUE; + return -1; + } + *all_newmails_forcopy = inst; + return 1; + } + } + *all_newmails_forcopy = NULL; + return 0; +} + +static void +dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx, + const struct dsync_mail *mail) +{ + if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0') + mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl); + if (mail->pop3_order > 0) + mailbox_save_set_pop3_order(save_ctx, mail->pop3_order); + mailbox_save_set_received_date(save_ctx, mail->received_date, 0); +} + +static struct mail_save_context * +dsync_mailbox_save_init(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail) +{ + struct mail_save_context *save_ctx; + + save_ctx = mailbox_save_alloc(importer->ext_trans); + mailbox_save_set_uid(save_ctx, newmail->final_uid); + if (*mail->guid != '\0') + mailbox_save_set_guid(save_ctx, mail->guid); + if (mail->saved_date != 0) + mailbox_save_set_save_date(save_ctx, mail->saved_date); + dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change); + + if (!mail->minimal_fields) + dsync_mailbox_save_set_nonminimal(save_ctx, mail); + return save_ctx; +} + +static bool +dsync_mailbox_save_body(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *newmail, + struct importer_new_mail **all_newmails_forcopy, + bool remote_mail) +{ + struct mail_save_context *save_ctx; + struct istream *input; + ssize_t ret; + bool save_failed = FALSE; + + /* try to save the mail by copying an existing mail */ + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) { + if (save_ctx == NULL) + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + } + if (ret <= 0 && mail->input_mail != NULL) { + /* copy using the source mail */ + i_assert(mail->input_mail->uid == mail->input_mail_uid); + if (mailbox_copy(&save_ctx, mail->input_mail) == 0) + ret = 1; + else { + enum mail_error error; + const char *errstr; + + errstr = mailbox_get_last_internal_error(importer->box, &error); + if (error != MAIL_ERROR_EXPUNGED) { + i_warning("Failed to copy source UID=%u mail: " + "%s - falling back to regular saving", + mail->input_mail->uid, errstr); + } + ret = -1; + save_ctx = dsync_mailbox_save_init(importer, mail, newmail); + } + + } + if (ret > 0) { + i_assert(save_ctx == NULL); + dsync_mailbox_import_saved_newmail(importer, newmail); + return TRUE; + } + /* fallback to saving from remote stream */ + if (!remote_mail) { + /* the mail isn't remote yet. we were just trying to copy a + local mail to avoid downloading the remote mail. */ + mailbox_save_cancel(&save_ctx); + return FALSE; + } + if (mail->minimal_fields) { + struct dsync_mail mail2; + const char *error_field; + + i_assert(mail->input_mail != NULL); + + if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2, + &error_field) < 0) { + i_error("Mailbox %s: Failed to read mail %s uid=%u: %s", + mailbox_get_vname(importer->box), + error_field, mail->uid, + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + mailbox_save_cancel(&save_ctx); + return TRUE; + } + dsync_mailbox_save_set_nonminimal(save_ctx, &mail2); + input = mail2.input; + } else { + input = mail->input; + } + + if (input == NULL) { + /* it was just expunged in remote, skip it */ + mailbox_save_cancel(&save_ctx); + return TRUE; + } + + i_stream_seek(input, 0); + if (mailbox_save_begin(&save_ctx, input) < 0) { + i_error("Mailbox %s: Saving failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + return TRUE; + } + while ((ret = i_stream_read(input)) > 0 || ret == -2) { + if (mailbox_save_continue(save_ctx) < 0) { + save_failed = TRUE; + ret = -1; + break; + } + } + i_assert(ret == -1); + + if (input->stream_errno != 0) { + i_error("Mailbox %s: read(msg input) failed: %s", + mailbox_get_vname(importer->box), + i_stream_get_error(input)); + mailbox_save_cancel(&save_ctx); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } else if (save_failed) { + i_error("Mailbox %s: Saving failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + mailbox_save_cancel(&save_ctx); + importer->failed = TRUE; + } else { + i_assert(input->eof); + if (mailbox_save_finish(&save_ctx) < 0) { + i_error("Mailbox %s: Saving failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } else { + dsync_mailbox_import_saved_newmail(importer, newmail); + } + } + return TRUE; +} + +static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail, + struct importer_new_mail *all_newmails, + bool remote_mail) +{ + struct importer_new_mail *newmail, *all_newmails_forcopy; + bool ret = TRUE; + + /* if all_newmails list is large, avoid scanning through the + uninteresting ones for each newmail */ + all_newmails_forcopy = all_newmails; + + /* save all instances of the message */ + for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) { + if (!newmail->skip) T_BEGIN { + if (!dsync_mailbox_save_body(importer, mail, newmail, + &all_newmails_forcopy, + remote_mail)) + ret = FALSE; + } T_END; + } + return ret; +} + +int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail) +{ + struct importer_new_mail *all_newmails; + + i_assert(mail->input == NULL || mail->input->seekable); + i_assert(importer->new_uids_assigned); + + if (importer->failed) + return -1; + if (importer->require_full_resync) + return 0; + + imp_debug(importer, "Import mail body for GUID=%s UID=%u", + mail->guid, mail->uid); + + all_newmails = *mail->guid != '\0' ? + hash_table_lookup(importer->import_guids, mail->guid) : + hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid)); + if (all_newmails == NULL) { + if (importer->want_mail_requests) { + i_error("Mailbox %s: Remote sent unwanted message body for " + "GUID=%s UID=%u", + mailbox_get_vname(importer->box), + mail->guid, mail->uid); + } else { + imp_debug(importer, "Skip unwanted mail body for " + "GUID=%s UID=%u", mail->guid, mail->uid); + } + return 0; + } + if (*mail->guid != '\0') + hash_table_remove(importer->import_guids, mail->guid); + else { + hash_table_remove(importer->import_uids, + POINTER_CAST(mail->uid)); + } + importer->import_pos++; + if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE)) + i_unreached(); + return importer->failed ? -1 : 0; +} + +static int +reassign_uids_in_seq_range(struct dsync_mailbox_importer *importer, + const ARRAY_TYPE(seq_range) *unwanted_uids) +{ + struct mailbox *box = importer->box; + const enum mailbox_transaction_flags trans_flags = + importer->transaction_flags | + MAILBOX_TRANSACTION_FLAG_EXTERNAL | + MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS; + struct mailbox_transaction_context *trans; + struct mail_search_args *search_args; + struct mail_search_arg *arg; + struct mail_search_context *search_ctx; + struct mail_save_context *save_ctx; + struct mail *mail; + unsigned int renumber_count = 0; + int ret = 1; + + if (array_count(unwanted_uids) == 0) + return 1; + + if (importer->debug) T_BEGIN { + string_t *str = t_str_new(256); + imap_write_seq_range(str, unwanted_uids); + imp_debug(importer, "Reassign UIDs: %s", str_c(str)); + } T_END; + + search_args = mail_search_build_init(); + arg = mail_search_build_add(search_args, SEARCH_UIDSET); + p_array_init(&arg->value.seqset, search_args->pool, + array_count(unwanted_uids)); + array_append_array(&arg->value.seqset, unwanted_uids); + + trans = mailbox_transaction_begin(box, trans_flags, __func__); + search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL); + mail_search_args_unref(&search_args); + + while (mailbox_search_next(search_ctx, &mail)) { + save_ctx = mailbox_save_alloc(trans); + mailbox_save_copy_flags(save_ctx, mail); + if (mailbox_move(&save_ctx, mail) < 0) { + i_error("Mailbox %s: Couldn't move mail within mailbox: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &importer->mail_error)); + ret = -1; + } else if (ret > 0) { + ret = 0; + } + renumber_count++; + } + if (mailbox_search_deinit(&search_ctx) < 0) { + i_error("Mailbox %s: mail search failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &importer->mail_error)); + ret = -1; + } + + if (mailbox_transaction_commit(&trans) < 0) { + i_error("Mailbox %s: UID reassign commit failed: %s", + mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &importer->mail_error)); + ret = -1; + } + if (ret == 0) { + imp_debug(importer, "Mailbox %s: Change during sync: " + "Renumbered %u of %u unwanted UIDs", + mailbox_get_vname(box), + renumber_count, array_count(unwanted_uids)); + } + return ret; +} + +static int +reassign_unwanted_uids(struct dsync_mailbox_importer *importer, + const char **changes_during_sync_r) +{ + ARRAY_TYPE(seq_range) unwanted_uids; + const uint32_t *wanted_uids, *saved_uids; + uint32_t highest_seen_uid; + unsigned int i, wanted_count, saved_count; + int ret = 0; + + wanted_uids = array_get(&importer->wanted_uids, &wanted_count); + saved_uids = array_get(&importer->saved_uids, &saved_count); + i_assert(wanted_count == saved_count); + if (wanted_count == 0) + return 0; + /* wanted_uids contains the UIDs we tried to save mails with. + if nothing changed during dsync, we should have the expected UIDs + (saved_uids) and all is well. + + if any new messages got inserted during dsync, we'll need to fix up + the UIDs and let the next dsync fix up the other side. for example: + + remote uids = 5,7,9 = wanted_uids + remote uidnext = 12 + locally added new uid=5 -> + saved_uids = 10,7,9 + + we'll now need to reassign UIDs 5 and 10. to be fully future-proof + we'll reassign all UIDs between [original local uidnext .. highest + UID we think we know] that aren't in saved_uids. */ + + /* create uidset for the list of UIDs we don't want to exist */ + t_array_init(&unwanted_uids, 8); + highest_seen_uid = I_MAX(importer->remote_uid_next-1, + importer->highest_wanted_uid); + i_assert(importer->local_uid_next <= highest_seen_uid); + seq_range_array_add_range(&unwanted_uids, + importer->local_uid_next, highest_seen_uid); + for (i = 0; i < wanted_count; i++) { + i_assert(i < wanted_count); + if (saved_uids[i] == wanted_uids[i]) + seq_range_array_remove(&unwanted_uids, saved_uids[i]); + } + + ret = reassign_uids_in_seq_range(importer, &unwanted_uids); + if (ret == 0) { + *changes_during_sync_r = t_strdup_printf( + "%u UIDs changed due to UID conflicts", + seq_range_count(&unwanted_uids)); + /* conflicting changes during sync, revert our last-common-uid + back to a safe value. */ + importer->last_common_uid = importer->local_uid_next - 1; + } + return ret < 0 ? -1 : 0; +} + +static int +dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final) +{ + struct mail_transaction_commit_changes changes; + struct seq_range_iter iter; + uint32_t uid; + unsigned int n; + int ret = importer->failed ? -1 : 0; + + mail_free(&importer->mail); + mail_free(&importer->ext_mail); + + /* commit saves */ + if (mailbox_transaction_commit_get_changes(&importer->ext_trans, + &changes) < 0) { + i_error("Mailbox %s: Save commit failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, &importer->mail_error)); + /* removed wanted_uids that weren't actually saved */ + array_delete(&importer->wanted_uids, + array_count(&importer->saved_uids), + array_count(&importer->wanted_uids) - + array_count(&importer->saved_uids)); + mailbox_transaction_rollback(&importer->trans); + ret = -1; + } else { + /* remember the UIDs that were successfully saved */ + if (importer->debug) T_BEGIN { + string_t *str = t_str_new(256); + imap_write_seq_range(str, &changes.saved_uids); + imp_debug(importer, "Saved UIDs: %s", str_c(str)); + } T_END; + seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0; + while (seq_range_array_iter_nth(&iter, n++, &uid)) + array_push_back(&importer->saved_uids, &uid); + pool_unref(&changes.pool); + + /* commit flag changes and expunges */ + if (mailbox_transaction_commit(&importer->trans) < 0) { + i_error("Mailbox %s: Commit failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + ret = -1; + } + } + + if (!final) + dsync_mailbox_import_transaction_begin(importer); + return ret; +} + +static int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer, + const char **changes_during_sync_r) +{ + struct mailbox_update update; + int ret; + + ret = dsync_mailbox_import_commit(importer, TRUE); + + if (ret == 0) { + /* update mailbox metadata if we successfully saved + everything. */ + i_zero(&update); + update.min_next_uid = importer->remote_uid_next; + update.min_first_recent_uid = + I_MIN(importer->last_common_uid+1, + importer->remote_first_recent_uid); + update.min_highest_modseq = importer->remote_highest_modseq; + update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq; + + imp_debug(importer, "Finish update: min_next_uid=%u " + "min_first_recent_uid=%u min_highest_modseq=%"PRIu64" " + "min_highest_pvt_modseq=%"PRIu64, + update.min_next_uid, update.min_first_recent_uid, + update.min_highest_modseq, + update.min_highest_pvt_modseq); + + if (mailbox_update(importer->box, &update) < 0) { + i_error("Mailbox %s: Update failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + ret = -1; + } + } + + /* sync mailbox to finish flag changes and expunges. */ + if (mailbox_sync(importer->box, 0) < 0) { + i_error("Mailbox %s: Sync failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + ret = -1; + } + if (ret == 0) { + /* give new UIDs to messages that got saved with unwanted UIDs. + do it only if the whole transaction succeeded. */ + if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0) + ret = -1; + } + return ret; +} + +static void +dsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer) +{ + struct hash_iterate_context *iter; + const char *key; + struct importer_new_mail *mail; + + iter = hash_table_iterate_init(importer->import_guids); + while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) { + for (; mail != NULL; mail = mail->next) { + if (mail->skip) + continue; + + i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)", + mailbox_get_vname(importer->box), + mail->guid, mail->remote_uid); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } + } + hash_table_iterate_deinit(&iter); +} + +static void +dsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer) +{ + struct hash_iterate_context *iter; + void *key; + struct importer_new_mail *mail; + + iter = hash_table_iterate_init(importer->import_uids); + while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) { + for (; mail != NULL; mail = mail->next) { + if (mail->skip) + continue; + + i_error("Mailbox %s: Remote didn't send mail UID=%u", + mailbox_get_vname(importer->box), + mail->remote_uid); + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } + } + hash_table_iterate_deinit(&iter); +} + +int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer, + bool success, + uint32_t *last_common_uid_r, + uint64_t *last_common_modseq_r, + uint64_t *last_common_pvt_modseq_r, + uint32_t *last_messages_count_r, + const char **changes_during_sync_r, + bool *require_full_resync_r, + enum mail_error *error_r) +{ + struct dsync_mailbox_importer *importer = *_importer; + struct mailbox_status status; + int ret; + + *_importer = NULL; + *changes_during_sync_r = NULL; + *require_full_resync_r = importer->require_full_resync; + + if ((!success || importer->require_full_resync) && !importer->failed) { + importer->mail_error = MAIL_ERROR_TEMP; + importer->failed = TRUE; + } + + if (!importer->new_uids_assigned && !importer->failed) + dsync_mailbox_import_assign_new_uids(importer); + + if (!importer->failed) { + dsync_mailbox_import_check_missing_guid_imports(importer); + dsync_mailbox_import_check_missing_uid_imports(importer); + } + + if (importer->search_ctx != NULL) { + if (mailbox_search_deinit(&importer->search_ctx) < 0) { + i_error("Mailbox %s: Search failed: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } + } + if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0) + importer->failed = TRUE; + + if (importer->virtual_mail != NULL) + mail_free(&importer->virtual_mail); + if (importer->virtual_trans != NULL) + (void)mailbox_transaction_commit(&importer->virtual_trans); + + hash_table_destroy(&importer->import_guids); + hash_table_destroy(&importer->import_uids); + array_free(&importer->maybe_expunge_uids); + array_free(&importer->maybe_saves); + array_free(&importer->wanted_uids); + array_free(&importer->saved_uids); + array_free(&importer->newmails); + if (array_is_created(&importer->mail_requests)) + array_free(&importer->mail_requests); + + *last_common_uid_r = importer->last_common_uid; + if (*changes_during_sync_r == NULL) { + *last_common_modseq_r = importer->remote_highest_modseq; + *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq; + } else { + /* local changes occurred during dsync. we exported changes up + to local_initial_highestmodseq, so all of the changes have + happened after it. we want the next run to see those changes, + so return it as the last common modseq */ + *last_common_modseq_r = importer->local_initial_highestmodseq; + *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq; + } + if (importer->delete_mailbox) { + if (mailbox_delete(importer->box) < 0) { + i_error("Couldn't delete mailbox %s: %s", + mailbox_get_vname(importer->box), + mailbox_get_last_internal_error(importer->box, + &importer->mail_error)); + importer->failed = TRUE; + } + *last_messages_count_r = 0; + } else { + mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status); + *last_messages_count_r = status.messages; + } + + i_assert(importer->failed == (importer->mail_error != 0)); + ret = importer->failed ? -1 : 0; + *error_r = importer->mail_error; + pool_unref(&importer->pool); + return ret; +} + +const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer) +{ + if (importer->search_ctx != NULL) + return ""; + return t_strdup_printf("%u/%u", importer->import_pos, + importer->import_count); +} diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h new file mode 100644 index 0000000..6a2020e --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-import.h @@ -0,0 +1,63 @@ +#ifndef DSYNC_MAILBOX_IMPORT_H +#define DSYNC_MAILBOX_IMPORT_H + +#include "mail-error.h" + +enum dsync_mailbox_import_flags { + DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN = 0x01, + DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02, + DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x04, + DSYNC_MAILBOX_IMPORT_FLAG_DEBUG = 0x08, + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10, + DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128 = 0x20, + DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY = 0x40, + DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND = 0x100 +}; + +struct mailbox; +struct dsync_mailbox_attribute; +struct dsync_mail; +struct dsync_mail_change; +struct dsync_transaction_log_scan; + +struct dsync_mailbox_importer * +dsync_mailbox_import_init(struct mailbox *box, + struct mailbox *virtual_all_box, + struct dsync_transaction_log_scan *log_scan, + uint32_t last_common_uid, + uint64_t last_common_modseq, + uint64_t last_common_pvt_modseq, + uint32_t remote_uid_next, + uint32_t remote_first_recent_uid, + uint64_t remote_highest_modseq, + uint64_t remote_highest_pvt_modseq, + time_t sync_since_timestamp, + time_t sync_until_timestamp, + uoff_t sync_max_size, + const char *sync_flag, + unsigned int commit_msgs_interval, + enum dsync_mailbox_import_flags flags, + unsigned int hdr_hash_version, + const char *const *hashed_headers); +int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer, + const struct dsync_mailbox_attribute *attr); +int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer, + const struct dsync_mail_change *change); +int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer); +const struct dsync_mail_request * +dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer); +int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer, + const struct dsync_mail *mail); +int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer, + bool success, + uint32_t *last_common_uid_r, + uint64_t *last_common_modseq_r, + uint64_t *last_common_pvt_modseq_r, + uint32_t *last_messages_count_r, + const char **changes_during_sync_r, + bool *require_full_resync_r, + enum mail_error *error_r); + +const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-state.c b/src/doveadm/dsync/dsync-mailbox-state.c new file mode 100644 index 0000000..cca24d4 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-state.c @@ -0,0 +1,127 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "base64.h" +#include "crc32.h" +#include "hash.h" +#include "dsync-mailbox-state.h" + +#define DSYNC_STATE_MAJOR_VERSION 1 +#define DSYNC_STATE_MINOR_VERSION 0 + +#define V0_MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8) +#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8 + 4) + +static void put_uint32(buffer_t *output, uint32_t num) +{ + uint8_t tmp[sizeof(uint32_t)]; + + cpu32_to_le_unaligned(num, tmp); + + buffer_append(output, tmp, sizeof(tmp)); +} + +void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states, + string_t *output) +{ + struct hash_iterate_context *iter; + struct dsync_mailbox_state *state; + uint8_t *guid; + buffer_t *buf = t_buffer_create(128); + uint32_t crc = 0; + + buffer_append_c(buf, DSYNC_STATE_MAJOR_VERSION); + buffer_append_c(buf, DSYNC_STATE_MINOR_VERSION); + buffer_append_c(buf, '\0'); + buffer_append_c(buf, '\0'); + + iter = hash_table_iterate_init(states); + while (hash_table_iterate(iter, states, &guid, &state)) { + buffer_append(buf, state->mailbox_guid, + sizeof(state->mailbox_guid)); + put_uint32(buf, state->last_uidvalidity); + put_uint32(buf, state->last_common_uid); + put_uint32(buf, state->last_common_modseq & 0xffffffffU); + put_uint32(buf, state->last_common_modseq >> 32); + put_uint32(buf, state->last_common_pvt_modseq & 0xffffffffU); + put_uint32(buf, state->last_common_pvt_modseq >> 32); + put_uint32(buf, state->last_messages_count); /* v1 */ + if (buf->used % 3 == 0) { + crc = crc32_data_more(crc, buf->data, buf->used); + base64_encode(buf->data, buf->used, output); + buffer_set_used_size(buf, 0); + } + } + hash_table_iterate_deinit(&iter); + + crc = crc32_data_more(crc, buf->data, buf->used); + put_uint32(buf, crc); + base64_encode(buf->data, buf->used, output); +} + +static int dsync_mailbox_states_retry_import_v0(const buffer_t *buf) +{ + const unsigned char *data = buf->data; + + /* v0 had no version header and no last_messages_count */ + + if ((buf->used-4) % V0_MAILBOX_SIZE != 0 || + le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4)) + return -1; + /* looks like valid v0 format, silently treat it as empty state */ + return 0; +} + +int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states, + pool_t pool, const char *input, + const char **error_r) +{ + struct dsync_mailbox_state *state; + buffer_t *buf; + uint8_t *guid_p; + const unsigned char *data; + unsigned int i, count; + + buf = t_buffer_create(strlen(input)); + if (base64_decode(input, strlen(input), NULL, buf) < 0) { + *error_r = "Invalid base64 data"; + return -1; + } + /* v1: 4 byte header, mailboxes[], CRC32 */ + data = buf->data; + + if (buf->used == 4 && le32_to_cpu_unaligned(data) == 0) { + /* v0: Empty state */ + return 0; + } + if (buf->used < 8) { + *error_r = "Input too small"; + return -1; + } + + if ((buf->used-8) % MAILBOX_SIZE != 0) { + *error_r = "Invalid input size"; + return dsync_mailbox_states_retry_import_v0(buf); + } + + if (le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4)) { + *error_r = "CRC32 mismatch"; + return dsync_mailbox_states_retry_import_v0(buf); + } + data += 4; + count = (buf->used-8) / MAILBOX_SIZE; + + for (i = 0; i < count; i++, data += MAILBOX_SIZE) { + state = p_new(pool, struct dsync_mailbox_state, 1); + memcpy(state->mailbox_guid, data, GUID_128_SIZE); + state->last_uidvalidity = le32_to_cpu_unaligned(data + GUID_128_SIZE); + state->last_common_uid = le32_to_cpu_unaligned(data + GUID_128_SIZE + 4); + state->last_common_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 8); + state->last_common_pvt_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 16); + state->last_messages_count = le32_to_cpu_unaligned(data + GUID_128_SIZE + 24); + guid_p = state->mailbox_guid; + hash_table_insert(states, guid_p, state); + } + return 0; +} diff --git a/src/doveadm/dsync/dsync-mailbox-state.h b/src/doveadm/dsync/dsync-mailbox-state.h new file mode 100644 index 0000000..a01f531 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-state.h @@ -0,0 +1,24 @@ +#ifndef DSYNC_MAILBOX_STATE_H +#define DSYNC_MAILBOX_STATE_H + +#include "guid.h" + +struct dsync_mailbox_state { + guid_128_t mailbox_guid; + uint32_t last_uidvalidity; + uint32_t last_common_uid; + uint64_t last_common_modseq; + uint64_t last_common_pvt_modseq; + uint32_t last_messages_count; + bool changes_during_sync; +}; +ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state); +HASH_TABLE_DEFINE_TYPE(dsync_mailbox_state, uint8_t *, struct dsync_mailbox_state *); + +void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states, + string_t *output); +int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states, + pool_t pool, const char *input, + const char **error_r); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-tree-fill.c b/src/doveadm/dsync/dsync-mailbox-tree-fill.c new file mode 100644 index 0000000..c523e6b --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c @@ -0,0 +1,405 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "guid.h" +#include "str.h" +#include "wildcard-match.h" +#include "mailbox-log.h" +#include "mail-namespace.h" +#include "mail-storage.h" +#include "mailbox-list-iter.h" +#include "dsync-mailbox-tree-private.h" + +static int +dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + struct dsync_mailbox_node **node_r) +{ + struct dsync_mailbox_node *node; + + node = dsync_mailbox_tree_get(tree, info->vname); + if (node->ns == info->ns) + ; + else if (node->ns == NULL) { + i_assert(tree->root.ns == NULL); + node->ns = info->ns; + } else { + i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'", + info->vname, node->ns->prefix, info->ns->prefix); + return -1; + } + *node_r = node; + return 0; +} + +static int +dsync_mailbox_tree_add_exists_node(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + struct dsync_mailbox_node **node_r, + enum mail_error *error_r) +{ + if (dsync_mailbox_tree_add_node(tree, info, node_r) < 0) { + *error_r = MAIL_ERROR_TEMP; + return -1; + } + (*node_r)->existence = DSYNC_MAILBOX_NODE_EXISTS; + return 0; +} + +static int +dsync_mailbox_tree_get_selectable(struct mailbox *box, + struct mailbox_metadata *metadata_r, + struct mailbox_status *status_r) +{ + /* try the fast path */ + if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0) + return -1; + if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0) + return -1; + + i_assert(!guid_128_is_empty(metadata_r->guid)); + if (status_r->uidvalidity != 0) + return 0; + + /* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */ + if (mailbox_sync(box, 0) < 0) + return -1; + if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0) + return -1; + i_assert(status_r->uidvalidity != 0); + return 0; +} + +static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree, + const struct mailbox_info *info, + const guid_128_t box_guid, + enum mail_error *error_r) +{ + struct dsync_mailbox_node *node; + struct mailbox *box; + struct mailbox_metadata metadata; + struct mailbox_status status; + const char *errstr; + enum mail_error error; + int ret = 0; + + if ((info->flags & MAILBOX_NONEXISTENT) != 0) + return 0; + if ((info->flags & MAILBOX_NOSELECT) != 0) { + return !guid_128_is_empty(box_guid) ? 0 : + dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r); + } + + /* get GUID and UIDVALIDITY for selectable mailbox */ + box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY); + if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) { + errstr = mailbox_get_last_internal_error(box, &error); + switch (error) { + case MAIL_ERROR_NOTFOUND: + /* mailbox was just deleted? */ + break; + case MAIL_ERROR_NOTPOSSIBLE: + /* invalid mbox files? ignore */ + break; + default: + i_error("Failed to access mailbox %s: %s", + info->vname, errstr); + *error_r = error; + ret = -1; + } + mailbox_free(&box); + return ret; + } + mailbox_free(&box); + + if (!guid_128_is_empty(box_guid) && + !guid_128_equals(box_guid, metadata.guid)) { + /* unwanted mailbox */ + return 0; + } + if (dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r) < 0) + return -1; + memcpy(node->mailbox_guid, metadata.guid, + sizeof(node->mailbox_guid)); + node->uid_validity = status.uidvalidity; + node->uid_next = status.uidnext; + return 0; +} + +static struct dsync_mailbox_node * +dsync_mailbox_tree_find_sha(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const guid_128_t sha128) +{ + struct dsync_mailbox_node *node; + + if (!hash_table_is_created(tree->name128_hash)) + dsync_mailbox_tree_build_name128_hash(tree); + + node = hash_table_lookup(tree->name128_hash, sha128); + return node == NULL || node->ns != ns ? NULL : node; +} + +static int +dsync_mailbox_tree_add_change_timestamps(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns) +{ + struct dsync_mailbox_node *node; + struct dsync_mailbox_delete *del; + struct mailbox_log *log; + struct mailbox_log_iter *iter; + const struct mailbox_log_record *rec; + const uint8_t *guid_p; + time_t timestamp; + + log = mailbox_list_get_changelog(ns->list); + if (log == NULL) + return 0; + + iter = mailbox_log_iter_init(log); + while ((rec = mailbox_log_iter_next(iter)) != NULL) { + /* For DELETE_MAILBOX the record_guid is the mailbox GUID. + Otherwise it's 128bit SHA1 of the mailbox vname. */ + node = rec->type == MAILBOX_LOG_RECORD_DELETE_MAILBOX ? NULL : + dsync_mailbox_tree_find_sha(tree, ns, rec->mailbox_guid); + + timestamp = mailbox_log_record_get_timestamp(rec); + switch (rec->type) { + case MAILBOX_LOG_RECORD_DELETE_MAILBOX: + guid_p = rec->mailbox_guid; + if (hash_table_lookup(tree->guid_hash, guid_p) != NULL) { + /* mailbox still exists. maybe it was restored + from backup or something. */ + break; + } + del = array_append_space(&tree->deletes); + del->type = DSYNC_MAILBOX_DELETE_TYPE_MAILBOX; + del->timestamp = timestamp; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + case MAILBOX_LOG_RECORD_DELETE_DIR: + if (node != NULL && + node->existence == DSYNC_MAILBOX_NODE_EXISTS) { + /* directory exists again, skip it */ + break; + } + /* we don't know what directory name was deleted, + just its hash. if the name still exists on the other + dsync side, it can match this deletion to the + name. */ + del = array_append_space(&tree->deletes); + del->type = DSYNC_MAILBOX_DELETE_TYPE_DIR; + del->timestamp = timestamp; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + case MAILBOX_LOG_RECORD_CREATE_DIR: + if (node == NULL) { + /* directory has been deleted again, skip it */ + break; + } + /* notify the remote that we want to keep this + directory created (unless remote has a newer delete + timestamp) */ + node->last_renamed_or_created = timestamp; + break; + case MAILBOX_LOG_RECORD_RENAME: + if (node != NULL) + node->last_renamed_or_created = timestamp; + break; + case MAILBOX_LOG_RECORD_SUBSCRIBE: + if (node != NULL) + node->last_subscription_change = timestamp; + break; + case MAILBOX_LOG_RECORD_UNSUBSCRIBE: + if (node != NULL) { + node->last_subscription_change = timestamp; + break; + } + /* The mailbox is already deleted, but it may still + exist on the other side (even the subscription + alone). */ + del = array_append_space(&tree->deletes); + del->type = DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE; + del->timestamp = timestamp; + memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid)); + break; + } + } + if (mailbox_log_iter_deinit(&iter) < 0) { + i_error("Mailbox log iteration for namespace '%s' failed", + ns->prefix); + return -1; + } + return 0; +} + +static int +dsync_mailbox_tree_fix_guid_duplicate(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + struct mailbox *box; + struct mailbox_update update; + struct dsync_mailbox_node *change_node; + const char *change_vname; + int ret = 0; + + i_zero(&update); + guid_128_generate(update.mailbox_guid); + + /* just in case the duplication exists in both sides, + make them choose the same node */ + if (strcmp(dsync_mailbox_node_get_full_name(tree, node1), + dsync_mailbox_node_get_full_name(tree, node2)) <= 0) + change_node = node1; + else + change_node = node2; + + change_vname = dsync_mailbox_node_get_full_name(tree, change_node); + i_error("Duplicate mailbox GUID %s for mailboxes %s and %s - " + "giving a new GUID %s to %s", + guid_128_to_string(node1->mailbox_guid), + dsync_mailbox_node_get_full_name(tree, node1), + dsync_mailbox_node_get_full_name(tree, node2), + guid_128_to_string(update.mailbox_guid), change_vname); + + i_assert(node1->ns != NULL && node2->ns != NULL); + box = mailbox_alloc(change_node->ns->list, change_vname, 0); + if (mailbox_update(box, &update) < 0) { + i_error("Couldn't update mailbox %s GUID: %s", + change_vname, mailbox_get_last_internal_error(box, NULL)); + ret = -1; + } else { + memcpy(change_node->mailbox_guid, update.mailbox_guid, + sizeof(change_node->mailbox_guid)); + } + mailbox_free(&box); + return ret; +} + +static bool +dsync_mailbox_info_is_wanted(const struct mailbox_info *info, + const char *box_name, + const char *const *exclude_mailboxes) +{ + const char *const *info_specialuses; + unsigned int i; + + if (exclude_mailboxes == NULL && + (box_name == NULL || box_name[0] != '\\')) + return TRUE; + + info_specialuses = info->special_use == NULL ? NULL : + t_strsplit(info->special_use, " "); + /* include */ + if (box_name != NULL && box_name[0] == '\\') { + if (info_specialuses == NULL || + !str_array_icase_find(info_specialuses, box_name)) + return FALSE; + } + /* exclude */ + if (exclude_mailboxes == NULL) + return TRUE; + for (i = 0; exclude_mailboxes[i] != NULL; i++) { + const char *exclude = exclude_mailboxes[i]; + + if (exclude[0] == '\\') { + /* special-use */ + if (info_specialuses != NULL && + str_array_icase_find(info_specialuses, exclude)) + return FALSE; + } else { + /* mailbox with wildcards */ + if (wildcard_match(info->vname, exclude)) + return FALSE; + } + } + return TRUE; +} + +int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const char *box_name, + const guid_128_t box_guid, + const char *const *exclude_mailboxes, + enum mail_error *error_r) +{ + const enum mailbox_list_iter_flags list_flags = + /* FIXME: we'll skip symlinks, because we can't handle them + currently. in future we could detect them and create them + by creating the symlink. */ + MAILBOX_LIST_ITER_SKIP_ALIASES | + MAILBOX_LIST_ITER_NO_AUTO_BOXES; + const enum mailbox_list_iter_flags subs_list_flags = + MAILBOX_LIST_ITER_NO_AUTO_BOXES | + MAILBOX_LIST_ITER_SELECT_SUBSCRIBED | + MAILBOX_LIST_ITER_RETURN_NO_FLAGS; + struct mailbox_list_iterate_context *iter; + struct dsync_mailbox_node *node, *dup_node1, *dup_node2; + const struct mailbox_info *info; + const char *list_pattern = + box_name != NULL && box_name[0] != '\\' ? box_name : "*"; + int ret = 0; + + i_assert(mail_namespace_get_sep(ns) == tree->sep); + + /* assign namespace to its root, so it gets copied to children */ + if (ns->prefix_len > 0) { + const char *vname = t_strndup(ns->prefix, ns->prefix_len-1); + node = dsync_mailbox_tree_get(tree, vname); + node->ns = ns; + + struct mailbox_info ns_info = { + .vname = vname, + .ns = ns, + }; + if (dsync_mailbox_tree_add(tree, &ns_info, box_guid, error_r) < 0) + return -1; + } else { + tree->root.ns = ns; + } + + /* first add all of the existing mailboxes */ + iter = mailbox_list_iter_init(ns->list, list_pattern, list_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN { + if (dsync_mailbox_info_is_wanted(info, box_name, + exclude_mailboxes)) { + if (dsync_mailbox_tree_add(tree, info, box_guid, error_r) < 0) + ret = -1; + } + } T_END; + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Mailbox listing for namespace '%s' failed: %s", + ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r)); + ret = -1; + } + + /* add subscriptions */ + iter = mailbox_list_iter_init(ns->list, list_pattern, subs_list_flags); + while ((info = mailbox_list_iter_next(iter)) != NULL) { + if (dsync_mailbox_tree_add_node(tree, info, &node) == 0) + node->subscribed = TRUE; + else { + *error_r = MAIL_ERROR_TEMP; + ret = -1; + } + } + if (mailbox_list_iter_deinit(&iter) < 0) { + i_error("Mailbox listing for namespace '%s' failed: %s", + ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r)); + ret = -1; + } + if (ret < 0) + return -1; + + while (dsync_mailbox_tree_build_guid_hash(tree, &dup_node1, + &dup_node2) < 0) { + if (dsync_mailbox_tree_fix_guid_duplicate(tree, dup_node1, dup_node2) < 0) + return -1; + } + + /* add timestamps */ + if (dsync_mailbox_tree_add_change_timestamps(tree, ns) < 0) + return -1; + return 0; +} diff --git a/src/doveadm/dsync/dsync-mailbox-tree-private.h b/src/doveadm/dsync/dsync-mailbox-tree-private.h new file mode 100644 index 0000000..0614151 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree-private.h @@ -0,0 +1,38 @@ +#ifndef DSYNC_MAILBOX_TREE_PRIVATE_H +#define DSYNC_MAILBOX_TREE_PRIVATE_H + +#include "dsync-mailbox-tree.h" + +struct dsync_mailbox_tree { + pool_t pool; + char sep, sep_str[2], remote_sep, alt_char; + char escape_char, remote_escape_char; + /* root node isn't part of the real mailbox tree. its name is "" and + it has no siblings */ + struct dsync_mailbox_node root; + + unsigned int iter_count; + + ARRAY(struct dsync_mailbox_delete) deletes; + + /* guid_128_t => struct dsync_mailbox_node */ + HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_hash; + HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_remotesep_hash; + HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) guid_hash; +}; + +void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree); + +int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1, + struct dsync_mailbox_node *const *n2); + +void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent); +void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node); + +struct dsync_mailbox_tree * +dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src); +bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c new file mode 100644 index 0000000..e4a50ae --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c @@ -0,0 +1,1473 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "buffer.h" +#include "str.h" +#include "md5.h" +#include "hex-binary.h" +#include "aqueue.h" +#include "hash.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox-tree-private.h" + +#define TEMP_MAX_NAME_LEN 100 +#define TEMP_SUFFIX_MAX_LEN (sizeof("temp-")-1 + 8) +#define TEMP_SUFFIX_FORMAT "temp-%x" + +struct dsync_mailbox_tree_bfs_iter { + struct dsync_mailbox_tree *tree; + + ARRAY(struct dsync_mailbox_node *) queue_arr; + struct aqueue *queue; + struct dsync_mailbox_node *cur; +}; + +struct dsync_mailbox_tree_sync_ctx { + pool_t pool; + struct dsync_mailbox_tree *local_tree, *remote_tree; + enum dsync_mailbox_trees_sync_type sync_type; + enum dsync_mailbox_trees_sync_flags sync_flags; + unsigned int combined_mailboxes_count; + + ARRAY(struct dsync_mailbox_tree_sync_change) changes; + unsigned int change_idx; + bool failed; +}; + +static struct dsync_mailbox_tree_bfs_iter * +dsync_mailbox_tree_bfs_iter_init(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_bfs_iter *iter; + + iter = i_new(struct dsync_mailbox_tree_bfs_iter, 1); + iter->tree = tree; + i_array_init(&iter->queue_arr, 32); + iter->queue = aqueue_init(&iter->queue_arr.arr); + iter->cur = tree->root.first_child; + return iter; +} + +static bool +dsync_mailbox_tree_bfs_iter_next(struct dsync_mailbox_tree_bfs_iter *iter, + struct dsync_mailbox_node **node_r) +{ + if (iter->cur == NULL) { + if (aqueue_count(iter->queue) == 0) + return FALSE; + iter->cur = array_idx_elem(&iter->queue_arr, + aqueue_idx(iter->queue, 0)); + aqueue_delete_tail(iter->queue); + } + *node_r = iter->cur; + + if (iter->cur->first_child != NULL) + aqueue_append(iter->queue, &iter->cur->first_child); + iter->cur = iter->cur->next; + return TRUE; +} + +static void +dsync_mailbox_tree_bfs_iter_deinit(struct dsync_mailbox_tree_bfs_iter **_iter) +{ + struct dsync_mailbox_tree_bfs_iter *iter = *_iter; + + *_iter = NULL; + + aqueue_deinit(&iter->queue); + array_free(&iter->queue_arr); + i_free(iter); +} + +static void +sync_add_dir_change(struct dsync_mailbox_tree_sync_ctx *ctx, + const struct dsync_mailbox_node *node, + enum dsync_mailbox_tree_sync_type type) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *name; + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + + name = dsync_mailbox_node_get_full_name(ctx->local_tree, node); + + change = array_append_space(&ctx->changes); + change->type = type; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); +} + +static void +sync_add_create_change(struct dsync_mailbox_tree_sync_ctx *ctx, + const struct dsync_mailbox_node *node, const char *name) +{ + struct dsync_mailbox_tree_sync_change *change; + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); + memcpy(change->mailbox_guid, node->mailbox_guid, + sizeof(change->mailbox_guid)); + change->uid_validity = node->uid_validity; +} + +static void sort_siblings(ARRAY_TYPE(dsync_mailbox_node) *siblings) +{ + struct dsync_mailbox_node *const *nodes; + unsigned int i, count; + + array_sort(siblings, dsync_mailbox_node_name_cmp); + + nodes = array_get(siblings, &count); + if (count == 0) + return; + + nodes[0]->parent->first_child = nodes[0]; + for (i = 1; i < count; i++) + nodes[i-1]->next = nodes[i]; + nodes[count-1]->next = NULL; +} + +static void +sync_set_node_deleted(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + const uint8_t *guid_p; + + /* for the rest of this sync assume that the mailbox has + already been deleted */ + guid_p = node->mailbox_guid; + hash_table_remove(tree->guid_hash, guid_p); + + node->existence = DSYNC_MAILBOX_NODE_DELETED; + memset(node->mailbox_guid, 0, sizeof(node->mailbox_guid)); + node->uid_validity = 0; +} + +static void +sync_delete_mailbox_node(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, const char *reason) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *name; + + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && + tree == ctx->local_tree) { + i_debug("brain %c: Deleting mailbox '%s' (GUID %s): %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + dsync_mailbox_node_get_full_name(tree, node), + guid_128_to_string(node->mailbox_guid), reason); + } + + if (tree == ctx->local_tree && + node->existence != DSYNC_MAILBOX_NODE_DELETED) { + /* delete this mailbox locally */ + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX; + change->ns = node->ns; + name = dsync_mailbox_node_get_full_name(tree, node); + change->full_name = p_strdup(ctx->pool, name); + memcpy(change->mailbox_guid, node->mailbox_guid, + sizeof(change->mailbox_guid)); + } + sync_set_node_deleted(tree, node); +} + +static void +sync_delete_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, const char *reason) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *other_node; + const uint8_t *guid_p; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + guid_p = node->mailbox_guid; + other_node = hash_table_lookup(other_tree->guid_hash, guid_p); + if (other_node == NULL) { + /* doesn't exist / already deleted */ + } else { + sync_delete_mailbox_node(ctx, other_tree, other_node, reason); + } + sync_delete_mailbox_node(ctx, tree, node, reason); +} + +static void +sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + bool twoway_sync) +{ + struct dsync_mailbox_tree_bfs_iter *iter; + struct dsync_mailbox_node *node, *parent = NULL; + ARRAY_TYPE(dsync_mailbox_node) siblings; + + t_array_init(&siblings, 64); + + iter = dsync_mailbox_tree_bfs_iter_init(tree); + while (dsync_mailbox_tree_bfs_iter_next(iter, &node)) { + if (node->parent != parent) { + sort_siblings(&siblings); + array_clear(&siblings); + parent = node->parent; + } + if (node->existence == DSYNC_MAILBOX_NODE_DELETED && + !dsync_mailbox_node_is_dir(node)) { + if (twoway_sync) { + /* this mailbox was deleted. delete it from the + other side as well */ + sync_delete_mailbox(ctx, tree, node, + "Mailbox has been deleted"); + } else { + /* treat the node as if it didn't exist. it'll + get either recreated or deleted later. in + any case this function must handle all + existence=DELETED mailbox nodes by changing + them into directories (setting GUID=0) or + we'll assert-crash later */ + sync_set_node_deleted(tree, node); + } + } + ctx->combined_mailboxes_count++; + array_push_back(&siblings, &node); + } + sort_siblings(&siblings); + dsync_mailbox_tree_bfs_iter_deinit(&iter); +} + +static bool node_names_equal(const struct dsync_mailbox_node *n1, + const struct dsync_mailbox_node *n2) +{ + while (n1 != NULL && n2 != NULL) { + if (strcmp(n1->name, n2->name) != 0) + return FALSE; + n1 = n1->parent; + n2 = n2->parent; + } + return n1 == NULL && n2 == NULL; +} + +static void +dsync_mailbox_tree_node_attach_sorted(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + struct dsync_mailbox_node **p; + + node->parent = parent; + for (p = &parent->first_child; *p != NULL; p = &(*p)->next) { + if (dsync_mailbox_node_name_cmp(p, &node) > 0) + break; + } + node->next = *p; + *p = node; +} + +static void +dsync_mailbox_tree_node_move_sorted(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + /* detach from old parent */ + dsync_mailbox_tree_node_detach(node); + /* attach to new parent */ + dsync_mailbox_tree_node_attach_sorted(node, parent); +} + +static struct dsync_mailbox_node * +sorted_tree_get(struct dsync_mailbox_tree *tree, const char *name) +{ + struct dsync_mailbox_node *node, *parent, *ret; + + node = ret = dsync_mailbox_tree_get(tree, name); + while (node->parent != NULL && + node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) { + parent = node->parent; + dsync_mailbox_tree_node_detach(node); + dsync_mailbox_tree_node_attach_sorted(node, parent); + node = parent; + } + return ret; +} + +static struct dsync_mailbox_node * +sync_node_new(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node **pos, + struct dsync_mailbox_node *parent, + const struct dsync_mailbox_node *src) +{ + struct dsync_mailbox_node *node; + + node = p_new(tree->pool, struct dsync_mailbox_node, 1); + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + node->name = p_strdup(tree->pool, src->name); + node->sync_temporary_name = src->sync_temporary_name; + node->ns = src->ns; + node->parent = parent; + node->next = *pos; + *pos = node; + return node; +} + +static struct dsync_mailbox_node * +sorted_tree_get_by_node_name(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + struct dsync_mailbox_node *other_node) +{ + const char *parent_name; + + if (other_node == &other_tree->root) + return &tree->root; + + parent_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + return sorted_tree_get(tree, parent_name); +} + +static bool node_has_child(struct dsync_mailbox_node *parent, const char *name) +{ + struct dsync_mailbox_node *node; + + for (node = parent->first_child; node != NULL; node = node->next) { + if (strcmp(node->name, name) == 0) + return TRUE; + } + return FALSE; +} + +static bool +node_has_existent_children(struct dsync_mailbox_node *node, bool dirs_ok) +{ + for (node = node->first_child; node != NULL; node = node->next) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + (dirs_ok || !dsync_mailbox_node_is_dir(node))) + return TRUE; + if (node_has_existent_children(node, dirs_ok)) + return TRUE; + } + return FALSE; +} + +static bool node_is_existent(struct dsync_mailbox_node *node) +{ + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) + return TRUE; + return node_has_existent_children(node, TRUE); +} + +static bool sync_node_is_namespace_prefix(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + const char *full_name; + size_t prefix_len = node->ns == NULL ? 0 : node->ns->prefix_len; + + if (strcmp(node->name, "INBOX") == 0 && node->parent == &tree->root) + return TRUE; + + if (prefix_len == 0) + return FALSE; + + full_name = dsync_mailbox_node_get_full_name(tree, node); + if (node->ns->prefix[prefix_len-1] == mail_namespace_get_sep(node->ns)) + prefix_len--; + return strncmp(full_name, node->ns->prefix, prefix_len) == 0 && + full_name[prefix_len] == '\0'; +} + +static void +sync_rename_node_to_temp(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node *new_parent, + const char **reason_r) +{ + struct dsync_mailbox_tree_sync_change *change; + const char *old_name, *new_name, *p; + char name[TEMP_MAX_NAME_LEN+1]; + buffer_t buf; + size_t prefix_len, max_prefix_len; + unsigned int counter = 1; + + i_assert(!sync_node_is_namespace_prefix(tree, node)); + + buffer_create_from_data(&buf, name, sizeof(name)); + max_prefix_len = TEMP_MAX_NAME_LEN - TEMP_SUFFIX_MAX_LEN - 1; + if (node->sync_temporary_name) { + /* the source name was also a temporary name. drop the + -<suffix> from it */ + p = strrchr(node->name, '-'); + i_assert(p != NULL); + if (max_prefix_len > (size_t)(p - node->name)) + max_prefix_len = p - node->name; + } + str_append_max(&buf, node->name, max_prefix_len); + str_append_c(&buf, '-'); + prefix_len = buf.used; + + do { + str_truncate(&buf, prefix_len); + str_printfa(&buf, TEMP_SUFFIX_FORMAT, counter++); + /* the generated name is quite unlikely to exist, + but check anyway.. */ + } while (node_has_child(node->parent, str_c(&buf))); + + old_name = tree != ctx->local_tree ? NULL : + dsync_mailbox_node_get_full_name(tree, node); + + *reason_r = t_strdup_printf("Renamed '%s' to '%s'", node->name, str_c(&buf)); + node->name = p_strdup(tree->pool, str_c(&buf)); + node->sync_temporary_name = TRUE; + node->last_renamed_or_created = 0; + dsync_mailbox_tree_node_move_sorted(node, new_parent); + + if (tree == ctx->local_tree && node_is_existent(node)) { + /* we're modifying a local tree. remember this change. */ + new_name = dsync_mailbox_node_get_full_name(tree, node); + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + i_assert(strcmp(old_name, "INBOX") != 0); + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, old_name); + change->rename_dest_name = p_strdup(ctx->pool, new_name); + } +} + +static bool node_has_parent(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + for (; node != NULL; node = node->parent) { + if (node == parent) + return TRUE; + } + return FALSE; +} + +static void +sync_rename_node(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *temp_node, + struct dsync_mailbox_node *node, + const struct dsync_mailbox_node *other_node, + const char **reason_r) +{ + struct dsync_mailbox_tree_sync_change *change; + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_node *parent; + const char *name, *other_name; + + i_assert(node != NULL); + i_assert(other_node != NULL); + + /* move/rename node in the tree, so that its position/name is identical + to other_node (in other_tree). temp_node's name is changed to + temporary name (i.e. it assumes that node's name becomes temp_node's + original name) */ + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + + parent = sorted_tree_get_by_node_name(tree, other_tree, + other_node->parent); + if (node_has_parent(parent, node)) { + /* don't introduce a loop. temporarily rename node + under root. */ + sync_rename_node_to_temp(ctx, tree, node, &tree->root, reason_r); + *reason_r = t_strconcat(*reason_r, " (Don't introduce loop)", NULL); + return; + } + sync_rename_node_to_temp(ctx, tree, temp_node, temp_node->parent, reason_r); + + /* get the old name before it's modified */ + name = dsync_mailbox_node_get_full_name(tree, node); + + /* set the new name */ + *reason_r = t_strdup_printf("%s + Renamed '%s' to '%s'", + *reason_r, name, other_node->name); + node->name = p_strdup(tree->pool, other_node->name); + node->sync_temporary_name = other_node->sync_temporary_name; + node->last_renamed_or_created = other_node->last_renamed_or_created; + /* change node's parent if necessary. in any case detach+reattach it + sorted, because the nodes must be sorted by name, and the node's + name (or its parent) changed. */ + dsync_mailbox_tree_node_move_sorted(node, parent); + + if (tree == ctx->local_tree && node_is_existent(node)) { + /* we're modifying a local tree. remember this change. */ + other_name = dsync_mailbox_node_get_full_name(other_tree, other_node); + + i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL); + i_assert(strcmp(name, "INBOX") != 0); + change = array_append_space(&ctx->changes); + change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME; + change->ns = node->ns; + change->full_name = p_strdup(ctx->pool, name); + change->rename_dest_name = p_strdup(ctx->pool, other_name); + } +} + +static int node_mailbox_guids_cmp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + int ret; + + while (node1 != NULL && node2 != NULL) { + if (node1->existence == DSYNC_MAILBOX_NODE_EXISTS && + node2->existence != DSYNC_MAILBOX_NODE_EXISTS) + return -1; + if (node2->existence == DSYNC_MAILBOX_NODE_EXISTS && + node1->existence != DSYNC_MAILBOX_NODE_EXISTS) + return 1; + + ret = memcmp(node1->mailbox_guid, node2->mailbox_guid, + sizeof(node1->mailbox_guid)); + if (ret != 0) + return ret; + + ret = node_mailbox_guids_cmp(node1->first_child, + node2->first_child); + if (ret != 0) + return ret; + node1 = node1->next; + node2 = node2->next; + } + if (node1 == NULL && node2 == NULL) + return 0; + return node1 != NULL ? -1 : 1; +} + +static int node_mailbox_names_cmp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + int ret; + + while (node1 != NULL && node2 != NULL) { + ret = strcmp(node1->name, node2->name); + if (ret != 0) + return ret; + + ret = node_mailbox_names_cmp(node1->first_child, + node2->first_child); + if (ret != 0) + return ret; + node1 = node1->next; + node2 = node2->next; + } + if (node1 == NULL && node2 == NULL) + return 0; + return node1 != NULL ? -1 : 1; +} + +static int node_mailbox_trees_cmp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + int ret; + + ret = node_mailbox_guids_cmp(node1, node2); + if (ret == 0) { + /* only a directory name changed and all the timestamps + are equal. just pick the alphabetically smaller. */ + ret = node_mailbox_names_cmp(node1, node2); + } + i_assert(ret != 0); + return ret; +} + +static time_t nodes_get_timestamp(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + time_t ts = 0; + + /* avoid using temporary names in case all the timestamps are 0 */ + if (node1 != NULL && !node1->sync_temporary_name) + ts = node1->last_renamed_or_created + 1; + if (node2 != NULL && !node2->sync_temporary_name && + ts <= node2->last_renamed_or_created) + ts = node2->last_renamed_or_created + 1; + return ts; +} + +static bool sync_node_is_namespace_root(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + if (node == NULL) + return FALSE; + if (node == &tree->root) + return TRUE; + return sync_node_is_namespace_prefix(tree, node); +} + +static bool ATTR_NULL(3, 4) +sync_rename_lower_ts(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node1, + struct dsync_mailbox_node *remote_node1, + struct dsync_mailbox_node *local_node2, + struct dsync_mailbox_node *remote_node2, + const char **reason_r) +{ + time_t local_ts, remote_ts; + + /* We're scanning the tree at the position of local_node1 + and remote_node2. They have identical names. We also know that + local_node1&remote_node1 and local_node2&remote_node2 are "the same" + either because their GUIDs or (in case of one being a directory) + their childrens' GUIDs match. We don't know where local_node2 or + remote_node1 are located in the mailbox tree, or if they exist + at all. Note that node1 and node2 may be the same node pointers. */ + i_assert(strcmp(local_node1->name, remote_node2->name) == 0); + + if (sync_node_is_namespace_root(ctx->remote_tree, remote_node1) || + sync_node_is_namespace_root(ctx->remote_tree, remote_node2) || + sync_node_is_namespace_root(ctx->local_tree, local_node1) || + sync_node_is_namespace_root(ctx->local_tree, local_node2)) { + local_node1->sync_delayed_guid_change = TRUE; + remote_node2->sync_delayed_guid_change = TRUE; + *reason_r = "Can't rename namespace prefixes - will be merged later"; + return FALSE; + } + + local_ts = nodes_get_timestamp(local_node1, local_node2); + remote_ts = nodes_get_timestamp(remote_node1, remote_node2); + + if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + local_ts = remote_ts+1; + else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + remote_ts = local_ts+1; + + /* The algorithm must be deterministic regardless of the sync direction, + so in case the timestamps are equal we need to resort to looking at + the other data. We'll start by looking at the nodes' mailbox GUIDs, + but if both of them don't exist continue looking into their + children. */ + if (local_ts > remote_ts || + (local_ts == remote_ts && + node_mailbox_trees_cmp(local_node1, remote_node2) < 0)) { + /* local nodes have a higher timestamp. we only want to do + renames where the destination parent is the current node's + (local_node1/remote_node2) parent. */ + + /* Numbers are GUIDs, letters are mailbox names: + + local 1A <-name conflict-> remote 2A + local 2B <- potentially -> remote 1[BC] + + Here we want to preserve the local 1A & 2B names: */ + if (local_node2 == NULL) { + /* local : 1A + remote: 1B, 2A -> 2A-temp, 1A */ + sync_rename_node(ctx, ctx->remote_tree, remote_node2, + remote_node1, local_node1, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: local_node2=NULL)", NULL); + return TRUE; + } else if (remote_node1 == remote_node2) { + /* FIXME: this fixes an infinite loop when in + rename2 test, think it through why :) */ + *reason_r = "local: remote_node1=remote_node2"; + } else if (remote_node1 != NULL) { + /* a) local_node1->parent == local_node2->parent + + local : 1A, 2B + remote: 1B, 2A -> 2A-temp, 1A(, 2B) + remote: 1C, 2A -> 2B, 1A + remote: 1C, 2A, 3B -> 2A-temp, 1A(, 3B-temp, 2B) + + b) local_node1->parent != local_node2->parent + + local : 1X/A, 2Y/B + remote: 1Y/B, 2X/A -> 2X/A-temp, 1X/A(, 2Y/B) + remote: 1Z/C, 2X/A -> 2X/A-temp, 1X/A + remote: 1Z/C, 2X/A, 3Y/B -> 2X/A-temp, 1X/A + + We can handle all of these more easily by simply + always renaming 2 to a temporary name and handling + it when we reach B handling. */ + sync_rename_node(ctx, ctx->remote_tree, remote_node2, + remote_node1, local_node1, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: remote_node1=NULL)", NULL); + return TRUE; + } else if (node_has_parent(local_node1, local_node2) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* node2 is a parent of node1, but it should be + vice versa */ + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node1, local_node2->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: node2 parent of node1)", NULL); + return TRUE; + } else if (node_has_parent(local_node2, local_node1) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* node1 is a parent of node2, but it should be + vice versa */ + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node2, local_node1->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: node1 parent of node2)", NULL); + return TRUE; + } else if (local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node2, remote_node2->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(local: local_node1 exists)", NULL); + return TRUE; + } else { + /* local : 1A, 2B + remote: 2A -> (2B) + remote: 2A, 3B -> (3B-temp, 2B) */ + *reason_r = "local: unchanged"; + } + } else { + /* remote nodes have a higher timestamp */ + if (remote_node1 == NULL) { + sync_rename_node(ctx, ctx->local_tree, local_node1, + local_node2, remote_node2, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: remote_node1=NULL)", NULL); + return TRUE; + } else if (local_node2 == local_node1) { + *reason_r = "remote: remote_node2=remote_node1"; + } else if (local_node2 != NULL) { + sync_rename_node(ctx, ctx->local_tree, local_node1, + local_node2, remote_node2, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: local_node2=NULL)", NULL); + return TRUE; + } else if (node_has_parent(remote_node1, remote_node2) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node1, remote_node2->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: node2 parent of node1)", NULL); + return TRUE; + } else if (node_has_parent(remote_node2, remote_node1) && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + sync_rename_node_to_temp(ctx, ctx->remote_tree, + remote_node2, remote_node1->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: node1 parent of node2)", NULL); + return TRUE; + } else if (remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS) { + sync_rename_node_to_temp(ctx, ctx->local_tree, + local_node1, local_node1->parent, reason_r); + *reason_r = t_strconcat(*reason_r, "(remote: remote_node2 exists)", NULL); + return TRUE; + } else { + *reason_r = "remote: unchanged"; + } + } + return FALSE; +} + +static bool sync_rename_conflict(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node1, + struct dsync_mailbox_node *remote_node2, + const char **reason_r) +{ + struct dsync_mailbox_node *local_node2, *remote_node1; + const uint8_t *guid_p; + bool ret; + + guid_p = local_node1->mailbox_guid; + remote_node1 = hash_table_lookup(ctx->remote_tree->guid_hash, guid_p); + guid_p = remote_node2->mailbox_guid; + local_node2 = hash_table_lookup(ctx->local_tree->guid_hash, guid_p); + + if ((remote_node1 != NULL && remote_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) || + (local_node2 != NULL && local_node2->existence == DSYNC_MAILBOX_NODE_EXISTS)) { + /* conflicting name, rename the one with lower timestamp */ + ret = sync_rename_lower_ts(ctx, local_node1, remote_node1, + local_node2, remote_node2, reason_r); + *reason_r = t_strconcat("conflicting name: ", *reason_r, NULL); + return ret; + } else if (dsync_mailbox_node_is_dir(local_node1) || + dsync_mailbox_node_is_dir(remote_node2)) { + /* one of the nodes is a directory, and the other is a mailbox + that doesn't exist on the other side. there is no conflict, + we'll just need to create the mailbox later. */ + *reason_r = "mailbox not selectable yet"; + return FALSE; + } else { + /* both nodes are mailboxes that don't exist on the other side. + we'll merge these mailboxes together later and change their + GUIDs and UIDVALIDITYs to be the same */ + local_node1->sync_delayed_guid_change = TRUE; + remote_node2->sync_delayed_guid_change = TRUE; + *reason_r = "GUIDs conflict - will be merged later"; + return FALSE; + } +} + +static struct dsync_mailbox_node * +sync_find_branch(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_tree *other_tree, + struct dsync_mailbox_node *dir_node) +{ + struct dsync_mailbox_node *node, *other_node; + const uint8_t *guid_p; + + for (node = dir_node->first_child; node != NULL; node = node->next) { + if (dsync_mailbox_node_is_dir(node)) { + other_node = sync_find_branch(tree, other_tree, node); + if (other_node != NULL) + return other_node; + } else { + guid_p = node->mailbox_guid; + other_node = hash_table_lookup(other_tree->guid_hash, + guid_p); + if (other_node != NULL) + return other_node->parent; + } + } + return NULL; +} + +static bool sync_rename_directory(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node1, + struct dsync_mailbox_node *remote_node2, + const char **reason_r) +{ + struct dsync_mailbox_node *remote_node1, *local_node2; + + /* see if we can find matching mailbox branches based on the nodes' + child mailboxes (with GUIDs). we can then rename the entire branch. + don't try to do this for namespace prefixes though. */ + remote_node1 = sync_find_branch(ctx->local_tree, + ctx->remote_tree, local_node1); + local_node2 = sync_find_branch(ctx->remote_tree, ctx->local_tree, + remote_node2); + if (remote_node1 == NULL || local_node2 == NULL) { + *reason_r = "Directory rename branch not found"; + return FALSE; + } + if (node_names_equal(remote_node1, local_node2)) { + *reason_r = "Directory name paths are equal"; + return FALSE; + } + + return sync_rename_lower_ts(ctx, local_node1, remote_node1, + local_node2, remote_node2, reason_r); +} + +static bool sync_rename_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_parent, + struct dsync_mailbox_node *remote_parent) +{ + struct dsync_mailbox_node **local_nodep = &local_parent->first_child; + struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child; + struct dsync_mailbox_node *local_node, *remote_node; + const char *reason; + string_t *debug = NULL; + bool changed; + + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) + debug = t_str_new(128); + + /* the nodes are sorted by their names. */ + while (*local_nodep != NULL || *remote_nodep != NULL) { + local_node = *local_nodep; + remote_node = *remote_nodep; + + if (local_node == NULL || + (remote_node != NULL && + strcmp(local_node->name, remote_node->name) > 0)) { + /* add a missing local node */ + local_node = sync_node_new(ctx->local_tree, local_nodep, + local_parent, remote_node); + } + if (remote_node == NULL || + strcmp(remote_node->name, local_node->name) > 0) { + /* add a missing remote node */ + remote_node = sync_node_new(ctx->remote_tree, remote_nodep, + remote_parent, local_node); + } + i_assert(strcmp(local_node->name, remote_node->name) == 0); + if (debug != NULL) { + str_truncate(debug, 0); + str_append(debug, "Mailbox "); + dsync_mailbox_node_append_full_name(debug, ctx->local_tree, local_node); + str_printfa(debug, ": local=%s/%ld/%d, remote=%s/%ld/%d", + guid_128_to_string(local_node->mailbox_guid), + (long)local_node->last_renamed_or_created, + local_node->existence, + guid_128_to_string(remote_node->mailbox_guid), + (long)remote_node->last_renamed_or_created, + remote_node->existence); + } + + if (dsync_mailbox_node_is_dir(local_node) && + dsync_mailbox_node_is_dir(remote_node)) { + /* both nodes are directories (or other side is + nonexistent). see if we can match them by their + child mailboxes */ + changed = sync_rename_directory(ctx, local_node, remote_node, &reason); + } else if (dsync_mailbox_node_guids_equal(local_node, + remote_node)) { + /* mailboxes are equal, no need to rename */ + reason = "Mailboxes are equal"; + changed = FALSE; + } else { + /* mailbox naming conflict */ + changed = sync_rename_conflict(ctx, local_node, + remote_node, &reason); + } + /* handle children, if there are any */ + if (debug != NULL) { + i_debug("brain %c: %s: %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + str_c(debug), reason); + } + + if (!changed) T_BEGIN { + changed = sync_rename_mailboxes(ctx, local_node, + remote_node); + } T_END; + if (changed) + return TRUE; + + local_nodep = &local_node->next; + remote_nodep = &remote_node->next; + } + return FALSE; +} + +static bool mailbox_node_hash_first_child(struct dsync_mailbox_node *node, + struct md5_context *md5) +{ + for (node = node->first_child; node != NULL; node = node->next) { + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) { + md5_update(md5, node->mailbox_guid, + sizeof(node->mailbox_guid)); + md5_update(md5, node->name, strlen(node->name)); + return TRUE; + } + if (node->first_child != NULL) { + if (mailbox_node_hash_first_child(node, md5)) + return TRUE; + } + } + return FALSE; +} + +static const char * +mailbox_node_generate_suffix(struct dsync_mailbox_node *node) +{ + struct md5_context md5; + unsigned char digest[MD5_RESULTLEN]; + + if (!dsync_mailbox_node_is_dir(node)) + return guid_128_to_string(node->mailbox_guid); + + md5_init(&md5); + if (!mailbox_node_hash_first_child(node, &md5)) + i_unreached(); /* we would have deleted it */ + md5_final(&md5, digest); + return binary_to_hex(digest, sizeof(digest)); +} + +static void suffix_inc(string_t *str) +{ + char *data; + size_t i; + + data = str_c_modifiable(str) + str_len(str)-1; + for (i = str_len(str); i > 0; i--, data--) { + if ((*data >= '0' && *data < '9') || + (*data >= 'a' && *data < 'f')) { + *data += 1; + return; + } else if (*data == '9') { + *data = 'a'; + return; + } else if (*data != 'f') { + i_unreached(); + } + } + i_unreached(); +} + +static void +sync_rename_temp_mailbox_node(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + const char **reason_r) +{ + const char *p, *new_suffix; + string_t *str = t_str_new(256); + size_t max_prefix_len; + + /* The name is currently <oldname>-<temp>. Both sides need to + use equivalent names, so we'll replace the <temp> if possible + with a) mailbox GUID, b) sha1 of childrens' (GUID|name)s. In the + very unlikely case of such name already existing, just increase + the last letters until it's not found. */ + new_suffix = mailbox_node_generate_suffix(node); + + p = strrchr(node->name, '-'); + i_assert(p != NULL); + p++; + max_prefix_len = TEMP_MAX_NAME_LEN - strlen(new_suffix) - 1; + if (max_prefix_len > (size_t)(p-node->name)) + max_prefix_len = p-node->name; + str_append_max(str, node->name, max_prefix_len); + str_append(str, new_suffix); + while (node_has_child(node->parent, str_c(str))) + suffix_inc(str); + + *reason_r = t_strdup_printf("Renamed '%s' to '%s'", + dsync_mailbox_node_get_full_name(tree, node), str_c(str)); + node->name = p_strdup(tree->pool, str_c(str)); + + dsync_mailbox_tree_node_move_sorted(node, node->parent); + node->sync_temporary_name = FALSE; +} + +static void +sync_rename_delete_node_dirs(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_node *child; + + for (child = node->first_child; child != NULL; child = child->next) + sync_rename_delete_node_dirs(ctx, tree, child); + + if (tree == ctx->local_tree && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL && + node->existence != DSYNC_MAILBOX_NODE_NONEXISTENT) { + sync_add_dir_change(ctx, node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR); + } + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + node->sync_temporary_name = FALSE; +} + +static bool +sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, bool *renames_r) +{ + const char *reason; + + for (; node != NULL; node = node->next) { + while (sync_rename_temp_mailboxes(ctx, tree, node->first_child, renames_r)) ; + + if (!node->sync_temporary_name) { + } else if (dsync_mailbox_node_is_dir(node) && + (node->first_child == NULL || + !node_has_existent_children(node, FALSE))) { + /* we can just delete this directory and + any child directories it may have */ + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) { + i_debug("brain %c: %s mailbox %s: Delete directory-only tree", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + tree == ctx->local_tree ? "local" : "remote", + dsync_mailbox_node_get_full_name(tree, node)); + } + sync_rename_delete_node_dirs(ctx, tree, node); + } else { + T_BEGIN { + *renames_r = TRUE; + sync_rename_temp_mailbox_node(tree, node, &reason); + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) { + i_debug("brain %c: %s mailbox %s: %s", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S', + tree == ctx->local_tree ? "local" : "remote", + dsync_mailbox_node_get_full_name(tree, node), reason); + } + } T_END; + return TRUE; + } + } + return FALSE; +} + +static int +dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx, + bool *renames_r) +{ + unsigned int max_renames, count = 0; + bool changed; + + *renames_r = FALSE; + + max_renames = ctx->combined_mailboxes_count * 3; + do { + T_BEGIN { + changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root, + &ctx->remote_tree->root); + } T_END; + if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 && + changed) { + i_debug("brain %c: -- Mailbox renamed, restart sync --", + (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S'); + } + } while (changed && ++count <= max_renames); + + if (changed) { + i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting"); + return -1; + } + + while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root, renames_r)) ; + while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root, renames_r)) ; + return 0; +} + +static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node, + const struct dsync_mailbox_node *wanted_node, + const char **reason_r) +{ + if (wanted_node->existence != DSYNC_MAILBOX_NODE_EXISTS) { + *reason_r = wanted_node->existence == DSYNC_MAILBOX_NODE_DELETED ? + "Mailbox has been deleted" : "Mailbox doesn't exist"; + return TRUE; + } + if (node->uid_validity != wanted_node->uid_validity) { + *reason_r = t_strdup_printf("UIDVALIDITY changed (%u -> %u)", + wanted_node->uid_validity, + node->uid_validity); + return TRUE; + } + if (node->uid_next > wanted_node->uid_next) { + /* we can't lower the UIDNEXT */ + *reason_r = t_strdup_printf("UIDNEXT is too high (%u > %u)", + node->uid_next, + wanted_node->uid_next); + return TRUE; + } + if (memcmp(node->mailbox_guid, wanted_node->mailbox_guid, + sizeof(node->mailbox_guid)) != 0) { + *reason_r = "GUID changed"; + return TRUE; + } + return FALSE; +} + +static void +sync_delete_wrong_mailboxes_branch(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_tree *wanted_tree, + struct dsync_mailbox_node *node, + const struct dsync_mailbox_node *wanted_node) +{ + const char *reason; + int ret; + + while (node != NULL && wanted_node != NULL) { + if (node->first_child != NULL) { + sync_delete_wrong_mailboxes_branch(ctx, + tree, wanted_tree, + node->first_child, wanted_node->first_child); + } + ret = strcmp(node->name, wanted_node->name); + if (ret < 0) { + /* node shouldn't exist */ + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) { + sync_delete_mailbox_node(ctx, tree, node, + "Mailbox doesn't exist"); + } + node = node->next; + } else if (ret > 0) { + /* wanted_node doesn't exist. it's created later. */ + wanted_node = wanted_node->next; + } else T_BEGIN { + if (sync_is_wrong_mailbox(node, wanted_node, &reason) && + node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) + sync_delete_mailbox_node(ctx, tree, node, reason); + node = node->next; + wanted_node = wanted_node->next; + } T_END; + } + for (; node != NULL; node = node->next) { + /* node and its children shouldn't exist */ + if (node->first_child != NULL) { + sync_delete_wrong_mailboxes_branch(ctx, + tree, wanted_tree, + node->first_child, NULL); + } + if (node->existence == DSYNC_MAILBOX_NODE_EXISTS && + !dsync_mailbox_node_is_dir(node)) + sync_delete_mailbox_node(ctx, tree, node, "Mailbox doesn't exist"); + } +} + +static void +sync_delete_wrong_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_tree *wanted_tree) +{ + sync_delete_wrong_mailboxes_branch(ctx, tree, wanted_tree, + tree->root.first_child, + wanted_tree->root.first_child); +} + +static void sync_create_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree *other_tree; + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node, *other_node; + const char *name; + const uint8_t *guid_p; + + other_tree = tree == ctx->local_tree ? + ctx->remote_tree : ctx->local_tree; + + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + /* make sure the renaming handled everything */ + i_assert(!node->sync_temporary_name); + if (dsync_mailbox_node_is_dir(node)) + continue; + + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + + guid_p = node->mailbox_guid; + other_node = hash_table_lookup(other_tree->guid_hash, guid_p); + if (other_node == NULL) + other_node = sorted_tree_get(other_tree, name); + if (dsync_mailbox_node_is_dir(other_node)) { + /* create a missing mailbox */ + other_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + other_node->ns = node->ns; + other_node->uid_validity = node->uid_validity; + memcpy(other_node->mailbox_guid, node->mailbox_guid, + sizeof(other_node->mailbox_guid)); + if (other_tree == ctx->local_tree) + sync_add_create_change(ctx, other_node, name); + } else if (!guid_128_equals(node->mailbox_guid, + other_node->mailbox_guid)) { + /* mailbox with same name exists both locally and + remotely, but they have different GUIDs and neither + side has the other's GUID. typically this means that + both sides had autocreated some mailboxes (e.g. + INBOX). we'll just change the GUID for one of + them. */ + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + i_assert(node->ns == other_node->ns); + + if (other_tree == ctx->local_tree) + sync_add_create_change(ctx, node, name); + } else { + /* existing mailbox. mismatching UIDVALIDITY is handled + later while syncing the mailbox. */ + i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS); + i_assert(node->ns == other_node->ns); + } + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +static void +sync_subscription(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_node, + struct dsync_mailbox_node *remote_node) +{ + bool use_local; + + if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + use_local = TRUE; + else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + use_local = FALSE; + else if (local_node->last_subscription_change > remote_node->last_subscription_change) + use_local = TRUE; + else if (local_node->last_subscription_change < remote_node->last_subscription_change) + use_local = FALSE; + else { + /* local and remote have equal timestamps. prefer to subscribe + rather than unsubscribe. */ + use_local = local_node->subscribed; + } + if (use_local) { + /* use local subscription state */ + remote_node->subscribed = local_node->subscribed; + } else { + /* use remote subscription state */ + local_node->subscribed = remote_node->subscribed; + sync_add_dir_change(ctx, local_node, local_node->subscribed ? + DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE : + DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE); + } +} + +static void sync_mailbox_child_dirs(struct dsync_mailbox_tree_sync_ctx *ctx, + struct dsync_mailbox_node *local_parent, + struct dsync_mailbox_node *remote_parent) +{ + struct dsync_mailbox_node **local_nodep = &local_parent->first_child; + struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child; + struct dsync_mailbox_node *local_node, *remote_node; + int ret; + + /* NOTE: the nodes are always sorted. renaming created all of the + interesting nodes, but it may have left some extra nonexistent nodes + lying around, which we will delete. */ + while (*local_nodep != NULL && *remote_nodep != NULL) { + local_node = *local_nodep; + remote_node = *remote_nodep; + + ret = strcmp(local_node->name, remote_node->name); + if (ret < 0) { + i_assert(!node_is_existent(local_node)); + *local_nodep = local_node->next; + continue; + } + if (ret > 0) { + i_assert(!node_is_existent(remote_node)); + *remote_nodep = remote_node->next; + continue; + } + + if (local_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + remote_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + /* create to remote */ + remote_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + } + if (remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + local_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* create to local */ + local_node->existence = DSYNC_MAILBOX_NODE_EXISTS; + sync_add_dir_change(ctx, local_node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR); + } + + /* create/delete child dirs */ + sync_mailbox_child_dirs(ctx, local_node, remote_node); + + if (local_node->subscribed != remote_node->subscribed) + sync_subscription(ctx, local_node, remote_node); + + if (local_node->existence == DSYNC_MAILBOX_NODE_DELETED && + !node_has_existent_children(local_node, TRUE) && + remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) { + /* delete from remote */ + i_assert(!node_has_existent_children(remote_node, TRUE)); + remote_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + } + if (remote_node->existence == DSYNC_MAILBOX_NODE_DELETED && + !node_has_existent_children(remote_node, TRUE) && + local_node->existence == DSYNC_MAILBOX_NODE_EXISTS && + ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) { + /* delete from local */ + i_assert(!node_has_existent_children(local_node, TRUE)); + local_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + sync_add_dir_change(ctx, local_node, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR); + } + + local_nodep = &local_node->next; + remote_nodep = &remote_node->next; + } + while (*local_nodep != NULL) { + i_assert(!node_is_existent(*local_nodep)); + *local_nodep = (*local_nodep)->next; + } + while (*remote_nodep != NULL) { + i_assert(!node_is_existent(*remote_nodep)); + *remote_nodep = (*remote_nodep)->next; + } +} + +static void sync_mailbox_dirs(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + sync_mailbox_child_dirs(ctx, &ctx->local_tree->root, + &ctx->remote_tree->root); +} + +static void +dsync_mailbox_tree_update_child_timestamps(struct dsync_mailbox_node *node, + time_t parent_timestamp) +{ + time_t ts; + + if (node->last_renamed_or_created < parent_timestamp) + node->last_renamed_or_created = parent_timestamp; + ts = node->last_renamed_or_created; + + for (node = node->first_child; node != NULL; node = node->next) + dsync_mailbox_tree_update_child_timestamps(node, ts); +} + +struct dsync_mailbox_tree_sync_ctx * +dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, + struct dsync_mailbox_tree *remote_tree, + enum dsync_mailbox_trees_sync_type sync_type, + enum dsync_mailbox_trees_sync_flags sync_flags) +{ + struct dsync_mailbox_tree_sync_ctx *ctx; + unsigned int rename_counter = 0; + bool renames; + pool_t pool; + + i_assert(hash_table_is_created(local_tree->guid_hash)); + i_assert(hash_table_is_created(remote_tree->guid_hash)); + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox trees sync", + 1024*64); + ctx = p_new(pool, struct dsync_mailbox_tree_sync_ctx, 1); + ctx->pool = pool; + ctx->local_tree = local_tree; + ctx->remote_tree = remote_tree; + ctx->sync_type = sync_type; + ctx->sync_flags = sync_flags; + i_array_init(&ctx->changes, 128); + +again: + renames = FALSE; + ctx->combined_mailboxes_count = 0; + sync_tree_sort_and_delete_mailboxes(ctx, remote_tree, + sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + sync_tree_sort_and_delete_mailboxes(ctx, local_tree, + sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY); + + dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0); + dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0); + if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) { + if (dsync_mailbox_tree_handle_renames(ctx, &renames) < 0) { + ctx->failed = TRUE; + return ctx; + } + } + + /* if we're not doing a two-way sync, delete now any mailboxes, which + a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY, + c) has a too high UIDNEXT */ + if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + sync_delete_wrong_mailboxes(ctx, remote_tree, local_tree); + else if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + sync_delete_wrong_mailboxes(ctx, local_tree, remote_tree); + + if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) + sync_create_mailboxes(ctx, remote_tree); + if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) + sync_create_mailboxes(ctx, local_tree); + if (renames && rename_counter++ <= ctx->combined_mailboxes_count*3) { + /* this rename algorithm is just horrible. we're retrying this + because the final sync_rename_temp_mailbox_node() calls + give different names to local & remote mailbox trees. + something's not right here, but this looping is better than + a crash in sync_mailbox_dirs() due to trees not matching. */ + goto again; + } + sync_mailbox_dirs(ctx); + return ctx; +} + +const struct dsync_mailbox_tree_sync_change * +dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx) +{ + if (ctx->change_idx == array_count(&ctx->changes)) + return NULL; + return array_idx(&ctx->changes, ctx->change_idx++); +} + +int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx) +{ + struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx; + int ret = ctx->failed ? -1 : 0; + + *_ctx = NULL; + + array_free(&ctx->changes); + pool_unref(&ctx->pool); + return ret; +} diff --git a/src/doveadm/dsync/dsync-mailbox-tree.c b/src/doveadm/dsync/dsync-mailbox-tree.c new file mode 100644 index 0000000..e34f335 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree.c @@ -0,0 +1,554 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "hash.h" +#include "str.h" +#include "sort.h" +#include "mailbox-list-private.h" +#include "dsync-mailbox-tree-private.h" + + +struct dsync_mailbox_tree_iter { + struct dsync_mailbox_tree *tree; + + struct dsync_mailbox_node *cur; + string_t *name; +}; + +struct dsync_mailbox_tree * +dsync_mailbox_tree_init(char sep, char escape_char, char alt_char) +{ + struct dsync_mailbox_tree *tree; + pool_t pool; + + i_assert(sep != '\0'); + i_assert(alt_char != '\0'); + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox tree", 4096); + tree = p_new(pool, struct dsync_mailbox_tree, 1); + tree->pool = pool; + tree->sep = tree->sep_str[0] = sep; + tree->escape_char = escape_char; + tree->alt_char = alt_char; + tree->root.name = ""; + i_array_init(&tree->deletes, 32); + return tree; +} + +void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **_tree) +{ + struct dsync_mailbox_tree *tree = *_tree; + + *_tree = NULL; + hash_table_destroy(&tree->name128_hash); + hash_table_destroy(&tree->guid_hash); + array_free(&tree->deletes); + pool_unref(&tree->pool); +} + +static struct dsync_mailbox_node * +dsync_mailbox_node_find(struct dsync_mailbox_node *nodes, const char *name) +{ + for (; nodes != NULL; nodes = nodes->next) { + if (strcmp(name, nodes->name) == 0) + return nodes; + } + return NULL; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree, + const char *full_name) +{ + struct dsync_mailbox_node *node = &tree->root; + + T_BEGIN { + const char *const *path; + + path = t_strsplit(full_name, tree->sep_str); + for (; *path != NULL && node != NULL; path++) + node = dsync_mailbox_node_find(node->first_child, *path); + } T_END; + return node; +} + +void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node, + struct dsync_mailbox_node *parent) +{ + node->parent = parent; + node->next = parent->first_child; + parent->first_child = node; +} + +void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node) +{ + struct dsync_mailbox_node **p; + + for (p = &node->parent->first_child;; p = &(*p)->next) { + if (*p == node) { + *p = node->next; + break; + } + } + node->parent = NULL; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name) +{ + struct dsync_mailbox_node *parent = NULL, *node = &tree->root; + + i_assert(tree->iter_count == 0); + + T_BEGIN { + const char *const *path; + + /* find the existing part */ + path = t_strsplit(full_name, tree->sep_str); + for (; *path != NULL; path++) { + parent = node; + node = dsync_mailbox_node_find(node->first_child, *path); + if (node == NULL) + break; + } + /* create the rest */ + for (; *path != NULL; path++) { + node = p_new(tree->pool, struct dsync_mailbox_node, 1); + node->name = p_strdup(tree->pool, *path); + node->ns = parent->ns; + dsync_mailbox_tree_node_attach(node, parent); + parent = node; + } + } T_END; + return node; +} + +static void +node_get_full_name_recurse(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node, string_t *str) +{ + if (node->parent != &tree->root) + node_get_full_name_recurse(tree, node->parent, str); + + str_append(str, node->name); + str_append_c(str, tree->sep); +} + +const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node) +{ + string_t *str = t_str_new(128); + dsync_mailbox_node_append_full_name(str, tree, node); + return str_c(str); +} + +void dsync_mailbox_node_append_full_name(string_t *str, + const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node) +{ + i_assert(node->parent != NULL); + + node_get_full_name_recurse(tree, node, str); + /* remove the trailing separator */ + str_truncate(str, str_len(str)-1); +} + +void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, + const struct dsync_mailbox_node *src) +{ + memcpy(dest->mailbox_guid, src->mailbox_guid, + sizeof(dest->mailbox_guid)); + dest->uid_validity = src->uid_validity; + dest->uid_next = src->uid_next; + dest->existence = src->existence; + dest->last_renamed_or_created = src->last_renamed_or_created; + dest->subscribed = src->subscribed; + dest->last_subscription_change = src->last_subscription_change; +} + +struct dsync_mailbox_tree_iter * +dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + + iter = i_new(struct dsync_mailbox_tree_iter, 1); + iter->tree = tree; + iter->name = str_new(default_pool, 128); + iter->cur = &tree->root; + + tree->iter_count++; + return iter; +} + +static size_t node_get_full_name_length(struct dsync_mailbox_node *node) +{ + if (node->parent->parent == NULL) + return strlen(node->name); + else { + return strlen(node->name) + 1 + + node_get_full_name_length(node->parent); + } +} + +bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter, + const char **full_name_r, + struct dsync_mailbox_node **node_r) +{ + size_t len; + + if (iter->cur->first_child != NULL) + iter->cur = iter->cur->first_child; + else { + while (iter->cur->next == NULL) { + if (iter->cur == &iter->tree->root) + return FALSE; + iter->cur = iter->cur->parent; + } + iter->cur = iter->cur->next; + len = iter->cur->parent == &iter->tree->root ? 0 : + node_get_full_name_length(iter->cur->parent); + str_truncate(iter->name, len); + } + if (str_len(iter->name) > 0) + str_append_c(iter->name, iter->tree->sep); + str_append(iter->name, iter->cur->name); + *full_name_r = str_c(iter->name); + *node_r = iter->cur; + return TRUE; +} + +void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **_iter) +{ + struct dsync_mailbox_tree_iter *iter = *_iter; + + *_iter = NULL; + + i_assert(iter->tree->iter_count > 0); + iter->tree->iter_count--; + + str_free(&iter->name); + i_free(iter); +} + +void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + guid_128_t *sha128; + uint8_t *guid_p; + + i_assert(!hash_table_is_created(tree->name128_hash)); + + hash_table_create(&tree->name128_hash, + tree->pool, 0, guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + sha128 = p_new(tree->pool, guid_128_t, 1); + mailbox_name_get_sha128(name, *sha128); + guid_p = *sha128; + hash_table_insert(tree->name128_hash, guid_p, node); + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +static const char * +convert_name_to_remote_sep(struct dsync_mailbox_tree *tree, const char *name) +{ + string_t *str = t_str_new(128); + + char remote_escape_chars[3] = { + tree->remote_escape_char, + tree->remote_sep, + '\0' + }; + + for (;;) { + const char *end = strchr(name, tree->sep); + const char *name_part = end == NULL ? name : + t_strdup_until(name, end++); + + if (tree->escape_char != '\0') + mailbox_list_name_unescape(&name_part, tree->escape_char); + if (remote_escape_chars[0] != '\0') { + /* The local name can be fully escaped to remote + name and back. */ + mailbox_list_name_escape(name_part, remote_escape_chars, + str); + } else { + /* There is no remote escape char, so for conflicting + separator use the alt_char. */ + for (; *name_part != '\0'; name_part++) { + if (*name_part == tree->remote_sep) + str_append_c(str, tree->alt_char); + else + str_append_c(str, *name_part); + } + } + if (end == NULL) + break; + str_append_c(str, tree->remote_sep); + } + return str_c(str); +} + +static void +dsync_mailbox_tree_build_name128_remotesep_hash(struct dsync_mailbox_tree *tree) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node; + const char *name; + guid_128_t *sha128; + uint8_t *guid_p; + + i_assert(tree->sep != tree->remote_sep); + i_assert(!hash_table_is_created(tree->name128_remotesep_hash)); + + hash_table_create(&tree->name128_remotesep_hash, tree->pool, 0, + guid_128_hash, guid_128_cmp); + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + sha128 = p_new(tree->pool, guid_128_t, 1); + T_BEGIN { + const char *remote_name = + convert_name_to_remote_sep(tree, name); + mailbox_name_get_sha128(remote_name, *sha128); + } T_END; + guid_p = *sha128; + hash_table_insert(tree->name128_remotesep_hash, guid_p, node); + } + dsync_mailbox_tree_iter_deinit(&iter); +} + +int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node **old_node_r) +{ + struct dsync_mailbox_node *old_node; + uint8_t *guid = node->mailbox_guid; + + if (guid_128_is_empty(node->mailbox_guid)) + return 0; + + *old_node_r = old_node = hash_table_lookup(tree->guid_hash, guid); + if (old_node == NULL) + hash_table_insert(tree->guid_hash, guid, node); + else if (old_node != node) + return -1; + return 0; +} + +int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node **dup_node1_r, + struct dsync_mailbox_node **dup_node2_r) +{ + struct dsync_mailbox_tree_iter *iter; + struct dsync_mailbox_node *node, *old_node; + const char *name; + int ret = 0; + + if (!hash_table_is_created(tree->guid_hash)) { + hash_table_create(&tree->guid_hash, tree->pool, 0, + guid_128_hash, guid_128_cmp); + } + iter = dsync_mailbox_tree_iter_init(tree); + while (dsync_mailbox_tree_iter_next(iter, &name, &node)) { + if (dsync_mailbox_tree_guid_hash_add(tree, node, &old_node) < 0) { + *dup_node1_r = node; + *dup_node2_r = old_node; + ret = -1; + } + } + dsync_mailbox_tree_iter_deinit(&iter); + return ret; +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree, + const guid_128_t guid) +{ + const uint8_t *guid_p = guid; + + return hash_table_lookup(tree->guid_hash, guid_p); +} + +const struct dsync_mailbox_delete * +dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree, + unsigned int *count_r) +{ + return array_get(&tree->deletes, count_r); +} + +struct dsync_mailbox_node * +dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_delete *del) +{ + const uint8_t *guid_p = del->guid; + + i_assert(hash_table_is_created(tree->guid_hash)); + i_assert(tree->remote_sep != '\0'); + + if (del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) { + /* find node by GUID */ + return hash_table_lookup(tree->guid_hash, guid_p); + } + + /* find node by name. this is a bit tricky, since the hierarchy + separator may differ from ours. */ + if (tree->sep == tree->remote_sep) { + if (!hash_table_is_created(tree->name128_hash)) + dsync_mailbox_tree_build_name128_hash(tree); + return hash_table_lookup(tree->name128_hash, guid_p); + } else { + if (!hash_table_is_created(tree->name128_remotesep_hash)) + dsync_mailbox_tree_build_name128_remotesep_hash(tree); + return hash_table_lookup(tree->name128_remotesep_hash, guid_p); + } +} + +void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree, + char remote_sep, char escape_char) +{ + i_assert(tree->remote_sep == '\0'); + i_assert(tree->remote_escape_char == '\0'); + + tree->remote_sep = remote_sep; + tree->remote_escape_char = escape_char; +} + +static void +dsync_mailbox_tree_dup_nodes(struct dsync_mailbox_tree *dest_tree, + const struct dsync_mailbox_node *src, + string_t *path) +{ + size_t prefix_len = str_len(path); + struct dsync_mailbox_node *node; + + if (prefix_len > 0) { + str_append_c(path, dest_tree->sep); + prefix_len++; + } + for (; src != NULL; src = src->next) { + str_truncate(path, prefix_len); + str_append(path, src->name); + node = dsync_mailbox_tree_get(dest_tree, str_c(path)); + + node->ns = src->ns; + memcpy(node->mailbox_guid, src->mailbox_guid, + sizeof(node->mailbox_guid)); + node->uid_validity = src->uid_validity; + node->uid_next = src->uid_next; + node->existence = src->existence; + node->last_renamed_or_created = src->last_renamed_or_created; + node->subscribed = src->subscribed; + node->last_subscription_change = src->last_subscription_change; + + if (src->first_child != NULL) { + dsync_mailbox_tree_dup_nodes(dest_tree, + src->first_child, path); + } + } +} + +struct dsync_mailbox_tree * +dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src) +{ + struct dsync_mailbox_tree *dest; + string_t *str = t_str_new(128); + + dest = dsync_mailbox_tree_init(src->sep, src->escape_char, + src->alt_char); + dsync_mailbox_tree_dup_nodes(dest, &src->root, str); + return dest; +} + +int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1, + struct dsync_mailbox_node *const *n2) +{ + return strcmp((*n1)->name, (*n2)->name); +} + +static bool +dsync_mailbox_nodes_equal(const struct dsync_mailbox_node *node1, + const struct dsync_mailbox_node *node2) +{ + return strcmp(node1->name, node2->name) == 0 && + node1->ns == node2->ns && + memcmp(node1->mailbox_guid, node2->mailbox_guid, + sizeof(node1->mailbox_guid)) == 0 && + node1->uid_validity == node2->uid_validity && + node1->existence == node2->existence && + node1->subscribed == node2->subscribed; +} + +static bool +dsync_mailbox_branches_equal(struct dsync_mailbox_node *node1, + struct dsync_mailbox_node *node2) +{ + /* this function is used only for unit tests, so performance doesn't + really matter */ + struct dsync_mailbox_node *n, **snodes1, **snodes2; + unsigned int i, count; + + for (n = node1, count = 0; n != NULL; n = n->next) count++; + for (n = node2, i = 0; n != NULL; n = n->next) i++; + if (i != count) + return FALSE; + if (count == 0) + return TRUE; + + /* sort the trees by name */ + snodes1 = t_new(struct dsync_mailbox_node *, count); + snodes2 = t_new(struct dsync_mailbox_node *, count); + for (n = node1, i = 0; n != NULL; n = n->next, i++) + snodes1[i] = n; + for (n = node2, i = 0; n != NULL; n = n->next, i++) + snodes2[i] = n; + i_qsort(snodes1, count, sizeof(*snodes1), dsync_mailbox_node_name_cmp); + i_qsort(snodes2, count, sizeof(*snodes2), dsync_mailbox_node_name_cmp); + + for (i = 0; i < count; i++) { + if (!dsync_mailbox_nodes_equal(snodes1[i], snodes2[i])) + return FALSE; + if (!dsync_mailbox_branches_equal(snodes1[i]->first_child, + snodes2[i]->first_child)) + return FALSE; + } + return TRUE; +} + +bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2) +{ + bool ret; + + T_BEGIN { + ret = dsync_mailbox_branches_equal(&tree1->root, &tree2->root); + } T_END; + return ret; +} + +const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node) +{ + return t_strdup_printf("guid=%s uid_validity=%u uid_next=%u subs=%s last_change=%ld last_subs=%ld", + guid_128_to_string(node->mailbox_guid), + node->uid_validity, node->uid_next, + node->subscribed ? "yes" : "no", + (long)node->last_renamed_or_created, + (long)node->last_subscription_change); +} + +const char * +dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type) +{ + switch (type) { + case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX: + return "mailbox"; + case DSYNC_MAILBOX_DELETE_TYPE_DIR: + return "dir"; + case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE: + return "unsubscribe"; + } + i_unreached(); +} diff --git a/src/doveadm/dsync/dsync-mailbox-tree.h b/src/doveadm/dsync/dsync-mailbox-tree.h new file mode 100644 index 0000000..6969813 --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox-tree.h @@ -0,0 +1,205 @@ +#ifndef DSYNC_MAILBOX_TREE_H +#define DSYNC_MAILBOX_TREE_H + +#include "guid.h" +#include "mail-error.h" + +struct mail_namespace; +struct dsync_brain; + +enum dsync_mailbox_trees_sync_type { + /* two-way sync for both mailboxes */ + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, + /* make remote tree look exactly like the local tree */ + DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL, + /* make local tree look exactly like the remote tree */ + DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE +}; + +enum dsync_mailbox_trees_sync_flags { + /* Enable debugging */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG = 0x01, + /* Show ourself as "master brain" in the debug output */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN = 0x02, + /* Disable mailbox renaming logic. This is just a kludge that should + be removed once the renaming logic has no more bugs.. */ + DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES = 0x04 +}; + +enum dsync_mailbox_node_existence { + /* this is just a filler node for children or for + subscription deletion */ + DSYNC_MAILBOX_NODE_NONEXISTENT = 0, + /* if mailbox GUID is set, the mailbox exists. + otherwise the directory exists. */ + DSYNC_MAILBOX_NODE_EXISTS, + /* if mailbox GUID is set, the mailbox has been deleted. + otherwise the directory has been deleted. */ + DSYNC_MAILBOX_NODE_DELETED +}; + +struct dsync_mailbox_node { + struct dsync_mailbox_node *parent, *next, *first_child; + + /* namespace where this node belongs to */ + struct mail_namespace *ns; + /* this node's name (not including parents) */ + const char *name; + /* mailbox GUID, or full of zeros if this is about a directory name */ + guid_128_t mailbox_guid; + /* mailbox's UIDVALIDITY/UIDNEXT (may be 0 if not assigned yet) */ + uint32_t uid_validity, uid_next; + + /* existence of this mailbox/directory. + doesn't affect subscription state. */ + enum dsync_mailbox_node_existence existence; + /* last time the mailbox/directory was created/renamed, + 0 if not known */ + time_t last_renamed_or_created; + + /* last time the subscription state was changed, 0 if not known */ + time_t last_subscription_change; + /* is this mailbox or directory subscribed? */ + bool subscribed:1; + + /* Internal syncing flags: */ + bool sync_delayed_guid_change:1; + bool sync_temporary_name:1; +}; +ARRAY_DEFINE_TYPE(dsync_mailbox_node, struct dsync_mailbox_node *); + +#define dsync_mailbox_node_guids_equal(node1, node2) \ + (memcmp((node1)->mailbox_guid, (node2)->mailbox_guid, \ + sizeof(guid_128_t)) == 0) + +#define dsync_mailbox_node_is_dir(node) \ + guid_128_is_empty((node)->mailbox_guid) + +enum dsync_mailbox_delete_type { + /* Delete mailbox by given GUID */ + DSYNC_MAILBOX_DELETE_TYPE_MAILBOX = 1, + /* Delete mailbox directory by given SHA1 name */ + DSYNC_MAILBOX_DELETE_TYPE_DIR, + /* Unsubscribe mailbox by given SHA1 name */ + DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE, +}; + +struct dsync_mailbox_delete { + enum dsync_mailbox_delete_type type; + guid_128_t guid; + time_t timestamp; +}; + +enum dsync_mailbox_tree_sync_type { + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX, + DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX, + DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR, + /* Rename given mailbox name and its children */ + DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME, + DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE, + DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE +}; + +struct dsync_mailbox_tree_sync_change { + enum dsync_mailbox_tree_sync_type type; + + /* for all types: */ + struct mail_namespace *ns; + const char *full_name; + + /* for create_box and delete_box: */ + guid_128_t mailbox_guid; + /* for create_box: */ + uint32_t uid_validity; + /* for rename: */ + const char *rename_dest_name; +}; + +struct dsync_mailbox_tree * +dsync_mailbox_tree_init(char sep, char escape_char, char alt_char); +void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **tree); + +/* Lookup a mailbox node by name. Returns NULL if not known. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree, + const char *full_name); +/* Lookup a mailbox node by GUID. Returns NULL if not known. + The mailbox GUID hash must have been build before calling this. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree, + const guid_128_t guid); +/* Lookup or create a mailbox node by name. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name); + +/* Returns full name for the given mailbox node. */ +const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node); +void dsync_mailbox_node_append_full_name(string_t *str, + const struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_node *node); + +/* Copy everything from src to dest, except name and hierarchy pointers */ +void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest, + const struct dsync_mailbox_node *src); + +/* Add nodes to tree from the given namespace. If box_name or box_guid is + non-NULL, add only that mailbox to the tree. */ +int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns, const char *box_name, + const guid_128_t box_guid, + const char *const *exclude_mailboxes, + enum mail_error *error_r); + +/* Return all known deleted mailboxes and directories. */ +const struct dsync_mailbox_delete * +dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree, + unsigned int *count_r); +/* Return mailbox node for a given delete record, or NULL if it doesn't exist. + The delete record is intended to come from another tree, possibly with + a different hierarchy separator. dsync_mailbox_tree_build_guid_hash() must + have been called before this. */ +struct dsync_mailbox_node * +dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree, + const struct dsync_mailbox_delete *del); +/* Build GUID lookup hash, if it's not already built. Returns 0 if ok, -1 if + there are duplicate GUIDs. The nodes with the duplicate GUIDs are + returned. */ +int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node **dup_node1_r, + struct dsync_mailbox_node **dup_node2_r); +/* Manually add a new node to hash. */ +int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree, + struct dsync_mailbox_node *node, + struct dsync_mailbox_node **old_node_r); +/* Set remote separator used for directory deletions in + dsync_mailbox_tree_find_delete() */ +void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree, + char remote_sep, + char remote_escape_char); + +/* Iterate through all nodes in a tree (depth-first) */ +struct dsync_mailbox_tree_iter * +dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree); +bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter, + const char **full_name_r, + struct dsync_mailbox_node **node_r); +void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **iter); + +/* Sync local and remote trees so at the end they're exactly the same. + Return changes done to local tree. */ +struct dsync_mailbox_tree_sync_ctx * +dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree, + struct dsync_mailbox_tree *remote_tree, + enum dsync_mailbox_trees_sync_type sync_type, + enum dsync_mailbox_trees_sync_flags sync_flags); +const struct dsync_mailbox_tree_sync_change * +dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx); +int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx); + +const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node); +const char * +dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type); + +#endif diff --git a/src/doveadm/dsync/dsync-mailbox.c b/src/doveadm/dsync/dsync-mailbox.c new file mode 100644 index 0000000..d6d06fd --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox.c @@ -0,0 +1,61 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "istream.h" +#include "mail-storage-private.h" +#include "dsync-brain-private.h" +#include "dsync-mailbox.h" + +void dsync_mailbox_attribute_dup(pool_t pool, + const struct dsync_mailbox_attribute *src, + struct dsync_mailbox_attribute *dest_r) +{ + dest_r->type = src->type; + dest_r->key = p_strdup(pool, src->key); + dest_r->value = p_strdup(pool, src->value); + if (src->value_stream != NULL) { + dest_r->value_stream = src->value_stream; + i_stream_ref(dest_r->value_stream); + } + + dest_r->deleted = src->deleted; + dest_r->last_change = src->last_change; + dest_r->modseq = src->modseq; +} + +int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box, + struct file_lock **lock_r) +{ + const char *path, *error; + int ret; + + /* Make sure the mailbox is open - locking requires it */ + if (mailbox_open(box) < 0) { + i_error("Can't open mailbox %s: %s", mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + return -1; + } + + ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path); + if (ret < 0) { + i_error("Can't get mailbox %s path: %s", mailbox_get_vname(box), + mailbox_get_last_internal_error(box, &brain->mail_error)); + return -1; + } + if (ret == 0) { + /* No index files - don't do any locking. In theory we still + could, but this lock is mainly meant to prevent replication + problems, and replication wouldn't work without indexes. */ + *lock_r = NULL; + return 0; + } + + if (mailbox_lock_file_create(box, DSYNC_MAILBOX_LOCK_FILENAME, + brain->mailbox_lock_timeout_secs, + lock_r, &error) <= 0) { + i_error("Failed to lock mailbox %s for dsyncing: %s", + box->vname, error); + return -1; + } + return 0; +} diff --git a/src/doveadm/dsync/dsync-mailbox.h b/src/doveadm/dsync/dsync-mailbox.h new file mode 100644 index 0000000..7e81c0c --- /dev/null +++ b/src/doveadm/dsync/dsync-mailbox.h @@ -0,0 +1,44 @@ +#ifndef DSYNC_MAILBOX_H +#define DSYNC_MAILBOX_H + +#include "mail-storage.h" + +struct dsync_brain; + +/* Mailbox that is going to be synced. Its name was already sent in the + mailbox tree. */ +struct dsync_mailbox { + guid_128_t mailbox_guid; + bool mailbox_lost; + bool mailbox_ignore; + bool have_guids, have_save_guids, have_only_guid128; + + uint32_t uid_validity, uid_next, messages_count, first_recent_uid; + uint64_t highest_modseq, highest_pvt_modseq; + ARRAY_TYPE(mailbox_cache_field) cache_fields; +}; + +struct dsync_mailbox_attribute { + enum mail_attribute_type type; + const char *key; + /* if both values are NULL = not looked up yet / deleted */ + const char *value; + struct istream *value_stream; + + time_t last_change; /* 0 = unknown */ + uint64_t modseq; /* 0 = unknown */ + + bool deleted; /* attribute is known to have been deleted */ + bool exported; /* internally used by exporting */ +}; +#define DSYNC_ATTR_HAS_VALUE(attr) \ + ((attr)->value != NULL || (attr)->value_stream != NULL) + +void dsync_mailbox_attribute_dup(pool_t pool, + const struct dsync_mailbox_attribute *src, + struct dsync_mailbox_attribute *dest_r); + +int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box, + struct file_lock **lock_r); + +#endif diff --git a/src/doveadm/dsync/dsync-serializer.c b/src/doveadm/dsync/dsync-serializer.c new file mode 100644 index 0000000..7b369b8 --- /dev/null +++ b/src/doveadm/dsync/dsync-serializer.c @@ -0,0 +1,117 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "str.h" +#include "strescape.h" +#include "dsync-serializer.h" + +struct dsync_serializer { + pool_t pool; + const char *const *keys; + unsigned int keys_count; +}; + +struct dsync_serializer_encoder { + pool_t pool; + struct dsync_serializer *serializer; + ARRAY_TYPE(const_string) values; +}; + +struct dsync_serializer *dsync_serializer_init(const char *const keys[]) +{ + struct dsync_serializer *serializer; + pool_t pool; + const char **dup_keys; + unsigned int i, count; + + pool = pool_alloconly_create("dsync serializer", 512); + serializer = p_new(pool, struct dsync_serializer, 1); + serializer->pool = pool; + + count = str_array_length(keys); + dup_keys = p_new(pool, const char *, count + 1); + for (i = 0; i < count; i++) + dup_keys[i] = p_strdup(pool, keys[i]); + serializer->keys = dup_keys; + serializer->keys_count = count; + return serializer; +} + +void dsync_serializer_deinit(struct dsync_serializer **_serializer) +{ + struct dsync_serializer *serializer = *_serializer; + + *_serializer = NULL; + + pool_unref(&serializer->pool); +} + +const char * +dsync_serializer_encode_header_line(struct dsync_serializer *serializer) +{ + string_t *str = t_str_new(128); + unsigned int i; + + for (i = 0; serializer->keys[i] != NULL; i++) { + if (i > 0) + str_append_c(str, '\t'); + str_append_tabescaped(str, serializer->keys[i]); + } + str_append_c(str, '\n'); + return str_c(str); +} + +struct dsync_serializer_encoder * +dsync_serializer_encode_begin(struct dsync_serializer *serializer) +{ + struct dsync_serializer_encoder *encoder; + pool_t pool = pool_alloconly_create("dsync serializer encode", 1024); + + encoder = p_new(pool, struct dsync_serializer_encoder, 1); + encoder->pool = pool; + encoder->serializer = serializer; + p_array_init(&encoder->values, pool, serializer->keys_count); + return encoder; +} + +void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder, + const char *key, const char *value) +{ + unsigned int i; + + for (i = 0; encoder->serializer->keys[i] != NULL; i++) { + if (strcmp(encoder->serializer->keys[i], key) == 0) { + value = p_strdup(encoder->pool, value); + array_idx_set(&encoder->values, i, &value); + return; + } + } + i_panic("Unknown key: %s", key); +} + +void dsync_serializer_encode_finish(struct dsync_serializer_encoder **_encoder, + string_t *output) +{ + struct dsync_serializer_encoder *encoder = *_encoder; + const char *const *values; + unsigned int i, count; + + *_encoder = NULL; + + values = array_get(&encoder->values, &count); + for (i = 0; i < count; i++) { + if (i > 0) + str_append_c(output, '\t'); + if (values[i] == NULL) + str_append_c(output, NULL_CHR); + else { + if (values[i][0] == NULL_CHR) + str_append_c(output, NULL_CHR); + str_append_tabescaped(output, values[i]); + } + } + str_append_c(output, '\n'); + + pool_unref(&encoder->pool); +} diff --git a/src/doveadm/dsync/dsync-serializer.h b/src/doveadm/dsync/dsync-serializer.h new file mode 100644 index 0000000..1ed6c31 --- /dev/null +++ b/src/doveadm/dsync/dsync-serializer.h @@ -0,0 +1,18 @@ +#ifndef DSYNC_SERIALIZER_H +#define DSYNC_SERIALIZER_H + +#define NULL_CHR '\002' + +struct dsync_serializer *dsync_serializer_init(const char *const keys[]); +void dsync_serializer_deinit(struct dsync_serializer **serializer); + +const char * +dsync_serializer_encode_header_line(struct dsync_serializer *serializer); +struct dsync_serializer_encoder * +dsync_serializer_encode_begin(struct dsync_serializer *serializer); +void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder, + const char *key, const char *value); +void dsync_serializer_encode_finish(struct dsync_serializer_encoder **encoder, + string_t *output); + +#endif diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.c b/src/doveadm/dsync/dsync-transaction-log-scan.c new file mode 100644 index 0000000..dd2834a --- /dev/null +++ b/src/doveadm/dsync/dsync-transaction-log-scan.c @@ -0,0 +1,608 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "hash.h" +#include "mail-index-modseq.h" +#include "mail-storage-private.h" +#include "dsync-mail.h" +#include "dsync-mailbox.h" +#include "dsync-transaction-log-scan.h" + +struct dsync_transaction_log_scan { + pool_t pool; + HASH_TABLE_TYPE(dsync_uid_mail_change) changes; + HASH_TABLE_TYPE(dsync_attr_change) attr_changes; + struct mail_index_view *view; + uint32_t highest_wanted_uid; + + uint32_t last_log_seq; + uoff_t last_log_offset; + + bool returned_all_changes; +}; + +static bool ATTR_NOWARN_UNUSED_RESULT +export_change_get(struct dsync_transaction_log_scan *ctx, uint32_t uid, + enum dsync_mail_change_type type, + struct dsync_mail_change **change_r) +{ + struct dsync_mail_change *change; + const char *orig_guid; + + i_assert(uid > 0); + i_assert(type != DSYNC_MAIL_CHANGE_TYPE_SAVE); + + *change_r = NULL; + + if (uid > ctx->highest_wanted_uid) + return FALSE; + + change = hash_table_lookup(ctx->changes, POINTER_CAST(uid)); + if (change == NULL) { + /* first change for this UID */ + change = p_new(ctx->pool, struct dsync_mail_change, 1); + change->uid = uid; + change->type = type; + hash_table_insert(ctx->changes, POINTER_CAST(uid), change); + } else if (type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* expunge overrides flag changes */ + orig_guid = change->guid; + i_zero(change); + change->type = type; + change->uid = uid; + change->guid = orig_guid; + } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) { + /* already expunged, this change doesn't matter */ + return FALSE; + } else { + /* another flag update */ + } + *change_r = change; + return TRUE; +} + +static void +log_add_expunge(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_expunge *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + return; + } + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change); + } + } +} + +static bool +log_add_expunge_uid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, uint32_t uid) +{ + const struct mail_transaction_expunge *rec = data, *end; + struct dsync_mail_change *change; + + if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) { + /* this is simply a request for expunge */ + return FALSE; + } + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (uid >= rec->uid1 && uid <= rec->uid2) { + export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change); + return TRUE; + } + } + return FALSE; +} + +static void +log_add_expunge_guid(struct dsync_transaction_log_scan *ctx, + struct mail_index_view *view, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_expunge_guid *rec = data, *end; + struct dsync_mail_change *change; + uint32_t seq; + bool external; + + external = (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (!external && mail_index_lookup_seq(view, rec->uid, &seq)) { + /* expunge request that hasn't been actually done yet. + we check non-external ones because they might have + the GUID while external ones don't. */ + continue; + } + if (export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change) && + !guid_128_is_empty(rec->guid_128)) T_BEGIN { + change->guid = p_strdup(ctx->pool, + guid_128_to_string(rec->guid_128)); + } T_END; + } +} + +static bool +log_add_expunge_guid_uid(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, uint32_t uid) +{ + const struct mail_transaction_expunge_guid *rec = data, *end; + struct dsync_mail_change *change; + + /* we're assuming UID is already known to be expunged */ + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + if (rec->uid != uid) + continue; + + if (!export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_EXPUNGE, + &change)) + i_unreached(); + if (!guid_128_is_empty(rec->guid_128)) T_BEGIN { + change->guid = p_strdup(ctx->pool, + guid_128_to_string(rec->guid_128)); + } T_END; + return TRUE; + } + return FALSE; +} + +static void +log_add_flag_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_flag_update *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + if (export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) { + change->add_flags |= rec->add_flags; + change->remove_flags &= ENUM_NEGATE(rec->add_flags); + change->remove_flags |= rec->remove_flags; + change->add_flags &= ENUM_NEGATE(rec->remove_flags); + } + } + } +} + +static void +log_add_keyword_reset(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_keyword_reset *rec = data, *end; + struct dsync_mail_change *change; + uint32_t uid; + + end = CONST_PTR_OFFSET(data, hdr->size); + for (; rec != end; rec++) { + for (uid = rec->uid1; uid <= rec->uid2; uid++) { + if (!export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + + change->keywords_reset = TRUE; + if (array_is_created(&change->keyword_changes)) + array_clear(&change->keyword_changes); + } + } +} + +static void +keywords_change_remove(struct dsync_mail_change *change, const char *name) +{ + const char *const *changes; + unsigned int i, count; + + changes = array_get(&change->keyword_changes, &count); + for (i = 0; i < count; i++) { + if (strcmp(changes[i]+1, name) == 0) { + array_delete(&change->keyword_changes, i, 1); + break; + } + } +} + +static void +log_add_keyword_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr) +{ + const struct mail_transaction_keyword_update *rec = data; + struct dsync_mail_change *change; + const char *kw_name, *change_str; + const uint32_t *uids, *end; + unsigned int uids_offset; + uint32_t uid; + + uids_offset = sizeof(*rec) + rec->name_size; + if ((uids_offset % 4) != 0) + uids_offset += 4 - (uids_offset % 4); + + kw_name = t_strndup((const void *)(rec+1), rec->name_size); + switch (rec->modify_type) { + case MODIFY_ADD: + change_str = p_strdup_printf(ctx->pool, "%c%s", + KEYWORD_CHANGE_ADD, kw_name); + break; + case MODIFY_REMOVE: + change_str = p_strdup_printf(ctx->pool, "%c%s", + KEYWORD_CHANGE_REMOVE, kw_name); + break; + default: + i_unreached(); + } + + uids = CONST_PTR_OFFSET(rec, uids_offset); + end = CONST_PTR_OFFSET(rec, hdr->size); + + for (; uids < end; uids += 2) { + for (uid = uids[0]; uid <= uids[1]; uid++) { + if (!export_change_get(ctx, uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + if (!array_is_created(&change->keyword_changes)) { + p_array_init(&change->keyword_changes, + ctx->pool, 4); + } else { + keywords_change_remove(change, kw_name); + } + array_push_back(&change->keyword_changes, &change_str); + } + } +} + +static void +log_add_modseq_update(struct dsync_transaction_log_scan *ctx, const void *data, + const struct mail_transaction_header *hdr, bool pvt_scan) +{ + const struct mail_transaction_modseq_update *rec = data, *end; + struct dsync_mail_change *change; + uint64_t modseq; + + /* update message's modseq, possibly by creating an empty flag change */ + end = CONST_PTR_OFFSET(rec, hdr->size); + for (; rec != end; rec++) { + if (rec->uid == 0) { + /* highestmodseq update */ + continue; + } + + if (!export_change_get(ctx, rec->uid, + DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE, + &change)) + continue; + + modseq = rec->modseq_low32 | + ((uint64_t)rec->modseq_high32 << 32); + if (!pvt_scan) { + if (change->modseq < modseq) + change->modseq = modseq; + } else { + if (change->pvt_modseq < modseq) + change->pvt_modseq = modseq; + } + } +} + +static void +log_add_attribute_update_key(struct dsync_transaction_log_scan *ctx, + const char *attr_change, uint64_t modseq) +{ + struct dsync_mailbox_attribute lookup_attr, *attr; + + i_assert(strlen(attr_change) > 2); /* checked by lib-index */ + + lookup_attr.type = attr_change[1] == 'p' ? + MAIL_ATTRIBUTE_TYPE_PRIVATE : MAIL_ATTRIBUTE_TYPE_SHARED; + lookup_attr.key = attr_change+2; + + attr = hash_table_lookup(ctx->attr_changes, &lookup_attr); + if (attr == NULL) { + attr = p_new(ctx->pool, struct dsync_mailbox_attribute, 1); + attr->type = lookup_attr.type; + attr->key = p_strdup(ctx->pool, lookup_attr.key); + hash_table_insert(ctx->attr_changes, attr, attr); + } + attr->deleted = attr_change[0] == '-'; + attr->modseq = modseq; +} + +static void +log_add_attribute_update(struct dsync_transaction_log_scan *ctx, + const void *data, + const struct mail_transaction_header *hdr, + uint64_t modseq) +{ + const char *attr_changes = data; + unsigned int i; + + for (i = 0; i < hdr->size && attr_changes[i] != '\0'; ) { + log_add_attribute_update_key(ctx, attr_changes+i, modseq); + i += strlen(attr_changes+i) + 1; + } +} + +static int +dsync_log_set(struct dsync_transaction_log_scan *ctx, + struct mail_index_view *view, bool pvt_scan, + struct mail_transaction_log_view *log_view, uint64_t modseq) +{ + uint32_t log_seq, end_seq; + uoff_t log_offset, end_offset; + const char *reason; + bool reset; + int ret; + + end_seq = view->log_file_head_seq; + end_offset = view->log_file_head_offset; + + if (modseq != 0 && + mail_index_modseq_get_next_log_offset(view, modseq, + &log_seq, &log_offset)) { + /* scan the view only up to end of the current view. + if there are more changes, we don't care about them until + the next sync. */ + ret = mail_transaction_log_view_set(log_view, + log_seq, log_offset, + end_seq, end_offset, + &reset, &reason); + if (ret != 0) + return ret; + } + + /* return everything we've got (until the end of the view) */ + if (!pvt_scan) + ctx->returned_all_changes = TRUE; + if (mail_transaction_log_view_set_all(log_view) < 0) + return -1; + + mail_transaction_log_view_get_prev_pos(log_view, &log_seq, &log_offset); + if (log_seq > end_seq || + (log_seq == end_seq && log_offset > end_offset)) { + end_seq = log_seq; + end_offset = log_offset; + } + ret = mail_transaction_log_view_set(log_view, + log_seq, log_offset, + end_seq, end_offset, + &reset, &reason); + if (ret == 0) { + /* we shouldn't get here. _view_set_all() already + reserved all the log files, the _view_set() only + removed unwanted ones. */ + i_error("%s: Couldn't set transaction log view (seq %u..%u): %s", + view->index->filepath, log_seq, end_seq, reason); + ret = -1; + } + if (ret < 0) + return -1; + if (modseq != 0) { + /* we didn't see all the changes that we wanted to */ + return 0; + } + return 1; +} + +static int +dsync_log_scan(struct dsync_transaction_log_scan *ctx, + struct mail_index_view *view, uint64_t modseq, bool pvt_scan) +{ + struct mail_transaction_log_view *log_view; + const struct mail_transaction_header *hdr; + const void *data; + uint32_t file_seq, max_seq; + uoff_t file_offset, max_offset; + uint64_t cur_modseq; + int ret; + + log_view = mail_transaction_log_view_open(view->index->log); + if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) { + mail_transaction_log_view_close(&log_view); + return -1; + } + + /* read the log only up to current position in view */ + max_seq = view->log_file_expunge_seq; + max_offset = view->log_file_expunge_offset; + + mail_transaction_log_view_get_prev_pos(log_view, &file_seq, + &file_offset); + + while (mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { + mail_transaction_log_view_get_prev_pos(log_view, &file_seq, + &file_offset); + if (file_offset >= max_offset && file_seq == max_seq) + break; + + if ((hdr->type & MAIL_TRANSACTION_SYNC) != 0) { + /* ignore changes done by dsync, unless we can get + expunged message's GUID from it */ + if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) != + MAIL_TRANSACTION_EXPUNGE_GUID) + continue; + } + + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + if (!pvt_scan) + log_add_expunge(ctx, data, hdr); + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + if (!pvt_scan) + log_add_expunge_guid(ctx, view, data, hdr); + break; + case MAIL_TRANSACTION_FLAG_UPDATE: + log_add_flag_update(ctx, data, hdr); + break; + case MAIL_TRANSACTION_KEYWORD_RESET: + log_add_keyword_reset(ctx, data, hdr); + break; + case MAIL_TRANSACTION_KEYWORD_UPDATE: + T_BEGIN { + log_add_keyword_update(ctx, data, hdr); + } T_END; + break; + case MAIL_TRANSACTION_MODSEQ_UPDATE: + log_add_modseq_update(ctx, data, hdr, pvt_scan); + break; + case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: + cur_modseq = mail_transaction_log_view_get_prev_modseq(log_view); + log_add_attribute_update(ctx, data, hdr, cur_modseq); + break; + } + } + + if (!pvt_scan) { + ctx->last_log_seq = file_seq; + ctx->last_log_offset = file_offset; + } + mail_transaction_log_view_close(&log_view); + return ret; +} + +static int +dsync_mailbox_attribute_cmp(const struct dsync_mailbox_attribute *attr1, + const struct dsync_mailbox_attribute *attr2) +{ + if (attr1->type < attr2->type) + return -1; + if (attr1->type > attr2->type) + return 1; + return strcmp(attr1->key, attr2->key); +} + +static unsigned int +dsync_mailbox_attribute_hash(const struct dsync_mailbox_attribute *attr) +{ + return str_hash(attr->key) ^ attr->type; +} + +int dsync_transaction_log_scan_init(struct mail_index_view *view, + struct mail_index_view *pvt_view, + uint32_t highest_wanted_uid, + uint64_t modseq, uint64_t pvt_modseq, + struct dsync_transaction_log_scan **scan_r, + bool *pvt_too_old_r) +{ + struct dsync_transaction_log_scan *ctx; + pool_t pool; + int ret, ret2; + + *pvt_too_old_r = FALSE; + + pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan", + 10240); + ctx = p_new(pool, struct dsync_transaction_log_scan, 1); + ctx->pool = pool; + hash_table_create_direct(&ctx->changes, pool, 0); + hash_table_create(&ctx->attr_changes, pool, 0, + dsync_mailbox_attribute_hash, + dsync_mailbox_attribute_cmp); + ctx->view = view; + ctx->highest_wanted_uid = highest_wanted_uid; + + if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0) + return -1; + if (pvt_view != NULL) { + if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0) + return -1; + if (ret2 == 0) { + ret = 0; + *pvt_too_old_r = TRUE; + } + } + + *scan_r = ctx; + return ret; +} + +HASH_TABLE_TYPE(dsync_uid_mail_change) +dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan) +{ + return scan->changes; +} + +HASH_TABLE_TYPE(dsync_attr_change) +dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan) +{ + return scan->attr_changes; +} + +bool +dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan) +{ + return scan->returned_all_changes; +} + +struct dsync_mail_change * +dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan, + uint32_t uid) +{ + struct mail_transaction_log_view *log_view; + const struct mail_transaction_header *hdr; + const void *data; + const char *reason; + bool reset, found = FALSE; + + i_assert(uid > 0); + + if (scan->highest_wanted_uid < uid) + scan->highest_wanted_uid = uid; + + log_view = mail_transaction_log_view_open(scan->view->index->log); + if (mail_transaction_log_view_set(log_view, + scan->last_log_seq, + scan->last_log_offset, + (uint32_t)-1, UOFF_T_MAX, + &reset, &reason) > 0) { + while (!found && + mail_transaction_log_view_next(log_view, &hdr, &data) > 0) { + switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) { + case MAIL_TRANSACTION_EXPUNGE: + if (log_add_expunge_uid(scan, data, hdr, uid)) + found = TRUE; + break; + case MAIL_TRANSACTION_EXPUNGE_GUID: + if (log_add_expunge_guid_uid(scan, data, hdr, uid)) + found = TRUE; + break; + } + } + } + mail_transaction_log_view_close(&log_view); + + return !found ? NULL : + hash_table_lookup(scan->changes, POINTER_CAST(uid)); +} + +void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **_scan) +{ + struct dsync_transaction_log_scan *scan = *_scan; + + *_scan = NULL; + + hash_table_destroy(&scan->changes); + hash_table_destroy(&scan->attr_changes); + pool_unref(&scan->pool); +} diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.h b/src/doveadm/dsync/dsync-transaction-log-scan.h new file mode 100644 index 0000000..458b775 --- /dev/null +++ b/src/doveadm/dsync/dsync-transaction-log-scan.h @@ -0,0 +1,32 @@ +#ifndef DSYNC_TRANSACTION_LOG_SCAN_H +#define DSYNC_TRANSACTION_LOG_SCAN_H + +HASH_TABLE_DEFINE_TYPE(dsync_uid_mail_change, + void *, struct dsync_mail_change *); +HASH_TABLE_DEFINE_TYPE(dsync_attr_change, + struct dsync_mailbox_attribute *, + struct dsync_mailbox_attribute *); + +struct mail_index_view; +struct dsync_transaction_log_scan; + +int dsync_transaction_log_scan_init(struct mail_index_view *view, + struct mail_index_view *pvt_view, + uint32_t highest_wanted_uid, + uint64_t modseq, uint64_t pvt_modseq, + struct dsync_transaction_log_scan **scan_r, + bool *pvt_too_old_r); +HASH_TABLE_TYPE(dsync_uid_mail_change) +dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan); +HASH_TABLE_TYPE(dsync_attr_change) +dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan); +/* Returns TRUE if the entire transaction log was scanned */ +bool dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan); +/* If the given UID has been expunged after the initial log scan, create/update + a change record for it and return it. */ +struct dsync_mail_change * +dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan, + uint32_t uid); +void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **scan); + +#endif diff --git a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c new file mode 100644 index 0000000..b068137 --- /dev/null +++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c @@ -0,0 +1,781 @@ +/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "sha1.h" +#include "str.h" +#include "mailbox-list-private.h" +#include "dsync-mailbox-tree-private.h" +#include "test-common.h" + +#include <stdio.h> + +#define MAX_DEPTH 4 +#define TEST_NAMESPACE_NAME "INBOX" + +static struct mail_namespace inbox_namespace = { + .prefix = TEST_NAMESPACE_NAME"/", + .prefix_len = sizeof(TEST_NAMESPACE_NAME)-1 + 1 +}; + +char mail_namespace_get_sep(struct mail_namespace *ns ATTR_UNUSED) +{ + return '/'; +} + +void mailbox_name_get_sha128(const char *name, guid_128_t guid_128_r) +{ + unsigned char sha[SHA1_RESULTLEN]; + + sha1_get_digest(name, strlen(name), sha); + memcpy(guid_128_r, sha, I_MIN(GUID_128_SIZE, sizeof(sha))); +} + +void mailbox_list_name_unescape(const char **name ATTR_UNUSED, + char escape_char ATTR_UNUSED) +{ +} + +void mailbox_list_name_escape(const char *name, + const char *escape_chars ATTR_UNUSED, + string_t *dest) +{ + str_append(dest, name); +} + +static struct dsync_mailbox_node * +node_create(struct dsync_mailbox_tree *tree, unsigned int counter, + const char *name, unsigned int last_renamed_or_created) +{ + struct dsync_mailbox_node *node; + + node = dsync_mailbox_tree_get(tree, name); + memcpy(node->mailbox_guid, &counter, sizeof(counter)); + node->uid_validity = counter; + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + node->last_renamed_or_created = last_renamed_or_created; + return node; +} + +static struct dsync_mailbox_node * +random_node_create(struct dsync_mailbox_tree *tree, unsigned int counter, + const char *name) +{ + return node_create(tree, counter, name, i_rand_limit(10)); +} + +static void nodes_create(struct dsync_mailbox_tree *tree, unsigned int *counter, + const char *const *names) +{ + for (; *names != NULL; names++) { + *counter += 1; + node_create(tree, *counter, *names, 0); + } +} + +static void nodes_delete(struct dsync_mailbox_tree *tree, unsigned int *counter, + const char *const *names) +{ + struct dsync_mailbox_node *node; + + for (; *names != NULL; names++) { + *counter += 1; + node = node_create(tree, *counter, *names, 0); + node->existence = DSYNC_MAILBOX_NODE_DELETED; + } +} + +static void +create_random_nodes(struct dsync_mailbox_tree *tree, const char *parent_name, + unsigned int depth, unsigned int *counter) +{ + unsigned int parent_len, i, nodes_count = i_rand_minmax(1, 3); + string_t *str; + + if (depth == MAX_DEPTH) + return; + + str = t_str_new(32); + if (*parent_name != '\0') + str_printfa(str, "%s/", parent_name); + parent_len = str_len(str); + + for (i = 0; i < nodes_count; i++) { + *counter += 1; + str_truncate(str, parent_len); + str_printfa(str, "%u.%u", depth, i); + random_node_create(tree, *counter, str_c(str)); + create_random_nodes(tree, str_c(str), depth+1, counter); + } +} + +static struct dsync_mailbox_tree *create_random_tree(void) +{ + struct dsync_mailbox_tree *tree; + unsigned int counter = 0; + + tree = dsync_mailbox_tree_init('/', '\0', '_'); + create_random_nodes(tree, "", 0, &counter); + return tree; +} + +static void test_tree_nodes_fixup(struct dsync_mailbox_node **pos, + unsigned int *newguid_counter) +{ + struct dsync_mailbox_node *node; + + for (node = *pos; node != NULL; node = node->next) { + if (node->sync_delayed_guid_change) { + /* the real code will pick one of the GUIDs. + we don't really care which one gets picked, so we'll + just change them to the same new one */ + memcpy(node->mailbox_guid, newguid_counter, + sizeof(*newguid_counter)); + node->uid_validity = *newguid_counter; + *newguid_counter += 1; + } + if (node->existence == DSYNC_MAILBOX_NODE_DELETED) + node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT; + test_tree_nodes_fixup(&node->first_child, newguid_counter); + if (node->existence != DSYNC_MAILBOX_NODE_EXISTS && + node->first_child == NULL) { + /* nonexistent node, drop it */ + *pos = node->next; + } else { + pos = &node->next; + } + } +} + +static void test_tree_fixup(struct dsync_mailbox_tree *tree) +{ + unsigned int newguid_counter = INT_MAX; + + test_tree_nodes_fixup(&tree->root.first_child, &newguid_counter); +} + +static void nodes_dump(const struct dsync_mailbox_node *node, unsigned int depth) +{ + unsigned int i; + + for (; node != NULL; node = node->next) { + for (i = 0; i < depth; i++) printf(" "); + printf("%-*s guid:%.5s uidv:%u %d%d %ld\n", 40-depth, node->name, + guid_128_to_string(node->mailbox_guid), node->uid_validity, + node->existence, node->subscribed ? 1 : 0, + (long)node->last_renamed_or_created); + nodes_dump(node->first_child, depth+1); + } +} + +static void trees_dump(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2) +{ + printf("tree1:\n"); + nodes_dump(tree1->root.first_child, 1); + printf("tree2:\n"); + nodes_dump(tree2->root.first_child, 1); +} + +static void test_trees_nofree(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree **_tree2) +{ + struct dsync_mailbox_tree *tree2 = *_tree2; + struct dsync_mailbox_tree *orig_tree1, *orig_tree2; + struct dsync_mailbox_tree_sync_ctx *ctx; + struct dsync_mailbox_node *dup_node1, *dup_node2; + + orig_tree1 = dsync_mailbox_tree_dup(tree1); + orig_tree2 = dsync_mailbox_tree_dup(tree2); + + /* test tree1 -> tree2 */ + dsync_mailbox_tree_build_guid_hash(tree1, &dup_node1, &dup_node2); + dsync_mailbox_tree_build_guid_hash(tree2, &dup_node1, &dup_node2); + ctx = dsync_mailbox_trees_sync_init(tree1, tree2, + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, + DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG); + while (dsync_mailbox_trees_sync_next(ctx) != NULL) { + } + dsync_mailbox_trees_sync_deinit(&ctx); + test_tree_fixup(tree1); + test_tree_fixup(tree2); + if (!dsync_mailbox_trees_equal(tree1, tree2)) { + test_assert(FALSE); + trees_dump(tree1, tree2); + } + + /* test tree2 -> tree1 */ + dsync_mailbox_tree_build_guid_hash(orig_tree1, &dup_node1, &dup_node2); + dsync_mailbox_tree_build_guid_hash(orig_tree2, &dup_node1, &dup_node2); + ctx = dsync_mailbox_trees_sync_init(orig_tree2, orig_tree1, + DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0); + while (dsync_mailbox_trees_sync_next(ctx) != NULL) { + } + dsync_mailbox_trees_sync_deinit(&ctx); + test_tree_fixup(orig_tree1); + test_tree_fixup(orig_tree2); + if (!dsync_mailbox_trees_equal(orig_tree1, orig_tree2)) { + test_assert(FALSE); + trees_dump(orig_tree1, orig_tree2); + } + + /* make sure both directions produced equal trees */ + if (!dsync_mailbox_trees_equal(tree1, orig_tree1)) { + test_assert(FALSE); + trees_dump(tree1, orig_tree1); + } + + dsync_mailbox_tree_deinit(_tree2); + dsync_mailbox_tree_deinit(&orig_tree1); + dsync_mailbox_tree_deinit(&orig_tree2); +} + +static void +test_tree_nodes_add_namespace(struct dsync_mailbox_node *node, + struct mail_namespace *ns) +{ + for (; node != NULL; node = node->next) { + node->ns = ns; + test_tree_nodes_add_namespace(node->first_child, ns); + } +} + +static void +test_tree_add_namespace(struct dsync_mailbox_tree *tree, + struct mail_namespace *ns) +{ + struct dsync_mailbox_node *node, *n; + + node = dsync_mailbox_tree_get(tree, TEST_NAMESPACE_NAME); + node->existence = DSYNC_MAILBOX_NODE_EXISTS; + i_assert(tree->root.first_child == node); + i_assert(node->first_child == NULL); + node->first_child = node->next; + for (n = node->first_child; n != NULL; n = n->next) + n->parent = node; + node->next = NULL; + + test_tree_nodes_add_namespace(&tree->root, ns); +} + +static void test_trees(struct dsync_mailbox_tree *tree1, + struct dsync_mailbox_tree *tree2) +{ + struct dsync_mailbox_tree *tree1_dup, *tree2_dup; + + tree1_dup = dsync_mailbox_tree_dup(tree1); + tree2_dup = dsync_mailbox_tree_dup(tree2); + + /* test without namespace prefix */ + test_trees_nofree(tree1, &tree2); + dsync_mailbox_tree_deinit(&tree1); + + /* test with namespace prefix */ + test_tree_add_namespace(tree1_dup, &inbox_namespace); + test_tree_add_namespace(tree2_dup, &inbox_namespace); + test_trees_nofree(tree1_dup, &tree2_dup); + dsync_mailbox_tree_deinit(&tree1_dup); +} + +static void test_dsync_mailbox_tree_sync_creates(void) +{ + static const char *common_nodes[] = { "foo", "foo/bar", NULL }; + static const char *create1_nodes[] = { "bar", "foo/baz", NULL }; + static const char *create2_nodes[] = { "foo/xyz", "foo/bar/3", NULL }; + struct dsync_mailbox_tree *tree1, *tree2; + unsigned int counter = 0; + + test_begin("dsync mailbox tree sync creates"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + nodes_create(tree1, &counter, common_nodes); + tree2 = dsync_mailbox_tree_dup(tree1); + nodes_create(tree1, &counter, create1_nodes); + nodes_create(tree2, &counter, create2_nodes); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_deletes(void) +{ + static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", NULL }; + static const char *delete1_nodes[] = { "1", "2", NULL }; + static const char *delete2_nodes[] = { "2/s1", "x/y", NULL }; + struct dsync_mailbox_tree *tree1, *tree2; + unsigned int counter = 0; + + test_begin("dsync mailbox tree sync deletes"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + nodes_create(tree1, &counter, common_nodes); + tree2 = dsync_mailbox_tree_dup(tree1); + nodes_delete(tree1, &counter, delete1_nodes); + nodes_delete(tree2, &counter, delete2_nodes); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames1(void) +{ + static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", "3/s3", NULL }; + struct dsync_mailbox_tree *tree1, *tree2; + struct dsync_mailbox_node *node; + unsigned int counter = 0; + + test_begin("dsync mailbox tree sync renames 1"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + nodes_create(tree1, &counter, common_nodes); + tree2 = dsync_mailbox_tree_dup(tree1); + + node = dsync_mailbox_tree_get(tree1, "1"); + node->name = "a"; + node->last_renamed_or_created = 1000; + node = dsync_mailbox_tree_get(tree2, "2"); + node->name = "b"; + node->last_renamed_or_created = 1000; + + node = dsync_mailbox_tree_get(tree1, "3/s3"); + node->name = "z"; + node->last_renamed_or_created = 1000; + dsync_mailbox_tree_node_detach(node); + dsync_mailbox_tree_node_attach(node, &tree1->root); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames2(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 2"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 1); + node_create(tree1, 2, "0/1/2", 3); + + node_create(tree2, 1, "0", 0); + node_create(tree2, 2, "0/1/2", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames3(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 3"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2", 1); + node_create(tree1, 2, "0/3", 1); + + node_create(tree2, 1, "0/4/5", 0); + node_create(tree2, 2, "1", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames4(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 4"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/b", 0); + node_create(tree1, 2, "c", 2); + + node_create(tree2, 2, "0/a", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames5(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 5"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "b", 0); + node_create(tree1, 2, "c", 2); + + node_create(tree2, 2, "0/a", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames6(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 6"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 0); + node_create(tree1, 2, "0/2", 1); + + node_create(tree2, 1, "0", 1); + node_create(tree2, 2, "0/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames7(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 7"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2", 0); + node_create(tree2, 1, "1/2", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames8(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 8"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 0); + node_create(tree1, 2, "0/2", 1); + + node_create(tree2, 1, "0", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames9(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 9"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1/2", 0); + node_create(tree1, 2, "0/3", 1); + + node_create(tree2, 1, "0", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames10(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 10"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 0); + node_create(tree1, 3, "0/2/3", 0); + + node_create(tree2, 1, "0", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames11(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 11"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/1", 2); + node_create(tree1, 0, "0/1/2", 0); + + node_create(tree2, 1, "0", 1); + node_create(tree2, 0, "0/1/2", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames12(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 12"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2", 0); + node_create(tree1, 2, "1", 0); + node_create(tree1, 3, "1/4", 0); + node_create(tree1, 4, "1/4/5", 1); + + node_create(tree2, 1, "1", 2); + node_create(tree2, 2, "1/4", 3); + node_create(tree2, 3, "1/4/6", 4); + node_create(tree2, 4, "1/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames13(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 13"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 4, "0.0/1.0/2.1", 0); + node_create(tree1, 5, "0.1", 2); + node_create(tree1, 6, "0.1/1.0", 2); + node_create(tree1, 7, "0.1/1.0/2.0", 8); + + node_create(tree2, 5, "0.1/1.0", 5); + node_create(tree2, 6, "0.1/1.0/2.0", 8); + node_create(tree2, 7, "0.1/1.1", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames14(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 14"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 0); + node_create(tree1, 2, "1/2", 0); + node_create(tree1, 3, "1/2/4", 1); + + node_create(tree2, 1, "1/2", 3); + node_create(tree2, 2, "1/2/5", 4); + node_create(tree2, 3, "1/2/4", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames15(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 15"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 0); + node_create(tree2, 2, "1", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames16(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 16"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1/2", 4); + node_create(tree1, 2, "1", 2); + + node_create(tree2, 1, "2", 1); + node_create(tree2, 2, "1/2", 3); + node_create(tree2, 3, "1", 5); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames17(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 17"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 1); + + node_create(tree2, 1, "1/2", 0); + node_create(tree2, 2, "1", 2); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames18(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 18"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 2, "a", 5); + node_create(tree1, 4, "a/c", 2); + node_create(tree1, 5, "b", 6); + + node_create(tree2, 1, "a", 7); + node_create(tree2, 2, "b", 3); + node_create(tree2, 3, "b/c", 4); + node_create(tree2, 4, "d", 1); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames19(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 19"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "0/2/1", 1); + node_create(tree1, 2, "0/4", 3); + node_create(tree1, 3, "0/2", 2); + + node_create(tree2, 1, "1", 0); + node_create(tree2, 2, "1/3", 4); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames20(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 20"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "1", 0); + node_create(tree1, 2, "0", 0); + node_create(tree1, 3, "0/2", 0); + /* rename 0 -> 1/0 */ + node_create(tree2, 1, "1", 0); + node_create(tree2, 2, "1/0", 1); + node_create(tree2, 3, "1/0/2", 0); + + test_trees_nofree(tree1, &tree2); + test_assert(tree1->root.first_child != NULL && + tree1->root.first_child->next == NULL); + dsync_mailbox_tree_deinit(&tree1); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_renames21(void) +{ +#if 0 + /* FIXME: we can't currently test this without crashing */ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 21"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 1, "INBOX", 0); + node_create(tree1, 2, "foo", 0); + /* swap INBOX and foo - the INBOX name is important since it's + treated specially */ + node_create(tree2, 1, "foo", 0); + node_create(tree2, 2, "INBOX", 1); + + test_trees(tree1, tree2); + test_end(); +#endif +} + +static void test_dsync_mailbox_tree_sync_renames22(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync renames 22"); + tree1 = dsync_mailbox_tree_init('/', '\0', '_'); + tree2 = dsync_mailbox_tree_init('/', '\0', '_'); + + node_create(tree1, 3, "p/a", 0); + node_create(tree1, 0, "p/2", 0); + node_create(tree1, 5, "p/2/h", 0); + + node_create(tree2, 4, "p/1/z", 0); + node_create(tree2, 1, "p/2", 0); + node_create(tree2, 2, "p/2/a", 0); + node_create(tree2, 5, "p/2/y", 0); + node_create(tree2, 3, "p/3", 0); + + test_trees(tree1, tree2); + test_end(); +} + +static void test_dsync_mailbox_tree_sync_random(void) +{ + struct dsync_mailbox_tree *tree1, *tree2; + + test_begin("dsync mailbox tree sync random"); + tree1 = create_random_tree(); + tree2 = create_random_tree(); + test_trees(tree1, tree2); + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_dsync_mailbox_tree_sync_creates, + test_dsync_mailbox_tree_sync_deletes, + test_dsync_mailbox_tree_sync_renames1, + test_dsync_mailbox_tree_sync_renames2, + test_dsync_mailbox_tree_sync_renames3, + test_dsync_mailbox_tree_sync_renames4, + test_dsync_mailbox_tree_sync_renames5, + test_dsync_mailbox_tree_sync_renames6, + test_dsync_mailbox_tree_sync_renames7, + test_dsync_mailbox_tree_sync_renames8, + test_dsync_mailbox_tree_sync_renames9, + test_dsync_mailbox_tree_sync_renames10, + test_dsync_mailbox_tree_sync_renames11, + test_dsync_mailbox_tree_sync_renames12, + test_dsync_mailbox_tree_sync_renames13, + test_dsync_mailbox_tree_sync_renames14, + test_dsync_mailbox_tree_sync_renames15, + test_dsync_mailbox_tree_sync_renames16, + test_dsync_mailbox_tree_sync_renames17, + test_dsync_mailbox_tree_sync_renames18, + test_dsync_mailbox_tree_sync_renames19, + test_dsync_mailbox_tree_sync_renames20, + test_dsync_mailbox_tree_sync_renames21, + test_dsync_mailbox_tree_sync_renames22, + test_dsync_mailbox_tree_sync_random, + NULL + }; + return test_run(test_functions); +} diff --git a/src/doveadm/main.c b/src/doveadm/main.c new file mode 100644 index 0000000..9635b88 --- /dev/null +++ b/src/doveadm/main.c @@ -0,0 +1,129 @@ +/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "restrict-access.h" +#include "process-title.h" +#include "master-service.h" +#include "settings-parser.h" +#include "dict.h" +#include "doveadm.h" +#include "client-connection.h" +#include "client-connection-private.h" +#include "doveadm-settings.h" +#include "doveadm-dump.h" +#include "doveadm-mail.h" +#include "doveadm-print-private.h" +#include "doveadm-server.h" +#include "ostream.h" + +const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[] = { + &doveadm_print_server_vfuncs, + &doveadm_print_json_vfuncs, + NULL +}; + +struct client_connection *doveadm_client; +int doveadm_exit_code = 0; + +static void doveadm_die(void) +{ + /* do nothing. doveadm connections should be over soon. */ +} + +static void client_connected(struct master_service_connection *conn) +{ + if (doveadm_client != NULL) { + i_error("doveadm server can handle only a single client"); + return; + } + + master_service_client_connection_accept(conn); + if (strcmp(conn->name, "http") == 0) { + doveadm_client = client_connection_http_create(conn->fd, conn->ssl); + } else { + doveadm_client = client_connection_tcp_create(conn->fd, conn->listen_fd, + conn->ssl); + } +} + +void help_ver2(const struct doveadm_cmd_ver2 *cmd) +{ + i_fatal("Client sent invalid command. Usage: %s %s", + cmd->name, cmd->usage); +} + +static void main_preinit(void) +{ + restrict_access_by_env(RESTRICT_ACCESS_FLAG_ALLOW_ROOT, NULL); + restrict_access_allow_coredumps(TRUE); +} + +static void main_init(void) +{ + doveadm_server = TRUE; + + doveadm_settings_init(); + doveadm_cmds_init(); + doveadm_register_auth_server_commands(); + doveadm_dump_init(); + doveadm_mail_init(); + dict_drivers_register_builtin(); + doveadm_load_modules(); + /* read settings only after loading doveadm plugins, which + may modify what settings are read */ + doveadm_read_settings(); + /* Load mail_plugins */ + doveadm_mail_init_finish(); + /* kludgy: Load the rest of the doveadm plugins after + mail_plugins have been loaded. */ + doveadm_load_modules(); + + doveadm_server_init(); + if (doveadm_verbose_proctitle) + process_title_set("[idling]"); +} + +static void main_deinit(void) +{ + doveadm_server_deinit(); + doveadm_mail_deinit(); + doveadm_dump_deinit(); + doveadm_unload_modules(); + dict_drivers_unregister_builtin(); + doveadm_print_deinit(); + doveadm_cmds_deinit(); + doveadm_settings_deinit(); +} + +int main(int argc, char *argv[]) +{ + enum master_service_flags service_flags = + MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN | + MASTER_SERVICE_FLAG_HAVE_STARTTLS; + int c; + + master_service = master_service_init("doveadm", service_flags, + &argc, &argv, "D"); + while ((c = master_getopt(master_service)) > 0) { + switch (c) { + case 'D': + doveadm_debug = TRUE; + doveadm_verbose = TRUE; + break; + default: + return FATAL_DEFAULT; + } + } + + master_service_init_log(master_service); + main_preinit(); + master_service_set_die_callback(master_service, doveadm_die); + + main_init(); + master_service_init_finish(master_service); + master_service_run(master_service, client_connected); + + main_deinit(); + master_service_deinit(&master_service); + return 0; +} diff --git a/src/doveadm/server-connection.c b/src/doveadm/server-connection.c new file mode 100644 index 0000000..a0e6500 --- /dev/null +++ b/src/doveadm/server-connection.c @@ -0,0 +1,691 @@ +/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "array.h" +#include "base64.h" +#include "ioloop.h" +#include "net.h" +#include "istream.h" +#include "istream-multiplex.h" +#include "ostream.h" +#include "ostream-dot.h" +#include "str.h" +#include "strescape.h" +#include "iostream-ssl.h" +#include "master-service.h" +#include "master-service-settings.h" +#include "settings-parser.h" +#include "doveadm.h" +#include "doveadm-print.h" +#include "doveadm-util.h" +#include "doveadm-server.h" +#include "doveadm-settings.h" +#include "server-connection.h" + +#include <sysexits.h> +#include <unistd.h> + +#define DOVEADM_LOG_CHANNEL_ID 'L' + +#define MAX_INBUF_SIZE (1024*32) + +enum server_reply_state { + SERVER_REPLY_STATE_DONE = 0, + SERVER_REPLY_STATE_PRINT, + SERVER_REPLY_STATE_RET +}; + +struct server_connection { + struct doveadm_server *server; + + pool_t pool; + struct doveadm_settings *set; + + int fd; + unsigned int minor; + + struct io *io; + struct io *io_log; + struct istream *input; + struct istream *log_input; + struct ostream *output; + struct ssl_iostream *ssl_iostream; + + struct istream *cmd_input; + struct ostream *cmd_output; + const char *delayed_cmd; + server_cmd_callback_t *callback; + void *context; + + enum server_reply_state state; + + bool version_received:1; + bool authenticate_sent:1; + bool authenticated:1; + bool streaming:1; + bool ssl_done:1; +}; + +static struct server_connection *printing_conn = NULL; +static ARRAY(struct doveadm_server *) print_pending_servers = ARRAY_INIT; + +static void server_connection_input(struct server_connection *conn); +static bool server_connection_input_one(struct server_connection *conn); +static int server_connection_init_ssl(struct server_connection *conn, + const char **error_r); + +static void server_set_print_pending(struct doveadm_server *server) +{ + struct doveadm_server *pending_server; + + if (!array_is_created(&print_pending_servers)) + i_array_init(&print_pending_servers, 16); + array_foreach_elem(&print_pending_servers, pending_server) { + if (pending_server == server) + return; + } + array_push_back(&print_pending_servers, &server); +} + +static void server_print_connection_released(struct doveadm_server *server) +{ + struct server_connection *const *conns; + unsigned int i, count; + + conns = array_get(&server->connections, &count); + for (i = 0; i < count; i++) { + if (conns[i]->io != NULL) + continue; + + conns[i]->io = io_add(conns[i]->fd, IO_READ, + server_connection_input, conns[i]); + io_set_pending(conns[i]->io); + } +} + +static void print_connection_released(void) +{ + struct doveadm_server *server; + + printing_conn = NULL; + if (!array_is_created(&print_pending_servers)) + return; + + array_foreach_elem(&print_pending_servers, server) + server_print_connection_released(server); + array_free(&print_pending_servers); +} + +static int server_connection_send_cmd_input_more(struct server_connection *conn) +{ + enum ostream_send_istream_result res; + int ret = -1; + + /* ostream-dot writes only up to max buffer size, so keep it non-zero */ + o_stream_set_max_buffer_size(conn->cmd_output, IO_BLOCK_SIZE); + res = o_stream_send_istream(conn->cmd_output, conn->cmd_input); + o_stream_set_max_buffer_size(conn->cmd_output, SIZE_MAX); + + switch (res) { + case OSTREAM_SEND_ISTREAM_RESULT_FINISHED: + break; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT: + return 1; + case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT: + return 0; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT: + i_error("read(%s) failed: %s", + i_stream_get_name(conn->cmd_input), + i_stream_get_error(conn->cmd_input)); + break; + case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT: + i_error("write(%s) failed: %s", + o_stream_get_name(conn->cmd_output), + o_stream_get_error(conn->cmd_output)); + break; + } + if (res == OSTREAM_SEND_ISTREAM_RESULT_FINISHED) { + if ((ret = o_stream_finish(conn->cmd_output)) == 0) + return 0; + else if (ret < 0) { + i_error("write(%s) failed: %s", + o_stream_get_name(conn->cmd_output), + o_stream_get_error(conn->cmd_output)); + } + } + + i_stream_destroy(&conn->cmd_input); + o_stream_destroy(&conn->cmd_output); + return ret; +} + +static void server_connection_send_cmd_input(struct server_connection *conn) +{ + if (conn->cmd_input == NULL) + return; + + conn->cmd_output = o_stream_create_dot(conn->output, TRUE); + (void)server_connection_send_cmd_input_more(conn); +} + +static int server_connection_output(struct server_connection *conn) +{ + int ret; + + ret = o_stream_flush(conn->output); + if (ret > 0 && conn->cmd_input != NULL && conn->delayed_cmd == NULL) + ret = server_connection_send_cmd_input_more(conn); + if (ret < 0) + server_connection_destroy(&conn); + return ret; +} + +static void +server_connection_callback(struct server_connection *conn, + int exit_code, const char *error) +{ + server_cmd_callback_t *callback = conn->callback; + + conn->callback = NULL; + callback(exit_code, error, conn->context); +} + +static void stream_data(string_t *str, const unsigned char *data, size_t size) +{ + str_truncate(str, 0); + str_append_tabunescaped(str, data, size); + doveadm_print_stream(str->data, str->used); +} + +static void server_flush_field(struct server_connection *conn, string_t *str, + const unsigned char *data, size_t size) +{ + if (conn->streaming) { + conn->streaming = FALSE; + if (size > 0) + stream_data(str, data, size); + doveadm_print_stream("", 0); + } else { + str_truncate(str, 0); + str_append_tabunescaped(str, data, size); + doveadm_print(str_c(str)); + } +} + +static void +server_handle_input(struct server_connection *conn, + const unsigned char *data, size_t size) +{ + string_t *str; + size_t i, start; + + if (printing_conn == conn) { + /* continue printing */ + } else if (printing_conn == NULL) { + printing_conn = conn; + } else { + /* someone else is printing. don't continue until it + goes away */ + server_set_print_pending(conn->server); + io_remove(&conn->io); + return; + } + + if (data[size-1] == '\001') { + /* last character is an escape */ + size--; + } + + str = t_str_new(128); + for (i = start = 0; i < size; i++) { + if (data[i] == '\n') { + if (i != start) { + i_error("doveadm server sent broken print input"); + server_connection_destroy(&conn); + return; + } + conn->state = SERVER_REPLY_STATE_RET; + i_stream_skip(conn->input, i + 1); + + print_connection_released(); + return; + } + if (data[i] == '\t') { + server_flush_field(conn, str, data + start, i - start); + start = i + 1; + } + } + if (start != size) { + conn->streaming = TRUE; + stream_data(str, data + start, size - start); + } + i_stream_skip(conn->input, size); +} + +static void server_connection_authenticated(struct server_connection *conn) +{ + conn->authenticated = TRUE; + if (conn->delayed_cmd != NULL) { + o_stream_nsend_str(conn->output, conn->delayed_cmd); + conn->delayed_cmd = NULL; + server_connection_send_cmd_input(conn); + } +} + +static int +server_connection_authenticate(struct server_connection *conn) +{ + string_t *plain = t_str_new(128); + string_t *cmd = t_str_new(128); + + if (*conn->set->doveadm_password == '\0') { + i_error("doveadm_password not set, " + "can't authenticate to remote server"); + return -1; + } + + str_append_c(plain, '\0'); + str_append(plain, conn->set->doveadm_username); + str_append_c(plain, '\0'); + str_append(plain, conn->set->doveadm_password); + + str_append(cmd, "PLAIN\t"); + base64_encode(plain->data, plain->used, cmd); + str_append_c(cmd, '\n'); + + o_stream_nsend(conn->output, cmd->data, cmd->used); + conn->authenticate_sent = TRUE; + return 0; +} + +static void server_log_disconnect_error(struct server_connection *conn) +{ + const char *error; + + error = conn->ssl_iostream == NULL ? NULL : + ssl_iostream_get_last_error(conn->ssl_iostream); + if (error == NULL) { + error = conn->input->stream_errno == 0 ? "EOF" : + strerror(conn->input->stream_errno); + } + i_error("doveadm server disconnected before handshake: %s", error); +} + +static void server_connection_print_log(struct server_connection *conn) +{ + const char *line; + struct failure_context ctx; + i_zero(&ctx); + + while((line = i_stream_read_next_line(conn->log_input))!=NULL) { + /* skip empty lines */ + if (*line == '\0') continue; + + if (!doveadm_log_type_from_char(line[0], &ctx.type)) + i_warning("Doveadm server sent invalid log type 0x%02x", + line[0]); + line++; + i_log_type(&ctx, "remote(%s): %s", conn->server->name, line); + } +} + +static void server_connection_start_multiplex(struct server_connection *conn) +{ + struct istream *is = conn->input; + conn->input = i_stream_create_multiplex(is, MAX_INBUF_SIZE); + i_stream_unref(&is); + io_remove(&conn->io); + conn->io = io_add_istream(conn->input, server_connection_input, conn); + conn->log_input = i_stream_multiplex_add_channel(conn->input, DOVEADM_LOG_CHANNEL_ID); + conn->io_log = io_add_istream(conn->log_input, server_connection_print_log, conn); + i_stream_set_return_partial_line(conn->log_input, TRUE); +} + +static void server_connection_input(struct server_connection *conn) +{ + const char *line; + const char *error; + + if (i_stream_read(conn->input) < 0) { + /* disconnected */ + server_log_disconnect_error(conn); + server_connection_destroy(&conn); + return; + } + + while (!conn->authenticated) { + if ((line = i_stream_next_line(conn->input)) == NULL) { + if (conn->input->eof) { + /* we'll also get here if the line is too long */ + server_log_disconnect_error(conn); + server_connection_destroy(&conn); + } + return; + } + /* Allow VERSION before or after the "+" or "-" line, + because v2.2.33 sent the version after and newer + versions send before. */ + if (!conn->version_received && + str_begins(line, "VERSION\t")) { + if (!version_string_verify_full(line, "doveadm-client", + DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR, + &conn->minor)) { + i_error("doveadm server not compatible with this client" + "(mixed old and new binaries?)"); + server_connection_destroy(&conn); + return; + } + conn->version_received = TRUE; + } else if (strcmp(line, "+") == 0) { + if (conn->minor > 0) + server_connection_start_multiplex(conn); + server_connection_authenticated(conn); + } else if (strcmp(line, "-") == 0) { + if (conn->authenticate_sent) { + i_error("doveadm authentication failed (%s)", + line+1); + server_connection_destroy(&conn); + return; + } + if (!conn->ssl_done && + (conn->server->ssl_flags & PROXY_SSL_FLAG_STARTTLS) != 0) { + io_remove(&conn->io); + if (conn->minor < 2) { + i_error("doveadm STARTTLS failed: Server does not support it"); + server_connection_destroy(&conn); + return; + } + /* send STARTTLS */ + o_stream_nsend_str(conn->output, "STARTTLS\n"); + if (server_connection_init_ssl(conn, &error) < 0) { + i_error("doveadm STARTTLS failed: %s", error); + server_connection_destroy(&conn); + return; + } + conn->ssl_done = TRUE; + conn->io = io_add_istream(conn->input, server_connection_input, conn); + } + if (server_connection_authenticate(conn) < 0) { + server_connection_destroy(&conn); + return; + } + } else { + i_error("doveadm server sent invalid handshake: %s", + line); + server_connection_destroy(&conn); + return; + } + } + + while (server_connection_input_one(conn)) ; +} + +static bool server_connection_input_one(struct server_connection *conn) +{ + const unsigned char *data; + size_t size; + const char *line; + int exit_code; + + /* check logs - NOTE: must be before i_stream_get_data() since checking + for logs may add data to our channel. */ + if (conn->log_input != NULL) + (void)server_connection_print_log(conn); + + data = i_stream_get_data(conn->input, &size); + if (size == 0) + return FALSE; + + switch (conn->state) { + case SERVER_REPLY_STATE_DONE: + i_error("doveadm server sent unexpected input"); + server_connection_destroy(&conn); + return FALSE; + case SERVER_REPLY_STATE_PRINT: + server_handle_input(conn, data, size); + if (conn->state != SERVER_REPLY_STATE_RET) + return FALSE; + /* fall through */ + case SERVER_REPLY_STATE_RET: + line = i_stream_next_line(conn->input); + if (line == NULL) + return FALSE; + if (line[0] == '+') + server_connection_callback(conn, 0, ""); + else if (line[0] == '-') { + line++; + exit_code = doveadm_str_to_exit_code(line); + if (exit_code == DOVEADM_EX_UNKNOWN && + str_to_int(line, &exit_code) < 0) { + /* old doveadm-server */ + exit_code = EX_TEMPFAIL; + } + server_connection_callback(conn, exit_code, line); + } else { + i_error("doveadm server sent broken input " + "(expected cmd reply): %s", line); + server_connection_destroy(&conn); + return FALSE; + } + if (conn->callback == NULL) { + /* we're finished, close the connection */ + server_connection_destroy(&conn); + return FALSE; + } + return TRUE; + } + i_unreached(); +} + +static int server_connection_read_settings(struct server_connection *conn, + const char **error_r) +{ + const struct setting_parser_info *set_roots[] = { + &doveadm_setting_parser_info, + NULL + }; + struct master_service_settings_input input; + struct master_service_settings_output output; + const char *error; + in_port_t port; + void *set; + + i_zero(&input); + input.roots = set_roots; + input.service = "doveadm"; + + (void)net_getsockname(conn->fd, &input.local_ip, &port); + (void)net_getpeername(conn->fd, &input.remote_ip, &port); + + if (master_service_settings_read(master_service, &input, + &output, &error) < 0) { + *error_r = t_strdup_printf( + "Error reading configuration: %s", error); + return -1; + } + set = master_service_settings_get_others(master_service)[0]; + conn->set = settings_dup(&doveadm_setting_parser_info, set, conn->pool); + return 0; +} + +static int server_connection_init_ssl(struct server_connection *conn, + const char **error_r) +{ + struct ssl_iostream_settings ssl_set; + const char *error; + + if (conn->server->ssl_flags == 0) + return 0; + + doveadm_get_ssl_settings(&ssl_set, pool_datastack_create()); + + if ((conn->server->ssl_flags & PROXY_SSL_FLAG_ANY_CERT) != 0) + ssl_set.allow_invalid_cert = TRUE; + if (ssl_set.allow_invalid_cert) + ssl_set.verbose_invalid_cert = TRUE; + + if (conn->server->ssl_ctx == NULL && + ssl_iostream_client_context_cache_get(&ssl_set, + &conn->server->ssl_ctx, + &error) < 0) { + *error_r = t_strdup_printf( + "Couldn't initialize SSL client: %s", error); + return -1; + } + + if (io_stream_create_ssl_client(conn->server->ssl_ctx, + conn->server->hostname, &ssl_set, + &conn->input, &conn->output, + &conn->ssl_iostream, &error) < 0) { + *error_r = t_strdup_printf( + "Couldn't initialize SSL client: %s", error); + return -1; + } + if (ssl_iostream_handshake(conn->ssl_iostream) < 0) { + *error_r = t_strdup_printf( + "SSL handshake failed: %s", + ssl_iostream_get_last_error(conn->ssl_iostream)); + return -1; + } + return 0; +} + +int server_connection_create(struct doveadm_server *server, + struct server_connection **conn_r, + const char **error_r) +{ + const char *target; + struct server_connection *conn; + pool_t pool; + + pool = pool_alloconly_create("doveadm server connection", 1024*16); + conn = p_new(pool, struct server_connection, 1); + conn->pool = pool; + conn->server = server; + if (server->ip.family != 0) { + (void)net_ipport2str(&server->ip, server->port, &target); + } else { + target = server->name; + } + conn->fd = doveadm_connect_with_default_port(target, + doveadm_settings->doveadm_port); + net_set_nonblock(conn->fd, TRUE); + conn->input = i_stream_create_fd(conn->fd, MAX_INBUF_SIZE); + conn->output = o_stream_create_fd(conn->fd, SIZE_MAX); + o_stream_set_flush_callback(conn->output, server_connection_output, conn); + o_stream_set_no_error_handling(conn->output, TRUE); + + i_stream_set_name(conn->input, server->name); + o_stream_set_name(conn->output, server->name); + + array_push_back(&conn->server->connections, &conn); + + if (server_connection_read_settings(conn, error_r) < 0 || + ((server->ssl_flags & PROXY_SSL_FLAG_STARTTLS) == 0 && + server_connection_init_ssl(conn, error_r) < 0)) { + server_connection_destroy(&conn); + return -1; + } + conn->io = io_add_istream(conn->input, server_connection_input, conn); + + conn->state = SERVER_REPLY_STATE_DONE; + o_stream_nsend_str(conn->output, DOVEADM_SERVER_PROTOCOL_VERSION_LINE"\n"); + + *conn_r = conn; + return 0; +} + +void server_connection_destroy(struct server_connection **_conn) +{ + struct server_connection *conn = *_conn; + struct server_connection *const *conns; + const char *error; + unsigned int i, count; + + *_conn = NULL; + + conns = array_get(&conn->server->connections, &count); + for (i = 0; i < count; i++) { + if (conns[i] == conn) { + array_delete(&conn->server->connections, i, 1); + break; + } + } + + if (conn->callback != NULL) { + error = conn->ssl_iostream == NULL ? NULL : + ssl_iostream_get_last_error(conn->ssl_iostream); + if (error == NULL) { + error = conn->input->stream_errno == 0 ? "EOF" : + strerror(conn->input->stream_errno); + } + server_connection_callback(conn, SERVER_EXIT_CODE_DISCONNECTED, + error); + } + if (printing_conn == conn) + print_connection_released(); + + i_stream_destroy(&conn->input); + o_stream_destroy(&conn->output); + i_stream_destroy(&conn->cmd_input); + /* close cmd_output after its parent, so the "." isn't sent */ + o_stream_destroy(&conn->cmd_output); + ssl_iostream_destroy(&conn->ssl_iostream); + io_remove(&conn->io_log); + /* make sure all logs got consumed */ + if (conn->log_input != NULL) + server_connection_print_log(conn); + i_stream_unref(&conn->log_input); + io_remove(&conn->io); + i_close_fd(&conn->fd); + pool_unref(&conn->pool); +} + +struct doveadm_server * +server_connection_get_server(struct server_connection *conn) +{ + return conn->server; +} + +void server_connection_cmd(struct server_connection *conn, const char *line, + struct istream *cmd_input, + server_cmd_callback_t *callback, void *context) +{ + i_assert(conn->delayed_cmd == NULL); + + conn->state = SERVER_REPLY_STATE_PRINT; + if (cmd_input != NULL) { + i_assert(conn->cmd_input == NULL); + i_stream_ref(cmd_input); + conn->cmd_input = cmd_input; + } + if (!conn->authenticated) + conn->delayed_cmd = p_strdup(conn->pool, line); + else { + o_stream_nsend_str(conn->output, line); + server_connection_send_cmd_input(conn); + } + conn->callback = callback; + conn->context = context; +} + +bool server_connection_is_idle(struct server_connection *conn) +{ + return conn->callback == NULL; +} + +void server_connection_extract(struct server_connection *conn, + struct istream **istream_r, + struct ostream **ostream_r, + struct ssl_iostream **ssl_iostream_r) +{ + *istream_r = conn->input; + *ostream_r = conn->output; + *ssl_iostream_r = conn->ssl_iostream; + + conn->input = NULL; + conn->output = NULL; + conn->ssl_iostream = NULL; + io_remove(&conn->io); + conn->fd = -1; +} diff --git a/src/doveadm/server-connection.h b/src/doveadm/server-connection.h new file mode 100644 index 0000000..1db30b9 --- /dev/null +++ b/src/doveadm/server-connection.h @@ -0,0 +1,35 @@ +#ifndef SERVER_CONNECTION_H +#define SERVER_CONNECTION_H + +#define SERVER_EXIT_CODE_DISCONNECTED 1000 + +struct doveadm_server; +struct server_connection; +struct ssl_iostream; + +typedef void server_cmd_callback_t(int exit_code, const char *error, + void *context); + +int server_connection_create(struct doveadm_server *server, + struct server_connection **conn_r, + const char **error_r); +void server_connection_destroy(struct server_connection **conn); + +/* Return the server given to create() */ +struct doveadm_server * +server_connection_get_server(struct server_connection *conn); + +void server_connection_cmd(struct server_connection *conn, const char *line, + struct istream *cmd_input, + server_cmd_callback_t *callback, void *context); +/* Returns TRUE if no command is being processed */ +bool server_connection_is_idle(struct server_connection *conn); + +/* Extract iostreams from connection. Afterwards the server_connection simply + waits for itself to be destroyed. */ +void server_connection_extract(struct server_connection *conn, + struct istream **istream_r, + struct ostream **ostream_r, + struct ssl_iostream **ssl_iostream_r); + +#endif diff --git a/src/doveadm/test-doveadm-util.c b/src/doveadm/test-doveadm-util.c new file mode 100644 index 0000000..aec1b39 --- /dev/null +++ b/src/doveadm/test-doveadm-util.c @@ -0,0 +1,48 @@ +/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */ + +#include "lib.h" +#include "test-common.h" +#include "doveadm-settings.h" +#include "doveadm-util.h" + +struct doveadm_settings *doveadm_settings; /* just to avoid linker error */ + +static void test_i_strccdascmp(void) +{ + test_begin("i_strccdascmp()"); + + test_assert(i_strccdascmp("", "")==0); + test_assert(i_strccdascmp("", "-")!=0); + test_assert(i_strccdascmp("-", "")!=0); + test_assert(i_strccdascmp("-", "-")==0); + test_assert(i_strccdascmp("-\0baz", "-\0bar")==0); + test_assert(i_strccdascmp("", "a")!=0); + test_assert(i_strccdascmp("a", "")!=0); + test_assert(i_strccdascmp("a", "a")==0); + test_assert(i_strccdascmp("a-", "a-")==0); + test_assert(i_strccdascmp("a-a", "a-a")==0); + test_assert(i_strccdascmp("ca", "ba")!=0); + + test_assert(i_strccdascmp("camel case", "camel case")==0); + test_assert(i_strccdascmp("camel case", "camel-case")==0); + test_assert(i_strccdascmp("camel case", "camelCase")==0); + + test_assert(i_strccdascmp("camel case", "camel-case")==0); + test_assert(i_strccdascmp("camel-case", "camel-case")==0); + test_assert(i_strccdascmp("camelCase", "camel-case")==0); + + test_assert(i_strccdascmp("camel case", "camel Case")==-i_strccdascmp("camel Case", "camel case")); + test_assert(i_strccdascmp("camel-case", "camel Case")==-i_strccdascmp("camel Case", "camel-case")); + test_assert(i_strccdascmp("camel dase", "camel case")==-i_strccdascmp("camel case", "camel dase")); + + test_end(); +} + +int main(void) +{ + static void (*const test_functions[])(void) = { + test_i_strccdascmp, + NULL + }; + return test_run(test_functions); +} |