summaryrefslogtreecommitdiffstats
path: root/src/doveadm
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 09:51:24 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-28 09:51:24 +0000
commitf7548d6d28c313cf80e6f3ef89aed16a19815df1 (patch)
treea3f6f2a3f247293bee59ecd28e8cd8ceb6ca064a /src/doveadm
parentInitial commit. (diff)
downloaddovecot-f7548d6d28c313cf80e6f3ef89aed16a19815df1.tar.xz
dovecot-f7548d6d28c313cf80e6f3ef89aed16a19815df1.zip
Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rw-r--r--src/doveadm/Makefile.am201
-rw-r--r--src/doveadm/Makefile.in1515
-rw-r--r--src/doveadm/client-connection-http.c1227
-rw-r--r--src/doveadm/client-connection-private.h22
-rw-r--r--src/doveadm/client-connection-tcp.c558
-rw-r--r--src/doveadm/client-connection.c127
-rw-r--r--src/doveadm/client-connection.h26
-rw-r--r--src/doveadm/doveadm-auth-server.c517
-rw-r--r--src/doveadm/doveadm-auth.c787
-rw-r--r--src/doveadm/doveadm-cmd.c469
-rw-r--r--src/doveadm/doveadm-cmd.h155
-rw-r--r--src/doveadm/doveadm-dict.c329
-rw-r--r--src/doveadm/doveadm-director.c1084
-rw-r--r--src/doveadm/doveadm-dsync.c1562
-rw-r--r--src/doveadm/doveadm-dsync.h10
-rw-r--r--src/doveadm/doveadm-dump-dbox.c231
-rw-r--r--src/doveadm/doveadm-dump-dcrypt-file.c93
-rw-r--r--src/doveadm/doveadm-dump-dcrypt-key.c215
-rw-r--r--src/doveadm/doveadm-dump-index.c833
-rw-r--r--src/doveadm/doveadm-dump-log.c568
-rw-r--r--src/doveadm/doveadm-dump-mailboxlog.c114
-rw-r--r--src/doveadm/doveadm-dump-thread.c139
-rw-r--r--src/doveadm/doveadm-dump.c162
-rw-r--r--src/doveadm/doveadm-dump.h27
-rw-r--r--src/doveadm/doveadm-fs.c608
-rw-r--r--src/doveadm/doveadm-instance.c155
-rw-r--r--src/doveadm/doveadm-kick.c235
-rw-r--r--src/doveadm/doveadm-log.c406
-rw-r--r--src/doveadm/doveadm-mail-altmove.c162
-rw-r--r--src/doveadm/doveadm-mail-batch.c186
-rw-r--r--src/doveadm/doveadm-mail-copymove.c224
-rw-r--r--src/doveadm/doveadm-mail-deduplicate.c146
-rw-r--r--src/doveadm/doveadm-mail-expunge.c286
-rw-r--r--src/doveadm/doveadm-mail-fetch.c683
-rw-r--r--src/doveadm/doveadm-mail-flags.c178
-rw-r--r--src/doveadm/doveadm-mail-import.c276
-rw-r--r--src/doveadm/doveadm-mail-index.c300
-rw-r--r--src/doveadm/doveadm-mail-iter.c176
-rw-r--r--src/doveadm/doveadm-mail-iter.h34
-rw-r--r--src/doveadm/doveadm-mail-mailbox-cache.c440
-rw-r--r--src/doveadm/doveadm-mail-mailbox-metadata.c425
-rw-r--r--src/doveadm/doveadm-mail-mailbox-status.c275
-rw-r--r--src/doveadm/doveadm-mail-mailbox.c857
-rw-r--r--src/doveadm/doveadm-mail-rebuild.c101
-rw-r--r--src/doveadm/doveadm-mail-save.c142
-rw-r--r--src/doveadm/doveadm-mail-search.c103
-rw-r--r--src/doveadm/doveadm-mail-server.c404
-rw-r--r--src/doveadm/doveadm-mail.c990
-rw-r--r--src/doveadm/doveadm-mail.h206
-rw-r--r--src/doveadm/doveadm-mailbox-list-iter.c195
-rw-r--r--src/doveadm/doveadm-mailbox-list-iter.h25
-rw-r--r--src/doveadm/doveadm-master.c297
-rw-r--r--src/doveadm/doveadm-mutf7.c56
-rw-r--r--src/doveadm/doveadm-oldstats.c604
-rw-r--r--src/doveadm/doveadm-penalty.c126
-rw-r--r--src/doveadm/doveadm-print-flow.c111
-rw-r--r--src/doveadm/doveadm-print-formatted.c92
-rw-r--r--src/doveadm/doveadm-print-json.c161
-rw-r--r--src/doveadm/doveadm-print-pager.c113
-rw-r--r--src/doveadm/doveadm-print-private.h31
-rw-r--r--src/doveadm/doveadm-print-server.c119
-rw-r--r--src/doveadm/doveadm-print-tab.c76
-rw-r--r--src/doveadm/doveadm-print-table.c268
-rw-r--r--src/doveadm/doveadm-print.c211
-rw-r--r--src/doveadm/doveadm-print.h48
-rw-r--r--src/doveadm/doveadm-proxy.c214
-rw-r--r--src/doveadm/doveadm-pw.c135
-rw-r--r--src/doveadm/doveadm-replicator.c372
-rw-r--r--src/doveadm/doveadm-server.h34
-rw-r--r--src/doveadm/doveadm-settings.c319
-rw-r--r--src/doveadm/doveadm-settings.h65
-rw-r--r--src/doveadm/doveadm-sis.c330
-rw-r--r--src/doveadm/doveadm-stats.c343
-rw-r--r--src/doveadm/doveadm-util.c221
-rw-r--r--src/doveadm/doveadm-util.h31
-rw-r--r--src/doveadm/doveadm-who.c364
-rw-r--r--src/doveadm/doveadm-who.h37
-rw-r--r--src/doveadm/doveadm-zlib.c297
-rw-r--r--src/doveadm/doveadm.c384
-rw-r--r--src/doveadm/doveadm.h34
-rw-r--r--src/doveadm/dsync/Makefile.am76
-rw-r--r--src/doveadm/dsync/Makefile.in1019
-rw-r--r--src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c229
-rw-r--r--src/doveadm/dsync/dsync-brain-mailbox-tree.c624
-rw-r--r--src/doveadm/dsync/dsync-brain-mailbox.c923
-rw-r--r--src/doveadm/dsync/dsync-brain-mails.c438
-rw-r--r--src/doveadm/dsync/dsync-brain-private.h163
-rw-r--r--src/doveadm/dsync/dsync-brain.c901
-rw-r--r--src/doveadm/dsync/dsync-brain.h130
-rw-r--r--src/doveadm/dsync/dsync-deserializer.c193
-rw-r--r--src/doveadm/dsync/dsync-deserializer.h27
-rw-r--r--src/doveadm/dsync/dsync-ibc-pipe.c599
-rw-r--r--src/doveadm/dsync/dsync-ibc-private.h96
-rw-r--r--src/doveadm/dsync/dsync-ibc-stream.c2138
-rw-r--r--src/doveadm/dsync/dsync-ibc.c239
-rw-r--r--src/doveadm/dsync/dsync-ibc.h178
-rw-r--r--src/doveadm/dsync/dsync-mail.c156
-rw-r--r--src/doveadm/dsync/dsync-mail.h108
-rw-r--r--src/doveadm/dsync/dsync-mailbox-export.c961
-rw-r--r--src/doveadm/dsync/dsync-mailbox-export.h37
-rw-r--r--src/doveadm/dsync/dsync-mailbox-import.c2997
-rw-r--r--src/doveadm/dsync/dsync-mailbox-import.h63
-rw-r--r--src/doveadm/dsync/dsync-mailbox-state.c127
-rw-r--r--src/doveadm/dsync/dsync-mailbox-state.h24
-rw-r--r--src/doveadm/dsync/dsync-mailbox-tree-fill.c405
-rw-r--r--src/doveadm/dsync/dsync-mailbox-tree-private.h38
-rw-r--r--src/doveadm/dsync/dsync-mailbox-tree-sync.c1473
-rw-r--r--src/doveadm/dsync/dsync-mailbox-tree.c554
-rw-r--r--src/doveadm/dsync/dsync-mailbox-tree.h205
-rw-r--r--src/doveadm/dsync/dsync-mailbox.c61
-rw-r--r--src/doveadm/dsync/dsync-mailbox.h44
-rw-r--r--src/doveadm/dsync/dsync-serializer.c117
-rw-r--r--src/doveadm/dsync/dsync-serializer.h18
-rw-r--r--src/doveadm/dsync/dsync-transaction-log-scan.c608
-rw-r--r--src/doveadm/dsync/dsync-transaction-log-scan.h32
-rw-r--r--src/doveadm/dsync/test-dsync-mailbox-tree-sync.c781
-rw-r--r--src/doveadm/main.c129
-rw-r--r--src/doveadm/server-connection.c691
-rw-r--r--src/doveadm/server-connection.h35
-rw-r--r--src/doveadm/test-doveadm-util.c48
120 files changed, 42367 insertions, 0 deletions
diff --git a/src/doveadm/Makefile.am b/src/doveadm/Makefile.am
new file mode 100644
index 0000000..440af73
--- /dev/null
+++ b/src/doveadm/Makefile.am
@@ -0,0 +1,201 @@
+doveadm_moduledir = $(moduledir)/doveadm
+pkglibexecdir = $(libexecdir)/dovecot
+
+SUBDIRS = dsync
+
+bin_PROGRAMS = doveadm
+pkglibexec_PROGRAMS = doveadm-server
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-auth \
+ -I$(top_srcdir)/src/lib-compression \
+ -I$(top_srcdir)/src/lib-dict \
+ -I$(top_srcdir)/src/lib-fs \
+ -I$(top_srcdir)/src/lib-ssl-iostream \
+ -I$(top_srcdir)/src/lib-master \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-imap \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage \
+ -I$(top_srcdir)/src/lib-imap-storage \
+ -I$(top_srcdir)/src/lib-http \
+ -I$(top_srcdir)/src/lib-dcrypt \
+ -I$(top_srcdir)/src/auth \
+ -I$(top_srcdir)/src/stats \
+ -DMODULEDIR=\""$(moduledir)"\" \
+ -DAUTH_MODULE_DIR=\""$(moduledir)/auth"\" \
+ -DDOVEADM_MODULEDIR=\""$(doveadm_moduledir)"\" \
+ -DPKG_RUNDIR=\""$(rundir)"\" \
+ -DPKG_STATEDIR=\""$(statedir)"\" \
+ -DPKG_LIBEXECDIR=\""$(pkglibexecdir)"\" \
+ -DBINDIR=\""$(bindir)"\" \
+ -DMANDIR=\""$(mandir)"\" \
+ $(BINARY_CFLAGS)
+
+cmd_pw_libs = \
+ ../auth/libpassword.la \
+ ../lib-otp/libotp.la
+
+libs = \
+ dsync/libdsync.la \
+ ../lib-compression/libcompression.la
+
+doveadm_LDADD = \
+ $(libs) \
+ $(cmd_pw_libs) \
+ $(CRYPT_LIBS) \
+ $(LIBDOVECOT_STORAGE) \
+ $(LIBDOVECOT) \
+ $(LIBSODIUM_LIBS) \
+ $(BINARY_LDFLAGS) \
+ -lm
+
+doveadm_DEPENDENCIES = \
+ $(libs) \
+ $(cmd_pw_libs) \
+ $(LIBDOVECOT_STORAGE_DEPS) \
+ $(LIBDOVECOT_DEPS)
+
+doveadm_server_LDADD = \
+ $(libs) \
+ $(LIBDOVECOT_STORAGE) \
+ $(LIBDOVECOT) \
+ $(BINARY_LDFLAGS) \
+ -lm
+
+doveadm_server_DEPENDENCIES = \
+ $(libs) \
+ $(LIBDOVECOT_STORAGE_DEPS) \
+ $(LIBDOVECOT_DEPS)
+
+doveadm_common_cmds = \
+ doveadm-auth.c \
+ doveadm-dict.c \
+ doveadm-director.c \
+ doveadm-fs.c \
+ doveadm-instance.c \
+ doveadm-kick.c \
+ doveadm-log.c \
+ doveadm-master.c \
+ doveadm-mutf7.c \
+ doveadm-penalty.c \
+ doveadm-proxy.c \
+ doveadm-replicator.c \
+ doveadm-sis.c \
+ doveadm-stats.c \
+ doveadm-oldstats.c \
+ doveadm-who.c
+
+doveadm_common_mail_cmds = \
+ doveadm-dsync.c \
+ doveadm-mail.c \
+ doveadm-mail-altmove.c \
+ doveadm-mail-batch.c \
+ doveadm-mail-deduplicate.c \
+ doveadm-mail-expunge.c \
+ doveadm-mail-fetch.c \
+ doveadm-mail-flags.c \
+ doveadm-mail-import.c \
+ doveadm-mail-index.c \
+ doveadm-mail-iter.c \
+ doveadm-mail-mailbox.c \
+ doveadm-mail-mailbox-metadata.c \
+ doveadm-mail-mailbox-status.c \
+ doveadm-mail-copymove.c \
+ doveadm-mailbox-list-iter.c \
+ doveadm-mail-save.c \
+ doveadm-mail-search.c \
+ doveadm-mail-server.c \
+ doveadm-mail-mailbox-cache.c \
+ doveadm-mail-rebuild.c
+
+# these aren't actually useful in doveadm-server, but plugins may implement
+# both dumping and some other commands inside a single plugin. not having the
+# dump functions in doveadm-server fails to load such plugins.
+doveadm_common_dump_cmds = \
+ doveadm-dump.c \
+ doveadm-dump-dbox.c \
+ doveadm-dump-index.c \
+ doveadm-dump-log.c \
+ doveadm-dump-mailboxlog.c \
+ doveadm-dump-thread.c \
+ doveadm-dump-dcrypt-file.c \
+ doveadm-dump-dcrypt-key.c \
+ doveadm-zlib.c
+
+common = \
+ $(doveadm_common_cmds) \
+ $(doveadm_common_mail_cmds) \
+ $(doveadm_common_dump_cmds) \
+ doveadm-cmd.c \
+ doveadm-print.c \
+ doveadm-settings.c \
+ doveadm-util.c \
+ server-connection.c \
+ doveadm-print-formatted.c
+
+doveadm_SOURCES = \
+ $(common) \
+ doveadm.c \
+ doveadm-print-flow.c \
+ doveadm-print-pager.c \
+ doveadm-print-tab.c \
+ doveadm-print-table.c \
+ doveadm-print-json.c \
+ doveadm-pw.c
+
+doveadm_server_SOURCES = \
+ $(common) \
+ doveadm-auth-server.c \
+ client-connection.c \
+ client-connection-tcp.c \
+ client-connection-http.c \
+ doveadm-print-server.c \
+ doveadm-print-json.c \
+ main.c
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+ doveadm.h \
+ doveadm-cmd.h \
+ doveadm-dsync.h \
+ doveadm-dump.h \
+ doveadm-mail.h \
+ doveadm-mail-iter.h \
+ doveadm-mailbox-list-iter.h \
+ doveadm-print.h \
+ doveadm-print-private.h \
+ doveadm-settings.h \
+ doveadm-util.h
+
+noinst_HEADERS = \
+ client-connection.h \
+ client-connection-private.h \
+ server-connection.h \
+ doveadm-server.h \
+ doveadm-who.h
+
+install-exec-local:
+ rm -f $(DESTDIR)$(bindir)/dsync
+ $(LN_S) doveadm $(DESTDIR)$(bindir)/dsync
+
+test_programs = \
+ test-doveadm-util
+noinst_PROGRAMS = $(test_programs)
+
+test_libs = \
+ ../lib-test/libtest.la \
+ ../lib/liblib.la
+test_deps = $(noinst_LTLIBRARIES) $(test_libs)
+
+test_doveadm_util_SOURCES = doveadm-util.c test-doveadm-util.c
+test_doveadm_util_LDADD = $(test_libs) $(MODULE_LIBS)
+test_doveadm_util_DEPENDENCIES = $(test_deps)
+
+check-local:
+ for bin in $(test_programs); do \
+ if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+ done
diff --git a/src/doveadm/Makefile.in b/src/doveadm/Makefile.in
new file mode 100644
index 0000000..b4e04fd
--- /dev/null
+++ b/src/doveadm/Makefile.in
@@ -0,0 +1,1515 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+ if test -z '$(MAKELEVEL)'; then \
+ false; \
+ elif test -n '$(MAKE_HOST)'; then \
+ true; \
+ elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+ true; \
+ else \
+ false; \
+ fi; \
+}
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+bin_PROGRAMS = doveadm$(EXEEXT)
+pkglibexec_PROGRAMS = doveadm-server$(EXEEXT)
+noinst_PROGRAMS = $(am__EXEEXT_1)
+subdir = src/doveadm
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+ $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+ $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+ $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+ $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+ $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+ $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+ $(top_srcdir)/m4/flexible_array_member.m4 \
+ $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+ $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+ $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+ $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+ $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+ $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+ $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+ $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+ $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+ $(top_srcdir)/m4/pr_set_dumpable.m4 \
+ $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+ $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+ $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+ $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+ $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+ $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+ $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+ $(top_srcdir)/m4/typeof_dev_t.m4 \
+ $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+ $(top_srcdir)/m4/want_apparmor.m4 \
+ $(top_srcdir)/m4/want_bsdauth.m4 \
+ $(top_srcdir)/m4/want_bzlib.m4 \
+ $(top_srcdir)/m4/want_cassandra.m4 \
+ $(top_srcdir)/m4/want_cdb.m4 \
+ $(top_srcdir)/m4/want_checkpassword.m4 \
+ $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+ $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+ $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+ $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+ $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+ $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+ $(top_srcdir)/m4/want_prefetch.m4 \
+ $(top_srcdir)/m4/want_shadow.m4 \
+ $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+ $(top_srcdir)/m4/want_sqlite.m4 \
+ $(top_srcdir)/m4/want_stemmer.m4 \
+ $(top_srcdir)/m4/want_systemd.m4 \
+ $(top_srcdir)/m4/want_textcat.m4 \
+ $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+ $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+ $(pkginc_lib_HEADERS) $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkglibexecdir)" \
+ "$(DESTDIR)$(pkginc_libdir)"
+am__EXEEXT_1 = test-doveadm-util$(EXEEXT)
+PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) $(pkglibexec_PROGRAMS)
+am__objects_1 = doveadm-auth.$(OBJEXT) doveadm-dict.$(OBJEXT) \
+ doveadm-director.$(OBJEXT) doveadm-fs.$(OBJEXT) \
+ doveadm-instance.$(OBJEXT) doveadm-kick.$(OBJEXT) \
+ doveadm-log.$(OBJEXT) doveadm-master.$(OBJEXT) \
+ doveadm-mutf7.$(OBJEXT) doveadm-penalty.$(OBJEXT) \
+ doveadm-proxy.$(OBJEXT) doveadm-replicator.$(OBJEXT) \
+ doveadm-sis.$(OBJEXT) doveadm-stats.$(OBJEXT) \
+ doveadm-oldstats.$(OBJEXT) doveadm-who.$(OBJEXT)
+am__objects_2 = doveadm-dsync.$(OBJEXT) doveadm-mail.$(OBJEXT) \
+ doveadm-mail-altmove.$(OBJEXT) doveadm-mail-batch.$(OBJEXT) \
+ doveadm-mail-deduplicate.$(OBJEXT) \
+ doveadm-mail-expunge.$(OBJEXT) doveadm-mail-fetch.$(OBJEXT) \
+ doveadm-mail-flags.$(OBJEXT) doveadm-mail-import.$(OBJEXT) \
+ doveadm-mail-index.$(OBJEXT) doveadm-mail-iter.$(OBJEXT) \
+ doveadm-mail-mailbox.$(OBJEXT) \
+ doveadm-mail-mailbox-metadata.$(OBJEXT) \
+ doveadm-mail-mailbox-status.$(OBJEXT) \
+ doveadm-mail-copymove.$(OBJEXT) \
+ doveadm-mailbox-list-iter.$(OBJEXT) \
+ doveadm-mail-save.$(OBJEXT) doveadm-mail-search.$(OBJEXT) \
+ doveadm-mail-server.$(OBJEXT) \
+ doveadm-mail-mailbox-cache.$(OBJEXT) \
+ doveadm-mail-rebuild.$(OBJEXT)
+am__objects_3 = doveadm-dump.$(OBJEXT) doveadm-dump-dbox.$(OBJEXT) \
+ doveadm-dump-index.$(OBJEXT) doveadm-dump-log.$(OBJEXT) \
+ doveadm-dump-mailboxlog.$(OBJEXT) \
+ doveadm-dump-thread.$(OBJEXT) \
+ doveadm-dump-dcrypt-file.$(OBJEXT) \
+ doveadm-dump-dcrypt-key.$(OBJEXT) doveadm-zlib.$(OBJEXT)
+am__objects_4 = $(am__objects_1) $(am__objects_2) $(am__objects_3) \
+ doveadm-cmd.$(OBJEXT) doveadm-print.$(OBJEXT) \
+ doveadm-settings.$(OBJEXT) doveadm-util.$(OBJEXT) \
+ server-connection.$(OBJEXT) doveadm-print-formatted.$(OBJEXT)
+am_doveadm_OBJECTS = $(am__objects_4) doveadm.$(OBJEXT) \
+ doveadm-print-flow.$(OBJEXT) doveadm-print-pager.$(OBJEXT) \
+ doveadm-print-tab.$(OBJEXT) doveadm-print-table.$(OBJEXT) \
+ doveadm-print-json.$(OBJEXT) doveadm-pw.$(OBJEXT)
+doveadm_OBJECTS = $(am_doveadm_OBJECTS)
+am__DEPENDENCIES_1 =
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 =
+am_doveadm_server_OBJECTS = $(am__objects_4) \
+ doveadm-auth-server.$(OBJEXT) client-connection.$(OBJEXT) \
+ client-connection-tcp.$(OBJEXT) \
+ client-connection-http.$(OBJEXT) \
+ doveadm-print-server.$(OBJEXT) doveadm-print-json.$(OBJEXT) \
+ main.$(OBJEXT)
+doveadm_server_OBJECTS = $(am_doveadm_server_OBJECTS)
+am_test_doveadm_util_OBJECTS = doveadm-util.$(OBJEXT) \
+ test-doveadm-util.$(OBJEXT)
+test_doveadm_util_OBJECTS = $(am_test_doveadm_util_OBJECTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/client-connection-http.Po \
+ ./$(DEPDIR)/client-connection-tcp.Po \
+ ./$(DEPDIR)/client-connection.Po \
+ ./$(DEPDIR)/doveadm-auth-server.Po ./$(DEPDIR)/doveadm-auth.Po \
+ ./$(DEPDIR)/doveadm-cmd.Po ./$(DEPDIR)/doveadm-dict.Po \
+ ./$(DEPDIR)/doveadm-director.Po ./$(DEPDIR)/doveadm-dsync.Po \
+ ./$(DEPDIR)/doveadm-dump-dbox.Po \
+ ./$(DEPDIR)/doveadm-dump-dcrypt-file.Po \
+ ./$(DEPDIR)/doveadm-dump-dcrypt-key.Po \
+ ./$(DEPDIR)/doveadm-dump-index.Po \
+ ./$(DEPDIR)/doveadm-dump-log.Po \
+ ./$(DEPDIR)/doveadm-dump-mailboxlog.Po \
+ ./$(DEPDIR)/doveadm-dump-thread.Po ./$(DEPDIR)/doveadm-dump.Po \
+ ./$(DEPDIR)/doveadm-fs.Po ./$(DEPDIR)/doveadm-instance.Po \
+ ./$(DEPDIR)/doveadm-kick.Po ./$(DEPDIR)/doveadm-log.Po \
+ ./$(DEPDIR)/doveadm-mail-altmove.Po \
+ ./$(DEPDIR)/doveadm-mail-batch.Po \
+ ./$(DEPDIR)/doveadm-mail-copymove.Po \
+ ./$(DEPDIR)/doveadm-mail-deduplicate.Po \
+ ./$(DEPDIR)/doveadm-mail-expunge.Po \
+ ./$(DEPDIR)/doveadm-mail-fetch.Po \
+ ./$(DEPDIR)/doveadm-mail-flags.Po \
+ ./$(DEPDIR)/doveadm-mail-import.Po \
+ ./$(DEPDIR)/doveadm-mail-index.Po \
+ ./$(DEPDIR)/doveadm-mail-iter.Po \
+ ./$(DEPDIR)/doveadm-mail-mailbox-cache.Po \
+ ./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po \
+ ./$(DEPDIR)/doveadm-mail-mailbox-status.Po \
+ ./$(DEPDIR)/doveadm-mail-mailbox.Po \
+ ./$(DEPDIR)/doveadm-mail-rebuild.Po \
+ ./$(DEPDIR)/doveadm-mail-save.Po \
+ ./$(DEPDIR)/doveadm-mail-search.Po \
+ ./$(DEPDIR)/doveadm-mail-server.Po ./$(DEPDIR)/doveadm-mail.Po \
+ ./$(DEPDIR)/doveadm-mailbox-list-iter.Po \
+ ./$(DEPDIR)/doveadm-master.Po ./$(DEPDIR)/doveadm-mutf7.Po \
+ ./$(DEPDIR)/doveadm-oldstats.Po ./$(DEPDIR)/doveadm-penalty.Po \
+ ./$(DEPDIR)/doveadm-print-flow.Po \
+ ./$(DEPDIR)/doveadm-print-formatted.Po \
+ ./$(DEPDIR)/doveadm-print-json.Po \
+ ./$(DEPDIR)/doveadm-print-pager.Po \
+ ./$(DEPDIR)/doveadm-print-server.Po \
+ ./$(DEPDIR)/doveadm-print-tab.Po \
+ ./$(DEPDIR)/doveadm-print-table.Po \
+ ./$(DEPDIR)/doveadm-print.Po ./$(DEPDIR)/doveadm-proxy.Po \
+ ./$(DEPDIR)/doveadm-pw.Po ./$(DEPDIR)/doveadm-replicator.Po \
+ ./$(DEPDIR)/doveadm-settings.Po ./$(DEPDIR)/doveadm-sis.Po \
+ ./$(DEPDIR)/doveadm-stats.Po ./$(DEPDIR)/doveadm-util.Po \
+ ./$(DEPDIR)/doveadm-who.Po ./$(DEPDIR)/doveadm-zlib.Po \
+ ./$(DEPDIR)/doveadm.Po ./$(DEPDIR)/main.Po \
+ ./$(DEPDIR)/server-connection.Po \
+ ./$(DEPDIR)/test-doveadm-util.Po
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+ $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo " CC " $@;
+am__v_CC_1 =
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo " CCLD " $@;
+am__v_CCLD_1 =
+SOURCES = $(doveadm_SOURCES) $(doveadm_server_SOURCES) \
+ $(test_doveadm_util_SOURCES)
+DIST_SOURCES = $(doveadm_SOURCES) $(doveadm_server_SOURCES) \
+ $(test_doveadm_util_SOURCES)
+RECURSIVE_TARGETS = all-recursive check-recursive cscopelist-recursive \
+ ctags-recursive dvi-recursive html-recursive info-recursive \
+ install-data-recursive install-dvi-recursive \
+ install-exec-recursive install-html-recursive \
+ install-info-recursive install-pdf-recursive \
+ install-ps-recursive install-recursive installcheck-recursive \
+ installdirs-recursive pdf-recursive ps-recursive \
+ tags-recursive uninstall-recursive
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+ test -z "$$files" \
+ || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+ $(am__cd) "$$dir" && rm -f $$files; }; \
+ }
+HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS)
+RECURSIVE_CLEAN_TARGETS = mostlyclean-recursive clean-recursive \
+ distclean-recursive maintainer-clean-recursive
+am__recursive_targets = \
+ $(RECURSIVE_TARGETS) \
+ $(RECURSIVE_CLEAN_TARGETS) \
+ $(am__extra_recursive_targets)
+AM_RECURSIVE_TARGETS = $(am__recursive_targets:-recursive=) TAGS CTAGS \
+ distdir distdir-am
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates. Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+ BEGIN { nonempty = 0; } \
+ { items[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique. This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+ list='$(am__tagged_files)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+DIST_SUBDIRS = $(SUBDIRS)
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+am__relativize = \
+ dir0=`pwd`; \
+ sed_first='s,^\([^/]*\)/.*$$,\1,'; \
+ sed_rest='s,^[^/]*/*,,'; \
+ sed_last='s,^.*/\([^/]*\)$$,\1,'; \
+ sed_butlast='s,/*[^/]*$$,,'; \
+ while test -n "$$dir1"; do \
+ first=`echo "$$dir1" | sed -e "$$sed_first"`; \
+ if test "$$first" != "."; then \
+ if test "$$first" = ".."; then \
+ dir2=`echo "$$dir0" | sed -e "$$sed_last"`/"$$dir2"; \
+ dir0=`echo "$$dir0" | sed -e "$$sed_butlast"`; \
+ else \
+ first2=`echo "$$dir2" | sed -e "$$sed_first"`; \
+ if test "$$first2" = "$$first"; then \
+ dir2=`echo "$$dir2" | sed -e "$$sed_rest"`; \
+ else \
+ dir2="../$$dir2"; \
+ fi; \
+ dir0="$$dir0"/"$$first"; \
+ fi; \
+ fi; \
+ dir1=`echo "$$dir1" | sed -e "$$sed_rest"`; \
+ done; \
+ reldir="$$dir2"
+pkglibexecdir = $(libexecdir)/dovecot
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+doveadm_moduledir = $(moduledir)/doveadm
+SUBDIRS = dsync
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-auth \
+ -I$(top_srcdir)/src/lib-compression \
+ -I$(top_srcdir)/src/lib-dict \
+ -I$(top_srcdir)/src/lib-fs \
+ -I$(top_srcdir)/src/lib-ssl-iostream \
+ -I$(top_srcdir)/src/lib-master \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-imap \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage \
+ -I$(top_srcdir)/src/lib-imap-storage \
+ -I$(top_srcdir)/src/lib-http \
+ -I$(top_srcdir)/src/lib-dcrypt \
+ -I$(top_srcdir)/src/auth \
+ -I$(top_srcdir)/src/stats \
+ -DMODULEDIR=\""$(moduledir)"\" \
+ -DAUTH_MODULE_DIR=\""$(moduledir)/auth"\" \
+ -DDOVEADM_MODULEDIR=\""$(doveadm_moduledir)"\" \
+ -DPKG_RUNDIR=\""$(rundir)"\" \
+ -DPKG_STATEDIR=\""$(statedir)"\" \
+ -DPKG_LIBEXECDIR=\""$(pkglibexecdir)"\" \
+ -DBINDIR=\""$(bindir)"\" \
+ -DMANDIR=\""$(mandir)"\" \
+ $(BINARY_CFLAGS)
+
+cmd_pw_libs = \
+ ../auth/libpassword.la \
+ ../lib-otp/libotp.la
+
+libs = \
+ dsync/libdsync.la \
+ ../lib-compression/libcompression.la
+
+doveadm_LDADD = \
+ $(libs) \
+ $(cmd_pw_libs) \
+ $(CRYPT_LIBS) \
+ $(LIBDOVECOT_STORAGE) \
+ $(LIBDOVECOT) \
+ $(LIBSODIUM_LIBS) \
+ $(BINARY_LDFLAGS) \
+ -lm
+
+doveadm_DEPENDENCIES = \
+ $(libs) \
+ $(cmd_pw_libs) \
+ $(LIBDOVECOT_STORAGE_DEPS) \
+ $(LIBDOVECOT_DEPS)
+
+doveadm_server_LDADD = \
+ $(libs) \
+ $(LIBDOVECOT_STORAGE) \
+ $(LIBDOVECOT) \
+ $(BINARY_LDFLAGS) \
+ -lm
+
+doveadm_server_DEPENDENCIES = \
+ $(libs) \
+ $(LIBDOVECOT_STORAGE_DEPS) \
+ $(LIBDOVECOT_DEPS)
+
+doveadm_common_cmds = \
+ doveadm-auth.c \
+ doveadm-dict.c \
+ doveadm-director.c \
+ doveadm-fs.c \
+ doveadm-instance.c \
+ doveadm-kick.c \
+ doveadm-log.c \
+ doveadm-master.c \
+ doveadm-mutf7.c \
+ doveadm-penalty.c \
+ doveadm-proxy.c \
+ doveadm-replicator.c \
+ doveadm-sis.c \
+ doveadm-stats.c \
+ doveadm-oldstats.c \
+ doveadm-who.c
+
+doveadm_common_mail_cmds = \
+ doveadm-dsync.c \
+ doveadm-mail.c \
+ doveadm-mail-altmove.c \
+ doveadm-mail-batch.c \
+ doveadm-mail-deduplicate.c \
+ doveadm-mail-expunge.c \
+ doveadm-mail-fetch.c \
+ doveadm-mail-flags.c \
+ doveadm-mail-import.c \
+ doveadm-mail-index.c \
+ doveadm-mail-iter.c \
+ doveadm-mail-mailbox.c \
+ doveadm-mail-mailbox-metadata.c \
+ doveadm-mail-mailbox-status.c \
+ doveadm-mail-copymove.c \
+ doveadm-mailbox-list-iter.c \
+ doveadm-mail-save.c \
+ doveadm-mail-search.c \
+ doveadm-mail-server.c \
+ doveadm-mail-mailbox-cache.c \
+ doveadm-mail-rebuild.c
+
+
+# these aren't actually useful in doveadm-server, but plugins may implement
+# both dumping and some other commands inside a single plugin. not having the
+# dump functions in doveadm-server fails to load such plugins.
+doveadm_common_dump_cmds = \
+ doveadm-dump.c \
+ doveadm-dump-dbox.c \
+ doveadm-dump-index.c \
+ doveadm-dump-log.c \
+ doveadm-dump-mailboxlog.c \
+ doveadm-dump-thread.c \
+ doveadm-dump-dcrypt-file.c \
+ doveadm-dump-dcrypt-key.c \
+ doveadm-zlib.c
+
+common = \
+ $(doveadm_common_cmds) \
+ $(doveadm_common_mail_cmds) \
+ $(doveadm_common_dump_cmds) \
+ doveadm-cmd.c \
+ doveadm-print.c \
+ doveadm-settings.c \
+ doveadm-util.c \
+ server-connection.c \
+ doveadm-print-formatted.c
+
+doveadm_SOURCES = \
+ $(common) \
+ doveadm.c \
+ doveadm-print-flow.c \
+ doveadm-print-pager.c \
+ doveadm-print-tab.c \
+ doveadm-print-table.c \
+ doveadm-print-json.c \
+ doveadm-pw.c
+
+doveadm_server_SOURCES = \
+ $(common) \
+ doveadm-auth-server.c \
+ client-connection.c \
+ client-connection-tcp.c \
+ client-connection-http.c \
+ doveadm-print-server.c \
+ doveadm-print-json.c \
+ main.c
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+ doveadm.h \
+ doveadm-cmd.h \
+ doveadm-dsync.h \
+ doveadm-dump.h \
+ doveadm-mail.h \
+ doveadm-mail-iter.h \
+ doveadm-mailbox-list-iter.h \
+ doveadm-print.h \
+ doveadm-print-private.h \
+ doveadm-settings.h \
+ doveadm-util.h
+
+noinst_HEADERS = \
+ client-connection.h \
+ client-connection-private.h \
+ server-connection.h \
+ doveadm-server.h \
+ doveadm-who.h
+
+test_programs = \
+ test-doveadm-util
+
+test_libs = \
+ ../lib-test/libtest.la \
+ ../lib/liblib.la
+
+test_deps = $(noinst_LTLIBRARIES) $(test_libs)
+test_doveadm_util_SOURCES = doveadm-util.c test-doveadm-util.c
+test_doveadm_util_LDADD = $(test_libs) $(MODULE_LIBS)
+test_doveadm_util_DEPENDENCIES = $(test_deps)
+all: all-recursive
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/doveadm/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --foreign src/doveadm/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+install-binPROGRAMS: $(bin_PROGRAMS)
+ @$(NORMAL_INSTALL)
+ @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \
+ fi; \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed 's/$(EXEEXT)$$//' | \
+ while read p p1; do if test -f $$p \
+ || test -f $$p1 \
+ ; then echo "$$p"; echo "$$p"; else :; fi; \
+ done | \
+ sed -e 'p;s,.*/,,;n;h' \
+ -e 's|.*|.|' \
+ -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
+ sed 'N;N;N;s,\n, ,g' | \
+ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
+ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+ if ($$2 == $$4) files[d] = files[d] " " $$1; \
+ else { print "f", $$3 "/" $$4, $$1; } } \
+ END { for (d in files) print "f", d, files[d] }' | \
+ while read type dir files; do \
+ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+ test -z "$$files" || { \
+ echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \
+ $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \
+ } \
+ ; done
+
+uninstall-binPROGRAMS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \
+ files=`for p in $$list; do echo "$$p"; done | \
+ sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
+ -e 's/$$/$(EXEEXT)/' \
+ `; \
+ test -n "$$list" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(bindir)" && rm -f $$files
+
+clean-binPROGRAMS:
+ @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+
+clean-noinstPROGRAMS:
+ @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+install-pkglibexecPROGRAMS: $(pkglibexec_PROGRAMS)
+ @$(NORMAL_INSTALL)
+ @list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pkglibexecdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pkglibexecdir)" || exit 1; \
+ fi; \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed 's/$(EXEEXT)$$//' | \
+ while read p p1; do if test -f $$p \
+ || test -f $$p1 \
+ ; then echo "$$p"; echo "$$p"; else :; fi; \
+ done | \
+ sed -e 'p;s,.*/,,;n;h' \
+ -e 's|.*|.|' \
+ -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \
+ sed 'N;N;N;s,\n, ,g' | \
+ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \
+ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \
+ if ($$2 == $$4) files[d] = files[d] " " $$1; \
+ else { print "f", $$3 "/" $$4, $$1; } } \
+ END { for (d in files) print "f", d, files[d] }' | \
+ while read type dir files; do \
+ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \
+ test -z "$$files" || { \
+ echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(pkglibexecdir)$$dir'"; \
+ $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(pkglibexecdir)$$dir" || exit $$?; \
+ } \
+ ; done
+
+uninstall-pkglibexecPROGRAMS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkglibexec_PROGRAMS)'; test -n "$(pkglibexecdir)" || list=; \
+ files=`for p in $$list; do echo "$$p"; done | \
+ sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \
+ -e 's/$$/$(EXEEXT)/' \
+ `; \
+ test -n "$$list" || exit 0; \
+ echo " ( cd '$(DESTDIR)$(pkglibexecdir)' && rm -f" $$files ")"; \
+ cd "$(DESTDIR)$(pkglibexecdir)" && rm -f $$files
+
+clean-pkglibexecPROGRAMS:
+ @list='$(pkglibexec_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+
+doveadm$(EXEEXT): $(doveadm_OBJECTS) $(doveadm_DEPENDENCIES) $(EXTRA_doveadm_DEPENDENCIES)
+ @rm -f doveadm$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(doveadm_OBJECTS) $(doveadm_LDADD) $(LIBS)
+
+doveadm-server$(EXEEXT): $(doveadm_server_OBJECTS) $(doveadm_server_DEPENDENCIES) $(EXTRA_doveadm_server_DEPENDENCIES)
+ @rm -f doveadm-server$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(doveadm_server_OBJECTS) $(doveadm_server_LDADD) $(LIBS)
+
+test-doveadm-util$(EXEEXT): $(test_doveadm_util_OBJECTS) $(test_doveadm_util_DEPENDENCIES) $(EXTRA_test_doveadm_util_DEPENDENCIES)
+ @rm -f test-doveadm-util$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(test_doveadm_util_OBJECTS) $(test_doveadm_util_LDADD) $(LIBS)
+
+mostlyclean-compile:
+ -rm -f *.$(OBJEXT)
+
+distclean-compile:
+ -rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-connection-http.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-connection-tcp.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/client-connection.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-auth-server.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-auth.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-cmd.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dict.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-director.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dsync.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-dbox.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-dcrypt-file.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-dcrypt-key.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-index.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-log.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-mailboxlog.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump-thread.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-dump.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-fs.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-instance.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-kick.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-log.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-altmove.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-batch.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-copymove.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-deduplicate.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-expunge.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-fetch.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-flags.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-import.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-index.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-iter.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox-cache.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox-status.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-mailbox.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-rebuild.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-save.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-search.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail-server.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mail.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mailbox-list-iter.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-master.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-mutf7.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-oldstats.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-penalty.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-flow.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-formatted.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-json.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-pager.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-server.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-tab.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print-table.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-print.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-proxy.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-pw.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-replicator.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-settings.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-sis.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-stats.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-util.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-who.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm-zlib.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/doveadm.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/main.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/server-connection.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-doveadm-util.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+ @$(MKDIR_P) $(@D)
+ @echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+ -rm -f *.lo
+
+clean-libtool:
+ -rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+ @$(NORMAL_INSTALL)
+ @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+ $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+ done
+
+uninstall-pkginc_libHEADERS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+# This directory's subdirectories are mostly independent; you can cd
+# into them and run 'make' without going through this Makefile.
+# To change the values of 'make' variables: instead of editing Makefiles,
+# (1) if the variable is set in 'config.status', edit 'config.status'
+# (which will cause the Makefiles to be regenerated when you run 'make');
+# (2) otherwise, pass the desired values on the 'make' command line.
+$(am__recursive_targets):
+ @fail=; \
+ if $(am__make_keepgoing); then \
+ failcom='fail=yes'; \
+ else \
+ failcom='exit 1'; \
+ fi; \
+ dot_seen=no; \
+ target=`echo $@ | sed s/-recursive//`; \
+ case "$@" in \
+ distclean-* | maintainer-clean-*) list='$(DIST_SUBDIRS)' ;; \
+ *) list='$(SUBDIRS)' ;; \
+ esac; \
+ for subdir in $$list; do \
+ echo "Making $$target in $$subdir"; \
+ if test "$$subdir" = "."; then \
+ dot_seen=yes; \
+ local_target="$$target-am"; \
+ else \
+ local_target="$$target"; \
+ fi; \
+ ($(am__cd) $$subdir && $(MAKE) $(AM_MAKEFLAGS) $$local_target) \
+ || eval $$failcom; \
+ done; \
+ if test "$$dot_seen" = "no"; then \
+ $(MAKE) $(AM_MAKEFLAGS) "$$target-am" || exit 1; \
+ fi; test -z "$$fail"
+
+ID: $(am__tagged_files)
+ $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-recursive
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ set x; \
+ here=`pwd`; \
+ if ($(ETAGS) --etags-include --version) >/dev/null 2>&1; then \
+ include_option=--etags-include; \
+ empty_fix=.; \
+ else \
+ include_option=--include; \
+ empty_fix=; \
+ fi; \
+ list='$(SUBDIRS)'; for subdir in $$list; do \
+ if test "$$subdir" = .; then :; else \
+ test ! -f $$subdir/TAGS || \
+ set "$$@" "$$include_option=$$here/$$subdir/TAGS"; \
+ fi; \
+ done; \
+ $(am__define_uniq_tagged_files); \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: ctags-recursive
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ $(am__define_uniq_tagged_files); \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-recursive
+
+cscopelist-am: $(am__tagged_files)
+ list='$(am__tagged_files)'; \
+ case "$(srcdir)" in \
+ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+ *) sdir=$(subdir)/$(srcdir) ;; \
+ esac; \
+ for i in $$list; do \
+ if test -f "$$i"; then \
+ echo "$(subdir)/$$i"; \
+ else \
+ echo "$$sdir/$$i"; \
+ fi; \
+ done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+ @list='$(DIST_SUBDIRS)'; for subdir in $$list; do \
+ if test "$$subdir" = .; then :; else \
+ $(am__make_dryrun) \
+ || test -d "$(distdir)/$$subdir" \
+ || $(MKDIR_P) "$(distdir)/$$subdir" \
+ || exit 1; \
+ dir1=$$subdir; dir2="$(distdir)/$$subdir"; \
+ $(am__relativize); \
+ new_distdir=$$reldir; \
+ dir1=$$subdir; dir2="$(top_distdir)"; \
+ $(am__relativize); \
+ new_top_distdir=$$reldir; \
+ echo " (cd $$subdir && $(MAKE) $(AM_MAKEFLAGS) top_distdir="$$new_top_distdir" distdir="$$new_distdir" \\"; \
+ echo " am__remove_distdir=: am__skip_length_check=: am__skip_mode_fix=: distdir)"; \
+ ($(am__cd) $$subdir && \
+ $(MAKE) $(AM_MAKEFLAGS) \
+ top_distdir="$$new_top_distdir" \
+ distdir="$$new_distdir" \
+ am__remove_distdir=: \
+ am__skip_length_check=: \
+ am__skip_mode_fix=: \
+ distdir) \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+ $(MAKE) $(AM_MAKEFLAGS) check-local
+check: check-recursive
+all-am: Makefile $(PROGRAMS) $(HEADERS)
+installdirs: installdirs-recursive
+installdirs-am:
+ for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(pkglibexecdir)" "$(DESTDIR)$(pkginc_libdir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-recursive
+install-exec: install-exec-recursive
+install-data: install-data-recursive
+uninstall: uninstall-recursive
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-recursive
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+clean: clean-recursive
+
+clean-am: clean-binPROGRAMS clean-generic clean-libtool \
+ clean-noinstPROGRAMS clean-pkglibexecPROGRAMS mostlyclean-am
+
+distclean: distclean-recursive
+ -rm -f ./$(DEPDIR)/client-connection-http.Po
+ -rm -f ./$(DEPDIR)/client-connection-tcp.Po
+ -rm -f ./$(DEPDIR)/client-connection.Po
+ -rm -f ./$(DEPDIR)/doveadm-auth-server.Po
+ -rm -f ./$(DEPDIR)/doveadm-auth.Po
+ -rm -f ./$(DEPDIR)/doveadm-cmd.Po
+ -rm -f ./$(DEPDIR)/doveadm-dict.Po
+ -rm -f ./$(DEPDIR)/doveadm-director.Po
+ -rm -f ./$(DEPDIR)/doveadm-dsync.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-dbox.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-file.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-key.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-index.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-log.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-mailboxlog.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-thread.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump.Po
+ -rm -f ./$(DEPDIR)/doveadm-fs.Po
+ -rm -f ./$(DEPDIR)/doveadm-instance.Po
+ -rm -f ./$(DEPDIR)/doveadm-kick.Po
+ -rm -f ./$(DEPDIR)/doveadm-log.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-altmove.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-batch.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-copymove.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-deduplicate.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-expunge.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-fetch.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-flags.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-import.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-index.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-iter.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-cache.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-status.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-rebuild.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-save.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-search.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-server.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail.Po
+ -rm -f ./$(DEPDIR)/doveadm-mailbox-list-iter.Po
+ -rm -f ./$(DEPDIR)/doveadm-master.Po
+ -rm -f ./$(DEPDIR)/doveadm-mutf7.Po
+ -rm -f ./$(DEPDIR)/doveadm-oldstats.Po
+ -rm -f ./$(DEPDIR)/doveadm-penalty.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-flow.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-formatted.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-json.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-pager.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-server.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-tab.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-table.Po
+ -rm -f ./$(DEPDIR)/doveadm-print.Po
+ -rm -f ./$(DEPDIR)/doveadm-proxy.Po
+ -rm -f ./$(DEPDIR)/doveadm-pw.Po
+ -rm -f ./$(DEPDIR)/doveadm-replicator.Po
+ -rm -f ./$(DEPDIR)/doveadm-settings.Po
+ -rm -f ./$(DEPDIR)/doveadm-sis.Po
+ -rm -f ./$(DEPDIR)/doveadm-stats.Po
+ -rm -f ./$(DEPDIR)/doveadm-util.Po
+ -rm -f ./$(DEPDIR)/doveadm-who.Po
+ -rm -f ./$(DEPDIR)/doveadm-zlib.Po
+ -rm -f ./$(DEPDIR)/doveadm.Po
+ -rm -f ./$(DEPDIR)/main.Po
+ -rm -f ./$(DEPDIR)/server-connection.Po
+ -rm -f ./$(DEPDIR)/test-doveadm-util.Po
+ -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+ distclean-tags
+
+dvi: dvi-recursive
+
+dvi-am:
+
+html: html-recursive
+
+html-am:
+
+info: info-recursive
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-recursive
+
+install-dvi-am:
+
+install-exec-am: install-binPROGRAMS install-exec-local \
+ install-pkglibexecPROGRAMS
+
+install-html: install-html-recursive
+
+install-html-am:
+
+install-info: install-info-recursive
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-recursive
+
+install-pdf-am:
+
+install-ps: install-ps-recursive
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-recursive
+ -rm -f ./$(DEPDIR)/client-connection-http.Po
+ -rm -f ./$(DEPDIR)/client-connection-tcp.Po
+ -rm -f ./$(DEPDIR)/client-connection.Po
+ -rm -f ./$(DEPDIR)/doveadm-auth-server.Po
+ -rm -f ./$(DEPDIR)/doveadm-auth.Po
+ -rm -f ./$(DEPDIR)/doveadm-cmd.Po
+ -rm -f ./$(DEPDIR)/doveadm-dict.Po
+ -rm -f ./$(DEPDIR)/doveadm-director.Po
+ -rm -f ./$(DEPDIR)/doveadm-dsync.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-dbox.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-file.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-dcrypt-key.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-index.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-log.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-mailboxlog.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump-thread.Po
+ -rm -f ./$(DEPDIR)/doveadm-dump.Po
+ -rm -f ./$(DEPDIR)/doveadm-fs.Po
+ -rm -f ./$(DEPDIR)/doveadm-instance.Po
+ -rm -f ./$(DEPDIR)/doveadm-kick.Po
+ -rm -f ./$(DEPDIR)/doveadm-log.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-altmove.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-batch.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-copymove.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-deduplicate.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-expunge.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-fetch.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-flags.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-import.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-index.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-iter.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-cache.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-metadata.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox-status.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-mailbox.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-rebuild.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-save.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-search.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail-server.Po
+ -rm -f ./$(DEPDIR)/doveadm-mail.Po
+ -rm -f ./$(DEPDIR)/doveadm-mailbox-list-iter.Po
+ -rm -f ./$(DEPDIR)/doveadm-master.Po
+ -rm -f ./$(DEPDIR)/doveadm-mutf7.Po
+ -rm -f ./$(DEPDIR)/doveadm-oldstats.Po
+ -rm -f ./$(DEPDIR)/doveadm-penalty.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-flow.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-formatted.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-json.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-pager.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-server.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-tab.Po
+ -rm -f ./$(DEPDIR)/doveadm-print-table.Po
+ -rm -f ./$(DEPDIR)/doveadm-print.Po
+ -rm -f ./$(DEPDIR)/doveadm-proxy.Po
+ -rm -f ./$(DEPDIR)/doveadm-pw.Po
+ -rm -f ./$(DEPDIR)/doveadm-replicator.Po
+ -rm -f ./$(DEPDIR)/doveadm-settings.Po
+ -rm -f ./$(DEPDIR)/doveadm-sis.Po
+ -rm -f ./$(DEPDIR)/doveadm-stats.Po
+ -rm -f ./$(DEPDIR)/doveadm-util.Po
+ -rm -f ./$(DEPDIR)/doveadm-who.Po
+ -rm -f ./$(DEPDIR)/doveadm-zlib.Po
+ -rm -f ./$(DEPDIR)/doveadm.Po
+ -rm -f ./$(DEPDIR)/main.Po
+ -rm -f ./$(DEPDIR)/server-connection.Po
+ -rm -f ./$(DEPDIR)/test-doveadm-util.Po
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-recursive
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool
+
+pdf: pdf-recursive
+
+pdf-am:
+
+ps: ps-recursive
+
+ps-am:
+
+uninstall-am: uninstall-binPROGRAMS uninstall-pkginc_libHEADERS \
+ uninstall-pkglibexecPROGRAMS
+
+.MAKE: $(am__recursive_targets) check-am install-am install-strip
+
+.PHONY: $(am__recursive_targets) CTAGS GTAGS TAGS all all-am \
+ am--depfiles check check-am check-local clean \
+ clean-binPROGRAMS clean-generic clean-libtool \
+ clean-noinstPROGRAMS clean-pkglibexecPROGRAMS cscopelist-am \
+ ctags ctags-am distclean distclean-compile distclean-generic \
+ distclean-libtool distclean-tags distdir dvi dvi-am html \
+ html-am info info-am install install-am install-binPROGRAMS \
+ install-data install-data-am install-dvi install-dvi-am \
+ install-exec install-exec-am install-exec-local install-html \
+ install-html-am install-info install-info-am install-man \
+ install-pdf install-pdf-am install-pkginc_libHEADERS \
+ install-pkglibexecPROGRAMS install-ps install-ps-am \
+ install-strip installcheck installcheck-am installdirs \
+ installdirs-am maintainer-clean maintainer-clean-generic \
+ mostlyclean mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \
+ uninstall-am uninstall-binPROGRAMS uninstall-pkginc_libHEADERS \
+ uninstall-pkglibexecPROGRAMS
+
+.PRECIOUS: Makefile
+
+
+install-exec-local:
+ rm -f $(DESTDIR)$(bindir)/dsync
+ $(LN_S) doveadm $(DESTDIR)$(bindir)/dsync
+
+check-local:
+ for bin in $(test_programs); do \
+ if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+ done
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/doveadm/client-connection-http.c b/src/doveadm/client-connection-http.c
new file mode 100644
index 0000000..c59d9dd
--- /dev/null
+++ b/src/doveadm/client-connection-http.c
@@ -0,0 +1,1227 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "compat.h"
+#include "lib-signals.h"
+#include "base64.h"
+#include "ioloop.h"
+#include "str.h"
+#include "str-sanitize.h"
+#include "istream.h"
+#include "ostream.h"
+#include "strescape.h"
+#include "settings-parser.h"
+#include "iostream-ssl.h"
+#include "iostream-temp.h"
+#include "istream-seekable.h"
+#include "master-service.h"
+#include "master-service-ssl.h"
+#include "master-service-settings.h"
+#include "mail-storage-service.h"
+#include "http-server.h"
+#include "http-request.h"
+#include "http-response.h"
+#include "http-url.h"
+#include "doveadm-util.h"
+#include "doveadm-server.h"
+#include "doveadm-mail.h"
+#include "doveadm-print.h"
+#include "doveadm-settings.h"
+#include "client-connection-private.h"
+#include "json-parser.h"
+
+#include <unistd.h>
+#include <ctype.h>
+
+enum client_request_parse_state {
+ CLIENT_REQUEST_PARSE_INIT,
+ CLIENT_REQUEST_PARSE_CMD,
+ CLIENT_REQUEST_PARSE_CMD_NAME,
+ CLIENT_REQUEST_PARSE_CMD_PARAMS,
+ CLIENT_REQUEST_PARSE_CMD_PARAM_KEY,
+ CLIENT_REQUEST_PARSE_CMD_PARAM_VALUE,
+ CLIENT_REQUEST_PARSE_CMD_PARAM_ARRAY,
+ CLIENT_REQUEST_PARSE_CMD_PARAM_ISTREAM,
+ CLIENT_REQUEST_PARSE_CMD_ID,
+ CLIENT_REQUEST_PARSE_CMD_DONE,
+ CLIENT_REQUEST_PARSE_DONE
+};
+
+struct client_request_http {
+ pool_t pool;
+ struct client_connection_http *conn;
+
+ struct http_server_request *http_request;
+
+ struct io *io;
+ struct istream *input;
+ struct ostream *output;
+
+ struct json_parser *json_parser;
+
+ const struct doveadm_cmd_ver2 *cmd;
+ struct doveadm_cmd_param *cmd_param;
+ struct ioloop *ioloop;
+ ARRAY_TYPE(doveadm_cmd_param_arr_t) pargv;
+ int method_err;
+ char *method_id;
+ bool first_row;
+ bool value_is_array;
+
+ enum client_request_parse_state parse_state;
+};
+
+struct client_connection_http {
+ struct client_connection conn;
+
+ struct http_server_connection *http_conn;
+
+ struct client_request_http *request;
+};
+
+typedef void doveadm_server_handler_t(struct client_request_http *req);
+
+struct doveadm_http_server_mount {
+ const char *verb;
+ const char *path;
+ doveadm_server_handler_t *handler;
+ bool auth;
+};
+
+static struct http_server *doveadm_http_server;
+
+static void doveadm_http_server_send_response(struct client_request_http *req);
+
+/*
+ * API
+ */
+
+static void doveadm_http_server_options_handler(struct client_request_http *);
+static void doveadm_http_server_print_mounts(struct client_request_http *);
+static void doveadm_http_server_send_api_v1(struct client_request_http *);
+static void doveadm_http_server_read_request_v1(struct client_request_http *);
+
+static struct doveadm_http_server_mount doveadm_http_server_mounts[] = {
+{
+ .verb = "OPTIONS",
+ .path = NULL,
+ .handler = doveadm_http_server_options_handler,
+ .auth = FALSE
+},{
+ .verb = "GET",
+ .path = "/",
+ .handler = doveadm_http_server_print_mounts,
+ .auth = TRUE
+},{
+ .verb = "GET",
+ .path = "/doveadm/v1",
+ .handler = doveadm_http_server_send_api_v1,
+ .auth = TRUE
+},{
+ .verb = "POST",
+ .path = "/doveadm/v1",
+ .handler = doveadm_http_server_read_request_v1,
+ .auth = TRUE
+}
+};
+
+static void doveadm_http_server_json_error(void *context, const char *error)
+{
+ struct client_request_http *req = context;
+ struct ostream *output = req->output;
+ string_t *escaped;
+
+ escaped = str_new(req->pool, 10);
+
+ o_stream_nsend_str(output, "[\"error\",{\"type\":\"");
+ json_append_escaped(escaped, error);
+ o_stream_nsend_str(output, str_c(escaped));
+ o_stream_nsend_str(output, "\", \"exitCode\":");
+ str_truncate(escaped,0);
+ str_printfa(escaped, "%d", doveadm_exit_code);
+ o_stream_nsend_str(output, str_c(escaped));
+ o_stream_nsend_str(output, "},\"");
+ str_truncate(escaped,0);
+ if (req->method_id != NULL) {
+ json_append_escaped(escaped, req->method_id);
+ o_stream_nsend_str(output, str_c(escaped));
+ }
+ o_stream_nsend_str(output, "\"]");
+}
+
+static void doveadm_http_server_json_success(void *context, struct istream *result)
+{
+ struct client_request_http *req = context;
+ struct ostream *output = req->output;
+ string_t *escaped;
+
+ escaped = str_new(req->pool, 10);
+
+ o_stream_nsend_str(output, "[\"doveadmResponse\",");
+ o_stream_nsend_istream(output, result);
+ o_stream_nsend_str(output, ",\"");
+ if (req->method_id != NULL) {
+ json_append_escaped(escaped, req->method_id);
+ o_stream_nsend_str(output, str_c(escaped));
+ }
+ o_stream_nsend_str(output, "\"]");
+}
+
+static void
+doveadm_http_server_command_execute(struct client_request_http *req)
+{
+ struct client_connection_http *conn = req->conn;
+ struct doveadm_cmd_context cctx;
+ struct istream *is;
+ const char *user;
+ struct ioloop *ioloop, *prev_ioloop;
+
+ /* final preflight check */
+ if (req->method_err == 0 &&
+ !doveadm_client_is_allowed_command(conn->conn.set,
+ req->cmd->name))
+ req->method_err = 403;
+ if (req->method_err != 0) {
+ if (req->method_err == 404) {
+ doveadm_http_server_json_error(req, "unknownMethod");
+ } else if (req->method_err == 403) {
+ doveadm_http_server_json_error(req, "unAuthorized");
+ } else if (req->method_err == 400) {
+ doveadm_http_server_json_error(req, "invalidRequest");
+ } else {
+ doveadm_http_server_json_error(req, "internalError");
+ }
+ return;
+ }
+
+ prev_ioloop = current_ioloop;
+ i_zero(&cctx);
+ cctx.conn_type = conn->conn.type;
+ cctx.input = req->input;
+ cctx.output = req->output;
+
+ // create iostream
+ doveadm_print_ostream = iostream_temp_create("/tmp/doveadm.", 0);
+ cctx.cmd = req->cmd;
+
+ if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0)
+ doveadm_print_init(DOVEADM_PRINT_TYPE_JSON);
+
+ /* then call it */
+ doveadm_cmd_params_null_terminate_arrays(&req->pargv);
+ cctx.argv = array_get(&req->pargv, (unsigned int*)&cctx.argc);
+ ioloop = io_loop_create();
+ doveadm_exit_code = 0;
+
+ cctx.local_ip = conn->conn.local_ip;
+ cctx.local_port = conn->conn.local_port;
+ cctx.remote_ip = conn->conn.remote_ip;
+ cctx.remote_port = conn->conn.remote_port;
+
+ if (doveadm_cmd_param_str(&cctx, "user", &user))
+ i_info("Executing command '%s' as '%s'", cctx.cmd->name, user);
+ else
+ i_info("Executing command '%s'", cctx.cmd->name);
+ client_connection_set_proctitle(&conn->conn, cctx.cmd->name);
+ cctx.cmd->cmd(&cctx);
+ client_connection_set_proctitle(&conn->conn, "");
+
+ o_stream_switch_ioloop_to(req->output, prev_ioloop);
+ io_loop_destroy(&ioloop);
+
+ if ((cctx.cmd->flags & CMD_FLAG_NO_PRINT) == 0)
+ doveadm_print_deinit();
+ if (o_stream_finish(doveadm_print_ostream) < 0) {
+ i_info("Error writing output in command %s: %s",
+ req->cmd->name,
+ o_stream_get_error(req->output));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+
+ is = iostream_temp_finish(&doveadm_print_ostream, 4096);
+
+ if (req->first_row == TRUE)
+ req->first_row = FALSE;
+ else
+ o_stream_nsend_str(req->output,",");
+
+ if (doveadm_exit_code != 0) {
+ if (doveadm_exit_code == 0 || doveadm_exit_code == EX_TEMPFAIL)
+ i_error("Command %s failed", req->cmd->name);
+ doveadm_http_server_json_error(req, "exitCode");
+ } else {
+ doveadm_http_server_json_success(req, is);
+ }
+ i_stream_unref(&is);
+}
+
+static int
+request_json_parse_init(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type != JSON_TYPE_ARRAY) {
+ /* request must be a JSON array */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Request must be a JSON array");
+ return -1;
+ }
+ req->first_row = TRUE;
+ o_stream_nsend_str(req->output,"[");
+
+ /* next: parse the next command */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD;
+ return 1;
+}
+
+static int
+request_json_parse_cmd(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type == JSON_TYPE_ARRAY_END) {
+ /* end of command list */
+ req->parse_state = CLIENT_REQUEST_PARSE_DONE;
+ return 1;
+ }
+ if (type != JSON_TYPE_ARRAY) {
+ /* command must be an array */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Command must be a JSON array");
+ return -1;
+ }
+ req->method_err = 0;
+ p_free_and_null(req->pool, req->method_id);
+ req->cmd = NULL;
+ doveadm_cmd_params_clean(&req->pargv);
+
+ /* next: parse the command name */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_NAME;
+ return 1;
+}
+
+static int
+request_json_parse_cmd_name(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ const struct doveadm_cmd_ver2 *ccmd;
+ struct doveadm_cmd_param *param;
+ bool found;
+ int pargc, ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type != JSON_TYPE_STRING) {
+ /* command name must be a string */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Command name must be a string");
+ return -1;
+ }
+
+ /* see if we can find it */
+ found = FALSE;
+ array_foreach(&doveadm_cmds_ver2, ccmd) {
+ if (i_strccdascmp(ccmd->name, value) == 0) {
+ req->cmd = ccmd;
+ found = TRUE;
+ break;
+ }
+ }
+ if (!found) {
+ /* command not found; skip to the command ID */
+ json_parse_skip_next(req->json_parser);
+ req->method_err = 404;
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_ID;
+ return 1;
+ }
+
+ /* initialize pargv */
+ for (pargc = 0; req->cmd->parameters[pargc].name != NULL; pargc++) {
+ param = array_append_space(&req->pargv);
+ *param = req->cmd->parameters[pargc];
+ param->value_set = FALSE;
+ }
+
+ /* next: parse the command parameters */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAMS;
+ return 1;
+}
+
+static int
+request_json_parse_cmd_params(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type == JSON_TYPE_OBJECT_END) {
+ /* empty command parameters object; parse command ID next */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_ID;
+ return 1;
+ }
+ if (type != JSON_TYPE_OBJECT) {
+ /* parameters must be contained in an object */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Parameters must be contained in a JSON object");
+ return -1;
+ }
+
+ /* next: parse parameter key */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY;
+ return 1;
+}
+
+static int
+request_json_parse_cmd_param_key(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ struct doveadm_cmd_param *par;
+ bool found;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type == JSON_TYPE_OBJECT_END) {
+ /* end of parameters; parse command ID next */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_ID;
+ return 1;
+ }
+ i_assert(type == JSON_TYPE_OBJECT_KEY);
+ /* find the parameter */
+ found = FALSE;
+ array_foreach_modifiable(&req->pargv, par) {
+ if (i_strccdascmp(par->name, value) == 0) {
+ req->cmd_param = par;
+ found = TRUE;
+ break;
+ }
+ }
+ if (found && req->cmd_param->value_set) {
+ /* it's already set, cannot have same key twice in json */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Parameter `%s' is duplicated",
+ req->cmd_param->name);
+ return -1;
+ }
+ /* skip remaining parameters if error has already occurred */
+ if (!found || req->method_err != 0) {
+ json_parse_skip_next(req->json_parser);
+ req->method_err = 400;
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY;
+ return 1;
+ }
+
+ /* next: parse parameter value */
+ req->value_is_array = FALSE;
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_VALUE;
+ return 1;
+}
+
+static int
+request_json_parse_param_value(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if (req->cmd_param->type == CMD_PARAM_ISTREAM) {
+ struct istream* is[2] = {0};
+
+ /* read the value as a stream */
+ ret = json_parse_next_stream(req->json_parser, &is[0]);
+ if (ret <= 0)
+ return ret;
+
+ req->cmd_param->value.v_istream =
+ i_stream_create_seekable_path(is,
+ IO_BLOCK_SIZE, "/tmp/doveadm.");
+ i_stream_unref(&is[0]);
+ req->cmd_param->value_set = TRUE;
+
+ /* read the seekable stream to its end so that the underlying
+ json istream is read to its conclusion. */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_ISTREAM;
+ return 1;
+ }
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (req->cmd_param->type == CMD_PARAM_ARRAY) {
+ const char *tmp;
+
+ /* expects either a singular value or an array of values */
+ p_array_init(&req->cmd_param->value.v_array, req->pool, 1);
+ req->cmd_param->value_set = TRUE;
+ if (type == JSON_TYPE_ARRAY) {
+ /* start of array */
+ req->value_is_array = TRUE;
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_ARRAY;
+ return 1;
+ }
+ /* singular value */
+ if (type != JSON_TYPE_STRING) {
+ /* FIXME: should handle other than string too */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Parameter `%s' must be string or array",
+ req->cmd_param->name);
+ return -1;
+ }
+ tmp = p_strdup(req->pool, value);
+ array_push_back(&req->cmd_param->value.v_array, &tmp);
+
+ /* next: continue with the next parameter */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY;
+ return 1;
+ }
+
+ /* expects just a value */
+ req->cmd_param->value_set = TRUE;
+ switch(req->cmd_param->type) {
+ case CMD_PARAM_BOOL:
+ req->cmd_param->value.v_bool = (strcmp(value, "true") == 0);
+ break;
+ case CMD_PARAM_INT64:
+ if (str_to_int64(value, &req->cmd_param->value.v_int64) != 0) {
+ req->method_err = 400;
+ }
+ break;
+ case CMD_PARAM_IP:
+ if (net_addr2ip(value, &req->cmd_param->value.v_ip) != 0) {
+ req->method_err = 400;
+ }
+ break;
+ case CMD_PARAM_STR:
+ req->cmd_param->value.v_string = p_strdup(req->pool, value);
+ break;
+ default:
+ break;
+ }
+
+ /* next: continue with the next parameter */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY;
+ return 1;
+}
+
+static int
+request_json_parse_param_array(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type == JSON_TYPE_ARRAY_END) {
+ /* end of array: continue with next parameter */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY;
+ return 1;
+ }
+ if (type != JSON_TYPE_STRING) {
+ /* array items must be string */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Command parameter array can only contain"
+ "string values");
+ return -1;
+ }
+
+ /* record entry */
+ value = p_strdup(req->pool, value);
+ array_push_back(&req->cmd_param->value.v_array, &value);
+
+ /* next: continue with the next array item */
+ return 1;
+}
+
+static int
+request_json_parse_param_istream(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ struct istream *v_input = req->cmd_param->value.v_istream;
+ const unsigned char *data;
+ size_t size;
+
+ while (i_stream_read_more(v_input, &data, &size) > 0)
+ i_stream_skip(v_input, size);
+ if (!v_input->eof) {
+ /* more to read */
+ return 0;
+ }
+
+ if (v_input->stream_errno != 0) {
+ i_error("read(%s) failed: %s",
+ i_stream_get_name(v_input),
+ i_stream_get_error(v_input));
+ req->method_err = 400;
+ if (req->input->stream_errno == 0) {
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Failed to read command parameter data");
+ }
+ return -1;
+ }
+
+ /* next: continue with the next parameter */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_PARAM_KEY;
+ return 1;
+}
+
+static int
+request_json_parse_cmd_id(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type != JSON_TYPE_STRING) {
+ /* command ID must be a string */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Command ID must be a string");
+ return -1;
+ }
+
+ /* next: parse end of command */
+ req->method_id = p_strdup(req->pool, value);
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD_DONE;
+ return 1;
+}
+
+static int
+request_json_parse_cmd_done(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ if (type != JSON_TYPE_ARRAY_END) {
+ /* command array must end here */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Unexpected JSON element at end of command");
+ return -1;
+ }
+
+ /* execute command */
+ doveadm_http_server_command_execute(req);
+
+ /* next: parse next command */
+ req->parse_state = CLIENT_REQUEST_PARSE_CMD;
+ return 1;
+}
+
+static int
+request_json_parse_done(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ enum json_type type;
+ const char *value;
+ int ret;
+
+ if ((ret=json_parse_next(req->json_parser, &type, &value)) <= 0)
+ return ret;
+ /* only gets here when there is spurious additional JSON */
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "Unexpected JSON element in input");
+ return -1;
+}
+
+static int
+doveadm_http_server_json_parse_v1(struct client_request_http *req)
+{
+ /* parser state machine */
+ switch (req->parse_state) {
+ /* command list: '[' */
+ case CLIENT_REQUEST_PARSE_INIT:
+ return request_json_parse_init(req);
+ /* command begin: '[' */
+ case CLIENT_REQUEST_PARSE_CMD:
+ return request_json_parse_cmd(req);
+ /* command name: string */
+ case CLIENT_REQUEST_PARSE_CMD_NAME:
+ return request_json_parse_cmd_name(req);
+ /* command parameters: '{' */
+ case CLIENT_REQUEST_PARSE_CMD_PARAMS:
+ return request_json_parse_cmd_params(req);
+ /* parameter key: string */
+ case CLIENT_REQUEST_PARSE_CMD_PARAM_KEY:
+ return request_json_parse_cmd_param_key(req);
+ /* parameter value */
+ case CLIENT_REQUEST_PARSE_CMD_PARAM_VALUE:
+ return request_json_parse_param_value(req);
+ /* parameter array value */
+ case CLIENT_REQUEST_PARSE_CMD_PARAM_ARRAY:
+ return request_json_parse_param_array(req);
+ /* parameter istream value */
+ case CLIENT_REQUEST_PARSE_CMD_PARAM_ISTREAM:
+ return request_json_parse_param_istream(req);
+ /* command ID: string */
+ case CLIENT_REQUEST_PARSE_CMD_ID:
+ return request_json_parse_cmd_id(req);
+ /* command end: ']' */
+ case CLIENT_REQUEST_PARSE_CMD_DONE:
+ return request_json_parse_cmd_done(req);
+ /* finished parsing request (seen final ']') */
+ case CLIENT_REQUEST_PARSE_DONE:
+ return request_json_parse_done(req);
+ default:
+ break;
+ }
+ i_unreached();
+}
+
+static void
+doveadm_http_server_read_request_v1(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ const char *error;
+ int ret;
+
+ if (req->json_parser == NULL) {
+ req->json_parser = json_parser_init_flags(
+ req->input, JSON_PARSER_NO_ROOT_OBJECT);
+ }
+
+ while ((ret=doveadm_http_server_json_parse_v1(req)) > 0);
+
+ if (http_server_request_get_response(http_sreq) != NULL) {
+ /* already responded */
+ io_remove(&req->io);
+ i_stream_destroy(&req->input);
+ return;
+ }
+ if (!req->input->eof && ret == 0)
+ return;
+ io_remove(&req->io);
+
+ doveadm_cmd_params_clean(&req->pargv);
+
+ if (req->input->stream_errno != 0) {
+ http_server_request_fail_close(http_sreq,
+ 400, "Client disconnected");
+ i_info("read(%s) failed: %s",
+ i_stream_get_name(req->input),
+ i_stream_get_error(req->input));
+ return;
+ }
+
+ if (json_parser_deinit(&req->json_parser, &error) != 0) {
+ http_server_request_fail_text(http_sreq,
+ 400, "Bad Request",
+ "JSON parse error: %s", error);
+ return;
+ }
+
+ i_stream_destroy(&req->input);
+ o_stream_nsend_str(req->output,"]");
+
+ doveadm_http_server_send_response(req);
+}
+
+static void doveadm_http_server_camelcase_value(string_t *value)
+{
+ size_t i, k;
+ char *ptr = str_c_modifiable(value);
+
+ for (i = 0, k = 0; i < strlen(ptr);) {
+ if (ptr[i] == ' ' || ptr[i] == '-') {
+ i++;
+ ptr[k++] = i_toupper(ptr[i++]);
+ } else {
+ ptr[k++] = ptr[i++];
+ }
+ }
+ str_truncate(value, k);
+}
+
+static void
+doveadm_http_server_send_api_v1(struct client_request_http *req)
+{
+ struct ostream *output = req->output;
+ const struct doveadm_cmd_ver2 *cmd;
+ const struct doveadm_cmd_param *par;
+ unsigned int i, k;
+ string_t *tmp;
+ bool sent;
+
+ tmp = str_new(req->pool, 8);
+
+ o_stream_nsend_str(output,"[\n");
+ for (i = 0; i < array_count(&doveadm_cmds_ver2); i++) {
+ cmd = array_idx(&doveadm_cmds_ver2, i);
+ if (i > 0)
+ o_stream_nsend_str(output, ",\n");
+ o_stream_nsend_str(output, "\t{\"command\":\"");
+ json_append_escaped(tmp, cmd->name);
+ doveadm_http_server_camelcase_value(tmp);
+ o_stream_nsend_str(output, str_c(tmp));
+ o_stream_nsend_str(output, "\", \"parameters\":[");
+
+ sent = FALSE;
+ for (k = 0; cmd->parameters[k].name != NULL; k++) {
+ str_truncate(tmp, 0);
+ par = &(cmd->parameters[k]);
+ if ((par->flags & CMD_PARAM_FLAG_DO_NOT_EXPOSE) != 0)
+ continue;
+ if (sent)
+ o_stream_nsend_str(output, ",\n");
+ else
+ o_stream_nsend_str(output, "\n");
+ sent = TRUE;
+ o_stream_nsend_str(output, "\t\t{\"name\":\"");
+ json_append_escaped(tmp, par->name);
+ doveadm_http_server_camelcase_value(tmp);
+ o_stream_nsend_str(output, str_c(tmp));
+ o_stream_nsend_str(output, "\",\"type\":\"");
+ switch(par->type) {
+ case CMD_PARAM_BOOL:
+ o_stream_nsend_str(output, "boolean");
+ break;
+ case CMD_PARAM_INT64:
+ o_stream_nsend_str(output, "integer");
+ break;
+ case CMD_PARAM_ARRAY:
+ o_stream_nsend_str(output, "array");
+ break;
+ case CMD_PARAM_IP:
+ case CMD_PARAM_ISTREAM:
+ case CMD_PARAM_STR:
+ o_stream_nsend_str(output, "string");
+ }
+ o_stream_nsend_str(output, "\"}");
+ }
+ if (k > 0)
+ o_stream_nsend_str(output,"\n\t");
+ o_stream_nsend_str(output,"]}");
+ str_truncate(tmp, 0);
+ }
+ o_stream_nsend_str(output,"\n]");
+ doveadm_http_server_send_response(req);
+}
+
+static void
+doveadm_http_server_options_handler(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ struct http_server_response *http_resp;
+
+ http_resp = http_server_response_create(http_sreq, 200, "OK");
+ http_server_response_add_header(http_resp,
+ "Access-Control-Allow-Origin", "*");
+ http_server_response_add_header(http_resp,
+ "Access-Control-Allow-Methods", "GET, POST, OPTIONS");
+ http_server_response_add_header(http_resp,
+ "Access-Control-Allow-Request-Headers",
+ "Content-Type, X-API-Key, Authorization");
+ http_server_response_add_header(http_resp,
+ "Access-Control-Allow-Headers",
+ "Content-Type, WWW-Authenticate");
+ http_server_response_submit(http_resp);
+}
+
+static void
+doveadm_http_server_print_mounts(struct client_request_http *req)
+{
+ struct ostream *output = req->output;
+ unsigned int i;
+
+ o_stream_nsend_str(output, "[\n");
+ for (i = 0; i < N_ELEMENTS(doveadm_http_server_mounts); i++) {
+ if (i > 0)
+ o_stream_nsend_str(output, ",\n");
+ o_stream_nsend_str(output, "{\"method\":\"");
+ if (doveadm_http_server_mounts[i].verb == NULL)
+ o_stream_nsend_str(output, "*");
+ else {
+ o_stream_nsend_str(output,
+ doveadm_http_server_mounts[i].verb);
+ }
+ o_stream_nsend_str(output, "\",\"path\":\"");
+ if (doveadm_http_server_mounts[i].path == NULL)
+ o_stream_nsend_str(output, "*");
+ else {
+ o_stream_nsend_str(output,
+ doveadm_http_server_mounts[i].path);
+ }
+ o_stream_nsend_str(output, "\"}");
+ }
+ o_stream_nsend_str(output, "\n]");
+ doveadm_http_server_send_response(req);
+}
+
+/*
+ * Request
+ */
+
+static void doveadm_http_server_send_response(struct client_request_http *req)
+{
+ struct http_server_request *http_sreq = req->http_request;
+ struct http_server_response *http_resp;
+ struct istream *payload = NULL;
+
+ if (req->output != NULL) {
+ if (o_stream_finish(req->output) == -1) {
+ i_info("error writing output: %s",
+ o_stream_get_error(req->output));
+ o_stream_destroy(&req->output);
+ http_server_request_fail(http_sreq,
+ 500, "Internal server error");
+ return;
+ }
+
+ payload = iostream_temp_finish(&req->output,
+ IO_BLOCK_SIZE);
+ }
+
+ http_resp = http_server_response_create(http_sreq, 200, "OK");
+ http_server_response_add_header(http_resp, "Content-Type",
+ "application/json; charset=utf-8");
+
+ if (payload != NULL) {
+ http_server_response_set_payload(http_resp, payload);
+ i_stream_unref(&payload);
+ }
+
+ http_server_response_submit(http_resp);
+}
+
+static void
+doveadm_http_server_request_destroy(struct client_request_http *req)
+{
+ struct client_connection_http *conn = req->conn;
+ struct http_server_request *http_sreq = req->http_request;
+ const struct http_request *http_req =
+ http_server_request_get(http_sreq);
+ struct http_server_response *http_resp =
+ http_server_request_get_response(http_sreq);
+
+ i_assert(conn->request == req);
+
+ if (http_resp != NULL) {
+ const char *agent, *url, *reason;
+ uoff_t size;
+ int status;
+
+ http_server_response_get_status(http_resp, &status, &reason);
+ size = http_server_response_get_total_size(http_resp);
+ agent = http_request_header_get(http_req, "User-Agent");
+ if (agent == NULL) agent = "";
+
+ url = http_url_create(http_req->target.url);
+ i_info("doveadm: %s %s %s \"%s %s "
+ "HTTP/%d.%d\" %d %"PRIuUOFF_T" \"%s\" \"%s\"",
+ net_ip2addr(&conn->conn.remote_ip), "-", "-",
+ http_req->method, http_req->target.url->path,
+ http_req->version_major, http_req->version_minor,
+ status, size, url, agent);
+ }
+ if (req->json_parser != NULL) {
+ const char *error ATTR_UNUSED;
+ (void)json_parser_deinit(&req->json_parser, &error);
+ // we've already failed, ignore error
+ }
+ if (req->output != NULL)
+ o_stream_set_no_error_handling(req->output, TRUE);
+ io_remove(&req->io);
+ o_stream_destroy(&req->output);
+ i_stream_destroy(&req->input);
+
+ http_server_request_unref(&req->http_request);
+ http_server_switch_ioloop(doveadm_http_server);
+
+ pool_unref(&req->pool);
+ conn->request = NULL;
+}
+
+static bool
+doveadm_http_server_auth_basic(struct client_request_http *req,
+ const struct http_auth_credentials *creds)
+{
+ struct client_connection_http *conn = req->conn;
+ const struct doveadm_settings *set = conn->conn.set;
+ string_t *b64_value;
+ char *value;
+
+ if (*set->doveadm_password == '\0') {
+ i_error("Invalid authentication attempt to HTTP API: "
+ "Basic authentication scheme not enabled");
+ return FALSE;
+ }
+
+ b64_value = str_new(conn->conn.pool, 32);
+ value = p_strdup_printf(conn->conn.pool,
+ "doveadm:%s", set->doveadm_password);
+ base64_encode(value, strlen(value), b64_value);
+ if (creds->data != NULL && strcmp(creds->data, str_c(b64_value)) == 0)
+ return TRUE;
+
+ i_error("Invalid authentication attempt to HTTP API "
+ "(using Basic authentication scheme)");
+ return FALSE;
+}
+
+static bool
+doveadm_http_server_auth_api_key(struct client_request_http *req,
+ const struct http_auth_credentials *creds)
+{
+ struct client_connection_http *conn = req->conn;
+ const struct doveadm_settings *set = doveadm_settings;
+ string_t *b64_value;
+
+ if (*set->doveadm_api_key == '\0') {
+ i_error("Invalid authentication attempt to HTTP API: "
+ "X-Dovecot-API authentication scheme not enabled");
+ return FALSE;
+ }
+
+ b64_value = str_new(conn->conn.pool, 32);
+ base64_encode(set->doveadm_api_key,
+ strlen(set->doveadm_api_key), b64_value);
+ if (creds->data != NULL && strcmp(creds->data, str_c(b64_value)) == 0)
+ return TRUE;
+
+ i_error("Invalid authentication attempt to HTTP API "
+ "(using X-Dovecot-API authentication scheme)");
+ return FALSE;
+}
+
+
+static bool
+doveadm_http_server_auth_verify(struct client_request_http *req,
+ const struct http_auth_credentials *creds)
+{
+ /* see if the mech is supported */
+ if (strcasecmp(creds->scheme, "Basic") == 0)
+ return doveadm_http_server_auth_basic(req, creds);
+ if (strcasecmp(creds->scheme, "X-Dovecot-API") == 0)
+ return doveadm_http_server_auth_api_key(req, creds);
+
+ i_error("Unsupported authentication scheme to HTTP API: %s",
+ str_sanitize(creds->scheme, 128));
+ return FALSE;
+}
+
+static bool
+doveadm_http_server_authorize_request(struct client_request_http *req)
+{
+ struct client_connection_http *conn = req->conn;
+ struct http_server_request *http_sreq = req->http_request;
+ bool auth = FALSE;
+ struct http_auth_credentials creds;
+
+ /* no authentication specified */
+ if (doveadm_settings->doveadm_api_key[0] == '\0' &&
+ *conn->conn.set->doveadm_password == '\0') {
+ http_server_request_fail(http_sreq,
+ 500, "Internal Server Error");
+ i_error("No authentication defined in configuration. "
+ "Add API key or password");
+ return FALSE;
+ }
+ if (http_server_request_get_auth(http_sreq, &creds) > 0)
+ auth = doveadm_http_server_auth_verify(req, &creds);
+ if (!auth) {
+ struct http_server_response *http_resp;
+
+ http_resp = http_server_response_create(http_sreq,
+ 401, "Authentication required");
+ if (doveadm_settings->doveadm_api_key[0] != '\0') {
+ http_server_response_add_header(http_resp,
+ "WWW-Authenticate", "X-Dovecot-API"
+ );
+ }
+ if (*conn->conn.set->doveadm_password != '\0') {
+ http_server_response_add_header(http_resp,
+ "WWW-Authenticate", "Basic Realm=\"doveadm\""
+ );
+ }
+ http_server_response_submit(http_resp);
+ }
+ return auth;
+}
+
+static void
+doveadm_http_server_handle_request(void *context,
+ struct http_server_request *http_sreq)
+{
+ struct client_connection_http *conn = context;
+ struct client_request_http *req;
+ const struct http_request *http_req =
+ http_server_request_get(http_sreq);
+ struct doveadm_http_server_mount *ep = NULL;
+ pool_t pool;
+ unsigned int i;
+
+ /* no pipelining possible due to synchronous handling of requests */
+ i_assert(conn->request == NULL);
+
+ pool = pool_alloconly_create("doveadm request", 1024*16);
+ req = p_new(pool, struct client_request_http, 1);
+ req->pool = pool;
+ req->conn = conn;
+
+ req->http_request = http_sreq;
+ http_server_request_ref(req->http_request);
+
+ http_server_request_connection_close(http_sreq, TRUE);
+ http_server_request_set_destroy_callback(http_sreq,
+ doveadm_http_server_request_destroy, req);
+
+ conn->request = req;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_http_server_mounts); i++) {
+ if (doveadm_http_server_mounts[i].verb == NULL ||
+ strcmp(http_req->method,
+ doveadm_http_server_mounts[i].verb) == 0) {
+ if (doveadm_http_server_mounts[i].path == NULL ||
+ strcmp(http_req->target.url->path,
+ doveadm_http_server_mounts[i].path) == 0) {
+ ep = &doveadm_http_server_mounts[i];
+ break;
+ }
+ }
+ }
+
+ if (ep == NULL) {
+ http_server_request_fail(http_sreq,
+ 404, "Path Not Found");
+ return;
+ }
+
+ if (ep->auth == TRUE && !doveadm_http_server_authorize_request(req))
+ return;
+
+ if (strcmp(http_req->method, "POST") == 0) {
+ /* handle request */
+ req->input = http_req->payload;
+ i_stream_set_name(req->input,
+ net_ip2addr(&conn->conn.remote_ip));
+ i_stream_ref(req->input);
+ req->io = io_add_istream(req->input, *ep->handler, req);
+ req->output = iostream_temp_create_named(
+ "/tmp/doveadm.", 0, net_ip2addr(&conn->conn.remote_ip));
+ p_array_init(&req->pargv, req->pool, 5);
+ ep->handler(req);
+ } else {
+ req->output = iostream_temp_create_named(
+ "/tmp/doveadm.", 0, net_ip2addr(&conn->conn.remote_ip));
+ ep->handler(req);
+ }
+}
+
+/*
+ * Connection
+ */
+
+static void doveadm_http_server_connection_destroy(void *context,
+ const char *reason);
+
+static const struct http_server_callbacks doveadm_http_callbacks = {
+ .connection_destroy = doveadm_http_server_connection_destroy,
+ .handle_request = doveadm_http_server_handle_request
+};
+
+static void
+client_connection_http_free(struct client_connection *_conn)
+{
+ struct client_connection_http *conn =
+ (struct client_connection_http *)_conn;
+
+ if (conn->http_conn != NULL) {
+ /* We're not in the lib-http/server's connection destroy
+ callback. */
+ http_server_connection_close(&conn->http_conn,
+ "Server shutting down");
+ }
+}
+
+struct client_connection *
+client_connection_http_create(int fd, bool ssl)
+{
+ struct client_connection_http *conn;
+ pool_t pool;
+
+ pool = pool_alloconly_create("doveadm client", 1024);
+ conn = p_new(pool, struct client_connection_http, 1);
+
+ if (client_connection_init(&conn->conn,
+ DOVEADM_CONNECTION_TYPE_HTTP, pool, fd) < 0) {
+ pool_unref(&conn->conn.pool);
+ return NULL;
+ }
+ conn->conn.free = client_connection_http_free;
+
+ conn->http_conn = http_server_connection_create(doveadm_http_server,
+ fd, fd, ssl, &doveadm_http_callbacks, conn);
+ return &conn->conn;
+}
+
+static void
+doveadm_http_server_connection_destroy(void *context,
+ const char *reason ATTR_UNUSED)
+{
+ struct client_connection_http *conn =
+ (struct client_connection_http *)context;
+ struct client_connection *bconn = &conn->conn;
+
+ if (conn->http_conn == NULL) {
+ /* already destroying client directly */
+ return;
+ }
+
+ /* HTTP connection is destroyed already now */
+ conn->http_conn = NULL;
+
+ /* destroy the connection itself */
+ client_connection_destroy(&bconn);
+}
+
+/*
+ * Server
+ */
+
+void doveadm_http_server_init(void)
+{
+ struct http_server_settings http_set = {
+ .rawlog_dir = doveadm_settings->doveadm_http_rawlog_dir,
+ };
+
+ doveadm_http_server = http_server_init(&http_set);
+}
+
+void doveadm_http_server_deinit(void)
+{
+ http_server_deinit(&doveadm_http_server);
+}
diff --git a/src/doveadm/client-connection-private.h b/src/doveadm/client-connection-private.h
new file mode 100644
index 0000000..8205531
--- /dev/null
+++ b/src/doveadm/client-connection-private.h
@@ -0,0 +1,22 @@
+#ifndef CLIENT_CONNECTION_PRIVATE_H
+#define CLIENT_CONNECTION_PRIVATE_H
+
+#include "client-connection.h"
+
+bool doveadm_client_is_allowed_command(const struct doveadm_settings *set,
+ const char *cmd_name);
+
+int client_connection_init(struct client_connection *conn,
+ enum doveadm_client_type type, pool_t pool, int fd);
+void client_connection_destroy(struct client_connection **_conn);
+
+void client_connection_set_proctitle(struct client_connection *conn,
+ const char *text);
+
+void doveadm_http_server_init(void);
+void doveadm_http_server_deinit(void);
+
+void doveadm_server_init(void);
+void doveadm_server_deinit(void);
+
+#endif
diff --git a/src/doveadm/client-connection-tcp.c b/src/doveadm/client-connection-tcp.c
new file mode 100644
index 0000000..b80d674
--- /dev/null
+++ b/src/doveadm/client-connection-tcp.c
@@ -0,0 +1,558 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "lib-signals.h"
+#include "str.h"
+#include "base64.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "ostream.h"
+#include "strescape.h"
+#include "iostream-ssl.h"
+#include "ostream-multiplex.h"
+#include "master-service.h"
+#include "master-service-ssl.h"
+#include "mail-storage-service.h"
+#include "doveadm-util.h"
+#include "doveadm-mail.h"
+#include "doveadm-print.h"
+#include "doveadm-server.h"
+#include "client-connection-private.h"
+
+#include <unistd.h>
+
+#define MAX_INBUF_SIZE (1024*1024)
+
+struct client_connection_tcp {
+ struct client_connection conn;
+
+ int fd;
+ struct io *io;
+ struct istream *input;
+ struct ostream *output;
+ struct ostream *log_out;
+ struct ssl_iostream *ssl_iostream;
+ struct ioloop *ioloop;
+
+ bool handshaked:1;
+ bool preauthenticated:1;
+ bool authenticated:1;
+ bool io_setup:1;
+ bool use_multiplex:1;
+};
+
+static void
+client_connection_tcp_input(struct client_connection_tcp *conn);
+static void
+client_connection_tcp_send_auth_handshake(struct client_connection_tcp *conn);
+static void
+client_connection_tcp_destroy(struct client_connection_tcp **_conn);
+static int
+client_connection_tcp_init_ssl(struct client_connection_tcp *conn);
+
+static failure_callback_t *orig_error_callback, *orig_fatal_callback;
+static failure_callback_t *orig_info_callback, *orig_debug_callback = NULL;
+
+static bool log_recursing = FALSE;
+
+static void ATTR_FORMAT(2, 0)
+doveadm_server_log_handler(const struct failure_context *ctx,
+ const char *format, va_list args)
+{
+ struct client_connection_tcp *conn = NULL;
+
+ if (doveadm_client != NULL &&
+ doveadm_client->type == DOVEADM_CONNECTION_TYPE_TCP)
+ conn = (struct client_connection_tcp *)doveadm_client;
+
+ if (!log_recursing && conn != NULL &&
+ conn->log_out != NULL) T_BEGIN {
+ struct ioloop *prev_ioloop = current_ioloop;
+ struct ostream *log_out = conn->log_out;
+ char c;
+ const char *ptr;
+ bool corked;
+ va_list va;
+
+ /* prevent re-entering this code if
+ any of the following code causes logging */
+ log_recursing = TRUE;
+
+ /* since we can get here from just about anywhere, make sure
+ the log ostream uses the connection's ioloop. */
+ if (conn->ioloop != NULL)
+ io_loop_set_current(conn->ioloop);
+
+ const char *log_prefix =
+ ctx->log_prefix != NULL ? ctx->log_prefix :
+ i_get_failure_prefix();
+ size_t log_prefix_len = strlen(log_prefix);
+ c = doveadm_log_type_to_char(ctx->type);
+ corked = o_stream_is_corked(log_out);
+
+ va_copy(va, args);
+ const char *str = t_strdup_vprintf(format, va);
+ va_end(va);
+
+ if (!corked)
+ o_stream_cork(log_out);
+ for (;;) {
+ ptr = strchr(str, '\n');
+ size_t len = ptr == NULL ? strlen(str) :
+ (size_t)(ptr - str);
+
+ o_stream_nsend(log_out, &c, 1);
+ o_stream_nsend(log_out, log_prefix, log_prefix_len);
+ o_stream_nsend(log_out, str, len);
+ o_stream_nsend(log_out, "\n", 1);
+
+ if (ptr == NULL)
+ break;
+ str = ptr+1;
+ }
+ o_stream_uncork(log_out);
+ if (corked)
+ o_stream_cork(log_out);
+ io_loop_set_current(prev_ioloop);
+
+ log_recursing = FALSE;
+ } T_END;
+
+ switch(ctx->type) {
+ case LOG_TYPE_DEBUG:
+ orig_debug_callback(ctx, format, args);
+ break;
+ case LOG_TYPE_INFO:
+ orig_info_callback(ctx, format, args);
+ break;
+ case LOG_TYPE_WARNING:
+ case LOG_TYPE_ERROR:
+ orig_error_callback(ctx, format, args);
+ break;
+ default:
+ i_unreached();
+ }
+}
+
+static void doveadm_server_capture_logs(void)
+{
+ i_assert(orig_debug_callback == NULL);
+ i_get_failure_handlers(&orig_fatal_callback, &orig_error_callback,
+ &orig_info_callback, &orig_debug_callback);
+ i_set_error_handler(doveadm_server_log_handler);
+ i_set_info_handler(doveadm_server_log_handler);
+ i_set_debug_handler(doveadm_server_log_handler);
+}
+
+static void doveadm_server_restore_logs(void)
+{
+ i_assert(orig_debug_callback != NULL);
+ i_set_error_handler(orig_error_callback);
+ i_set_info_handler(orig_info_callback);
+ i_set_debug_handler(orig_debug_callback);
+ orig_fatal_callback = NULL;
+ orig_error_callback = NULL;
+ orig_info_callback = NULL;
+ orig_debug_callback = NULL;
+}
+
+static void
+doveadm_cmd_server_post(struct client_connection_tcp *conn, const char *cmd_name)
+{
+ const char *str = NULL;
+
+ if (doveadm_exit_code == 0) {
+ o_stream_nsend(conn->output, "\n+\n", 3);
+ return;
+ }
+
+ str = doveadm_exit_code_to_str(doveadm_exit_code);
+
+ if (str != NULL) {
+ o_stream_nsend_str(conn->output,
+ t_strdup_printf("\n-%s\n", str));
+ } else {
+ o_stream_nsend_str(conn->output, "\n-\n");
+ i_error("BUG: Command '%s' returned unknown error code %d",
+ cmd_name, doveadm_exit_code);
+ }
+}
+
+static void
+doveadm_cmd_server_run_ver2(struct client_connection_tcp *conn,
+ int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx)
+{
+ i_getopt_reset();
+ if (doveadm_cmd_run_ver2(argc, argv, cctx) < 0)
+ doveadm_exit_code = EX_USAGE;
+ doveadm_cmd_server_post(conn, cctx->cmd->name);
+}
+
+static int doveadm_cmd_handle(struct client_connection_tcp *conn,
+ const char *cmd_name,
+ int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx)
+{
+ struct ioloop *prev_ioloop = current_ioloop;
+ const struct doveadm_cmd_ver2 *cmd_ver2;
+
+ if ((cmd_ver2 = doveadm_cmd_find_with_args_ver2(cmd_name, &argc, &argv)) == NULL) {
+ i_error("doveadm: Client sent unknown command: %s", cmd_name);
+ return -1;
+ }
+ cctx->cmd = cmd_ver2;
+
+ /* some commands will want to call io_loop_run(), but we're already
+ running one and we can't call the original one recursively, so
+ create a new ioloop. */
+ conn->ioloop = io_loop_create();
+ o_stream_switch_ioloop(conn->output);
+ if (conn->log_out != NULL)
+ o_stream_switch_ioloop(conn->log_out);
+
+ doveadm_cmd_server_run_ver2(conn, argc, argv, cctx);
+
+ o_stream_switch_ioloop_to(conn->output, prev_ioloop);
+ if (conn->log_out != NULL)
+ o_stream_switch_ioloop_to(conn->log_out, prev_ioloop);
+ io_loop_destroy(&conn->ioloop);
+
+ /* clear all headers */
+ doveadm_print_deinit();
+ doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER);
+
+ /* We already sent the success/failure reply to the client. Return 0
+ so caller never adds another failure reply. */
+ return 0;
+}
+
+static bool client_handle_command(struct client_connection_tcp *conn,
+ const char *const *args)
+{
+ struct doveadm_cmd_context cctx;
+ const char *flags, *cmd_name;
+ unsigned int argc = str_array_length(args);
+
+ if (argc < 3) {
+ i_error("doveadm client: No command given");
+ return FALSE;
+ }
+ i_zero(&cctx);
+ cctx.conn_type = conn->conn.type;
+ cctx.input = conn->input;
+ cctx.output = conn->output;
+ cctx.local_ip = conn->conn.local_ip;
+ cctx.remote_ip = conn->conn.remote_ip;
+ cctx.local_port = conn->conn.local_port;
+ cctx.remote_port = conn->conn.remote_port;
+ doveadm_exit_code = 0;
+
+ flags = args[0];
+ cctx.username = args[1];
+ cmd_name = args[2];
+
+ doveadm_debug = FALSE;
+ doveadm_verbose = FALSE;
+
+ for (; *flags != '\0'; flags++) {
+ switch (*flags) {
+ case 'D':
+ doveadm_debug = TRUE;
+ doveadm_verbose = TRUE;
+ break;
+ case 'v':
+ doveadm_verbose = TRUE;
+ break;
+ default:
+ i_error("doveadm client: Unknown flag: %c", *flags);
+ return FALSE;
+ }
+ }
+
+ if (!doveadm_client_is_allowed_command(conn->conn.set, cmd_name)) {
+ i_error("doveadm client isn't allowed to use command: %s",
+ cmd_name);
+ return FALSE;
+ }
+
+ client_connection_set_proctitle(&conn->conn, cmd_name);
+ o_stream_cork(conn->output);
+ /* Disable IO while running a command. This is required for commands
+ that do IO themselves (e.g. dsync-server). */
+ io_remove(&conn->io);
+ if (doveadm_cmd_handle(conn, cmd_name, argc-2, args+2, &cctx) < 0)
+ o_stream_nsend(conn->output, "\n-\n", 3);
+ o_stream_uncork(conn->output);
+ conn->io = io_add_istream(conn->input, client_connection_tcp_input, conn);
+ client_connection_set_proctitle(&conn->conn, "");
+
+ /* Try to flush the output. It might finish later. */
+ (void)o_stream_flush(conn->output);
+ return TRUE;
+}
+
+static int
+client_connection_tcp_authenticate(struct client_connection_tcp *conn)
+{
+ const struct doveadm_settings *set = conn->conn.set;
+ const char *line, *pass;
+ buffer_t *plain;
+ const unsigned char *data;
+ size_t size;
+
+ if ((line = i_stream_read_next_line(conn->input)) == NULL) {
+ if (conn->input->eof)
+ return -1;
+ return 0;
+ }
+
+ if (*set->doveadm_password == '\0') {
+ i_error("doveadm_password not set, "
+ "remote authentication disabled");
+ return -1;
+ }
+
+ if (strcmp(line, "STARTTLS") == 0) {
+ io_remove(&conn->io);
+ if (client_connection_tcp_init_ssl(conn) < 0)
+ return -1;
+ conn->io = io_add_istream(conn->input, client_connection_tcp_input, conn);
+ return 0;
+ }
+
+ /* FIXME: some day we should probably let auth process do this and
+ support all kinds of authentication */
+ if (!str_begins(line, "PLAIN\t")) {
+ i_error("doveadm client attempted non-PLAIN authentication: %s", line);
+ return -1;
+ }
+
+ plain = t_buffer_create(128);
+ if (base64_decode(line + 6, strlen(line + 6), NULL, plain) < 0) {
+ i_error("doveadm client sent invalid base64 auth PLAIN data");
+ return -1;
+ }
+ data = plain->data;
+ size = plain->used;
+
+ if (size < 10 || data[0] != '\0' ||
+ memcmp(data+1, "doveadm", 7) != 0 || data[8] != '\0') {
+ i_error("doveadm client didn't authenticate as 'doveadm'");
+ return -1;
+ }
+ pass = t_strndup(data + 9, size - 9);
+ if (strlen(pass) != strlen(set->doveadm_password) ||
+ !mem_equals_timing_safe(pass, set->doveadm_password,
+ strlen(pass))) {
+ i_error("doveadm client authenticated with wrong password");
+ return -1;
+ }
+ return 1;
+}
+
+static void client_log_disconnect_error(struct client_connection_tcp *conn)
+{
+ const char *error;
+
+ error = conn->ssl_iostream == NULL ? NULL :
+ ssl_iostream_get_last_error(conn->ssl_iostream);
+ if (error == NULL) {
+ error = conn->input->stream_errno == 0 ? "EOF" :
+ strerror(conn->input->stream_errno);
+ }
+ i_error("doveadm client disconnected before handshake: %s", error);
+}
+
+static void
+client_connection_tcp_input(struct client_connection_tcp *conn)
+{
+ const char *line;
+ bool ok = TRUE;
+ int ret;
+ unsigned int minor;
+
+ if (!conn->handshaked) {
+ if ((line = i_stream_read_next_line(conn->input)) == NULL) {
+ if (conn->input->eof || conn->input->stream_errno != 0) {
+ client_log_disconnect_error(conn);
+ client_connection_tcp_destroy(&conn);
+ }
+ return;
+ }
+ if (!version_string_verify_full(line, "doveadm-server",
+ DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR, &minor)) {
+ i_error("doveadm client not compatible with this server "
+ "(mixed old and new binaries?)");
+ client_connection_tcp_destroy(&conn);
+ return;
+ }
+ if (minor > 0) {
+ /* send version reply */
+ o_stream_nsend_str(conn->output,
+ DOVEADM_CLIENT_PROTOCOL_VERSION_LINE"\n");
+ conn->use_multiplex = TRUE;
+ }
+ client_connection_tcp_send_auth_handshake(conn);
+ conn->handshaked = TRUE;
+ }
+ if (!conn->authenticated) {
+ if ((ret = client_connection_tcp_authenticate(conn)) <= 0) {
+ if (ret < 0) {
+ o_stream_nsend(conn->output, "-\n", 2);
+ client_connection_tcp_destroy(&conn);
+ }
+ return;
+ }
+ o_stream_nsend(conn->output, "+\n", 2);
+ conn->authenticated = TRUE;
+ }
+
+ if (!conn->io_setup) {
+ conn->io_setup = TRUE;
+ if (conn->use_multiplex) {
+ struct ostream *os = conn->output;
+ conn->output = o_stream_create_multiplex(os, SIZE_MAX);
+ o_stream_set_name(conn->output, o_stream_get_name(os));
+ o_stream_set_no_error_handling(conn->output, TRUE);
+ o_stream_unref(&os);
+ conn->log_out =
+ o_stream_multiplex_add_channel(conn->output,
+ DOVEADM_LOG_CHANNEL_ID);
+ o_stream_set_no_error_handling(conn->log_out, TRUE);
+ o_stream_set_name(conn->log_out, t_strdup_printf("%s (log)",
+ o_stream_get_name(conn->output)));
+ doveadm_server_capture_logs();
+ }
+ doveadm_print_ostream = conn->output;
+ }
+
+ while (ok && !conn->input->closed &&
+ (line = i_stream_read_next_line(conn->input)) != NULL) {
+ T_BEGIN {
+ const char *const *args;
+
+ args = t_strsplit_tabescaped(line);
+ ok = client_handle_command(conn, args);
+ } T_END;
+ }
+ if (conn->input->eof || conn->input->stream_errno != 0 || !ok)
+ client_connection_tcp_destroy(&conn);
+}
+
+static int
+client_connection_tcp_init_ssl(struct client_connection_tcp *conn)
+{
+ const char *error;
+
+ if (master_service_ssl_init(master_service,
+ &conn->input, &conn->output,
+ &conn->ssl_iostream, &error) < 0) {
+ i_error("SSL init failed: %s", error);
+ return -1;
+ }
+ if (ssl_iostream_handshake(conn->ssl_iostream) < 0) {
+ i_error("SSL handshake failed: %s",
+ ssl_iostream_get_last_error(conn->ssl_iostream));
+ return -1;
+ }
+ return 0;
+}
+
+static bool
+client_connection_is_preauthenticated(int listen_fd)
+{
+ const char *listen_path;
+ struct stat st;
+
+ /* we'll have to do this with stat(), because at least in Linux
+ fstat() always returns mode as 0777 */
+ return net_getunixname(listen_fd, &listen_path) == 0 &&
+ stat(listen_path, &st) == 0 && S_ISSOCK(st.st_mode) &&
+ (st.st_mode & 0777) == 0600;
+}
+
+static void
+client_connection_tcp_send_auth_handshake(struct client_connection_tcp *conn)
+{
+ if (conn->preauthenticated) {
+ /* no need for client to authenticate */
+ conn->authenticated = TRUE;
+ o_stream_nsend(conn->output, "+\n", 2);
+ } else {
+ o_stream_nsend(conn->output, "-\n", 2);
+ }
+}
+
+static void
+client_connection_tcp_free(struct client_connection *_conn)
+{
+ struct client_connection_tcp *conn =
+ (struct client_connection_tcp *)_conn;
+
+ i_assert(_conn->type == DOVEADM_CONNECTION_TYPE_TCP);
+
+ doveadm_print_deinit();
+ doveadm_print_ostream = NULL;
+
+ if (conn->log_out != NULL) {
+ doveadm_server_restore_logs();
+ o_stream_unref(&conn->log_out);
+ }
+ ssl_iostream_destroy(&conn->ssl_iostream);
+
+ io_remove(&conn->io);
+ o_stream_destroy(&conn->output);
+ i_stream_destroy(&conn->input);
+ i_close_fd(&conn->fd);
+}
+
+struct client_connection *
+client_connection_tcp_create(int fd, int listen_fd, bool ssl)
+{
+ struct client_connection_tcp *conn;
+ pool_t pool;
+
+ pool = pool_alloconly_create("doveadm client", 1024*16);
+ conn = p_new(pool, struct client_connection_tcp, 1);
+ conn->fd = fd;
+
+ if (client_connection_init(&conn->conn,
+ DOVEADM_CONNECTION_TYPE_TCP, pool, fd) < 0) {
+ client_connection_tcp_destroy(&conn);
+ return NULL;
+ }
+ conn->conn.free = client_connection_tcp_free;
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_SERVER);
+
+ conn->input = i_stream_create_fd(fd, MAX_INBUF_SIZE);
+ conn->output = o_stream_create_fd(fd, SIZE_MAX);
+ i_stream_set_name(conn->input, conn->conn.name);
+ o_stream_set_name(conn->output, conn->conn.name);
+ o_stream_set_no_error_handling(conn->output, TRUE);
+
+ if (ssl) {
+ if (client_connection_tcp_init_ssl(conn) < 0) {
+ client_connection_tcp_destroy(&conn);
+ return NULL;
+ }
+ }
+ /* add IO after SSL istream is created */
+ conn->io = io_add_istream(conn->input, client_connection_tcp_input, conn);
+ conn->preauthenticated =
+ client_connection_is_preauthenticated(listen_fd);
+ client_connection_set_proctitle(&conn->conn, "");
+
+ return &conn->conn;
+}
+
+static void
+client_connection_tcp_destroy(struct client_connection_tcp **_conn)
+{
+ struct client_connection_tcp *conn = *_conn;
+ struct client_connection *bconn = &conn->conn;
+
+ *_conn = NULL;
+ client_connection_destroy(&bconn);
+}
diff --git a/src/doveadm/client-connection.c b/src/doveadm/client-connection.c
new file mode 100644
index 0000000..02a64a6
--- /dev/null
+++ b/src/doveadm/client-connection.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "process-title.h"
+#include "settings-parser.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "doveadm.h"
+#include "doveadm-settings.h"
+#include "doveadm-server.h"
+#include "client-connection-private.h"
+
+bool doveadm_client_is_allowed_command(const struct doveadm_settings *set,
+ const char *cmd_name)
+{
+ bool ret = FALSE;
+
+ if (*set->doveadm_allowed_commands == '\0')
+ return TRUE;
+
+ T_BEGIN {
+ const char *const *cmds =
+ t_strsplit(set->doveadm_allowed_commands, ",");
+ for (; *cmds != NULL; cmds++) {
+ if (strcmp(*cmds, cmd_name) == 0) {
+ ret = TRUE;
+ break;
+ }
+ }
+ } T_END;
+ return ret;
+}
+
+static int client_connection_read_settings(struct client_connection *conn)
+{
+ const struct setting_parser_info *set_roots[] = {
+ &doveadm_setting_parser_info,
+ NULL
+ };
+ struct master_service_settings_input input;
+ struct master_service_settings_output output;
+ const char *error;
+ void *set;
+
+ i_zero(&input);
+ input.roots = set_roots;
+ input.service = "doveadm";
+ input.local_ip = conn->local_ip;
+ input.remote_ip = conn->remote_ip;
+
+ if (master_service_settings_read(master_service, &input,
+ &output, &error) < 0) {
+ i_error("Error reading configuration: %s", error);
+ return -1;
+ }
+ set = master_service_settings_get_others(master_service)[0];
+ conn->set = settings_dup(&doveadm_setting_parser_info, set, conn->pool);
+ return 0;
+}
+
+int client_connection_init(struct client_connection *conn,
+ enum doveadm_client_type type, pool_t pool, int fd)
+{
+ const char *ip;
+
+ i_assert(type != DOVEADM_CONNECTION_TYPE_CLI);
+
+ conn->type = type;
+ conn->pool = pool;
+
+ (void)net_getsockname(fd, &conn->local_ip, &conn->local_port);
+ (void)net_getpeername(fd, &conn->remote_ip, &conn->remote_port);
+
+ ip = net_ip2addr(&conn->remote_ip);
+ if (ip[0] != '\0')
+ i_set_failure_prefix("doveadm(%s): ", ip);
+
+ conn->name = conn->remote_ip.family == 0 ? "<local>" :
+ p_strdup(pool, net_ip2addr(&conn->remote_ip));
+
+ return client_connection_read_settings(conn);
+}
+
+void client_connection_destroy(struct client_connection **_conn)
+{
+ struct client_connection *conn = *_conn;
+
+ *_conn = NULL;
+
+ if (conn->free != NULL)
+ conn->free(conn);
+
+ doveadm_client = NULL;
+ master_service_client_connection_destroyed(master_service);
+
+ if (doveadm_verbose_proctitle)
+ process_title_set("[idling]");
+
+ pool_unref(&conn->pool);
+}
+
+void client_connection_set_proctitle(struct client_connection *conn,
+ const char *text)
+{
+ const char *str;
+
+ if (!doveadm_verbose_proctitle)
+ return;
+
+ if (text[0] == '\0')
+ str = t_strdup_printf("[%s]", conn->name);
+ else
+ str = t_strdup_printf("[%s %s]", conn->name, text);
+ process_title_set(str);
+}
+
+void doveadm_server_init(void)
+{
+ doveadm_http_server_init();
+}
+
+void doveadm_server_deinit(void)
+{
+ if (doveadm_client != NULL)
+ client_connection_destroy(&doveadm_client);
+ doveadm_http_server_deinit();
+}
diff --git a/src/doveadm/client-connection.h b/src/doveadm/client-connection.h
new file mode 100644
index 0000000..bb3797e
--- /dev/null
+++ b/src/doveadm/client-connection.h
@@ -0,0 +1,26 @@
+#ifndef CLIENT_CONNECTION_H
+#define CLIENT_CONNECTION_H
+
+#include "net.h"
+
+#define DOVEADM_LOG_CHANNEL_ID 'L'
+
+struct client_connection {
+ pool_t pool;
+ enum doveadm_client_type type;
+ const char *name;
+
+ struct ip_addr local_ip, remote_ip;
+ in_port_t local_port, remote_port;
+
+ const struct doveadm_settings *set;
+
+ void (*free)(struct client_connection *conn);
+};
+
+struct client_connection *
+client_connection_tcp_create(int fd, int listen_fd, bool ssl);
+struct client_connection *
+client_connection_http_create(int fd, bool ssl);
+
+#endif
diff --git a/src/doveadm/doveadm-auth-server.c b/src/doveadm/doveadm-auth-server.c
new file mode 100644
index 0000000..f50fa5f
--- /dev/null
+++ b/src/doveadm/doveadm-auth-server.c
@@ -0,0 +1,517 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "str.h"
+#include "var-expand.h"
+#include "wildcard-match.h"
+#include "settings-parser.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "auth-client.h"
+#include "auth-master.h"
+#include "master-auth.h"
+#include "master-login-auth.h"
+#include "mail-storage-service.h"
+#include "mail-user.h"
+#include "ostream.h"
+#include "json-parser.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+struct authtest_input {
+ pool_t pool;
+ const char *username;
+ const char *master_user;
+ const char *password;
+ struct auth_user_info info;
+ bool success;
+
+ struct auth_client_request *request;
+ struct master_auth_request master_auth_req;
+
+ unsigned int auth_id;
+ unsigned int auth_pid;
+ const char *auth_cookie;
+
+};
+
+static struct auth_master_connection *
+doveadm_get_auth_master_conn(const char *auth_socket_path)
+{
+ enum auth_master_flags flags = 0;
+
+ if (doveadm_debug)
+ flags |= AUTH_MASTER_FLAG_DEBUG;
+ return auth_master_init(auth_socket_path, flags);
+}
+
+static int
+cmd_user_input(struct auth_master_connection *conn,
+ const struct authtest_input *input,
+ const char *show_field, bool userdb)
+{
+ const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup";
+ pool_t pool;
+ const char *updated_username = NULL, *const *fields, *p;
+ int ret;
+
+ pool = pool_alloconly_create("auth master lookup", 1024);
+
+ if (userdb) {
+ ret = auth_master_user_lookup(conn, input->username, &input->info,
+ pool, &updated_username, &fields);
+ } else {
+ ret = auth_master_pass_lookup(conn, input->username, &input->info,
+ pool, &fields);
+ }
+ if (ret < 0) {
+ const char *msg;
+ if (fields[0] == NULL) {
+ msg = t_strdup_printf("\"error\":\"%s failed\"",
+ lookup_name);
+ } else {
+ msg = t_strdup_printf("\"error\":\"%s failed: %s\"",
+ lookup_name,
+ fields[0]);
+ }
+ o_stream_nsend_str(doveadm_print_ostream, msg);
+ ret = -1;
+ } else if (ret == 0) {
+ o_stream_nsend_str(doveadm_print_ostream,
+ t_strdup_printf("\"error\":\"%s: user doesn't exist\"",
+ lookup_name));
+ } else if (show_field != NULL) {
+ size_t show_field_len = strlen(show_field);
+ string_t *json_field = t_str_new(show_field_len+1);
+ json_append_escaped(json_field, show_field);
+ o_stream_nsend_str(doveadm_print_ostream, t_strdup_printf("\"%s\":", str_c(json_field)));
+ for (; *fields != NULL; fields++) {
+ if (strncmp(*fields, show_field, show_field_len) == 0 &&
+ (*fields)[show_field_len] == '=') {
+ string_t *jsonval = t_str_new(32);
+ json_append_escaped(jsonval, *fields + show_field_len + 1);
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ }
+ }
+ } else {
+ string_t *jsonval = t_str_new(64);
+ o_stream_nsend_str(doveadm_print_ostream, "\"source\":\"");
+ o_stream_nsend_str(doveadm_print_ostream, userdb ? "userdb\"" : "passdb\"");
+
+ if (updated_username != NULL) {
+ o_stream_nsend_str(doveadm_print_ostream, ",\"updated_username\":\"");
+ str_truncate(jsonval, 0);
+ json_append_escaped(jsonval, updated_username);
+ o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ }
+ for (; *fields != NULL; fields++) {
+ const char *field = *fields;
+ if (*field == '\0') continue;
+ p = strchr(*fields, '=');
+ str_truncate(jsonval, 0);
+ if (p != NULL) {
+ field = t_strcut(*fields, '=');
+ }
+ str_truncate(jsonval, 0);
+ json_append_escaped(jsonval, field);
+ o_stream_nsend_str(doveadm_print_ostream, ",\"");
+ o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+ o_stream_nsend_str(doveadm_print_ostream, "\":");
+ if (p != NULL) {
+ str_truncate(jsonval, 0);
+ json_append_escaped(jsonval, p+1);
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ o_stream_nsend_str(doveadm_print_ostream, str_c(jsonval));
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ } else {
+ o_stream_nsend_str(doveadm_print_ostream, "true");
+ }
+ }
+ }
+ return ret;
+}
+
+static void auth_user_info_parse(struct auth_user_info *info, const char *arg)
+{
+ if (str_begins(arg, "service="))
+ info->service = arg + 8;
+ else if (str_begins(arg, "lip=")) {
+ if (net_addr2ip(arg + 4, &info->local_ip) < 0)
+ i_fatal("lip: Invalid ip");
+ } else if (str_begins(arg, "rip=")) {
+ if (net_addr2ip(arg + 4, &info->remote_ip) < 0)
+ i_fatal("rip: Invalid ip");
+ } else if (str_begins(arg, "lport=")) {
+ if (net_str2port(arg + 6, &info->local_port) < 0)
+ i_fatal("lport: Invalid port number");
+ } else if (str_begins(arg, "rport=")) {
+ if (net_str2port(arg + 6, &info->remote_port) < 0)
+ i_fatal("rport: Invalid port number");
+ } else {
+ i_fatal("Unknown -x argument: %s", arg);
+ }
+}
+
+static void
+cmd_user_list(struct auth_master_connection *conn,
+ const struct authtest_input *input,
+ char *const *users)
+{
+ struct auth_master_user_list_ctx *ctx;
+ const char *username, *user_mask = "*";
+ string_t *escaped = t_str_new(256);
+ bool first = TRUE;
+ unsigned int i;
+
+ if (users[0] != NULL && users[1] == NULL)
+ user_mask = users[0];
+
+ o_stream_nsend_str(doveadm_print_ostream, "{\"userList\":[");
+
+ ctx = auth_master_user_list_init(conn, user_mask, &input->info);
+ while ((username = auth_master_user_list_next(ctx)) != NULL) {
+ for (i = 0; users[i] != NULL; i++) {
+ if (wildcard_match_icase(username, users[i]))
+ break;
+ }
+ if (users[i] != NULL) {
+ if (first)
+ first = FALSE;
+ else
+ o_stream_nsend_str(doveadm_print_ostream, ",");
+ str_truncate(escaped, 0);
+ str_append_c(escaped, '"');
+ json_append_escaped(escaped, username);
+ str_append_c(escaped, '"');
+ o_stream_nsend(doveadm_print_ostream, escaped->data, escaped->used);
+ }
+ }
+ if (auth_master_user_list_deinit(&ctx) < 0) {
+ i_error("user listing failed");
+ doveadm_exit_code = EX_DATAERR;
+ }
+
+ o_stream_nsend_str(doveadm_print_ostream, "]}");
+}
+
+static void cmd_auth_cache_flush(struct doveadm_cmd_context *cctx)
+{
+ const char *master_socket_path, *const *users;
+ struct auth_master_connection *conn;
+ unsigned int count;
+
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &master_socket_path)) {
+ master_socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/auth-master", NULL);
+ }
+ if (!doveadm_cmd_param_array(cctx, "user", &users))
+ i_fatal("Missing user parameter");
+
+ conn = doveadm_get_auth_master_conn(master_socket_path);
+ if (auth_master_cache_flush(conn, users, &count) < 0) {
+ i_error("Cache flush failed");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else {
+ doveadm_print_init("formatted");
+ doveadm_print_formatted_set_format("%{entries} cache entries flushed\n");
+ doveadm_print_header_simple("entries");
+ doveadm_print_num(count);
+ }
+ auth_master_deinit(&conn);
+}
+
+static void cmd_user_mail_input_field(const char *key, const char *value,
+ const char *show_field, bool *first)
+{
+ string_t *jvalue = t_str_new(128);
+ if (show_field != NULL && strcmp(show_field, key) != 0) return;
+ /* do not emit comma on first field. we need to keep track
+ of when the first field actually gets printed as it
+ might change due to show_field */
+ if (!*first)
+ o_stream_nsend_str(doveadm_print_ostream, ",");
+ *first = FALSE;
+ json_append_escaped(jvalue, key);
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue));
+ o_stream_nsend_str(doveadm_print_ostream, "\":\"");
+ str_truncate(jvalue, 0);
+ json_append_escaped(jvalue, value);
+ o_stream_nsend_str(doveadm_print_ostream, str_c(jvalue));
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+}
+
+static void
+cmd_user_mail_print_fields(const struct authtest_input *input,
+ struct mail_user *user,
+ const char *const *userdb_fields,
+ const char *show_field)
+{
+ const struct mail_storage_settings *mail_set;
+ const char *key, *value;
+ unsigned int i;
+ bool first = TRUE;
+
+ if (strcmp(input->username, user->username) != 0)
+ cmd_user_mail_input_field("user", user->username, show_field, &first);
+ cmd_user_mail_input_field("uid", user->set->mail_uid, show_field, &first);
+ cmd_user_mail_input_field("gid", user->set->mail_gid, show_field, &first);
+ cmd_user_mail_input_field("home", user->set->mail_home, show_field, &first);
+
+ mail_set = mail_user_set_get_storage_set(user);
+ cmd_user_mail_input_field("mail", mail_set->mail_location, show_field, &first);
+
+ if (userdb_fields != NULL) {
+ for (i = 0; userdb_fields[i] != NULL; i++) {
+ value = strchr(userdb_fields[i], '=');
+ if (value != NULL)
+ key = t_strdup_until(userdb_fields[i], value++);
+ else {
+ key = userdb_fields[i];
+ value = "";
+ }
+ if (strcmp(key, "uid") != 0 &&
+ strcmp(key, "gid") != 0 &&
+ strcmp(key, "home") != 0 &&
+ strcmp(key, "mail") != 0 &&
+ *key != '\0') {
+ cmd_user_mail_input_field(key, value, show_field, &first);
+ }
+ }
+ }
+}
+
+static int
+cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
+ const struct authtest_input *input,
+ const char *show_field, const char *expand_field)
+{
+ struct mail_storage_service_input service_input;
+ struct mail_storage_service_user *service_user;
+ struct mail_user *user;
+ const char *error, *const *userdb_fields;
+ pool_t pool;
+ int ret;
+
+ i_zero(&service_input);
+ service_input.module = "mail";
+ service_input.service = input->info.service;
+ service_input.username = input->username;
+ service_input.local_ip = input->info.local_ip;
+ service_input.local_port = input->info.local_port;
+ service_input.remote_ip = input->info.remote_ip;
+ service_input.remote_port = input->info.remote_port;
+ service_input.debug = input->info.debug;
+
+ pool = pool_alloconly_create("userdb fields", 1024);
+ mail_storage_service_save_userdb_fields(storage_service, pool,
+ &userdb_fields);
+
+ if ((ret = mail_storage_service_lookup_next(storage_service, &service_input,
+ &service_user, &user,
+ &error)) <= 0) {
+ pool_unref(&pool);
+ if (ret < 0)
+ return -1;
+ string_t *username = t_str_new(32);
+ json_append_escaped(username, input->username);
+ o_stream_nsend_str(doveadm_print_ostream,
+ t_strdup_printf("\"error\":\"userdb lookup: user %s doesn't exist\"", str_c(username))
+ );
+ return 0;
+ }
+
+ if (expand_field == NULL)
+ cmd_user_mail_print_fields(input, user, userdb_fields, show_field);
+ else {
+ string_t *str = t_str_new(128);
+ if (var_expand_with_funcs(str, expand_field,
+ mail_user_var_expand_table(user),
+ mail_user_var_expand_func_table, user,
+ &error) <= 0) {
+ string_t *str = t_str_new(128);
+ str_printfa(str, "\"error\":\"Failed to expand field: ");
+ json_append_escaped(str, error);
+ str_append_c(str, '"');
+ o_stream_nsend(doveadm_print_ostream, str_data(str), str_len(str));
+ } else {
+ string_t *value = t_str_new(128);
+ json_append_escaped(value, expand_field);
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ o_stream_nsend_str(doveadm_print_ostream, str_c(value));
+ o_stream_nsend_str(doveadm_print_ostream, "\":\"");
+ str_truncate(value, 0);
+ json_append_escaped(value, str_c(str));
+ o_stream_nsend_str(doveadm_print_ostream, str_c(value));
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ }
+
+ }
+
+ mail_user_deinit(&user);
+ mail_storage_service_user_unref(&service_user);
+ pool_unref(&pool);
+ return 1;
+}
+
+static void cmd_user_ver2(struct doveadm_cmd_context *cctx)
+{
+ const char * const *optval;
+
+ const char *auth_socket_path = NULL;
+ struct auth_master_connection *conn;
+ struct authtest_input input;
+ const char *show_field = NULL, *expand_field = NULL;
+ struct mail_storage_service_ctx *storage_service = NULL;
+ bool have_wildcards, userdb_only = FALSE, first = TRUE;
+ int ret;
+
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path))
+ auth_socket_path = doveadm_settings->auth_socket_path;
+
+ (void)doveadm_cmd_param_str(cctx, "expand-field", &expand_field);
+ (void)doveadm_cmd_param_str(cctx, "field", &show_field);
+ (void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only);
+
+ i_zero(&input);
+ if (doveadm_cmd_param_array(cctx, "auth-info", &optval))
+ for(;*optval != NULL; optval++)
+ auth_user_info_parse(&input.info, *optval);
+
+ if (!doveadm_cmd_param_array(cctx, "user-mask", &optval)) {
+ doveadm_exit_code = EX_USAGE;
+ i_error("No user(s) specified");
+ return;
+ }
+
+ if (expand_field != NULL && userdb_only) {
+ i_error("-e can't be used with -u");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+ if (expand_field != NULL && show_field != NULL) {
+ i_error("-e can't be used with -f");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ conn = doveadm_get_auth_master_conn(auth_socket_path);
+
+ have_wildcards = FALSE;
+
+ for(const char *const *val = optval; *val != NULL; val++) {
+ if (strchr(*val, '*') != NULL ||
+ strchr(*val, '?') != NULL) {
+ have_wildcards = TRUE;
+ break;
+ }
+ }
+
+ if (have_wildcards) {
+ cmd_user_list(conn, &input, (char*const*)optval);
+ auth_master_deinit(&conn);
+ return;
+ }
+
+ if (!userdb_only) {
+ storage_service = mail_storage_service_init(master_service, NULL,
+ MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP |
+ MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR |
+ MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT |
+ MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS |
+ MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES |
+ MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS);
+ mail_storage_service_set_auth_conn(storage_service, conn);
+ conn = NULL;
+ }
+
+ string_t *json = t_str_new(64);
+ o_stream_nsend_str(doveadm_print_ostream, "{");
+
+ input.info.local_ip = cctx->local_ip;
+ input.info.local_port = cctx->local_port;
+ input.info.remote_ip = cctx->remote_ip;
+ input.info.remote_port = cctx->remote_port;
+
+ for(const char *const *val = optval; *val != NULL; val++) {
+ str_truncate(json, 0);
+ json_append_escaped(json, *val);
+
+ input.username = *val;
+ if (first)
+ first = FALSE;
+ else
+ o_stream_nsend_str(doveadm_print_ostream, ",");
+
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ o_stream_nsend_str(doveadm_print_ostream, str_c(json));
+ o_stream_nsend_str(doveadm_print_ostream, "\"");
+ o_stream_nsend_str(doveadm_print_ostream, ":{");
+
+ ret = !userdb_only ?
+ cmd_user_mail_input(storage_service, &input, show_field, expand_field) :
+ cmd_user_input(conn, &input, show_field, TRUE);
+
+ o_stream_nsend_str(doveadm_print_ostream, "}");
+
+ switch (ret) {
+ case -1:
+ doveadm_exit_code = EX_TEMPFAIL;
+ break;
+ case 0:
+ doveadm_exit_code = EX_NOUSER;
+ break;
+ }
+ }
+
+ o_stream_nsend_str(doveadm_print_ostream,"}");
+
+ if (storage_service != NULL)
+ mail_storage_service_deinit(&storage_service);
+ if (conn != NULL)
+ auth_master_deinit(&conn);
+}
+
+static
+struct doveadm_cmd_ver2 doveadm_cmd_auth_server[] = {
+{
+ .name = "auth cache flush",
+ .cmd = cmd_auth_cache_flush,
+ .usage = "[-a <master socket path>] [<user> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "user",
+ .cmd = cmd_user_ver2,
+ .usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] [-e <value>] [-u] <user mask> [...]",
+ .flags = CMD_FLAG_NO_PRINT,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('e', "expand-field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('u', "userdb-only", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+void doveadm_register_auth_server_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth_server); i++) {
+ doveadm_cmd_register_ver2(&doveadm_cmd_auth_server[i]);
+ }
+}
diff --git a/src/doveadm/doveadm-auth.c b/src/doveadm/doveadm-auth.c
new file mode 100644
index 0000000..d89befe
--- /dev/null
+++ b/src/doveadm/doveadm-auth.c
@@ -0,0 +1,787 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "array.h"
+#include "askpass.h"
+#include "base64.h"
+#include "hex-binary.h"
+#include "str.h"
+#include "strescape.h"
+#include "var-expand.h"
+#include "wildcard-match.h"
+#include "settings-parser.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "auth-client.h"
+#include "auth-master.h"
+#include "master-auth.h"
+#include "master-login-auth.h"
+#include "mail-storage-service.h"
+#include "mail-user.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+static struct event_category event_category_auth = {
+ .name = "auth",
+};
+
+struct authtest_input {
+ pool_t pool;
+ const char *username;
+ const char *master_user;
+ const char *password;
+ struct auth_user_info info;
+ bool success;
+
+ struct auth_client_request *request;
+ struct master_auth_request master_auth_req;
+
+ unsigned int auth_id;
+ unsigned int auth_pid;
+ const char *auth_cookie;
+
+};
+
+static void auth_cmd_help(struct doveadm_cmd_context *cctx);
+
+static struct auth_master_connection *
+doveadm_get_auth_master_conn(const char *auth_socket_path)
+{
+ enum auth_master_flags flags = 0;
+
+ if (doveadm_debug)
+ flags |= AUTH_MASTER_FLAG_DEBUG;
+ return auth_master_init(auth_socket_path, flags);
+}
+
+static int
+cmd_user_input(struct auth_master_connection *conn,
+ const struct authtest_input *input,
+ const char *show_field, bool userdb)
+{
+ const char *lookup_name = userdb ? "userdb lookup" : "passdb lookup";
+ pool_t pool;
+ const char *updated_username = NULL, *const *fields, *p;
+ int ret;
+
+ pool = pool_alloconly_create("auth master lookup", 1024);
+
+ if (userdb) {
+ ret = auth_master_user_lookup(conn, input->username, &input->info,
+ pool, &updated_username, &fields);
+ } else {
+ ret = auth_master_pass_lookup(conn, input->username, &input->info,
+ pool, &fields);
+ }
+ if (ret < 0) {
+ if (fields[0] == NULL)
+ i_error("%s failed for %s", lookup_name, input->username);
+ else {
+ i_error("%s failed for %s: %s", lookup_name,
+ input->username, fields[0]);
+ }
+ ret = -1;
+ } else if (ret == 0) {
+ fprintf(show_field == NULL ? stdout : stderr,
+ "%s: user %s doesn't exist\n", lookup_name,
+ input->username);
+ } else if (show_field != NULL) {
+ size_t show_field_len = strlen(show_field);
+
+ for (; *fields != NULL; fields++) {
+ if (strncmp(*fields, show_field, show_field_len) == 0 &&
+ (*fields)[show_field_len] == '=')
+ printf("%s\n", *fields + show_field_len + 1);
+ }
+ } else {
+ printf("%s: %s\n", userdb ? "userdb" : "passdb", input->username);
+
+ if (updated_username != NULL)
+ printf(" %-10s: %s\n", "user", updated_username);
+ for (; *fields != NULL; fields++) {
+ p = strchr(*fields, '=');
+ if (p == NULL)
+ printf(" %-10s\n", *fields);
+ else {
+ printf(" %-10s: %s\n",
+ t_strcut(*fields, '='), p + 1);
+ }
+ }
+ }
+ pool_unref(&pool);
+ return ret;
+}
+
+static void
+auth_callback(struct auth_client_request *request ATTR_UNUSED,
+ enum auth_request_status status,
+ const char *data_base64 ATTR_UNUSED,
+ const char *const *args, void *context)
+{
+ struct authtest_input *input = context;
+
+ input->request = NULL;
+ input->auth_id = auth_client_request_get_id(request);
+ input->auth_pid = auth_client_request_get_server_pid(request);
+ input->auth_cookie = input->pool == NULL ? NULL :
+ p_strdup(input->pool, auth_client_request_get_cookie(request));
+
+ if (!io_loop_is_running(current_ioloop))
+ return;
+
+ if (status == 0)
+ i_fatal("passdb expects SASL continuation");
+
+ switch (status) {
+ case AUTH_REQUEST_STATUS_ABORT:
+ i_unreached();
+ case AUTH_REQUEST_STATUS_INTERNAL_FAIL:
+ case AUTH_REQUEST_STATUS_FAIL:
+ printf("passdb: %s auth failed\n", input->username);
+ break;
+ case AUTH_REQUEST_STATUS_CONTINUE:
+ printf("passdb: %s auth unexpectedly requested continuation\n",
+ input->username);
+ break;
+ case AUTH_REQUEST_STATUS_OK:
+ input->success = TRUE;
+ printf("passdb: %s auth succeeded\n", input->username);
+ break;
+ }
+
+ if (args != NULL && *args != NULL) {
+ printf("extra fields:\n");
+ for (; *args != NULL; args++)
+ printf(" %s\n", *args);
+ }
+ io_loop_stop(current_ioloop);
+}
+
+static void auth_connected(struct auth_client *client,
+ bool connected, void *context)
+{
+ struct event *event_auth;
+ struct authtest_input *input = context;
+ struct auth_request_info info;
+ string_t *init_resp, *base64_resp;
+
+ if (!connected)
+ i_fatal("Couldn't connect to auth socket");
+ event_auth = event_create(NULL);
+ event_add_category(event_auth, &event_category_auth);
+
+ init_resp = t_str_new(128);
+ str_append(init_resp, input->username);
+ str_append_c(init_resp, '\0');
+ if (input->master_user != NULL)
+ str_append(init_resp, input->master_user);
+ else
+ str_append(init_resp, input->username);
+ str_append_c(init_resp, '\0');
+ str_append(init_resp, input->password);
+
+ base64_resp = t_str_new(128);
+ base64_encode(str_data(init_resp), str_len(init_resp), base64_resp);
+
+ i_zero(&info);
+ info.mech = "PLAIN";
+ info.service = input->info.service;
+ info.session_id = input->info.session_id;
+ info.local_name = input->info.local_name;
+ info.local_ip = input->info.local_ip;
+ info.local_port = input->info.local_port;
+ info.remote_ip = input->info.remote_ip;
+ info.remote_port = input->info.remote_port;
+ info.real_local_ip = input->info.real_local_ip;
+ info.real_remote_ip = input->info.real_remote_ip;
+ info.real_local_port = input->info.real_local_port;
+ info.real_remote_port = input->info.real_remote_port;
+ info.extra_fields = input->info.extra_fields;
+ info.forward_fields = input->info.forward_fields;
+ info.initial_resp_base64 = str_c(base64_resp);
+ if (doveadm_settings->auth_debug ||
+ event_want_debug_log(event_auth))
+ info.flags |= AUTH_REQUEST_FLAG_DEBUG;
+
+ input->request = auth_client_request_new(client, &info,
+ auth_callback, input);
+ event_unref(&event_auth);
+}
+
+static void
+cmd_auth_input(const char *auth_socket_path, struct authtest_input *input)
+{
+ struct auth_client *client;
+
+ if (auth_socket_path == NULL) {
+ auth_socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/auth-client", NULL);
+ }
+
+ client = auth_client_init(auth_socket_path, getpid(), FALSE);
+ auth_client_connect(client);
+ auth_client_set_connect_notify(client, auth_connected, input);
+
+ if (!auth_client_is_disconnected(client))
+ io_loop_run(current_ioloop);
+
+ auth_client_set_connect_notify(client, NULL, NULL);
+ auth_client_deinit(&client);
+}
+
+static void
+auth_user_info_parse_arg(struct auth_user_info *info, const char *arg)
+{
+ if (str_begins(arg, "service="))
+ info->service = arg + 8;
+ else if (str_begins(arg, "session="))
+ info->session_id = arg + 8;
+ else if (str_begins(arg, "local_name="))
+ info->local_name = arg + 11;
+ else if (str_begins(arg, "lip=")) {
+ if (net_addr2ip(arg + 4, &info->local_ip) < 0)
+ i_fatal("lip: Invalid ip");
+ } else if (str_begins(arg, "rip=")) {
+ if (net_addr2ip(arg + 4, &info->remote_ip) < 0)
+ i_fatal("rip: Invalid ip");
+ } else if (str_begins(arg, "lport=")) {
+ if (net_str2port(arg + 6, &info->local_port) < 0)
+ i_fatal("lport: Invalid port number");
+ } else if (str_begins(arg, "rport=")) {
+ if (net_str2port(arg + 6, &info->remote_port) < 0)
+ i_fatal("rport: Invalid port number");
+ } else if (str_begins(arg, "real_lip=")) {
+ if (net_addr2ip(arg + 9, &info->real_local_ip) < 0)
+ i_fatal("real_lip: Invalid ip");
+ } else if (str_begins(arg, "real_rip=")) {
+ if (net_addr2ip(arg + 9, &info->real_remote_ip) < 0)
+ i_fatal("real_rip: Invalid ip");
+ } else if (str_begins(arg, "real_lport=")) {
+ if (net_str2port(arg + 11, &info->real_local_port) < 0)
+ i_fatal("real_lport: Invalid port number");
+ } else if (str_begins(arg, "real_rport=")) {
+ if (net_str2port(arg + 11, &info->real_remote_port) < 0)
+ i_fatal("real_rport: Invalid port number");
+ } else if (str_begins(arg, "forward_")) {
+ const char *key = arg+8;
+ const char *value = strchr(arg+8, '=');
+
+ if (value == NULL)
+ value = "";
+ else
+ key = t_strdup_until(key, value++);
+ key = str_tabescape(key);
+ value = str_tabescape(value);
+ if (info->forward_fields == NULL) {
+ info->forward_fields =
+ t_strdup_printf("%s=%s", key, value);
+ } else {
+ info->forward_fields =
+ t_strdup_printf("%s\t%s=%s", info->forward_fields, key, value);
+ }
+ } else {
+ if (!array_is_created(&info->extra_fields))
+ t_array_init(&info->extra_fields, 4);
+ array_push_back(&info->extra_fields, &arg);
+ }
+}
+
+static void
+auth_user_info_parse(struct auth_user_info *info, const char *const *args)
+{
+ for (unsigned int i = 0; args[i] != NULL; i++)
+ auth_user_info_parse_arg(info, args[i]);
+}
+
+static void
+cmd_user_list(struct auth_master_connection *conn,
+ const struct authtest_input *input,
+ const char *const *users)
+{
+ struct auth_master_user_list_ctx *ctx;
+ const char *username, *user_mask = "*";
+ unsigned int i;
+
+ if (users[0] != NULL && users[1] == NULL)
+ user_mask = users[0];
+
+ ctx = auth_master_user_list_init(conn, user_mask, &input->info);
+ while ((username = auth_master_user_list_next(ctx)) != NULL) {
+ for (i = 0; users[i] != NULL; i++) {
+ if (wildcard_match_icase(username, users[i]))
+ break;
+ }
+ if (users[i] != NULL)
+ printf("%s\n", username);
+ }
+ if (auth_master_user_list_deinit(&ctx) < 0)
+ i_fatal("user listing failed");
+}
+
+static void cmd_auth_cache_flush(struct doveadm_cmd_context *cctx)
+{
+ const char *master_socket_path;
+ struct auth_master_connection *conn;
+ const char *const *users = NULL;
+ unsigned int count;
+
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &master_socket_path)) {
+ master_socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/auth-master", NULL);
+ }
+ (void)doveadm_cmd_param_array(cctx, "user", &users);
+
+ conn = doveadm_get_auth_master_conn(master_socket_path);
+ if (auth_master_cache_flush(conn, users, &count) < 0) {
+ i_error("Cache flush failed");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else {
+ printf("%u cache entries flushed\n", count);
+ }
+ auth_master_deinit(&conn);
+}
+
+static void authtest_input_init(struct authtest_input *input)
+{
+ i_zero(input);
+ input->info.service = "doveadm";
+ input->info.debug = doveadm_settings->auth_debug;
+}
+
+static void cmd_auth_test(struct doveadm_cmd_context *cctx)
+{
+ const char *auth_socket_path = NULL;
+ const char *const *auth_info;
+ struct authtest_input input;
+
+ authtest_input_init(&input);
+ (void)doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path);
+ (void)doveadm_cmd_param_str(cctx, "master-user", &input.master_user);
+ if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info))
+ auth_user_info_parse(&input.info, auth_info);
+
+ if (!doveadm_cmd_param_str(cctx, "user", &input.username))
+ auth_cmd_help(cctx);
+ if (!doveadm_cmd_param_str(cctx, "password", &input.password))
+ input.password = t_askpass("Password: ");
+ cmd_auth_input(auth_socket_path, &input);
+ if (!input.success)
+ doveadm_exit_code = EX_NOPERM;
+}
+
+static void
+master_auth_callback(const char *const *auth_args,
+ const char *errormsg, void *context)
+{
+ struct authtest_input *input = context;
+ unsigned int i;
+
+ io_loop_stop(current_ioloop);
+ if (errormsg != NULL) {
+ i_error("userdb lookup failed: %s", errormsg);
+ return;
+ }
+ printf("userdb extra fields:\n");
+ for (i = 0; auth_args[i] != NULL; i++)
+ printf(" %s\n", auth_args[i]);
+ input->success = TRUE;
+}
+
+static void
+cmd_auth_master_input(const char *auth_master_socket_path,
+ struct authtest_input *input)
+{
+ struct master_login_auth *master_auth;
+ struct master_auth_request master_auth_req;
+ buffer_t buf;
+
+ i_zero(&master_auth_req);
+ master_auth_req.tag = 1;
+ master_auth_req.auth_pid = input->auth_pid;
+ master_auth_req.auth_id = input->auth_id;
+ master_auth_req.client_pid = getpid();
+ master_auth_req.local_ip = input->info.local_ip;
+ master_auth_req.remote_ip = input->info.remote_ip;
+
+ buffer_create_from_data(&buf, master_auth_req.cookie,
+ sizeof(master_auth_req.cookie));
+ if (strlen(input->auth_cookie) == MASTER_AUTH_COOKIE_SIZE*2)
+ (void)hex_to_binary(input->auth_cookie, &buf);
+
+ input->success = FALSE;
+ master_auth = master_login_auth_init(auth_master_socket_path, FALSE);
+ io_loop_set_running(current_ioloop);
+ master_login_auth_request(master_auth, &master_auth_req,
+ master_auth_callback, input);
+ if (io_loop_is_running(current_ioloop))
+ io_loop_run(current_ioloop);
+ master_login_auth_deinit(&master_auth);
+}
+
+static void cmd_auth_login(struct doveadm_cmd_context *cctx)
+{
+ const char *auth_login_socket_path, *auth_master_socket_path;
+ const char *const *auth_info;
+ struct auth_client *auth_client;
+ struct authtest_input input;
+
+ authtest_input_init(&input);
+ if (!doveadm_cmd_param_str(cctx, "auth-login-socket-path",
+ &auth_login_socket_path)) {
+ auth_login_socket_path =
+ t_strconcat(doveadm_settings->base_dir,
+ "/auth-login", NULL);
+ }
+ if (!doveadm_cmd_param_str(cctx, "auth-master-socket-path",
+ &auth_master_socket_path)) {
+ auth_master_socket_path =
+ t_strconcat(doveadm_settings->base_dir,
+ "/auth-master", NULL);
+ }
+ (void)doveadm_cmd_param_str(cctx, "master-user", &input.master_user);
+ if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info))
+ auth_user_info_parse(&input.info, auth_info);
+ if (!doveadm_cmd_param_str(cctx, "user", &input.username))
+ auth_cmd_help(cctx);
+ if (!doveadm_cmd_param_str(cctx, "password", &input.password))
+ input.password = t_askpass("Password: ");
+
+ input.pool = pool_alloconly_create("auth login", 256);
+ /* authenticate */
+ auth_client = auth_client_init(auth_login_socket_path, getpid(), FALSE);
+ auth_client_connect(auth_client);
+ auth_client_set_connect_notify(auth_client, auth_connected, &input);
+ if (!auth_client_is_disconnected(auth_client))
+ io_loop_run(current_ioloop);
+ auth_client_set_connect_notify(auth_client, NULL, NULL);
+ /* finish login with userdb lookup */
+ if (input.success)
+ cmd_auth_master_input(auth_master_socket_path, &input);
+ if (!input.success)
+ doveadm_exit_code = EX_NOPERM;
+ auth_client_deinit(&auth_client);
+ pool_unref(&input.pool);
+}
+
+static void cmd_auth_lookup(struct doveadm_cmd_context *cctx)
+{
+ const char *auth_socket_path;
+ struct auth_master_connection *conn;
+ struct authtest_input input;
+ const char *show_field = NULL;
+ const char *const *auth_info, *const *users;
+ bool first = TRUE;
+ int ret;
+
+ authtest_input_init(&input);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path))
+ auth_socket_path = doveadm_settings->auth_socket_path;
+ (void)doveadm_cmd_param_str(cctx, "field", &show_field);
+ if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info))
+ auth_user_info_parse(&input.info, auth_info);
+ if (!doveadm_cmd_param_array(cctx, "user", &users))
+ auth_cmd_help(cctx);
+
+ conn = doveadm_get_auth_master_conn(auth_socket_path);
+ for (unsigned int i = 0; users[i] != NULL; i++) {
+ input.username = users[i];
+ if (first)
+ first = FALSE;
+ else
+ putchar('\n');
+
+ ret = cmd_user_input(conn, &input, show_field, FALSE);
+ switch (ret) {
+ case -1:
+ doveadm_exit_code = EX_TEMPFAIL;
+ break;
+ case 0:
+ doveadm_exit_code = EX_NOUSER;
+ break;
+ }
+ }
+ auth_master_deinit(&conn);
+}
+
+static void cmd_user_mail_input_field(const char *key, const char *value,
+ const char *show_field)
+{
+ if (show_field == NULL) {
+ doveadm_print(key);
+ doveadm_print(value);
+ } else if (strcmp(show_field, key) == 0) {
+ printf("%s\n", value);
+ }
+}
+
+static void
+cmd_user_mail_print_fields(const struct authtest_input *input,
+ struct mail_user *user,
+ const char *const *userdb_fields,
+ const char *show_field)
+{
+ const struct mail_storage_settings *mail_set;
+ const char *key, *value;
+ unsigned int i;
+
+ if (strcmp(input->username, user->username) != 0)
+ cmd_user_mail_input_field("user", user->username, show_field);
+ cmd_user_mail_input_field("uid", user->set->mail_uid, show_field);
+ cmd_user_mail_input_field("gid", user->set->mail_gid, show_field);
+ cmd_user_mail_input_field("home", user->set->mail_home, show_field);
+
+ mail_set = mail_user_set_get_storage_set(user);
+ cmd_user_mail_input_field("mail", mail_set->mail_location, show_field);
+
+ if (userdb_fields != NULL) {
+ for (i = 0; userdb_fields[i] != NULL; i++) {
+ value = strchr(userdb_fields[i], '=');
+ if (value != NULL)
+ key = t_strdup_until(userdb_fields[i], value++);
+ else {
+ key = userdb_fields[i];
+ value = "";
+ }
+ if (strcmp(key, "uid") != 0 &&
+ strcmp(key, "gid") != 0 &&
+ strcmp(key, "home") != 0 &&
+ strcmp(key, "mail") != 0)
+ cmd_user_mail_input_field(key, value, show_field);
+ }
+ }
+}
+
+static int
+cmd_user_mail_input(struct mail_storage_service_ctx *storage_service,
+ const struct authtest_input *input,
+ const char *show_field, const char *expand_field)
+{
+ struct mail_storage_service_input service_input;
+ struct mail_storage_service_user *service_user;
+ struct mail_user *user;
+ const char *error, *const *userdb_fields;
+ pool_t pool;
+ int ret;
+
+ i_zero(&service_input);
+ service_input.module = "mail";
+ service_input.service = input->info.service;
+ service_input.username = input->username;
+ service_input.local_ip = input->info.local_ip;
+ service_input.local_port = input->info.local_port;
+ service_input.remote_ip = input->info.remote_ip;
+ service_input.remote_port = input->info.remote_port;
+ service_input.debug = input->info.debug;
+
+ pool = pool_alloconly_create("userdb fields", 1024);
+ mail_storage_service_save_userdb_fields(storage_service, pool,
+ &userdb_fields);
+
+ if ((ret = mail_storage_service_lookup_next(storage_service, &service_input,
+ &service_user, &user,
+ &error)) <= 0) {
+ pool_unref(&pool);
+ if (ret < 0)
+ return -1;
+ fprintf(show_field == NULL && expand_field == NULL ? stdout : stderr,
+ "\nuserdb lookup: user %s doesn't exist\n",
+ input->username);
+ return 0;
+ }
+
+ if (expand_field == NULL)
+ cmd_user_mail_print_fields(input, user, userdb_fields, show_field);
+ else {
+ string_t *str = t_str_new(128);
+ if (var_expand_with_funcs(str, expand_field,
+ mail_user_var_expand_table(user),
+ mail_user_var_expand_func_table, user,
+ &error) <= 0) {
+ i_error("Failed to expand %s: %s", expand_field, error);
+ } else {
+ printf("%s\n", str_c(str));
+ }
+ }
+
+ mail_user_deinit(&user);
+ mail_storage_service_user_unref(&service_user);
+ pool_unref(&pool);
+ return 1;
+}
+
+static void cmd_user(struct doveadm_cmd_context *cctx)
+{
+ const char *auth_socket_path;
+ struct auth_master_connection *conn;
+ struct authtest_input input;
+ const char *show_field = NULL, *expand_field = NULL;
+ const char *const *user_masks, *const *auth_info;
+ struct mail_storage_service_ctx *storage_service = NULL;
+ unsigned int i;
+ bool have_wildcards, userdb_only = FALSE, first = TRUE;
+ int ret;
+
+ authtest_input_init(&input);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &auth_socket_path))
+ auth_socket_path = doveadm_settings->auth_socket_path;
+ (void)doveadm_cmd_param_str(cctx, "field", &show_field);
+ (void)doveadm_cmd_param_str(cctx, "expand-field", &expand_field);
+ (void)doveadm_cmd_param_bool(cctx, "userdb-only", &userdb_only);
+ if (doveadm_cmd_param_array(cctx, "auth-info", &auth_info))
+ auth_user_info_parse(&input.info, auth_info);
+ if (!doveadm_cmd_param_array(cctx, "user-mask", &user_masks))
+ auth_cmd_help(cctx);
+
+ if (expand_field != NULL && userdb_only) {
+ i_error("-e can't be used with -u");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+ if (expand_field != NULL && show_field != NULL) {
+ i_error("-e can't be used with -f");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ conn = doveadm_get_auth_master_conn(auth_socket_path);
+
+ have_wildcards = FALSE;
+ for (i = 0; user_masks[i] != NULL; i++) {
+ if (strchr(user_masks[i], '*') != NULL ||
+ strchr(user_masks[i], '?') != NULL) {
+ have_wildcards = TRUE;
+ break;
+ }
+ }
+
+ if (have_wildcards) {
+ cmd_user_list(conn, &input, user_masks);
+ auth_master_deinit(&conn);
+ return;
+ }
+
+ if (!userdb_only) {
+ storage_service = mail_storage_service_init(master_service, NULL,
+ MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP |
+ MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR |
+ MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT |
+ MAIL_STORAGE_SERVICE_FLAG_NO_PLUGINS |
+ MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES |
+ MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS);
+ mail_storage_service_set_auth_conn(storage_service, conn);
+ conn = NULL;
+ if (show_field == NULL && expand_field == NULL) {
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TAB);
+ doveadm_print_header_simple("field");
+ doveadm_print_header_simple("value");
+ }
+ }
+
+ for (i = 0; user_masks[i] != NULL; i++) {
+ input.username = user_masks[i];
+ if (first)
+ first = FALSE;
+ else
+ putchar('\n');
+
+ ret = !userdb_only ?
+ cmd_user_mail_input(storage_service, &input, show_field, expand_field) :
+ cmd_user_input(conn, &input, show_field, TRUE);
+ switch (ret) {
+ case -1:
+ doveadm_exit_code = EX_TEMPFAIL;
+ break;
+ case 0:
+ doveadm_exit_code = EX_NOUSER;
+ break;
+ }
+ }
+ if (storage_service != NULL)
+ mail_storage_service_deinit(&storage_service);
+ if (conn != NULL)
+ auth_master_deinit(&conn);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_auth[] = {
+{
+ .cmd = cmd_auth_test,
+ .name = "auth test",
+ .usage = "[-a <auth socket path>] [-x <auth info>] [-M <master user>] <user> [<password>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('M', "master-user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "password", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .cmd = cmd_auth_login,
+ .name = "auth login",
+ .usage = "[-a <auth-login socket path>] [-m <auth-master socket path>] [-x <auth info>] [-M <master user>] <user> [<password>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "auth-login-socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('m', "auth-master-socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('M', "master-user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "password", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .cmd = cmd_auth_lookup,
+ .name = "auth lookup",
+ .usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] <user> [<user> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .cmd = cmd_auth_cache_flush,
+ .name = "auth cache flush",
+ .usage = "[-a <master socket path>] [<user> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .cmd = cmd_user,
+ .name = "user",
+ .usage = "[-a <userdb socket path>] [-x <auth info>] [-f field] [-e <value>] [-u] <user mask> [<user mask> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('x', "auth-info", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('e', "expand-field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('u', "userdb-only", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+static void auth_cmd_help(struct doveadm_cmd_context *cctx)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth); i++) {
+ if (doveadm_cmd_auth[i].cmd == cctx->cmd->cmd)
+ help_ver2(&doveadm_cmd_auth[i]);
+ }
+ i_unreached();
+}
+
+void doveadm_register_auth_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_auth); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_auth[i]);
+}
diff --git a/src/doveadm/doveadm-cmd.c b/src/doveadm/doveadm-cmd.c
new file mode 100644
index 0000000..5de976d
--- /dev/null
+++ b/src/doveadm/doveadm-cmd.c
@@ -0,0 +1,469 @@
+/* Copyright (c) 2009-2r016 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "str.h"
+#include "net.h"
+#include "doveadm.h"
+#include "doveadm-cmd.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <getopt.h>
+
+static struct doveadm_cmd_ver2 *doveadm_commands_ver2[] = {
+ &doveadm_cmd_mailbox_mutf7,
+ &doveadm_cmd_service_stop_ver2,
+ &doveadm_cmd_service_status_ver2,
+ &doveadm_cmd_sis_deduplicate,
+ &doveadm_cmd_sis_find,
+ &doveadm_cmd_process_status_ver2,
+ &doveadm_cmd_stop_ver2,
+ &doveadm_cmd_reload_ver2,
+ &doveadm_cmd_stats_dump_ver2,
+ &doveadm_cmd_stats_add_ver2,
+ &doveadm_cmd_stats_remove_ver2,
+ &doveadm_cmd_oldstats_dump_ver2,
+ &doveadm_cmd_oldstats_reset_ver2,
+ &doveadm_cmd_penalty_ver2,
+ &doveadm_cmd_kick_ver2,
+ &doveadm_cmd_who_ver2
+};
+
+static const struct exit_code_str {
+ int code;
+ const char *str;
+} exit_code_strings[] = {
+ { DOVEADM_EX_UNKNOWN, "UNKNOWN" },
+ { EX_TEMPFAIL, "TEMPFAIL" },
+ { EX_USAGE, "USAGE" },
+ { EX_NOUSER, "NOUSER" },
+ { EX_NOPERM, "NOPERM" },
+ { EX_PROTOCOL, "PROTOCOL" },
+ { EX_DATAERR, "DATAERR" },
+ { DOVEADM_EX_NOREPLICATE, "NOREPLICATE" },
+ { DOVEADM_EX_NOTFOUND, "NOTFOUND" }
+};
+
+ARRAY_TYPE(doveadm_cmd_ver2) doveadm_cmds_ver2;
+ARRAY_DEFINE_TYPE(getopt_option_array, struct option);
+
+const char *doveadm_exit_code_to_str(int code)
+{
+ for(size_t i = 0; i < N_ELEMENTS(exit_code_strings); i++) {
+ const struct exit_code_str *ptr = &exit_code_strings[i];
+ if (ptr->code == code)
+ return ptr->str;
+ }
+ return "UNKNOWN";
+}
+
+int doveadm_str_to_exit_code(const char *reason)
+{
+ for(size_t i = 0; i < N_ELEMENTS(exit_code_strings); i++) {
+ const struct exit_code_str *ptr = &exit_code_strings[i];
+ if (strcmp(ptr->str, reason) == 0)
+ return ptr->code;
+ }
+ return DOVEADM_EX_UNKNOWN;
+}
+
+void doveadm_cmd_register_ver2(struct doveadm_cmd_ver2 *cmd)
+{
+ if (cmd->cmd == NULL) {
+ if (cmd->mail_cmd != NULL)
+ cmd->cmd = doveadm_cmd_ver2_to_mail_cmd_wrapper;
+ else i_unreached();
+ }
+ array_push_back(&doveadm_cmds_ver2, cmd);
+}
+
+const struct doveadm_cmd_ver2 *doveadm_cmd_find_ver2(const char *cmd_name)
+{
+ const struct doveadm_cmd_ver2 *cmd;
+
+ array_foreach(&doveadm_cmds_ver2, cmd) {
+ if (strcmp(cmd_name, cmd->name) == 0)
+ return cmd;
+ }
+ return NULL;
+}
+
+const struct doveadm_cmd_ver2 *
+doveadm_cmd_find_with_args_ver2(const char *cmd_name, int *argc,
+ const char *const *argv[])
+{
+ int i, k;
+ const struct doveadm_cmd_ver2 *cmd;
+ const char *cptr;
+
+ for (i = 0; i < *argc; i++) {
+ if (strcmp((*argv)[i], cmd_name) == 0)
+ break;
+ }
+
+ i_assert(i != *argc);
+
+ array_foreach(&doveadm_cmds_ver2, cmd) {
+ cptr = cmd->name;
+ /* cannot reuse i here because this needs be
+ done more than once */
+ for (k = 0; *cptr != '\0' && i + k < *argc; k++) {
+ size_t alen = strlen((*argv)[i + k]);
+ /* make sure we don't overstep */
+ if (strlen(cptr) < alen)
+ break;
+ /* did not match */
+ if (strncmp(cptr, (*argv)[i+k], alen) != 0)
+ break;
+ /* do not accept abbreviations */
+ if (cptr[alen] != ' ' && cptr[alen] != '\0')
+ break;
+ cptr += alen;
+ if (*cptr != '\0')
+ cptr++; /* consume space */
+ }
+ /* name was fully consumed */
+ if (*cptr == '\0') {
+ if (k > 1) {
+ *argc -= k-1;
+ *argv += k-1;
+ }
+ return cmd;
+ }
+ }
+
+ return NULL;
+}
+
+void doveadm_cmds_init(void)
+{
+ unsigned int i;
+
+ i_array_init(&doveadm_cmds_ver2, 2);
+
+ for (i = 0; i < N_ELEMENTS(doveadm_commands_ver2); i++)
+ doveadm_cmd_register_ver2(doveadm_commands_ver2[i]);
+
+ doveadm_register_director_commands();
+ doveadm_register_instance_commands();
+ doveadm_register_proxy_commands();
+ doveadm_register_log_commands();
+ doveadm_register_replicator_commands();
+ doveadm_register_dict_commands();
+ doveadm_register_fs_commands();
+}
+
+void doveadm_cmds_deinit(void)
+{
+ array_free(&doveadm_cmds_ver2);
+}
+
+static const struct doveadm_cmd_param *
+doveadm_cmd_param_get(const struct doveadm_cmd_context *cctx,
+ const char *name)
+{
+ i_assert(cctx != NULL);
+ i_assert(cctx->argv != NULL);
+ for(int i = 0; i < cctx->argc; i++) {
+ if (strcmp(cctx->argv[i].name, name) == 0 &&
+ cctx->argv[i].value_set)
+ return &cctx->argv[i];
+ }
+ return NULL;
+}
+
+bool doveadm_cmd_param_bool(const struct doveadm_cmd_context *cctx,
+ const char *name, bool *value_r)
+{
+ const struct doveadm_cmd_param *param;
+ if ((param = doveadm_cmd_param_get(cctx, name)) == NULL)
+ return FALSE;
+
+ if (param->type == CMD_PARAM_BOOL) {
+ *value_r = param->value.v_bool;
+ return TRUE;
+ }
+ return FALSE;
+}
+
+bool doveadm_cmd_param_int64(const struct doveadm_cmd_context *cctx,
+ const char *name, int64_t *value_r)
+{
+ const struct doveadm_cmd_param *param;
+ if ((param = doveadm_cmd_param_get(cctx, name)) == NULL)
+ return FALSE;
+
+ if (param->type == CMD_PARAM_INT64) {
+ *value_r = param->value.v_int64;
+ return TRUE;
+ }
+ return FALSE;
+}
+
+bool doveadm_cmd_param_str(const struct doveadm_cmd_context *cctx,
+ const char *name, const char **value_r)
+{
+ const struct doveadm_cmd_param *param;
+ if ((param = doveadm_cmd_param_get(cctx, name)) == NULL)
+ return FALSE;
+
+ if (param->type == CMD_PARAM_STR) {
+ *value_r = param->value.v_string;
+ return TRUE;
+ }
+ return FALSE;
+}
+
+bool doveadm_cmd_param_ip(const struct doveadm_cmd_context *cctx,
+ const char *name, struct ip_addr *value_r)
+{
+ const struct doveadm_cmd_param *param;
+ if ((param = doveadm_cmd_param_get(cctx, name)) == NULL)
+ return FALSE;
+
+ if (param->type == CMD_PARAM_IP) {
+ memcpy(value_r, &param->value.v_ip, sizeof(struct ip_addr));
+ return TRUE;
+ }
+ return FALSE;
+}
+
+bool doveadm_cmd_param_array(const struct doveadm_cmd_context *cctx,
+ const char *name, const char *const **value_r)
+{
+ const struct doveadm_cmd_param *param;
+ unsigned int count;
+
+ if ((param = doveadm_cmd_param_get(cctx, name)) == NULL)
+ return FALSE;
+ if (param->type == CMD_PARAM_ARRAY) {
+ *value_r = array_get(&param->value.v_array, &count);
+ /* doveadm_cmd_params_null_terminate_arrays() should have been
+ called, which guarantees that we're NULL-terminated */
+ i_assert((*value_r)[count] == NULL);
+ return TRUE;
+ }
+ return FALSE;
+}
+
+bool doveadm_cmd_param_istream(const struct doveadm_cmd_context *cctx,
+ const char *name, struct istream **value_r)
+{
+ const struct doveadm_cmd_param *param;
+ if ((param = doveadm_cmd_param_get(cctx, name)) == NULL)
+ return FALSE;
+
+ if (param->type == CMD_PARAM_ISTREAM) {
+ *value_r = param->value.v_istream;
+ return TRUE;
+ }
+ return FALSE;
+}
+
+void doveadm_cmd_params_clean(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv)
+{
+ struct doveadm_cmd_param *param;
+
+ array_foreach_modifiable(pargv, param) {
+ if (param->type == CMD_PARAM_ISTREAM &&
+ param->value.v_istream != NULL)
+ i_stream_destroy(&param->value.v_istream);
+ }
+ array_clear(pargv);
+}
+
+void doveadm_cmd_params_null_terminate_arrays(
+ ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv)
+{
+ struct doveadm_cmd_param *param;
+
+ array_foreach_modifiable(pargv, param) {
+ if (param->type == CMD_PARAM_ARRAY &&
+ array_is_created(&param->value.v_array)) {
+ array_append_zero(&param->value.v_array);
+ array_pop_back(&param->value.v_array);
+ }
+ }
+}
+
+static void
+doveadm_build_options(const struct doveadm_cmd_param par[],
+ string_t *shortopts,
+ ARRAY_TYPE(getopt_option_array) *longopts)
+{
+ for (size_t i = 0; par[i].name != NULL; i++) {
+ struct option longopt;
+
+ i_zero(&longopt);
+ longopt.name = par[i].name;
+ if (par[i].short_opt != '\0') {
+ longopt.val = par[i].short_opt;
+ str_append_c(shortopts, par[i].short_opt);
+ if (par[i].type != CMD_PARAM_BOOL)
+ str_append_c(shortopts, ':');
+ }
+ if (par[i].type != CMD_PARAM_BOOL)
+ longopt.has_arg = 1;
+ array_push_back(longopts, &longopt);
+ }
+ array_append_zero(longopts);
+}
+
+static void
+doveadm_fill_param(struct doveadm_cmd_param *param,
+ const char *value, pool_t pool)
+{
+ param->value_set = TRUE;
+ switch (param->type) {
+ case CMD_PARAM_BOOL:
+ param->value.v_bool = TRUE;
+ break;
+ case CMD_PARAM_INT64:
+ if (str_to_int64(value, &param->value.v_int64) != 0)
+ param->value_set = FALSE;
+ break;
+ case CMD_PARAM_IP:
+ if (net_addr2ip(value, &param->value.v_ip) != 0)
+ param->value_set = FALSE;
+ break;
+ case CMD_PARAM_STR:
+ param->value.v_string = p_strdup(pool, value);
+ break;
+ case CMD_PARAM_ARRAY:
+ if (!array_is_created(&param->value.v_array))
+ p_array_init(&param->value.v_array, pool, 8);
+ const char *val = p_strdup(pool, value);
+ array_push_back(&param->value.v_array, &val);
+ break;
+ case CMD_PARAM_ISTREAM: {
+ struct istream *is;
+ if (strcmp(value,"-") == 0)
+ is = i_stream_create_fd(STDIN_FILENO, IO_BLOCK_SIZE);
+ else
+ is = i_stream_create_file(value, IO_BLOCK_SIZE);
+ param->value.v_istream = is;
+ break;
+ }
+ }
+}
+
+bool doveadm_cmd_try_run_ver2(const char *cmd_name,
+ int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx)
+{
+ const struct doveadm_cmd_ver2 *cmd;
+
+ cmd = doveadm_cmd_find_with_args_ver2(cmd_name, &argc, &argv);
+ if (cmd == NULL)
+ return FALSE;
+
+ cctx->cmd = cmd;
+ if (doveadm_cmd_run_ver2(argc, argv, cctx) < 0)
+ doveadm_exit_code = EX_USAGE;
+ return TRUE;
+}
+
+static int
+doveadm_cmd_process_options(int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx, pool_t pool,
+ ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv)
+{
+ struct doveadm_cmd_param *param;
+ ARRAY_TYPE(getopt_option_array) opts;
+ string_t *optbuf = str_new(pool, 64);
+
+ p_array_init(&opts, pool, 4);
+
+ // build parameters
+ if ((cctx->cmd->flags & CMD_FLAG_NO_UNORDERED_OPTIONS) != 0)
+ str_append_c(optbuf, '+');
+ doveadm_build_options(cctx->cmd->parameters, optbuf, &opts);
+
+ unsigned int pargc;
+ for (pargc = 0; cctx->cmd->parameters[pargc].name != NULL; pargc++) {
+ param = array_append_space(pargv);
+ memcpy(param, &cctx->cmd->parameters[pargc],
+ sizeof(struct doveadm_cmd_param));
+ param->value_set = FALSE;
+ }
+ i_assert(pargc == array_count(&opts)-1); /* opts is NULL-terminated */
+
+ if ((cctx->cmd->flags & CMD_FLAG_NO_OPTIONS) != 0) {
+ /* process -parameters as if they were regular parameters */
+ optind = 1;
+ return 0;
+ }
+
+ int c, li;
+ while ((c = getopt_long(argc, (char *const *)argv, str_c(optbuf),
+ array_front(&opts), &li)) > -1) {
+ switch (c) {
+ case 0:
+ for (unsigned int i = 0; i < array_count(pargv); i++) {
+ const struct option *opt = array_idx(&opts, li);
+ param = array_idx_modifiable(pargv, i);
+ if (opt->name == param->name)
+ doveadm_fill_param(param, optarg, pool);
+ }
+ break;
+ case '?':
+ case ':':
+ doveadm_cmd_params_clean(pargv);
+ return -1;
+ default:
+ // hunt the option
+ for (unsigned int i = 0; i < pargc; i++) {
+ const struct option *longopt =
+ array_idx(&opts, i);
+ if (longopt->val == c)
+ doveadm_fill_param(array_idx_modifiable(pargv, i),
+ optarg, pool);
+ }
+ }
+ }
+ return 0;
+}
+
+int doveadm_cmd_run_ver2(int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx)
+{
+ ARRAY_TYPE(doveadm_cmd_param_arr_t) pargv;
+ unsigned int pargc;
+ pool_t pool = pool_datastack_create();
+
+ p_array_init(&pargv, pool, 20);
+ if (doveadm_cmd_process_options(argc, argv, cctx, pool, &pargv) < 0)
+ return -1;
+
+ /* process positional arguments */
+ for (; optind < argc; optind++) {
+ struct doveadm_cmd_param *ptr;
+ bool found = FALSE;
+ array_foreach_modifiable(&pargv, ptr) {
+ if ((ptr->flags & CMD_PARAM_FLAG_POSITIONAL) != 0 &&
+ (ptr->value_set == FALSE ||
+ ptr->type == CMD_PARAM_ARRAY)) {
+ doveadm_fill_param(ptr, argv[optind], pool);
+ found = TRUE;
+ break;
+ }
+ }
+ if (!found) {
+ i_error("Extraneous arguments found: %s",
+ t_strarray_join(argv + optind, " "));
+ doveadm_cmd_params_clean(&pargv);
+ return -1;
+ }
+ }
+
+ doveadm_cmd_params_null_terminate_arrays(&pargv);
+ cctx->argv = array_get_modifiable(&pargv, &pargc);
+ cctx->argc = pargc;
+
+ cctx->cmd->cmd(cctx);
+
+ doveadm_cmd_params_clean(&pargv);
+ return 0;
+}
diff --git a/src/doveadm/doveadm-cmd.h b/src/doveadm/doveadm-cmd.h
new file mode 100644
index 0000000..a988575
--- /dev/null
+++ b/src/doveadm/doveadm-cmd.h
@@ -0,0 +1,155 @@
+#ifndef DOVEADM_CMD_H
+#define DOVEADM_CMD_H
+
+#include "net.h"
+
+#define DOVEADM_CMD_PARAMS_START .parameters = (const struct doveadm_cmd_param[]){
+#define DOVEADM_CMD_PARAM(optP, nameP, typeP, flagP ) { .short_opt = optP, .name = nameP, .type = typeP, .flags = flagP },
+#define DOVEADM_CMD_PARAMS_END { .short_opt = '\0', .name = NULL, .type = CMD_PARAM_BOOL, .flags = CMD_PARAM_FLAG_NONE } }
+
+struct doveadm_cmd_ver2;
+struct doveadm_cmd_context;
+struct doveadm_mail_cmd_context;
+
+typedef void doveadm_command_t(int argc, char *argv[]);
+
+typedef enum {
+ CMD_PARAM_BOOL = 0, /* value will contain 1 (not pointer) */
+ CMD_PARAM_INT64, /* ditto but contains number (not pointer) */
+ CMD_PARAM_IP, /* value contains struct ip_addr */
+ CMD_PARAM_STR, /* value contains const char* */
+ CMD_PARAM_ARRAY, /* value contains const char*[] */
+ CMD_PARAM_ISTREAM /* value contains struct istream* */
+} doveadm_cmd_param_t;
+
+typedef enum {
+ CMD_PARAM_FLAG_NONE = 0x0,
+ CMD_PARAM_FLAG_POSITIONAL = 0x1,
+ CMD_PARAM_FLAG_DO_NOT_EXPOSE = 0x2,
+} doveadm_cmd_param_flag_t;
+
+typedef enum {
+ CMD_FLAG_NONE = 0x0,
+ CMD_FLAG_HIDDEN = 0x1,
+ CMD_FLAG_NO_PRINT = 0x2,
+ /* Don't parse any -options for the command. */
+ CMD_FLAG_NO_OPTIONS = 0x4,
+ /* Prevent GNU getopt() from finding options after the first
+ non-option is seen (e.g. "-1 arg -2" would parse -1 but not -2
+ as option). */
+ CMD_FLAG_NO_UNORDERED_OPTIONS = 0x8,
+} doveadm_cmd_flag_t;
+
+struct doveadm_cmd_param {
+ char short_opt;
+ const char *name;
+ doveadm_cmd_param_t type;
+ bool value_set;
+ struct {
+ bool v_bool;
+ int64_t v_int64;
+ const char* v_string;
+ ARRAY_TYPE(const_string) v_array;
+ struct ip_addr v_ip;
+ struct istream* v_istream;
+ } value;
+ doveadm_cmd_param_flag_t flags;
+};
+ARRAY_DEFINE_TYPE(doveadm_cmd_param_arr_t, struct doveadm_cmd_param);
+
+typedef void doveadm_command_ver2_t(struct doveadm_cmd_context *cctx);
+
+struct doveadm_cmd_ver2 {
+ doveadm_command_ver2_t *cmd;
+ struct doveadm_mail_cmd_context *(*mail_cmd)(void);
+ const char *name;
+ const char *usage;
+ doveadm_cmd_flag_t flags;
+ const struct doveadm_cmd_param *parameters;
+};
+
+struct doveadm_cmd_context {
+ const struct doveadm_cmd_ver2 *cmd; /* for help */
+
+ int argc;
+ const struct doveadm_cmd_param *argv;
+
+ const char *username;
+ struct ip_addr local_ip, remote_ip;
+ in_port_t local_port, remote_port;
+
+ enum doveadm_client_type conn_type;
+ struct istream *input;
+ struct ostream *output;
+};
+
+ARRAY_DEFINE_TYPE(doveadm_cmd_ver2, struct doveadm_cmd_ver2);
+extern ARRAY_TYPE(doveadm_cmd_ver2) doveadm_cmds_ver2;
+
+void doveadm_register_auth_commands(void);
+void doveadm_register_auth_server_commands(void);
+void doveadm_register_director_commands(void);
+void doveadm_register_proxy_commands(void);
+void doveadm_register_log_commands(void);
+void doveadm_register_instance_commands(void);
+void doveadm_register_mount_commands(void);
+void doveadm_register_replicator_commands(void);
+void doveadm_register_dict_commands(void);
+void doveadm_register_fs_commands(void);
+
+void doveadm_cmds_init(void);
+void doveadm_cmds_deinit(void);
+
+void doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx);
+
+void doveadm_cmd_register_ver2(struct doveadm_cmd_ver2 *cmd);
+const struct doveadm_cmd_ver2 *
+doveadm_cmd_find_with_args_ver2(const char *cmd_name, int *argc,
+ const char *const *argv[]);
+const struct doveadm_cmd_ver2 *doveadm_cmd_find_ver2(const char *cmd_name);
+/* Returns FALSE if cmd_name doesn't exist, TRUE if it exists. */
+bool doveadm_cmd_try_run_ver2(const char *cmd_name,
+ int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx);
+/* Returns 0 if success, -1 if parameters were invalid. */
+int doveadm_cmd_run_ver2(int argc, const char *const argv[],
+ struct doveadm_cmd_context *cctx);
+
+bool doveadm_cmd_param_bool(const struct doveadm_cmd_context *cctx,
+ const char *name, bool *value_r);
+bool doveadm_cmd_param_int64(const struct doveadm_cmd_context *cctx,
+ const char *name, int64_t *value_r);
+bool doveadm_cmd_param_str(const struct doveadm_cmd_context *cctx,
+ const char *name, const char **value_r);
+bool doveadm_cmd_param_ip(const struct doveadm_cmd_context *cctx,
+ const char *name, struct ip_addr *value_r);
+bool doveadm_cmd_param_array(const struct doveadm_cmd_context *cctx,
+ const char *name, const char *const **value_r);
+bool doveadm_cmd_param_istream(const struct doveadm_cmd_context *cctx,
+ const char *name, struct istream **value_r);
+
+void doveadm_cmd_params_clean(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv);
+void doveadm_cmd_params_null_terminate_arrays(ARRAY_TYPE(doveadm_cmd_param_arr_t) *pargv);
+
+extern struct doveadm_cmd_ver2 doveadm_cmd_dump;
+extern struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_service_status_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_process_status_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_stop_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_reload_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_stats_dump_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_stats_add_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_stats_remove_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_mutf7;
+extern struct doveadm_cmd_ver2 doveadm_cmd_oldstats_reset_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_oldstats_dump_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_oldstats_top_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_penalty_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_pw;
+extern struct doveadm_cmd_ver2 doveadm_cmd_kick_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_who_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_sis_deduplicate;
+extern struct doveadm_cmd_ver2 doveadm_cmd_sis_find;
+extern struct doveadm_cmd_ver2 doveadm_cmd_zlibconnect;
+
+#endif
diff --git a/src/doveadm/doveadm-dict.c b/src/doveadm/doveadm-dict.c
new file mode 100644
index 0000000..12b901a
--- /dev/null
+++ b/src/doveadm/doveadm-dict.c
@@ -0,0 +1,329 @@
+/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dict.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+static int
+cmd_dict_init_full(struct doveadm_cmd_context *cctx,
+ doveadm_command_ver2_t *cmd ATTR_UNUSED, enum dict_iterate_flags *iter_flags,
+ struct dict **dict_r, struct dict_op_settings *dopset_r)
+{
+ struct dict_settings dict_set;
+ struct dict *dict;
+ bool set = FALSE;
+ const char *dict_uri, *error, *key, *username = "";
+ i_zero(dopset_r);
+
+ if (doveadm_cmd_param_bool(cctx, "exact", &set) && set)
+ *iter_flags |= DICT_ITERATE_FLAG_EXACT_KEY;
+ if (doveadm_cmd_param_bool(cctx, "recurse", &set) && set)
+ *iter_flags |= DICT_ITERATE_FLAG_RECURSE;
+ if (doveadm_cmd_param_bool(cctx, "no-value", &set) && set)
+ *iter_flags |= DICT_ITERATE_FLAG_NO_VALUE;
+ (void)doveadm_cmd_param_str(cctx, "user", &username);
+ dopset_r->username = username;
+
+ if (!doveadm_cmd_param_str(cctx, "dict-uri", &dict_uri)) {
+ i_error("dictionary URI must be specified");
+ doveadm_exit_code = EX_USAGE;
+ return -1;
+ }
+
+ if (!doveadm_cmd_param_str(cctx, "prefix", &key) &&
+ !doveadm_cmd_param_str(cctx, "key", &key))
+ key = "";
+
+ if (!str_begins(key, DICT_PATH_PRIVATE) &&
+ !str_begins(key, DICT_PATH_SHARED)) {
+ i_error("Key must begin with '"DICT_PATH_PRIVATE
+ "' or '"DICT_PATH_SHARED"': %s", key);
+ doveadm_exit_code = EX_USAGE;
+ return -1;
+ }
+ if (username[0] == '\0' &&
+ str_begins(key, DICT_PATH_PRIVATE)) {
+ i_error("-u must be specified for "DICT_PATH_PRIVATE" keys");
+ doveadm_exit_code = EX_USAGE;
+ return -1;
+ }
+
+ dict_drivers_register_builtin();
+ i_zero(&dict_set);
+ dict_set.base_dir = doveadm_settings->base_dir;
+ if (dict_init(dict_uri, &dict_set, &dict, &error) < 0) {
+ i_error("dict_init(%s) failed: %s", dict_uri, error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ return -1;
+ }
+ *dict_r = dict;
+ return 0;
+}
+
+static int
+cmd_dict_init(struct doveadm_cmd_context *cctx,
+ doveadm_command_ver2_t *cmd, struct dict **dict_r,
+ struct dict_op_settings *set_r)
+{
+ enum dict_iterate_flags iter_flags = 0;
+ return cmd_dict_init_full(cctx, cmd, &iter_flags, dict_r, set_r);
+}
+
+struct doveadm_dict_ctx {
+ pool_t pool;
+ int ret;
+ const char *const *values;
+ const char *error;
+};
+
+static void dict_lookup_callback(const struct dict_lookup_result *result,
+ struct doveadm_dict_ctx *ctx)
+{
+ ctx->ret = result->ret;
+ ctx->values = result->values == NULL ? NULL :
+ p_strarray_dup(ctx->pool, result->values);
+ ctx->error = p_strdup(ctx->pool, result->error);
+}
+
+static void cmd_dict_get(struct doveadm_cmd_context *cctx)
+{
+ struct doveadm_dict_ctx ctx;
+ struct dict *dict;
+ const char *key;
+ struct dict_op_settings set;
+
+ if (!doveadm_cmd_param_str(cctx, "key", &key)) {
+ i_error("dict-get: Missing key");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ if (cmd_dict_init(cctx, cmd_dict_get, &dict, &set) < 0)
+ return;
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("value", "", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+
+ i_zero(&ctx);
+ ctx.pool = pool_alloconly_create("doveadm dict lookup", 512);
+ ctx.ret = -2;
+ dict_lookup_async(dict, &set, key, dict_lookup_callback, &ctx);
+ while (ctx.ret == -2)
+ dict_wait(dict);
+ if (ctx.ret < 0) {
+ i_error("dict_lookup(%s) failed: %s", key, ctx.error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (ctx.ret == 0) {
+ i_error("%s doesn't exist", key);
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else {
+ unsigned int i, values_count = str_array_length(ctx.values);
+
+ for (i = 1; i < values_count; i++)
+ doveadm_print_header("value", "", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ for (i = 0; i < values_count; i++)
+ doveadm_print(ctx.values[i]);
+ }
+ pool_unref(&ctx.pool);
+ dict_deinit(&dict);
+}
+
+static void cmd_dict_set(struct doveadm_cmd_context *cctx)
+{
+ struct dict *dict;
+ struct dict_transaction_context *trans;
+ const char *error;
+ const char *key, *value = "";
+ struct dict_op_settings set;
+
+ if (!doveadm_cmd_param_str(cctx, "key", &key) ||
+ !doveadm_cmd_param_str(cctx, "value", &value)) {
+ i_error("dict set: Missing parameters");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ if (cmd_dict_init(cctx, cmd_dict_set, &dict, &set) < 0)
+ return;
+
+ trans = dict_transaction_begin(dict, &set);
+ dict_set(trans, key, value);
+ if (dict_transaction_commit(&trans, &error) <= 0) {
+ i_error("dict_transaction_commit() failed: %s", error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ dict_deinit(&dict);
+}
+
+static void cmd_dict_unset(struct doveadm_cmd_context *cctx)
+{
+ struct dict *dict;
+ struct dict_transaction_context *trans;
+ const char *error;
+ const char *key;
+ struct dict_op_settings set;
+
+ if (!doveadm_cmd_param_str(cctx, "key", &key)) {
+ i_error("dict unset: Missing key");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ if (cmd_dict_init(cctx, cmd_dict_unset, &dict, &set) < 0)
+ return;
+
+ trans = dict_transaction_begin(dict, &set);
+ dict_unset(trans, key);
+ if (dict_transaction_commit(&trans, &error) <= 0) {
+ i_error("dict_transaction_commit() failed: %s", error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ dict_deinit(&dict);
+}
+
+static void cmd_dict_inc(struct doveadm_cmd_context *cctx)
+{
+ struct dict *dict;
+ struct dict_transaction_context *trans;
+ const char *error;
+ const char *key;
+ int64_t diff;
+ int ret;
+ struct dict_op_settings set;
+
+ if (!doveadm_cmd_param_str(cctx, "key", &key) ||
+ !doveadm_cmd_param_int64(cctx, "difference", &diff)) {
+ i_error("dict-inc: Missing parameters");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ if (cmd_dict_init(cctx, cmd_dict_inc, &dict, &set) < 0)
+ return;
+
+ trans = dict_transaction_begin(dict, &set);
+ dict_atomic_inc(trans, key, diff);
+ ret = dict_transaction_commit(&trans, &error);
+ if (ret < 0) {
+ i_error("dict_transaction_commit() failed: %s", error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (ret == 0) {
+ i_error("%s doesn't exist", key);
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ }
+ dict_deinit(&dict);
+}
+
+static void cmd_dict_iter(struct doveadm_cmd_context *cctx)
+{
+ struct dict *dict;
+ struct dict_iterate_context *iter;
+ enum dict_iterate_flags iter_flags = 0;
+ const char *prefix, *key, *const *values, *error;
+ bool header_printed = FALSE;
+ struct dict_op_settings set;
+
+ if (!doveadm_cmd_param_str(cctx, "prefix", &prefix)) {
+ i_error("dict-iter: Missing prefix");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ if (cmd_dict_init_full(cctx, cmd_dict_iter, &iter_flags, &dict, &set) < 0)
+ return;
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TAB);
+ doveadm_print_header_simple("key");
+ if ((iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0)
+ doveadm_print_header_simple("value");
+
+ iter = dict_iterate_init(dict, &set, prefix, iter_flags);
+ while (dict_iterate_values(iter, &key, &values)) {
+ unsigned int values_count = str_array_length(values);
+ if (!header_printed) {
+ for (unsigned int i = 1; i < values_count; i++)
+ doveadm_print_header_simple("value");
+ header_printed = TRUE;
+ }
+ doveadm_print(key);
+ if ((iter_flags & DICT_ITERATE_FLAG_NO_VALUE) == 0) {
+ for (unsigned int i = 0; i < values_count; i++)
+ doveadm_print(values[i]);
+ }
+ }
+ if (dict_iterate_deinit(&iter, &error) < 0) {
+ i_error("dict_iterate_deinit(%s) failed: %s", prefix, error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ dict_deinit(&dict);
+}
+
+static struct doveadm_cmd_ver2 doveadm_cmd_dict[] = {
+{
+ .name = "dict get",
+ .cmd = cmd_dict_get,
+ .usage = "[-u <user>] <dict uri> <key>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "dict set",
+ .cmd = cmd_dict_set,
+ .usage = "[-u <user>] <dict uri> <key> <value>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "value", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "dict unset",
+ .cmd = cmd_dict_unset,
+ .usage = "[-u <user>] <dict uri> <key>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "dict inc",
+ .cmd = cmd_dict_inc,
+ .usage = "[-u <user>] <dict uri> <key> <diff>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "difference", CMD_PARAM_INT64, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "dict iter",
+ .cmd = cmd_dict_iter,
+ .usage = "[-u <user>] [-1RV] <dict uri> <prefix>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('1', "exact", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('R', "recurse", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('V', "no-value", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "dict-uri", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "prefix", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+void doveadm_register_dict_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_dict); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_dict[i]);
+}
diff --git a/src/doveadm/doveadm-director.c b/src/doveadm/doveadm-director.c
new file mode 100644
index 0000000..167b091
--- /dev/null
+++ b/src/doveadm/doveadm-director.c
@@ -0,0 +1,1084 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "md5.h"
+#include "hash.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "istream.h"
+#include "write-full.h"
+#include "master-service.h"
+#include "auth-master.h"
+#include "mail-user-hash.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+struct director_context {
+ const char *socket_path;
+ const char *users_path;
+ const char *tag;
+ const char *user;
+ const char *host;
+ const char *ip;
+ const char *port;
+ const char *vhost_count;
+ const char *passdb_field;
+
+ struct istream *users_input;
+ struct istream *input;
+ bool explicit_socket_path;
+ bool hash_map, user_map, force_flush;
+ int64_t max_parallel;
+};
+
+struct user_list {
+ struct user_list *next;
+ const char *name;
+};
+
+HASH_TABLE_DEFINE_TYPE(user_list, void *, struct user_list *);
+
+static void director_cmd_help(const struct doveadm_cmd_ver2 *);
+static int director_get_host(const char *host, struct ip_addr **ips_r,
+ unsigned int *ips_count_r) ATTR_WARN_UNUSED_RESULT;
+static void
+director_send(struct director_context *ctx, const char *data)
+{
+ if (write_full(i_stream_get_fd(ctx->input), data, strlen(data)) < 0)
+ i_fatal("write(%s) failed: %m", ctx->socket_path);
+}
+
+static void director_connect(struct director_context *ctx)
+{
+#define DIRECTOR_HANDSHAKE "VERSION\tdirector-doveadm\t1\t0\n"
+ const char *line;
+ int fd;
+
+ fd = doveadm_connect(ctx->socket_path);
+ net_set_nonblock(fd, FALSE);
+
+ ctx->input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+ director_send(ctx, DIRECTOR_HANDSHAKE);
+
+ alarm(5);
+ line = i_stream_read_next_line(ctx->input);
+ alarm(0);
+ if (line == NULL) {
+ if (ctx->input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->socket_path,
+ i_stream_get_error(ctx->input));
+ } else if (ctx->input->eof) {
+ i_fatal("%s disconnected", ctx->socket_path);
+ } else {
+ i_fatal("read(%s) timed out (is director configured?)",
+ ctx->socket_path);
+ }
+ }
+ if (!version_string_verify(line, "director-doveadm", 1)) {
+ i_fatal_status(EX_PROTOCOL,
+ "%s not a compatible director-doveadm socket",
+ ctx->socket_path);
+ }
+}
+
+static void director_disconnect(struct director_context *ctx)
+{
+ if (ctx->input != NULL) {
+ if (ctx->input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->socket_path,
+ i_stream_get_error(ctx->input));
+ }
+ i_stream_destroy(&ctx->input);
+ }
+}
+
+static struct director_context *
+cmd_director_init(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ ctx = t_new(struct director_context, 1);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx->socket_path)))
+ ctx->socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/director-admin", NULL);
+ else
+ ctx->explicit_socket_path = TRUE;
+ if (!doveadm_cmd_param_bool(cctx, "user-map", &(ctx->user_map)))
+ ctx->user_map = FALSE;
+ if (!doveadm_cmd_param_bool(cctx, "hash-map", &(ctx->hash_map)))
+ ctx->hash_map = FALSE;
+ if (!doveadm_cmd_param_bool(cctx, "force-flush", &(ctx->force_flush)))
+ ctx->force_flush = FALSE;
+ if (!doveadm_cmd_param_istream(cctx, "users-file", &(ctx->users_input)))
+ ctx->users_input = NULL;
+ if (!doveadm_cmd_param_str(cctx, "tag", &(ctx->tag)))
+ ctx->tag = NULL;
+ if (!doveadm_cmd_param_str(cctx, "user", &(ctx->user)))
+ ctx->user = NULL;
+ if (!doveadm_cmd_param_str(cctx, "host", &(ctx->host)))
+ ctx->host = NULL;
+ if (!doveadm_cmd_param_str(cctx, "ip", &(ctx->ip)))
+ ctx->ip = NULL;
+ if (!doveadm_cmd_param_str(cctx, "port", &(ctx->port)))
+ ctx->port = NULL;
+ if (!doveadm_cmd_param_str(cctx, "vhost-count", &(ctx->vhost_count)))
+ ctx->vhost_count = NULL;
+ if (!doveadm_cmd_param_str(cctx, "passdb-field", &(ctx->passdb_field)))
+ ctx->passdb_field = NULL;
+ if (!doveadm_cmd_param_int64(cctx, "max-parallel", &(ctx->max_parallel)))
+ ctx->max_parallel = 0;
+ if (!ctx->user_map)
+ director_connect(ctx);
+ return ctx;
+}
+
+static void director_disconnected(struct director_context *ctx)
+{
+ i_assert(ctx->input->eof);
+ if (ctx->input->stream_errno != 0) {
+ i_error("read(%s) failed: %s", ctx->socket_path,
+ i_stream_get_error(ctx->input));
+ } else {
+ i_error("%s unexpectedly disconnected", ctx->socket_path);
+ }
+ doveadm_exit_code = EX_TEMPFAIL;
+}
+
+static void
+cmd_director_status_user(struct director_context *ctx)
+{
+ const char *line, *const *args;
+ time_t expires;
+
+ director_send(ctx, t_strdup_printf("USER-LOOKUP\t%s\t%s\n", ctx->user,
+ ctx->tag != NULL ? ctx->tag : ""));
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ director_disconnected(ctx);
+ return;
+ }
+
+ args = t_strsplit_tabescaped(line);
+ if (str_array_length(args) != 4 ||
+ str_to_time(args[1], &expires) < 0) {
+ i_error("Invalid reply from director");
+ doveadm_exit_code = EX_PROTOCOL;
+ return;
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+
+ doveadm_print_header_simple("status");
+ doveadm_print_header_simple("expires");
+ doveadm_print_header_simple("hashed");
+ doveadm_print_header_simple("initial-config");
+
+ doveadm_print_formatted_set_format("Current: %{status} (expires %{expires})\n" \
+ "Hashed: %{hashed}\n" \
+ "Initial config: %{initial-config}\n");
+
+ if (args[0][0] != '\0') {
+ doveadm_print(args[0]);
+ doveadm_print(unixdate2str(expires));
+ } else {
+ doveadm_print("n/a");
+ doveadm_print("-1");
+ }
+ doveadm_print(args[2]);
+ doveadm_print(args[3]);
+
+ director_disconnect(ctx);
+}
+
+static void cmd_director_status(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ const char *line, *const *args;
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->user != NULL) {
+ cmd_director_status_user(ctx);
+ return;
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header_simple("mail server ip");
+ doveadm_print_header_simple("tag");
+ doveadm_print_header_simple("vhosts");
+ doveadm_print_header_simple("state");
+ doveadm_print_header("state-changed", "state changed", 0);
+ doveadm_print_header_simple("users");
+
+ director_send(ctx, "HOST-LIST\n");
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ unsigned int arg_count;
+ time_t ts;
+
+ args = t_strsplit_tabescaped(line);
+ arg_count = str_array_length(args);
+ if (arg_count >= 6) {
+ /* ip vhosts users tag updown updown-ts */
+ doveadm_print(args[0]);
+ doveadm_print(args[3]);
+ doveadm_print(args[1]);
+ doveadm_print(args[4][0] == 'D' ? "down" : "up");
+ if (str_to_time(args[5], &ts) < 0 ||
+ ts <= 0)
+ doveadm_print("-");
+ else
+ doveadm_print(unixdate2str(ts));
+ doveadm_print(args[2]);
+ }
+ } T_END;
+ }
+ if (line == NULL)
+ director_disconnected(ctx);
+ director_disconnect(ctx);
+}
+
+static bool user_hash_expand(const char *username, unsigned int *hash_r)
+{
+ const char *error;
+
+ if (!mail_user_hash(username, doveadm_settings->director_username_hash,
+ hash_r, &error)) {
+ i_error("Failed to expand director_username_hash=%s: %s",
+ doveadm_settings->director_username_hash, error);
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static void
+user_list_add(const char *username, pool_t pool,
+ HASH_TABLE_TYPE(user_list) users)
+{
+ struct user_list *user, *old_user;
+ unsigned int user_hash;
+
+ if (!user_hash_expand(username, &user_hash))
+ return;
+
+ user = p_new(pool, struct user_list, 1);
+ user->name = p_strdup(pool, username);
+
+ old_user = hash_table_lookup(users, POINTER_CAST(user_hash));
+ if (old_user != NULL)
+ user->next = old_user;
+ hash_table_update(users, POINTER_CAST(user_hash), user);
+}
+
+static void ATTR_NULL(1)
+userdb_get_user_list(const char *auth_socket_path, pool_t pool,
+ HASH_TABLE_TYPE(user_list) users)
+{
+ struct auth_master_user_list_ctx *ctx;
+ struct auth_master_connection *conn;
+ const char *username;
+
+ if (auth_socket_path == NULL) {
+ auth_socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/auth-userdb", NULL);
+ }
+
+ conn = auth_master_init(auth_socket_path, 0);
+ ctx = auth_master_user_list_init(conn, "", NULL);
+ while ((username = auth_master_user_list_next(ctx)) != NULL)
+ user_list_add(username, pool, users);
+ if (auth_master_user_list_deinit(&ctx) < 0) {
+ i_error("user listing failed");
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ auth_master_deinit(&conn);
+}
+
+static void
+user_file_get_user_list(struct istream *input, pool_t pool,
+ HASH_TABLE_TYPE(user_list) users)
+{
+ const char *username;
+
+ while ((username = i_stream_read_next_line(input)) != NULL)
+ user_list_add(username, pool, users);
+}
+
+static int director_get_host(const char *host, struct ip_addr **ips_r,
+ unsigned int *ips_count_r)
+{
+ struct ip_addr ip;
+ int ret = 0;
+
+ if (net_addr2ip(host, &ip) == 0) {
+ *ips_r = t_new(struct ip_addr, 1);
+ **ips_r = ip;
+ *ips_count_r = 1;
+ } else {
+ ret = net_gethostbyname(host, ips_r, ips_count_r);
+ if (ret != 0) {
+ i_error("gethostname(%s) failed: %s", host,
+ net_gethosterror(ret));
+ doveadm_exit_code = EX_TEMPFAIL;
+ return ret;
+ }
+ }
+
+ return ret;
+}
+
+static bool ip_find(const struct ip_addr *ips, unsigned int ips_count,
+ const struct ip_addr *match_ip)
+{
+ unsigned int i;
+
+ for (i = 0; i < ips_count; i++) {
+ if (net_ip_compare(&ips[i], match_ip))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void cmd_director_map(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ const char *line, *const *args;
+ struct ip_addr *ips, user_ip;
+ pool_t pool;
+ HASH_TABLE_TYPE(user_list) users;
+ struct user_list *user;
+ unsigned int ips_count, user_hash;
+ time_t expires;
+
+ ctx = cmd_director_init(cctx);
+
+ if ((ctx->hash_map || ctx->user_map) && ctx->host == NULL) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ if (ctx->user_map) {
+ /* user -> hash mapping */
+ if (user_hash_expand(ctx->host, &user_hash)) {
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("hash", "hash", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ doveadm_print(t_strdup_printf("%u", user_hash));
+ }
+ director_disconnect(ctx);
+ return;
+ }
+
+ if (ctx->host == NULL || ctx->hash_map)
+ ips_count = 0;
+ else if (director_get_host(ctx->host, &ips, &ips_count) != 0) {
+ director_disconnect(ctx);
+ return;
+ }
+
+ pool = pool_alloconly_create("director map users", 1024*128);
+ hash_table_create_direct(&users, pool, 0);
+ if (ctx->users_input == NULL)
+ userdb_get_user_list(NULL, pool, users);
+ else
+ user_file_get_user_list(ctx->users_input, pool, users);
+
+ if (ctx->hash_map) {
+ /* hash -> usernames mapping */
+ if (str_to_uint(ctx->host, &user_hash) < 0)
+ i_fatal("Invalid username hash: %s", ctx->host);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ user = hash_table_lookup(users, POINTER_CAST(user_hash));
+ for (; user != NULL; user = user->next)
+ doveadm_print(user->name);
+ goto deinit;
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("user", "user", DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ doveadm_print_header_simple("hash");
+ doveadm_print_header_simple("mail server ip");
+ doveadm_print_header_simple("expire time");
+
+ if (ips_count != 1)
+ director_send(ctx, "USER-LIST\n");
+ else {
+ director_send(ctx, t_strdup_printf(
+ "USER-LIST\t%s\n", net_ip2addr(&ips[0])));
+ }
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ args = t_strsplit_tabescaped(line);
+ if (str_array_length(args) < 3 ||
+ str_to_uint(args[0], &user_hash) < 0 ||
+ str_to_time(args[1], &expires) < 0 ||
+ net_addr2ip(args[2], &user_ip) < 0) {
+ i_error("Invalid USER-LIST reply: %s", line);
+ doveadm_exit_code = EX_PROTOCOL;
+ } else if (ips_count == 0 ||
+ ip_find(ips, ips_count, &user_ip)) {
+ user = hash_table_lookup(users,
+ POINTER_CAST(user_hash));
+ if (user == NULL) {
+ doveadm_print("<unknown>");
+ doveadm_print(args[0]);
+ doveadm_print(args[2]);
+ doveadm_print(unixdate2str(expires));
+ }
+ for (; user != NULL; user = user->next) {
+ doveadm_print(user->name);
+ doveadm_print(args[0]);
+ doveadm_print(args[2]);
+ doveadm_print(unixdate2str(expires));
+ }
+ }
+ } T_END;
+ }
+ if (line == NULL)
+ director_disconnected(ctx);
+deinit:
+ director_disconnect(ctx);
+ hash_table_destroy(&users);
+ pool_unref(&pool);
+}
+
+static void
+cmd_director_add_or_update(struct doveadm_cmd_context *cctx, bool update)
+{
+ const char *director_cmd = update ? "HOST-UPDATE" : "HOST-SET";
+ struct director_context *ctx;
+ struct ip_addr *ips;
+ unsigned int i, ips_count, vhost_count = UINT_MAX;
+ const char *line, *host;
+ string_t *cmd;
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->tag != NULL && ctx->tag[0] == '\0')
+ ctx->tag = NULL;
+ if (ctx->host == NULL) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+ if (ctx->vhost_count != NULL) {
+ if (str_to_uint(ctx->vhost_count, &vhost_count) < 0) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+ } else if (update) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+ if (str_to_uint(ctx->host, &i) == 0) {
+ /* host is a number. this would translate to an IP address,
+ which is probably a mistake. */
+ i_error("Invalid host '%s'", ctx->host);
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ host = ctx->host;
+ if (ctx->tag == NULL) {
+ ctx->tag = strchr(ctx->host, '@');
+ if (ctx->tag != NULL)
+ host = t_strdup_until(ctx->host, ctx->tag++);
+ }
+ if (director_get_host(host, &ips, &ips_count) != 0) {
+ director_disconnect(ctx);
+ return;
+ }
+ cmd = t_str_new(128);
+ for (i = 0; i < ips_count; i++) {
+ str_truncate(cmd, 0);
+ str_printfa(cmd, "%s\t%s", director_cmd, net_ip2addr(&ips[i]));
+ if (ctx->tag != NULL)
+ str_printfa(cmd, "@%s", ctx->tag);
+ if (vhost_count != UINT_MAX)
+ str_printfa(cmd, "\t%u", vhost_count);
+ str_append_c(cmd, '\n');
+ director_send(ctx, str_c(cmd));
+ }
+ for (i = 0; i < ips_count; i++) {
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL)
+ director_disconnected(ctx);
+ else if (strcmp(line, "OK") != 0) {
+ i_error("%s: %s", net_ip2addr(&ips[i]),
+ strcmp(line, "NOTFOUND") == 0 ?
+ "doesn't exist" : line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (doveadm_verbose) {
+ i_info("%s: OK", net_ip2addr(&ips[i]));
+ }
+ }
+ director_disconnect(ctx);
+}
+
+static void cmd_director_add(struct doveadm_cmd_context *cctx)
+{
+ cmd_director_add_or_update(cctx, FALSE);
+}
+
+static void cmd_director_update(struct doveadm_cmd_context *cctx)
+{
+ cmd_director_add_or_update(cctx, TRUE);
+}
+
+static void
+cmd_director_ipcmd(const char *cmd_name, const char *success_result,
+ struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ struct ip_addr *ips;
+ unsigned int i, ips_count;
+ const char *host, *line;
+
+ ctx = cmd_director_init(cctx);
+ host = ctx->host;
+ if (host == NULL) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ if (director_get_host(host, &ips, &ips_count) != 0) {
+ director_disconnect(ctx);
+ return;
+ }
+ for (i = 0; i < ips_count; i++) {
+ director_send(ctx, t_strdup_printf(
+ "%s\t%s\n", cmd_name, net_ip2addr(&ips[i])));
+ }
+ for (i = 0; i < ips_count; i++) {
+ line = i_stream_read_next_line(ctx->input);
+ if (line != NULL && strcmp(line, "NOTFOUND") == 0) {
+ i_error("%s: doesn't exist",
+ net_ip2addr(&ips[i]));
+ if (doveadm_exit_code == 0)
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else if (line == NULL) {
+ director_disconnected(ctx);
+ } else if (strcmp(line, "OK") != 0) {
+ i_error("%s: %s", net_ip2addr(&ips[i]), line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (doveadm_verbose) {
+ i_info("%s: %s", net_ip2addr(&ips[i]), success_result);
+ }
+ }
+ director_disconnect(ctx);
+}
+
+static void cmd_director_remove(struct doveadm_cmd_context *cctx)
+{
+ cmd_director_ipcmd("HOST-REMOVE", "removed", cctx);
+}
+
+static void cmd_director_up(struct doveadm_cmd_context *cctx)
+{
+ cmd_director_ipcmd("HOST-UP", "up", cctx);
+}
+
+static void cmd_director_down(struct doveadm_cmd_context *cctx)
+{
+ cmd_director_ipcmd("HOST-DOWN", "down", cctx);
+}
+
+static void cmd_director_move(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ struct ip_addr *ips;
+ unsigned int ips_count, user_hash;
+ const char *line, *ip_str;
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->user == NULL || ctx->host == NULL) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ if (!user_hash_expand(ctx->user, &user_hash) ||
+ director_get_host(ctx->host, &ips, &ips_count) != 0) {
+ director_disconnect(ctx);
+ return;
+ }
+ ip_str = net_ip2addr(&ips[0]);
+ director_send(ctx, t_strdup_printf(
+ "USER-MOVE\t%u\t%s\n", user_hash, ip_str));
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ director_disconnected(ctx);
+ } else if (strcmp(line, "OK") == 0) {
+ if (doveadm_verbose)
+ i_info("User hash %u moved to %s", user_hash, ip_str);
+ } else if (strcmp(line, "NOTFOUND") == 0) {
+ i_error("Host '%s' doesn't exist", ip_str);
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else if (strcmp(line, "TRYAGAIN") == 0) {
+ i_error("User is already being moved, "
+ "wait a while for it to be finished");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else {
+ i_error("failed: %s", line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ director_disconnect(ctx);
+}
+
+static void cmd_director_kick(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ const char *line;
+ string_t *cmd = t_str_new(64);
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->user == NULL) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ if (ctx->passdb_field == NULL) {
+ str_append(cmd, "USER-KICK\t");
+ str_append_tabescaped(cmd, ctx->user);
+ str_append_c(cmd, '\n');
+ } else {
+ str_append(cmd, "USER-KICK-ALT\t");
+ str_append_tabescaped(cmd, ctx->passdb_field);
+ str_append_c(cmd, '\t');
+ str_append_tabescaped(cmd, ctx->user);
+ str_append_c(cmd, '\n');
+ }
+ director_send(ctx, str_c(cmd));
+
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ director_disconnected(ctx);
+ } else if (strcmp(line, "OK") == 0) {
+ if (doveadm_verbose)
+ i_info("User %s kicked", ctx->user);
+ } else {
+ i_error("failed: %s", line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ director_disconnect(ctx);
+}
+
+static void cmd_director_flush_all(struct director_context *ctx)
+{
+ const char *line;
+
+ if (ctx->force_flush)
+ line = "HOST-FLUSH\n";
+ else if (ctx->max_parallel > 0) {
+ line = t_strdup_printf("HOST-RESET-USERS\t\t%lld\n",
+ (long long)ctx->max_parallel);
+ } else {
+ line = "HOST-RESET-USERS\n";
+ }
+ director_send(ctx, line);
+
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ director_disconnected(ctx);
+ } else if (strcmp(line, "OK") != 0) {
+ i_error("failed: %s", line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (doveadm_verbose)
+ i_info("flushed");
+ director_disconnect(ctx);
+}
+
+static void cmd_director_flush(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ struct ip_addr *ips;
+ unsigned int i, ips_count;
+ struct ip_addr ip;
+ const char *line;
+ string_t *cmd;
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->host == NULL) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ if (strcmp(ctx->host, "all") == 0) {
+ cmd_director_flush_all(ctx);
+ return;
+ }
+ if (net_addr2ip(ctx->host, &ip) == 0) {
+ ips = &ip;
+ ips_count = 1;
+ } else if (director_get_host(ctx->host, &ips, &ips_count) != 0) {
+ director_disconnect(ctx);
+ return;
+ }
+
+ cmd = t_str_new(64);
+ for (i = 0; i < ips_count; i++) {
+ ip = ips[i];
+ str_truncate(cmd, 0);
+ if (ctx->force_flush)
+ str_printfa(cmd, "HOST-FLUSH\t%s\n", net_ip2addr(&ip));
+ else {
+ str_printfa(cmd, "HOST-RESET-USERS\t%s", net_ip2addr(&ip));
+ if (ctx->max_parallel > 0) {
+ str_printfa(cmd, "\t%lld",
+ (long long)ctx->max_parallel);
+ }
+ str_append_c(cmd, '\n');
+ }
+ director_send(ctx, str_c(cmd));
+ }
+ for (i = 0; i < ips_count; i++) {
+ line = i_stream_read_next_line(ctx->input);
+ if (line != NULL && strcmp(line, "NOTFOUND") == 0) {
+ i_warning("%s: doesn't exist",
+ net_ip2addr(&ips[i]));
+ if (doveadm_exit_code == 0)
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else if (line == NULL) {
+ director_disconnected(ctx);
+ } else if (strcmp(line, "OK") != 0) {
+ i_warning("%s: %s", net_ip2addr(&ips[i]), line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (doveadm_verbose) {
+ i_info("%s: flushed", net_ip2addr(&ips[i]));
+ }
+ }
+ director_disconnect(ctx);
+}
+
+static void cmd_director_dump(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ const char *line, *const *args;
+
+ ctx = cmd_director_init(cctx);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ if (ctx->explicit_socket_path)
+ doveadm_print_formatted_set_format("doveadm director %{command} -a %{socket-path} %{host} %{vhost_count}\n");
+ else
+ doveadm_print_formatted_set_format("doveadm director %{command} %{host} %{vhost_count}\n");
+
+ doveadm_print_header_simple("command");
+ doveadm_print_header_simple("socket-path");
+ doveadm_print_header_simple("host");
+ doveadm_print_header_simple("vhost_count");
+
+ director_send(ctx, "HOST-LIST\n");
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ args = t_strsplit_tabescaped(line);
+ if (str_array_length(args) >= 2) {
+ const char *host = args[0];
+ const char *tag = args[3];
+ /* this is guaranteed to be at least NULL */
+ if (tag != NULL &&
+ *tag != '\0')
+ host = t_strdup_printf("%s@%s", host,
+ tag);
+ doveadm_print("add");
+ doveadm_print(ctx->socket_path);
+ doveadm_print(host);
+ doveadm_print(args[1]);
+ }
+ } T_END;
+ }
+
+ director_send(ctx, "HOST-LIST-REMOVED\n");
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ doveadm_print("remove");
+ doveadm_print(ctx->socket_path);
+ doveadm_print(line);
+ doveadm_print("");
+ }
+ if (line == NULL)
+ director_disconnected(ctx);
+ director_disconnect(ctx);
+}
+
+
+static void director_read_ok_reply(struct director_context *ctx)
+{
+ const char *line;
+
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ director_disconnected(ctx);
+ } else if (strcmp(line, "NOTFOUND") == 0) {
+ i_error("Not found");
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else if (strcmp(line, "OK") != 0) {
+ i_error("Failed: %s", line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+}
+
+static void cmd_director_ring_add(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ struct ip_addr ip;
+ in_port_t port = 0;
+ string_t *str = t_str_new(64);
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->ip == NULL ||
+ net_addr2ip(ctx->ip, &ip) < 0 ||
+ (ctx->port != 0 && net_str2port(ctx->port, &port) < 0)) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ str_printfa(str, "DIRECTOR-ADD\t%s", net_ip2addr(&ip));
+ if (port != 0)
+ str_printfa(str, "\t%u", port);
+ str_append_c(str, '\n');
+ director_send(ctx, str_c(str));
+ director_read_ok_reply(ctx);
+ director_disconnect(ctx);
+}
+
+static void cmd_director_ring_remove(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ struct ip_addr ip;
+ string_t *str = t_str_new(64);
+ in_port_t port = 0;
+
+ ctx = cmd_director_init(cctx);
+ if (ctx->ip == NULL ||
+ net_addr2ip(ctx->ip, &ip) < 0 ||
+ (ctx->port != NULL && net_str2port(ctx->port, &port) < 0)) {
+ director_cmd_help(cctx->cmd);
+ return;
+ }
+
+ str_printfa(str, "DIRECTOR-REMOVE\t%s", net_ip2addr(&ip));
+ if (port != 0)
+ str_printfa(str, "\t%u", port);
+ str_append_c(str, '\n');
+ director_send(ctx, str_c(str));
+ director_read_ok_reply(ctx);
+ director_disconnect(ctx);
+}
+
+static void cmd_director_ring_status(struct doveadm_cmd_context *cctx)
+{
+ struct director_context *ctx;
+ const char *line, *const *args;
+
+ ctx = cmd_director_init(cctx);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header_simple("director ip");
+ doveadm_print_header_simple("port");
+ doveadm_print_header_simple("type");
+ doveadm_print_header_simple("last failed");
+ doveadm_print_header_simple("status");
+ doveadm_print_header_simple("ping ms");
+ doveadm_print_header_simple("input");
+ doveadm_print_header_simple("output");
+ doveadm_print_header_simple("buffered");
+ doveadm_print_header_simple("buffered peak");
+ doveadm_print_header_simple("last read");
+ doveadm_print_header_simple("last write");
+
+ director_send(ctx, "DIRECTOR-LIST\n");
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ unsigned int i;
+ time_t ts;
+
+ args = t_strsplit_tabescaped(line);
+ for (i = 0; i < 12 && args[i] != NULL; i++) {
+ if ((i == 3 || i == 10 || i == 11) &&
+ str_to_time(args[i], &ts) == 0) {
+ if (ts == 0)
+ doveadm_print("never");
+ else
+ doveadm_print(unixdate2str(ts));
+ } else {
+ doveadm_print(args[i]);
+ }
+ }
+ for (; i < 12; i++)
+ doveadm_print("-");
+ } T_END;
+ }
+ if (line == NULL)
+ director_disconnected(ctx);
+ director_disconnect(ctx);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_director[] = {
+{
+ .name = "director status",
+ .cmd = cmd_director_status,
+ .usage = "[-a <director socket path>] [<user>] [<tag>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "tag", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director map",
+ .cmd = cmd_director_map,
+ .usage = "[-a <director socket path>] [-f <users file>] [-h | -u] [<host>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('f', "users-file", CMD_PARAM_ISTREAM, 0)
+DOVEADM_CMD_PARAM('h', "hash-map", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('u', "user-map", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director add",
+ .cmd = cmd_director_add,
+ .usage = "[-a <director socket path>] [-t <tag>] <host> [<vhost count>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('t', "tag", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director update",
+ .cmd = cmd_director_update,
+ .usage = "[-a <director socket path>] <host> <vhost count>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "vhost-count", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director up",
+ .cmd = cmd_director_up,
+ .usage = "[-a <director socket path>] <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director down",
+ .cmd = cmd_director_down,
+ .usage = "[-a <director socket path>] <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director remove",
+ .cmd = cmd_director_remove,
+ .usage = "[-a <director socket path>] <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director move",
+ .cmd = cmd_director_move,
+ .usage = "[-a <director socket path>] <user> <host>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director kick",
+ .cmd = cmd_director_kick,
+ .usage = "[-a <director socket path>] [-f <passdb field>] <user>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('f', "passdb-field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director flush",
+ .cmd = cmd_director_flush,
+ .usage = "[-a <director socket path>] [-F] [--max-parallel <n>] <host>|all",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('F', "force-flush", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "max-parallel", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director dump",
+ .cmd = cmd_director_dump,
+ .usage = "[-a <director socket path>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director ring add",
+ .cmd = cmd_director_ring_add,
+ .usage = "[-a <director socket path>] <ip> [<port>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director ring remove",
+ .cmd = cmd_director_ring_remove,
+ .usage = "[-a <director socket path>] <ip> [<port>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "ip", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "director ring status",
+ .cmd = cmd_director_ring_status,
+ .usage = "[-a <director socket path>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+static void director_cmd_help(const struct doveadm_cmd_ver2 *cmd)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++) {
+ if (doveadm_cmd_director[i].cmd == cmd->cmd)
+ help_ver2(&doveadm_cmd_director[i]);
+ }
+ i_unreached();
+}
+
+void doveadm_register_director_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_director); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_director[i]);
+}
diff --git a/src/doveadm/doveadm-dsync.c b/src/doveadm/doveadm-dsync.c
new file mode 100644
index 0000000..e490def
--- /dev/null
+++ b/src/doveadm/doveadm-dsync.c
@@ -0,0 +1,1562 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "lib-signals.h"
+#include "array.h"
+#include "execv-const.h"
+#include "child-wait.h"
+#include "istream.h"
+#include "ostream.h"
+#include "iostream-ssl.h"
+#include "iostream-rawlog.h"
+#include "write-full.h"
+#include "str.h"
+#include "strescape.h"
+#include "var-expand.h"
+#include "process-title.h"
+#include "settings-parser.h"
+#include "imap-util.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "master-service-ssl-settings.h"
+#include "mail-storage.h"
+#include "mail-storage-service.h"
+#include "mail-user.h"
+#include "mail-namespace.h"
+#include "mailbox-list-private.h"
+#include "doveadm-settings.h"
+#include "doveadm-mail.h"
+#include "doveadm-print.h"
+#include "doveadm-server.h"
+#include "client-connection.h"
+#include "server-connection.h"
+#include "dsync/dsync-brain.h"
+#include "dsync/dsync-ibc.h"
+#include "doveadm-dsync.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <sys/wait.h>
+
+#define DSYNC_COMMON_GETOPT_ARGS "+1a:dDEfg:I:l:m:n:NO:Pr:Rs:t:e:T:Ux:"
+#define DSYNC_REMOTE_CMD_EXIT_WAIT_SECS 30
+/* The default vname_escape_char to use unless overridden by BROKENCHAR
+ setting. Note that it's only used for internal dsync names, so it won't end
+ up in permanent storage names. The only requirement for it is that it's not
+ the same as the hierarchy separator. */
+#define DSYNC_LIST_VNAME_ESCAPE_CHAR '%'
+/* In case DSYNC_LIST_VNAME_ESCAPE_CHAR is the hierarchy separator,
+ use this instead. */
+#define DSYNC_LIST_VNAME_ALT_ESCAPE_CHAR '~'
+
+#define DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS (60*10)
+
+enum dsync_run_type {
+ DSYNC_RUN_TYPE_LOCAL,
+ DSYNC_RUN_TYPE_STREAM,
+ DSYNC_RUN_TYPE_CMD
+};
+
+struct dsync_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ enum dsync_brain_sync_type sync_type;
+ const char *mailbox;
+ const char *sync_flags;
+ const char *virtual_all_box;
+ guid_128_t mailbox_guid;
+ const char *state_input, *rawlog_path;
+ ARRAY_TYPE(const_string) exclude_mailboxes;
+ ARRAY_TYPE(const_string) namespace_prefixes;
+ time_t sync_since_timestamp;
+ time_t sync_until_timestamp;
+ uoff_t sync_max_size;
+ unsigned int io_timeout_secs;
+
+ const char *remote_name;
+ const char *local_location;
+ pid_t remote_pid;
+ const char *const *remote_cmd_args;
+ struct child_wait *child_wait;
+ int exit_status;
+
+ int fd_in, fd_out, fd_err;
+ struct io *io_err;
+ struct istream *input, *err_stream;
+ struct ostream *output;
+ size_t input_orig_bufsize, output_orig_bufsize;
+
+ struct ssl_iostream_context *ssl_ctx;
+ struct ssl_iostream *ssl_iostream;
+
+ enum dsync_run_type run_type;
+ struct server_connection *tcp_conn;
+ const char *error;
+
+ unsigned int lock_timeout;
+ unsigned int import_commit_msgs_interval;
+
+ bool lock:1;
+ bool purge_remote:1;
+ bool sync_visible_namespaces:1;
+ bool default_replica_location:1;
+ bool oneway:1;
+ bool backup:1;
+ bool reverse_backup:1;
+ bool remote_user_prefix:1;
+ bool no_mail_sync:1;
+ bool local_location_from_arg:1;
+ bool replicator_notify:1;
+ bool exited:1;
+ bool empty_hdr_workaround:1;
+};
+
+static bool legacy_dsync = FALSE;
+
+static void dsync_cmd_switch_ioloop_to(struct dsync_cmd_context *ctx,
+ struct ioloop *ioloop)
+{
+ if (ctx->input != NULL)
+ i_stream_switch_ioloop_to(ctx->input, ioloop);
+ if (ctx->output != NULL)
+ o_stream_switch_ioloop_to(ctx->output, ioloop);
+}
+
+static void remote_error_input(struct dsync_cmd_context *ctx)
+{
+ const unsigned char *data;
+ size_t size;
+ const char *line;
+
+ switch (i_stream_read(ctx->err_stream)) {
+ case -2:
+ data = i_stream_get_data(ctx->err_stream, &size);
+ fprintf(stderr, "%.*s", (int)size, data);
+ i_stream_skip(ctx->err_stream, size);
+ break;
+ case -1:
+ io_remove(&ctx->io_err);
+ break;
+ default:
+ while ((line = i_stream_next_line(ctx->err_stream)) != NULL)
+ fprintf(stderr, "%s\n", line);
+ break;
+ }
+}
+
+static void
+run_cmd(struct dsync_cmd_context *ctx, const char *const *args)
+{
+ struct doveadm_cmd_context *cctx = ctx->ctx.cctx;
+ int fd_in[2], fd_out[2], fd_err[2];
+
+ ctx->remote_cmd_args = p_strarray_dup(ctx->ctx.pool, args);
+
+ if (pipe(fd_in) < 0 || pipe(fd_out) < 0 || pipe(fd_err) < 0)
+ i_fatal("pipe() failed: %m");
+
+ ctx->remote_pid = fork();
+ switch (ctx->remote_pid) {
+ case -1:
+ i_fatal("fork() failed: %m");
+ case 0:
+ /* child, which will execute the proxy server. stdin/stdout
+ goes to pipes which we'll pass to proxy client. */
+ if (dup2(fd_in[0], STDIN_FILENO) < 0 ||
+ dup2(fd_out[1], STDOUT_FILENO) < 0 ||
+ dup2(fd_err[1], STDERR_FILENO) < 0)
+ i_fatal("dup2() failed: %m");
+
+ i_close_fd(&fd_in[0]);
+ i_close_fd(&fd_in[1]);
+ i_close_fd(&fd_out[0]);
+ i_close_fd(&fd_out[1]);
+ i_close_fd(&fd_err[0]);
+ i_close_fd(&fd_err[1]);
+
+ execvp_const(args[0], args);
+ default:
+ /* parent */
+ break;
+ }
+
+ i_close_fd(&fd_in[0]);
+ i_close_fd(&fd_out[1]);
+ i_close_fd(&fd_err[1]);
+ ctx->fd_in = fd_out[0];
+ ctx->fd_out = fd_in[1];
+ ctx->fd_err = fd_err[0];
+
+ if (ctx->remote_user_prefix) {
+ const char *prefix =
+ t_strdup_printf("%s\n", cctx->username);
+ if (write_full(ctx->fd_out, prefix, strlen(prefix)) < 0)
+ i_fatal("write(remote out) failed: %m");
+ }
+
+ fd_set_nonblock(ctx->fd_err, TRUE);
+ ctx->err_stream = i_stream_create_fd(ctx->fd_err, IO_BLOCK_SIZE);
+ i_stream_set_return_partial_line(ctx->err_stream, TRUE);
+}
+
+static void
+mirror_get_remote_cmd_line(const char *const *argv,
+ const char *const **cmd_args_r)
+{
+ ARRAY_TYPE(const_string) cmd_args;
+ unsigned int i;
+ const char *p;
+
+ i_assert(argv[0] != NULL);
+
+ t_array_init(&cmd_args, 16);
+ for (i = 0; argv[i] != NULL; i++) {
+ p = argv[i];
+ array_push_back(&cmd_args, &p);
+ }
+
+ if (legacy_dsync) {
+ /* we're executing dsync */
+ p = "server";
+ } else if (i > 0 && strcmp(argv[i-1], "dsync-server") == 0) {
+ /* Caller already specified dsync-server in parameters.
+ This is a common misconfiguration, so just allow it. */
+ p = NULL;
+ } else {
+ /* we're executing doveadm */
+ p = "dsync-server";
+ }
+ if (p != NULL)
+ array_push_back(&cmd_args, &p);
+ array_append_zero(&cmd_args);
+ *cmd_args_r = array_front(&cmd_args);
+}
+
+static const char *const *
+get_ssh_cmd_args(const char *host, const char *login, const char *mail_user)
+{
+ static struct var_expand_table static_tab[] = {
+ { 'u', NULL, "user" },
+ { '\0', NULL, "login" },
+ { '\0', NULL, "host" },
+ { '\0', NULL, NULL }
+ };
+ struct var_expand_table *tab;
+ ARRAY_TYPE(const_string) cmd_args;
+ string_t *str, *str2;
+ const char *value, *const *args, *error;
+
+ tab = t_malloc_no0(sizeof(static_tab));
+ memcpy(tab, static_tab, sizeof(static_tab));
+
+ tab[0].value = mail_user;
+ tab[1].value = login;
+ tab[2].value = host;
+
+ t_array_init(&cmd_args, 8);
+ str = t_str_new(128);
+ str2 = t_str_new(128);
+ args = t_strsplit(doveadm_settings->dsync_remote_cmd, " ");
+ for (; *args != NULL; args++) {
+ if (strchr(*args, '%') == NULL)
+ value = *args;
+ else {
+ /* some automation: if parameter's all %variables
+ expand to empty, but the %variable isn't the only
+ text in the parameter, skip it. */
+ str_truncate(str, 0);
+ str_truncate(str2, 0);
+ if (var_expand(str, *args, tab, &error) <= 0 ||
+ var_expand(str2, *args, static_tab, &error) <= 0) {
+ i_error("Failed to expand dsync_remote_cmd=%s: %s",
+ *args, error);
+ }
+ if (strcmp(str_c(str), str_c(str2)) == 0 &&
+ str_len(str) > 0)
+ continue;
+ value = t_strdup(str_c(str));
+ }
+ array_push_back(&cmd_args, &value);
+ }
+ array_append_zero(&cmd_args);
+ return array_front(&cmd_args);
+}
+
+static bool mirror_get_remote_cmd(struct dsync_cmd_context *ctx,
+ const char *user,
+ const char *const **cmd_args_r)
+{
+ const char *p, *host, *const *argv = ctx->ctx.args;
+
+ if (argv[1] != NULL) {
+ /* more than one parameter, so it contains a full command
+ (e.g. ssh host dsync) */
+ mirror_get_remote_cmd_line(argv, cmd_args_r);
+ return TRUE;
+ }
+
+ /* if it begins with /[a-z0-9]+:/, it's a mail location
+ (e.g. mdbox:~/mail) */
+ for (p = argv[0]; *p != '\0'; p++) {
+ if (!i_isalnum(*p)) {
+ if (*p == ':')
+ return FALSE;
+ break;
+ }
+ }
+
+ if (strchr(argv[0], ' ') != NULL || strchr(argv[0], '/') != NULL) {
+ /* a) the whole command is in one string. this is mainly for
+ backwards compatibility.
+ b) script/path */
+ mirror_get_remote_cmd_line(t_strsplit(argv[0], " "),
+ cmd_args_r);
+ return TRUE;
+ }
+
+ /* [user@]host */
+ host = strchr(argv[0], '@');
+ if (host != NULL)
+ user = t_strdup_until(argv[0], host++);
+ else
+ host = argv[0];
+
+ /* we'll assume virtual users, so in user@host it really means not to
+ give ssh a username, but to give dsync -u user parameter. */
+ *cmd_args_r = get_ssh_cmd_args(host, "", user);
+ return TRUE;
+}
+
+static void doveadm_user_init_dsync(struct mail_user *user)
+{
+ struct mail_namespace *ns;
+ char ns_sep = mail_namespaces_get_root_sep(user->namespaces);
+
+ user->dsyncing = TRUE;
+ for (ns = user->namespaces; ns != NULL; ns = ns->next) {
+ struct dsync_mailbox_list *dlist =
+ p_new(ns->list->pool, struct dsync_mailbox_list, 1);
+ MODULE_CONTEXT_SET(ns->list, dsync_mailbox_list_module, dlist);
+
+ if (ns->list->set.vname_escape_char == '\0') {
+ ns->list->set.vname_escape_char =
+ ns_sep != DSYNC_LIST_VNAME_ESCAPE_CHAR ?
+ DSYNC_LIST_VNAME_ESCAPE_CHAR :
+ DSYNC_LIST_VNAME_ALT_ESCAPE_CHAR;
+ } else {
+ dlist->have_orig_escape_char = TRUE;
+ }
+ }
+}
+
+static bool
+paths_are_equal(struct mail_namespace *ns1, struct mail_namespace *ns2,
+ enum mailbox_list_path_type type)
+{
+ const char *path1, *path2;
+
+ return mailbox_list_get_root_path(ns1->list, type, &path1) &&
+ mailbox_list_get_root_path(ns2->list, type, &path2) &&
+ strcmp(path1, path2) == 0;
+}
+
+static int
+get_dsync_verify_namespace(struct dsync_cmd_context *ctx,
+ struct mail_user *user, struct mail_namespace **ns_r)
+{
+ struct mail_namespace *ns;
+
+ /* Use the first -n namespace if given */
+ if (array_count(&ctx->namespace_prefixes) > 0) {
+ const char *prefix =
+ array_idx_elem(&ctx->namespace_prefixes, 0);
+ ns = mail_namespace_find(user->namespaces, prefix);
+ if (ns == NULL) {
+ i_error("Namespace not found: '%s'", prefix);
+ ctx->ctx.exit_code = DOVEADM_EX_NOTFOUND;
+ return -1;
+ }
+ *ns_r = ns;
+ return 0;
+ }
+
+ /* Prefer prefix="" namespace over inbox=yes namespace. Either it uses
+ the global mail_location, which is good, or it might have
+ overwritten location in case of e.g. using subscriptions file for
+ all namespaces. This isn't necessarily obvious, so lets make it
+ clearer by failing if it happens. */
+ if ((user->namespaces->flags & NAMESPACE_FLAG_UNUSABLE) == 0) {
+ *ns_r = user->namespaces;
+ i_assert((*ns_r)->prefix_len == 0);
+ } else {
+ /* fallback to inbox=yes */
+ *ns_r = mail_namespace_find_inbox(user->namespaces);
+ }
+ return 0;
+}
+
+static int
+cmd_dsync_run_local(struct dsync_cmd_context *ctx, struct mail_user *user,
+ struct dsync_brain *brain, struct dsync_ibc *ibc2,
+ const char **changes_during_sync_r,
+ enum mail_error *mail_error_r)
+{
+ struct dsync_brain *brain2;
+ struct mail_user *user2;
+ struct mail_namespace *ns, *ns2;
+ struct setting_parser_context *set_parser;
+ const char *location, *error;
+ bool brain1_running, brain2_running, changed1, changed2;
+ bool remote_only_changes;
+ int ret;
+
+ *mail_error_r = 0;
+
+ if (ctx->local_location_from_arg)
+ location = ctx->ctx.args[0];
+ else {
+ i_assert(ctx->local_location != NULL);
+ location = ctx->local_location;
+ }
+
+ i_set_failure_prefix("dsync(%s): ", user->username);
+
+ /* update mail_location and create another user for the
+ second location. */
+ set_parser = mail_storage_service_user_get_settings_parser(ctx->ctx.cur_service_user);
+ if (settings_parse_keyvalue(set_parser, "mail_location", location) < 0)
+ i_unreached();
+ ret = mail_storage_service_next(ctx->ctx.storage_service,
+ ctx->ctx.cur_service_user,
+ &user2, &error);
+ if (ret < 0) {
+ i_error("Failed to initialize user: %s", error);
+ ctx->ctx.exit_code = ret == -1 ? EX_TEMPFAIL : EX_CONFIG;
+ return -1;
+ }
+ doveadm_user_init_dsync(user2);
+
+ if (get_dsync_verify_namespace(ctx, user, &ns) < 0 ||
+ get_dsync_verify_namespace(ctx, user2, &ns2) < 0)
+ return -1;
+ if (mail_namespace_get_sep(ns) != mail_namespace_get_sep(ns2)) {
+ i_error("Mail locations must use the same hierarchy separator "
+ "(specify namespace prefix=\"%s\" "
+ "{ separator } explicitly)", ns->prefix);
+ ctx->ctx.exit_code = EX_CONFIG;
+ mail_user_deinit(&user2);
+ return -1;
+ }
+ if (paths_are_equal(ns, ns2, MAILBOX_LIST_PATH_TYPE_MAILBOX) &&
+ paths_are_equal(ns, ns2, MAILBOX_LIST_PATH_TYPE_INDEX)) {
+ i_error("Both source and destination mail_location "
+ "points to same directory: %s (namespace "
+ "prefix=\"%s\" { location } is set explicitly?)",
+ mailbox_list_get_root_forced(user->namespaces->list,
+ MAILBOX_LIST_PATH_TYPE_MAILBOX),
+ ns->prefix);
+ ctx->ctx.exit_code = EX_CONFIG;
+ mail_user_deinit(&user2);
+ return -1;
+ }
+
+ brain2 = dsync_brain_slave_init(user2, ibc2, TRUE, "",
+ doveadm_settings->dsync_alt_char[0]);
+ mail_user_unref(&user2);
+
+ brain1_running = brain2_running = TRUE;
+ changed1 = changed2 = TRUE;
+ while (brain1_running || brain2_running) {
+ if (dsync_brain_has_failed(brain) ||
+ dsync_brain_has_failed(brain2))
+ break;
+ if (doveadm_is_killed()) {
+ i_warning("Killed with signal %d", doveadm_killed_signo());
+ break;
+ }
+
+ i_assert(changed1 || changed2);
+ brain1_running = dsync_brain_run(brain, &changed1);
+ brain2_running = dsync_brain_run(brain2, &changed2);
+ }
+ *changes_during_sync_r = t_strdup(dsync_brain_get_unexpected_changes_reason(brain2, &remote_only_changes));
+ if (dsync_brain_deinit(&brain2, mail_error_r) < 0)
+ return -1;
+ return doveadm_is_killed() ? -1 : 0;
+}
+
+static void cmd_dsync_remote_exited(const struct child_wait_status *status,
+ struct dsync_cmd_context *ctx)
+{
+ ctx->exited = TRUE;
+ ctx->exit_status = status->status;
+ io_loop_stop(current_ioloop);
+}
+
+static void cmd_dsync_wait_remote(struct dsync_cmd_context *ctx)
+{
+ struct timeout *to;
+
+ /* wait in ioloop for the remote process to die. while we're running
+ we're also reading and printing all errors that still coming from
+ it. */
+ to = timeout_add(DSYNC_REMOTE_CMD_EXIT_WAIT_SECS*1000,
+ io_loop_stop, current_ioloop);
+ io_loop_run(current_ioloop);
+ timeout_remove(&to);
+
+ /* io_loop_run() deactivates the context - put it back */
+ mail_storage_service_io_activate_user(ctx->ctx.cur_service_user);
+
+ if (!ctx->exited) {
+ i_error("Remote command process isn't dying, killing it");
+ if (kill(ctx->remote_pid, SIGKILL) < 0 && errno != ESRCH) {
+ i_error("kill(%ld, SIGKILL) failed: %m",
+ (long)ctx->remote_pid);
+ }
+ }
+}
+
+static void cmd_dsync_log_remote_status(int status, bool remote_errors_logged,
+ const char *const *remote_cmd_args)
+{
+ if (status == -1)
+ ;
+ else if (WIFSIGNALED(status)) {
+ i_error("Remote command died with signal %d: %s", WTERMSIG(status),
+ t_strarray_join(remote_cmd_args, " "));
+ } else if (!WIFEXITED(status)) {
+ i_error("Remote command failed with status %d: %s", status,
+ t_strarray_join(remote_cmd_args, " "));
+ } else if (WEXITSTATUS(status) == EX_TEMPFAIL && remote_errors_logged) {
+ /* remote most likely already logged the error.
+ don't bother logging another line about it */
+ } else if (WEXITSTATUS(status) != 0) {
+ i_error("Remote command returned error %d: %s", WEXITSTATUS(status),
+ t_strarray_join(remote_cmd_args, " "));
+ }
+}
+
+static void cmd_dsync_run_remote(struct mail_user *user)
+{
+ i_set_failure_prefix("dsync-local(%s)<%s>: ", user->username, user->session_id);
+ io_loop_run(current_ioloop);
+}
+
+static const char *const *
+parse_ssh_location(const char *location, const char *username)
+{
+ const char *host, *login;
+
+ host = strrchr(location, '@');
+ if (host != NULL)
+ login = t_strdup_until(location, host++);
+ else {
+ host = location;
+ login = "";
+ }
+ return get_ssh_cmd_args(host, login, username);
+}
+
+static struct dsync_ibc *
+cmd_dsync_ibc_stream_init(struct dsync_cmd_context *ctx,
+ const char *name, const char *temp_prefix)
+{
+ if (ctx->input == NULL) {
+ fd_set_nonblock(ctx->fd_in, TRUE);
+ fd_set_nonblock(ctx->fd_out, TRUE);
+ ctx->input = i_stream_create_fd(ctx->fd_in, SIZE_MAX);
+ ctx->output = o_stream_create_fd(ctx->fd_out, SIZE_MAX);
+ } else {
+ i_assert(ctx->fd_in == -1 && ctx->fd_out == -1);
+ ctx->fd_in = i_stream_get_fd(ctx->input);
+ ctx->fd_out = o_stream_get_fd(ctx->output);
+ ctx->input_orig_bufsize = i_stream_get_max_buffer_size(ctx->input);
+ ctx->output_orig_bufsize = o_stream_get_max_buffer_size(ctx->output);
+ i_stream_set_max_buffer_size(ctx->input, SIZE_MAX);
+ o_stream_set_max_buffer_size(ctx->output, SIZE_MAX);
+ }
+ if (ctx->rawlog_path != NULL) {
+ iostream_rawlog_create_path(ctx->rawlog_path,
+ &ctx->input, &ctx->output);
+ }
+ return dsync_ibc_init_stream(ctx->input, ctx->output,
+ name, temp_prefix, ctx->io_timeout_secs);
+}
+
+static void
+dsync_replicator_notify(struct dsync_cmd_context *ctx,
+ enum dsync_brain_sync_type sync_type,
+ const char *state_str)
+{
+#define REPLICATOR_HANDSHAKE "VERSION\treplicator-doveadm-client\t1\t0\n"
+ const char *path;
+ string_t *str;
+ int fd;
+
+ path = t_strdup_printf("%s/replicator-doveadm",
+ ctx->ctx.cur_mail_user->set->base_dir);
+ fd = net_connect_unix(path);
+ if (fd == -1) {
+ if (errno == ECONNREFUSED || errno == ENOENT) {
+ /* replicator not running on this server. ignore. */
+ return;
+ }
+ i_error("net_connect_unix(%s) failed: %m", path);
+ return;
+ }
+ fd_set_nonblock(fd, FALSE);
+
+ str = t_str_new(128);
+ str_append(str, REPLICATOR_HANDSHAKE"NOTIFY\t");
+ str_append_tabescaped(str, ctx->ctx.cur_mail_user->username);
+ str_append_c(str, '\t');
+ if (sync_type == DSYNC_BRAIN_SYNC_TYPE_FULL)
+ str_append_c(str, 'f');
+ str_append_c(str, '\t');
+ str_append_tabescaped(str, state_str);
+ str_append_c(str, '\n');
+ if (write_full(fd, str_data(str), str_len(str)) < 0)
+ i_error("write(%s) failed: %m", path);
+ /* we only wanted to notify replicator. we don't care enough about the
+ answer to wait for it. */
+ if (close(fd) < 0)
+ i_error("close(%s) failed: %m", path);
+}
+
+static int
+cmd_dsync_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
+ struct doveadm_cmd_context *cctx = _ctx->cctx;
+ struct dsync_ibc *ibc, *ibc2 = NULL;
+ struct dsync_brain *brain;
+ struct dsync_brain_settings set;
+ struct mail_namespace *ns;
+ const char *const *strp;
+ enum dsync_brain_flags brain_flags;
+ enum mail_error mail_error = 0, mail_error2;
+ bool remote_errors_logged = FALSE;
+ bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI);
+ const char *changes_during_sync, *changes_during_sync2 = NULL;
+ bool remote_only_changes;
+ int ret = 0;
+
+ /* replicator_notify indicates here automated attempt,
+ we still want to allow manual sync/backup */
+ if (!cli && ctx->replicator_notify &&
+ mail_user_plugin_getenv_bool(_ctx->cur_mail_user, "noreplicate")) {
+ ctx->ctx.exit_code = DOVEADM_EX_NOREPLICATE;
+ return -1;
+ }
+
+ i_zero(&set);
+ if (cctx->remote_ip.family != 0) {
+ /* include the doveadm client's IP address in the ps output */
+ set.process_title_prefix = t_strdup_printf(
+ "%s ", net_ip2addr(&cctx->remote_ip));
+ }
+ set.sync_since_timestamp = ctx->sync_since_timestamp;
+ set.sync_until_timestamp = ctx->sync_until_timestamp;
+
+ if (set.sync_since_timestamp > 0 && set.sync_until_timestamp > 0 &&
+ set.sync_since_timestamp > set.sync_until_timestamp) {
+ i_fatal("start date is later than end date");
+ }
+
+ set.sync_max_size = ctx->sync_max_size;
+ set.sync_box = ctx->mailbox;
+ set.sync_flag = ctx->sync_flags;
+ set.virtual_all_box = ctx->virtual_all_box;
+ memcpy(set.sync_box_guid, ctx->mailbox_guid, sizeof(set.sync_box_guid));
+ set.lock_timeout_secs = ctx->lock_timeout;
+ set.import_commit_msgs_interval = ctx->import_commit_msgs_interval;
+ set.state = ctx->state_input;
+ set.mailbox_alt_char = doveadm_settings->dsync_alt_char[0];
+ if (*doveadm_settings->dsync_hashed_headers == '\0') {
+ i_error("dsync_hashed_headers must not be empty");
+ ctx->ctx.exit_code = EX_USAGE;
+ return -1;
+ }
+ set.hashed_headers =
+ t_strsplit_spaces(doveadm_settings->dsync_hashed_headers, " ,");
+ if (array_count(&ctx->exclude_mailboxes) > 0) {
+ /* array is NULL-terminated in init() */
+ set.exclude_mailboxes = array_front(&ctx->exclude_mailboxes);
+ }
+ doveadm_user_init_dsync(user);
+
+ t_array_init(&set.sync_namespaces, array_count(&ctx->namespace_prefixes));
+ array_foreach(&ctx->namespace_prefixes, strp) {
+ ns = mail_namespace_find(user->namespaces, *strp);
+ if (ns == NULL) {
+ i_error("Namespace not found: '%s'", *strp);
+ ctx->ctx.exit_code = EX_USAGE;
+ return -1;
+ }
+ array_push_back(&set.sync_namespaces, &ns);
+ }
+
+ if (ctx->run_type == DSYNC_RUN_TYPE_LOCAL)
+ dsync_ibc_init_pipe(&ibc, &ibc2);
+ else {
+ string_t *temp_prefix = t_str_new(64);
+ mail_user_set_get_temp_prefix(temp_prefix, user->set);
+ ibc = cmd_dsync_ibc_stream_init(ctx, ctx->remote_name,
+ str_c(temp_prefix));
+ if (ctx->fd_err != -1) {
+ ctx->io_err = io_add(ctx->fd_err, IO_READ,
+ remote_error_input, ctx);
+ }
+ }
+
+ brain_flags = DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
+ if (ctx->sync_visible_namespaces)
+ brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES;
+ if (ctx->purge_remote)
+ brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE;
+
+ if (ctx->backup) {
+ if (ctx->reverse_backup)
+ brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV;
+ else
+ brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
+ }
+
+ if (ctx->no_mail_sync)
+ brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC;
+ if (ctx->oneway)
+ brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE;
+ if (ctx->empty_hdr_workaround)
+ brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
+ if (doveadm_debug)
+ brain_flags |= DSYNC_BRAIN_FLAG_DEBUG;
+
+ child_wait_init();
+ brain = dsync_brain_master_init(user, ibc, ctx->sync_type,
+ brain_flags, &set);
+
+ switch (ctx->run_type) {
+ case DSYNC_RUN_TYPE_LOCAL:
+ if (cmd_dsync_run_local(ctx, user, brain, ibc2,
+ &changes_during_sync2, &mail_error) < 0)
+ ret = -1;
+ break;
+ case DSYNC_RUN_TYPE_CMD:
+ ctx->child_wait = child_wait_new_with_pid(ctx->remote_pid,
+ cmd_dsync_remote_exited, ctx);
+ /* fall through */
+ case DSYNC_RUN_TYPE_STREAM:
+ cmd_dsync_run_remote(user);
+ /* io_loop_run() deactivates the context - put it back */
+ mail_storage_service_io_activate_user(ctx->ctx.cur_service_user);
+ break;
+ }
+
+ if (ctx->state_input != NULL) {
+ string_t *state_str = t_str_new(128);
+ dsync_brain_get_state(brain, state_str);
+ doveadm_print(str_c(state_str));
+ }
+
+ changes_during_sync = dsync_brain_get_unexpected_changes_reason(brain, &remote_only_changes);
+ if (changes_during_sync != NULL || changes_during_sync2 != NULL) {
+ /* don't log a warning when running via doveadm server
+ (e.g. called by replicator) */
+ if (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI) {
+ i_warning("Mailbox changes caused a desync. "
+ "You may want to run dsync again: %s",
+ changes_during_sync == NULL ||
+ (remote_only_changes && changes_during_sync2 != NULL) ?
+ changes_during_sync2 : changes_during_sync);
+ }
+ ctx->ctx.exit_code = 2;
+ }
+ if (dsync_brain_deinit(&brain, &mail_error2) < 0)
+ ret = -1;
+ if (ret < 0) {
+ /* tempfail is the default error. prefer to use a non-tempfail
+ if that exists. */
+ if (mail_error2 != 0 &&
+ (mail_error == 0 || mail_error == MAIL_ERROR_TEMP))
+ mail_error = mail_error2;
+ doveadm_mail_failed_error(&ctx->ctx, mail_error);
+ }
+ dsync_ibc_deinit(&ibc);
+ if (ibc2 != NULL)
+ dsync_ibc_deinit(&ibc2);
+ ssl_iostream_destroy(&ctx->ssl_iostream);
+ if (ctx->ssl_ctx != NULL)
+ ssl_iostream_context_unref(&ctx->ssl_ctx);
+ if (ctx->input != NULL) {
+ i_stream_set_max_buffer_size(ctx->input, ctx->input_orig_bufsize);
+ i_stream_unref(&ctx->input);
+ }
+ if (ctx->output != NULL) {
+ o_stream_set_max_buffer_size(ctx->output, ctx->output_orig_bufsize);
+ o_stream_unref(&ctx->output);
+ }
+ if (ctx->fd_in != -1) {
+ if (ctx->fd_out != ctx->fd_in)
+ i_close_fd(&ctx->fd_out);
+ i_close_fd(&ctx->fd_in);
+ }
+ /* print any final errors after the process has died. not closing
+ stdin/stdout before wait() may cause the process to hang, but stderr
+ shouldn't (at least with ssh) and we need stderr to be open to be
+ able to print the final errors */
+ if (ctx->run_type == DSYNC_RUN_TYPE_CMD) {
+ cmd_dsync_wait_remote(ctx);
+ remote_error_input(ctx);
+ remote_errors_logged = ctx->err_stream->v_offset > 0;
+ i_stream_destroy(&ctx->err_stream);
+ cmd_dsync_log_remote_status(ctx->exit_status, remote_errors_logged,
+ ctx->remote_cmd_args);
+ } else {
+ i_assert(ctx->err_stream == NULL);
+ }
+ io_remove(&ctx->io_err);
+ i_close_fd(&ctx->fd_err);
+
+ if (ctx->child_wait != NULL)
+ child_wait_free(&ctx->child_wait);
+ child_wait_deinit();
+ return ret;
+}
+
+static void dsync_connected_callback(int exit_code, const char *error,
+ void *context)
+{
+ struct dsync_cmd_context *ctx = context;
+
+ ctx->ctx.exit_code = exit_code;
+ switch (exit_code) {
+ case 0:
+ server_connection_extract(ctx->tcp_conn, &ctx->input,
+ &ctx->output, &ctx->ssl_iostream);
+ break;
+ case SERVER_EXIT_CODE_DISCONNECTED:
+ ctx->ctx.exit_code = EX_TEMPFAIL;
+ ctx->error = p_strdup_printf(ctx->ctx.pool,
+ "Disconnected from remote: %s", error);
+ break;
+ case EX_NOUSER:
+ ctx->error = "Unknown user in remote";
+ break;
+ case DOVEADM_EX_NOREPLICATE:
+ if (doveadm_debug)
+ i_debug("user is disabled for replication");
+ break;
+ default:
+ ctx->error = p_strdup_printf(ctx->ctx.pool,
+ "Failed to start remote dsync-server command: "
+ "Remote exit_code=%u %s",
+ exit_code, error == NULL ? "" : error);
+ break;
+ }
+ io_loop_stop(current_ioloop);
+}
+
+static int dsync_init_ssl_ctx(struct dsync_cmd_context *ctx,
+ const struct master_service_ssl_settings *ssl_set,
+ const char **error_r)
+{
+ struct ssl_iostream_settings ssl_ctx_set;
+
+ if (ctx->ssl_ctx != NULL)
+ return 0;
+
+ master_service_ssl_client_settings_to_iostream_set(ssl_set,
+ pool_datastack_create(), &ssl_ctx_set);
+ return ssl_iostream_client_context_cache_get(&ssl_ctx_set,
+ &ctx->ssl_ctx, error_r);
+}
+
+static void dsync_server_run_command(struct dsync_cmd_context *ctx,
+ struct server_connection *conn)
+{
+ struct doveadm_cmd_context *cctx = ctx->ctx.cctx;
+ /* <flags> <username> <command> [<args>] */
+ string_t *cmd = t_str_new(256);
+ if (doveadm_debug)
+ str_append_c(cmd, 'D');
+ str_append_c(cmd, '\t');
+ str_append_tabescaped(cmd, cctx->username);
+ str_append(cmd, "\tdsync-server\t-u");
+ str_append_tabescaped(cmd, cctx->username);
+ if (ctx->replicator_notify)
+ str_append(cmd, "\t-U");
+ str_append_c(cmd, '\n');
+
+ ctx->tcp_conn = conn;
+ server_connection_cmd(conn, str_c(cmd), NULL,
+ dsync_connected_callback, ctx);
+ io_loop_run(current_ioloop);
+ ctx->tcp_conn = NULL;
+}
+
+static int
+dsync_connect_tcp(struct dsync_cmd_context *ctx,
+ const struct master_service_ssl_settings *ssl_set,
+ const char *target, bool ssl, const char **error_r)
+{
+ struct doveadm_server *server;
+ struct server_connection *conn;
+ struct ioloop *prev_ioloop, *ioloop;
+ const char *p, *error;
+
+ server = p_new(ctx->ctx.pool, struct doveadm_server, 1);
+ server->name = p_strdup(ctx->ctx.pool, target);
+ p = strrchr(server->name, ':');
+ server->hostname = p == NULL ? server->name :
+ p_strdup_until(ctx->ctx.pool, server->name, p);
+ if (ssl) {
+ if (dsync_init_ssl_ctx(ctx, ssl_set, &error) < 0) {
+ *error_r = t_strdup_printf(
+ "Couldn't initialize SSL context: %s", error);
+ return -1;
+ }
+ server->ssl_flags = PROXY_SSL_FLAG_YES;
+ server->ssl_ctx = ctx->ssl_ctx;
+ }
+ p_array_init(&server->connections, ctx->ctx.pool, 1);
+ p_array_init(&server->queue, ctx->ctx.pool, 1);
+
+ prev_ioloop = current_ioloop;
+ ioloop = io_loop_create();
+ dsync_cmd_switch_ioloop_to(ctx, ioloop);
+
+ if (doveadm_verbose_proctitle) {
+ process_title_set(t_strdup_printf(
+ "[dsync - connecting to %s]", server->name));
+ }
+ if (server_connection_create(server, &conn, &error) < 0) {
+ ctx->error = p_strdup_printf(ctx->ctx.pool,
+ "Couldn't create server connection: %s", error);
+ } else {
+ if (doveadm_verbose_proctitle) {
+ process_title_set(t_strdup_printf(
+ "[dsync - running dsync-server on %s]", server->name));
+ }
+
+ dsync_server_run_command(ctx, conn);
+ }
+
+ if (array_count(&server->connections) > 0)
+ server_connection_destroy(&conn);
+
+ dsync_cmd_switch_ioloop_to(ctx, prev_ioloop);
+ io_loop_destroy(&ioloop);
+
+ if (ctx->error != NULL) {
+ ssl_iostream_context_unref(&ctx->ssl_ctx);
+ *error_r = ctx->error;
+ ctx->error = NULL;
+ return -1;
+ }
+ ctx->run_type = DSYNC_RUN_TYPE_STREAM;
+ return 0;
+}
+
+static int
+parse_location(struct dsync_cmd_context *ctx,
+ const struct master_service_ssl_settings *ssl_set,
+ const char *location,
+ const char *const **remote_cmd_args_r, const char **error_r)
+{
+ struct doveadm_cmd_context *cctx = ctx->ctx.cctx;
+
+ if (str_begins(location, "tcp:")) {
+ /* TCP connection to remote dsync */
+ ctx->remote_name = location+4;
+ return dsync_connect_tcp(ctx, ssl_set, ctx->remote_name,
+ FALSE, error_r);
+ }
+ if (str_begins(location, "tcps:")) {
+ /* TCP+SSL connection to remote dsync */
+ ctx->remote_name = location+5;
+ return dsync_connect_tcp(ctx, ssl_set, ctx->remote_name,
+ TRUE, error_r);
+ }
+
+ if (str_begins(location, "remote:")) {
+ /* this is a remote (ssh) command */
+ ctx->remote_name = location+7;
+ } else if (str_begins(location, "remoteprefix:")) {
+ /* this is a remote (ssh) command with a "user\n"
+ prefix sent before dsync actually starts */
+ ctx->remote_name = location+13;
+ ctx->remote_user_prefix = TRUE;
+ } else {
+ /* local with e.g. maildir:path */
+ ctx->remote_name = NULL;
+ return 0;
+ }
+ *remote_cmd_args_r =
+ parse_ssh_location(ctx->remote_name, cctx->username);
+ return 0;
+}
+
+static int cmd_dsync_prerun(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_storage_service_user *service_user,
+ const char **error_r)
+{
+ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
+ struct doveadm_cmd_context *cctx = _ctx->cctx;
+ const char *const *remote_cmd_args = NULL;
+ const struct mail_user_settings *user_set;
+ const struct master_service_ssl_settings *ssl_set;
+ const char *username = "";
+
+ user_set = mail_storage_service_user_get_set(service_user)[0];
+ ssl_set = mail_storage_service_user_get_ssl_settings(service_user);
+
+ ctx->fd_in = -1;
+ ctx->fd_out = -1;
+ ctx->fd_err = -1;
+ ctx->run_type = DSYNC_RUN_TYPE_LOCAL;
+ ctx->remote_name = "remote";
+
+ if (ctx->default_replica_location) {
+ ctx->local_location =
+ mail_user_set_plugin_getenv(user_set, "mail_replica");
+ if (ctx->local_location == NULL ||
+ *ctx->local_location == '\0') {
+ *error_r = "User has no mail_replica in userdb";
+ _ctx->exit_code = DOVEADM_EX_NOTFOUND;
+ return -1;
+ }
+ } else {
+ /* if we're executing remotely, give -u parameter if we also
+ did a userdb lookup. */
+ if ((_ctx->service_flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) != 0)
+ username = cctx->username;
+
+ if (!mirror_get_remote_cmd(ctx, username, &remote_cmd_args)) {
+ /* it's a mail_location */
+ if (_ctx->args[1] != NULL)
+ doveadm_mail_help_name(_ctx->cmd->name);
+ ctx->local_location = _ctx->args[0];
+ ctx->local_location_from_arg = TRUE;
+ }
+ }
+
+ if (remote_cmd_args == NULL && ctx->local_location != NULL) {
+ if (parse_location(ctx, ssl_set, ctx->local_location,
+ &remote_cmd_args, error_r) < 0)
+ return -1;
+ }
+
+ if (remote_cmd_args != NULL) {
+ /* do this before mail_storage_service_next() in case it
+ drops process privileges */
+ run_cmd(ctx, remote_cmd_args);
+ ctx->run_type = DSYNC_RUN_TYPE_CMD;
+ }
+
+ if (ctx->sync_visible_namespaces &&
+ ctx->run_type == DSYNC_RUN_TYPE_LOCAL)
+ i_fatal("-N parameter requires syncing with remote host");
+ return 0;
+}
+
+static void cmd_dsync_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
+
+ if (ctx->default_replica_location) {
+ if (args[0] != NULL)
+ i_error("Don't give mail location with -d parameter");
+ } else {
+ if (args[0] == NULL)
+ doveadm_mail_help_name(_ctx->cmd->name);
+ }
+ if (array_count(&ctx->exclude_mailboxes) > 0)
+ array_append_zero(&ctx->exclude_mailboxes);
+
+ lib_signals_ignore(SIGHUP, TRUE);
+}
+
+static void cmd_dsync_preinit(struct doveadm_mail_cmd_context *ctx)
+{
+ if ((ctx->service_flags & MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) == 0)
+ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_CHDIR;
+}
+
+static bool
+cmd_mailbox_dsync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
+ const char *str, *error;
+ bool utc;
+
+ switch (c) {
+ case '1':
+ ctx->oneway = TRUE;
+ ctx->backup = TRUE;
+ break;
+ case 'a':
+ ctx->virtual_all_box = optarg;
+ break;
+ case 'd':
+ ctx->default_replica_location = TRUE;
+ break;
+ case 'E':
+ /* dsync wrapper detection flag */
+ legacy_dsync = TRUE;
+ break;
+ case 'f':
+ ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
+ break;
+ case 'O': {
+ const char *str = optarg;
+
+ if (strchr(str, ' ') != NULL)
+ i_fatal("-O parameter doesn't support multiple flags currently");
+ if (str[0] == '-')
+ str++;
+ if (str[0] == '\\' && imap_parse_system_flag(str) == 0)
+ i_fatal("Invalid system flag given for -O parameter: '%s'", str);
+ ctx->sync_flags = optarg;
+ break;
+ }
+ case 'g':
+ if (optarg[0] == '\0')
+ ctx->no_mail_sync = TRUE;
+ else if (guid_128_from_string(optarg, ctx->mailbox_guid) < 0 ||
+ guid_128_is_empty(ctx->mailbox_guid))
+ i_fatal("Invalid -g parameter: %s", optarg);
+ break;
+ case 'l':
+ ctx->lock = TRUE;
+ if (str_to_uint(optarg, &ctx->lock_timeout) < 0)
+ i_fatal("Invalid -l parameter: %s", optarg);
+ break;
+ case 'm':
+ if (optarg[0] == '\0')
+ ctx->no_mail_sync = TRUE;
+ else
+ ctx->mailbox = optarg;
+ break;
+ case 'x':
+ str = optarg;
+ array_push_back(&ctx->exclude_mailboxes, &str);
+ break;
+ case 'n':
+ str = optarg;
+ array_push_back(&ctx->namespace_prefixes, &str);
+ break;
+ case 'N':
+ ctx->sync_visible_namespaces = TRUE;
+ break;
+ case 'P':
+ ctx->purge_remote = TRUE;
+ break;
+ case 'r':
+ ctx->rawlog_path = optarg;
+ break;
+ case 'R':
+ ctx->reverse_backup = TRUE;
+ break;
+ case 's':
+ if (ctx->sync_type != DSYNC_BRAIN_SYNC_TYPE_FULL &&
+ *optarg != '\0')
+ ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
+ ctx->state_input = optarg;
+ break;
+ case 't':
+ if (mail_parse_human_timestamp(optarg, &ctx->sync_since_timestamp, &utc) < 0)
+ i_fatal("Invalid -t parameter: %s", optarg);
+ break;
+ case 'e':
+ if (mail_parse_human_timestamp(optarg, &ctx->sync_until_timestamp, &utc) < 0)
+ i_fatal("Invalid -e parameter: %s", optarg);
+ break;
+ case 'I':
+ if (settings_get_size(optarg, &ctx->sync_max_size, &error) < 0)
+ i_fatal("Invalid -I parameter '%s': %s", optarg, error);
+ break;
+ case 'T':
+ if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0)
+ i_fatal("Invalid -T parameter: %s", optarg);
+ break;
+ case 'U':
+ ctx->replicator_notify = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_dsync_alloc(void)
+{
+ struct dsync_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context);
+ ctx->io_timeout_secs = DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS;
+ ctx->ctx.getopt_args = DSYNC_COMMON_GETOPT_ARGS;
+ ctx->ctx.v.parse_arg = cmd_mailbox_dsync_parse_arg;
+ ctx->ctx.v.preinit = cmd_dsync_preinit;
+ ctx->ctx.v.init = cmd_dsync_init;
+ ctx->ctx.v.prerun = cmd_dsync_prerun;
+ ctx->ctx.v.run = cmd_dsync_run;
+ ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ doveadm_print_header("state", "state",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ p_array_init(&ctx->exclude_mailboxes, ctx->ctx.pool, 4);
+ p_array_init(&ctx->namespace_prefixes, ctx->ctx.pool, 4);
+ if ((doveadm_settings->parsed_features & DSYNC_FEATURE_EMPTY_HDR_WORKAROUND) != 0)
+ ctx->empty_hdr_workaround = TRUE;
+ ctx->import_commit_msgs_interval = doveadm_settings->dsync_commit_msgs_interval;
+ return &ctx->ctx;
+}
+
+static struct doveadm_mail_cmd_context *cmd_dsync_backup_alloc(void)
+{
+ struct doveadm_mail_cmd_context *_ctx;
+ struct dsync_cmd_context *ctx;
+
+ _ctx = cmd_dsync_alloc();
+ ctx = (struct dsync_cmd_context *)_ctx;
+ ctx->backup = TRUE;
+ return _ctx;
+}
+
+static int
+cmd_dsync_server_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
+ struct doveadm_cmd_context *cctx = _ctx->cctx;
+ bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI);
+ struct dsync_ibc *ibc;
+ struct dsync_brain *brain;
+ string_t *temp_prefix, *state_str = NULL;
+ enum dsync_brain_sync_type sync_type;
+ const char *name, *process_title_prefix = "";
+ enum mail_error mail_error;
+
+ if (!cli) {
+ /* replicator_notify indicates here automated attempt,
+ we still want to allow manual sync/backup */
+ if (ctx->replicator_notify &&
+ mail_user_plugin_getenv_bool(_ctx->cur_mail_user, "noreplicate")) {
+ _ctx->exit_code = DOVEADM_EX_NOREPLICATE;
+ return -1;
+ }
+
+ /* doveadm-server connection. start with a success reply.
+ after that follows the regular dsync protocol. */
+ ctx->fd_in = ctx->fd_out = -1;
+ ctx->input = cctx->input;
+ ctx->output = cctx->output;
+ i_stream_ref(ctx->input);
+ o_stream_ref(ctx->output);
+ o_stream_set_finish_also_parent(ctx->output, FALSE);
+ o_stream_nsend(ctx->output, "\n+\n", 3);
+ i_set_failure_prefix("dsync-server(%s): ", user->username);
+ name = i_stream_get_name(ctx->input);
+
+ if (cctx->remote_ip.family != 0) {
+ /* include the doveadm client's IP address in the ps output */
+ process_title_prefix = t_strdup_printf(
+ "%s ", net_ip2addr(&cctx->remote_ip));
+ }
+ } else {
+ /* the log messages go via stderr to the remote dsync,
+ so the names are reversed */
+ i_set_failure_prefix("dsync-remote(%s)<%s>: ", user->username, user->session_id);
+ name = "local";
+ }
+
+ doveadm_user_init_dsync(user);
+
+ temp_prefix = t_str_new(64);
+ mail_user_set_get_temp_prefix(temp_prefix, user->set);
+
+ ibc = cmd_dsync_ibc_stream_init(ctx, name, str_c(temp_prefix));
+ brain = dsync_brain_slave_init(user, ibc, FALSE, process_title_prefix,
+ doveadm_settings->dsync_alt_char[0]);
+
+ io_loop_run(current_ioloop);
+ /* io_loop_run() deactivates the context - put it back */
+ mail_storage_service_io_activate_user(ctx->ctx.cur_service_user);
+
+ if (ctx->replicator_notify) {
+ state_str = t_str_new(128);
+ dsync_brain_get_state(brain, state_str);
+ }
+ sync_type = dsync_brain_get_sync_type(brain);
+
+ if (dsync_brain_deinit(&brain, &mail_error) < 0)
+ doveadm_mail_failed_error(_ctx, mail_error);
+ dsync_ibc_deinit(&ibc);
+
+ if (!cli) {
+ /* make sure nothing more is written by the generic doveadm
+ connection code */
+ o_stream_close(cctx->output);
+ }
+ i_stream_unref(&ctx->input);
+ o_stream_unref(&ctx->output);
+
+ if (ctx->replicator_notify && _ctx->exit_code == 0)
+ dsync_replicator_notify(ctx, sync_type, str_c(state_str));
+ return _ctx->exit_code == 0 ? 0 : -1;
+}
+
+static bool
+cmd_mailbox_dsync_server_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct dsync_cmd_context *ctx = (struct dsync_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'E':
+ /* dsync wrapper detection flag */
+ legacy_dsync = TRUE;
+ break;
+ case 'r':
+ ctx->rawlog_path = optarg;
+ break;
+ case 'T':
+ if (str_to_uint(optarg, &ctx->io_timeout_secs) < 0)
+ i_fatal("Invalid -T parameter: %s", optarg);
+ break;
+ case 'U':
+ ctx->replicator_notify = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_dsync_server_alloc(void)
+{
+ struct dsync_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct dsync_cmd_context);
+ ctx->io_timeout_secs = DSYNC_DEFAULT_IO_STREAM_TIMEOUT_SECS;
+ ctx->ctx.getopt_args = "Er:T:U";
+ ctx->ctx.v.parse_arg = cmd_mailbox_dsync_server_parse_arg;
+ ctx->ctx.v.run = cmd_dsync_server_run;
+ ctx->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED;
+
+ ctx->fd_in = STDIN_FILENO;
+ ctx->fd_out = STDOUT_FILENO;
+ return &ctx->ctx;
+}
+
+#define DSYNC_COMMON_PARAMS \
+DOVEADM_CMD_MAIL_COMMON \
+DOVEADM_CMD_PARAM('f', "full-sync", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('P', "purge-remote", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('R', "reverse-sync", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('U', "replicator-notify", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('l', "lock-timeout", CMD_PARAM_INT64, 0) \
+DOVEADM_CMD_PARAM('r', "rawlog", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('m', "mailbox", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('n', "namespace", CMD_PARAM_ARRAY, 0) \
+DOVEADM_CMD_PARAM('N', "all-namespaces", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('x', "exclude-mailbox", CMD_PARAM_ARRAY, 0) \
+DOVEADM_CMD_PARAM('a', "all-mailbox", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('s', "state", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('t', "sync-since-time", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('e', "sync-until-time", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('O', "sync-flags", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('I', "sync-max-size", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('T', "timeout", CMD_PARAM_INT64, 0) \
+DOVEADM_CMD_PARAM('d', "default-destination", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('E', "legacy-dsync", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('\0', "destination", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+
+#define DSYNC_COMMON_USAGE \
+ "[-l <secs>] [-r <rawlog path>] " \
+ "[-m <mailbox>] [-g <mailbox guid>] [-n <namespace> | -N] " \
+ "[-x <exclude>] [-a <all mailbox>] [-s <state>] [-T <secs>] " \
+ "[-t <start date>] [-e <end date>] [-O <sync flag>] [-I <max size>] " \
+ "-d|<dest>"
+
+struct doveadm_cmd_ver2 doveadm_cmd_dsync_mirror = {
+ .mail_cmd = cmd_dsync_alloc,
+ .name = "sync",
+ .usage = "[-1fPRU] "DSYNC_COMMON_USAGE,
+ .flags = CMD_FLAG_NO_UNORDERED_OPTIONS,
+DOVEADM_CMD_PARAMS_START
+DSYNC_COMMON_PARAMS
+DOVEADM_CMD_PARAM('1', "oneway-sync", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAMS_END
+};
+struct doveadm_cmd_ver2 doveadm_cmd_dsync_backup = {
+ .mail_cmd = cmd_dsync_backup_alloc,
+ .name = "backup",
+ .usage = "[-fPRU] "DSYNC_COMMON_USAGE,
+ .flags = CMD_FLAG_NO_UNORDERED_OPTIONS,
+DOVEADM_CMD_PARAMS_START
+DSYNC_COMMON_PARAMS
+DOVEADM_CMD_PARAMS_END
+};
+struct doveadm_cmd_ver2 doveadm_cmd_dsync_server = {
+ .mail_cmd = cmd_dsync_server_alloc,
+ .name = "dsync-server",
+ .usage = "[-E] [-r <rawlog path>] [-T <timeout secs>] [-U]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('E', "legacy-dsync", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('r', "rawlog", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('T', "timeout", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('U', "replicator-notify", CMD_PARAM_BOOL, 0)
+/* previously dsync-server could have been added twice to the parameters */
+DOVEADM_CMD_PARAM('\0', "ignore-arg", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+void doveadm_dsync_main(int *_argc, char **_argv[])
+{
+ int argc = *_argc;
+ const char *getopt_str;
+ char **argv = *_argv;
+ char **new_argv, *mailbox = NULL, *alt_char = NULL, *username = NULL;
+ char *p, *dup, new_flags[6];
+ int max_argc, src, dest, i, j;
+ bool flag_f = FALSE, flag_R = FALSE, flag_m, flag_u, flag_C, has_arg;
+ bool dsync_server = FALSE;
+
+ p = strrchr(argv[0], '/');
+ if (p == NULL) p = argv[0];
+ if (strstr(p, "dsync") == NULL)
+ return;
+
+ /* @UNSAFE: this is called when the "doveadm" binary is called as
+ "dsync" (for backwards compatibility) */
+ max_argc = argc + 7;
+ new_argv = t_new(char *, max_argc);
+ new_argv[0] = argv[0];
+ dest = 1;
+ getopt_str = master_service_getopt_string();
+
+ /* add global doveadm flags */
+ for (src = 1; src < argc; src++) {
+ if (argv[src][0] != '-')
+ break;
+
+ flag_m = FALSE; flag_C = FALSE; has_arg = FALSE; flag_u = FALSE;
+ dup = t_strdup_noconst(argv[src]);
+ for (i = j = 1; argv[src][i] != '\0'; i++) {
+ switch (argv[src][i]) {
+ case 'C':
+ flag_C = TRUE;
+ break;
+ case 'f':
+ flag_f = TRUE;
+ break;
+ case 'R':
+ flag_R = TRUE;
+ break;
+ case 'm':
+ flag_m = TRUE;
+ break;
+ case 'u':
+ flag_u = TRUE;
+ break;
+ default:
+ p = strchr(getopt_str, argv[src][i]);
+ if (p != NULL && p[1] == ':')
+ has_arg = TRUE;
+ dup[j++] = argv[src][i];
+ break;
+ }
+ }
+ if (j > 1) {
+ dup[j++] = '\0';
+ new_argv[dest++] = dup;
+ if (has_arg && src+1 < argc)
+ new_argv[dest++] = argv[++src];
+ }
+ if (flag_m) {
+ if (src+1 == argc)
+ i_fatal("-m missing parameter");
+ mailbox = argv[++src];
+ }
+ if (flag_u) {
+ if (src+1 == argc)
+ i_fatal("-u missing parameter");
+ username = argv[++src];
+ }
+ if (flag_C) {
+ if (src+1 == argc)
+ i_fatal("-C missing parameter");
+ alt_char = argv[++src];
+ }
+ }
+ if (alt_char != NULL) {
+ new_argv[dest++] = "-o";
+ new_argv[dest++] =
+ p_strconcat(pool_datastack_create(),
+ "dsync_alt_char=", alt_char, NULL);
+ }
+
+ /* mirror|backup|server */
+ if (src == argc)
+ i_fatal("Missing mirror or backup parameter");
+ if (strcmp(argv[src], "sync") == 0 ||
+ strcmp(argv[src], "dsync-server") == 0) {
+ /* we're re-executing dsync due to doveconf.
+ "backup" re-exec detection is later. */
+ return;
+ }
+ if (strcmp(argv[src], "mirror") == 0)
+ new_argv[dest] = "sync";
+ else if (strcmp(argv[src], "backup") == 0)
+ new_argv[dest] = "backup";
+ else if (strcmp(argv[src], "server") == 0) {
+ new_argv[dest] = "dsync-server";
+ dsync_server = TRUE;
+ } else
+ i_fatal("Invalid parameter: %s", argv[src]);
+ src++; dest++;
+
+ if (src < argc && str_begins(argv[src], "-E")) {
+ /* we're re-executing dsync due to doveconf */
+ return;
+ }
+
+ /* dsync flags */
+ new_flags[0] = '-';
+ new_flags[1] = 'E'; i = 2;
+ if (!dsync_server) {
+ if (flag_f)
+ new_flags[i++] = 'f';
+ if (flag_R)
+ new_flags[i++] = 'R';
+ if (mailbox != NULL)
+ new_flags[i++] = 'm';
+ }
+ i_assert((unsigned int)i < sizeof(new_flags));
+ new_flags[i] = '\0';
+
+ if (i > 1) {
+ new_argv[dest++] = strdup(new_flags);
+ if (mailbox != NULL)
+ new_argv[dest++] = mailbox;
+ }
+ if (username != NULL) {
+ new_argv[dest++] = "-u";
+ new_argv[dest++] = username;
+ }
+
+ /* rest of the parameters */
+ for (; src < argc; src++)
+ new_argv[dest++] = argv[src];
+ i_assert(dest < max_argc);
+ new_argv[dest] = NULL;
+
+ legacy_dsync = TRUE;
+ *_argc = dest;
+ *_argv = new_argv;
+ i_getopt_reset();
+}
diff --git a/src/doveadm/doveadm-dsync.h b/src/doveadm/doveadm-dsync.h
new file mode 100644
index 0000000..c4bf08b
--- /dev/null
+++ b/src/doveadm/doveadm-dsync.h
@@ -0,0 +1,10 @@
+#ifndef DOVEADM_DSYNC_H
+#define DOVEADM_DSYNC_H
+
+extern struct doveadm_cmd_ver2 doveadm_cmd_dsync_mirror;
+extern struct doveadm_cmd_ver2 doveadm_cmd_dsync_backup;
+extern struct doveadm_cmd_ver2 doveadm_cmd_dsync_server;
+
+void doveadm_dsync_main(int *_argc, char **_argv[]);
+
+#endif
diff --git a/src/doveadm/doveadm-dump-dbox.c b/src/doveadm/doveadm-dump-dbox.c
new file mode 100644
index 0000000..429b4a1
--- /dev/null
+++ b/src/doveadm/doveadm-dump-dbox.c
@@ -0,0 +1,231 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "hex-dec.h"
+#include "istream.h"
+#include "index/dbox-common/dbox-file.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+static void
+dump_timestamp(struct istream *input, const char *name, const char *value)
+{
+ time_t t;
+
+ if (strcmp(value, "0") == 0)
+ t = 0;
+ else {
+ t = hex2dec((const void *)value, strlen(value));
+ if (t == 0) {
+ i_fatal("Invalid %s at %"PRIuUOFF_T": %s",
+ name, input->v_offset, value);
+ }
+ }
+ printf("%s = %ld (%s)\n", name, (long)t, unixdate2str(t));
+}
+
+static uoff_t
+dump_size(struct istream *input, const char *name, const char *value)
+{
+ uoff_t size;
+
+ if (strcmp(value, "0") == 0)
+ size = 0;
+ else {
+ size = hex2dec((const void *)value, strlen(value));
+ if (size == 0) {
+ i_fatal("Invalid %s at %"PRIuUOFF_T": %s",
+ name, input->v_offset, value);
+ }
+ }
+ printf("%s = %"PRIuUOFF_T"\n", name, size);
+ return size;
+}
+
+static unsigned int dump_file_hdr(struct istream *input)
+{
+ const char *line, *const *arg, *version;
+ unsigned int msg_hdr_size = 0;
+
+ if ((line = i_stream_read_next_line(input)) == NULL)
+ i_fatal("Empty file");
+ arg = t_strsplit(line, " ");
+
+ /* check version */
+ version = *arg;
+ if (version == NULL || !str_is_numeric(version, ' '))
+ i_fatal("%s is not a dbox file", i_stream_get_name(input));
+ if (strcmp(version, "2") != 0)
+ i_fatal("Unsupported dbox file version %s", version);
+ arg++;
+
+ for (; *arg != NULL; arg++) {
+ switch (**arg) {
+ case DBOX_HEADER_MSG_HEADER_SIZE:
+ msg_hdr_size = hex2dec((const void *)(*arg + 1),
+ strlen(*arg + 1));
+ if (msg_hdr_size == 0) {
+ i_fatal("Invalid msg_header_size header: %s",
+ *arg + 1);
+ }
+ printf("file.msg_header_size = %u\n", msg_hdr_size);
+ break;
+ case DBOX_HEADER_CREATE_STAMP:
+ dump_timestamp(input, "file.create_stamp", *arg + 1);
+ break;
+ default:
+ printf("file.unknown-%c = %s\n", **arg, *arg + 1);
+ break;
+ }
+ }
+ if (msg_hdr_size == 0)
+ i_fatal("Missing msg_header_size in file header");
+ return msg_hdr_size;
+}
+
+static bool
+dump_msg_hdr(struct istream *input, unsigned int hdr_size, uoff_t *msg_size_r)
+{
+ struct dbox_message_header hdr;
+ const unsigned char *data;
+ size_t size;
+ uoff_t msg_size;
+
+ if (i_stream_read_bytes(input, &data, &size, hdr_size) <= 0) {
+ if (size == 0)
+ return FALSE;
+ i_fatal("Partial message header read at %"PRIuUOFF_T": %zu bytes",
+ input->v_offset, size);
+ }
+ printf("offset %"PRIuUOFF_T":\n", input->v_offset);
+
+ if (hdr_size < sizeof(hdr))
+ i_fatal("file.hdr_size too small: %u", hdr_size);
+ memcpy(&hdr, data, sizeof(hdr));
+
+ if (memcmp(hdr.magic_pre, DBOX_MAGIC_PRE, sizeof(hdr.magic_pre)) != 0)
+ i_fatal("dbox wrong pre-magic at %"PRIuUOFF_T, input->v_offset);
+
+ msg_size = dump_size(input, "msg.size",
+ t_strndup(hdr.message_size_hex, sizeof(hdr.message_size_hex)));
+
+ i_stream_skip(input, hdr_size);
+ *msg_size_r = msg_size;
+ return TRUE;
+}
+
+static void dump_msg_metadata(struct istream *input)
+{
+ struct dbox_metadata_header hdr;
+ const unsigned char *data;
+ size_t size;
+ const char *line;
+
+ /* verify magic */
+ if (i_stream_read_bytes(input, &data, &size, sizeof(hdr)) <= 0) {
+ i_fatal("dbox missing metadata at %"PRIuUOFF_T,
+ input->v_offset);
+ }
+ memcpy(&hdr, data, sizeof(hdr));
+ if (memcmp(hdr.magic_post, DBOX_MAGIC_POST, sizeof(hdr.magic_post)) != 0)
+ i_fatal("dbox wrong post-magic at %"PRIuUOFF_T, input->v_offset);
+ i_stream_skip(input, sizeof(hdr));
+
+ /* dump the metadata */
+ for (;;) {
+ if ((line = i_stream_read_next_line(input)) == NULL)
+ i_fatal("dbox metadata ended unexpectedly at EOF");
+ if (*line == '\0')
+ break;
+
+ switch (*line) {
+ case DBOX_METADATA_GUID:
+ printf("msg.guid = %s\n", line + 1);
+ break;
+ case DBOX_METADATA_POP3_UIDL:
+ printf("msg.pop3-uidl = %s\n", line + 1);
+ break;
+ case DBOX_METADATA_POP3_ORDER:
+ printf("msg.pop3-order = %s\n", line + 1);
+ break;
+ case DBOX_METADATA_RECEIVED_TIME:
+ dump_timestamp(input, "msg.received", line + 1);
+ break;
+ case DBOX_METADATA_PHYSICAL_SIZE:
+ (void)dump_size(input, "msg.physical-size", line + 1);
+ break;
+ case DBOX_METADATA_VIRTUAL_SIZE:
+ (void)dump_size(input, "msg.virtual-size", line + 1);
+ break;
+ case DBOX_METADATA_EXT_REF:
+ printf("msg.ext-ref = %s\n", line + 1);
+ break;
+ case DBOX_METADATA_ORIG_MAILBOX:
+ printf("msg.orig-mailbox = %s\n", line + 1);
+ break;
+
+ case DBOX_METADATA_OLDV1_EXPUNGED:
+ case DBOX_METADATA_OLDV1_FLAGS:
+ case DBOX_METADATA_OLDV1_KEYWORDS:
+ case DBOX_METADATA_OLDV1_SAVE_TIME:
+ case DBOX_METADATA_OLDV1_SPACE:
+ printf("msg.obsolete-%c = %s\n", *line, line + 1);
+ break;
+ }
+ }
+}
+
+static bool dump_msg(struct istream *input, unsigned int hdr_size)
+{
+ uoff_t msg_size;
+
+ if (!dump_msg_hdr(input, hdr_size, &msg_size))
+ return FALSE;
+ i_stream_skip(input, msg_size);
+ dump_msg_metadata(input);
+ return TRUE;
+}
+
+static void cmd_dump_dbox(const char *path, const char *const *args ATTR_UNUSED)
+{
+ struct istream *input;
+ int fd;
+ unsigned int hdr_size;
+ bool ret;
+
+ fd = open(path, O_RDONLY);
+ if (fd < 0)
+ i_fatal("open(%s) failed: %m", path);
+
+ input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+ i_stream_set_name(input, path);
+ hdr_size = dump_file_hdr(input);
+ do {
+ printf("\n");
+ T_BEGIN {
+ ret = dump_msg(input, hdr_size);
+ } T_END;
+ } while (ret);
+ i_stream_destroy(&input);
+}
+
+static bool test_dump_dbox(const char *path)
+{
+ const char *p;
+
+ p = strrchr(path, '/');
+ if (p == NULL)
+ p = path;
+ else
+ p++;
+ return str_begins(p, "m.") || str_begins(p, "u.");
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_dbox = {
+ "dbox",
+ test_dump_dbox,
+ cmd_dump_dbox
+};
diff --git a/src/doveadm/doveadm-dump-dcrypt-file.c b/src/doveadm/doveadm-dump-dcrypt-file.c
new file mode 100644
index 0000000..3703bf4
--- /dev/null
+++ b/src/doveadm/doveadm-dump-dcrypt-file.c
@@ -0,0 +1,93 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dcrypt.h"
+#include "istream.h"
+#include "istream-decrypt.h"
+#include "dcrypt-iostream.h"
+#include "doveadm-dump.h"
+#include <stdio.h>
+
+static int get_digest(const char *digest,
+ struct dcrypt_private_key **priv_key_r ATTR_UNUSED,
+ const char **error_r ATTR_UNUSED,
+ void *context)
+{
+ const char **digest_r = (const char**)context;
+ *digest_r = t_strdup(digest);
+ return 0;
+}
+
+static void dcrypt_istream_dump_metadata(const struct istream *stream)
+{
+ enum io_stream_encrypt_flags flags = i_stream_encrypt_get_flags(stream);
+ if ((flags & IO_STREAM_ENC_INTEGRITY_HMAC) != 0)
+ printf("flags: IO_STREAM_ENC_INTEGRITY_HMAC\n");
+ if ((flags & IO_STREAM_ENC_INTEGRITY_AEAD) != 0)
+ printf("flags: IO_STREAM_ENC_INTEGRITY_AEAD\n");
+ if ((flags & IO_STREAM_ENC_INTEGRITY_NONE) != 0)
+ printf("flags: IO_STREAM_ENC_INTEGRITY_NONE\n");
+ if ((flags & IO_STREAM_ENC_VERSION_1) != 0)
+ printf("flags: IO_STREAM_ENC_VERSION_1\n");
+
+ enum decrypt_istream_format format = i_stream_encrypt_get_format(stream);
+ switch (format) {
+ case DECRYPT_FORMAT_V1:
+ printf("format: DECRYPT_FORMAT_V1\n");
+ break;
+ case DECRYPT_FORMAT_V2:
+ printf("format: DECRYPT_FORMAT_V2\n");
+ break;
+ }
+}
+
+static bool dcrypt_file_dump_metadata(const char *filename, bool print)
+{
+ bool ret = FALSE;
+ struct istream *is = i_stream_create_file(filename, IO_BLOCK_SIZE);
+ const char *key_digest = NULL;
+ struct istream *ds = i_stream_create_decrypt_callback(is,
+ get_digest, &key_digest);
+
+ ssize_t size = i_stream_read(ds);
+ i_assert(size < 0);
+
+ if (key_digest != NULL) {
+ ret = TRUE;
+ if (print) {
+ dcrypt_istream_dump_metadata(ds);
+ printf("decrypt key digest: %s\n", key_digest);
+ }
+ } else if (print && ds->stream_errno != 0) {
+ i_error("read(%s) failed: %s",
+ i_stream_get_name(ds),
+ i_stream_get_error(ds));
+ }
+
+ i_stream_unref(&ds);
+ i_stream_unref(&is);
+ return ret;
+}
+
+static bool test_dump_dcrypt_file(const char *path)
+{
+ if (!dcrypt_initialize("openssl", NULL, NULL))
+ return FALSE;
+ bool ret = dcrypt_file_dump_metadata(path, FALSE);
+ return ret;
+}
+
+static void
+cmd_dump_dcrypt_file(const char *path, const char *const *args ATTR_UNUSED)
+{
+ const char *error = NULL;
+ if (!dcrypt_initialize("openssl", NULL, &error))
+ i_fatal("dcrypt_initialize failed: %s", error);
+ (void)dcrypt_file_dump_metadata(path, TRUE);
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_file = {
+ "dcrypt-file",
+ test_dump_dcrypt_file,
+ cmd_dump_dcrypt_file
+};
diff --git a/src/doveadm/doveadm-dump-dcrypt-key.c b/src/doveadm/doveadm-dump-dcrypt-key.c
new file mode 100644
index 0000000..cecd27f
--- /dev/null
+++ b/src/doveadm/doveadm-dump-dcrypt-key.c
@@ -0,0 +1,215 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dcrypt.h"
+#include "dcrypt-iostream.h"
+#include "ostream-encrypt.h"
+#include "istream-private.h"
+#include "istream-decrypt.h"
+#include "doveadm-dump.h"
+#include "hex-binary.h"
+#include "str.h"
+#include <stdio.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#define KEY_BUF_SIZE 4096
+
+static void dcrypt_dump_public_key_metadata(const char *buf)
+{
+ const char *error = NULL;
+ struct dcrypt_public_key *pub_key;
+
+ bool ret = dcrypt_key_load_public(&pub_key, buf, &error);
+ if (ret == FALSE) {
+ i_error("dcrypt_key_load_public failed: %s", error);
+ return;
+ }
+ enum dcrypt_key_type key_type = dcrypt_key_type_public(pub_key);
+ if (key_type == DCRYPT_KEY_RSA)
+ printf("key type: DCRYPT_KEY_RSA\n");
+ else if (key_type == DCRYPT_KEY_EC)
+ printf("key type: DCRYPT_KEY_EC\n");
+
+ string_t *hash = t_str_new(128);
+ if (!dcrypt_key_id_public(pub_key, "sha256", hash, &error)) {
+ i_error("dcrypt_key_id_public failed: %s", error);
+ } else {
+ const char *v2_hash = binary_to_hex(hash->data, hash->used);
+ printf("v2 hash: %s\n", v2_hash);
+
+ if (key_type == DCRYPT_KEY_EC) {
+ buffer_set_used_size(hash, 0);
+ if (!dcrypt_key_id_public_old(pub_key, hash, &error)) {
+ i_error("dcrypt_key_id_public_old failed: %s",
+ error);
+ } else {
+ const char *v1_hash = binary_to_hex(hash->data,
+ hash->used);
+ printf("v1 hash: %s\n", v1_hash);
+ }
+ }
+ }
+ dcrypt_key_unref_public(&pub_key);
+}
+
+static void dcrypt_dump_private_key_metadata(const char *buf)
+{
+ const char *error = NULL;
+ struct dcrypt_private_key *priv_key;
+
+ bool ret = dcrypt_key_load_private(&priv_key, buf, NULL, NULL,
+ &error);
+ if (ret == FALSE) {
+ i_error("dcrypt_key_load_private failed: %s", error);
+ return;
+ }
+ enum dcrypt_key_type key_type = dcrypt_key_type_private(priv_key);
+ if (key_type == DCRYPT_KEY_RSA)
+ printf("key type: DCRYPT_KEY_RSA\n");
+ else if (key_type == DCRYPT_KEY_EC)
+ printf("key type: DCRYPT_KEY_EC\n");
+
+ string_t *hash = t_str_new(128);
+ if (!dcrypt_key_id_private(priv_key, "sha256", hash, &error)) {
+ i_error("dcrypt_key_id_private failed: %s", error);
+ } else {
+ const char *v2_hash = binary_to_hex(hash->data, hash->used);
+ printf("v2 hash: %s\n", v2_hash);
+
+ if (key_type == DCRYPT_KEY_EC) {
+ buffer_set_used_size(hash, 0);
+ if (!dcrypt_key_id_private_old(priv_key, hash, &error)) {
+ i_error("dcrypt_key_id_private_old failed: %s", error);
+ } else {
+ const char *v1_hash = binary_to_hex(hash->data,
+ hash->used);
+ printf("v1 hash: %s\n", v1_hash);
+ }
+ }
+ }
+ dcrypt_key_unref_private(&priv_key);
+}
+
+static bool dcrypt_key_dump_metadata(const char *filename, bool print)
+{
+ bool ret = TRUE;
+ int fd = open(filename, O_RDONLY);
+ if (fd < 0) {
+ if (print) i_error("open(%s) failed: %m", filename);
+ return FALSE;
+ }
+
+ char buf[KEY_BUF_SIZE+1];
+ ssize_t res = read(fd, buf, KEY_BUF_SIZE);
+ if (res < 0) {
+ if (print) i_error("read(%s) failed: %m", filename);
+ i_close_fd(&fd);
+ return FALSE;
+ }
+ i_close_fd(&fd);
+
+ buf[res] = '\0';
+ enum dcrypt_key_format format;
+ enum dcrypt_key_version version;
+ enum dcrypt_key_kind kind;
+ enum dcrypt_key_encryption_type encryption_type;
+ const char *encryption_key_hash;
+ const char *key_hash;
+ const char *error;
+
+ ret = dcrypt_key_string_get_info(buf, &format, &version,
+ &kind, &encryption_type, &encryption_key_hash,
+ &key_hash, &error);
+ if (ret == FALSE) {
+ if (print) i_error("dcrypt_key_string_get_info failed: %s", error);
+ return FALSE;
+ }
+ if (!print) return TRUE;
+
+ switch (format) {
+ case DCRYPT_FORMAT_PEM:
+ printf("format: DCRYPT_FORMAT_PEM\n");
+ break;
+ case DCRYPT_FORMAT_DOVECOT:
+ printf("format: DCRYPT_FORMAT_DOVECOT\n");
+ break;
+ case DCRYPT_FORMAT_JWK:
+ printf("format: DCRYPT_FORMAT_JWK\n");
+ }
+
+ switch (version) {
+ case DCRYPT_KEY_VERSION_1:
+ printf("version: DCRYPT_KEY_VERSION_1\n");
+ break;
+ case DCRYPT_KEY_VERSION_2:
+ printf("version: DCRYPT_KEY_VERSION_2\n");
+ break;
+ case DCRYPT_KEY_VERSION_NA:
+ printf("version: DCRYPT_KEY_VERSION_NA\n");
+ break;
+ }
+
+ switch (kind) {
+ case DCRYPT_KEY_KIND_PUBLIC:
+ printf("kind: DCRYPT_KEY_KIND_PUBLIC\n");
+ break;
+ case DCRYPT_KEY_KIND_PRIVATE:
+ printf("kind: DCRYPT_KEY_KIND_PRIVATE\n");
+ break;
+ }
+
+ switch (encryption_type) {
+ case DCRYPT_KEY_ENCRYPTION_TYPE_NONE:
+ printf("encryption_type: DCRYPT_KEY_ENCRYPTION_TYPE_NONE\n");
+ break;
+ case DCRYPT_KEY_ENCRYPTION_TYPE_KEY:
+ printf("encryption_type: DCRYPT_KEY_ENCRYPTION_TYPE_KEY\n");
+ break;
+ case DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD:
+ printf("encryption_type: DCRYPT_KEY_ENCRYPTION_TYPE_PASSWORD\n");
+ break;
+ }
+
+ if (encryption_key_hash != NULL)
+ printf("encryption_key_hash: %s\n", encryption_key_hash);
+ if (key_hash != NULL)
+ printf("key_hash: %s\n", key_hash);
+
+ const char *data = t_str_rtrim(buf, "\r\n\t ");
+ switch (kind) {
+ case DCRYPT_KEY_KIND_PUBLIC:
+ dcrypt_dump_public_key_metadata(data);
+ break;
+ case DCRYPT_KEY_KIND_PRIVATE:
+ if (encryption_type == DCRYPT_KEY_ENCRYPTION_TYPE_NONE)
+ dcrypt_dump_private_key_metadata(data);
+ break;
+ }
+ return TRUE;
+}
+
+static bool test_dump_dcrypt_key(const char *path)
+{
+ if (!dcrypt_initialize("openssl", NULL, NULL))
+ return FALSE;
+ bool ret = dcrypt_key_dump_metadata(path, FALSE);
+ return ret;
+}
+
+static void
+cmd_dump_dcrypt_key(const char *path, const char *const *args ATTR_UNUSED)
+{
+ const char *error = NULL;
+ if (!dcrypt_initialize("openssl", NULL, &error))
+ i_fatal("dcrypt_initialize: %s", error);
+ (void)dcrypt_key_dump_metadata(path, TRUE);
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_key = {
+ "dcrypt-key",
+ test_dump_dcrypt_key,
+ cmd_dump_dcrypt_key
+};
diff --git a/src/doveadm/doveadm-dump-index.c b/src/doveadm/doveadm-dump-index.c
new file mode 100644
index 0000000..ec21406
--- /dev/null
+++ b/src/doveadm/doveadm-dump-index.c
@@ -0,0 +1,833 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "file-lock.h"
+#include "message-parser.h"
+#include "message-part-serialize.h"
+#include "mail-cache-private.h"
+#include "mail-index-modseq.h"
+#include "mail-storage-private.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+#include <time.h>
+
+struct index_vsize_header {
+ uint64_t vsize;
+ uint32_t highest_uid;
+ uint32_t message_count;
+};
+struct maildir_index_header {
+ uint32_t new_check_time, new_mtime, new_mtime_nsecs;
+ uint32_t cur_check_time, cur_mtime, cur_mtime_nsecs;
+ uint32_t uidlist_mtime, uidlist_mtime_nsecs, uidlist_size;
+};
+struct mbox_index_header {
+ uint64_t sync_size;
+ uint32_t sync_mtime;
+ uint8_t dirty_flag;
+ uint8_t unused[3];
+ uint8_t mailbox_guid[16];
+};
+struct sdbox_index_header {
+ uint32_t rebuild_count;
+ guid_128_t mailbox_guid;
+ uint8_t flags;
+ uint8_t unused[3];
+};
+struct mdbox_index_header {
+ uint32_t map_uid_validity;
+ guid_128_t mailbox_guid;
+ uint8_t flags;
+ uint8_t unused[3];
+};
+struct mdbox_mail_index_record {
+ uint32_t map_uid;
+ uint32_t save_date;
+};
+struct obox_mail_index_record {
+ unsigned char guid[GUID_128_SIZE];
+ unsigned char oid[GUID_128_SIZE];
+};
+struct mobox_mail_index_header {
+ uint32_t rebuild_count;
+ uint32_t map_uid_validity;
+ uint8_t unused[4];
+ guid_128_t mailbox_guid;
+};
+struct mobox_mail_index_record {
+ uint32_t map_uid;
+ uint32_t save_date;
+};
+struct mobox_map_mail_index_header {
+ uint32_t rebuild_count;
+};
+
+struct mobox_map_mail_index_record {
+ uint32_t offset;
+ uint32_t size;
+ guid_128_t oid;
+};
+struct mailbox_list_index_header {
+ uint8_t refresh_flag;
+ /* array of { uint32_t id; char name[]; } */
+};
+struct mailbox_list_index_record {
+ uint32_t name_id;
+ uint32_t parent_uid;
+ guid_128_t guid;
+ uint32_t uid_validity;
+};
+struct mailbox_list_index_msgs_record {
+ uint32_t messages;
+ uint32_t unseen;
+ uint32_t recent;
+ uint32_t uidnext;
+};
+
+struct fts_index_header {
+ uint32_t last_indexed_uid;
+ uint32_t settings_checksum;
+ uint32_t unused;
+};
+struct virtual_mail_index_header {
+ uint32_t change_counter;
+ uint32_t mailbox_count;
+ uint32_t highest_mailbox_id;
+ uint32_t search_args_crc32;
+};
+struct virtual_mail_index_mailbox_record {
+ uint32_t id;
+ uint32_t name_len;
+ uint32_t uid_validity;
+ uint32_t next_uid;
+ uint64_t highest_modseq;
+};
+struct virtual_mail_index_record {
+ uint32_t mailbox_id;
+ uint32_t real_uid;
+};
+
+struct mdbox_mail_index_map_record {
+ uint32_t file_id;
+ uint32_t offset;
+ uint32_t size;
+};
+
+static void dump_hdr(struct mail_index *index)
+{
+ const struct mail_index_header *hdr = &index->map->hdr;
+ unsigned int i;
+
+ printf("version .................. = %u.%u\n", hdr->major_version, hdr->minor_version);
+ printf("base header size ......... = %u\n", hdr->base_header_size);
+ printf("header size .............. = %u\n", hdr->header_size);
+ printf("record size .............. = %u\n", hdr->record_size);
+ printf("compat flags ............. = %u\n", hdr->compat_flags);
+ printf("index id ................. = %u (%s)\n", hdr->indexid, unixdate2str(hdr->indexid));
+ printf("flags .................... = %u\n", hdr->flags);
+ printf("uid validity ............. = %u (%s)\n", hdr->uid_validity, unixdate2str(hdr->uid_validity));
+ printf("next uid ................. = %u\n", hdr->next_uid);
+ printf("messages count ........... = %u\n", hdr->messages_count);
+ printf("seen messages count ...... = %u\n", hdr->seen_messages_count);
+ printf("deleted messages count ... = %u\n", hdr->deleted_messages_count);
+ printf("first recent uid ......... = %u\n", hdr->first_recent_uid);
+ printf("first unseen uid lowwater = %u\n", hdr->first_unseen_uid_lowwater);
+ printf("first deleted uid lowwater = %u\n", hdr->first_deleted_uid_lowwater);
+ printf("log file seq ............. = %u\n", hdr->log_file_seq);
+ if (hdr->minor_version == 0) {
+ printf("log file int offset ...... = %u\n", hdr->log_file_tail_offset);
+ printf("log file ext offset ...... = %u\n", hdr->log_file_head_offset);
+ } else {
+ printf("log file tail offset ..... = %u\n", hdr->log_file_tail_offset);
+ printf("log file head offset ..... = %u\n", hdr->log_file_head_offset);
+ }
+ if (hdr->minor_version >= 3) {
+ printf("log2 rotate time ......... = %u (%s)\n", hdr->log2_rotate_time, unixdate2str(hdr->log2_rotate_time));
+ printf("last temp file scan ...... = %u (%s)\n", hdr->last_temp_file_scan, unixdate2str(hdr->last_temp_file_scan));
+ }
+ printf("day stamp ................ = %u (%s)\n", hdr->day_stamp, unixdate2str(hdr->day_stamp));
+ for (i = 0; i < N_ELEMENTS(hdr->day_first_uid); i++)
+ printf("day first uid[%u] ......... = %u\n", i, hdr->day_first_uid[i]);
+}
+
+static void dump_list_header(const void *data, size_t size)
+{
+ const struct mailbox_list_index_header *hdr = data;
+ const void *name_start, *p;
+ size_t i, len;
+ uint32_t id;
+
+ printf(" - refresh_flag = %d\n", hdr->refresh_flag);
+ for (i = sizeof(*hdr); i < size; ) {
+ /* get id */
+ if (i + sizeof(id) > size) {
+ printf(" - corrupted\n");
+ break;
+ }
+ memcpy(&id, CONST_PTR_OFFSET(data, i), sizeof(id));
+ i += sizeof(id);
+
+ if (id == 0)
+ break;
+
+ /* get name */
+ p = memchr(CONST_PTR_OFFSET(data, i), '\0', size-i);
+ if (p == NULL) {
+ printf(" - corrupted\n");
+ break;
+ }
+ name_start = CONST_PTR_OFFSET(data, i);
+ len = (const char *)p - (const char *)name_start;
+
+ printf(" - %d : %.*s\n", id, (int)len, (const char *)name_start);
+
+ i += len + 1;
+ }
+}
+
+static void dump_box_name_header(const void *data, size_t size)
+{
+ char *dest = t_malloc0(size + 1);
+ memcpy(dest, data, size);
+ for (size_t i = 0; i < size; i++) {
+ if (dest[i] == '\0')
+ dest[i] = '\n';
+ }
+ printf(" %s\n", t_strarray_join(t_strsplit(dest, "\n"), "\n "));
+}
+
+static void dump_extension_header(struct mail_index *index,
+ const struct mail_index_ext *ext)
+{
+ const void *data;
+ void *buf;
+
+ if (strcmp(ext->name, MAIL_INDEX_EXT_KEYWORDS) == 0)
+ return;
+
+ /* add some padding, since we don't bother to handle undersized
+ headers correctly */
+ buf = t_malloc0(MALLOC_ADD(ext->hdr_size, 128));
+ data = MAIL_INDEX_MAP_HDR_OFFSET(index->map, ext->hdr_offset);
+ memcpy(buf, data, ext->hdr_size);
+ data = buf;
+
+ if (strcmp(ext->name, "hdr-vsize") == 0) {
+ const struct index_vsize_header *hdr = data;
+
+ printf("header\n");
+ printf(" - highest uid . = %u\n", hdr->highest_uid);
+ printf(" - message count = %u\n", hdr->message_count);
+ printf(" - vsize ....... = %"PRIu64"\n", hdr->vsize);
+ } else if (strcmp(ext->name, "maildir") == 0) {
+ const struct maildir_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - new_check_time .... = %s\n", unixdate2str(hdr->new_check_time));
+ printf(" - new_mtime ......... = %s\n", unixdate2str(hdr->new_mtime));
+ printf(" - new_mtime_nsecs ... = %u\n", hdr->new_mtime_nsecs);
+ printf(" - cur_check_time .... = %s\n", unixdate2str(hdr->cur_check_time));
+ printf(" - cur_mtime ......... = %s\n", unixdate2str(hdr->cur_mtime));
+ printf(" - cur_mtime_nsecs.... = %u\n", hdr->cur_mtime_nsecs);
+ printf(" - uidlist_mtime ..... = %s\n", unixdate2str(hdr->uidlist_mtime));
+ printf(" - uidlist_mtime_nsecs = %u\n", hdr->uidlist_mtime_nsecs);
+ printf(" - uidlist_size ...... = %u\n", hdr->uidlist_size);
+ } else if (strcmp(ext->name, "mbox") == 0) {
+ const struct mbox_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - sync_mtime . = %s\n", unixdate2str(hdr->sync_mtime));
+ printf(" - sync_size .. = %"PRIu64"\n", hdr->sync_size);
+ printf(" - dirty_flag . = %d\n", hdr->dirty_flag);
+ printf(" - mailbox_guid = %s\n",
+ guid_128_to_string(hdr->mailbox_guid));
+ } else if (strcmp(ext->name, "mdbox-hdr") == 0) {
+ const struct mdbox_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - map_uid_validity .. = %u\n", hdr->map_uid_validity);
+ printf(" - mailbox_guid ...... = %s\n",
+ guid_128_to_string(hdr->mailbox_guid));
+ printf(" - flags ............. = 0x%x\n", hdr->flags);
+ } else if (strcmp(ext->name, "dbox-hdr") == 0) {
+ const struct sdbox_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - rebuild_count . = %u\n", hdr->rebuild_count);
+ printf(" - mailbox_guid .. = %s\n",
+ guid_128_to_string(hdr->mailbox_guid));
+ printf(" - flags ......... = 0x%x\n", hdr->flags);
+ } else if (strcmp(ext->name, "mobox-hdr") == 0) {
+ const struct mobox_mail_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - rebuild_count .. = %u\n", hdr->rebuild_count);
+ printf(" - map_uid_validity .. = %u\n", hdr->map_uid_validity);
+ printf(" - mailbox_guid ...... = %s\n",
+ guid_128_to_string(hdr->mailbox_guid));
+ } else if (strcmp(ext->name, "mobox-map") == 0) {
+ const struct mobox_map_mail_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - rebuild_count .. = %u\n", hdr->rebuild_count);
+ } else if (strcmp(ext->name, "modseq") == 0) {
+ const struct mail_index_modseq_header *hdr = data;
+
+ printf("header\n");
+ printf(" - highest_modseq = %"PRIu64"\n", hdr->highest_modseq);
+ printf(" - log_seq ...... = %u\n", hdr->log_seq);
+ printf(" - log_offset ... = %u\n", hdr->log_offset);
+ } else if (strcmp(ext->name, "fts") == 0) {
+ const struct fts_index_header *hdr = data;
+
+ printf("header\n");
+ printf(" - last_indexed_uid ..... = %u\n",
+ hdr->last_indexed_uid);
+ printf(" - settings_checksum .... = %u\n",
+ hdr->settings_checksum);
+ } else if (strcmp(ext->name, "virtual") == 0) {
+ const struct virtual_mail_index_header *hdr = data;
+ const struct virtual_mail_index_mailbox_record *rec;
+ const unsigned char *name;
+ unsigned int i;
+
+ printf("header\n");
+ printf(" - change_counter ... = %u\n", hdr->change_counter);
+ printf(" - mailbox_count .... = %u\n", hdr->mailbox_count);
+ printf(" - highest_mailbox_id = %u\n", hdr->highest_mailbox_id);
+ printf(" - search_args_crc32 = %u\n", hdr->search_args_crc32);
+
+ rec = CONST_PTR_OFFSET(hdr, sizeof(*hdr));
+ name = CONST_PTR_OFFSET(rec, sizeof(*rec) * hdr->mailbox_count);
+ for (i = 0; i < hdr->mailbox_count; i++, rec++) {
+ printf("mailbox %s:\n", t_strndup(name, rec->name_len));
+ printf(" - id ........... = %u\n", rec->id);
+ printf(" - uid_validity . = %u\n", rec->uid_validity);
+ printf(" - next_uid ..... = %u\n", rec->next_uid);
+ printf(" - highest_modseq = %"PRIu64"\n",
+ rec->highest_modseq);
+
+ name += rec->name_len;
+ }
+ } else if (strcmp(ext->name, "list") == 0) {
+ printf("header ........ = %s\n",
+ binary_to_hex(data, ext->hdr_size));
+ dump_list_header(data, ext->hdr_size);
+ } else if (strcmp(ext->name, "box-name") == 0) {
+ printf("header ........ = %s\n",
+ binary_to_hex(data, ext->hdr_size));
+ dump_box_name_header(data, ext->hdr_size);
+ } else if (strcmp(ext->name, "hdr-pop3-uidl") == 0) {
+ const struct mailbox_index_pop3_uidl *hdr = data;
+
+ printf("header\n");
+ printf(" - max_uid_with_pop3_uidl = %u\n",
+ hdr->max_uid_with_pop3_uidl);
+ } else {
+ printf("header ........ = %s\n",
+ binary_to_hex(data, ext->hdr_size));
+ }
+}
+
+static void dump_extensions(struct mail_index *index)
+{
+ const struct mail_index_ext *extensions;
+ unsigned int i, count;
+
+ if (array_is_created(&index->map->extensions))
+ extensions = array_get(&index->map->extensions, &count);
+ else
+ count = 0;
+ if (count == 0) {
+ printf("no extensions\n");
+ return;
+ }
+
+ for (i = 0; i < count; i++) {
+ const struct mail_index_ext *ext = &extensions[i];
+
+ printf("-- Extension %u --\n", i);
+ printf("name ........ = %s\n", ext->name);
+ printf("hdr_size .... = %u\n", ext->hdr_size);
+ printf("reset_id .... = %u\n", ext->reset_id);
+ printf("record_offset = %u\n", ext->record_offset);
+ printf("record_size . = %u\n", ext->record_size);
+ printf("record_align = %u\n", ext->record_align);
+ if (ext->hdr_size > 0) T_BEGIN {
+ dump_extension_header(index, ext);
+ } T_END;
+ }
+}
+
+static void dump_keywords(struct mail_index *index)
+{
+ const unsigned int *kw_indexes;
+ const char *const *keywords;
+ unsigned int i, count;
+
+ printf("-- Keywords --\n");
+ if (!array_is_created(&index->map->keyword_idx_map))
+ return;
+
+ kw_indexes = array_get(&index->map->keyword_idx_map, &count);
+ if (count == 0)
+ return;
+
+ keywords = array_front(&index->keywords);
+ for (i = 0; i < count; i++)
+ printf("%3u = %s\n", i, keywords[kw_indexes[i]]);
+}
+
+static const char *cache_decision2str(enum mail_cache_decision_type type)
+{
+ const char *str;
+
+ switch (type & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) {
+ case MAIL_CACHE_DECISION_NO:
+ str = "no";
+ break;
+ case MAIL_CACHE_DECISION_TEMP:
+ str = "tmp";
+ break;
+ case MAIL_CACHE_DECISION_YES:
+ str = "yes";
+ break;
+ default:
+ return t_strdup_printf("0x%x", type);
+ }
+
+ if ((type & MAIL_CACHE_DECISION_FORCED) != 0)
+ str = t_strconcat(str, "!", NULL);
+ return str;
+}
+
+#define CACHE_TYPE_IS_FIXED_SIZE(type) \
+ ((type) == MAIL_CACHE_FIELD_FIXED_SIZE || \
+ (type) == MAIL_CACHE_FIELD_BITMASK)
+static const char *cache_type2str(enum mail_cache_field_type type)
+{
+ switch (type) {
+ case MAIL_CACHE_FIELD_FIXED_SIZE:
+ return "fix";
+ case MAIL_CACHE_FIELD_VARIABLE_SIZE:
+ return "var";
+ case MAIL_CACHE_FIELD_STRING:
+ return "str";
+ case MAIL_CACHE_FIELD_BITMASK:
+ return "bit";
+ case MAIL_CACHE_FIELD_HEADER:
+ return "hdr";
+ default:
+ return t_strdup_printf("0x%x", type);
+ }
+}
+
+static void dump_cache_hdr(struct mail_cache *cache)
+{
+ const struct mail_cache_header *hdr;
+ const struct mail_cache_field *fields, *field;
+ unsigned int i, count, cache_idx;
+
+ (void)mail_cache_open_and_verify(cache);
+ if (MAIL_CACHE_IS_UNUSABLE(cache)) {
+ printf("cache is unusable\n");
+ return;
+ }
+
+ hdr = cache->hdr;
+ printf("major version ........ = %u\n", hdr->major_version);
+ printf("minor version ........ = %u\n", hdr->minor_version);
+ printf("indexid .............. = %u (%s)\n", hdr->indexid, unixdate2str(hdr->indexid));
+ printf("file_seq ............. = %u (%s) (%d purges)\n",
+ hdr->file_seq, unixdate2str(hdr->file_seq),
+ hdr->file_seq - hdr->indexid);
+ printf("continued_record_count = %u\n", hdr->continued_record_count);
+ printf("record_count ......... = %u\n", hdr->record_count);
+ printf("used_file_size (old) . = %u\n", hdr->backwards_compat_used_file_size);
+ printf("deleted_record_count . = %u\n", hdr->deleted_record_count);
+ printf("field_header_offset .. = %u (0x%08x nontranslated)\n",
+ mail_index_offset_to_uint32(hdr->field_header_offset),
+ hdr->field_header_offset);
+
+ printf("-- Cache fields --\n");
+ fields = mail_cache_register_get_list(cache, pool_datastack_create(),
+ &count);
+ printf(
+" # Name Type Size Dec Last used\n");
+ for (i = 0; i < cache->file_fields_count; i++) {
+ cache_idx = cache->file_field_map[i];
+ field = &fields[cache_idx];
+
+ printf("%2u: %-44s %-4s ", i, field->name,
+ cache_type2str(field->type));
+ if (field->field_size != (uint32_t)-1 ||
+ CACHE_TYPE_IS_FIXED_SIZE(field->type))
+ printf("%4u ", field->field_size);
+ else
+ printf(" - ");
+ printf("%-4s %.16s\n",
+ cache_decision2str(field->decision),
+ unixdate2str(field->last_used));
+ }
+}
+
+static void dump_message_part(string_t *str, const struct message_part *part)
+{
+ for (; part != NULL; part = part->next) {
+ str_append_c(str, '(');
+ str_printfa(str, "pos=%"PRIuUOFF_T" ", part->physical_pos);
+ str_printfa(str, "hdr.p=%"PRIuUOFF_T" ", part->header_size.physical_size);
+ str_printfa(str, "hdr.v=%"PRIuUOFF_T" ", part->header_size.virtual_size);
+ str_printfa(str, "body.p=%"PRIuUOFF_T" ", part->body_size.physical_size);
+ str_printfa(str, "body.v=%"PRIuUOFF_T" ", part->body_size.virtual_size);
+ str_printfa(str, "flags=%x", part->flags);
+ if (part->children != NULL) {
+ str_append_c(str, ' ');
+ dump_message_part(str, part->children);
+ }
+ str_append_c(str, ')');
+ }
+}
+
+static void
+dump_cache_mime_parts(string_t *str, const void *data, unsigned int size)
+{
+ const struct message_part *part;
+ const char *error;
+
+ str_append_c(str, ' ');
+
+ part = message_part_deserialize(pool_datastack_create(), data, size, &error);
+ if (part == NULL) {
+ str_printfa(str, "error: %s", error);
+ return;
+ }
+
+ dump_message_part(str, part);
+}
+
+static void
+dump_cache_append_string(string_t *str, const unsigned char *data,
+ unsigned int size)
+{
+ /* cached strings end with NUL */
+ if (size > 0 && data[size-1] == '\0')
+ size--;
+ str_append_data(str, data, size);
+}
+
+static void
+dump_cache_snippet(string_t *str, const unsigned char *data, unsigned int size)
+{
+ if (size == 0)
+ return;
+ str_printfa(str, " (version=%u: ", data[0]);
+ dump_cache_append_string(str, data+1, size-1);
+ str_append_c(str, ')');
+}
+
+static void dump_cache(struct mail_cache_view *cache_view, unsigned int seq)
+{
+ struct mail_cache_lookup_iterate_ctx iter;
+ const struct mail_cache_record *prev_rec = NULL;
+ const struct mail_cache_field *field;
+ struct mail_cache_iterate_field iter_field;
+ const void *data;
+ unsigned int size;
+ string_t *str;
+ int ret;
+
+ str = t_str_new(512);
+ mail_cache_lookup_iter_init(cache_view, seq, &iter);
+ while ((ret = mail_cache_lookup_iter_next(&iter, &iter_field)) > 0) {
+ if (iter.rec != prev_rec) {
+ printf(" - cache offset=%u size=%u, prev_offset = %u\n",
+ iter.offset, iter.rec->size,
+ iter.rec->prev_offset);
+ prev_rec = iter.rec;
+ }
+
+ field = &cache_view->cache->fields[iter_field.field_idx].field;
+ data = iter_field.data;
+ size = iter_field.size;
+
+ str_truncate(str, 0);
+ str_printfa(str, " - %s: ", field->name);
+ switch (field->type) {
+ case MAIL_CACHE_FIELD_FIXED_SIZE:
+ if (size == sizeof(uint32_t)) {
+ uint32_t value;
+ memcpy(&value, data, sizeof(value));
+ str_printfa(str, "%u ", value);
+ } else if (size == sizeof(uint64_t)) {
+ uint64_t value;
+ memcpy(&value, data, sizeof(value));
+ str_printfa(str, "%"PRIu64, value);
+ }
+ /* fall through */
+ case MAIL_CACHE_FIELD_VARIABLE_SIZE:
+ case MAIL_CACHE_FIELD_BITMASK:
+ str_printfa(str, "(%s)", binary_to_hex(data, size));
+ if (strcmp(field->name, "mime.parts") == 0)
+ dump_cache_mime_parts(str, data, size);
+ else if (strcmp(field->name, "body.snippet") == 0)
+ dump_cache_snippet(str, data, size);
+ break;
+ case MAIL_CACHE_FIELD_STRING:
+ dump_cache_append_string(str, data, size);
+ break;
+ case MAIL_CACHE_FIELD_HEADER: {
+ const uint32_t *lines = data;
+ int i;
+
+ for (i = 0;; i++) {
+ if (size < sizeof(uint32_t)) {
+ if (i == 0 && size == 0) {
+ /* header doesn't exist */
+ break;
+ }
+
+ str_append(str, "\n - BROKEN: header field doesn't end with 0 line");
+ size = 0;
+ break;
+ }
+
+ size -= sizeof(uint32_t);
+ data = CONST_PTR_OFFSET(data, sizeof(uint32_t));
+ if (lines[i] == 0)
+ break;
+
+ if (i > 0)
+ str_append(str, ", ");
+ str_printfa(str, "%u", lines[i]);
+ }
+
+ if (i == 1 && size > 0 &&
+ ((const char *)data)[size-1] == '\n')
+ size--;
+ if (size > 0)
+ str_printfa(str, ": %.*s", (int)size, (const char *)data);
+ break;
+ }
+ case MAIL_CACHE_FIELD_COUNT:
+ i_unreached();
+ break;
+ }
+
+ fwrite(str_data(str), 1, str_len(str), stdout);
+ putchar('\n');
+ }
+ if (ret < 0)
+ printf(" - broken cache\n");
+}
+
+static const char *flags2str(enum mail_flags flags)
+{
+ string_t *str;
+
+ str = t_str_new(64);
+ str_append_c(str, '(');
+ if ((flags & MAIL_SEEN) != 0)
+ str_append(str, "Seen ");
+ if ((flags & MAIL_ANSWERED) != 0)
+ str_append(str, "Answered ");
+ if ((flags & MAIL_FLAGGED) != 0)
+ str_append(str, "Flagged ");
+ if ((flags & MAIL_DELETED) != 0)
+ str_append(str, "Deleted ");
+ if ((flags & MAIL_DRAFT) != 0)
+ str_append(str, "Draft ");
+ if (str_len(str) == 1)
+ return "";
+
+ str_truncate(str, str_len(str)-1);
+ str_append_c(str, ')');
+ return str_c(str);
+}
+
+static void dump_record(struct mail_index_view *view, unsigned int seq)
+{
+ struct mail_index *index = mail_index_view_get_index(view);
+ const struct mail_index_record *rec;
+ const struct mail_index_registered_ext *ext;
+ const void *data;
+ unsigned int i, ext_count;
+ string_t *str;
+ bool expunged;
+
+ rec = mail_index_lookup(view, seq);
+ printf("RECORD: seq=%u, uid=%u, flags=0x%02x %s\n",
+ seq, rec->uid, rec->flags, flags2str(rec->flags));
+
+ str = t_str_new(256);
+ ext = array_get(&index->extensions, &ext_count);
+ for (i = 0; i < ext_count; i++) {
+ mail_index_lookup_ext(view, seq, i, &data, &expunged);
+ if (data == NULL || ext[i].record_size == 0)
+ continue;
+
+ str_truncate(str, 0);
+ str_printfa(str, " - ext %d %-10s: ", i, ext[i].name);
+ if (ext[i].record_size == sizeof(uint16_t) &&
+ ext[i].record_align == sizeof(uint16_t))
+ str_printfa(str, "%10u", *((const uint16_t *)data));
+ else if (ext[i].record_size == sizeof(uint32_t) &&
+ ext[i].record_align == sizeof(uint32_t))
+ str_printfa(str, "%10u", *((const uint32_t *)data));
+ else if (ext[i].record_size == sizeof(uint64_t) &&
+ ext[i].record_align == sizeof(uint64_t)) {
+ uint64_t value = *((const uint64_t *)data);
+ str_printfa(str, "%10"PRIu64, value);
+ } else {
+ str_append(str, " ");
+ }
+ str_printfa(str, " (%s)",
+ binary_to_hex(data, ext[i].record_size));
+ printf("%s\n", str_c(str));
+ if (strcmp(ext[i].name, "virtual") == 0) {
+ const struct virtual_mail_index_record *vrec = data;
+ printf(" : mailbox_id = %u\n", vrec->mailbox_id);
+ printf(" : real_uid = %u\n", vrec->real_uid);
+ } else if (strcmp(ext[i].name, "map") == 0) {
+ const struct mdbox_mail_index_map_record *mrec = data;
+ printf(" : file_id = %u\n", mrec->file_id);
+ printf(" : offset = %u\n", mrec->offset);
+ printf(" : size = %u\n", mrec->size);
+ } else if (strcmp(ext[i].name, "mdbox") == 0) {
+ const struct mdbox_mail_index_record *drec = data;
+ printf(" : map_uid = %u\n", drec->map_uid);
+ printf(" : save_date = %u (%s)\n", drec->save_date, unixdate2str(drec->save_date));
+ } else if (strcmp(ext[i].name, "obox") == 0) {
+ const struct obox_mail_index_record *orec = data;
+ printf(" : guid = %s\n", guid_128_to_string(orec->guid));
+ printf(" : oid = %s\n", binary_to_hex(orec->oid, ext[i].record_size - sizeof(orec->guid)));
+ } else if (strcmp(ext[i].name, "mobox") == 0) {
+ const struct mobox_mail_index_record *orec = data;
+ printf(" : map_uid = %u\n", orec->map_uid);
+ printf(" : save_date = %u (%s)\n", orec->save_date, unixdate2str(orec->save_date));
+ } else if (strcmp(ext[i].name, "mobox-map") == 0) {
+ const struct mobox_map_mail_index_record *orec = data;
+ printf(" : offset = %u\n", orec->offset);
+ printf(" : size = %u\n", orec->size);
+ printf(" : oid = %s\n", guid_128_to_string(orec->oid));
+ } else if (strcmp(ext[i].name, "list") == 0) {
+ const struct mailbox_list_index_record *lrec = data;
+ printf(" : name_id = %u\n", lrec->name_id);
+ printf(" : parent_uid = %u\n", lrec->parent_uid);
+ printf(" : guid = %s\n", guid_128_to_string(lrec->guid));
+ printf(" : uid_validity = %u\n", lrec->uid_validity);
+ } else if (strcmp(ext[i].name, "msgs") == 0) {
+ const struct mailbox_list_index_msgs_record *lrec = data;
+ printf(" : messages = %u\n", lrec->messages);
+ printf(" : unseen = %u\n", lrec->unseen);
+ printf(" : recent = %u\n", lrec->recent);
+ printf(" : uidnext = %u\n", lrec->uidnext);
+ } else if (strcmp(ext[i].name, "vsize") == 0 &&
+ ext[i].record_size >= sizeof(struct mailbox_index_vsize)) {
+ /* this is "vsize" in dovecot.list.index, not the
+ 32bit "vsize" in dovecot.index */
+ const struct mailbox_index_vsize *vrec = data;
+ printf(" : vsize = %"PRIu64"\n", vrec->vsize);
+ printf(" : highest_uid = %u\n", vrec->highest_uid);
+ printf(" : message_count = %u\n", vrec->message_count);
+ }
+ }
+}
+
+static bool dir_has_index(const char *dir, const char *name)
+{
+ struct stat st;
+
+ return stat(t_strconcat(dir, "/", name, NULL), &st) == 0 ||
+ stat(t_strconcat(dir, "/", name, ".log", NULL), &st) == 0;
+}
+
+static struct mail_index *path_open_index(const char *path)
+{
+ struct stat st;
+ const char *p;
+
+ if (stat(path, &st) == 0 && S_ISDIR(st.st_mode)) {
+ if (dir_has_index(path, "dovecot.index"))
+ return mail_index_alloc(NULL, path, "dovecot.index");
+ else if (dir_has_index(path, "dovecot.map.index"))
+ return mail_index_alloc(NULL, path, "dovecot.map.index");
+ else
+ return NULL;
+ } else if ((p = strrchr(path, '/')) != NULL)
+ return mail_index_alloc(NULL, t_strdup_until(path, p), p + 1);
+ else
+ return mail_index_alloc(NULL, ".", path);
+}
+
+static void
+cmd_dump_index(const char *path, const char *const *args)
+{
+ struct mail_index *index;
+ struct mail_index_view *view;
+ struct mail_cache_view *cache_view;
+ unsigned int seq, uid = 0;
+
+ index = path_open_index(path);
+ if (index == NULL ||
+ mail_index_open(index, MAIL_INDEX_OPEN_FLAG_READONLY) <= 0)
+ i_fatal("Couldn't open index %s", path);
+ if (args[0] != NULL) {
+ if (str_to_uint(args[0], &uid) < 0)
+ i_fatal("Invalid uid number %s", args[0]);
+ }
+
+ view = mail_index_view_open(index);
+ cache_view = mail_cache_view_open(index->cache, view);
+
+ if (uid == 0) {
+ printf("-- INDEX: %s\n", index->filepath);
+ dump_hdr(index);
+ dump_extensions(index);
+ dump_keywords(index);
+
+ printf("\n-- CACHE: %s\n", index->cache->filepath);
+ dump_cache_hdr(index->cache);
+
+ printf("\n-- RECORDS: %u\n", index->map->hdr.messages_count);
+ }
+ for (seq = 1; seq <= index->map->hdr.messages_count; seq++) {
+ if (uid == 0 || mail_index_lookup(view, seq)->uid == uid) {
+ T_BEGIN {
+ dump_record(view, seq);
+ dump_cache(cache_view, seq);
+ printf("\n");
+ } T_END;
+ }
+ }
+ mail_cache_view_close(&cache_view);
+ mail_index_view_close(&view);
+ mail_index_close(index);
+ mail_index_free(&index);
+}
+
+static bool test_dump_index(const char *path)
+{
+ struct mail_index *index;
+ bool ret;
+
+ index = path_open_index(path);
+ if (index == NULL)
+ return FALSE;
+
+ ret = mail_index_open(index, MAIL_INDEX_OPEN_FLAG_READONLY) > 0;
+ if (ret)
+ mail_index_close(index);
+ mail_index_free(&index);
+ return ret;
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_index = {
+ "index",
+ test_dump_index,
+ cmd_dump_index
+};
diff --git a/src/doveadm/doveadm-dump-log.c b/src/doveadm/doveadm-dump-log.c
new file mode 100644
index 0000000..0725b28
--- /dev/null
+++ b/src/doveadm/doveadm-dump-log.c
@@ -0,0 +1,568 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "hex-binary.h"
+#include "mail-index-private.h"
+#include "mail-transaction-log-private.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+
+static struct mail_transaction_ext_intro prev_intro;
+
+static void dump_hdr(struct istream *input, uint64_t *modseq_r,
+ unsigned int *version_r)
+{
+ struct mail_transaction_log_header hdr;
+ const unsigned char *data;
+ size_t size;
+ int ret;
+
+ ret = i_stream_read_bytes(input, &data, &size, sizeof(hdr));
+ if (ret < 0 && input->stream_errno != 0)
+ i_fatal("read() failed: %s", i_stream_get_error(input));
+ if (ret <= 0) {
+ i_fatal("file hdr read() %zu != %zu",
+ size, sizeof(hdr));
+ }
+ memcpy(&hdr, data, sizeof(hdr));
+ if (hdr.hdr_size < sizeof(hdr)) {
+ memset(PTR_OFFSET(&hdr, hdr.hdr_size), 0,
+ sizeof(hdr) - hdr.hdr_size);
+ }
+ i_stream_skip(input, hdr.hdr_size);
+
+ printf("version = %u.%u\n", hdr.major_version, hdr.minor_version);
+ printf("hdr size = %u\n", hdr.hdr_size);
+ printf("index id = %u\n", hdr.indexid);
+ printf("file seq = %u\n", hdr.file_seq);
+ printf("prev file = %u/%u\n", hdr.prev_file_seq, hdr.prev_file_offset);
+ printf("create stamp = %u\n", hdr.create_stamp);
+ printf("initial modseq = %"PRIu64"\n", hdr.initial_modseq);
+ printf("compat flags = %x\n", hdr.compat_flags);
+ *modseq_r = hdr.initial_modseq;
+ *version_r = MAIL_TRANSACTION_LOG_HDR_VERSION(&hdr);
+}
+
+static const char *log_record_type(unsigned int type)
+{
+ const char *name;
+
+ switch (type & MAIL_TRANSACTION_TYPE_MASK) {
+ case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT:
+ name = "expunge";
+ break;
+ case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT:
+ name = "expunge-guid";
+ break;
+ case MAIL_TRANSACTION_APPEND:
+ name = "append";
+ break;
+ case MAIL_TRANSACTION_FLAG_UPDATE:
+ name = "flag-update";
+ break;
+ case MAIL_TRANSACTION_HEADER_UPDATE:
+ name = "header-update";
+ break;
+ case MAIL_TRANSACTION_EXT_INTRO:
+ name = "ext-intro";
+ break;
+ case MAIL_TRANSACTION_EXT_RESET:
+ name = "ext-reset";
+ break;
+ case MAIL_TRANSACTION_EXT_HDR_UPDATE:
+ name = "ext-hdr";
+ break;
+ case MAIL_TRANSACTION_EXT_HDR_UPDATE32:
+ name = "ext-hdr32";
+ break;
+ case MAIL_TRANSACTION_EXT_REC_UPDATE:
+ name = "ext-rec";
+ break;
+ case MAIL_TRANSACTION_KEYWORD_UPDATE:
+ name = "keyword-update";
+ break;
+ case MAIL_TRANSACTION_KEYWORD_RESET:
+ name = "keyword-reset";
+ break;
+ case MAIL_TRANSACTION_EXT_ATOMIC_INC:
+ name = "ext-atomic-inc";
+ break;
+ case MAIL_TRANSACTION_MODSEQ_UPDATE:
+ name = "modseq-update";
+ break;
+ case MAIL_TRANSACTION_INDEX_DELETED:
+ name = "index-deleted";
+ break;
+ case MAIL_TRANSACTION_INDEX_UNDELETED:
+ name = "index-undeleted";
+ break;
+ case MAIL_TRANSACTION_BOUNDARY:
+ name = "boundary";
+ break;
+ case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
+ name = "attribute-update";
+ break;
+ default:
+ name = t_strdup_printf("unknown: %x", type);
+ break;
+ }
+
+ if ((type & MAIL_TRANSACTION_EXTERNAL) != 0)
+ name = t_strconcat(name, " (ext)", NULL);
+ if ((type & MAIL_TRANSACTION_SYNC) != 0)
+ name = t_strconcat(name, " (sync)", NULL);
+ return name;
+}
+
+static void print_data(const void *data, size_t size)
+{
+ size_t i;
+
+ for (i = 0; i < size; i++)
+ printf("%02x", ((const unsigned char *)data)[i]);
+ if (size == 4) {
+ const uint32_t *n = (const uint32_t *)data;
+
+ printf(" (dec=%u)", *n);
+ }
+}
+
+static void print_try_uint(const void *data, size_t size)
+{
+ size_t i;
+
+ switch (size) {
+ case 1: {
+ const uint8_t *n = data;
+ printf("%u", *n);
+ break;
+ }
+ case 2: {
+ const uint16_t *n = data;
+ uint32_t n16;
+
+ memcpy(&n16, n, sizeof(n16));
+ printf("%u", n16);
+ break;
+ }
+ case 4: {
+ const uint32_t *n = data;
+ uint32_t n32;
+
+ memcpy(&n32, n, sizeof(n32));
+ printf("%u", n32);
+ break;
+ }
+ case 8: {
+ const uint64_t *n = data;
+ uint64_t n64;
+
+ memcpy(&n64, n, sizeof(n64));
+ printf("%"PRIu64, n64);
+ break;
+ }
+ default:
+ for (i = 0; i < size; i++)
+ printf("%02x", ((const unsigned char *)data)[i]);
+ }
+}
+
+#define HDRF(field) { \
+ #field, offsetof(struct mail_index_header, field), \
+ sizeof(((struct mail_index_header *)0)->field) }
+
+static struct {
+ const char *name;
+ unsigned int offset, size;
+} header_fields[] = {
+ HDRF(minor_version),
+ HDRF(base_header_size),
+ HDRF(header_size),
+ HDRF(record_size),
+ HDRF(compat_flags),
+ HDRF(indexid),
+ HDRF(flags),
+ HDRF(uid_validity),
+ HDRF(next_uid),
+ HDRF(messages_count),
+ HDRF(unused_old_recent_messages_count),
+ HDRF(seen_messages_count),
+ HDRF(deleted_messages_count),
+ HDRF(first_recent_uid),
+ HDRF(first_unseen_uid_lowwater),
+ HDRF(first_deleted_uid_lowwater),
+ HDRF(log_file_seq),
+ HDRF(log_file_tail_offset),
+ HDRF(log_file_head_offset),
+ HDRF(day_stamp)
+};
+
+static void log_header_update(const struct mail_transaction_header_update *u,
+ size_t data_size)
+{
+ const void *data = u + 1;
+ unsigned int offset = u->offset, size = u->size;
+ unsigned int i;
+
+ if (sizeof(*u) + size > data_size) {
+ printf(" - offset = %u, size = %u (too large)\n", offset, size);
+ return;
+ }
+
+ while (size > 0) {
+ /* don't bother trying to handle header updates that include
+ unknown/unexpected fields offsets/sizes */
+ for (i = 0; i < N_ELEMENTS(header_fields); i++) {
+ if (header_fields[i].offset == offset &&
+ header_fields[i].size <= size)
+ break;
+ }
+
+ if (i == N_ELEMENTS(header_fields)) {
+ printf(" - offset = %u, size = %u: ", offset, size);
+ print_data(data, size);
+ printf("\n");
+ break;
+ }
+
+ printf(" - %s = ", header_fields[i].name);
+ print_try_uint(data, header_fields[i].size);
+ printf("\n");
+
+ data = CONST_PTR_OFFSET(data, header_fields[i].size);
+ offset += header_fields[i].size;
+ size -= header_fields[i].size;
+ }
+}
+
+static void log_record_print(const struct mail_transaction_header *hdr,
+ const void *data, size_t data_size,
+ uint64_t *modseq)
+{
+ unsigned int size = mail_index_offset_to_uint32(hdr->size) - sizeof(*hdr);
+
+ switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+ case MAIL_TRANSACTION_EXPUNGE|MAIL_TRANSACTION_EXPUNGE_PROT: {
+ const struct mail_transaction_expunge *exp = data;
+
+ printf(" - uids=");
+ for (; size > 0; size -= sizeof(*exp), exp++) {
+ printf("%u-%u,", exp->uid1, exp->uid2);
+ }
+ printf("\n");
+ break;
+ }
+ case MAIL_TRANSACTION_EXPUNGE_GUID|MAIL_TRANSACTION_EXPUNGE_PROT: {
+ const struct mail_transaction_expunge_guid *exp = data;
+
+ for (; size > 0; size -= sizeof(*exp), exp++) {
+ printf(" - uid=%u (guid ", exp->uid);
+ print_data(exp->guid_128, sizeof(exp->guid_128));
+ printf(")\n");
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_APPEND: {
+ const struct mail_index_record *rec = data;
+
+ printf(" - uids=");
+ for (; size > 0; size -= sizeof(*rec), rec++) {
+ printf("%u", rec->uid);
+ if (rec->flags != 0)
+ printf(" (flags=%x)", rec->flags);
+ printf(",");
+ }
+ printf("\n");
+ break;
+ }
+ case MAIL_TRANSACTION_FLAG_UPDATE: {
+ const struct mail_transaction_flag_update *u = data;
+
+ for (; size > 0; size -= sizeof(*u), u++) {
+ printf(" - uids=%u-%u (flags +%x-%x, modseq_inc_flag=%d)\n",
+ u->uid1, u->uid2, u->add_flags, u->remove_flags, u->modseq_inc_flag);
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_HEADER_UPDATE: {
+ const struct mail_transaction_header_update *u = data;
+
+ log_header_update(u, data_size);
+ break;
+ }
+ case MAIL_TRANSACTION_EXT_INTRO: {
+ const struct mail_transaction_ext_intro *intro = data;
+
+ prev_intro = *intro;
+ printf(" - ext_id = %u\n", intro->ext_id);
+ printf(" - reset_id = %u\n", intro->reset_id);
+ printf(" - hdr_size = %u\n", intro->hdr_size);
+ printf(" - record_size = %u\n", intro->record_size);
+ printf(" - record_align = %u\n", intro->record_align);
+ printf(" - flags = %u\n", intro->flags);
+ printf(" - name_size = %u\n", intro->name_size);
+ if (intro->name_size > 0) {
+ const char *name = (const char *)(intro+1);
+
+ printf(" - name = '%.*s'\n", intro->name_size, name);
+ if (*modseq == 0 && intro->name_size == 6 &&
+ memcmp(name, "modseq", 6) == 0)
+ *modseq = 1;
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_EXT_RESET: {
+ const struct mail_transaction_ext_reset *reset = data;
+
+ printf(" - new_reset_id = %u\n", reset->new_reset_id);
+ printf(" - preserve_data = %u\n", reset->preserve_data);
+ break;
+ }
+ case MAIL_TRANSACTION_EXT_HDR_UPDATE: {
+ const struct mail_transaction_ext_hdr_update *u = data;
+
+ printf(" - offset = %u, size = %u", u->offset, u->size);
+ if (sizeof(*u) + u->size <= data_size) {
+ printf(": ");
+ print_data(u + 1, u->size);
+ } else {
+ printf(" (too large)");
+ }
+ printf("\n");
+ break;
+ }
+ case MAIL_TRANSACTION_EXT_HDR_UPDATE32: {
+ const struct mail_transaction_ext_hdr_update32 *u = data;
+
+ printf(" - offset = %u, size = %u", u->offset, u->size);
+ if (sizeof(*u) + u->size <= data_size) {
+ printf(": ");
+ print_data(u + 1, u->size);
+ } else {
+ printf(" (too large)");
+ }
+ printf("\n");
+ break;
+ }
+ case MAIL_TRANSACTION_EXT_REC_UPDATE: {
+ const struct mail_transaction_ext_rec_update *rec = data, *end;
+ size_t record_size;
+
+ end = CONST_PTR_OFFSET(data, size);
+ record_size = (sizeof(*rec) + prev_intro.record_size + 3) & ~3U;
+ while (rec < end) {
+ printf(" - uid=%u: ", rec->uid);
+ size_t bytes_left = (const char *)end - (const char *)(rec + 1);
+ if (prev_intro.record_size <= bytes_left)
+ print_data(rec + 1, prev_intro.record_size);
+ else
+ printf("(record_size too large)");
+ printf("\n");
+ rec = CONST_PTR_OFFSET(rec, record_size);
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_EXT_ATOMIC_INC: {
+ const struct mail_transaction_ext_atomic_inc *rec = data, *end;
+
+ end = CONST_PTR_OFFSET(data, size);
+ for (; rec < end; rec++) {
+ printf(" - uid=%u: ", rec->uid);
+ if (rec->diff > 0)
+ printf("+%d\n", rec->diff);
+ else
+ printf("%d\n", rec->diff);
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_KEYWORD_UPDATE: {
+ const struct mail_transaction_keyword_update *u = data;
+ const uint32_t *uid;
+ unsigned int uid_offset;
+
+ printf(" - modify=%d, name=%.*s, uids=",
+ u->modify_type, u->name_size, (const char *)(u+1));
+
+ uid_offset = sizeof(*u) + u->name_size +
+ ((u->name_size % 4) == 0 ? 0 : 4 - (u->name_size%4));
+ uid = (const uint32_t *)((const char *)u + uid_offset);
+ size -= uid_offset;
+
+ for (; size > 0; size -= sizeof(*uid)*2, uid += 2) {
+ printf("%u-%u,", uid[0], uid[1]);
+ }
+ printf("\n");
+ break;
+ }
+ case MAIL_TRANSACTION_KEYWORD_RESET: {
+ const struct mail_transaction_keyword_reset *u = data;
+
+ printf(" - uids=");
+ for (; size > 0; size -= sizeof(*u), u++) {
+ printf("%u-%u, ", u->uid1, u->uid2);
+ }
+ printf("\n");
+ break;
+ }
+ case MAIL_TRANSACTION_MODSEQ_UPDATE: {
+ const struct mail_transaction_modseq_update *rec, *end;
+
+ end = CONST_PTR_OFFSET(data, size);
+ for (rec = data; rec < end; rec++) {
+ printf(" - uid=%u modseq=%"PRIu64"\n", rec->uid,
+ ((uint64_t)rec->modseq_high32 << 32) |
+ rec->modseq_low32);
+ }
+ break;
+ }
+ case MAIL_TRANSACTION_INDEX_DELETED:
+ case MAIL_TRANSACTION_INDEX_UNDELETED:
+ break;
+ case MAIL_TRANSACTION_BOUNDARY: {
+ const struct mail_transaction_boundary *rec = data;
+
+ printf(" - size=%u\n", rec->size);
+ break;
+ }
+ case MAIL_TRANSACTION_ATTRIBUTE_UPDATE: {
+ const char *keys = data;
+ const uint32_t *extra;
+ unsigned int i, extra_pos, extra_count = 0;
+
+ for (i = 0; i < size && keys[i] != '\0'; ) {
+ if (keys[i] == '+')
+ extra_count++;
+ extra_count++;
+ i += strlen(keys+i) + 1;
+ }
+ if (i % sizeof(uint32_t) != 0)
+ i += sizeof(uint32_t) - i%sizeof(uint32_t);
+ extra = (const void *)(keys+i);
+
+ if ((size-i) != extra_count*sizeof(uint32_t)) {
+ printf(" - broken entry\n");
+ break;
+ }
+
+ extra_pos = 0;
+ for (i = 0; i < size && keys[i] != '\0'; ) {
+ printf(" - %s: %s/%s : timestamp=%s",
+ keys[i] == '+' ? "add" : keys[i] == '-' ? "remove" : "?",
+ keys[i+1] == 'p' ? "private" :
+ keys[i+1] == 's' ? "shared" : "?error?",
+ keys+i+2, unixdate2str(extra[extra_pos++]));
+ if (keys[i] == '+')
+ printf(" value_len=%u", extra[extra_pos++]);
+ printf("\n");
+ i += strlen(keys+i) + 1;
+ }
+
+ break;
+ }
+ default:
+ break;
+ }
+}
+
+static int dump_record(struct istream *input, uint64_t *modseq,
+ unsigned int version)
+{
+ struct mail_transaction_header hdr;
+ unsigned int hdr_size;
+ const unsigned char *data;
+ size_t size;
+ int ret;
+
+ ret = i_stream_read_bytes(input, &data, &size, sizeof(hdr));
+ if (ret < 0 && input->stream_errno != 0)
+ i_fatal("read() failed: %s", i_stream_get_error(input));
+ if (ret <= 0) {
+ if (size == 0)
+ return 0;
+ i_fatal("rec hdr read() %zu != %zu",
+ size, sizeof(hdr));
+ }
+ memcpy(&hdr, data, sizeof(hdr));
+
+ hdr_size = mail_index_offset_to_uint32(hdr.size);
+ if (hdr_size < sizeof(hdr)) {
+ printf("record: offset=%"PRIuUOFF_T", "
+ "type=%s, size=broken (%x)\n",
+ input->v_offset, log_record_type(hdr.type), hdr.size);
+ return 0;
+ }
+
+ printf("record: offset=%"PRIuUOFF_T", type=%s, size=%u",
+ input->v_offset, log_record_type(hdr.type), hdr_size);
+
+ i_stream_skip(input, sizeof(hdr));
+ size_t data_size = hdr_size - sizeof(hdr);
+ ret = i_stream_read_bytes(input, &data, &size, data_size);
+ if (ret < 0 && input->stream_errno != 0)
+ i_fatal("read() failed: %s", i_stream_get_error(input));
+ if (ret <= 0) {
+ i_fatal("rec data read() %zu != %zu",
+ size, data_size);
+ }
+
+ uint64_t prev_modseq = *modseq;
+ mail_transaction_update_modseq(&hdr, data, modseq, version);
+ if (*modseq > prev_modseq)
+ printf(", modseq=%"PRIu64, *modseq);
+ printf("\n");
+
+ log_record_print(&hdr, data, data_size, modseq);
+ i_stream_skip(input, data_size);
+ return 1;
+}
+
+static void cmd_dump_log(const char *path, const char *const *args ATTR_UNUSED)
+{
+ struct istream *input;
+ uint64_t modseq;
+ unsigned int version;
+ int ret;
+
+ input = i_stream_create_file(path, SIZE_MAX);
+ dump_hdr(input, &modseq, &version);
+ do {
+ T_BEGIN {
+ ret = dump_record(input, &modseq, version);
+ } T_END;
+ } while (ret > 0);
+ i_stream_unref(&input);
+}
+
+static bool test_dump_log(const char *path)
+{
+ struct mail_transaction_log_header hdr;
+ const char *p;
+ bool ret = FALSE;
+ int fd;
+
+ p = strrchr(path, '/');
+ if (p == NULL)
+ p = path;
+ p = strstr(p, ".log");
+ if (p == NULL || !(p[4] == '\0' || p[4] == '.'))
+ return FALSE;
+
+ fd = open(path, O_RDONLY);
+ if (fd == -1)
+ return FALSE;
+
+ if (read(fd, &hdr, sizeof(hdr)) >= MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE &&
+ hdr.major_version == MAIL_TRANSACTION_LOG_MAJOR_VERSION &&
+ hdr.hdr_size >= MAIL_TRANSACTION_LOG_HEADER_MIN_SIZE)
+ ret = TRUE;
+ i_close_fd(&fd);
+ return ret;
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_log = {
+ "log",
+ test_dump_log,
+ cmd_dump_log
+};
diff --git a/src/doveadm/doveadm-dump-mailboxlog.c b/src/doveadm/doveadm-dump-mailboxlog.c
new file mode 100644
index 0000000..8ff5a54
--- /dev/null
+++ b/src/doveadm/doveadm-dump-mailboxlog.c
@@ -0,0 +1,114 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "hex-binary.h"
+#include "mailbox-log.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <time.h>
+
+static int dump_record(int fd)
+{
+ off_t offset;
+ ssize_t ret;
+ struct mailbox_log_record rec;
+ time_t timestamp;
+
+ offset = lseek(fd, 0, SEEK_CUR);
+
+ ret = read(fd, &rec, sizeof(rec));
+ if (ret == 0)
+ return 0;
+
+ if (ret != sizeof(rec)) {
+ i_fatal("rec read() %zu != %zu",
+ ret, sizeof(rec));
+ }
+
+ printf("#%"PRIuUOFF_T": ", offset);
+ switch (rec.type) {
+ case MAILBOX_LOG_RECORD_DELETE_MAILBOX:
+ printf("delete-mailbox");
+ break;
+ case MAILBOX_LOG_RECORD_DELETE_DIR:
+ printf("delete-dir");
+ break;
+ case MAILBOX_LOG_RECORD_RENAME:
+ printf("rename");
+ break;
+ case MAILBOX_LOG_RECORD_SUBSCRIBE:
+ printf("subscribe");
+ break;
+ case MAILBOX_LOG_RECORD_UNSUBSCRIBE:
+ printf("unsubscribe");
+ break;
+ case MAILBOX_LOG_RECORD_CREATE_DIR:
+ printf("create-dir");
+ break;
+ }
+ printf(" %s", binary_to_hex(rec.mailbox_guid,
+ sizeof(rec.mailbox_guid)));
+
+ timestamp = be32_to_cpu_unaligned(rec.timestamp);
+ printf(" (%s)\n", unixdate2str(timestamp));
+ return 1;
+}
+
+static void
+cmd_dump_mailboxlog(const char *path, const char *const *args ATTR_UNUSED)
+{
+ int fd, ret;
+
+ fd = open(path, O_RDONLY);
+ if (fd < 0)
+ i_fatal("open(%s) failed: %m", path);
+
+ do {
+ T_BEGIN {
+ ret = dump_record(fd);
+ } T_END;
+ } while (ret > 0);
+ i_close_fd(&fd);
+}
+
+static bool test_dump_mailboxlog(const char *path)
+{
+ const char *p;
+ int fd;
+ struct mailbox_log_record rec;
+ bool ret = FALSE;
+
+ p = strrchr(path, '.');
+ if (p == NULL || strcmp(p, ".log") != 0)
+ return FALSE;
+
+ fd = open(path, O_RDONLY);
+ if (fd == -1)
+ return FALSE;
+
+ if (read(fd, &rec, sizeof(rec)) == sizeof(rec) &&
+ rec.padding[0] == 0 && rec.padding[1] == 0 && rec.padding[2] == 0) {
+ enum mailbox_log_record_type type = rec.type;
+ switch (type) {
+ case MAILBOX_LOG_RECORD_DELETE_MAILBOX:
+ case MAILBOX_LOG_RECORD_DELETE_DIR:
+ case MAILBOX_LOG_RECORD_RENAME:
+ case MAILBOX_LOG_RECORD_SUBSCRIBE:
+ case MAILBOX_LOG_RECORD_UNSUBSCRIBE:
+ case MAILBOX_LOG_RECORD_CREATE_DIR:
+ ret = TRUE;
+ break;
+ }
+ }
+ i_close_fd(&fd);
+ return ret;
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_mailboxlog = {
+ "mailboxlog",
+ test_dump_mailboxlog,
+ cmd_dump_mailboxlog
+};
diff --git a/src/doveadm/doveadm-dump-thread.c b/src/doveadm/doveadm-dump-thread.c
new file mode 100644
index 0000000..026b07f
--- /dev/null
+++ b/src/doveadm/doveadm-dump-thread.c
@@ -0,0 +1,139 @@
+/* Copyright (c) 2007-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mmap-util.h"
+#include "mail-index-private.h"
+#include "mail-index-strmap.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/stat.h>
+
+static uint32_t max_likely_index;
+
+static size_t dump_hdr(const struct mail_index_strmap_header *hdr)
+{
+ printf("version = %u\n", hdr->version);
+ printf("uid validity = %u\n", hdr->uid_validity);
+ return sizeof(*hdr);
+}
+
+static int dump_record(const uint8_t **p, const uint8_t *end, uint32_t *uid)
+{
+ uint32_t uid_diff, n, i, count, crc32, idx;
+ size_t size;
+
+ /* <uid diff> <n> <crc32>*count <str_idx>*count */
+ if (mail_index_unpack_num(p, end, &uid_diff) < 0)
+ return -1;
+ *uid += uid_diff;
+
+ if (mail_index_unpack_num(p, end, &n) < 0)
+ return -1;
+ printf(" - uid %u: n=%u\n", *uid, n);
+
+ count = n < 2 ? n + 1 : n;
+ size = sizeof(crc32)*count + sizeof(idx)*count;
+ if (*p + size > end)
+ return -1;
+ for (i = 0; i < count; i++) {
+ if (i == 0)
+ printf(" - message-id: ");
+ else if (i == 1) {
+ if (n == 1)
+ printf(" - in-reply-to: ");
+ else
+ printf(" - references[1]: ");
+ } else {
+ printf(" - references[%u]: ", i);
+ }
+ memcpy(&crc32, *p + sizeof(crc32)*i, sizeof(crc32));
+ memcpy(&idx, *p + sizeof(crc32)*count + sizeof(idx)*i, sizeof(idx));
+ printf("crc32=%08x index=%u\n", crc32, idx);
+ if (idx > max_likely_index)
+ printf(" - index probably broken\n");
+ }
+ *p += size;
+ return 0;
+}
+
+static int dump_block(const uint8_t *data, const uint8_t *end, uint32_t *uid)
+{
+ const uint8_t *p;
+ uint32_t block_size;
+
+ if (data + 4 >= end)
+ return -1;
+
+ memcpy(&block_size, data, sizeof(block_size));
+ block_size = mail_index_offset_to_uint32(block_size) >> 2;
+ printf(" - block_size=%u\n", block_size);
+ if (block_size == 0) {
+ /* finished */
+ return -1;
+ }
+ if (data + sizeof(block_size) + block_size > end) {
+ printf(" - broken!\n");
+ return -1;
+ }
+ p = data + sizeof(block_size);
+ end = p + block_size;
+
+ *uid += 1;
+ while (p != end) {
+ if (dump_record(&p, end, uid) < 0) {
+ printf(" - broken\n");
+ return -1;
+ }
+ }
+ return p - data;
+}
+
+static void
+cmd_dump_thread(const char *path, const char *const *args ATTR_UNUSED)
+{
+ unsigned int pos;
+ const void *map, *end;
+ struct stat st;
+ uint32_t uid;
+ int fd, ret;
+
+ fd = open(path, O_RDONLY);
+ if (fd < 0)
+ i_fatal("open(%s) failed: %m", path);
+
+ if (fstat(fd, &st) < 0)
+ i_fatal("fstat(%s) failed: %m", path);
+ max_likely_index = (st.st_size / 8) * 2;
+
+ map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+ if (map == MAP_FAILED)
+ i_fatal("mmap() failed: %m");
+ end = CONST_PTR_OFFSET(map, st.st_size);
+ pos = dump_hdr(map);
+ uid = 0;
+ do {
+ printf("block at offset %u:\n", pos);
+ T_BEGIN {
+ ret = dump_block(CONST_PTR_OFFSET(map, pos), end, &uid);
+ pos += ret;
+ } T_END;
+ } while (ret > 0);
+ i_close_fd(&fd);
+}
+
+static bool test_dump_thread(const char *path)
+{
+ const char *p;
+
+ p = strrchr(path, '.');
+ return p != NULL && strcmp(p, ".thread") == 0;
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_thread = {
+ "thread",
+ test_dump_thread,
+ cmd_dump_thread
+};
diff --git a/src/doveadm/doveadm-dump.c b/src/doveadm/doveadm-dump.c
new file mode 100644
index 0000000..eb08e96
--- /dev/null
+++ b/src/doveadm/doveadm-dump.c
@@ -0,0 +1,162 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "istream-multiplex.h"
+#include "doveadm.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+static ARRAY(const struct doveadm_cmd_dump *) dumps;
+
+void doveadm_dump_register(const struct doveadm_cmd_dump *dump)
+{
+ array_push_back(&dumps, &dump);
+}
+
+static const struct doveadm_cmd_dump *
+dump_find_name(const char *name)
+{
+ const struct doveadm_cmd_dump *dump;
+
+ array_foreach_elem(&dumps, dump) {
+ if (strcmp(dump->name, name) == 0)
+ return dump;
+ }
+ return NULL;
+}
+
+static const struct doveadm_cmd_dump *
+dump_find_test(const char *path)
+{
+ const struct doveadm_cmd_dump *dump;
+
+ array_foreach_elem(&dumps, dump) {
+ if (dump->test != NULL && dump->test(path))
+ return dump;
+ }
+ return NULL;
+}
+
+static void cmd_dump(struct doveadm_cmd_context *cctx)
+{
+ const struct doveadm_cmd_dump *dump;
+ const char *path, *type = NULL, *const *args = NULL;
+ const char *no_args = NULL;
+
+ if (!doveadm_cmd_param_str(cctx, "path", &path))
+ help_ver2(&doveadm_cmd_dump);
+ (void)doveadm_cmd_param_str(cctx, "type", &type);
+ (void)doveadm_cmd_param_array(cctx, "args", &args);
+
+ dump = type != NULL ? dump_find_name(type) : dump_find_test(path);
+ if (dump == NULL) {
+ if (type != NULL) {
+ print_dump_types();
+ i_fatal_status(EX_USAGE, "Unknown type: %s", type);
+ } else {
+ i_fatal_status(EX_DATAERR,
+ "Can't autodetect file type: %s", path);
+ }
+ } else {
+ if (type == NULL)
+ printf("Detected file type: %s\n", dump->name);
+ }
+ dump->cmd(path, args != NULL ? args : &no_args);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_dump = {
+ .name = "dump",
+ .cmd = cmd_dump,
+ .usage = "[-t <type>] <path> [<type-specific args>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('t', "type", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+static void
+cmd_dump_multiplex(const char *path, const char *const *args ATTR_UNUSED)
+{
+ const unsigned int channels_count = 256;
+ struct istream *file_input, *channels[channels_count];
+ const unsigned char *data;
+ size_t size;
+ unsigned int i;
+
+ file_input = i_stream_create_file(path, IO_BLOCK_SIZE);
+ /* A bit kludgy: istream-multiplex returns 0 if a wrong channel is
+ being read from. This causes a panic with blocking istreams.
+ Work around this by assuming that the file istream isn't blocking. */
+ file_input->blocking = FALSE;
+ channels[0] = i_stream_create_multiplex(file_input, IO_BLOCK_SIZE);
+ i_stream_unref(&file_input);
+
+ for (i = 1; i < channels_count; i++)
+ channels[i] = i_stream_multiplex_add_channel(channels[0], i);
+
+ bool have_input;
+ do {
+ have_input = FALSE;
+ for (i = 0; i < channels_count; i++) {
+ if (i_stream_read_more(channels[i], &data, &size) > 0) {
+ printf("CHANNEL %u: %zu bytes:\n", i, size);
+ fwrite(data, 1, size, stdout);
+ printf("\n");
+ have_input = TRUE;
+ i_stream_skip(channels[i], size);
+ }
+ }
+ } while (have_input);
+
+ if (channels[0]->stream_errno != 0)
+ i_error("read() failed: %s", i_stream_get_error(channels[0]));
+ for (i = 0; i < channels_count; i++)
+ i_stream_unref(&channels[i]);
+}
+
+struct doveadm_cmd_dump doveadm_cmd_dump_multiplex = {
+ "multiplex",
+ NULL,
+ cmd_dump_multiplex
+};
+
+static const struct doveadm_cmd_dump *dumps_builtin[] = {
+ &doveadm_cmd_dump_dbox,
+ &doveadm_cmd_dump_index,
+ &doveadm_cmd_dump_log,
+ &doveadm_cmd_dump_mailboxlog,
+ &doveadm_cmd_dump_thread,
+ &doveadm_cmd_dump_zlib,
+ &doveadm_cmd_dump_dcrypt_file,
+ &doveadm_cmd_dump_dcrypt_key,
+ &doveadm_cmd_dump_multiplex,
+};
+
+void print_dump_types(void)
+{
+ unsigned int i;
+
+ fprintf(stderr, "Available dump types: %s", dumps_builtin[0]->name);
+ for (i = 1; i < N_ELEMENTS(dumps_builtin); i++)
+ fprintf(stderr, " %s", dumps_builtin[i]->name);
+ fprintf(stderr, "\n");
+}
+
+void doveadm_dump_init(void)
+{
+ unsigned int i;
+
+ i_array_init(&dumps, N_ELEMENTS(dumps_builtin) + 8);
+ for (i = 0; i < N_ELEMENTS(dumps_builtin); i++)
+ doveadm_dump_register(dumps_builtin[i]);
+}
+
+void doveadm_dump_deinit(void)
+{
+ array_free(&dumps);
+}
diff --git a/src/doveadm/doveadm-dump.h b/src/doveadm/doveadm-dump.h
new file mode 100644
index 0000000..5776260
--- /dev/null
+++ b/src/doveadm/doveadm-dump.h
@@ -0,0 +1,27 @@
+#ifndef DOVEADM_DUMP_H
+#define DOVEADM_DUMP_H
+
+#include "doveadm.h"
+
+struct doveadm_cmd_dump {
+ const char *name;
+ bool (*test)(const char *path);
+ void (*cmd)(const char *path, const char *const *args);
+};
+
+extern struct doveadm_cmd_dump doveadm_cmd_dump_dbox;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_index;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_log;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_mailboxlog;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_thread;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_zlib;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_file;
+extern struct doveadm_cmd_dump doveadm_cmd_dump_dcrypt_key;
+
+void doveadm_dump_register(const struct doveadm_cmd_dump *dump);
+
+void print_dump_types(void);
+void doveadm_dump_init(void);
+void doveadm_dump_deinit(void);
+
+#endif
diff --git a/src/doveadm/doveadm-fs.c b/src/doveadm/doveadm-fs.c
new file mode 100644
index 0000000..17f6129
--- /dev/null
+++ b/src/doveadm/doveadm-fs.c
@@ -0,0 +1,608 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "md5.h"
+#include "sha2.h"
+#include "hash-method.h"
+#include "hex-binary.h"
+#include "istream.h"
+#include "ostream.h"
+#include "iostream-ssl.h"
+#include "fs-api.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+static void fs_cmd_help(struct doveadm_cmd_context *cctx) ATTR_NORETURN;
+static void cmd_fs_delete(struct doveadm_cmd_context *cctx);
+
+static struct fs *
+cmd_fs_init(struct doveadm_cmd_context *cctx)
+{
+ struct ssl_iostream_settings ssl_set;
+ struct fs_settings fs_set;
+ struct fs *fs;
+ const char *fs_driver, *fs_args, *error;
+
+ if (!doveadm_cmd_param_str(cctx, "fs-driver", &fs_driver) ||
+ !doveadm_cmd_param_str(cctx, "fs-args", &fs_args))
+ fs_cmd_help(cctx);
+
+ doveadm_get_ssl_settings(&ssl_set, pool_datastack_create());
+ ssl_set.verbose = doveadm_debug;
+ i_zero(&fs_set);
+ fs_set.ssl_client_set = &ssl_set;
+ fs_set.temp_dir = doveadm_settings->mail_temp_dir;
+ fs_set.base_dir = doveadm_settings->base_dir;
+ fs_set.debug = doveadm_debug;
+
+ if (fs_init(fs_driver, fs_args, &fs_set, &fs, &error) < 0)
+ i_fatal("fs_init() failed: %s", error);
+ return fs;
+}
+
+static void cmd_fs_get(struct doveadm_cmd_context *cctx)
+{
+ struct fs *fs;
+ struct fs_file *file;
+ struct istream *input;
+ const char *path;
+ const unsigned char *data;
+ size_t size;
+ ssize_t ret;
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_PAGER);
+ doveadm_print_header("content", "content", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_str(cctx, "path", &path))
+ fs_cmd_help(cctx);
+
+ file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+ input = fs_read_stream(file, IO_BLOCK_SIZE);
+ while ((ret = i_stream_read_more(input, &data, &size)) > 0) {
+ doveadm_print_stream(data, size);
+ i_stream_skip(input, size);
+ }
+ doveadm_print_stream("", 0);
+ i_assert(ret == -1);
+ if (input->stream_errno == ENOENT) {
+ i_error("%s doesn't exist: %s", fs_file_path(file),
+ i_stream_get_error(input));
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else if (input->stream_errno != 0) {
+ i_error("read(%s) failed: %s", fs_file_path(file),
+ i_stream_get_error(input));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ i_stream_unref(&input);
+ fs_file_deinit(&file);
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_put(struct doveadm_cmd_context *cctx)
+{
+ struct fs *fs;
+ enum fs_properties props;
+ const char *hash_str, *src_path, *dest_path;
+ struct fs_file *file;
+ struct istream *input;
+ struct ostream *output;
+ buffer_t *hash = NULL;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_str(cctx, "input-path", &src_path) ||
+ !doveadm_cmd_param_str(cctx, "path", &dest_path))
+ fs_cmd_help(cctx);
+ if (doveadm_cmd_param_str(cctx, "hash", &hash_str)) {
+ hash = t_buffer_create(32);
+ if (hex_to_binary(optarg, hash) < 0)
+ i_fatal("Invalid -h parameter: Hash not in hex");
+ }
+
+ file = fs_file_init(fs, dest_path, FS_OPEN_MODE_REPLACE);
+ props = fs_get_properties(fs);
+ if (hash == NULL)
+ ;
+ else if (hash->used == hash_method_md5.digest_size) {
+ if ((props & FS_PROPERTY_WRITE_HASH_MD5) == 0)
+ i_fatal("fs backend doesn't support MD5 hashes");
+ fs_write_set_hash(file,
+ hash_method_lookup(hash_method_md5.name), hash->data);
+ } else if (hash->used == hash_method_sha256.digest_size) {
+ if ((props & FS_PROPERTY_WRITE_HASH_SHA256) == 0)
+ i_fatal("fs backend doesn't support SHA256 hashes");
+ fs_write_set_hash(file,
+ hash_method_lookup(hash_method_sha256.name), hash->data);
+ }
+
+ output = fs_write_stream(file);
+ input = i_stream_create_file(src_path, IO_BLOCK_SIZE);
+ o_stream_nsend_istream(output, input);
+ i_stream_destroy(&input);
+ if (fs_write_stream_finish(file, &output) < 0) {
+ i_error("fs_write_stream_finish() failed: %s",
+ fs_file_last_error(file));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ fs_file_deinit(&file);
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_copy(struct doveadm_cmd_context *cctx)
+{
+ struct fs *fs;
+ struct fs_file *src_file, *dest_file;
+ const char *src_path, *dest_path;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_str(cctx, "source-path", &src_path) ||
+ !doveadm_cmd_param_str(cctx, "destination-path", &dest_path))
+ fs_cmd_help(cctx);
+
+ src_file = fs_file_init(fs, src_path, FS_OPEN_MODE_READONLY);
+ dest_file = fs_file_init(fs, dest_path, FS_OPEN_MODE_REPLACE);
+ if (fs_copy(src_file, dest_file) == 0) ;
+ else if (errno == ENOENT) {
+ i_error("%s doesn't exist: %s", src_path,
+ fs_file_last_error(dest_file));
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else {
+ i_error("fs_copy(%s, %s) failed: %s",
+ src_path, dest_path, fs_file_last_error(dest_file));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ fs_file_deinit(&src_file);
+ fs_file_deinit(&dest_file);
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_stat(struct doveadm_cmd_context *cctx)
+{
+ struct fs *fs;
+ struct fs_file *file;
+ struct stat st;
+ const char *path;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_str(cctx, "path", &path))
+ fs_cmd_help(cctx);
+
+ file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ doveadm_print_formatted_set_format("%{path} size=%{size}");
+ doveadm_print_header_simple("path");
+ doveadm_print_header("size", "size", DOVEADM_PRINT_HEADER_FLAG_NUMBER);
+
+ if (fs_stat(file, &st) == 0) {
+ doveadm_print(fs_file_path(file));
+ doveadm_print(dec2str(st.st_size));
+ } else if (errno == ENOENT) {
+ i_error("%s doesn't exist: %s", fs_file_path(file),
+ fs_file_last_error(file));
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else {
+ i_error("fs_stat(%s) failed: %s",
+ fs_file_path(file), fs_file_last_error(file));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ fs_file_deinit(&file);
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_metadata(struct doveadm_cmd_context *cctx)
+{
+ struct fs *fs;
+ struct fs_file *file;
+ const struct fs_metadata *m;
+ const ARRAY_TYPE(fs_metadata) *metadata;
+ const char *path;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_str(cctx, "path", &path))
+ fs_cmd_help(cctx);
+
+ file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ doveadm_print_formatted_set_format("%{key}=%{value}\n");
+ doveadm_print_header_simple("key");
+ doveadm_print_header_simple("value");
+
+ if (fs_get_metadata(file, &metadata) == 0) {
+ array_foreach(metadata, m) {
+ doveadm_print(m->key);
+ doveadm_print(m->value);
+ }
+ } else if (errno == ENOENT) {
+ i_error("%s doesn't exist: %s", fs_file_path(file),
+ fs_file_last_error(file));
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ } else {
+ i_error("fs_stat(%s) failed: %s",
+ fs_file_path(file), fs_file_last_error(file));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ fs_file_deinit(&file);
+ fs_deinit(&fs);
+}
+
+struct fs_delete_ctx {
+ struct fs *fs;
+ const char *path_prefix;
+
+ unsigned int files_count;
+ struct fs_file **files;
+};
+
+static int cmd_fs_delete_ctx_run(struct fs_delete_ctx *ctx)
+{
+ unsigned int i;
+ int ret = 0;
+
+ for (i = 0; i < ctx->files_count; i++) {
+ if (ctx->files[i] == NULL)
+ ;
+ else if (fs_delete(ctx->files[i]) == 0)
+ fs_file_deinit(&ctx->files[i]);
+ else if (errno == EAGAIN) {
+ if (ret == 0)
+ ret = 1;
+ } else if (errno == ENOENT) {
+ i_error("%s doesn't exist: %s", fs_file_path(ctx->files[i]),
+ fs_file_last_error(ctx->files[i]));
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ ret = -1;
+ } else {
+ i_error("fs_delete(%s) failed: %s",
+ fs_file_path(ctx->files[i]),
+ fs_file_last_error(ctx->files[i]));
+ doveadm_exit_code = EX_TEMPFAIL;
+ ret = -1;
+ }
+ }
+ return ret;
+}
+
+static int doveadm_fs_delete_async_fname(struct fs_delete_ctx *ctx,
+ const char *fname)
+{
+ unsigned int i;
+ int ret;
+
+ for (i = 0; i < ctx->files_count; i++) {
+ if (ctx->files[i] != NULL)
+ continue;
+
+ ctx->files[i] = fs_file_init(ctx->fs,
+ t_strdup_printf("%s%s", ctx->path_prefix, fname),
+ FS_OPEN_MODE_READONLY | FS_OPEN_FLAG_ASYNC |
+ FS_OPEN_FLAG_ASYNC_NOQUEUE);
+ fname = NULL;
+ break;
+ }
+ if ((ret = cmd_fs_delete_ctx_run(ctx)) < 0)
+ return -1;
+ if (fname != NULL) {
+ if (ret > 0)
+ fs_wait_async(ctx->fs);
+ return doveadm_fs_delete_async_fname(ctx, fname);
+ }
+ return 0;
+}
+
+static void doveadm_fs_delete_async_finish(struct fs_delete_ctx *ctx)
+{
+ unsigned int i;
+
+ while (doveadm_exit_code == 0 && cmd_fs_delete_ctx_run(ctx) > 0) {
+ fs_wait_async(ctx->fs);
+ }
+ for (i = 0; i < ctx->files_count; i++) {
+ fs_file_deinit(&ctx->files[i]);
+ }
+}
+
+static void
+cmd_fs_delete_dir_recursive(struct fs *fs, unsigned int async_count,
+ const char *path_prefix)
+{
+ struct fs_iter *iter;
+ ARRAY_TYPE(const_string) fnames;
+ struct fs_delete_ctx ctx;
+ const char *fname, *error;
+ int ret;
+
+ i_zero(&ctx);
+ ctx.fs = fs;
+ ctx.path_prefix = path_prefix;
+ ctx.files_count = I_MAX(async_count, 1);
+ ctx.files = t_new(struct fs_file *, ctx.files_count);
+
+ /* delete subdirs first. all fs backends can't handle recursive
+ lookups, so save the list first. */
+ t_array_init(&fnames, 8);
+ iter = fs_iter_init(fs, path_prefix, FS_ITER_FLAG_DIRS);
+ while ((fname = fs_iter_next(iter)) != NULL) {
+ /* append "/" so that if FS_PROPERTY_DIRECTORIES is set,
+ we'll include the "/" suffix in the filename when deleting
+ it. */
+ fname = t_strconcat(fname, "/", NULL);
+ array_push_back(&fnames, &fname);
+ }
+ if (fs_iter_deinit(&iter, &error) < 0) {
+ i_error("fs_iter_deinit(%s) failed: %s",
+ path_prefix, error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ array_foreach_elem(&fnames, fname) T_BEGIN {
+ cmd_fs_delete_dir_recursive(fs, async_count,
+ t_strdup_printf("%s%s", path_prefix, fname));
+ } T_END;
+
+ /* delete files. again because we're doing this asynchronously finish
+ the iteration first. */
+ if ((fs_get_properties(fs) & FS_PROPERTY_DIRECTORIES) != 0) {
+ /* we need to explicitly delete also the directories */
+ } else {
+ array_clear(&fnames);
+ }
+ iter = fs_iter_init(fs, path_prefix, 0);
+ while ((fname = fs_iter_next(iter)) != NULL) {
+ fname = t_strdup(fname);
+ array_push_back(&fnames, &fname);
+ }
+ if (fs_iter_deinit(&iter, &error) < 0) {
+ i_error("fs_iter_deinit(%s) failed: %s",
+ path_prefix, error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+
+ array_foreach_elem(&fnames, fname) {
+ T_BEGIN {
+ ret = doveadm_fs_delete_async_fname(&ctx, fname);
+ } T_END;
+ if (ret < 0)
+ break;
+ }
+ doveadm_fs_delete_async_finish(&ctx);
+}
+
+static void cmd_fs_delete_recursive_path(struct fs *fs, const char *path,
+ unsigned int async_count)
+{
+ struct fs_file *file;
+ size_t path_len;
+
+ path_len = strlen(path);
+ if (path_len > 0 && path[path_len-1] != '/')
+ path = t_strconcat(path, "/", NULL);
+
+ cmd_fs_delete_dir_recursive(fs, async_count, path);
+ if ((fs_get_properties(fs) & FS_PROPERTY_DIRECTORIES) != 0) {
+ /* delete the root itself */
+ file = fs_file_init(fs, path, FS_OPEN_MODE_READONLY);
+ if (fs_delete(file) < 0) {
+ i_error("fs_delete(%s) failed: %s",
+ fs_file_path(file), fs_file_last_error(file));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ fs_file_deinit(&file);
+ }
+}
+
+static void
+cmd_fs_delete_recursive(struct doveadm_cmd_context *cctx,
+ unsigned int async_count)
+{
+ struct fs *fs;
+ const char *const *paths;
+ unsigned int i;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_array(cctx, "path", &paths))
+ fs_cmd_help(cctx);
+
+ for (i = 0; paths[i] != NULL; i++)
+ cmd_fs_delete_recursive_path(fs, paths[i], async_count);
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_delete_paths(struct doveadm_cmd_context *cctx,
+ unsigned int async_count)
+{
+ struct fs *fs;
+ struct fs_delete_ctx ctx;
+ const char *const *paths;
+ unsigned int i;
+ int ret;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_array(cctx, "path", &paths))
+ fs_cmd_help(cctx);
+
+ i_zero(&ctx);
+ ctx.fs = fs;
+ ctx.path_prefix = "";
+ ctx.files_count = I_MAX(async_count, 1);
+ ctx.files = t_new(struct fs_file *, ctx.files_count);
+
+ for (i = 0; paths[i] != NULL; i++) {
+ T_BEGIN {
+ ret = doveadm_fs_delete_async_fname(&ctx, paths[i]);
+ } T_END;
+ if (ret < 0)
+ break;
+ }
+ doveadm_fs_delete_async_finish(&ctx);
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_delete(struct doveadm_cmd_context *cctx)
+{
+ bool recursive = FALSE;
+ int64_t async_count = 0;
+
+ (void)doveadm_cmd_param_bool(cctx, "recursive", &recursive);
+ (void)doveadm_cmd_param_int64(cctx, "max-parallel", &async_count);
+
+ if (recursive)
+ cmd_fs_delete_recursive(cctx, async_count);
+ else
+ cmd_fs_delete_paths(cctx, async_count);
+}
+
+static void cmd_fs_iter_full(struct doveadm_cmd_context *cctx,
+ enum fs_iter_flags flags)
+{
+ struct fs *fs;
+ struct fs_iter *iter;
+ const char *path, *fname, *error;
+ bool b;
+
+ if (doveadm_cmd_param_bool(cctx, "no-cache", &b) && b)
+ flags |= FS_ITER_FLAG_NOCACHE;
+ if (doveadm_cmd_param_bool(cctx, "object-ids", &b) && b)
+ flags |= FS_ITER_FLAG_OBJECTIDS;
+
+ fs = cmd_fs_init(cctx);
+ if (!doveadm_cmd_param_str(cctx, "path", &path))
+ fs_cmd_help(cctx);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ doveadm_print_formatted_set_format("%{path}\n");
+ doveadm_print_header_simple("path");
+
+ iter = fs_iter_init(fs, path, flags);
+ while ((fname = fs_iter_next(iter)) != NULL) {
+ doveadm_print(fname);
+ }
+ if (fs_iter_deinit(&iter, &error) < 0) {
+ i_error("fs_iter_deinit(%s) failed: %s", path, error);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ fs_deinit(&fs);
+}
+
+static void cmd_fs_iter(struct doveadm_cmd_context *cctx)
+{
+ cmd_fs_iter_full(cctx, 0);
+}
+
+static void cmd_fs_iter_dirs(struct doveadm_cmd_context *cctx)
+{
+ cmd_fs_iter_full(cctx, FS_ITER_FLAG_DIRS);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_fs[] = {
+{
+ .name = "fs get",
+ .cmd = cmd_fs_get,
+ .usage = "<fs-driver> <fs-args> <path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs put",
+ .cmd = cmd_fs_put,
+ .usage = "[-h <hash>] <fs-driver> <fs-args> <input path> <path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('h', "hash", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "input-path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs copy",
+ .cmd = cmd_fs_copy,
+ .usage = "<fs-driver> <fs-args> <source path> <dest path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "source-path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "destination-path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs stat",
+ .cmd = cmd_fs_stat,
+ .usage = "<fs-driver> <fs-args> <path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs metadata",
+ .cmd = cmd_fs_metadata,
+ .usage = "<fs-driver> <fs-args> <path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs delete",
+ .cmd = cmd_fs_delete,
+ .usage = "[-R] [-n <count>] <fs-driver> <fs-args> <path> [<path> ...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('R', "recursive", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('n', "max-parallel", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs iter",
+ .cmd = cmd_fs_iter,
+ .usage = "[--no-cache] [--object-ids] <fs-driver> <fs-args> <path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('C', "no-cache", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('O', "object-ids", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "fs iter-dirs",
+ .cmd = cmd_fs_iter_dirs,
+ .usage = "<fs-driver> <fs-args> <path>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "fs-driver", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "fs-args", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "path", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+static void fs_cmd_help(struct doveadm_cmd_context *cctx)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_fs); i++) {
+ if (doveadm_cmd_fs[i].cmd == cctx->cmd->cmd)
+ help_ver2(&doveadm_cmd_fs[i]);
+ }
+ i_unreached();
+}
+
+void doveadm_register_fs_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_fs); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_fs[i]);
+}
diff --git a/src/doveadm/doveadm-instance.c b/src/doveadm/doveadm-instance.c
new file mode 100644
index 0000000..cd3d55e
--- /dev/null
+++ b/src/doveadm/doveadm-instance.c
@@ -0,0 +1,155 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "master-instance.h"
+#include "master-service-settings.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <signal.h>
+
+extern struct doveadm_cmd_ver2 doveadm_cmd_instance[];
+
+static void instance_cmd_help(const struct doveadm_cmd_ver2 *cmd) ATTR_NORETURN;
+
+static bool pid_file_read(const char *path)
+{
+ char buf[32];
+ int fd;
+ ssize_t ret;
+ pid_t pid;
+ bool found = FALSE;
+
+ fd = open(path, O_RDONLY);
+ if (fd == -1) {
+ if (errno != ENOENT)
+ i_error("open(%s) failed: %m", path);
+ return FALSE;
+ }
+
+ ret = read(fd, buf, sizeof(buf));
+ if (ret < 0)
+ i_error("read(%s) failed: %m", path);
+ else if (ret > 0 && buf[ret-1] == '\n') {
+ buf[ret-1] = '\0';
+ if (str_to_pid(buf, &pid) == 0) {
+ found = !(pid == getpid() ||
+ (kill(pid, 0) < 0 && errno == ESRCH));
+ }
+ }
+ i_close_fd(&fd);
+ return found;
+}
+
+static void cmd_instance_list(struct doveadm_cmd_context *cctx)
+{
+ struct master_instance_list *list;
+ struct master_instance_list_iter *iter;
+ const struct master_instance *inst;
+ const char *instance_path, *pidfile_path;
+ bool show_config = FALSE;
+ const char *name = NULL;
+
+ (void)doveadm_cmd_param_bool(cctx, "show-config", &show_config);
+ (void)doveadm_cmd_param_str(cctx, "name", &name);
+
+ if (!show_config) {
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("path", "path", DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ doveadm_print_header_simple("name");
+ doveadm_print_header_simple("last used");
+ doveadm_print_header_simple("running");
+ }
+
+ instance_path = t_strconcat(service_set->state_dir,
+ "/"MASTER_INSTANCE_FNAME, NULL);
+ list = master_instance_list_init(instance_path);
+ iter = master_instance_list_iterate_init(list);
+ while ((inst = master_instance_iterate_list_next(iter)) != NULL) {
+ if (name != NULL && strcmp(name, inst->name) != 0)
+ continue;
+
+ if (show_config) {
+ printf("%s\n", inst->config_path == NULL ? "" :
+ inst->config_path);
+ continue;
+ }
+ doveadm_print(inst->base_dir);
+ doveadm_print(inst->name);
+ doveadm_print(unixdate2str(inst->last_used));
+ pidfile_path = t_strconcat(inst->base_dir, "/master.pid", NULL);
+ if (pid_file_read(pidfile_path))
+ doveadm_print("yes");
+ else
+ doveadm_print("no");
+ }
+ master_instance_iterate_list_deinit(&iter);
+ master_instance_list_deinit(&list);
+}
+
+static void cmd_instance_remove(struct doveadm_cmd_context *cctx)
+{
+ struct master_instance_list *list;
+ const struct master_instance *inst;
+ const char *base_dir, *instance_path, *name;
+ int ret;
+
+ if (!doveadm_cmd_param_str(cctx, "name", &name))
+ instance_cmd_help(cctx->cmd);
+
+ instance_path = t_strconcat(service_set->state_dir,
+ "/"MASTER_INSTANCE_FNAME, NULL);
+ list = master_instance_list_init(instance_path);
+ inst = master_instance_list_find_by_name(list, name);
+ base_dir = inst != NULL ? inst->base_dir : name;
+ if ((ret = master_instance_list_remove(list, base_dir)) < 0) {
+ i_error("Failed to remove instance");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (ret == 0) {
+ i_error("Instance already didn't exist");
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ }
+ master_instance_list_deinit(&list);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_instance[] = {
+{
+ .name = "instance list",
+ .cmd = cmd_instance_list,
+ .usage = "[-c] [<name>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('c', "show-config", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "instance remove",
+ .cmd = cmd_instance_remove,
+ .usage = "<name> | <base dir>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+static void instance_cmd_help(const struct doveadm_cmd_ver2 *cmd)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_instance); i++) {
+ if (doveadm_cmd_instance[i].cmd == cmd->cmd)
+ help_ver2(&doveadm_cmd_instance[i]);
+ }
+ i_unreached();
+}
+
+void doveadm_register_instance_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_instance); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_instance[i]);
+}
diff --git a/src/doveadm/doveadm-kick.c b/src/doveadm/doveadm-kick.c
new file mode 100644
index 0000000..a0b06f3
--- /dev/null
+++ b/src/doveadm/doveadm-kick.c
@@ -0,0 +1,235 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "net.h"
+#include "hash.h"
+#include "doveadm.h"
+#include "doveadm-who.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <signal.h>
+
+struct kick_user {
+ const char *username;
+ bool kick_me; /* true if username and/or ip[/mask] matches.
+ ignored when the -f switch is given. */
+};
+
+struct kick_pid {
+ pid_t pid;
+ ARRAY(struct kick_user) users;
+ bool kick;
+};
+
+struct kick_context {
+ struct who_context who;
+ HASH_TABLE(void *, struct kick_pid *) pids;
+ enum doveadm_client_type conn_type;
+ bool force_kick;
+ ARRAY(const char *) kicked_users;
+};
+
+static void
+kick_aggregate_line(struct who_context *_ctx, const struct who_line *line)
+{
+ struct kick_context *ctx = (struct kick_context *)_ctx;
+ const bool user_match = who_line_filter_match(line, &ctx->who.filter);
+ struct kick_pid *k_pid;
+ struct kick_user new_user, *user;
+
+ i_zero(&new_user);
+
+ k_pid = hash_table_lookup(ctx->pids, POINTER_CAST(line->pid));
+ if (k_pid == NULL) {
+ k_pid = p_new(ctx->who.pool, struct kick_pid, 1);
+ k_pid->pid = line->pid;
+ p_array_init(&k_pid->users, ctx->who.pool, 5);
+ hash_table_insert(ctx->pids, POINTER_CAST(line->pid), k_pid);
+ }
+
+ array_foreach_modifiable(&k_pid->users, user) {
+ if (strcmp(line->username, user->username) == 0) {
+ if (user_match)
+ user->kick_me = TRUE;
+ return;
+ }
+ }
+ new_user.username = p_strdup(ctx->who.pool, line->username);
+ new_user.kick_me = user_match;
+ array_push_back(&k_pid->users, &new_user);
+}
+
+static bool
+kick_pid_want_kicked(struct kick_context *ctx, const struct kick_pid *k_pid,
+ bool *show_warning)
+{
+ unsigned int kick_count = 0;
+ const struct kick_user *user;
+
+ if (array_count(&k_pid->users) == 1) {
+ user = array_front(&k_pid->users);
+ if (!user->kick_me)
+ return FALSE;
+ } else {
+ array_foreach(&k_pid->users, user) {
+ if (user->kick_me)
+ kick_count++;
+ }
+ if (kick_count == 0)
+ return FALSE;
+ if (kick_count < array_count(&k_pid->users) &&
+ !ctx->force_kick) {
+ array_foreach(&k_pid->users, user) {
+ if (!user->kick_me) {
+ array_push_back(&ctx->kicked_users,
+ &user->username);
+ }
+ }
+ *show_warning = TRUE;
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
+static void
+kick_print_kicked(struct kick_context *ctx, const bool show_warning)
+{
+ unsigned int i, count;
+ const char *const *users;
+ bool cli = (ctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI);
+
+ if (array_count(&ctx->kicked_users) == 0) {
+ if (cli)
+ printf("no users kicked\n");
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ return;
+ }
+
+ if (cli) {
+ if (show_warning) {
+ printf("warning: other connections would also be "
+ "kicked from following users:\n");
+ } else {
+ printf("kicked connections from the following users:\n");
+ }
+ }
+
+ array_sort(&ctx->kicked_users, i_strcmp_p);
+ users = array_get(&ctx->kicked_users, &count);
+ doveadm_print(users[0]);
+ for (i = 1; i < count; i++) {
+ if (strcmp(users[i-1], users[i]) != 0)
+ doveadm_print(users[i]);
+ }
+
+ doveadm_print_flush();
+
+ if (cli)
+ printf("\n");
+
+ if (show_warning)
+ printf("Use the '-f' option to enforce the disconnect.\n");
+}
+
+static void kick_users(struct kick_context *ctx)
+{
+ bool show_enforce_warning = FALSE;
+ struct hash_iterate_context *iter;
+ void *key;
+ struct kick_pid *k_pid;
+ const struct kick_user *user;
+
+ p_array_init(&ctx->kicked_users, ctx->who.pool, 10);
+
+ iter = hash_table_iterate_init(ctx->pids);
+ while (hash_table_iterate(iter, ctx->pids, &key, &k_pid)) {
+ if (kick_pid_want_kicked(ctx, k_pid, &show_enforce_warning))
+ k_pid->kick = TRUE;
+ }
+ hash_table_iterate_deinit(&iter);
+
+ if (show_enforce_warning) {
+ kick_print_kicked(ctx, show_enforce_warning);
+ return;
+ }
+
+ iter = hash_table_iterate_init(ctx->pids);
+ while (hash_table_iterate(iter, ctx->pids, &key, &k_pid)) {
+ if (!k_pid->kick)
+ continue;
+
+ if (kill(k_pid->pid, SIGTERM) < 0 && errno != ESRCH) {
+ fprintf(stderr, "kill(%s, SIGTERM) failed: %m\n",
+ dec2str(k_pid->pid));
+ } else {
+ array_foreach(&k_pid->users, user) {
+ array_push_back(&ctx->kicked_users,
+ &user->username);
+ }
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+
+ kick_print_kicked(ctx, show_enforce_warning);
+}
+
+static void cmd_kick(struct doveadm_cmd_context *cctx)
+{
+ const char *const *masks;
+ struct kick_context ctx;
+
+ i_zero(&ctx);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx.who.anvil_path)))
+ ctx.who.anvil_path = t_strconcat(doveadm_settings->base_dir, "/anvil", NULL);
+ (void)doveadm_cmd_param_bool(cctx, "force", &(ctx.force_kick));
+ if (!doveadm_cmd_param_array(cctx, "mask", &masks)) {
+ doveadm_exit_code = EX_USAGE;
+ i_error("user and/or ip[/bits] must be specified.");
+ return;
+ }
+ ctx.conn_type = cctx->conn_type;
+ if (ctx.conn_type != DOVEADM_CONNECTION_TYPE_CLI) {
+ /* force-kick is a pretty ugly option. its output can't be
+ nicely translated to an API reply. it also wouldn't be very
+ useful in scripts, only for preventing a new admin from
+ accidentally kicking too many users. it's also useful only
+ in a non-recommended setup where processes are handling
+ multiple connections. so for now we'll preserve the option
+ for CLI, but always do a force-kick with non-CLI. */
+ ctx.force_kick = TRUE;
+ }
+ ctx.who.pool = pool_alloconly_create("kick pids", 10240);
+ hash_table_create_direct(&ctx.pids, ctx.who.pool, 0);
+
+ if (who_parse_args(&ctx.who, masks)!=0) {
+ hash_table_destroy(&ctx.pids);
+ pool_unref(&ctx.who.pool);
+ return;
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ doveadm_print_formatted_set_format("%{result} ");
+ doveadm_print_header_simple("result");
+
+ who_lookup(&ctx.who, kick_aggregate_line);
+ kick_users(&ctx);
+
+ hash_table_destroy(&ctx.pids);
+ pool_unref(&ctx.who.pool);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_kick_ver2 = {
+ .name = "kick",
+ .cmd = cmd_kick,
+ .usage = "[-a <anvil socket path>] <user mask>[|]<ip/bits>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a',"socket-path",CMD_PARAM_STR,0)
+DOVEADM_CMD_PARAM('f',"force",CMD_PARAM_BOOL,0)
+DOVEADM_CMD_PARAM('\0',"mask",CMD_PARAM_ARRAY,CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-log.c b/src/doveadm/doveadm-log.c
new file mode 100644
index 0000000..2654d9b
--- /dev/null
+++ b/src/doveadm/doveadm-log.c
@@ -0,0 +1,406 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "hash.h"
+#include "str.h"
+#include "strescape.h"
+#include "time-util.h"
+#include "master-service-private.h"
+#include "master-service-settings.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <signal.h>
+#include <sys/stat.h>
+
+#define LAST_LOG_TYPE LOG_TYPE_PANIC
+#define TEST_LOG_MSG_PREFIX "This is Dovecot's "
+#define LOG_ERRORS_FNAME "log-errors"
+#define LOG_TIMESTAMP_FORMAT "%b %d %H:%M:%S"
+
+static void ATTR_NULL(2)
+cmd_log_test(struct doveadm_cmd_context *cctx ATTR_UNUSED)
+{
+ struct failure_context ctx;
+ unsigned int i;
+
+ master_service->log_initialized = FALSE;
+ master_service->flags |= MASTER_SERVICE_FLAG_DONT_LOG_TO_STDERR;
+ master_service_init_log(master_service);
+
+ i_zero(&ctx);
+ for (i = 0; i < LAST_LOG_TYPE; i++) {
+ const char *prefix = failure_log_type_prefixes[i];
+
+ /* add timestamp so that syslog won't just write
+ "repeated message" text */
+ ctx.type = i;
+ i_log_type(&ctx, TEST_LOG_MSG_PREFIX"%s log (%u)",
+ t_str_lcase(t_strcut(prefix, ':')),
+ (unsigned int)ioloop_time);
+ }
+}
+
+static void cmd_log_reopen(struct doveadm_cmd_context *cctx ATTR_UNUSED)
+{
+ doveadm_master_send_signal(SIGUSR1);
+}
+
+struct log_find_file {
+ const char *path;
+ uoff_t size;
+
+ /* 1 << enum log_type */
+ unsigned int mask;
+};
+
+struct log_find_context {
+ pool_t pool;
+ HASH_TABLE(char *, struct log_find_file *) files;
+};
+
+static void cmd_log_find_add(struct log_find_context *ctx,
+ const char *path, enum log_type type)
+{
+ struct log_find_file *file;
+ char *key;
+
+ file = hash_table_lookup(ctx->files, path);
+ if (file == NULL) {
+ file = p_new(ctx->pool, struct log_find_file, 1);
+ file->path = key = p_strdup(ctx->pool, path);
+ hash_table_insert(ctx->files, key, file);
+ }
+
+ file->mask |= 1 << type;
+}
+
+static void
+cmd_log_find_syslog_files(struct log_find_context *ctx, const char *path)
+{
+ struct log_find_file *file;
+ DIR *dir;
+ struct dirent *d;
+ struct stat st;
+ char *key;
+ string_t *full_path;
+ size_t dir_len;
+
+ dir = opendir(path);
+ if (dir == NULL) {
+ i_error("opendir(%s) failed: %m", path);
+ return;
+ }
+
+ full_path = t_str_new(256);
+ str_append(full_path, path);
+ str_append_c(full_path, '/');
+ dir_len = str_len(full_path);
+
+ while ((d = readdir(dir)) != NULL) {
+ if (d->d_name[0] == '.')
+ continue;
+
+ str_truncate(full_path, dir_len);
+ str_append(full_path, d->d_name);
+ if (stat(str_c(full_path), &st) < 0)
+ continue;
+
+ if (S_ISDIR(st.st_mode)) {
+ /* recursively go through all subdirectories */
+ cmd_log_find_syslog_files(ctx, str_c(full_path));
+ } else if (hash_table_lookup(ctx->files,
+ str_c(full_path)) == NULL) {
+ file = p_new(ctx->pool, struct log_find_file, 1);
+ file->size = st.st_size;
+ file->path = key =
+ p_strdup(ctx->pool, str_c(full_path));
+ hash_table_insert(ctx->files, key, file);
+ }
+ }
+
+ (void)closedir(dir);
+}
+
+static bool log_type_find(const char *str, enum log_type *type_r)
+{
+ unsigned int i;
+ size_t len = strlen(str);
+
+ for (i = 0; i < LAST_LOG_TYPE; i++) {
+ if (strncasecmp(str, failure_log_type_prefixes[i], len) == 0 &&
+ failure_log_type_prefixes[i][len] == ':') {
+ *type_r = i;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static void cmd_log_find_syslog_file_messages(struct log_find_file *file)
+{
+ struct istream *input;
+ const char *line, *p;
+ enum log_type type;
+ int fd;
+
+ fd = open(file->path, O_RDONLY);
+ if (fd == -1)
+ return;
+
+ input = i_stream_create_fd_autoclose(&fd, 1024);
+ i_stream_seek(input, file->size);
+ while ((line = i_stream_read_next_line(input)) != NULL) {
+ p = strstr(line, TEST_LOG_MSG_PREFIX);
+ if (p == NULL)
+ continue;
+ p += strlen(TEST_LOG_MSG_PREFIX);
+
+ /* <type> log */
+ T_BEGIN {
+ if (log_type_find(t_strcut(p, ' '), &type))
+ file->mask |= 1 << type;
+ } T_END;
+ }
+ i_stream_destroy(&input);
+}
+
+static void cmd_log_find_syslog_messages(struct log_find_context *ctx)
+{
+ struct hash_iterate_context *iter;
+ struct stat st;
+ char *key;
+ struct log_find_file *file;
+
+ iter = hash_table_iterate_init(ctx->files);
+ while (hash_table_iterate(iter, ctx->files, &key, &file)) {
+ if (stat(file->path, &st) < 0 ||
+ (uoff_t)st.st_size <= file->size)
+ continue;
+
+ cmd_log_find_syslog_file_messages(file);
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+static void
+cmd_log_find_syslog(struct log_find_context *ctx,
+ struct doveadm_cmd_context *cctx)
+{
+ const char *log_dir;
+ struct stat st;
+
+ if (doveadm_cmd_param_str(cctx, "log-dir", &log_dir))
+ ;
+ else if (stat("/var/log", &st) == 0 && S_ISDIR(st.st_mode))
+ log_dir = "/var/log";
+ else if (stat("/var/adm", &st) == 0 && S_ISDIR(st.st_mode))
+ log_dir = "/var/adm";
+ else
+ return;
+
+ printf("Looking for log files from %s\n", log_dir);
+ cmd_log_find_syslog_files(ctx, log_dir);
+ cmd_log_test(cctx);
+
+ /* give syslog some time to write the messages to files */
+ sleep(1);
+ cmd_log_find_syslog_messages(ctx);
+}
+
+static void cmd_log_find(struct doveadm_cmd_context *cctx)
+{
+ const struct master_service_settings *set;
+ const char *log_file_path;
+ struct log_find_context ctx;
+ unsigned int i;
+
+ i_zero(&ctx);
+ ctx.pool = pool_alloconly_create("log file", 1024*32);
+ hash_table_create(&ctx.files, ctx.pool, 0, str_hash, strcmp);
+
+ /* first get the paths that we know are used */
+ set = master_service_settings_get(master_service);
+ log_file_path = set->log_path;
+ if (strcmp(log_file_path, "syslog") == 0)
+ log_file_path = "";
+ if (*log_file_path != '\0') {
+ cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_WARNING);
+ cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_ERROR);
+ cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_FATAL);
+ }
+
+ if (strcmp(set->info_log_path, "syslog") != 0) {
+ if (*set->info_log_path != '\0')
+ log_file_path = set->info_log_path;
+ if (*log_file_path != '\0')
+ cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_INFO);
+ }
+
+ if (strcmp(set->debug_log_path, "syslog") != 0) {
+ if (*set->debug_log_path != '\0')
+ log_file_path = set->debug_log_path;
+ if (*log_file_path != '\0')
+ cmd_log_find_add(&ctx, log_file_path, LOG_TYPE_DEBUG);
+ }
+
+ if (*set->log_path == '\0' ||
+ strcmp(set->log_path, "syslog") == 0 ||
+ strcmp(set->info_log_path, "syslog") == 0 ||
+ strcmp(set->debug_log_path, "syslog") == 0) {
+ /* at least some logs were logged via syslog */
+ cmd_log_find_syslog(&ctx, cctx);
+ }
+
+ /* print them */
+ for (i = 0; i < LAST_LOG_TYPE; i++) {
+ struct hash_iterate_context *iter;
+ char *key;
+ struct log_find_file *file;
+ bool found = FALSE;
+
+ iter = hash_table_iterate_init(ctx.files);
+ while (hash_table_iterate(iter, ctx.files, &key, &file)) {
+ if ((file->mask & (1 << i)) != 0) {
+ printf("%s%s\n", failure_log_type_prefixes[i],
+ file->path);
+ found = TRUE;
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+
+ if (!found)
+ printf("%sNot found\n", failure_log_type_prefixes[i]);
+ }
+ hash_table_destroy(&ctx.files);
+ pool_unref(&ctx.pool);
+}
+
+static const char *t_cmd_log_error_trim(const char *orig)
+{
+ size_t pos;
+
+ /* Trim whitespace from suffix and remove ':' if it exists */
+ for (pos = strlen(orig); pos > 0; pos--) {
+ if (orig[pos-1] != ' ') {
+ if (orig[pos-1] == ':')
+ pos--;
+ break;
+ }
+ }
+ return orig[pos] == '\0' ? orig : t_strndup(orig, pos);
+}
+
+static void cmd_log_error_write(const char *const *args, time_t min_timestamp)
+{
+ /* <type> <timestamp> <prefix> <text> */
+ const char *type_prefix = "?";
+ unsigned int type;
+ time_t t;
+
+ /* find type's prefix */
+ for (type = 0; type < LOG_TYPE_COUNT; type++) {
+ if (strcmp(args[0], failure_log_type_names[type]) == 0) {
+ type_prefix = failure_log_type_prefixes[type];
+ break;
+ }
+ }
+
+ if (str_to_time(args[1], &t) < 0) {
+ i_error("Invalid timestamp: %s", args[1]);
+ t = 0;
+ }
+ if (t >= min_timestamp) {
+ doveadm_print(t_strflocaltime(LOG_TIMESTAMP_FORMAT, t));
+ doveadm_print(t_cmd_log_error_trim(args[2]));
+ doveadm_print(t_cmd_log_error_trim(type_prefix));
+ doveadm_print(args[3]);
+ }
+}
+
+static void cmd_log_errors(struct doveadm_cmd_context *cctx)
+{
+ struct istream *input;
+ const char *path, *line, *const *args;
+ time_t min_timestamp = 0;
+ int64_t since_int64;
+ int fd;
+
+ if (doveadm_cmd_param_int64(cctx, "since", &since_int64))
+ min_timestamp = since_int64;
+
+ path = t_strconcat(doveadm_settings->base_dir,
+ "/"LOG_ERRORS_FNAME, NULL);
+ fd = net_connect_unix(path);
+ if (fd == -1)
+ i_fatal("net_connect_unix(%s) failed: %m", path);
+ net_set_nonblock(fd, FALSE);
+
+ input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ doveadm_print_formatted_set_format("%{timestamp} %{type}: %{prefix}: %{text}\n");
+
+ doveadm_print_header_simple("timestamp");
+ doveadm_print_header_simple("prefix");
+ doveadm_print_header_simple("type");
+ doveadm_print_header_simple("text");
+
+ while ((line = i_stream_read_next_line(input)) != NULL) T_BEGIN {
+ args = t_strsplit_tabescaped(line);
+ if (str_array_length(args) == 4)
+ cmd_log_error_write(args, min_timestamp);
+ else {
+ i_error("Invalid input from log: %s", line);
+ doveadm_exit_code = EX_PROTOCOL;
+ }
+ } T_END;
+ i_stream_destroy(&input);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_log[] = {
+{
+ .name = "log test",
+ .cmd = cmd_log_test,
+ .usage = "",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "log reopen",
+ .cmd = cmd_log_reopen,
+ .usage = "",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "log find",
+ .cmd = cmd_log_find,
+ .usage = "[<dir>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "log-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "log errors",
+ .usage = "[-s <min_timestamp>]",
+ .cmd = cmd_log_errors,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('s', "since", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+void doveadm_register_log_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_log); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_log[i]);
+}
diff --git a/src/doveadm/doveadm-mail-altmove.c b/src/doveadm/doveadm-mail-altmove.c
new file mode 100644
index 0000000..7bcf7b0
--- /dev/null
+++ b/src/doveadm/doveadm-mail-altmove.c
@@ -0,0 +1,162 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-index.h"
+#include "mail-storage.h"
+#include "mail-namespace.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+struct altmove_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ bool reverse;
+};
+
+static int
+cmd_altmove_box(struct doveadm_mail_cmd_context *ctx,
+ const struct mailbox_info *info,
+ struct mail_search_args *search_args, bool reverse)
+{
+ struct doveadm_mail_iter *iter;
+ struct mail *mail;
+ enum modify_type modify_type =
+ !reverse ? MODIFY_ADD : MODIFY_REMOVE;
+
+ if (doveadm_mail_iter_init(ctx, info, search_args, 0, NULL, 0,
+ &iter) < 0)
+ return -1;
+
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ if (doveadm_debug) {
+ i_debug("altmove: box=%s uid=%u",
+ info->vname, mail->uid);
+ }
+ mail_update_flags(mail, modify_type,
+ (enum mail_flags)MAIL_INDEX_MAIL_FLAG_BACKEND);
+ }
+ return doveadm_mail_iter_deinit_sync(&iter);
+}
+
+static int
+ns_purge(struct doveadm_mail_cmd_context *ctx, struct mail_namespace *ns,
+ struct mail_storage *storage)
+{
+ if (mail_storage_purge(storage) < 0) {
+ i_error("Purging namespace '%s' failed: %s", ns->prefix,
+ mail_storage_get_last_internal_error(storage, NULL));
+ doveadm_mail_failed_storage(ctx, storage);
+ return -1;
+ }
+ return 0;
+}
+
+static int
+cmd_altmove_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct altmove_cmd_context *ctx = (struct altmove_cmd_context *)_ctx;
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ struct mail_namespace *ns, *prev_ns = NULL;
+ ARRAY(struct mail_storage *) purged_storages;
+ struct mail_storage *const *storages, *ns_storage, *prev_storage = NULL;
+ unsigned int i, count;
+ int ret = 0;
+
+ t_array_init(&purged_storages, 8);
+ iter = doveadm_mailbox_list_iter_init(_ctx, user, _ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ ns_storage = mail_namespace_get_default_storage(info->ns);
+ if (ns_storage != prev_storage) {
+ if (prev_storage != NULL) {
+ if (ns_purge(_ctx, prev_ns, prev_storage) < 0)
+ ret = -1;
+ array_push_back(&purged_storages,
+ &prev_storage);
+ }
+ prev_storage = ns_storage;
+ prev_ns = info->ns;
+ }
+ if (cmd_altmove_box(_ctx, info, _ctx->search_args, ctx->reverse) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+
+ if (prev_storage != NULL) {
+ if (ns_purge(_ctx, prev_ns, prev_storage) < 0)
+ ret = -1;
+ array_push_back(&purged_storages, &prev_storage);
+ }
+
+ /* make sure all private storages have been purged */
+ storages = array_get(&purged_storages, &count);
+ for (ns = user->namespaces; ns != NULL; ns = ns->next) {
+ if (ns->type != MAIL_NAMESPACE_TYPE_PRIVATE)
+ continue;
+
+ ns_storage = mail_namespace_get_default_storage(ns);
+ for (i = 0; i < count; i++) {
+ if (ns_storage == storages[i])
+ break;
+ }
+ if (i == count) {
+ if (ns_purge(_ctx, ns, ns_storage) < 0)
+ ret = -1;
+ array_push_back(&purged_storages, &ns_storage);
+ storages = array_get(&purged_storages, &count);
+ }
+ }
+ return ret;
+}
+
+static void cmd_altmove_init(struct doveadm_mail_cmd_context *ctx,
+ const char *const args[])
+{
+ if (args[0] == NULL)
+ doveadm_mail_help_name("altmove");
+ ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static bool
+cmd_mailbox_altmove_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct altmove_cmd_context *ctx = (struct altmove_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'r':
+ ctx->reverse = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_altmove_alloc(void)
+{
+ struct altmove_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct altmove_cmd_context);
+ ctx->ctx.getopt_args = "r";
+ ctx->ctx.v.parse_arg = cmd_mailbox_altmove_parse_arg;
+ ctx->ctx.v.init = cmd_altmove_init;
+ ctx->ctx.v.run = cmd_altmove_run;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_altmove_ver2 = {
+ .name = "altmove",
+ .mail_cmd = cmd_altmove_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-r] <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('r', "reverse", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-batch.c b/src/doveadm/doveadm-mail-batch.c
new file mode 100644
index 0000000..2dbbb89
--- /dev/null
+++ b/src/doveadm/doveadm-mail-batch.c
@@ -0,0 +1,186 @@
+/* Copyright (c) 2012-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "doveadm-mail.h"
+
+#include <unistd.h>
+
+struct batch_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ ARRAY(struct doveadm_mail_cmd_context *) commands;
+};
+
+static int cmd_batch_prerun(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_storage_service_user *service_user,
+ const char **error_r)
+{
+ struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx;
+ struct doveadm_mail_cmd_context *cmd;
+ int ret = 0;
+
+ array_foreach_elem(&ctx->commands, cmd) {
+ if (cmd->v.prerun != NULL &&
+ cmd->v.prerun(cmd, service_user, error_r) < 0) {
+ ret = -1;
+ break;
+ }
+ }
+ return ret;
+}
+
+static int cmd_batch_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx;
+ struct doveadm_mail_cmd_context *cmd;
+ int ret = 0;
+
+ array_foreach_elem(&ctx->commands, cmd) {
+ cmd->cur_mail_user = user;
+ const char *reason_code =
+ event_reason_code_prefix("doveadm", "cmd_",
+ cmd->cmd->name);
+ struct event_reason *reason = event_reason_begin(reason_code);
+ ret = cmd->v.run(cmd, user);
+ event_reason_end(&reason);
+ if (ret < 0) {
+ i_assert(cmd->exit_code != 0);
+ _ctx->exit_code = cmd->exit_code;
+ break;
+ }
+ cmd->cur_mail_user = NULL;
+ }
+ return ret;
+}
+
+static void
+cmd_batch_add(struct batch_cmd_context *batchctx,
+ int argc, const char *const *argv)
+{
+ struct doveadm_mail_cmd_context *subctx;
+ const struct doveadm_cmd_ver2 *cmd_ver2;
+ const struct doveadm_mail_cmd *cmd;
+ const char *getopt_args;
+ int c;
+
+ cmd_ver2 = doveadm_cmd_find_with_args_ver2(argv[0], &argc, &argv);
+ if (cmd_ver2 == NULL)
+ i_fatal_status(EX_USAGE, "doveadm batch: '%s' mail command doesn't exist", argv[0]);
+
+ struct doveadm_mail_cmd *dyncmd =
+ p_new(batchctx->ctx.pool, struct doveadm_mail_cmd, 1);
+ dyncmd->usage_args = cmd_ver2->usage;
+ dyncmd->name = cmd_ver2->name;
+ dyncmd->alloc = cmd_ver2->mail_cmd;
+ cmd = dyncmd;
+
+ subctx = doveadm_mail_cmd_init(cmd, doveadm_settings);
+ subctx->full_args = argv + 1;
+ subctx->service_flags |= batchctx->ctx.service_flags;
+
+ i_getopt_reset();
+ getopt_args = subctx->getopt_args != NULL ? subctx->getopt_args : "";
+ while ((c = getopt(argc, (void *)argv, getopt_args)) > 0) {
+ if (subctx->v.parse_arg == NULL ||
+ !subctx->v.parse_arg(subctx, c))
+ doveadm_mail_help(cmd);
+ }
+ argv += optind;
+ if (argv[0] != NULL && cmd->usage_args == NULL) {
+ i_fatal_status(EX_USAGE, "doveadm %s: Unknown parameter: %s",
+ cmd->name, argv[0]);
+ }
+ subctx->args = argv;
+ if (subctx->v.preinit != NULL)
+ subctx->v.preinit(subctx);
+ array_push_back(&batchctx->commands, &subctx);
+}
+
+static void
+cmd_batch_preinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ const char *const *args = _ctx->args;
+ struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx;
+ ARRAY_TYPE(const_string) sep_args;
+ const char *sep = args[0];
+ unsigned int i, start;
+ int argc;
+ const char *const *argv;
+
+ if (sep == NULL || args[1] == NULL)
+ doveadm_mail_help_name("batch");
+ args++;
+
+ p_array_init(&ctx->commands, _ctx->pool, 8);
+ p_array_init(&sep_args, _ctx->pool, 16);
+ for (i = start = 0;; i++) {
+ if (args[i] != NULL && strcmp(args[i], sep) != 0) {
+ array_push_back(&sep_args, &args[i]);
+ continue;
+ }
+ if (i > start) {
+ (void)array_append_space(&sep_args);
+ argc = i - start;
+ argv = array_idx(&sep_args, start);
+ cmd_batch_add(ctx, argc, argv);
+ start = i+1;
+ }
+ if (args[i] == NULL)
+ break;
+ }
+ (void)array_append_space(&sep_args);
+}
+
+static void
+cmd_batch_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[] ATTR_UNUSED)
+{
+ struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx;
+ struct doveadm_mail_cmd_context *cmd;
+ struct batch_cmd_context *subctx;
+
+ array_foreach_elem(&ctx->commands, cmd) {
+ subctx = (struct batch_cmd_context *)cmd;
+ subctx->ctx.storage_service = _ctx->storage_service;
+ if (subctx->ctx.v.init != NULL)
+ subctx->ctx.v.init(&subctx->ctx, subctx->ctx.args);
+ }
+}
+
+static void cmd_batch_deinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ struct batch_cmd_context *ctx = (struct batch_cmd_context *)_ctx;
+ struct doveadm_mail_cmd_context *cmd;
+
+ array_foreach_elem(&ctx->commands, cmd) {
+ doveadm_mail_cmd_deinit(cmd);
+ doveadm_mail_cmd_free(cmd);
+ }
+}
+
+static struct doveadm_mail_cmd_context *cmd_batch_alloc(void)
+{
+ struct batch_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct batch_cmd_context);
+ ctx->ctx.getopt_args = "+"; /* disable processing -args in the middle */
+ ctx->ctx.v.preinit = cmd_batch_preinit;
+ ctx->ctx.v.init = cmd_batch_init;
+ ctx->ctx.v.prerun = cmd_batch_prerun;
+ ctx->ctx.v.run = cmd_batch_run;
+ ctx->ctx.v.deinit = cmd_batch_deinit;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_batch = {
+ .name = "batch",
+ .mail_cmd = cmd_batch_alloc,
+ .usage = "<sep> <cmd1> [<sep> <cmd2> [..]]",
+ .flags = CMD_FLAG_NO_UNORDERED_OPTIONS,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "separator", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-copymove.c b/src/doveadm/doveadm-mail-copymove.c
new file mode 100644
index 0000000..7e26ed8
--- /dev/null
+++ b/src/doveadm/doveadm-mail-copymove.c
@@ -0,0 +1,224 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-storage.h"
+#include "mail-namespace.h"
+#include "doveadm-print.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+#include <stdio.h>
+
+struct copy_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+
+ const char *source_username;
+ struct mail_storage_service_user *source_service_user;
+ struct mail_user *source_user;
+
+ const char *destname;
+ bool move;
+};
+
+static int
+cmd_copy_box(struct copy_cmd_context *ctx, struct mailbox *destbox,
+ const struct mailbox_info *info)
+{
+ struct doveadm_mail_iter *iter;
+ struct mailbox_transaction_context *desttrans;
+ struct mail_save_context *save_ctx;
+ struct mail *mail;
+ int ret = 0, ret2;
+
+ if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args, 0,
+ NULL, 0, &iter) < 0)
+ return -1;
+
+ /* use a separately committed transaction for each mailbox.
+ this guarantees that mails aren't expunged without actually having
+ been copied. */
+ desttrans = mailbox_transaction_begin(destbox,
+ MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+ ctx->ctx.transaction_flags, __func__);
+
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ save_ctx = mailbox_save_alloc(desttrans);
+ mailbox_save_copy_flags(save_ctx, mail);
+ if (ctx->move)
+ ret2 = mailbox_move(&save_ctx, mail);
+ else
+ ret2 = mailbox_copy(&save_ctx, mail);
+ if (ret2 < 0) {
+ i_error("%s message UID %u from '%s' failed: %s",
+ ctx->move ? "Moving" : "Copying",
+ mail->uid, info->vname,
+ mailbox_get_last_internal_error(destbox, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, destbox);
+ ret = -1;
+ }
+ }
+
+ if (mailbox_transaction_commit(&desttrans) < 0) {
+ i_error("Committing %s mails failed: %s",
+ ctx->move ? "moved" : "copied",
+ mailbox_get_last_internal_error(destbox, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, destbox);
+ /* rollback expunges */
+ doveadm_mail_iter_deinit_rollback(&iter);
+ ret = -1;
+ } else {
+ if (doveadm_mail_iter_deinit_sync(&iter) < 0)
+ ret = -1;
+ }
+ return ret;
+}
+
+static void
+cmd_copy_alloc_source_user(struct copy_cmd_context *ctx)
+{
+ struct mail_storage_service_input input;
+ const char *error;
+
+ input = ctx->ctx.storage_service_input;
+ input.username = ctx->source_username;
+
+ mail_storage_service_io_deactivate_user(ctx->ctx.cur_service_user);
+ if (mail_storage_service_lookup_next(ctx->ctx.storage_service, &input,
+ &ctx->source_service_user,
+ &ctx->source_user,
+ &error) < 0)
+ i_fatal("Couldn't lookup user %s: %s", input.username, error);
+ mail_storage_service_io_deactivate_user(ctx->source_service_user);
+ mail_storage_service_io_activate_user(ctx->ctx.cur_service_user);
+}
+
+static int
+cmd_copy_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct copy_cmd_context *ctx = (struct copy_cmd_context *)_ctx;
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ struct mail_user *src_user;
+ struct mail_namespace *ns;
+ struct mailbox *destbox;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ if (ctx->source_username != NULL && ctx->source_user == NULL)
+ cmd_copy_alloc_source_user(ctx);
+
+ ns = mail_namespace_find(user->namespaces, ctx->destname);
+ destbox = mailbox_alloc(ns->list, ctx->destname, MAILBOX_FLAG_SAVEONLY);
+ if (mailbox_open(destbox) < 0) {
+ i_error("Can't open mailbox '%s': %s", ctx->destname,
+ mailbox_get_last_internal_error(destbox, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, destbox);
+ mailbox_free(&destbox);
+ return -1;
+ }
+
+ src_user = ctx->source_user != NULL ? ctx->source_user : user;
+ iter = doveadm_mailbox_list_iter_init(_ctx, src_user, _ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_copy_box(ctx, destbox, info) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+
+ if (mailbox_sync(destbox, 0) < 0) {
+ i_error("Syncing mailbox '%s' failed: %s", ctx->destname,
+ mailbox_get_last_internal_error(destbox, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, destbox);
+ ret = -1;
+ }
+ mailbox_free(&destbox);
+ return ret;
+}
+
+static void cmd_copy_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct copy_cmd_context *ctx = (struct copy_cmd_context *)_ctx;
+ const char *destname = args[0], *cmdname = ctx->move ? "move" : "copy";
+
+ if (destname == NULL || args[1] == NULL)
+ doveadm_mail_help_name(cmdname);
+ args++;
+
+ if (args[0] != NULL && args[1] != NULL &&
+ strcasecmp(args[0], "user") == 0) {
+ if ((_ctx->service_flags &
+ MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP) == 0)
+ i_fatal("Use -u parameter to specify destination user");
+
+ ctx->source_username = p_strdup(_ctx->pool, args[1]);
+ args += 2;
+ }
+
+ ctx->destname = p_strdup(ctx->ctx.pool, destname);
+ _ctx->search_args = doveadm_mail_build_search_args(args);
+ if (ctx->move)
+ expunge_search_args_check(ctx->ctx.search_args, cmdname);
+}
+
+static void cmd_copy_deinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ struct copy_cmd_context *ctx = (struct copy_cmd_context *)_ctx;
+
+ if (ctx->source_user != NULL) {
+ mail_storage_service_user_unref(&ctx->source_service_user);
+ mail_user_deinit(&ctx->source_user);
+ }
+}
+
+static struct doveadm_mail_cmd_context *cmd_copy_alloc(void)
+{
+ struct copy_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct copy_cmd_context);
+ ctx->ctx.v.init = cmd_copy_init;
+ ctx->ctx.v.deinit = cmd_copy_deinit;
+ ctx->ctx.v.run = cmd_copy_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return &ctx->ctx;
+}
+
+static struct doveadm_mail_cmd_context *cmd_move_alloc(void)
+{
+ struct copy_cmd_context *ctx;
+
+ ctx = (struct copy_cmd_context *)cmd_copy_alloc();
+ ctx->move = TRUE;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_copy_ver2 = {
+ .name = "copy",
+ .mail_cmd = cmd_copy_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<destination> [user <source user>] <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "destination-mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "source-type", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "source-user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_move_ver2 = {
+ .name = "move",
+ .mail_cmd = cmd_move_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<destination> [user <source user>] <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "destination-mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "source-type", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "source-user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-deduplicate.c b/src/doveadm/doveadm-mail-deduplicate.c
new file mode 100644
index 0000000..f990287
--- /dev/null
+++ b/src/doveadm/doveadm-mail-deduplicate.c
@@ -0,0 +1,146 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "hash.h"
+#include "mail-storage.h"
+#include "mail-search-build.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+struct deduplicate_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ bool by_msgid;
+};
+
+static int
+cmd_deduplicate_box(struct doveadm_mail_cmd_context *_ctx,
+ const struct mailbox_info *info,
+ struct mail_search_args *search_args)
+{
+ struct deduplicate_cmd_context *ctx =
+ (struct deduplicate_cmd_context *)_ctx;
+ struct doveadm_mail_iter *iter;
+ struct mail *mail;
+ enum mail_error error;
+ pool_t pool;
+ HASH_TABLE(const char *, void *) hash;
+ const char *key, *errstr;
+ int ret = 0;
+
+ if (doveadm_mail_iter_init(_ctx, info, search_args, 0, NULL, 0,
+ &iter) < 0)
+ return -1;
+
+ pool = pool_alloconly_create("deduplicate", 10240);
+ hash_table_create(&hash, pool, 0, str_hash, strcmp);
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ if (ctx->by_msgid) {
+ if (mail_get_first_header(mail, "Message-ID", &key) < 0) {
+ errstr = mailbox_get_last_internal_error(mail->box, &error);
+ if (error == MAIL_ERROR_NOTFOUND)
+ continue;
+ i_error("Couldn't lookup Message-ID: for UID=%u: %s",
+ mail->uid, errstr);
+ doveadm_mail_failed_error(_ctx, error);
+ ret = -1;
+ break;
+ }
+ } else {
+ if (mail_get_special(mail, MAIL_FETCH_GUID, &key) < 0) {
+ errstr = mailbox_get_last_internal_error(mail->box, &error);
+ if (error == MAIL_ERROR_NOTFOUND)
+ continue;
+ i_error("Couldn't lookup GUID: for UID=%u: %s",
+ mail->uid, errstr);
+ doveadm_mail_failed_error(_ctx, error);
+ ret = -1;
+ break;
+ }
+ }
+ if (key != NULL && *key != '\0') {
+ if (hash_table_lookup(hash, key) != NULL)
+ mail_expunge(mail);
+ else {
+ key = p_strdup(pool, key);
+ hash_table_insert(hash, key, POINTER_CAST(1));
+ }
+ }
+ }
+
+ if (doveadm_mail_iter_deinit_sync(&iter) < 0)
+ ret = -1;
+
+ hash_table_destroy(&hash);
+ pool_unref(&pool);
+ return ret;
+}
+
+static int
+cmd_deduplicate_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
+{
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_deduplicate_box(ctx, info, ctx->search_args) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_deduplicate_init(struct doveadm_mail_cmd_context *ctx,
+ const char *const args[])
+{
+ if (args[0] == NULL)
+ doveadm_mail_help_name("deduplicate");
+
+ ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static bool
+cmd_deduplicate_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct deduplicate_cmd_context *ctx =
+ (struct deduplicate_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'm':
+ ctx->by_msgid = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_deduplicate_alloc(void)
+{
+ struct deduplicate_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct deduplicate_cmd_context);
+ ctx->ctx.getopt_args = "m";
+ ctx->ctx.v.parse_arg = cmd_deduplicate_parse_arg;
+ ctx->ctx.v.init = cmd_deduplicate_init;
+ ctx->ctx.v.run = cmd_deduplicate_run;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_deduplicate_ver2 = {
+ .name = "deduplicate",
+ .mail_cmd = cmd_deduplicate_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-m] <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('m', "by-msgid", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-expunge.c b/src/doveadm/doveadm-mail-expunge.c
new file mode 100644
index 0000000..4ab7e3d
--- /dev/null
+++ b/src/doveadm/doveadm-mail-expunge.c
@@ -0,0 +1,286 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "mail-index.h"
+#include "mail-storage.h"
+#include "mail-search.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+struct expunge_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ bool delete_empty_mailbox;
+};
+
+static int
+cmd_expunge_box(struct doveadm_mail_cmd_context *_ctx,
+ const struct mailbox_info *info,
+ struct mail_search_args *search_args)
+{
+ struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx;
+ struct doveadm_mail_iter *iter;
+ struct mailbox *box;
+ struct mail *mail;
+ enum mail_error error;
+ int ret = 0;
+
+ if (doveadm_mail_iter_init(_ctx, info, search_args, 0, NULL, 0,
+ &iter) < 0)
+ return -1;
+
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ if (doveadm_debug) {
+ i_debug("expunge: box=%s uid=%u",
+ info->vname, mail->uid);
+ }
+ mail_expunge(mail);
+ }
+
+ if (doveadm_mail_iter_deinit_keep_box(&iter, &box) < 0)
+ ret = -1;
+ else if (mailbox_sync(box, 0) < 0) {
+ i_error("Syncing mailbox '%s' failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+
+ if (ctx->delete_empty_mailbox && ret == 0) {
+ if (mailbox_delete_empty(box) < 0) {
+ error = mailbox_get_last_mail_error(box);
+ if (error != MAIL_ERROR_EXISTS) {
+ i_error("Deleting mailbox '%s' failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ } else {
+ if (mailbox_set_subscribed(box, FALSE) < 0) {
+ i_error("Unsubscribing mailbox '%s' failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ }
+ }
+ mailbox_free(&box);
+ return ret;
+}
+
+static bool
+expunge_search_args_is_mailbox_ok(struct mail_search_arg *args);
+
+static bool
+expunge_search_args_is_mailbox_or_ok(struct mail_search_arg *args)
+{
+ struct mail_search_arg *arg;
+
+ for (arg = args; arg != NULL; arg = arg->next) {
+ switch (arg->type) {
+ case SEARCH_OR:
+ if (!expunge_search_args_is_mailbox_or_ok(arg->value.subargs))
+ return FALSE;
+ break;
+ case SEARCH_SUB:
+ case SEARCH_INTHREAD:
+ if (!expunge_search_args_is_mailbox_ok(arg->value.subargs))
+ return FALSE;
+ break;
+ case SEARCH_MAILBOX:
+ case SEARCH_MAILBOX_GUID:
+ case SEARCH_MAILBOX_GLOB:
+ break;
+ default:
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
+static bool
+expunge_search_args_is_mailbox_ok(struct mail_search_arg *args)
+{
+ struct mail_search_arg *arg;
+ bool have_or = FALSE;
+
+ /* a) we find one mailbox here in the SUB block */
+ for (arg = args; arg != NULL; arg = arg->next) {
+ switch (arg->type) {
+ case SEARCH_MAILBOX:
+ case SEARCH_MAILBOX_GUID:
+ case SEARCH_MAILBOX_GLOB:
+ return TRUE;
+ case SEARCH_OR:
+ have_or = TRUE;
+ break;
+ case SEARCH_SUB:
+ case SEARCH_INTHREAD:
+ if (expunge_search_args_is_mailbox_ok(arg->value.subargs))
+ return TRUE;
+ break;
+ default:
+ break;
+ }
+ }
+
+ /* b) there is at least one OR block, and all of the ORs must have
+ mailbox */
+ if (!have_or)
+ return FALSE;
+
+ for (arg = args; arg != NULL; arg = arg->next) {
+ if (arg->type == SEARCH_OR &&
+ !expunge_search_args_is_mailbox_or_ok(arg->value.subargs))
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static bool
+expunge_search_args_is_msgset_ok(struct mail_search_arg *args);
+
+static bool
+expunge_search_args_is_msgset_or_ok(struct mail_search_arg *args)
+{
+ struct mail_search_arg *arg;
+
+ /* we're done if all OR branches contain something else besides
+ MAILBOXes */
+ for (arg = args; arg != NULL; arg = arg->next) {
+ switch (arg->type) {
+ case SEARCH_MAILBOX:
+ case SEARCH_MAILBOX_GUID:
+ case SEARCH_MAILBOX_GLOB:
+ return FALSE;
+ case SEARCH_OR:
+ if (!expunge_search_args_is_msgset_or_ok(arg->value.subargs))
+ return FALSE;
+ break;
+ case SEARCH_SUB:
+ if (!expunge_search_args_is_msgset_ok(arg->value.subargs))
+ return FALSE;
+ break;
+ default:
+ break;
+ }
+ }
+ return TRUE;
+}
+
+static bool
+expunge_search_args_is_msgset_ok(struct mail_search_arg *args)
+{
+ struct mail_search_arg *arg;
+
+ /* all args can't be just MAILBOXes */
+ for (arg = args; arg != NULL; arg = arg->next) {
+ switch (arg->type) {
+ case SEARCH_MAILBOX:
+ case SEARCH_MAILBOX_GUID:
+ case SEARCH_MAILBOX_GLOB:
+ break;
+ case SEARCH_OR:
+ /* if each OR branch has something else than just
+ MAILBOXes, we're ok */
+ if (expunge_search_args_is_msgset_or_ok(arg->value.subargs))
+ return TRUE;
+ break;
+ case SEARCH_SUB:
+ if (expunge_search_args_is_msgset_ok(arg->value.subargs))
+ return TRUE;
+ break;
+ default:
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static int
+cmd_expunge_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
+{
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_expunge_box(ctx, info, ctx->search_args) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+void expunge_search_args_check(struct mail_search_args *args, const char *cmd)
+{
+ if (!expunge_search_args_is_mailbox_ok(args->args)) {
+ i_fatal_status(EX_USAGE,
+ "%s: To avoid accidents, search query "
+ "must contain MAILBOX in all search branches", cmd);
+ }
+ if (!expunge_search_args_is_msgset_ok(args->args)) {
+ i_fatal_status(EX_USAGE,
+ "%s: To avoid accidents, each branch in search query "
+ "must contain something else besides MAILBOX "
+ "(e.g. just add \"all\" if you want everything)", cmd);
+ }
+ mail_search_args_simplify(args);
+}
+
+static void cmd_expunge_init(struct doveadm_mail_cmd_context *ctx,
+ const char *const args[])
+{
+ if (args[0] == NULL)
+ doveadm_mail_help_name("expunge");
+
+ ctx->search_args = doveadm_mail_build_search_args(args);
+ expunge_search_args_check(ctx->search_args, "expunge");
+}
+
+static bool cmd_expunge_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct expunge_cmd_context *ctx = (struct expunge_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'd':
+ ctx->delete_empty_mailbox = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_expunge_alloc(void)
+{
+ struct expunge_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct expunge_cmd_context);
+ ctx->ctx.getopt_args = "d";
+ ctx->ctx.v.parse_arg = cmd_expunge_parse_arg;
+ ctx->ctx.v.init = cmd_expunge_init;
+ ctx->ctx.v.run = cmd_expunge_run;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_expunge_ver2 = {
+ .name = "expunge",
+ .mail_cmd = cmd_expunge_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-m] <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('d', "delete-empty-mailbox", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-fetch.c b/src/doveadm/doveadm-mail-fetch.c
new file mode 100644
index 0000000..932a54d
--- /dev/null
+++ b/src/doveadm/doveadm-mail-fetch.c
@@ -0,0 +1,683 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "str.h"
+#include "message-address.h"
+#include "message-size.h"
+#include "message-parser.h"
+#include "message-header-decode.h"
+#include "message-decoder.h"
+#include "imap-util.h"
+#include "mail-user.h"
+#include "mail-storage.h"
+#include "mail-search.h"
+#include "mail-namespace.h"
+#include "imap-msgpart.h"
+#include "doveadm-print.h"
+#include "doveadm-mail.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+
+#include <stdio.h>
+
+struct fetch_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+
+ struct mail *mail;
+
+ ARRAY(struct fetch_field) fields;
+ ARRAY_TYPE(const_string) header_fields;
+ enum mail_fetch_field wanted_fields;
+
+ const struct fetch_field *cur_field;
+ /* if print() returns -1, log this error if non-NULL. otherwise log
+ the storage error. */
+ const char *print_error;
+};
+
+struct fetch_field {
+ const char *name;
+ enum mail_fetch_field wanted_fields;
+ int (*print)(struct fetch_cmd_context *ctx);
+};
+
+static int fetch_user(struct fetch_cmd_context *ctx)
+{
+ doveadm_print(ctx->ctx.cur_mail_user->username);
+ return 0;
+}
+
+static int fetch_mailbox(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_MAILBOX_NAME, &value) < 0)
+ return -1;
+
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_mailbox_guid(struct fetch_cmd_context *ctx)
+{
+ struct mailbox_metadata metadata;
+
+ if (mailbox_get_metadata(ctx->mail->box, MAILBOX_METADATA_GUID,
+ &metadata) < 0)
+ return -1;
+ doveadm_print(guid_128_to_string(metadata.guid));
+ return 0;
+}
+
+static int fetch_seq(struct fetch_cmd_context *ctx)
+{
+ doveadm_print_num(ctx->mail->seq);
+ return 0;
+}
+
+static int fetch_uid(struct fetch_cmd_context *ctx)
+{
+ doveadm_print_num(ctx->mail->uid);
+ return 0;
+}
+
+static int fetch_guid(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_GUID, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_flags(struct fetch_cmd_context *ctx)
+{
+ string_t *str = t_str_new(64);
+
+ imap_write_flags(str, mail_get_flags(ctx->mail),
+ mail_get_keywords(ctx->mail));
+ doveadm_print(str_c(str));
+ return 0;
+}
+
+static int fetch_modseq(struct fetch_cmd_context *ctx)
+{
+ doveadm_print_num(mail_get_modseq(ctx->mail));
+ return 0;
+}
+
+static void
+fetch_set_istream_error(struct fetch_cmd_context *ctx, struct istream *input)
+{
+ ctx->print_error = t_strdup_printf("read(%s) failed: %s",
+ i_stream_get_name(input), i_stream_get_error(input));
+}
+
+static int fetch_hdr(struct fetch_cmd_context *ctx)
+{
+ struct istream *input;
+ struct message_size hdr_size;
+ int ret;
+
+ if (mail_get_hdr_stream(ctx->mail, &hdr_size, &input) < 0)
+ return -1;
+
+ input = i_stream_create_limit(input, hdr_size.physical_size);
+ if ((ret = doveadm_print_istream(input)) < 0)
+ fetch_set_istream_error(ctx, input);
+ i_stream_unref(&input);
+ return ret;
+}
+
+static int fetch_hdr_field(struct fetch_cmd_context *ctx)
+{
+ const char *const *value, *filter, *name = ctx->cur_field->name;
+ string_t *str = t_str_new(256);
+ bool add_lf = FALSE;
+
+ filter = strchr(name, '.');
+ if (filter != NULL)
+ name = t_strdup_until(name, filter++);
+
+ if (filter != NULL && strcmp(filter, "utf8") == 0) {
+ if (mail_get_headers_utf8(ctx->mail, name, &value) < 0)
+ return -1;
+ } else {
+ if (mail_get_headers(ctx->mail, name, &value) < 0)
+ return -1;
+ }
+
+ for (; *value != NULL; value++) {
+ if (add_lf)
+ str_append_c(str, '\n');
+ str_append(str, *value);
+ add_lf = TRUE;
+ }
+
+ if (filter == NULL || strcmp(filter, "utf8") == 0) {
+ /* print the header as-is */
+ } else if (strcmp(filter, "address") == 0 ||
+ strcmp(filter, "address_name") == 0 ||
+ strcmp(filter, "address_name.utf8") == 0) {
+ struct message_address *addr;
+
+ addr = message_address_parse(pool_datastack_create(),
+ str_data(str), str_len(str),
+ UINT_MAX, 0);
+ str_truncate(str, 0);
+ add_lf = FALSE;
+ for (; addr != NULL; addr = addr->next) {
+ if (add_lf)
+ str_append_c(str, '\n');
+ if (strcmp(filter, "address") == 0) {
+ if (addr->mailbox != NULL)
+ str_append(str, addr->mailbox);
+ if (addr->domain != NULL) {
+ str_append_c(str, '@');
+ str_append(str, addr->domain);
+ }
+ } else if (addr->name != NULL) {
+ if (strcmp(filter, "address_name") == 0)
+ str_append(str, addr->name);
+ else {
+ message_header_decode_utf8(
+ (const void *)addr->name,
+ strlen(addr->name), str, NULL);
+ }
+ }
+ add_lf = TRUE;
+ }
+ } else {
+ i_fatal("Unknown header filter: %s", filter);
+ }
+ doveadm_print(str_c(str));
+ return 0;
+}
+
+static int fetch_body_field(struct fetch_cmd_context *ctx)
+{
+ const char *name = ctx->cur_field->name;
+ struct imap_msgpart *msgpart;
+ struct imap_msgpart_open_result result;
+ bool binary;
+ int ret;
+
+ binary = str_begins(name, "binary.");
+ name += binary ? 7 : 5;
+ if (imap_msgpart_parse(name, &msgpart) < 0)
+ i_unreached(); /* we already verified this was ok */
+ if (binary)
+ imap_msgpart_set_decode_to_binary(msgpart);
+
+ if (imap_msgpart_open(ctx->mail, msgpart, &result) < 0) {
+ imap_msgpart_free(&msgpart);
+ return -1;
+ }
+ if ((ret = doveadm_print_istream(result.input)) < 0)
+ fetch_set_istream_error(ctx, result.input);
+ i_stream_unref(&result.input);
+ imap_msgpart_free(&msgpart);
+ return ret;
+}
+
+static int fetch_body(struct fetch_cmd_context *ctx)
+{
+ struct istream *input;
+ struct message_size hdr_size;
+ int ret;
+
+ if (mail_get_stream(ctx->mail, &hdr_size, NULL, &input) < 0)
+ return -1;
+
+ i_stream_skip(input, hdr_size.physical_size);
+ if ((ret = doveadm_print_istream(input)) < 0)
+ fetch_set_istream_error(ctx, input);
+ return ret;
+}
+
+static int fetch_body_snippet(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_BODY_SNIPPET, &value) < 0)
+ return -1;
+ /* [0] contains the snippet algorithm, skip over it */
+ i_assert(value[0] != '\0');
+ doveadm_print(value + 1);
+ return 0;
+}
+
+static int fetch_text(struct fetch_cmd_context *ctx)
+{
+ struct istream *input;
+ int ret;
+
+ if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0)
+ return -1;
+ if ((ret = doveadm_print_istream(input)) < 0)
+ fetch_set_istream_error(ctx, input);
+ return ret;
+}
+
+static int fetch_text_utf8(struct fetch_cmd_context *ctx)
+{
+ const struct message_parser_settings parser_set = {
+ .hdr_flags = MESSAGE_HEADER_PARSER_FLAG_CLEAN_ONELINE,
+ };
+ struct istream *input;
+ struct message_parser_ctx *parser;
+ struct message_decoder_context *decoder;
+ struct message_block raw_block, block;
+ struct message_part *parts;
+ int ret = 0;
+
+ if (mail_get_stream(ctx->mail, NULL, NULL, &input) < 0)
+ return -1;
+
+ parser = message_parser_init(pool_datastack_create(), input, &parser_set);
+ decoder = message_decoder_init(NULL, 0);
+
+ while ((ret = message_parser_parse_next_block(parser, &raw_block)) > 0) {
+ if (!message_decoder_decode_next_block(decoder, &raw_block,
+ &block))
+ continue;
+
+ if (block.hdr == NULL) {
+ if (block.size > 0)
+ doveadm_print_stream(block.data, block.size);
+ } else if (block.hdr->eoh)
+ doveadm_print_stream("\n", 1);
+ else {
+ i_assert(block.hdr->name_len > 0);
+ doveadm_print_stream(block.hdr->name,
+ block.hdr->name_len);
+ doveadm_print_stream(": ", 2);
+ if (block.hdr->full_value_len > 0) {
+ doveadm_print_stream(block.hdr->full_value,
+ block.hdr->full_value_len);
+ }
+ doveadm_print_stream("\n", 1);
+ }
+ }
+ i_assert(ret != 0);
+ message_decoder_deinit(&decoder);
+ message_parser_deinit(&parser, &parts);
+
+ doveadm_print_stream("", 0);
+ if (input->stream_errno != 0) {
+ i_error("read(%s) failed: %s", i_stream_get_name(input),
+ i_stream_get_error(input));
+ return -1;
+ }
+ return 0;
+}
+
+static int fetch_size_physical(struct fetch_cmd_context *ctx)
+{
+ uoff_t size;
+
+ if (mail_get_physical_size(ctx->mail, &size) < 0)
+ return -1;
+ doveadm_print_num(size);
+ return 0;
+}
+
+static int fetch_size_virtual(struct fetch_cmd_context *ctx)
+{
+ uoff_t size;
+
+ if (mail_get_virtual_size(ctx->mail, &size) < 0)
+ return -1;
+ doveadm_print_num(size);
+ return 0;
+}
+
+static int fetch_date_received(struct fetch_cmd_context *ctx)
+{
+ time_t t;
+
+ if (mail_get_received_date(ctx->mail, &t) < 0)
+ return -1;
+ doveadm_print(unixdate2str(t));
+ return 0;
+}
+
+static int fetch_date_sent(struct fetch_cmd_context *ctx)
+{
+ time_t t;
+ int tz;
+ char chr;
+
+ if (mail_get_date(ctx->mail, &t, &tz) < 0)
+ return -1;
+
+ chr = tz < 0 ? '-' : '+';
+ if (tz < 0) tz = -tz;
+ doveadm_print(t_strdup_printf("%s (%c%02u%02u)", unixdate2str(t),
+ chr, tz/60, tz%60));
+ return 0;
+}
+
+static int fetch_date_saved(struct fetch_cmd_context *ctx)
+{
+ time_t t;
+
+ if (mail_get_save_date(ctx->mail, &t) < 0)
+ return -1;
+ doveadm_print(unixdate2str(t));
+ return 0;
+}
+
+static int fetch_date_received_unixtime(struct fetch_cmd_context *ctx)
+{
+ time_t t;
+
+ if (mail_get_received_date(ctx->mail, &t) < 0)
+ return -1;
+ doveadm_print(dec2str(t));
+ return 0;
+}
+
+static int fetch_date_sent_unixtime(struct fetch_cmd_context *ctx)
+{
+ time_t t;
+ int tz;
+
+ if (mail_get_date(ctx->mail, &t, &tz) < 0)
+ return -1;
+
+ doveadm_print(dec2str(t));
+ return 0;
+}
+
+static int fetch_date_saved_unixtime(struct fetch_cmd_context *ctx)
+{
+ time_t t;
+
+ if (mail_get_save_date(ctx->mail, &t) < 0)
+ return -1;
+ doveadm_print(dec2str(t));
+ return 0;
+}
+
+static int fetch_imap_envelope(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_IMAP_ENVELOPE, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_imap_body(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_IMAP_BODY, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_imap_bodystructure(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_IMAP_BODYSTRUCTURE, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+static int fetch_pop3_uidl(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_UIDL_BACKEND, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_pop3_order(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_POP3_ORDER, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_refcount(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_REFCOUNT, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static int fetch_storageid(struct fetch_cmd_context *ctx)
+{
+ const char *value;
+
+ if (mail_get_special(ctx->mail, MAIL_FETCH_STORAGE_ID, &value) < 0)
+ return -1;
+ doveadm_print(value);
+ return 0;
+}
+
+static const struct fetch_field fetch_fields[] = {
+ { "user", 0, fetch_user },
+ { "mailbox", 0, fetch_mailbox },
+ { "mailbox-guid", 0, fetch_mailbox_guid },
+ { "seq", 0, fetch_seq },
+ { "uid", 0, fetch_uid },
+ { "guid", 0, fetch_guid },
+ { "flags", MAIL_FETCH_FLAGS, fetch_flags },
+ { "modseq", 0, fetch_modseq },
+ { "hdr", MAIL_FETCH_STREAM_HEADER, fetch_hdr },
+ { "body", MAIL_FETCH_STREAM_BODY, fetch_body },
+ { "body.preview", MAIL_FETCH_BODY_SNIPPET, fetch_body_snippet },
+ { "body.snippet", MAIL_FETCH_BODY_SNIPPET, fetch_body_snippet },
+ { "text", MAIL_FETCH_STREAM_HEADER |
+ MAIL_FETCH_STREAM_BODY, fetch_text },
+ { "text.utf8", MAIL_FETCH_STREAM_HEADER |
+ MAIL_FETCH_STREAM_BODY, fetch_text_utf8 },
+ { "size.physical", MAIL_FETCH_PHYSICAL_SIZE, fetch_size_physical },
+ { "size.virtual", MAIL_FETCH_VIRTUAL_SIZE, fetch_size_virtual },
+ { "date.received", MAIL_FETCH_RECEIVED_DATE, fetch_date_received },
+ { "date.sent", MAIL_FETCH_DATE, fetch_date_sent },
+ { "date.saved", MAIL_FETCH_SAVE_DATE, fetch_date_saved },
+ { "date.received.unixtime", MAIL_FETCH_RECEIVED_DATE, fetch_date_received_unixtime },
+ { "date.sent.unixtime", MAIL_FETCH_DATE, fetch_date_sent_unixtime },
+ { "date.saved.unixtime", MAIL_FETCH_SAVE_DATE, fetch_date_saved_unixtime },
+ { "imap.envelope", MAIL_FETCH_IMAP_ENVELOPE, fetch_imap_envelope },
+ { "imap.body", MAIL_FETCH_IMAP_BODY, fetch_imap_body },
+ { "imap.bodystructure", MAIL_FETCH_IMAP_BODYSTRUCTURE, fetch_imap_bodystructure },
+ { "pop3.uidl", MAIL_FETCH_UIDL_BACKEND, fetch_pop3_uidl },
+ { "pop3.order", MAIL_FETCH_POP3_ORDER, fetch_pop3_order },
+ { "refcount", MAIL_FETCH_REFCOUNT, fetch_refcount },
+ { "storageid", MAIL_FETCH_STORAGE_ID, fetch_storageid }
+};
+
+static const struct fetch_field *fetch_field_find(const char *name)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(fetch_fields); i++) {
+ if (strcmp(fetch_fields[i].name, name) == 0)
+ return &fetch_fields[i];
+ }
+ return NULL;
+}
+
+static void print_fetch_fields(void)
+{
+ unsigned int i;
+
+ fprintf(stderr, "Available fetch fields: hdr.<name> body.<section> binary.<section> %s", fetch_fields[0].name);
+ for (i = 1; i < N_ELEMENTS(fetch_fields); i++)
+ fprintf(stderr, " %s", fetch_fields[i].name);
+ fprintf(stderr, "\n");
+}
+
+static void parse_fetch_fields(struct fetch_cmd_context *ctx, const char *str)
+{
+ const char *const *fields, *name;
+ const struct fetch_field *field;
+ struct fetch_field hdr_field, body_field;
+ struct imap_msgpart *msgpart;
+
+ i_zero(&hdr_field);
+ hdr_field.print = fetch_hdr_field;
+
+ i_zero(&body_field);
+ body_field.print = fetch_body_field;
+
+ t_array_init(&ctx->fields, 32);
+ t_array_init(&ctx->header_fields, 32);
+ fields = t_strsplit_spaces(str, " ");
+ for (; *fields != NULL; fields++) {
+ name = t_str_lcase(*fields);
+
+ doveadm_print_header_simple(name);
+ if ((field = fetch_field_find(name)) != NULL) {
+ ctx->wanted_fields |= field->wanted_fields;
+ array_push_back(&ctx->fields, field);
+ } else if (str_begins(name, "hdr.")) {
+ name += 4;
+ hdr_field.name = name;
+ array_push_back(&ctx->fields, &hdr_field);
+ name = t_strcut(name, '.');
+ array_push_back(&ctx->header_fields, &name);
+ } else if (str_begins(name, "body.") ||
+ str_begins(name, "binary.")) {
+ bool binary = str_begins(name, "binary.");
+ body_field.name = t_strarray_join(t_strsplit(name, ","), " ");
+
+ name += binary ? 7 : 5;
+ if (imap_msgpart_parse(name, &msgpart) < 0) {
+ print_fetch_fields();
+ i_fatal("Unknown fetch section: %s", name);
+ }
+ array_push_back(&ctx->fields, &body_field);
+ ctx->wanted_fields |= imap_msgpart_get_fetch_data(msgpart);
+ imap_msgpart_free(&msgpart);
+ } else {
+ print_fetch_fields();
+ i_fatal("Unknown fetch field: %s", name);
+ }
+ }
+ array_append_zero(&ctx->header_fields);
+}
+
+static int cmd_fetch_mail(struct fetch_cmd_context *ctx)
+{
+ const struct fetch_field *field;
+ struct mail *mail = ctx->mail;
+ int ret = 0;
+
+ array_foreach(&ctx->fields, field) {
+ ctx->cur_field = field;
+ if (field->print(ctx) < 0) {
+ i_error("fetch(%s) failed for box=%s uid=%u: %s",
+ field->name, mailbox_get_vname(mail->box),
+ mail->uid,
+ ctx->print_error != NULL ? ctx->print_error :
+ mailbox_get_last_internal_error(mail->box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, mail->box);
+ ctx->print_error = NULL;
+ ret = -1;
+ }
+ }
+ return ret;
+}
+
+static int
+cmd_fetch_box(struct fetch_cmd_context *ctx, const struct mailbox_info *info)
+{
+ struct doveadm_mail_iter *iter;
+ int ret = 0;
+
+ if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args,
+ ctx->wanted_fields,
+ array_front(&ctx->header_fields),
+ DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT,
+ &iter) < 0)
+ return -1;
+
+ while (doveadm_mail_iter_next(iter, &ctx->mail)) {
+ T_BEGIN {
+ if (cmd_fetch_mail(ctx) < 0)
+ ret = -1;
+ } T_END;
+ }
+ if (doveadm_mail_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static int
+cmd_fetch_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct fetch_cmd_context *ctx = (struct fetch_cmd_context *)_ctx;
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(_ctx, user, _ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_fetch_box(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_fetch_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct fetch_cmd_context *ctx = (struct fetch_cmd_context *)_ctx;
+ const char *fetch_fields = args[0];
+
+ if (fetch_fields == NULL || args[1] == NULL)
+ doveadm_mail_help_name("fetch");
+
+ parse_fetch_fields(ctx, fetch_fields);
+ _ctx->search_args = doveadm_mail_build_search_args(args + 1);
+}
+
+static struct doveadm_mail_cmd_context *cmd_fetch_alloc(void)
+{
+ struct fetch_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct fetch_cmd_context);
+ ctx->ctx.v.init = cmd_fetch_init;
+ ctx->ctx.v.run = cmd_fetch_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_PAGER);
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_fetch_ver2 = {
+ .name = "fetch",
+ .mail_cmd = cmd_fetch_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<fields> <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "field", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('\0', "fieldstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL | CMD_PARAM_FLAG_DO_NOT_EXPOSE) /* FIXME: horrible hack, remove me when possible */
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-flags.c b/src/doveadm/doveadm-mail-flags.c
new file mode 100644
index 0000000..7c252a7
--- /dev/null
+++ b/src/doveadm/doveadm-mail-flags.c
@@ -0,0 +1,178 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "imap-util.h"
+#include "mail-storage.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+#include <stdio.h>
+
+struct flags_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ enum modify_type modify_type;
+ enum mail_flags flags;
+ const char *const *keywords;
+};
+
+static int
+cmd_flags_run_box(struct flags_cmd_context *ctx,
+ const struct mailbox_info *info)
+{
+ struct doveadm_mail_iter *iter;
+ struct mailbox *box;
+ struct mail *mail;
+ struct mail_keywords *kw = NULL;
+
+ if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args,
+ 0, NULL, 0, &iter) < 0)
+ return -1;
+ box = doveadm_mail_iter_get_mailbox(iter);
+
+ if (ctx->keywords != NULL) {
+ if (mailbox_keywords_create(box, ctx->keywords, &kw) < 0) {
+ i_error("Invalid keywords: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ (void)doveadm_mail_iter_deinit(&iter);
+ ctx->ctx.exit_code = DOVEADM_EX_NOTPOSSIBLE;
+ return -1;
+ }
+ }
+
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ mail_update_flags(mail, ctx->modify_type, ctx->flags);
+ if (kw != NULL)
+ mail_update_keywords(mail, ctx->modify_type, kw);
+ }
+ if (kw != NULL)
+ mailbox_keywords_unref(&kw);
+ return doveadm_mail_iter_deinit_sync(&iter);
+}
+
+static int
+cmd_flags_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct flags_cmd_context *ctx = (struct flags_cmd_context *)_ctx;
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(_ctx, user, _ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_flags_run_box(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_flags_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct flags_cmd_context *ctx = (struct flags_cmd_context *)_ctx;
+ const char *const *tmp;
+ enum mail_flags flag;
+ ARRAY_TYPE(const_string) keywords;
+
+ if (args[0] == NULL || args[1] == NULL) {
+ switch (ctx->modify_type) {
+ case MODIFY_ADD:
+ doveadm_mail_help_name("flags add");
+ case MODIFY_REMOVE:
+ doveadm_mail_help_name("flags remove");
+ case MODIFY_REPLACE:
+ doveadm_mail_help_name("flags replace");
+ }
+ i_unreached();
+ }
+
+ p_array_init(&keywords, _ctx->pool, 8);
+ for (tmp = t_strsplit(args[0], " "); *tmp != NULL; tmp++) {
+ const char *str = *tmp;
+
+ if (str[0] == '\\') {
+ flag = imap_parse_system_flag(str);
+ if (flag == 0)
+ i_fatal("Invalid system flag: %s", str);
+ ctx->flags |= flag;
+ } else {
+ str = p_strdup(_ctx->pool, str);
+ array_push_back(&keywords, &str);
+ }
+ }
+ if (array_count(&keywords) > 0 || ctx->modify_type == MODIFY_REPLACE) {
+ array_append_zero(&keywords);
+ ctx->keywords = array_front(&keywords);
+ }
+
+ _ctx->search_args = doveadm_mail_build_search_args(args+1);
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_flag_alloc(enum modify_type modify_type)
+{
+ struct flags_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct flags_cmd_context);
+ ctx->modify_type = modify_type;
+ ctx->ctx.v.init = cmd_flags_init;
+ ctx->ctx.v.run = cmd_flags_run;
+ return &ctx->ctx;
+}
+
+static struct doveadm_mail_cmd_context *cmd_flags_add_alloc(void)
+{
+ return cmd_flag_alloc(MODIFY_ADD);
+}
+
+static struct doveadm_mail_cmd_context *cmd_flags_remove_alloc(void)
+{
+ return cmd_flag_alloc(MODIFY_REMOVE);
+}
+
+static struct doveadm_mail_cmd_context *cmd_flags_replace_alloc(void)
+{
+ return cmd_flag_alloc(MODIFY_REPLACE);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_flags_add_ver2 = {
+ .name = "flags add",
+ .mail_cmd = cmd_flags_add_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<flags> <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "flag", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('\0', "flagstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL|CMD_PARAM_FLAG_DO_NOT_EXPOSE)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_flags_remove_ver2 = {
+ .name = "flags remove",
+ .mail_cmd = cmd_flags_remove_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<flags> <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "flag", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('\0', "flagstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL|CMD_PARAM_FLAG_DO_NOT_EXPOSE)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_flags_replace_ver2 = {
+ .name = "flags replace",
+ .mail_cmd = cmd_flags_replace_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<flags> <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "flag", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('\0', "flagstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL|CMD_PARAM_FLAG_DO_NOT_EXPOSE)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-import.c b/src/doveadm/doveadm-mail-import.c
new file mode 100644
index 0000000..bd6ef6d
--- /dev/null
+++ b/src/doveadm/doveadm-mail-import.c
@@ -0,0 +1,276 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "mail-storage.h"
+#include "mail-storage-service.h"
+#include "mail-namespace.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+struct import_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+
+ const char *src_location;
+ const char *src_username;
+ struct mail_user *src_user;
+ const char *dest_parent;
+ bool subscribe;
+};
+
+static const char *
+convert_vname_separators(const char *vname, char src_sep, char dest_sep)
+{
+ string_t *str = t_str_new(128);
+
+ for (; *vname != '\0'; vname++) {
+ if (*vname == src_sep)
+ str_append_c(str, dest_sep);
+ else if (*vname == dest_sep)
+ str_append_c(str, '_');
+ else
+ str_append_c(str, *vname);
+ }
+ return str_c(str);
+}
+
+static int
+dest_mailbox_open_or_create(struct import_cmd_context *ctx,
+ struct mail_user *user,
+ const struct mailbox_info *info,
+ struct mailbox **box_r)
+{
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ enum mail_error error;
+ const char *name, *errstr;
+
+ if (*ctx->dest_parent != '\0') {
+ /* prefix destination mailbox name with given parent mailbox */
+ ns = mail_namespace_find(user->namespaces, ctx->dest_parent);
+ } else {
+ ns = mail_namespace_find(user->namespaces, info->vname);
+ }
+ name = convert_vname_separators(info->vname,
+ mail_namespace_get_sep(info->ns),
+ mail_namespace_get_sep(ns));
+
+ if (*ctx->dest_parent != '\0') {
+ name = t_strdup_printf("%s%c%s", ctx->dest_parent,
+ mail_namespace_get_sep(ns), name);
+ }
+
+ box = mailbox_alloc(ns->list, name, MAILBOX_FLAG_SAVEONLY);
+ if (mailbox_create(box, NULL, FALSE) < 0) {
+ errstr = mailbox_get_last_internal_error(box, &error);
+ if (error != MAIL_ERROR_EXISTS) {
+ i_error("Couldn't create mailbox %s: %s", name, errstr);
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ mailbox_free(&box);
+ return -1;
+ }
+ }
+ if (ctx->subscribe) {
+ if (mailbox_set_subscribed(box, TRUE) < 0) {
+ i_error("Couldn't subscribe to mailbox %s: %s",
+ name, mailbox_get_last_internal_error(box, NULL));
+ }
+ }
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ i_error("Syncing mailbox %s failed: %s", name,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ mailbox_free(&box);
+ return -1;
+ }
+ *box_r = box;
+ return 0;
+}
+
+static int
+cmd_import_box_contents(struct doveadm_mail_cmd_context *ctx,
+ struct doveadm_mail_iter *iter, struct mail *src_mail,
+ struct mailbox *dest_box)
+{
+ struct mail_save_context *save_ctx;
+ struct mailbox_transaction_context *dest_trans;
+ const char *mailbox = mailbox_get_vname(dest_box);
+ int ret = 0;
+
+ dest_trans = mailbox_transaction_begin(dest_box,
+ MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+ ctx->transaction_flags, __func__);
+ do {
+ if (doveadm_debug) {
+ i_debug("import: box=%s uid=%u",
+ mailbox, src_mail->uid);
+ }
+ save_ctx = mailbox_save_alloc(dest_trans);
+ mailbox_save_copy_flags(save_ctx, src_mail);
+ if (mailbox_copy(&save_ctx, src_mail) < 0) {
+ i_error("Copying box=%s uid=%u failed: %s",
+ mailbox, src_mail->uid,
+ mailbox_get_last_internal_error(dest_box, NULL));
+ ret = -1;
+ }
+ } while (doveadm_mail_iter_next(iter, &src_mail));
+
+ if (mailbox_transaction_commit(&dest_trans) < 0) {
+ i_error("Committing copied mails to %s failed: %s", mailbox,
+ mailbox_get_last_internal_error(dest_box, NULL));
+ ret = -1;
+ }
+ return ret;
+}
+
+static int
+cmd_import_box(struct import_cmd_context *ctx, struct mail_user *dest_user,
+ const struct mailbox_info *info,
+ struct mail_search_args *search_args)
+{
+ struct doveadm_mail_iter *iter;
+ struct mailbox *box;
+ struct mail *mail;
+ int ret = 0;
+
+ if (doveadm_mail_iter_init(&ctx->ctx, info, search_args, 0, NULL,
+ DOVEADM_MAIL_ITER_FLAG_READONLY,
+ &iter) < 0)
+ return -1;
+
+ if (doveadm_mail_iter_next(iter, &mail)) {
+ /* at least one mail matches in this mailbox */
+ if (dest_mailbox_open_or_create(ctx, dest_user, info, &box) < 0)
+ ret = -1;
+ else {
+ if (cmd_import_box_contents(&ctx->ctx, iter, mail, box) < 0) {
+ doveadm_mail_failed_mailbox(&ctx->ctx, mail->box);
+ ret = -1;
+ }
+ mailbox_free(&box);
+ }
+ }
+ if (doveadm_mail_iter_deinit_sync(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_import_init_source_user(struct import_cmd_context *ctx, struct mail_user *dest_user)
+{
+ struct mail_storage_service_input input;
+ struct mail_storage_service_user *service_user;
+ struct mail_user *user;
+ const char *error;
+
+ /* create a user for accessing the source storage */
+ i_zero(&input);
+ input.module = "mail";
+ input.username = ctx->src_username != NULL ?
+ ctx->src_username :
+ dest_user->username;
+
+ mail_storage_service_io_deactivate_user(ctx->ctx.cur_service_user);
+ input.flags_override_add = MAIL_STORAGE_SERVICE_FLAG_NO_NAMESPACES |
+ MAIL_STORAGE_SERVICE_FLAG_NO_RESTRICT_ACCESS;
+ if (mail_storage_service_lookup_next(ctx->ctx.storage_service, &input,
+ &service_user, &user, &error) < 0)
+ i_fatal("Import user initialization failed: %s", error);
+ if (mail_namespaces_init_location(user, ctx->src_location, &error) < 0)
+ i_fatal("Import namespace initialization failed: %s", error);
+
+ ctx->src_user = user;
+ mail_storage_service_io_deactivate_user(service_user);
+ mail_storage_service_user_unref(&service_user);
+ mail_storage_service_io_activate_user(ctx->ctx.cur_service_user);
+}
+
+static int
+cmd_import_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx;
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ if (ctx->src_user == NULL)
+ cmd_import_init_source_user(ctx, user);
+
+ iter = doveadm_mailbox_list_iter_init(_ctx, ctx->src_user,
+ _ctx->search_args, iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_import_box(ctx, user, info, _ctx->search_args) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_import_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx;
+
+ if (str_array_length(args) < 3)
+ doveadm_mail_help_name("import");
+ ctx->src_location = p_strdup(_ctx->pool, args[0]);
+ ctx->dest_parent = p_strdup(_ctx->pool, args[1]);
+ ctx->ctx.search_args = doveadm_mail_build_search_args(args+2);
+}
+
+static void cmd_import_deinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx;
+
+ if (ctx->src_user != NULL)
+ mail_user_deinit(&ctx->src_user);
+}
+
+static bool cmd_import_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct import_cmd_context *ctx = (struct import_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'U':
+ ctx->src_username = p_strdup(_ctx->pool, optarg);
+ break;
+ case 's':
+ ctx->subscribe = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_import_alloc(void)
+{
+ struct import_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct import_cmd_context);
+ ctx->ctx.getopt_args = "s";
+ ctx->ctx.v.parse_arg = cmd_import_parse_arg;
+ ctx->ctx.v.init = cmd_import_init;
+ ctx->ctx.v.deinit = cmd_import_deinit;
+ ctx->ctx.v.run = cmd_import_run;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_import_ver2 = {
+ .name = "import",
+ .mail_cmd = cmd_import_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-U source-user] [-s] <source mail location> <dest parent mailbox> <search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('U', "source-user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('s', "subscribe", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "source-location", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "dest-parent-mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-index.c b/src/doveadm/doveadm-mail-index.c
new file mode 100644
index 0000000..524f373
--- /dev/null
+++ b/src/doveadm/doveadm-mail-index.c
@@ -0,0 +1,300 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "net.h"
+#include "write-full.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "mail-search-build.h"
+#include "mailbox-list-iter.h"
+#include "doveadm-settings.h"
+#include "doveadm-mail.h"
+
+#include <stdio.h>
+
+#define INDEXER_SOCKET_NAME "indexer"
+#define INDEXER_HANDSHAKE "VERSION\tindexer\t1\t0\n"
+
+struct index_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+
+ int queue_fd;
+ unsigned int max_recent_msgs;
+ bool queue:1;
+ bool have_wildcards:1;
+};
+
+static int cmd_index_box_precache(struct doveadm_mail_cmd_context *dctx,
+ struct mailbox *box)
+{
+ struct mailbox_status status;
+ struct mailbox_transaction_context *trans;
+ struct mail_search_args *search_args;
+ struct mail_search_context *ctx;
+ struct mail *mail;
+ struct mailbox_metadata metadata;
+ uint32_t seq;
+ unsigned int counter = 0, max;
+ int ret = 0;
+
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_PRECACHE_FIELDS,
+ &metadata) < 0) {
+ i_error("Mailbox %s: Precache-fields lookup failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ }
+ if (mailbox_get_status(box, STATUS_MESSAGES | STATUS_LAST_CACHED_SEQ,
+ &status) < 0) {
+ i_error("Mailbox %s: Status lookup failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ return -1;
+ }
+
+ seq = status.last_cached_seq + 1;
+ if (seq > status.messages) {
+ if (doveadm_verbose) {
+ i_info("%s: Cache is already up to date",
+ mailbox_get_vname(box));
+ }
+ return 0;
+ }
+ if (doveadm_verbose) {
+ i_info("%s: Caching mails seq=%u..%u",
+ mailbox_get_vname(box), seq, status.messages);
+ }
+
+ trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_NO_CACHE_DEC |
+ dctx->transaction_flags, __func__);
+ search_args = mail_search_build_init();
+ mail_search_build_add_seqset(search_args, seq, status.messages);
+ ctx = mailbox_search_init(trans, search_args, NULL,
+ metadata.precache_fields, NULL);
+ mail_search_args_unref(&search_args);
+
+ max = status.messages - seq + 1;
+ while (mailbox_search_next(ctx, &mail)) {
+ if (mail_precache(mail) < 0) {
+ i_error("Mailbox %s: Precache for UID=%u failed: %s",
+ mailbox_get_vname(box), mail->uid,
+ mailbox_get_last_internal_error(box, NULL));
+ ret = -1;
+ break;
+ }
+ if (doveadm_verbose && ++counter % 100 == 0) {
+ printf("\r%u/%u", counter, max);
+ fflush(stdout);
+ }
+ }
+ if (doveadm_verbose)
+ printf("\r%u/%u\n", counter, max);
+ if (mailbox_search_deinit(&ctx) < 0) {
+ i_error("Mailbox %s: Mail search failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ ret = -1;
+ }
+ if (mailbox_transaction_commit(&trans) < 0) {
+ i_error("Mailbox %s: Transaction commit failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ ret = -1;
+ }
+ return ret;
+}
+
+static int
+cmd_index_box(struct index_cmd_context *ctx, const struct mailbox_info *info)
+{
+ struct mailbox *box;
+ struct mailbox_status status;
+ int ret = 0;
+
+ box = mailbox_alloc(info->ns->list, info->vname,
+ MAILBOX_FLAG_IGNORE_ACLS);
+ if (ctx->max_recent_msgs != 0) {
+ /* index only if there aren't too many recent messages.
+ don't bother syncing the mailbox, that alone can take a
+ while with large maildirs. */
+ if (mailbox_open(box) < 0) {
+ i_error("Opening mailbox %s failed: %s", info->vname,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ mailbox_free(&box);
+ return -1;
+ }
+
+ mailbox_get_open_status(box, STATUS_RECENT, &status);
+ if (status.recent > ctx->max_recent_msgs) {
+ mailbox_free(&box);
+ return 0;
+ }
+ }
+
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ i_error("Syncing mailbox %s failed: %s", info->vname,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ ret = -1;
+ } else {
+ if (cmd_index_box_precache(&ctx->ctx, box) < 0) {
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ ret = -1;
+ }
+ }
+ mailbox_free(&box);
+ return ret;
+}
+
+static void index_queue_connect(struct index_cmd_context *ctx)
+{
+ const char *path;
+
+ path = t_strconcat(doveadm_settings->base_dir,
+ "/"INDEXER_SOCKET_NAME, NULL);
+ ctx->queue_fd = net_connect_unix(path);
+ if (ctx->queue_fd == -1)
+ i_fatal("net_connect_unix(%s) failed: %m", path);
+ if (write_full(ctx->queue_fd, INDEXER_HANDSHAKE,
+ strlen(INDEXER_HANDSHAKE)) < 0)
+ i_fatal("write(indexer) failed: %m");
+}
+
+static void cmd_index_queue(struct index_cmd_context *ctx,
+ struct mail_user *user, const char *mailbox)
+{
+ if (ctx->queue_fd == -1)
+ index_queue_connect(ctx);
+ i_assert(ctx->queue_fd != -1);
+
+ T_BEGIN {
+ string_t *str = t_str_new(256);
+
+ str_append(str, "APPEND\t0\t");
+ str_append_tabescaped(str, user->username);
+ str_append_c(str, '\t');
+ str_append_tabescaped(str, mailbox);
+ str_printfa(str, "\t%u\n", ctx->max_recent_msgs);
+ if (write_full(ctx->queue_fd, str_data(str), str_len(str)) < 0)
+ i_fatal("write(indexer) failed: %m");
+ } T_END;
+}
+
+static int
+cmd_index_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
+ MAILBOX_LIST_ITER_STAR_WITHIN_NS;
+ const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
+ struct mailbox_list_iterate_context *iter;
+ const struct mailbox_info *info;
+ unsigned int i;
+ int ret = 0;
+
+ if (ctx->queue && !ctx->have_wildcards) {
+ /* we can do this quickly without going through the mailboxes */
+ for (i = 0; _ctx->args[i] != NULL; i++)
+ cmd_index_queue(ctx, user, _ctx->args[i]);
+ return 0;
+ }
+
+ iter = mailbox_list_iter_init_namespaces(user->namespaces, _ctx->args,
+ ns_mask, iter_flags);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ if ((info->flags & (MAILBOX_NOSELECT |
+ MAILBOX_NONEXISTENT)) == 0) T_BEGIN {
+ if (ctx->queue)
+ cmd_index_queue(ctx, user, info->vname);
+ else {
+ if (cmd_index_box(ctx, info) < 0)
+ ret = -1;
+ }
+ } T_END;
+ }
+ if (mailbox_list_iter_deinit(&iter) < 0) {
+ i_error("Listing mailboxes failed: %s",
+ mailbox_list_get_last_internal_error(user->namespaces->list, NULL));
+ doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP);
+ ret = -1;
+ }
+ return ret;
+}
+
+static void cmd_index_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
+ unsigned int i;
+
+ if (args[0] == NULL)
+ doveadm_mail_help_name("index");
+ for (i = 0; args[i] != NULL; i++) {
+ if (strchr(args[i], '*') != NULL ||
+ strchr(args[i], '%') != NULL) {
+ ctx->have_wildcards = TRUE;
+ break;
+ }
+ }
+}
+
+static void cmd_index_deinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
+
+ if (ctx->queue_fd != -1) {
+ net_disconnect(ctx->queue_fd);
+ ctx->queue_fd = -1;
+ }
+}
+
+static bool
+cmd_index_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct index_cmd_context *ctx = (struct index_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'q':
+ ctx->queue = TRUE;
+ break;
+ case 'n':
+ if (str_to_uint(optarg, &ctx->max_recent_msgs) < 0) {
+ i_fatal_status(EX_USAGE,
+ "Invalid -n parameter number: %s", optarg);
+ }
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_index_alloc(void)
+{
+ struct index_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct index_cmd_context);
+ ctx->queue_fd = -1;
+ ctx->ctx.getopt_args = "qn:";
+ ctx->ctx.v.parse_arg = cmd_index_parse_arg;
+ ctx->ctx.v.init = cmd_index_init;
+ ctx->ctx.v.deinit = cmd_index_deinit;
+ ctx->ctx.v.run = cmd_index_run;
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_index_ver2 = {
+ .name = "index",
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-q] [-n <max recent>] <mailbox mask>",
+ .mail_cmd = cmd_index_alloc,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('q',"queue",CMD_PARAM_BOOL,0)
+DOVEADM_CMD_PARAM('n',"max-recent",CMD_PARAM_STR,0)
+DOVEADM_CMD_PARAM('\0',"mailbox-mask",CMD_PARAM_STR,CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-iter.c b/src/doveadm/doveadm-mail-iter.c
new file mode 100644
index 0000000..eca5421
--- /dev/null
+++ b/src/doveadm/doveadm-mail-iter.c
@@ -0,0 +1,176 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ostream.h"
+#include "mail-storage.h"
+#include "mail-namespace.h"
+#include "mail-search.h"
+#include "doveadm-print.h"
+#include "doveadm-mail.h"
+#include "doveadm-mail-iter.h"
+
+struct doveadm_mail_iter {
+ struct doveadm_mail_cmd_context *ctx;
+ struct mail_search_args *search_args;
+ enum doveadm_mail_iter_flags flags;
+
+ struct mailbox *box;
+ struct mailbox_transaction_context *t;
+ struct mail_search_context *search_ctx;
+ bool killed;
+};
+
+int doveadm_mail_iter_init(struct doveadm_mail_cmd_context *ctx,
+ const struct mailbox_info *info,
+ struct mail_search_args *search_args,
+ enum mail_fetch_field wanted_fields,
+ const char *const *wanted_headers,
+ enum doveadm_mail_iter_flags flags,
+ struct doveadm_mail_iter **iter_r)
+{
+ struct doveadm_mail_iter *iter;
+ struct mailbox_header_lookup_ctx *headers_ctx;
+ const char *errstr;
+ enum mail_error error;
+
+ enum mailbox_flags readonly_flag =
+ (flags & DOVEADM_MAIL_ITER_FLAG_READONLY) != 0 ?
+ MAILBOX_FLAG_READONLY : 0;
+
+ iter = i_new(struct doveadm_mail_iter, 1);
+ iter->ctx = ctx;
+ iter->flags = flags;
+ iter->box = mailbox_alloc(info->ns->list, info->vname,
+ MAILBOX_FLAG_IGNORE_ACLS | readonly_flag);
+ iter->search_args = search_args;
+
+ if (mailbox_sync(iter->box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ errstr = mailbox_get_last_internal_error(iter->box, &error);
+ if (error == MAIL_ERROR_NOTFOUND) {
+ /* just ignore this mailbox */
+ *iter_r = iter;
+ return 0;
+ }
+ i_error("Syncing mailbox %s failed: %s", info->vname, errstr);
+ doveadm_mail_failed_mailbox(ctx, iter->box);
+ mailbox_free(&iter->box);
+ i_free(iter);
+ return -1;
+ }
+
+ headers_ctx = wanted_headers == NULL || wanted_headers[0] == NULL ?
+ NULL : mailbox_header_lookup_init(iter->box, wanted_headers);
+
+ mail_search_args_init(search_args, iter->box, FALSE, NULL);
+ iter->t = mailbox_transaction_begin(iter->box, ctx->transaction_flags,
+ ctx->cmd->name);
+ iter->search_ctx = mailbox_search_init(iter->t, search_args, NULL,
+ wanted_fields, headers_ctx);
+ mailbox_header_lookup_unref(&headers_ctx);
+ *iter_r = iter;
+ return 0;
+}
+
+static int
+doveadm_mail_iter_deinit_transaction(struct doveadm_mail_iter *iter,
+ bool commit)
+{
+ int ret = 0;
+
+ if (iter->search_ctx != NULL) {
+ if (mailbox_search_deinit(&iter->search_ctx) < 0) {
+ i_error("Searching mailbox %s failed: %s",
+ mailbox_get_vname(iter->box),
+ mailbox_get_last_internal_error(iter->box, NULL));
+ ret = -1;
+ }
+ }
+ if (iter->t == NULL)
+ ;
+ else if (commit) {
+ if (mailbox_transaction_commit(&iter->t) < 0) {
+ i_error("Committing mailbox %s failed: %s",
+ mailbox_get_vname(iter->box),
+ mailbox_get_last_internal_error(iter->box, NULL));
+ ret = -1;
+ }
+ } else {
+ mailbox_transaction_rollback(&iter->t);
+ }
+ mail_search_args_deinit(iter->search_args);
+ return ret;
+}
+
+static int
+doveadm_mail_iter_deinit_full(struct doveadm_mail_iter **_iter,
+ bool sync, bool commit, bool keep_box)
+{
+ struct doveadm_mail_iter *iter = *_iter;
+ int ret;
+
+ *_iter = NULL;
+
+ ret = doveadm_mail_iter_deinit_transaction(iter, commit);
+ if (ret == 0 && sync) {
+ ret = mailbox_sync(iter->box, 0);
+ if (ret < 0) {
+ i_error("Mailbox %s: Mailbox sync failed: %s",
+ mailbox_get_vname(iter->box),
+ mailbox_get_last_internal_error(iter->box, NULL));
+ }
+ }
+ if (ret < 0)
+ doveadm_mail_failed_mailbox(iter->ctx, iter->box);
+ else if (iter->killed) {
+ iter->ctx->exit_code = EX_TEMPFAIL;
+ ret = -1;
+ }
+ if (!keep_box)
+ mailbox_free(&iter->box);
+ i_free(iter);
+ return ret;
+}
+
+int doveadm_mail_iter_deinit(struct doveadm_mail_iter **_iter)
+{
+ return doveadm_mail_iter_deinit_full(_iter, FALSE, TRUE, FALSE);
+}
+
+int doveadm_mail_iter_deinit_sync(struct doveadm_mail_iter **_iter)
+{
+ return doveadm_mail_iter_deinit_full(_iter, TRUE, TRUE, FALSE);
+}
+
+int doveadm_mail_iter_deinit_keep_box(struct doveadm_mail_iter **iter,
+ struct mailbox **box_r)
+{
+ *box_r = (*iter)->box;
+ return doveadm_mail_iter_deinit_full(iter, FALSE, TRUE, TRUE);
+}
+
+void doveadm_mail_iter_deinit_rollback(struct doveadm_mail_iter **_iter)
+{
+ (void)doveadm_mail_iter_deinit_full(_iter, FALSE, FALSE, FALSE);
+}
+
+bool doveadm_mail_iter_next(struct doveadm_mail_iter *iter,
+ struct mail **mail_r)
+{
+ if (iter->search_ctx == NULL)
+ return FALSE;
+ if (doveadm_is_killed()) {
+ iter->killed = TRUE;
+ return FALSE;
+ }
+ if ((iter->flags & DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT) != 0 &&
+ doveadm_print_ostream->stream_errno != 0) {
+ iter->killed = TRUE;
+ return FALSE;
+ }
+ return mailbox_search_next(iter->search_ctx, mail_r);
+}
+
+struct mailbox *doveadm_mail_iter_get_mailbox(struct doveadm_mail_iter *iter)
+{
+ return iter->box;
+}
diff --git a/src/doveadm/doveadm-mail-iter.h b/src/doveadm/doveadm-mail-iter.h
new file mode 100644
index 0000000..32cc232
--- /dev/null
+++ b/src/doveadm/doveadm-mail-iter.h
@@ -0,0 +1,34 @@
+#ifndef DOVEADM_MAIL_ITER_H
+#define DOVEADM_MAIL_ITER_H
+
+#include "mailbox-list-iter.h"
+
+enum doveadm_mail_iter_flags {
+ /* Open the mailbox with MAILBOX_FLAG_READONLY */
+ DOVEADM_MAIL_ITER_FLAG_READONLY = BIT(0),
+ /* Stop the iteration if client is detected to be disconnected. */
+ DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT = BIT(1),
+};
+
+struct doveadm_mail_iter;
+struct doveadm_mail_cmd_context;
+
+int doveadm_mail_iter_init(struct doveadm_mail_cmd_context *ctx,
+ const struct mailbox_info *info,
+ struct mail_search_args *search_args,
+ enum mail_fetch_field wanted_fields,
+ const char *const *wanted_headers,
+ enum doveadm_mail_iter_flags flags,
+ struct doveadm_mail_iter **iter_r) ATTR_NULL(6);
+int doveadm_mail_iter_deinit(struct doveadm_mail_iter **iter);
+int doveadm_mail_iter_deinit_sync(struct doveadm_mail_iter **iter);
+int doveadm_mail_iter_deinit_keep_box(struct doveadm_mail_iter **iter,
+ struct mailbox **box_r);
+void doveadm_mail_iter_deinit_rollback(struct doveadm_mail_iter **iter);
+struct mailbox *doveadm_mail_iter_get_mailbox(struct doveadm_mail_iter *iter);
+
+bool doveadm_mail_iter_next(struct doveadm_mail_iter *iter,
+ struct mail **mail_r);
+
+#endif
+
diff --git a/src/doveadm/doveadm-mail-mailbox-cache.c b/src/doveadm/doveadm-mail-mailbox-cache.c
new file mode 100644
index 0000000..718fca4
--- /dev/null
+++ b/src/doveadm/doveadm-mail-mailbox-cache.c
@@ -0,0 +1,440 @@
+/* Copyright (c) 2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "time-util.h"
+#include "mail-index-private.h"
+#include "mail-cache-private.h"
+#include "mail-namespace.h"
+#include "mail-storage-private.h"
+#include "doveadm-print.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail.h"
+
+struct mailbox_cache_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+
+ const char *const *boxes;
+ const char *const *fields;
+ uint64_t last_used;
+ enum mail_cache_decision_type decision;
+ bool all_fields;
+ bool set_decision;
+ bool set_last_used;
+ bool remove;
+};
+
+static int cmd_mailbox_cache_open_box(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user,
+ const char *boxname,
+ struct mailbox **box_r)
+{
+ struct mailbox *box = doveadm_mailbox_find(user, boxname);
+
+ if (mailbox_open(box) < 0 || mailbox_sync(box, 0) < 0) {
+ i_error("Cannot open mailbox %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(ctx, box);
+ mailbox_free(&box);
+ return -1;
+ }
+
+ *box_r = box;
+
+ return 0;
+}
+
+static void cmd_mailbox_cache_decision_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+ const char *fields;
+
+ doveadm_print_header("mailbox", "mailbox", DOVEADM_PRINT_HEADER_FLAG_STICKY);
+ doveadm_print_header_simple("field");
+ doveadm_print_header_simple("decision");
+ doveadm_print_header_simple("last-used");
+
+ if (!ctx->all_fields &&
+ !doveadm_cmd_param_str(_ctx->cctx, "fieldstr", &fields)) {
+ i_fatal("Missing fields parameter");
+ } else if (!ctx->all_fields) {
+ ctx->fields = t_strsplit_spaces(fields, ", ");
+ }
+
+ ctx->boxes = args;
+}
+
+static bool
+cmd_mailbox_cache_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+
+ switch(c) {
+ case 'a':
+ ctx->all_fields = TRUE;
+ return TRUE;
+ /* this is handled in doveadm-mail as 'fieldstr' field */
+ case 'f':
+ return TRUE;
+ case 'l':
+ if (str_to_uint64(optarg, &ctx->last_used) < 0) {
+ i_error("Invalid last-used '%s': not a number", optarg);
+ return FALSE;
+ }
+ ctx->set_last_used = TRUE;
+ return TRUE;
+ case 'd':
+ if (ctx->set_decision) {
+ i_error("Only one decision flag allowed");
+ return FALSE;
+ }
+ if (strcmp(optarg, "no") == 0) {
+ ctx->decision = MAIL_CACHE_DECISION_NO;
+ } else if (strcmp(optarg, "temp") == 0) {
+ ctx->decision = MAIL_CACHE_DECISION_TEMP;
+ } else if (strcmp(optarg, "yes") == 0) {
+ ctx->decision = MAIL_CACHE_DECISION_YES;
+ } else {
+ i_error("Invalid decision '%s': " \
+ "must be one of yes, temp, no",
+ optarg);
+ return FALSE;
+ }
+ ctx->set_decision = TRUE;
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static const char *
+cmd_mailbox_cache_decision_to_str(enum mail_cache_decision_type decision)
+{
+ string_t *ret = t_str_new(10);
+ switch((decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED))) {
+ case MAIL_CACHE_DECISION_NO:
+ str_append(ret, "no");
+ break;
+ case MAIL_CACHE_DECISION_TEMP:
+ str_append(ret, "temp");
+ break;
+ case MAIL_CACHE_DECISION_YES:
+ str_append(ret, "yes");
+ break;
+ }
+ return str_c(ret);
+}
+
+static void
+cmd_mailbox_cache_decision_process_field(struct mailbox_cache_cmd_context *ctx,
+ struct mail_cache_field_private *field)
+{
+ if (ctx->set_decision) {
+ field->field.decision = ctx->decision;
+ field->decision_dirty = TRUE;
+ }
+
+ if (ctx->set_last_used) {
+ field->field.last_used = (time_t)ctx->last_used;
+ field->decision_dirty = TRUE;
+ }
+
+ doveadm_print(cmd_mailbox_cache_decision_to_str(field->field.decision));
+ doveadm_print(t_strflocaltime("%F %T %Z", field->field.last_used));
+}
+
+static void
+cmd_mailbox_cache_decision_run_per_field(struct mailbox_cache_cmd_context *ctx,
+ struct mail_cache *cache)
+{
+ const char *const *field_name;
+ for(field_name = ctx->fields; *field_name != NULL; field_name++) {
+ doveadm_print(*field_name);
+ /* see if the field exists */
+ unsigned int idx = mail_cache_register_lookup(cache,
+ *field_name);
+ if (idx == UINT_MAX) {
+ doveadm_print("<not found>");
+ doveadm_print("");
+ continue;
+ }
+
+ cmd_mailbox_cache_decision_process_field(ctx, &cache->fields[idx]);
+ }
+}
+
+static void
+cmd_mailbox_cache_decision_run_all_fields(struct mailbox_cache_cmd_context *ctx,
+ struct mail_cache *cache)
+{
+ /* get all fields */
+ for(unsigned int i = 0; i < cache->fields_count; i++) {
+ doveadm_print(cache->fields[i].field.name);
+ cmd_mailbox_cache_decision_process_field(ctx, &cache->fields[i]);
+ }
+}
+
+static int cmd_mailbox_cache_decision_run_box(struct mailbox_cache_cmd_context *ctx,
+ struct mailbox *box)
+{
+ struct mailbox_transaction_context *t =
+ mailbox_transaction_begin(box, ctx->ctx.transaction_flags,
+ "mailbox cache decision");
+ struct mail_cache *cache = t->box->cache;
+ struct mail_cache_view *view;
+
+ if (mail_cache_open_and_verify(cache) < 0 ||
+ MAIL_CACHE_IS_UNUSABLE(cache)) {
+ mailbox_transaction_rollback(&t);
+ i_error("Cache is unusable");
+ ctx->ctx.exit_code = EX_TEMPFAIL;
+ return -1;
+ }
+
+ view = mail_cache_view_open(cache, t->box->view);
+
+ if (ctx->all_fields)
+ cmd_mailbox_cache_decision_run_all_fields(ctx, cache);
+ else
+ cmd_mailbox_cache_decision_run_per_field(ctx, cache);
+
+ /* update headers */
+ if (ctx->set_decision || ctx->set_last_used)
+ mail_cache_header_fields_update(cache);
+
+ mail_cache_view_close(&view);
+
+ if (mailbox_transaction_commit(&t) < 0) {
+ i_error("mailbox_transaction_commit() failed: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ return -1;
+ }
+ return 0;
+}
+
+static int cmd_mailbox_cache_decision_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+ const char *const *boxname;
+ int ret = 0;
+
+ if (_ctx->exit_code != 0)
+ return -1;
+
+ for(boxname = ctx->boxes; ret == 0 && *boxname != NULL; boxname++) {
+ struct mailbox *box;
+ if ((ret = cmd_mailbox_cache_open_box(_ctx, user, *boxname, &box)) < 0)
+ break;
+ doveadm_print_sticky("mailbox", mailbox_get_vname(box));
+ ret = cmd_mailbox_cache_decision_run_box(ctx, box);
+ mailbox_free(&box);
+ }
+
+ return ret;
+}
+
+static int cmd_mailbox_cache_remove_box(struct mailbox_cache_cmd_context *ctx,
+ const struct mailbox_info *info)
+{
+ struct doveadm_mail_iter *iter;
+ struct mailbox *box;
+ struct mail *mail;
+ void *empty = NULL;
+ int ret = 0, count = 0;
+
+ if (doveadm_mail_iter_init(&ctx->ctx, info, ctx->ctx.search_args,
+ 0, NULL, 0, &iter) < 0)
+ return -1;
+
+ box = doveadm_mail_iter_get_mailbox(iter);
+
+ struct mail_index_transaction *t =
+ mail_index_transaction_begin(box->view, MAIL_INDEX_TRANSACTION_FLAG_EXTERNAL);
+ struct mail_cache_view *view =
+ mail_cache_view_open(box->cache, box->view);
+
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ count++;
+ doveadm_print(mailbox_get_vname(box));
+ doveadm_print(dec2str(mail->uid));
+ /* drop cache pointer */
+ mail_index_update_ext(t, mail->seq, view->cache->ext_id, &empty, NULL);
+ doveadm_print("ok");
+ }
+
+ if (mail_index_transaction_commit(&t) < 0) {
+ i_error("mail_index_transaction_commit() failed: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ ret = -1;
+ } else {
+ mail_cache_expunge_count(view->cache, count);
+ }
+
+ mail_cache_view_close(&view);
+
+ if (doveadm_mail_iter_deinit(&iter) < 0)
+ ret = -1;
+
+ return ret;
+}
+
+static int cmd_mailbox_cache_remove_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(&ctx->ctx, user, ctx->ctx.search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_mailbox_cache_remove_box(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_mailbox_cache_remove_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+
+ if (args[0] == NULL)
+ doveadm_mail_help_name("mailbox cache remove");
+
+ doveadm_print_header_simple("mailbox");
+ doveadm_print_header_simple("uid");
+ doveadm_print_header_simple("result");
+
+ ctx->ctx.search_args = doveadm_mail_build_search_args(args);
+}
+
+static int cmd_mailbox_cache_purge_run_box(struct mailbox_cache_cmd_context *ctx,
+ struct mailbox *box)
+{
+ if (mail_cache_purge(box->cache, (uint32_t)-1,
+ "doveadm mailbox cache purge") < 0) {
+ mailbox_set_index_error(box);
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ return -1;
+ }
+ return 0;
+}
+
+static int cmd_mailbox_cache_purge_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+ const char *const *boxname;
+ int ret = 0;
+
+ if (_ctx->exit_code != 0)
+ return -1;
+
+ for(boxname = ctx->boxes; ret == 0 && *boxname != NULL; boxname++) {
+ struct mailbox *box;
+ if ((ret = cmd_mailbox_cache_open_box(_ctx, user, *boxname, &box)) < 0)
+ break;
+ ret = cmd_mailbox_cache_purge_run_box(ctx, box);
+ mailbox_free(&box);
+ }
+
+ return ret;
+}
+
+static void cmd_mailbox_cache_purge_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct mailbox_cache_cmd_context *ctx =
+ container_of(_ctx, struct mailbox_cache_cmd_context, ctx);
+
+ ctx->boxes = args;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_cache_decision_alloc(void)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ doveadm_mail_cmd_alloc(struct mailbox_cache_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_cache_decision_init;
+ ctx->ctx.v.parse_arg = cmd_mailbox_cache_parse_arg;
+ ctx->ctx.v.run = cmd_mailbox_cache_decision_run;
+ ctx->ctx.getopt_args = "al:f:d:";
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ return &ctx->ctx;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_cache_remove_alloc(void)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ doveadm_mail_cmd_alloc(struct mailbox_cache_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_cache_remove_init;
+ ctx->ctx.v.parse_arg = cmd_mailbox_cache_parse_arg;
+ ctx->ctx.v.run = cmd_mailbox_cache_remove_run;
+ ctx->ctx.getopt_args = "";
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ return &ctx->ctx;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_cache_purge_alloc(void)
+{
+ struct mailbox_cache_cmd_context *ctx =
+ doveadm_mail_cmd_alloc(struct mailbox_cache_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_cache_purge_init;
+ ctx->ctx.v.run = cmd_mailbox_cache_purge_run;
+ ctx->ctx.getopt_args = "";
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_decision = {
+ .name = "mailbox cache decision",
+ .mail_cmd = cmd_mailbox_cache_decision_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"--all --fields <fields> " \
+ "--last-used <timestamp> --decision <decision> " \
+ "<mailbox> [<mailbox> ... ]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('a', "all", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('f', "fieldstr", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('l', "last-used", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('d', "decision", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_remove = {
+ .name = "mailbox cache remove",
+ .mail_cmd = cmd_mailbox_cache_remove_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<search string>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_purge = {
+ .name = "mailbox cache purge",
+ .mail_cmd = cmd_mailbox_cache_purge_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<mailbox> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-mailbox-metadata.c b/src/doveadm/doveadm-mail-mailbox-metadata.c
new file mode 100644
index 0000000..593555b
--- /dev/null
+++ b/src/doveadm/doveadm-mail-mailbox-metadata.c
@@ -0,0 +1,425 @@
+/* Copyright (c) 2014-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "istream.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "doveadm-print.h"
+#include "doveadm-mail.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "imap-metadata.h"
+
+enum doveadm_metadata_op {
+ DOVEADM_METADATA_OP_SET = 0,
+ DOVEADM_METADATA_OP_GET,
+ DOVEADM_METADATA_OP_LIST,
+};
+
+const char *doveadm_metadata_op_names[] = {
+ "set attribute",
+ "get attribute",
+ "list attribute",
+};
+
+struct metadata_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ const char *mailbox;
+ enum mail_attribute_type key_type;
+ const char *key;
+ struct mail_attribute_value value;
+ bool empty_mailbox_name;
+ bool allow_empty_mailbox_name;
+ bool prepend_prefix;
+};
+
+static int
+cmd_mailbox_metadata_get_mailbox(struct metadata_cmd_context *mctx,
+ struct mail_user *user,
+ enum doveadm_metadata_op op,
+ struct mail_namespace **ns_r,
+ struct mailbox **box_r)
+{
+ mctx->empty_mailbox_name = mctx->mailbox[0] == '\0';
+
+ if (mctx->empty_mailbox_name) {
+ if (!mctx->allow_empty_mailbox_name) {
+ const char *op_str = doveadm_metadata_op_names[op];
+ i_error("Failed to %s: %s", op_str,
+ "mailbox name cannot be empty");
+ mctx->ctx.exit_code = EX_USAGE;
+ return -1;
+ }
+
+ /* Server attribute. It shouldn't depend on INBOX's ACLs,
+ so ignore them. */
+ *ns_r = mail_namespace_find_inbox(user->namespaces);
+ *box_r = mailbox_alloc((*ns_r)->list, "INBOX",
+ MAILBOX_FLAG_IGNORE_ACLS |
+ MAILBOX_FLAG_ATTRIBUTE_SESSION);
+
+ mctx->key = t_strconcat(MAILBOX_ATTRIBUTE_PREFIX_DOVECOT_PVT_SERVER,
+ mctx->key, NULL);
+ } else {
+ /* mailbox attributes */
+ *ns_r = mail_namespace_find(user->namespaces, mctx->mailbox);
+ *box_r = mailbox_alloc((*ns_r)->list, mctx->mailbox,
+ MAILBOX_FLAG_ATTRIBUTE_SESSION);
+ }
+
+ if (op == DOVEADM_METADATA_OP_SET &&
+ mailbox_open(*box_r) < 0) {
+ i_error("Failed to open mailbox: %s",
+ mailbox_get_last_internal_error(*box_r, NULL));
+ doveadm_mail_failed_mailbox(&mctx->ctx, *box_r);
+ mailbox_free(box_r);
+ return -1;
+ }
+
+ return 0;
+}
+
+static int
+cmd_mailbox_metadata_set_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ struct mailbox_transaction_context *trans;
+ int ret;
+
+ ret = cmd_mailbox_metadata_get_mailbox(ctx, user, DOVEADM_METADATA_OP_SET,
+ &ns, &box);
+ if (ret != 0)
+ return ret;
+
+ trans = mailbox_transaction_begin(box, (ctx->empty_mailbox_name ?
+ MAILBOX_TRANSACTION_FLAG_EXTERNAL : 0) |
+ ctx->ctx.transaction_flags, __func__);
+
+ ret = ctx->value.value == NULL ?
+ mailbox_attribute_unset(trans, ctx->key_type, ctx->key) :
+ mailbox_attribute_set(trans, ctx->key_type, ctx->key, &ctx->value);
+ if (ret < 0) {
+ i_error("Failed to set attribute: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ mailbox_transaction_rollback(&trans);
+ } else if (mailbox_transaction_commit(&trans) < 0) {
+ i_error("Failed to commit transaction: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+
+ mailbox_free(&box);
+ return ret;
+}
+
+static void
+cmd_mailbox_metadata_parse_key(const char *arg,
+ enum mail_attribute_type *type_r,
+ const char **key_r)
+{
+ arg = t_str_lcase(arg);
+
+ if (str_begins(arg, "/private/")) {
+ *type_r = MAIL_ATTRIBUTE_TYPE_PRIVATE;
+ *key_r = arg + 9;
+ } else if (str_begins(arg, "/shared/")) {
+ *type_r = MAIL_ATTRIBUTE_TYPE_SHARED;
+ *key_r = arg + 8;
+ } else if (strcmp(arg, "/private") == 0) {
+ *type_r = MAIL_ATTRIBUTE_TYPE_PRIVATE;
+ *key_r = "";
+ } else if (strcmp(arg, "/shared") == 0) {
+ *type_r = MAIL_ATTRIBUTE_TYPE_SHARED;
+ *key_r = "";
+ } else {
+ i_fatal_status(EX_USAGE, "Invalid metadata key '%s': "
+ "Must begin with /private or /shared", arg);
+ }
+}
+
+static void
+cmd_mailbox_metadata_set_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ const char *key;
+
+ if (str_array_length(args) != 3)
+ doveadm_mail_help_name("mailbox metadata set");
+ cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key);
+
+ ctx->mailbox = p_strdup(_ctx->pool, args[0]);
+ ctx->key = p_strdup(_ctx->pool, key);
+ ctx->value.value = p_strdup(_ctx->pool, args[2]);
+}
+
+static bool
+cmd_mailbox_metadata_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct metadata_cmd_context *ctx =
+ (struct metadata_cmd_context *)_ctx;
+
+ switch (c) {
+ case 's':
+ ctx->allow_empty_mailbox_name = TRUE;
+ break;
+ case 'p':
+ ctx->prepend_prefix = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_set_alloc(void)
+{
+ struct metadata_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_metadata_set_init;
+ ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg;
+ ctx->ctx.v.run = cmd_mailbox_metadata_set_run;
+ return &ctx->ctx;
+}
+
+static void
+cmd_mailbox_metadata_unset_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ const char *key;
+
+ if (str_array_length(args) != 2)
+ doveadm_mail_help_name("mailbox metadata unset");
+ cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key);
+
+ ctx->mailbox = p_strdup(_ctx->pool, args[0]);
+ ctx->key = p_strdup(_ctx->pool, key);
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_unset_alloc(void)
+{
+ struct metadata_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_metadata_unset_init;
+ ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg;
+ ctx->ctx.v.run = cmd_mailbox_metadata_set_run;
+ return &ctx->ctx;
+}
+
+static int
+cmd_mailbox_metadata_get_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ struct mail_attribute_value value;
+ int ret;
+
+ ret = cmd_mailbox_metadata_get_mailbox(ctx, user, DOVEADM_METADATA_OP_GET,
+ &ns, &box);
+ if (ret != 0)
+ return ret;
+
+ ret = mailbox_attribute_get_stream(box, ctx->key_type, ctx->key, &value);
+ if (ret < 0) {
+ i_error("Failed to get attribute: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ } else if (ret == 0) {
+ /* not found, print as empty */
+ doveadm_print("");
+ } else if (value.value_stream != NULL) {
+ if (doveadm_print_istream(value.value_stream) < 0) {
+ i_error("read(%s) failed: %s",
+ i_stream_get_name(value.value_stream),
+ i_stream_get_error(value.value_stream));
+ ret = -1;
+ }
+ } else {
+ doveadm_print(value.value);
+ }
+
+ mailbox_free(&box);
+ return ret;
+}
+
+static void
+cmd_mailbox_metadata_get_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ const char *key;
+
+ if (str_array_length(args) != 2)
+ doveadm_mail_help_name("mailbox metadata get");
+ cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key);
+
+ ctx->mailbox = p_strdup(_ctx->pool, args[0]);
+ ctx->key = p_strdup(_ctx->pool, key);
+ doveadm_print_header("value", "value",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_get_alloc(void)
+{
+ struct metadata_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_metadata_get_init;
+ ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg;
+ ctx->ctx.v.run = cmd_mailbox_metadata_get_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return &ctx->ctx;
+}
+
+static int
+cmd_mailbox_metadata_list_run_iter(struct metadata_cmd_context *ctx,
+ struct mailbox *box,
+ enum mail_attribute_type type)
+{
+ struct mailbox_attribute_iter *iter;
+ const char *key;
+ string_t *outp = t_str_new(64);
+
+ iter = mailbox_attribute_iter_init(box, type, ctx->key);
+ while ((key = mailbox_attribute_iter_next(iter)) != NULL) {
+ if (ctx->prepend_prefix) {
+ if (type == MAIL_ATTRIBUTE_TYPE_PRIVATE)
+ str_append(outp, IMAP_METADATA_PRIVATE_PREFIX"/");
+ else if (type == MAIL_ATTRIBUTE_TYPE_SHARED)
+ str_append(outp, IMAP_METADATA_SHARED_PREFIX"/");
+ else
+ i_unreached();
+ str_append(outp, key);
+ doveadm_print(str_c(outp));
+ str_truncate(outp, 0);
+ } else {
+ doveadm_print(key);
+ }
+ }
+ if (mailbox_attribute_iter_deinit(&iter) < 0) {
+ i_error("Mailbox %s: Failed to iterate mailbox attributes: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ return -1;
+ }
+ return 0;
+}
+
+static int
+cmd_mailbox_metadata_list_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ int ret = 0;
+
+ ret = cmd_mailbox_metadata_get_mailbox(ctx, user, DOVEADM_METADATA_OP_LIST,
+ &ns, &box);
+ if (ret != 0)
+ return ret;
+
+ if (ctx->key[0] == '\0' || ctx->key_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
+ if (cmd_mailbox_metadata_list_run_iter(ctx, box, MAIL_ATTRIBUTE_TYPE_PRIVATE) < 0) {
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ }
+ if (ctx->key[0] == '\0' || ctx->key_type == MAIL_ATTRIBUTE_TYPE_SHARED) {
+ if (cmd_mailbox_metadata_list_run_iter(ctx, box, MAIL_ATTRIBUTE_TYPE_SHARED) < 0) {
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ }
+ mailbox_free(&box);
+ return ret;
+}
+
+static void
+cmd_mailbox_metadata_list_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct metadata_cmd_context *ctx = (struct metadata_cmd_context *)_ctx;
+ const char *key = NULL;
+
+ if (args[0] == NULL)
+ doveadm_mail_help_name("mailbox metadata list");
+ if (args[1] != NULL)
+ cmd_mailbox_metadata_parse_key(args[1], &ctx->key_type, &key);
+ ctx->mailbox = p_strdup(_ctx->pool, args[0]);
+ ctx->key = key == NULL ? "" : p_strdup(_ctx->pool, key);
+ doveadm_print_header("key", "key",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_metadata_list_alloc(void)
+{
+ struct metadata_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct metadata_cmd_context);
+ ctx->ctx.v.init = cmd_mailbox_metadata_list_init;
+ ctx->ctx.v.parse_arg = cmd_mailbox_metadata_parse_arg;
+ ctx->ctx.v.run = cmd_mailbox_metadata_list_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_set_ver2 = {
+ .name = "mailbox metadata set",
+ .mail_cmd = cmd_mailbox_metadata_set_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> <key> <value>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "value", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_unset_ver2 = {
+ .name = "mailbox metadata unset",
+ .mail_cmd = cmd_mailbox_metadata_unset_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> <key>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_get_ver2 = {
+ .name = "mailbox metadata get",
+ .mail_cmd = cmd_mailbox_metadata_get_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <mailbox> <key>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_list_ver2 = {
+ .name = "mailbox metadata list",
+ .mail_cmd = cmd_mailbox_metadata_list_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] [-p] <mailbox> [<key prefix>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('s', "allow-empty-mailbox-name", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('p', "prepend-prefix", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "key-prefix", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-mailbox-status.c b/src/doveadm/doveadm-mail-mailbox-status.c
new file mode 100644
index 0000000..9f47095
--- /dev/null
+++ b/src/doveadm/doveadm-mail-mailbox-status.c
@@ -0,0 +1,275 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "mail-search.h"
+#include "doveadm-print.h"
+#include "doveadm-mail.h"
+#include "doveadm-mailbox-list-iter.h"
+
+#define ALL_STATUS_ITEMS \
+ (STATUS_MESSAGES | STATUS_RECENT | \
+ STATUS_UIDNEXT | STATUS_UIDVALIDITY | \
+ STATUS_UNSEEN | STATUS_HIGHESTMODSEQ)
+#define ALL_METADATA_ITEMS \
+ (MAILBOX_METADATA_VIRTUAL_SIZE | MAILBOX_METADATA_GUID | \
+ MAILBOX_METADATA_FIRST_SAVE_DATE)
+
+#define TOTAL_STATUS_ITEMS \
+ (STATUS_MESSAGES | STATUS_RECENT | STATUS_UNSEEN)
+#define TOTAL_METADATA_ITEMS \
+ (MAILBOX_METADATA_VIRTUAL_SIZE)
+
+struct status_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ struct mail_search_args *search_args;
+
+ enum mailbox_status_items status_items;
+ enum mailbox_metadata_items metadata_items;
+ struct mailbox_status total_status;
+ struct mailbox_metadata total_metadata;
+
+ bool total_sum:1;
+};
+
+static void status_parse_fields(struct status_cmd_context *ctx,
+ const char *const *fields)
+{
+ if (*fields == NULL)
+ i_fatal_status(EX_USAGE, "No status fields");
+
+ for (; *fields != NULL; fields++) {
+ const char *field = *fields;
+
+ if (strcmp(field, "all") == 0) {
+ if (ctx->total_sum) {
+ ctx->status_items |= TOTAL_STATUS_ITEMS;
+ ctx->metadata_items |= TOTAL_METADATA_ITEMS;
+ } else {
+ ctx->status_items |= ALL_STATUS_ITEMS;
+ ctx->metadata_items |= ALL_METADATA_ITEMS;
+ }
+ } else if (strcmp(field, "messages") == 0)
+ ctx->status_items |= STATUS_MESSAGES;
+ else if (strcmp(field, "recent") == 0)
+ ctx->status_items |= STATUS_RECENT;
+ else if (strcmp(field, "uidnext") == 0)
+ ctx->status_items |= STATUS_UIDNEXT;
+ else if (strcmp(field, "uidvalidity") == 0)
+ ctx->status_items |= STATUS_UIDVALIDITY;
+ else if (strcmp(field, "unseen") == 0)
+ ctx->status_items |= STATUS_UNSEEN;
+ else if (strcmp(field, "highestmodseq") == 0)
+ ctx->status_items |= STATUS_HIGHESTMODSEQ;
+ else if (strcmp(field, "vsize") == 0)
+ ctx->metadata_items |= MAILBOX_METADATA_VIRTUAL_SIZE;
+ else if (strcmp(field, "guid") == 0)
+ ctx->metadata_items |= MAILBOX_METADATA_GUID;
+ else if (strcmp(field, "firstsaved") == 0)
+ ctx->metadata_items |= MAILBOX_METADATA_FIRST_SAVE_DATE;
+ else {
+ i_fatal_status(EX_USAGE,
+ "Unknown status field: %s", field);
+ }
+
+ if (ctx->total_sum &&
+ ((ctx->status_items & ENUM_NEGATE(TOTAL_STATUS_ITEMS)) != 0 ||
+ (ctx->metadata_items & ENUM_NEGATE(TOTAL_METADATA_ITEMS)) != 0)) {
+ i_fatal_status(EX_USAGE,
+ "Status field %s can't be used with -t", field);
+ }
+ }
+}
+
+static void ATTR_NULL(2)
+status_output(struct status_cmd_context *ctx, struct mailbox *box,
+ const struct mailbox_status *status,
+ const struct mailbox_metadata *metadata)
+{
+ if (box != NULL)
+ doveadm_print(mailbox_get_vname(box));
+
+ if ((ctx->status_items & STATUS_MESSAGES) != 0)
+ doveadm_print_num(status->messages);
+ if ((ctx->status_items & STATUS_RECENT) != 0)
+ doveadm_print_num(status->recent);
+ if ((ctx->status_items & STATUS_UIDNEXT) != 0)
+ doveadm_print_num(status->uidnext);
+ if ((ctx->status_items & STATUS_UIDVALIDITY) != 0)
+ doveadm_print_num(status->uidvalidity);
+ if ((ctx->status_items & STATUS_UNSEEN) != 0)
+ doveadm_print_num(status->unseen);
+ if ((ctx->status_items & STATUS_HIGHESTMODSEQ) != 0)
+ doveadm_print_num(status->highest_modseq);
+ if ((ctx->metadata_items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0)
+ doveadm_print_num(metadata->virtual_size);
+ if ((ctx->metadata_items & MAILBOX_METADATA_GUID) != 0)
+ doveadm_print(guid_128_to_string(metadata->guid));
+ if ((ctx->metadata_items & MAILBOX_METADATA_FIRST_SAVE_DATE) > 0) {
+ if (metadata->first_save_date > -1)
+ doveadm_print_num(metadata->first_save_date);
+ else
+ doveadm_print("never");
+ }
+}
+
+static void
+status_sum(struct status_cmd_context *ctx,
+ const struct mailbox_status *status,
+ const struct mailbox_metadata *metadata)
+{
+ struct mailbox_status *dest = &ctx->total_status;
+
+ dest->messages += status->messages;
+ dest->recent += status->recent;
+ dest->unseen += status->unseen;
+ ctx->total_metadata.virtual_size += metadata->virtual_size;
+}
+
+static int
+status_mailbox(struct status_cmd_context *ctx, const struct mailbox_info *info)
+{
+ struct mailbox *box;
+ struct mailbox_status status;
+ struct mailbox_metadata metadata;
+
+ box = doveadm_mailbox_find(ctx->ctx.cur_mail_user, info->vname);
+ if (mailbox_get_status(box, ctx->status_items, &status) < 0 ||
+ mailbox_get_metadata(box, ctx->metadata_items, &metadata) < 0) {
+ i_error("Mailbox %s: Failed to lookup mailbox status: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(&ctx->ctx, box);
+ mailbox_free(&box);
+ return -1;
+ }
+ if (!ctx->total_sum)
+ status_output(ctx, box, &status, &metadata);
+ else
+ status_sum(ctx, &status, &metadata);
+ mailbox_free(&box);
+ return 0;
+}
+
+static int
+cmd_mailbox_status_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx;
+ enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ i_zero(&ctx->total_status);
+ i_zero(&ctx->total_metadata);
+
+ iter = doveadm_mailbox_list_iter_init(_ctx, user, ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) {
+ T_BEGIN {
+ if (status_mailbox(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ }
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+
+ if (ctx->total_sum) {
+ status_output(ctx, NULL, &ctx->total_status,
+ &ctx->total_metadata);
+ }
+ return ret;
+}
+
+static void cmd_mailbox_status_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx;
+ const char *fields = args[0];
+
+ if (fields == NULL || args[1] == NULL)
+ doveadm_mail_help_name("mailbox status");
+
+ status_parse_fields(ctx, t_strsplit_spaces(fields, " "));
+ ctx->search_args = doveadm_mail_mailbox_search_args_build(args+1);
+
+ if (!ctx->total_sum) {
+ doveadm_print_header("mailbox", "mailbox",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ }
+ if ((ctx->status_items & STATUS_MESSAGES) != 0)
+ doveadm_print_header_simple("messages");
+ if ((ctx->status_items & STATUS_RECENT) != 0)
+ doveadm_print_header_simple("recent");
+ if ((ctx->status_items & STATUS_UIDNEXT) != 0)
+ doveadm_print_header_simple("uidnext");
+ if ((ctx->status_items & STATUS_UIDVALIDITY) != 0)
+ doveadm_print_header_simple("uidvalidity");
+ if ((ctx->status_items & STATUS_UNSEEN) != 0)
+ doveadm_print_header_simple("unseen");
+ if ((ctx->status_items & STATUS_HIGHESTMODSEQ) != 0)
+ doveadm_print_header_simple("highestmodseq");
+ if ((ctx->metadata_items & MAILBOX_METADATA_VIRTUAL_SIZE) != 0)
+ doveadm_print_header_simple("vsize");
+ if ((ctx->metadata_items & MAILBOX_METADATA_GUID) != 0)
+ doveadm_print_header_simple("guid");
+ if ((ctx->metadata_items & MAILBOX_METADATA_FIRST_SAVE_DATE) != 0)
+ doveadm_print_header_simple("firstsaved");
+}
+
+static void cmd_mailbox_status_deinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx;
+
+ if (ctx->search_args != NULL)
+ mail_search_args_unref(&ctx->search_args);
+}
+
+static bool
+cmd_mailbox_status_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct status_cmd_context *ctx = (struct status_cmd_context *)_ctx;
+
+ switch (c) {
+ case 't':
+ ctx->total_sum = TRUE;
+ break;
+ case 'f':
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_status_alloc(void)
+{
+ struct status_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct status_cmd_context);
+ ctx->ctx.getopt_args = "t";
+ ctx->ctx.v.parse_arg = cmd_mailbox_status_parse_arg;
+ ctx->ctx.v.init = cmd_mailbox_status_init;
+ ctx->ctx.v.deinit = cmd_mailbox_status_deinit;
+ ctx->ctx.v.run = cmd_mailbox_status_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_status_ver2 = {
+ .name = "mailbox status",
+ .mail_cmd = cmd_mailbox_status_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<fields> <mailbox> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('t', "total-sum", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('f', "field", CMD_PARAM_ARRAY, 0)
+DOVEADM_CMD_PARAM('\0', "fieldstr", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL | CMD_PARAM_FLAG_DO_NOT_EXPOSE) /* FIXME: horrible hack, remove me when possible */
+DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-mailbox.c b/src/doveadm/doveadm-mail-mailbox.c
new file mode 100644
index 0000000..2b2c453
--- /dev/null
+++ b/src/doveadm/doveadm-mail-mailbox.c
@@ -0,0 +1,857 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "unichar.h"
+#include "imap-utf7.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "mail-search-build.h"
+#include "doveadm-print.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail.h"
+
+#include <stdio.h>
+
+struct doveadm_mailbox_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ bool subscriptions;
+};
+
+struct mailbox_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ ARRAY_TYPE(const_string) mailboxes;
+};
+
+struct create_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ ARRAY_TYPE(const_string) mailboxes;
+ struct mailbox_update update;
+};
+
+struct delete_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ ARRAY_TYPE(const_string) mailboxes;
+ bool recursive;
+ bool require_empty;
+ bool unsafe;
+};
+
+struct rename_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ const char *oldname, *newname;
+};
+
+struct list_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ struct mail_search_args *search_args;
+ bool mutf7;
+};
+
+struct update_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ const char *mailbox;
+ struct mailbox_update update;
+};
+
+struct path_cmd_context {
+ struct doveadm_mailbox_cmd_context ctx;
+ const char *mailbox;
+ enum mailbox_list_path_type path_type;
+};
+
+static const char *mailbox_list_path_type_names[] = {
+ "dir", "alt-dir", "mailbox", "alt-mailbox",
+ "control", "index", "index-private"
+};
+
+void doveadm_mailbox_args_check(const char *const args[])
+{
+ unsigned int i;
+
+ for (i = 0; args[i] != NULL; i++) {
+ if (!uni_utf8_str_is_valid(args[i])) {
+ i_fatal_status(EX_DATAERR,
+ "Mailbox name not valid UTF-8: %s", args[i]);
+ }
+ }
+}
+
+static bool cmd_mailbox_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct doveadm_mailbox_cmd_context *ctx =
+ (struct doveadm_mailbox_cmd_context *)_ctx;
+
+ switch (c) {
+ case 's':
+ ctx->subscriptions = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+#define doveadm_mailbox_cmd_alloc(type) \
+ (type *)doveadm_mailbox_cmd_alloc_size(sizeof(type))
+static struct doveadm_mail_cmd_context *
+doveadm_mailbox_cmd_alloc_size(size_t size)
+{
+ struct doveadm_mail_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc_size(size);
+ ctx->getopt_args = "s";
+ ctx->v.parse_arg = cmd_mailbox_parse_arg;
+ return ctx;
+}
+
+static bool
+cmd_mailbox_list_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx;
+
+ switch (c) {
+ case '7':
+ ctx->mutf7 = TRUE;
+ break;
+ case '8':
+ ctx->mutf7 = FALSE;
+ break;
+ case 's':
+ ctx->ctx.subscriptions = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static int
+cmd_mailbox_list_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx;
+ enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ string_t *str = t_str_new(256);
+
+ if (ctx->ctx.subscriptions)
+ iter_flags |= MAILBOX_LIST_ITER_SELECT_SUBSCRIBED;
+
+ iter = doveadm_mailbox_list_iter_full_init(_ctx, user, ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) {
+ if (!ctx->mutf7)
+ doveadm_print(info->vname);
+ else {
+ str_truncate(str, 0);
+ if (imap_utf8_to_utf7(info->vname, str) < 0)
+ i_unreached();
+ doveadm_print(str_c(str));
+ }
+ }
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ return -1;
+ return 0;
+}
+
+struct mail_search_args *
+doveadm_mail_mailbox_search_args_build(const char *const args[])
+{
+ struct mail_search_args *search_args;
+ struct mail_search_arg *arg;
+ enum mail_search_arg_type type;
+ unsigned int i;
+
+ doveadm_mailbox_args_check(args);
+ search_args = mail_search_build_init();
+ for (i = 0; args[i] != NULL; i++) {
+ if (strchr(args[i], '*') != NULL ||
+ strchr(args[i], '%') != NULL)
+ type = SEARCH_MAILBOX_GLOB;
+ else
+ type = SEARCH_MAILBOX;
+ arg = mail_search_build_add(search_args, type);
+ arg->value.str = p_strdup(search_args->pool, args[i]);
+ }
+ if (i > 1) {
+ struct mail_search_arg *subargs = search_args->args;
+
+ search_args->args = NULL;
+ arg = mail_search_build_add(search_args, SEARCH_OR);
+ arg->value.subargs = subargs;
+ }
+ return search_args;
+}
+
+static void cmd_mailbox_list_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx;
+
+ doveadm_print_header("mailbox", "mailbox",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ ctx->search_args = doveadm_mail_mailbox_search_args_build(args);
+}
+
+static void cmd_mailbox_list_deinit(struct doveadm_mail_cmd_context *_ctx)
+{
+ struct list_cmd_context *ctx = (struct list_cmd_context *)_ctx;
+
+ if (ctx->search_args != NULL)
+ mail_search_args_unref(&ctx->search_args);
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_list_alloc(void)
+{
+ struct list_cmd_context *ctx;
+
+ ctx = doveadm_mailbox_cmd_alloc(struct list_cmd_context);
+ ctx->ctx.ctx.v.init = cmd_mailbox_list_init;
+ ctx->ctx.ctx.v.deinit = cmd_mailbox_list_deinit;
+ ctx->ctx.ctx.v.run = cmd_mailbox_list_run;
+ ctx->ctx.ctx.v.parse_arg = cmd_mailbox_list_parse_arg;
+ ctx->ctx.ctx.getopt_args = "78s";
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return &ctx->ctx.ctx;
+}
+
+static int
+cmd_mailbox_create_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct create_cmd_context *ctx = (struct create_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ const char *name;
+ int ret = 0;
+
+ array_foreach_elem(&ctx->mailboxes, name) {
+ size_t len;
+ bool directory = FALSE;
+
+ ns = mail_namespace_find(user->namespaces, name);
+ len = strlen(name);
+ if (len > 0 && name[len-1] == mail_namespace_get_sep(ns)) {
+ name = t_strndup(name, len-1);
+ directory = TRUE;
+ }
+
+ box = mailbox_alloc(ns->list, name, 0);
+ if (mailbox_create(box, &ctx->update, directory) < 0) {
+ i_error("Can't create mailbox %s: %s", name,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ if (ctx->ctx.subscriptions) {
+ if (mailbox_set_subscribed(box, TRUE) < 0) {
+ i_error("Can't subscribe to mailbox %s: %s", name,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ }
+ mailbox_free(&box);
+ }
+ return ret;
+}
+
+static void cmd_mailbox_create_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx;
+ const char *name;
+ unsigned int i;
+
+ if (args[0] == NULL)
+ doveadm_mail_help_name("mailbox create");
+ doveadm_mailbox_args_check(args);
+
+ for (i = 0; args[i] != NULL; i++) {
+ name = p_strdup(ctx->ctx.ctx.pool, args[i]);
+ array_push_back(&ctx->mailboxes, &name);
+ }
+}
+
+static bool
+cmd_mailbox_create_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct create_cmd_context *ctx = (struct create_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'g':
+ if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0)
+ doveadm_mail_help_name("mailbox create");
+ break;
+ case 's':
+ ctx->ctx.subscriptions = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_create_alloc(void)
+{
+ struct create_cmd_context *ctx;
+
+ ctx = doveadm_mailbox_cmd_alloc(struct create_cmd_context);
+ ctx->ctx.ctx.v.init = cmd_mailbox_create_init;
+ ctx->ctx.ctx.v.run = cmd_mailbox_create_run;
+ ctx->ctx.ctx.v.parse_arg = cmd_mailbox_create_parse_arg;
+ ctx->ctx.ctx.getopt_args = "g:s";
+ p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16);
+ return &ctx->ctx.ctx;
+}
+
+static int i_strcmp_reverse_p(const char *const *s1, const char *const *s2)
+{
+ return -strcmp(*s1, *s2);
+}
+
+static int
+get_child_mailboxes(struct mail_user *user, ARRAY_TYPE(const_string) *mailboxes,
+ const char *name)
+{
+ struct mailbox_list_iterate_context *iter;
+ struct mail_namespace *ns;
+ const struct mailbox_info *info;
+ const char *pattern, *child_name;
+
+ ns = mail_namespace_find(user->namespaces, name);
+ pattern = name[0] == '\0' ? "*" :
+ t_strdup_printf("%s%c*", name, mail_namespace_get_sep(ns));
+ iter = mailbox_list_iter_init(ns->list, pattern,
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ child_name = t_strdup(info->vname);
+ array_push_back(mailboxes, &child_name);
+ }
+ return mailbox_list_iter_deinit(&iter);
+}
+
+static int
+cmd_mailbox_delete_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct delete_cmd_context *ctx = (struct delete_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ struct mail_storage *storage;
+ const char *name;
+ ARRAY_TYPE(const_string) recursive_mailboxes;
+ const ARRAY_TYPE(const_string) *mailboxes = &ctx->mailboxes;
+ enum mailbox_flags mailbox_flags = 0;
+ int ret = 0, ret2;
+
+ if (ctx->unsafe)
+ mailbox_flags |= MAILBOX_FLAG_DELETE_UNSAFE;
+ if (ctx->recursive) {
+ t_array_init(&recursive_mailboxes, 32);
+ array_foreach_elem(&ctx->mailboxes, name) {
+ if (get_child_mailboxes(user, &recursive_mailboxes,
+ name) < 0) {
+ doveadm_mail_failed_error(_ctx, MAIL_ERROR_TEMP);
+ ret = -1;
+ }
+ if (name[0] != '\0')
+ array_push_back(&recursive_mailboxes, &name);
+ }
+ array_sort(&recursive_mailboxes, i_strcmp_reverse_p);
+ mailboxes = &recursive_mailboxes;
+ }
+
+ array_foreach_elem(mailboxes, name) {
+ ns = mail_namespace_find(user->namespaces, name);
+ box = mailbox_alloc(ns->list, name, mailbox_flags);
+ storage = mailbox_get_storage(box);
+ ret2 = ctx->require_empty ? mailbox_delete_empty(box) :
+ mailbox_delete(box);
+ if (ret2 < 0) {
+ i_error("Can't delete mailbox %s: %s", name,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ if (ctx->ctx.subscriptions) {
+ if (mailbox_set_subscribed(box, FALSE) < 0) {
+ i_error("Can't unsubscribe mailbox %s: %s", name,
+ mail_storage_get_last_internal_error(storage, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ }
+ mailbox_free(&box);
+ }
+ return ret;
+}
+
+static void cmd_mailbox_delete_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct delete_cmd_context *ctx = (struct delete_cmd_context *)_ctx;
+ const char *name;
+ unsigned int i;
+
+ if (args[0] == NULL)
+ doveadm_mail_help_name("mailbox delete");
+ doveadm_mailbox_args_check(args);
+
+ for (i = 0; args[i] != NULL; i++) {
+ name = p_strdup(ctx->ctx.ctx.pool, args[i]);
+ array_push_back(&ctx->mailboxes, &name);
+ }
+ array_sort(&ctx->mailboxes, i_strcmp_reverse_p);
+}
+
+static bool
+cmd_mailbox_delete_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct delete_cmd_context *ctx = (struct delete_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'r':
+ ctx->recursive = TRUE;
+ break;
+ case 's':
+ ctx->ctx.subscriptions = TRUE;
+ break;
+ case 'e':
+ ctx->require_empty = TRUE;
+ break;
+ case 'Z':
+ ctx->unsafe = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_delete_alloc(void)
+{
+ struct delete_cmd_context *ctx;
+
+ ctx = doveadm_mailbox_cmd_alloc(struct delete_cmd_context);
+ ctx->ctx.ctx.v.init = cmd_mailbox_delete_init;
+ ctx->ctx.ctx.v.run = cmd_mailbox_delete_run;
+ ctx->ctx.ctx.v.parse_arg = cmd_mailbox_delete_parse_arg;
+ ctx->ctx.ctx.getopt_args = "ersZ";
+ p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16);
+ return &ctx->ctx.ctx;
+}
+
+static int
+cmd_mailbox_rename_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct rename_cmd_context *ctx = (struct rename_cmd_context *)_ctx;
+ struct mail_namespace *oldns, *newns;
+ struct mailbox *oldbox, *newbox;
+ const char *oldname = ctx->oldname;
+ const char *newname = ctx->newname;
+ int ret = 0;
+
+ oldns = mail_namespace_find(user->namespaces, oldname);
+ newns = mail_namespace_find(user->namespaces, newname);
+ oldbox = mailbox_alloc(oldns->list, oldname, 0);
+ newbox = mailbox_alloc(newns->list, newname, 0);
+ if (mailbox_rename(oldbox, newbox) < 0) {
+ i_error("Can't rename mailbox %s to %s: %s", oldname, newname,
+ mailbox_get_last_internal_error(oldbox, NULL));
+ doveadm_mail_failed_mailbox(_ctx, oldbox);
+ ret = -1;
+ }
+ if (ctx->ctx.subscriptions) {
+ if (mailbox_set_subscribed(oldbox, FALSE) < 0) {
+ i_error("Can't unsubscribe mailbox %s: %s", ctx->oldname,
+ mailbox_get_last_internal_error(oldbox, NULL));
+ doveadm_mail_failed_mailbox(_ctx, oldbox);
+ ret = -1;
+ }
+ if (mailbox_set_subscribed(newbox, TRUE) < 0) {
+ i_error("Can't subscribe to mailbox %s: %s", ctx->newname,
+ mailbox_get_last_internal_error(newbox, NULL));
+ doveadm_mail_failed_mailbox(_ctx, newbox);
+ ret = -1;
+ }
+ }
+
+ mailbox_free(&oldbox);
+ mailbox_free(&newbox);
+ return ret;
+}
+
+static void cmd_mailbox_rename_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct rename_cmd_context *ctx = (struct rename_cmd_context *)_ctx;
+
+ if (str_array_length(args) != 2)
+ doveadm_mail_help_name("mailbox rename");
+ doveadm_mailbox_args_check(args);
+
+ ctx->oldname = p_strdup(ctx->ctx.ctx.pool, args[0]);
+ ctx->newname = p_strdup(ctx->ctx.ctx.pool, args[1]);
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_rename_alloc(void)
+{
+ struct rename_cmd_context *ctx;
+
+ ctx = doveadm_mailbox_cmd_alloc(struct rename_cmd_context);
+ ctx->ctx.ctx.v.init = cmd_mailbox_rename_init;
+ ctx->ctx.ctx.v.run = cmd_mailbox_rename_run;
+ return &ctx->ctx.ctx;
+}
+
+static int
+cmd_mailbox_subscribe_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ const char *name;
+ int ret = 0;
+
+ array_foreach_elem(&ctx->mailboxes, name) {
+ ns = mail_namespace_find(user->namespaces, name);
+ box = mailbox_alloc(ns->list, name, 0);
+ if (mailbox_set_subscribed(box, ctx->ctx.subscriptions) < 0) {
+ i_error("Can't %s mailbox %s: %s", name,
+ ctx->ctx.subscriptions ? "subscribe to" :
+ "unsubscribe",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ mailbox_free(&box);
+ }
+ return ret;
+}
+
+static void cmd_mailbox_subscribe_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct mailbox_cmd_context *ctx = (struct mailbox_cmd_context *)_ctx;
+ const char *name;
+ unsigned int i;
+
+ if (args[0] == NULL) {
+ doveadm_mail_help_name(ctx->ctx.subscriptions ?
+ "mailbox subscribe" :
+ "mailbox unsubscribe");
+ }
+ doveadm_mailbox_args_check(args);
+
+ for (i = 0; args[i] != NULL; i++) {
+ name = p_strdup(ctx->ctx.ctx.pool, args[i]);
+ array_push_back(&ctx->mailboxes, &name);
+ }
+}
+
+static struct doveadm_mail_cmd_context *
+cmd_mailbox_subscriptions_alloc(bool subscriptions)
+{
+ struct mailbox_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct mailbox_cmd_context);
+ ctx->ctx.subscriptions = subscriptions;
+
+ ctx->ctx.ctx.v.parse_arg = cmd_mailbox_parse_arg;
+ ctx->ctx.ctx.v.init = cmd_mailbox_subscribe_init;
+ ctx->ctx.ctx.v.run = cmd_mailbox_subscribe_run;
+ p_array_init(&ctx->mailboxes, ctx->ctx.ctx.pool, 16);
+ return &ctx->ctx.ctx;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_subscribe_alloc(void)
+{
+ return cmd_mailbox_subscriptions_alloc(TRUE);
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_unsubscribe_alloc(void)
+{
+ return cmd_mailbox_subscriptions_alloc(FALSE);
+}
+
+static
+void cmd_mailbox_update_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+
+ if (str_array_length(args) != 1)
+ doveadm_mail_help_name("mailbox update");
+
+ doveadm_mailbox_args_check(args);
+
+ ctx->mailbox = args[0];
+
+ if ((ctx->update.min_first_recent_uid != 0 ||
+ ctx->update.min_next_uid != 0) &&
+ ctx->update.min_first_recent_uid > ctx->update.min_next_uid) {
+ i_fatal_status(EX_DATAERR,
+ "min_first_recent_uid > min_next_uid");
+ }
+}
+
+static
+bool cmd_mailbox_update_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'g':
+ if (guid_128_from_string(optarg, ctx->update.mailbox_guid) < 0)
+ doveadm_mail_help_name("mailbox update");
+ break;
+ case 'V':
+ if (str_to_uint32(optarg, &(ctx->update.uid_validity)) < 0)
+ doveadm_mail_help_name("mailbox update");
+ break;
+ case 'N':
+ if (str_to_uint32(optarg, &(ctx->update.min_next_uid)) < 0)
+ doveadm_mail_help_name("mailbox update");
+ break;
+ case 'R':
+ if (str_to_uint32(optarg, &(ctx->update.min_first_recent_uid)) < 0)
+ doveadm_mail_help_name("mailbox update");
+ break;
+ case 'H':
+ if (str_to_uint64(optarg, &(ctx->update.min_highest_modseq)) < 0)
+ doveadm_mail_help_name("mailbox update");
+ break;
+ case 'P':
+ if (str_to_uint64(optarg, &(ctx->update.min_highest_pvt_modseq)) < 0)
+ doveadm_mail_help_name("mailbox update");
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static
+int cmd_mailbox_update_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ enum mail_error mail_error;
+ int ret = 0;
+
+ ns = mail_namespace_find(user->namespaces, ctx->mailbox);
+ box = mailbox_alloc(ns->list, ctx->mailbox, 0);
+
+ if ((ret = mailbox_update(box, &(ctx->update))) != 0) {
+ i_error("Cannot update %s: %s",
+ ctx->mailbox,
+ mailbox_get_last_internal_error(box, &mail_error));
+ doveadm_mail_failed_error(_ctx, mail_error);
+ }
+
+ mailbox_free(&box);
+
+ return ret;
+}
+
+static
+struct doveadm_mail_cmd_context *cmd_mailbox_update_alloc(void)
+{
+ struct update_cmd_context *ctx;
+ ctx = doveadm_mail_cmd_alloc(struct update_cmd_context);
+ ctx->ctx.ctx.v.parse_arg = cmd_mailbox_update_parse_arg;
+ ctx->ctx.ctx.v.init = cmd_mailbox_update_init;
+ ctx->ctx.ctx.v.run = cmd_mailbox_update_run;
+ return &ctx->ctx.ctx;
+}
+
+static void
+cmd_mailbox_path_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[])
+{
+ struct update_cmd_context *ctx = (struct update_cmd_context *)_ctx;
+
+ if (str_array_length(args) != 1)
+ doveadm_mail_help_name("mailbox path");
+
+ doveadm_mailbox_args_check(args);
+
+ ctx->mailbox = args[0];
+ doveadm_print_header("path", "path", DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+}
+
+static bool
+mailbox_list_path_type_name_parse(const char *name,
+ enum mailbox_list_path_type *type_r)
+{
+ enum mailbox_list_path_type type;
+
+ for (type = 0; type < N_ELEMENTS(mailbox_list_path_type_names); type++) {
+ if (strcmp(mailbox_list_path_type_names[type], name) == 0) {
+ *type_r = type;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static bool
+cmd_mailbox_path_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct path_cmd_context *ctx = (struct path_cmd_context *)_ctx;
+
+ switch (c) {
+ case 't':
+ if (!mailbox_list_path_type_name_parse(optarg, &ctx->path_type))
+ doveadm_mail_help_name("mailbox path");
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static int
+cmd_mailbox_path_run(struct doveadm_mail_cmd_context *_ctx,
+ struct mail_user *user)
+{
+ struct path_cmd_context *ctx = (struct path_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ enum mail_error mail_error;
+ const char *storage_name, *path;
+ int ret;
+
+ ns = mail_namespace_find(user->namespaces, ctx->mailbox);
+ storage_name = mailbox_list_get_storage_name(ns->list, ctx->mailbox);
+ ret = mailbox_list_get_path(ns->list, storage_name, ctx->path_type, &path);
+ if (ret < 0) {
+ i_error("Failed to lookup mailbox %s path: %s",
+ ctx->mailbox,
+ mailbox_list_get_last_internal_error(ns->list, &mail_error));
+ doveadm_mail_failed_error(_ctx, mail_error);
+ } else if (ret > 0) {
+ doveadm_print(path);
+ }
+ return ret;
+}
+
+static struct doveadm_mail_cmd_context *cmd_mailbox_path_alloc(void)
+{
+ struct path_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct path_cmd_context);
+ ctx->path_type = MAILBOX_LIST_PATH_TYPE_INDEX;
+ ctx->ctx.ctx.v.parse_arg = cmd_mailbox_path_parse_arg;
+ ctx->ctx.ctx.v.init = cmd_mailbox_path_init;
+ ctx->ctx.ctx.v.run = cmd_mailbox_path_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return &ctx->ctx.ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_list_ver2 = {
+ .name = "mailbox list",
+ .mail_cmd = cmd_mailbox_list_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-7|-8] [-s] [<mailbox mask> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('7', "mutf7", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('8', "utf8", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_create_ver2 = {
+ .name = "mailbox create",
+ .mail_cmd = cmd_mailbox_create_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] [-g <guid>] <mailbox> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('g', "guid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2 = {
+ .name = "mailbox delete",
+ .mail_cmd = cmd_mailbox_delete_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-e] [-r] [-s] [-Z] <mailbox> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('e', "require-empty", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('r', "recursive", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('Z', "unsafe", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_rename_ver2 = {
+ .name = "mailbox rename",
+ .mail_cmd = cmd_mailbox_rename_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-s] <old name> <new name>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('s', "subscriptions", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "new-name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_subscribe_ver2 = {
+ .name = "mailbox subscribe",
+ .mail_cmd = cmd_mailbox_subscribe_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<mailbox> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_unsubscribe_ver2 = {
+ .name = "mailbox unsubscribe",
+ .mail_cmd = cmd_mailbox_unsubscribe_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"<mailbox> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2 = {
+ .name = "mailbox update",
+ .mail_cmd = cmd_mailbox_update_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[--mailbox-guid guid] [--uid-validity uid] [--min-next-uid uid] [--min-first-recent-uid uid] [--min-highest-modseq seq] [--min-highest-pvt-modseq seq] <mailbox>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('g', "mailbox-guid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('V', "uid-validity", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('N', "min-next-uid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('R', "min-first-recent-uid", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('H', "min-highest-modseq", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('P', "min-highest-pvt-modseq", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_path_ver2 = {
+ .name = "mailbox path",
+ .mail_cmd = cmd_mailbox_path_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-t <type>] <mailbox>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "mailbox", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('t', "type", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+};
+
diff --git a/src/doveadm/doveadm-mail-rebuild.c b/src/doveadm/doveadm-mail-rebuild.c
new file mode 100644
index 0000000..211c3c3
--- /dev/null
+++ b/src/doveadm/doveadm-mail-rebuild.c
@@ -0,0 +1,101 @@
+/* Copyright (c) 2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-storage.h"
+#include "doveadm-print.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+#include "mail-storage-private.h"
+
+static int
+cmd_rebuild_attachment_box(struct doveadm_mail_cmd_context *ctx,
+ const struct mailbox_info *info)
+{
+ struct doveadm_mail_iter *iter;
+ struct mail *mail;
+ int ret = 0;
+
+ if (doveadm_mail_iter_init(ctx, info, ctx->search_args,
+ MAIL_FETCH_IMAP_BODYSTRUCTURE|
+ MAIL_FETCH_MESSAGE_PARTS, NULL, 0,
+ &iter) < 0)
+ return -1;
+
+ while (doveadm_mail_iter_next(iter, &mail) && ret >= 0) {
+ T_BEGIN {
+ doveadm_print(dec2str(mail->uid));
+ switch(mail_set_attachment_keywords(mail)) {
+ case -1:
+ doveadm_print("error");
+ doveadm_mail_failed_mailbox(ctx, mail->box);
+ ret = -1;
+ break;
+ case 0:
+ doveadm_print("no");
+ break;
+ case 1:
+ doveadm_print("yes");
+ break;
+ default:
+ i_unreached();
+ }
+ } T_END;
+ }
+
+ if (doveadm_mail_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static int
+cmd_rebuild_attachment_run(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user)
+{
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_rebuild_attachment_box(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_rebuild_attachment_init(struct doveadm_mail_cmd_context *ctx,
+ const char *const args[])
+{
+ doveadm_print_header_simple("uid");
+ doveadm_print_header_simple("attachment");
+ ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+
+static struct doveadm_mail_cmd_context *cmd_rebuild_attachment_alloc(void)
+{
+ struct doveadm_mail_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+ ctx->v.init = cmd_rebuild_attachment_init;
+ ctx->v.run = cmd_rebuild_attachment_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ return ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_rebuild_attachments = {
+ .name = "rebuild attachments",
+ .mail_cmd = cmd_rebuild_attachment_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-save.c b/src/doveadm/doveadm-mail-save.c
new file mode 100644
index 0000000..d8312c8
--- /dev/null
+++ b/src/doveadm/doveadm-mail-save.c
@@ -0,0 +1,142 @@
+/* Copyright (c) 2015-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "mail-storage.h"
+#include "doveadm-mail.h"
+
+struct save_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ const char *mailbox;
+};
+
+static int
+cmd_save_to_mailbox(struct save_cmd_context *ctx, struct mailbox *box,
+ struct istream *input)
+{
+ struct mail_storage *storage = mailbox_get_storage(box);
+ struct mailbox_transaction_context *trans;
+ struct mail_save_context *save_ctx;
+ ssize_t ret;
+ bool save_failed = FALSE;
+
+ if (input->stream_errno != 0) {
+ i_error("open(%s) failed: %s",
+ i_stream_get_name(input),
+ i_stream_get_error(input));
+ ctx->ctx.exit_code = EX_TEMPFAIL;
+ return -1;
+ }
+
+ if (mailbox_open(box) < 0) {
+ i_error("Failed to open mailbox %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_storage(&ctx->ctx, storage);
+ return -1;
+ }
+
+ trans = mailbox_transaction_begin(box, MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+ ctx->ctx.transaction_flags, __func__);
+ save_ctx = mailbox_save_alloc(trans);
+ if (mailbox_save_begin(&save_ctx, input) < 0) {
+ i_error("Saving failed: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_storage(&ctx->ctx, storage);
+ mailbox_transaction_rollback(&trans);
+ return -1;
+ }
+ do {
+ if (mailbox_save_continue(save_ctx) < 0) {
+ save_failed = TRUE;
+ ret = -1;
+ break;
+ }
+ } while ((ret = i_stream_read(input)) > 0);
+ i_assert(ret == -1);
+
+ if (input->stream_errno != 0) {
+ i_error("read(msg input) failed: %s", i_stream_get_error(input));
+ doveadm_mail_failed_error(&ctx->ctx, MAIL_ERROR_TEMP);
+ } else if (save_failed) {
+ i_error("Saving failed: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_storage(&ctx->ctx, storage);
+ } else if (mailbox_save_finish(&save_ctx) < 0) {
+ i_error("Saving failed: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_storage(&ctx->ctx, storage);
+ } else if (mailbox_transaction_commit(&trans) < 0) {
+ i_error("Save transaction commit failed: %s",
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_storage(&ctx->ctx, storage);
+ } else {
+ ret = 0;
+ }
+ if (save_ctx != NULL)
+ mailbox_save_cancel(&save_ctx);
+ if (trans != NULL)
+ mailbox_transaction_rollback(&trans);
+ i_assert(input->eof);
+ return ret < 0 ? -1 : 0;
+}
+
+static int
+cmd_save_run(struct doveadm_mail_cmd_context *_ctx, struct mail_user *user)
+{
+ struct save_cmd_context *ctx = (struct save_cmd_context *)_ctx;
+ struct mail_namespace *ns;
+ struct mailbox *box;
+ int ret;
+
+ ns = mail_namespace_find(user->namespaces, ctx->mailbox);
+ box = mailbox_alloc(ns->list, ctx->mailbox, MAILBOX_FLAG_SAVEONLY);
+ ret = cmd_save_to_mailbox(ctx, box, _ctx->cmd_input);
+ mailbox_free(&box);
+ return ret;
+}
+
+static void cmd_save_init(struct doveadm_mail_cmd_context *_ctx,
+ const char *const args[] ATTR_UNUSED)
+{
+ doveadm_mail_get_input(_ctx);
+}
+
+static bool
+cmd_mailbox_save_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct save_cmd_context *ctx = (struct save_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'm':
+ ctx->mailbox = optarg;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_save_alloc(void)
+{
+ struct save_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct save_cmd_context);
+ ctx->ctx.getopt_args = "m:";
+ ctx->ctx.v.parse_arg = cmd_mailbox_save_parse_arg;
+ ctx->ctx.v.init = cmd_save_init;
+ ctx->ctx.v.run = cmd_save_run;
+ ctx->mailbox = "INBOX";
+ return &ctx->ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_save_ver2 = {
+ .name = "save",
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX"[-m mailbox]",
+ .mail_cmd = cmd_save_alloc,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('m', "mailbox", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "file", CMD_PARAM_ISTREAM, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-search.c b/src/doveadm/doveadm-mail-search.c
new file mode 100644
index 0000000..37c47e3
--- /dev/null
+++ b/src/doveadm/doveadm-mail-search.c
@@ -0,0 +1,103 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-storage.h"
+#include "doveadm-print.h"
+#include "doveadm-mailbox-list-iter.h"
+#include "doveadm-mail-iter.h"
+#include "doveadm-mail.h"
+
+#include <stdio.h>
+
+static int
+cmd_search_box(struct doveadm_mail_cmd_context *ctx,
+ const struct mailbox_info *info)
+{
+ struct doveadm_mail_iter *iter;
+ struct mailbox *box;
+ struct mail *mail;
+ struct mailbox_metadata metadata;
+ const char *guid_str;
+ int ret = 0;
+
+ if (doveadm_mail_iter_init(ctx, info, ctx->search_args, 0, NULL,
+ DOVEADM_MAIL_ITER_FLAG_STOP_WITH_CLIENT,
+ &iter) < 0)
+ return -1;
+ box = doveadm_mail_iter_get_mailbox(iter);
+
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) {
+ i_error("Couldn't get mailbox '%s' GUID: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, NULL));
+ ret = -1;
+ doveadm_mail_failed_mailbox(ctx, box);
+ } else {
+ guid_str = guid_128_to_string(metadata.guid);
+ while (doveadm_mail_iter_next(iter, &mail)) {
+ doveadm_print(guid_str);
+ T_BEGIN {
+ doveadm_print(dec2str(mail->uid));
+ } T_END;
+ }
+ }
+ if (doveadm_mail_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static int
+cmd_search_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
+{
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct doveadm_mailbox_list_iter *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = doveadm_mailbox_list_iter_init(ctx, user, ctx->search_args,
+ iter_flags);
+ while ((info = doveadm_mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (cmd_search_box(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ if (doveadm_mailbox_list_iter_deinit(&iter) < 0)
+ ret = -1;
+ return ret;
+}
+
+static void cmd_search_init(struct doveadm_mail_cmd_context *ctx,
+ const char *const args[])
+{
+ if (args[0] == NULL)
+ doveadm_mail_help_name("search");
+
+ doveadm_print_header("mailbox-guid", "mailbox-guid",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ doveadm_print_header("uid", "uid",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+
+ ctx->search_args = doveadm_mail_build_search_args(args);
+}
+
+static struct doveadm_mail_cmd_context *cmd_search_alloc(void)
+{
+ struct doveadm_mail_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+ ctx->v.init = cmd_search_init;
+ ctx->v.run = cmd_search_run;
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ return ctx;
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_search_ver2 = {
+ .name = "search",
+ .mail_cmd = cmd_search_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "<search query>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('\0', "query", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mail-server.c b/src/doveadm/doveadm-mail-server.c
new file mode 100644
index 0000000..a829a24
--- /dev/null
+++ b/src/doveadm/doveadm-mail-server.c
@@ -0,0 +1,404 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "strescape.h"
+#include "ioloop.h"
+#include "master-service.h"
+#include "iostream-ssl.h"
+#include "auth-master.h"
+#include "mail-storage.h"
+#include "mail-storage-service.h"
+#include "server-connection.h"
+#include "doveadm-settings.h"
+#include "doveadm-print.h"
+#include "doveadm-server.h"
+#include "doveadm-mail.h"
+
+#define DOVEADM_SERVER_CONNECTIONS_MAX 4
+#define DOVEADM_SERVER_QUEUE_MAX 16
+
+#define DOVEADM_MAIL_SERVER_FAILED() \
+ (internal_failure || master_service_is_killed(master_service))
+
+struct doveadm_mail_server_cmd {
+ struct server_connection *conn;
+ char *username;
+};
+
+static HASH_TABLE(char *, struct doveadm_server *) servers;
+static pool_t server_pool;
+static struct doveadm_mail_cmd_context *cmd_ctx;
+static bool internal_failure = FALSE;
+
+static void doveadm_mail_server_handle(struct server_connection *conn,
+ const char *username);
+
+static struct doveadm_server *
+doveadm_server_get(struct doveadm_mail_cmd_context *ctx, const char *name)
+{
+ struct doveadm_server *server;
+ const char *p;
+ char *dup_name;
+
+ if (!hash_table_is_created(servers)) {
+ server_pool = pool_alloconly_create("doveadm servers", 1024*16);
+ hash_table_create(&servers, server_pool, 0, str_hash, strcmp);
+ }
+ server = hash_table_lookup(servers, name);
+ if (server == NULL) {
+ server = p_new(server_pool, struct doveadm_server, 1);
+ server->name = dup_name = p_strdup(server_pool, name);
+ p = strrchr(server->name, ':');
+ server->hostname = p == NULL ? server->name :
+ p_strdup_until(server_pool, server->name, p);
+
+ p_array_init(&server->connections, server_pool,
+ ctx->set->doveadm_worker_count);
+ p_array_init(&server->queue, server_pool,
+ DOVEADM_SERVER_QUEUE_MAX);
+ hash_table_insert(servers, dup_name, server);
+ }
+ return server;
+}
+
+static struct server_connection *
+doveadm_server_find_unused_conn(struct doveadm_server *server)
+{
+ struct server_connection *conn;
+
+ array_foreach_elem(&server->connections, conn) {
+ if (server_connection_is_idle(conn))
+ return conn;
+ }
+ return NULL;
+}
+
+static bool doveadm_server_have_used_connections(struct doveadm_server *server)
+{
+ struct server_connection *conn;
+
+ array_foreach_elem(&server->connections, conn) {
+ if (!server_connection_is_idle(conn))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void doveadm_cmd_callback(int exit_code, const char *error,
+ void *context)
+{
+ struct doveadm_mail_server_cmd *servercmd = context;
+ struct doveadm_server *server =
+ server_connection_get_server(servercmd->conn);
+ const char *username = t_strdup(servercmd->username);
+
+ i_free(servercmd->username);
+ i_free(servercmd);
+
+ switch (exit_code) {
+ case 0:
+ break;
+ case SERVER_EXIT_CODE_DISCONNECTED:
+ i_error("%s: Command %s failed for %s: %s",
+ server->name, cmd_ctx->cmd->name, username, error);
+ internal_failure = TRUE;
+ io_loop_stop(current_ioloop);
+ return;
+ case EX_NOUSER:
+ i_error("%s: No such user: %s", server->name, username);
+ if (cmd_ctx->exit_code == 0)
+ cmd_ctx->exit_code = EX_NOUSER;
+ break;
+ default:
+ if (cmd_ctx->exit_code == 0 || exit_code == EX_TEMPFAIL)
+ cmd_ctx->exit_code = exit_code;
+ break;
+ }
+
+ if (array_count(&server->queue) > 0) {
+ struct server_connection *conn;
+ char *const *usernamep = array_front(&server->queue);
+ char *username = *usernamep;
+
+ conn = doveadm_server_find_unused_conn(server);
+ if (conn != NULL) {
+ array_pop_front(&server->queue);
+ doveadm_mail_server_handle(conn, username);
+ i_free(username);
+ }
+ }
+
+ io_loop_stop(current_ioloop);
+}
+
+static void doveadm_mail_server_handle(struct server_connection *conn,
+ const char *username)
+{
+ struct doveadm_mail_server_cmd *servercmd;
+ string_t *cmd;
+ unsigned int i;
+
+ /* <flags> <username> <command> [<args>] */
+ cmd = t_str_new(256);
+ if (doveadm_debug)
+ str_append_c(cmd, 'D');
+ else if (doveadm_verbose)
+ str_append_c(cmd, 'v');
+ str_append_c(cmd, '\t');
+
+ str_append_tabescaped(cmd, username);
+ str_append_c(cmd, '\t');
+ str_append_tabescaped(cmd, cmd_ctx->cmd->name);
+ for (i = 0; cmd_ctx->full_args[i] != NULL; i++) {
+ str_append_c(cmd, '\t');
+ str_append_tabescaped(cmd, cmd_ctx->full_args[i]);
+ }
+ str_append_c(cmd, '\n');
+
+ servercmd = i_new(struct doveadm_mail_server_cmd, 1);
+ servercmd->conn = conn;
+ servercmd->username = i_strdup(username);
+ server_connection_cmd(conn, str_c(cmd), cmd_ctx->cmd_input,
+ doveadm_cmd_callback, servercmd);
+}
+
+static void doveadm_server_flush_one(struct doveadm_server *server)
+{
+ unsigned int count = array_count(&server->queue);
+
+ do {
+ io_loop_run(current_ioloop);
+ } while (array_count(&server->queue) == count &&
+ doveadm_server_have_used_connections(server) &&
+ !DOVEADM_MAIL_SERVER_FAILED());
+}
+
+static int
+doveadm_mail_server_user_get_host(struct doveadm_mail_cmd_context *ctx,
+ const struct mail_storage_service_input *input,
+ const char **user_r, const char **host_r,
+ struct ip_addr *hostip_r, in_port_t *port_r,
+ enum doveadm_proxy_ssl_flags *ssl_flags_r,
+ const char **error_r)
+{
+ struct auth_master_connection *auth_conn;
+ struct auth_user_info info;
+ pool_t pool;
+ const char *auth_socket_path, *proxy_host, *proxy_hostip, *const *fields;
+ unsigned int i;
+ in_port_t proxy_port;
+ bool proxying;
+ int ret;
+
+ *user_r = input->username;
+ *host_r = ctx->set->doveadm_socket_path;
+ *port_r = ctx->set->doveadm_port;
+
+ if (ctx->set->doveadm_port == 0)
+ return 0;
+
+ if (strcmp(ctx->set->doveadm_ssl, "ssl") == 0)
+ *ssl_flags_r |= PROXY_SSL_FLAG_YES;
+ else if (strcmp(ctx->set->doveadm_ssl, "starttls") == 0)
+ *ssl_flags_r |= PROXY_SSL_FLAG_YES | PROXY_SSL_FLAG_STARTTLS;
+
+ /* make sure we have an auth connection */
+ mail_storage_service_init_settings(ctx->storage_service, input);
+
+ i_zero(&info);
+ info.service = master_service_get_name(master_service);
+ info.local_ip = input->local_ip;
+ info.remote_ip = input->remote_ip;
+ info.local_port = input->local_port;
+ info.remote_port = input->remote_port;
+
+ pool = pool_alloconly_create("auth lookup", 1024);
+ auth_conn = mail_storage_service_get_auth_conn(ctx->storage_service);
+ auth_socket_path = auth_master_get_socket_path(auth_conn);
+ ret = auth_master_pass_lookup(auth_conn, input->username, &info,
+ pool, &fields);
+ if (ret < 0) {
+ *error_r = fields[0] != NULL ?
+ t_strdup(fields[0]) : "passdb lookup failed";
+ *error_r = t_strdup_printf("%s: %s (to see if user is proxied, "
+ "because doveadm_port is set)",
+ auth_socket_path, *error_r);
+ } else if (ret == 0) {
+ /* user not found from passdb. it could be in userdb though,
+ so just continue with the default host */
+ } else {
+ proxy_host = NULL; proxy_hostip = NULL; proxying = FALSE;
+ proxy_port = ctx->set->doveadm_port;
+ for (i = 0; fields[i] != NULL; i++) {
+ if (str_begins(fields[i], "proxy") &&
+ (fields[i][5] == '\0' || fields[i][5] == '='))
+ proxying = TRUE;
+ else if (str_begins(fields[i], "host="))
+ proxy_host = fields[i]+5;
+ else if (str_begins(fields[i], "hostip="))
+ proxy_hostip = fields[i]+7;
+ else if (str_begins(fields[i], "user="))
+ *user_r = t_strdup(fields[i]+5);
+ else if (str_begins(fields[i], "destuser="))
+ *user_r = t_strdup(fields[i]+9);
+ else if (str_begins(fields[i], "port=")) {
+ if (net_str2port(fields[i]+5, &proxy_port) < 0)
+ proxy_port = 0;
+ } else if (str_begins(fields[i], "ssl=")) {
+ *ssl_flags_r |= PROXY_SSL_FLAG_YES;
+ if (strcmp(fields[i]+4, "any-cert") == 0)
+ *ssl_flags_r |= PROXY_SSL_FLAG_ANY_CERT;
+ } else if (str_begins(fields[i], "starttls=")) {
+ *ssl_flags_r |= PROXY_SSL_FLAG_YES |
+ PROXY_SSL_FLAG_STARTTLS;
+ if (strcmp(fields[i]+9, "any-cert") == 0)
+ *ssl_flags_r |= PROXY_SSL_FLAG_ANY_CERT;
+ }
+ }
+ if (proxy_hostip != NULL &&
+ net_addr2ip(proxy_hostip, hostip_r) < 0) {
+ *error_r = t_strdup_printf("%s Invalid hostip value '%s'",
+ auth_socket_path, proxy_hostip);
+ ret = -1;
+ }
+ if (!proxying)
+ ret = 0;
+ else if (proxy_host == NULL) {
+ *error_r = t_strdup_printf("%s: Proxy is missing destination host",
+ auth_socket_path);
+ if (strstr(auth_socket_path, "/auth-userdb") != NULL) {
+ *error_r = t_strdup_printf(
+ "%s (maybe set auth_socket_path=director-userdb)",
+ *error_r);
+ }
+ ret = -1;
+ } else {
+ *port_r = proxy_port;
+ *host_r = t_strdup_printf("%s:%u", proxy_host, proxy_port);
+ }
+ }
+ pool_unref(&pool);
+ return ret;
+}
+
+int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx,
+ const struct mail_storage_service_input *input,
+ const char **error_r)
+{
+ struct doveadm_server *server;
+ struct server_connection *conn;
+ const char *user, *host;
+ struct ip_addr hostip;
+ enum doveadm_proxy_ssl_flags ssl_flags = 0;
+ char *username_dup;
+ int ret;
+ in_port_t port;
+
+ i_assert(cmd_ctx == ctx || cmd_ctx == NULL);
+ cmd_ctx = ctx;
+
+ i_zero(&hostip);
+ ret = doveadm_mail_server_user_get_host(ctx, input, &user, &host, &hostip,
+ &port, &ssl_flags, error_r);
+ if (ret < 0)
+ return ret;
+ if (ret == 0 &&
+ (ctx->set->doveadm_worker_count == 0 || doveadm_server)) {
+ /* run it ourself */
+ return 0;
+ }
+
+ /* server sends the sticky headers for each row as well,
+ so undo any sticks we might have added already */
+ doveadm_print_unstick_headers();
+
+ server = doveadm_server_get(ctx, host);
+ server->ip = hostip;
+ server->ssl_flags = ssl_flags;
+ server->port = port;
+ conn = doveadm_server_find_unused_conn(server);
+ if (conn != NULL)
+ doveadm_mail_server_handle(conn, user);
+ else if (array_count(&server->connections) <
+ I_MAX(ctx->set->doveadm_worker_count, 1)) {
+ if (server_connection_create(server, &conn, error_r) < 0) {
+ internal_failure = TRUE;
+ return -1;
+ } else {
+ doveadm_mail_server_handle(conn, user);
+ }
+ } else {
+ if (array_count(&server->queue) >= DOVEADM_SERVER_QUEUE_MAX)
+ doveadm_server_flush_one(server);
+
+ username_dup = i_strdup(user);
+ array_push_back(&server->queue, &username_dup);
+ }
+ *error_r = "doveadm server failure";
+ return DOVEADM_MAIL_SERVER_FAILED() ? -1 : 1;
+}
+
+static struct doveadm_server *doveadm_server_find_used(void)
+{
+ struct hash_iterate_context *iter;
+ struct doveadm_server *ret = NULL;
+ char *key;
+ struct doveadm_server *server;
+
+ iter = hash_table_iterate_init(servers);
+ while (hash_table_iterate(iter, servers, &key, &server)) {
+ if (doveadm_server_have_used_connections(server)) {
+ ret = server;
+ break;
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+ return ret;
+}
+
+static void doveadm_servers_destroy_all_connections(void)
+{
+ struct hash_iterate_context *iter;
+ char *key;
+ struct doveadm_server *server;
+
+ iter = hash_table_iterate_init(servers);
+ while (hash_table_iterate(iter, servers, &key, &server)) {
+ while (array_count(&server->connections) > 0) {
+ struct server_connection *const *connp, *conn;
+
+ connp = array_front(&server->connections);
+ conn = *connp;
+ server_connection_destroy(&conn);
+ }
+ ssl_iostream_context_unref(&server->ssl_ctx);
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+void doveadm_mail_server_flush(void)
+{
+ struct doveadm_server *server;
+
+ if (!hash_table_is_created(servers)) {
+ cmd_ctx = NULL;
+ return;
+ }
+
+ while ((server = doveadm_server_find_used()) != NULL &&
+ !DOVEADM_MAIL_SERVER_FAILED())
+ doveadm_server_flush_one(server);
+
+ doveadm_servers_destroy_all_connections();
+ if (master_service_is_killed(master_service))
+ i_error("Aborted");
+ if (DOVEADM_MAIL_SERVER_FAILED())
+ doveadm_mail_failed_error(cmd_ctx, MAIL_ERROR_TEMP);
+
+ hash_table_destroy(&servers);
+ pool_unref(&server_pool);
+ cmd_ctx = NULL;
+}
diff --git a/src/doveadm/doveadm-mail.c b/src/doveadm/doveadm-mail.c
new file mode 100644
index 0000000..574c3b7
--- /dev/null
+++ b/src/doveadm/doveadm-mail.c
@@ -0,0 +1,990 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "lib-signals.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "istream-dot.h"
+#include "istream-seekable.h"
+#include "str.h"
+#include "unichar.h"
+#include "module-dir.h"
+#include "wildcard-match.h"
+#include "master-service.h"
+#include "mail-user.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "mail-storage-settings.h"
+#include "mail-storage-service.h"
+#include "mail-storage-hooks.h"
+#include "mail-search-build.h"
+#include "mail-search-parser.h"
+#include "mailbox-list-iter.h"
+#include "doveadm.h"
+#include "client-connection.h"
+#include "doveadm-settings.h"
+#include "doveadm-print.h"
+#include "doveadm-dsync.h"
+#include "doveadm-mail.h"
+
+#include <stdio.h>
+
+#define DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS (5*60*1000)
+
+struct force_resync_cmd_context {
+ struct doveadm_mail_cmd_context ctx;
+ bool fsck;
+};
+
+void (*hook_doveadm_mail_init)(struct doveadm_mail_cmd_context *ctx);
+struct doveadm_mail_cmd_module_register
+ doveadm_mail_cmd_module_register = { 0 };
+char doveadm_mail_cmd_hide = '\0';
+
+static int killed_signo = 0;
+
+bool doveadm_is_killed(void)
+{
+ return killed_signo != 0;
+}
+
+int doveadm_killed_signo(void)
+{
+ return killed_signo;
+}
+
+void doveadm_mail_failed_error(struct doveadm_mail_cmd_context *ctx,
+ enum mail_error error)
+{
+ int exit_code = EX_TEMPFAIL;
+
+ switch (error) {
+ case MAIL_ERROR_NONE:
+ i_unreached();
+ case MAIL_ERROR_TEMP:
+ case MAIL_ERROR_UNAVAILABLE:
+ break;
+ case MAIL_ERROR_NOTPOSSIBLE:
+ case MAIL_ERROR_EXISTS:
+ case MAIL_ERROR_CONVERSION:
+ case MAIL_ERROR_INVALIDDATA:
+ exit_code = DOVEADM_EX_NOTPOSSIBLE;
+ break;
+ case MAIL_ERROR_PARAMS:
+ exit_code = EX_USAGE;
+ break;
+ case MAIL_ERROR_PERM:
+ exit_code = EX_NOPERM;
+ break;
+ case MAIL_ERROR_NOQUOTA:
+ exit_code = EX_CANTCREAT;
+ break;
+ case MAIL_ERROR_NOTFOUND:
+ exit_code = DOVEADM_EX_NOTFOUND;
+ break;
+ case MAIL_ERROR_EXPUNGED:
+ break;
+ case MAIL_ERROR_INUSE:
+ case MAIL_ERROR_LIMIT:
+ exit_code = DOVEADM_EX_NOTPOSSIBLE;
+ break;
+ case MAIL_ERROR_LOOKUP_ABORTED:
+ break;
+ }
+ /* tempfail overrides all other exit codes, otherwise use whatever
+ error happened first */
+ if (ctx->exit_code == 0 || exit_code == EX_TEMPFAIL)
+ ctx->exit_code = exit_code;
+}
+
+void doveadm_mail_failed_storage(struct doveadm_mail_cmd_context *ctx,
+ struct mail_storage *storage)
+{
+ enum mail_error error;
+
+ mail_storage_get_last_error(storage, &error);
+ doveadm_mail_failed_error(ctx, error);
+}
+
+void doveadm_mail_failed_mailbox(struct doveadm_mail_cmd_context *ctx,
+ struct mailbox *box)
+{
+ doveadm_mail_failed_storage(ctx, mailbox_get_storage(box));
+}
+
+void doveadm_mail_failed_list(struct doveadm_mail_cmd_context *ctx,
+ struct mailbox_list *list)
+{
+ enum mail_error error;
+
+ mailbox_list_get_last_error(list, &error);
+ doveadm_mail_failed_error(ctx, error);
+}
+
+struct doveadm_mail_cmd_context *
+doveadm_mail_cmd_alloc_size(size_t size)
+{
+ struct doveadm_mail_cmd_context *ctx;
+ pool_t pool;
+
+ i_assert(size >= sizeof(struct doveadm_mail_cmd_context));
+
+ pool = pool_alloconly_create("doveadm mail cmd", 1024);
+ ctx = p_malloc(pool, size);
+ ctx->pool = pool;
+ ctx->cmd_input_fd = -1;
+ return ctx;
+}
+
+static int
+cmd_purge_run(struct doveadm_mail_cmd_context *ctx, struct mail_user *user)
+{
+ struct mail_namespace *ns;
+ struct mail_storage *storage;
+ int ret = 0;
+
+ for (ns = user->namespaces; ns != NULL; ns = ns->next) {
+ if (ns->type != MAIL_NAMESPACE_TYPE_PRIVATE ||
+ ns->alias_for != NULL)
+ continue;
+
+ storage = mail_namespace_get_default_storage(ns);
+ if (mail_storage_purge(storage) < 0) {
+ i_error("Purging namespace '%s' failed: %s", ns->prefix,
+ mail_storage_get_last_internal_error(storage, NULL));
+ doveadm_mail_failed_storage(ctx, storage);
+ ret = -1;
+ }
+ }
+ return ret;
+}
+
+static struct doveadm_mail_cmd_context *cmd_purge_alloc(void)
+{
+ struct doveadm_mail_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct doveadm_mail_cmd_context);
+ ctx->v.run = cmd_purge_run;
+ return ctx;
+}
+
+static void doveadm_mail_cmd_input_input(struct doveadm_mail_cmd_context *ctx)
+{
+ const unsigned char *data;
+ size_t size;
+
+ while (i_stream_read_more(ctx->cmd_input, &data, &size) > 0)
+ i_stream_skip(ctx->cmd_input, size);
+ if (!ctx->cmd_input->eof)
+ return;
+
+ if (ctx->cmd_input->stream_errno != 0) {
+ i_error("read(%s) failed: %s",
+ i_stream_get_name(ctx->cmd_input),
+ i_stream_get_error(ctx->cmd_input));
+ }
+ io_loop_stop(current_ioloop);
+}
+
+static void doveadm_mail_cmd_input_timeout(struct doveadm_mail_cmd_context *ctx)
+{
+ struct istream *input;
+
+ input = i_stream_create_error_str(ETIMEDOUT, "Timed out in %u secs",
+ DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS/1000);
+ i_stream_set_name(input, i_stream_get_name(ctx->cmd_input));
+ i_stream_destroy(&ctx->cmd_input);
+ ctx->cmd_input = input;
+ ctx->exit_code = EX_TEMPFAIL;
+ io_loop_stop(current_ioloop);
+}
+
+static void doveadm_mail_cmd_input_read(struct doveadm_mail_cmd_context *ctx)
+{
+ struct ioloop *ioloop;
+ struct io *io;
+ struct timeout *to;
+
+ ioloop = io_loop_create();
+ /* Read the pending input from stream. Delay adding the IO in case
+ we're reading from a file. That would cause a panic with epoll. */
+ io_loop_set_running(ioloop);
+ doveadm_mail_cmd_input_input(ctx);
+ if (io_loop_is_running(ioloop)) {
+ io = io_add(ctx->cmd_input_fd, IO_READ,
+ doveadm_mail_cmd_input_input, ctx);
+ to = timeout_add(DOVEADM_MAIL_CMD_INPUT_TIMEOUT_MSECS,
+ doveadm_mail_cmd_input_timeout, ctx);
+ io_loop_run(ioloop);
+ io_remove(&io);
+ timeout_remove(&to);
+ }
+ io_loop_destroy(&ioloop);
+
+ i_assert(ctx->cmd_input->eof);
+ i_stream_seek(ctx->cmd_input, 0);
+}
+
+void doveadm_mail_get_input(struct doveadm_mail_cmd_context *ctx)
+{
+ const struct doveadm_cmd_context *cctx = ctx->cctx;
+ bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI);
+ struct istream *inputs[2];
+
+ if (ctx->cmd_input != NULL)
+ return;
+
+ if (!cli && cctx->input == NULL) {
+ ctx->cmd_input = i_stream_create_error_str(EINVAL, "Input stream missing (provide with file parameter)");
+ return;
+ }
+
+ if (!cli)
+ inputs[0] = i_stream_create_dot(cctx->input, FALSE);
+ else {
+ inputs[0] = i_stream_create_fd(STDIN_FILENO, 1024*1024);
+ i_stream_set_name(inputs[0], "stdin");
+ }
+ inputs[1] = NULL;
+ ctx->cmd_input_fd = i_stream_get_fd(inputs[0]);
+ ctx->cmd_input = i_stream_create_seekable_path(inputs, 1024*256,
+ "/tmp/doveadm.");
+ i_stream_set_name(ctx->cmd_input, i_stream_get_name(inputs[0]));
+ i_stream_unref(&inputs[0]);
+
+ doveadm_mail_cmd_input_read(ctx);
+}
+
+struct mailbox *
+doveadm_mailbox_find(struct mail_user *user, const char *mailbox)
+{
+ struct mail_namespace *ns;
+
+ if (!uni_utf8_str_is_valid(mailbox)) {
+ i_fatal_status(EX_DATAERR,
+ "Mailbox name not valid UTF-8: %s", mailbox);
+ }
+
+ ns = mail_namespace_find(user->namespaces, mailbox);
+ return mailbox_alloc(ns->list, mailbox, MAILBOX_FLAG_IGNORE_ACLS);
+}
+
+struct mail_search_args *
+doveadm_mail_build_search_args(const char *const args[])
+{
+ struct mail_search_parser *parser;
+ struct mail_search_args *sargs;
+ const char *error, *charset = "UTF-8";
+
+ parser = mail_search_parser_init_cmdline(args);
+ if (mail_search_build(mail_search_register_get_human(),
+ parser, &charset, &sargs, &error) < 0)
+ i_fatal("%s", error);
+ mail_search_parser_deinit(&parser);
+ return sargs;
+}
+
+static int cmd_force_resync_box(struct doveadm_mail_cmd_context *_ctx,
+ const struct mailbox_info *info)
+{
+ struct force_resync_cmd_context *ctx =
+ (struct force_resync_cmd_context *)_ctx;
+ enum mailbox_flags flags = MAILBOX_FLAG_IGNORE_ACLS;
+ struct mailbox *box;
+ int ret = 0;
+
+ if (ctx->fsck)
+ flags |= MAILBOX_FLAG_FSCK;
+
+ box = mailbox_alloc(info->ns->list, info->vname, flags);
+ if (mailbox_open(box) < 0) {
+ i_error("Opening mailbox %s failed: %s", info->vname,
+ mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ } else if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FORCE_RESYNC |
+ MAILBOX_SYNC_FLAG_FIX_INCONSISTENT) < 0) {
+ i_error("Forcing a resync on mailbox %s failed: %s",
+ info->vname, mailbox_get_last_internal_error(box, NULL));
+ doveadm_mail_failed_mailbox(_ctx, box);
+ ret = -1;
+ }
+ mailbox_free(&box);
+ return ret;
+}
+
+static int cmd_force_resync_prerun(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
+ struct mail_storage_service_user *service_user,
+ const char **error_r)
+{
+ if (mail_storage_service_user_set_setting(service_user,
+ "mailbox_list_index_very_dirty_syncs",
+ "no",
+ error_r) <= 0)
+ i_unreached();
+ return 0;
+}
+
+static int cmd_force_resync_run(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user)
+{
+ const enum mailbox_list_iter_flags iter_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS |
+ MAILBOX_LIST_ITER_STAR_WITHIN_NS;
+ const enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
+ struct mailbox_list_iterate_context *iter;
+ const struct mailbox_info *info;
+ int ret = 0;
+
+ iter = mailbox_list_iter_init_namespaces(user->namespaces, ctx->args,
+ ns_mask, iter_flags);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ if ((info->flags & (MAILBOX_NOSELECT |
+ MAILBOX_NONEXISTENT)) == 0) T_BEGIN {
+ if (cmd_force_resync_box(ctx, info) < 0)
+ ret = -1;
+ } T_END;
+ }
+ if (mailbox_list_iter_deinit(&iter) < 0) {
+ i_error("Listing mailboxes failed: %s",
+ mailbox_list_get_last_internal_error(user->namespaces->list, NULL));
+ doveadm_mail_failed_list(ctx, user->namespaces->list);
+ ret = -1;
+ }
+ return ret;
+}
+
+static void
+cmd_force_resync_init(struct doveadm_mail_cmd_context *_ctx ATTR_UNUSED,
+ const char *const args[])
+{
+ if (args[0] == NULL)
+ doveadm_mail_help_name("force-resync");
+}
+
+static bool
+cmd_force_resync_parse_arg(struct doveadm_mail_cmd_context *_ctx, int c)
+{
+ struct force_resync_cmd_context *ctx =
+ (struct force_resync_cmd_context *)_ctx;
+
+ switch (c) {
+ case 'f':
+ ctx->fsck = TRUE;
+ break;
+ default:
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static struct doveadm_mail_cmd_context *cmd_force_resync_alloc(void)
+{
+ struct force_resync_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_alloc(struct force_resync_cmd_context);
+ ctx->ctx.getopt_args = "f";
+ ctx->ctx.v.parse_arg = cmd_force_resync_parse_arg;
+ ctx->ctx.v.init = cmd_force_resync_init;
+ ctx->ctx.v.run = cmd_force_resync_run;
+ ctx->ctx.v.prerun = cmd_force_resync_prerun;
+ return &ctx->ctx;
+}
+
+static void
+doveadm_cctx_to_storage_service_input(const struct doveadm_cmd_context *cctx,
+ struct mail_storage_service_input *input_r)
+{
+ i_zero(input_r);
+ input_r->service = "doveadm";
+ input_r->remote_ip = cctx->remote_ip;
+ input_r->remote_port = cctx->remote_port;
+ input_r->local_ip = cctx->local_ip;
+ input_r->local_port = cctx->local_port;
+ input_r->username = cctx->username;
+}
+
+static int
+doveadm_mail_next_user(struct doveadm_mail_cmd_context *ctx,
+ const char **error_r)
+{
+ const struct doveadm_cmd_context *cctx = ctx->cctx;
+ struct mail_storage_service_input input;
+ const char *error, *ip;
+ int ret;
+
+ i_assert(cctx != NULL);
+
+ ip = net_ip2addr(&cctx->remote_ip);
+ if (ip[0] == '\0')
+ i_set_failure_prefix("doveadm(%s): ", cctx->username);
+ else
+ i_set_failure_prefix("doveadm(%s,%s): ", ip, cctx->username);
+ doveadm_cctx_to_storage_service_input(cctx, &input);
+ if (ctx->cmd_input != NULL)
+ i_stream_seek(ctx->cmd_input, 0);
+
+ /* see if we want to execute this command via (another)
+ doveadm server */
+ ret = doveadm_mail_server_user(ctx, &input, error_r);
+ if (ret != 0)
+ return ret;
+
+ ret = mail_storage_service_lookup(ctx->storage_service, &input,
+ &ctx->cur_service_user, &error);
+ if (ret <= 0) {
+ if (ret < 0) {
+ *error_r = t_strdup_printf("User lookup failed: %s",
+ error);
+ }
+ return ret;
+ }
+
+ if (ctx->v.prerun != NULL) {
+ if (ctx->v.prerun(ctx, ctx->cur_service_user, error_r) < 0) {
+ mail_storage_service_user_unref(&ctx->cur_service_user);
+ return -1;
+ }
+ }
+
+ ret = mail_storage_service_next(ctx->storage_service,
+ ctx->cur_service_user,
+ &ctx->cur_mail_user, error_r);
+ if (ret < 0) {
+ mail_storage_service_user_unref(&ctx->cur_service_user);
+ return ret;
+ }
+
+ struct event_reason *reason =
+ event_reason_begin(event_reason_code_prefix("doveadm", "cmd_",
+ ctx->cmd->name));
+ T_BEGIN {
+ if (ctx->v.run(ctx, ctx->cur_mail_user) < 0) {
+ i_assert(ctx->exit_code != 0);
+ }
+ } T_END;
+ mail_user_deinit(&ctx->cur_mail_user);
+ /* user deinit may still do some work, so finish the reason after it */
+ event_reason_end(&reason);
+
+ mail_storage_service_user_unref(&ctx->cur_service_user);
+ return 1;
+}
+
+static void sig_die(const siginfo_t *si, void *context ATTR_UNUSED)
+{
+ killed_signo = si->si_signo;
+}
+
+int doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx,
+ const char **error_r)
+{
+ const struct doveadm_cmd_context *cctx = ctx->cctx;
+
+ i_assert(cctx->username != NULL);
+
+ doveadm_cctx_to_storage_service_input(cctx, &ctx->storage_service_input);
+ ctx->storage_service = mail_storage_service_init(master_service, NULL,
+ ctx->service_flags);
+ ctx->v.init(ctx, ctx->args);
+ if (hook_doveadm_mail_init != NULL)
+ hook_doveadm_mail_init(ctx);
+
+ lib_signals_set_handler(SIGINT, 0, sig_die, NULL);
+ lib_signals_set_handler(SIGTERM, 0, sig_die, NULL);
+
+ return doveadm_mail_next_user(ctx, error_r);
+}
+
+static void
+doveadm_mail_all_users(struct doveadm_mail_cmd_context *ctx,
+ const char *wildcard_user)
+{
+ struct doveadm_cmd_context *cctx = ctx->cctx;
+ unsigned int user_idx;
+ const char *ip, *user, *error;
+ int ret;
+
+ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP;
+
+ doveadm_cctx_to_storage_service_input(cctx, &ctx->storage_service_input);
+ ctx->storage_service = mail_storage_service_init(master_service, NULL,
+ ctx->service_flags);
+ lib_signals_set_handler(SIGINT, 0, sig_die, NULL);
+ lib_signals_set_handler(SIGTERM, 0, sig_die, NULL);
+
+ ctx->v.init(ctx, ctx->args);
+
+ mail_storage_service_all_init_mask(ctx->storage_service,
+ wildcard_user != NULL ? wildcard_user : "");
+
+ if (hook_doveadm_mail_init != NULL)
+ hook_doveadm_mail_init(ctx);
+
+ user_idx = 0;
+ while ((ret = ctx->v.get_next_user(ctx, &user)) > 0) {
+ if (wildcard_user != NULL) {
+ if (!wildcard_match_icase(user, wildcard_user))
+ continue;
+ }
+ cctx->username = user;
+ doveadm_print_sticky("username", user);
+ T_BEGIN {
+ ret = doveadm_mail_next_user(ctx, &error);
+ if (ret < 0)
+ i_error("%s", error);
+ else if (ret == 0)
+ i_info("User no longer exists, skipping");
+ } T_END;
+ if (ret == -1)
+ break;
+ if (doveadm_verbose) {
+ if (++user_idx % 100 == 0) {
+ printf("\r%d", user_idx);
+ fflush(stdout);
+ }
+ }
+ if (killed_signo != 0) {
+ i_warning("Killed with signal %d", killed_signo);
+ ret = -1;
+ break;
+ }
+ }
+ if (doveadm_verbose)
+ printf("\n");
+ ip = net_ip2addr(&cctx->remote_ip);
+ if (ip[0] == '\0')
+ i_set_failure_prefix("doveadm: ");
+ else
+ i_set_failure_prefix("doveadm(%s): ", ip);
+ if (ret < 0) {
+ i_error("Failed to iterate through some users");
+ ctx->exit_code = EX_TEMPFAIL;
+ }
+}
+
+static void
+doveadm_mail_cmd_init_noop(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED,
+ const char *const args[] ATTR_UNUSED)
+{
+}
+
+static int
+doveadm_mail_cmd_get_next_user(struct doveadm_mail_cmd_context *ctx,
+ const char **username_r)
+{
+ if (ctx->users_list_input == NULL)
+ return mail_storage_service_all_next(ctx->storage_service, username_r);
+
+ *username_r = i_stream_read_next_line(ctx->users_list_input);
+ if (ctx->users_list_input->stream_errno != 0) {
+ i_error("read(%s) failed: %s",
+ i_stream_get_name(ctx->users_list_input),
+ i_stream_get_error(ctx->users_list_input));
+ return -1;
+ }
+ return *username_r != NULL ? 1 : 0;
+}
+
+static void
+doveadm_mail_cmd_deinit_noop(struct doveadm_mail_cmd_context *ctx ATTR_UNUSED)
+{
+}
+
+struct doveadm_mail_cmd_context *
+doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd,
+ const struct doveadm_settings *set)
+{
+ struct doveadm_mail_cmd_context *ctx;
+
+ ctx = cmd->alloc();
+ ctx->set = set;
+ ctx->cmd = cmd;
+ if (ctx->v.init == NULL)
+ ctx->v.init = doveadm_mail_cmd_init_noop;
+ if (ctx->v.get_next_user == NULL)
+ ctx->v.get_next_user = doveadm_mail_cmd_get_next_user;
+ if (ctx->v.deinit == NULL)
+ ctx->v.deinit = doveadm_mail_cmd_deinit_noop;
+
+ p_array_init(&ctx->module_contexts, ctx->pool, 5);
+ return ctx;
+}
+
+static struct doveadm_mail_cmd_context *
+doveadm_mail_cmdline_init(const struct doveadm_mail_cmd *cmd)
+{
+ struct doveadm_mail_cmd_context *ctx;
+
+ ctx = doveadm_mail_cmd_init(cmd, doveadm_settings);
+ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_NO_LOG_INIT;
+ if (doveadm_debug)
+ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_DEBUG;
+ return ctx;
+}
+
+static void
+doveadm_mail_cmd_exec(struct doveadm_mail_cmd_context *ctx,
+ const char *wildcard_user)
+{
+ const struct doveadm_cmd_context *cctx = ctx->cctx;
+ bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI);
+ int ret;
+ const char *error;
+
+ if (ctx->v.preinit != NULL)
+ ctx->v.preinit(ctx);
+
+ ctx->iterate_single_user =
+ !ctx->iterate_all_users && wildcard_user == NULL;
+ if (doveadm_print_is_initialized() &&
+ (!ctx->iterate_single_user || ctx->add_username_header)) {
+ doveadm_print_header("username", "Username",
+ DOVEADM_PRINT_HEADER_FLAG_STICKY |
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ }
+
+ if (ctx->iterate_single_user) {
+ if (cctx->username == NULL)
+ i_fatal_status(EX_USAGE, "USER environment is missing and -u option not used");
+ if (!cli) {
+ /* we may access multiple users */
+ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP;
+ }
+
+ if (ctx->add_username_header)
+ doveadm_print_sticky("username", cctx->username);
+ ret = doveadm_mail_single_user(ctx, &error);
+ if (ret < 0) {
+ /* user lookup/init failed somehow */
+ doveadm_exit_code = EX_TEMPFAIL;
+ i_error("%s", error);
+ } else if (ret == 0) {
+ doveadm_exit_code = EX_NOUSER;
+ i_error("User doesn't exist");
+ }
+ } else {
+ ctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_TEMP_PRIV_DROP;
+ doveadm_mail_all_users(ctx, wildcard_user);
+ }
+ doveadm_mail_server_flush();
+ doveadm_mail_cmd_deinit(ctx);
+ doveadm_print_flush();
+
+ /* service deinit unloads mail plugins, so do it late */
+ mail_storage_service_deinit(&ctx->storage_service);
+
+ if (ctx->exit_code != 0)
+ doveadm_exit_code = ctx->exit_code;
+}
+
+void doveadm_mail_cmd_deinit(struct doveadm_mail_cmd_context *ctx)
+{
+ ctx->v.deinit(ctx);
+ if (ctx->search_args != NULL)
+ mail_search_args_unref(&ctx->search_args);
+}
+
+void doveadm_mail_cmd_free(struct doveadm_mail_cmd_context *ctx)
+{
+ i_stream_unref(&ctx->users_list_input);
+ i_stream_unref(&ctx->cmd_input);
+ pool_unref(&ctx->pool);
+}
+
+void doveadm_mail_help(const struct doveadm_mail_cmd *cmd)
+{
+ fprintf(stderr, "doveadm %s "DOVEADM_CMD_MAIL_USAGE_PREFIX" %s\n",
+ cmd->name, cmd->usage_args == NULL ? "" : cmd->usage_args);
+ lib_exit(EX_USAGE);
+}
+
+void doveadm_mail_try_help_name(const char *cmd_name)
+{
+ const struct doveadm_cmd_ver2 *cmd2;
+
+ cmd2 = doveadm_cmd_find_ver2(cmd_name);
+ if (cmd2 != NULL)
+ help_ver2(cmd2);
+}
+
+void doveadm_mail_help_name(const char *cmd_name)
+{
+ doveadm_mail_try_help_name(cmd_name);
+ i_fatal("Missing help for command %s", cmd_name);
+}
+
+static struct doveadm_cmd_ver2 doveadm_cmd_force_resync_ver2 = {
+ .name = "force-resync",
+ .mail_cmd = cmd_force_resync_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX "[-f] <mailbox mask>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAM('f', "fsck", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "mailbox-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+static struct doveadm_cmd_ver2 doveadm_cmd_purge_ver2 = {
+ .name = "purge",
+ .mail_cmd = cmd_purge_alloc,
+ .usage = DOVEADM_CMD_MAIL_USAGE_PREFIX,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_MAIL_COMMON
+DOVEADM_CMD_PARAMS_END
+};
+
+static struct doveadm_cmd_ver2 *mail_commands_ver2[] = {
+ &doveadm_cmd_batch,
+ &doveadm_cmd_dsync_backup,
+ &doveadm_cmd_dsync_mirror,
+ &doveadm_cmd_dsync_server,
+ &doveadm_cmd_mailbox_metadata_set_ver2,
+ &doveadm_cmd_mailbox_metadata_unset_ver2,
+ &doveadm_cmd_mailbox_metadata_get_ver2,
+ &doveadm_cmd_mailbox_metadata_list_ver2,
+ &doveadm_cmd_mailbox_status_ver2,
+ &doveadm_cmd_mailbox_list_ver2,
+ &doveadm_cmd_mailbox_create_ver2,
+ &doveadm_cmd_mailbox_delete_ver2,
+ &doveadm_cmd_mailbox_rename_ver2,
+ &doveadm_cmd_mailbox_subscribe_ver2,
+ &doveadm_cmd_mailbox_unsubscribe_ver2,
+ &doveadm_cmd_mailbox_update_ver2,
+ &doveadm_cmd_mailbox_path_ver2,
+ &doveadm_cmd_fetch_ver2,
+ &doveadm_cmd_save_ver2,
+ &doveadm_cmd_index_ver2,
+ &doveadm_cmd_altmove_ver2,
+ &doveadm_cmd_deduplicate_ver2,
+ &doveadm_cmd_expunge_ver2,
+ &doveadm_cmd_flags_add_ver2,
+ &doveadm_cmd_flags_remove_ver2,
+ &doveadm_cmd_flags_replace_ver2,
+ &doveadm_cmd_import_ver2,
+ &doveadm_cmd_force_resync_ver2,
+ &doveadm_cmd_purge_ver2,
+ &doveadm_cmd_search_ver2,
+ &doveadm_cmd_copy_ver2,
+ &doveadm_cmd_move_ver2,
+ &doveadm_cmd_mailbox_cache_decision,
+ &doveadm_cmd_mailbox_cache_remove,
+ &doveadm_cmd_mailbox_cache_purge,
+ &doveadm_cmd_rebuild_attachments,
+};
+
+void doveadm_mail_init(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(mail_commands_ver2); i++)
+ doveadm_cmd_register_ver2(mail_commands_ver2[i]);
+}
+
+void doveadm_mail_init_finish(void)
+{
+ struct module_dir_load_settings mod_set;
+
+ i_zero(&mod_set);
+ mod_set.abi_version = DOVECOT_ABI_VERSION;
+ mod_set.require_init_funcs = TRUE;
+ mod_set.debug = doveadm_debug;
+ mod_set.binary_name = "doveadm";
+
+ /* load all configured mail plugins */
+ mail_storage_service_modules =
+ module_dir_load_missing(mail_storage_service_modules,
+ doveadm_settings->mail_plugin_dir,
+ doveadm_settings->mail_plugins,
+ &mod_set);
+ /* keep mail_storage_init() referenced so that its _deinit() doesn't
+ try to free doveadm plugins' hooks too early. */
+ mail_storage_init();
+}
+
+void doveadm_mail_deinit(void)
+{
+ mail_storage_deinit();
+ module_dir_unload(&mail_storage_service_modules);
+}
+
+static int doveadm_cmd_parse_arg(struct doveadm_mail_cmd_context *mctx,
+ const struct doveadm_cmd_param *arg,
+ ARRAY_TYPE(const_string) *full_args)
+{
+ const char *short_opt_str =
+ p_strdup_printf(mctx->pool, "-%c", arg->short_opt);
+ const char *arg_value = NULL;
+
+ switch (arg->type) {
+ case CMD_PARAM_BOOL:
+ break;
+ case CMD_PARAM_INT64:
+ arg_value = dec2str(arg->value.v_int64);
+ break;
+ case CMD_PARAM_IP:
+ arg_value = net_ip2addr(&arg->value.v_ip);
+ break;
+ case CMD_PARAM_STR:
+ arg_value = arg->value.v_string;
+ break;
+ case CMD_PARAM_ARRAY: {
+ const char *str;
+
+ array_foreach_elem(&arg->value.v_array, str) {
+ optarg = (char *)str;
+ if (!mctx->v.parse_arg(mctx, arg->short_opt))
+ return -1;
+ array_push_back(full_args, &short_opt_str);
+ array_push_back(full_args, &str);
+ }
+ return 0;
+ }
+ default:
+ i_panic("Cannot convert parameter %s to short opt", arg->name);
+ }
+
+ optarg = (char *)arg_value;
+ if (!mctx->v.parse_arg(mctx, arg->short_opt))
+ return -1;
+
+ array_push_back(full_args, &short_opt_str);
+ if (arg_value != NULL)
+ array_push_back(full_args, &arg_value);
+ return 0;
+}
+
+void
+doveadm_cmd_ver2_to_mail_cmd_wrapper(struct doveadm_cmd_context *cctx)
+{
+ struct doveadm_mail_cmd_context *mctx;
+ const char *wildcard_user;
+ const char *fieldstr;
+ ARRAY_TYPE(const_string) pargv, full_args;
+ int i;
+ bool cli = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_CLI);
+ bool tcp_server = (cctx->conn_type == DOVEADM_CONNECTION_TYPE_TCP);
+ struct doveadm_mail_cmd mail_cmd = {
+ cctx->cmd->mail_cmd, cctx->cmd->name, cctx->cmd->usage
+ };
+
+ if (!cli) {
+ mctx = doveadm_mail_cmd_init(&mail_cmd, doveadm_settings);
+ /* doveadm-server always does userdb lookups */
+ mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP;
+ } else {
+ mctx = doveadm_mail_cmdline_init(&mail_cmd);
+ }
+ mctx->cctx = cctx;
+ mctx->iterate_all_users = FALSE;
+ wildcard_user = NULL;
+ p_array_init(&full_args, mctx->pool, 8);
+ p_array_init(&pargv, mctx->pool, 8);
+
+ for(i=0;i<cctx->argc;i++) {
+ const struct doveadm_cmd_param *arg = &cctx->argv[i];
+
+ if (!arg->value_set)
+ continue;
+
+ if (strcmp(arg->name, "all-users") == 0) {
+ if (tcp_server)
+ mctx->add_username_header = TRUE;
+ else
+ mctx->iterate_all_users = arg->value.v_bool;
+ fieldstr = "-A";
+ array_push_back(&full_args, &fieldstr);
+ } else if (strcmp(arg->name, "socket-path") == 0) {
+ doveadm_settings->doveadm_socket_path = arg->value.v_string;
+ if (doveadm_settings->doveadm_worker_count == 0)
+ doveadm_settings->doveadm_worker_count = 1;
+ } else if (strcmp(arg->name, "user") == 0) {
+ mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP;
+ if (!tcp_server)
+ cctx->username = arg->value.v_string;
+
+ fieldstr = "-u";
+ array_push_back(&full_args, &fieldstr);
+ array_push_back(&full_args, &arg->value.v_string);
+ if (strchr(arg->value.v_string, '*') != NULL ||
+ strchr(arg->value.v_string, '?') != NULL) {
+ if (tcp_server)
+ mctx->add_username_header = TRUE;
+ else {
+ wildcard_user = arg->value.v_string;
+ cctx->username = NULL;
+ }
+ }
+ } else if (strcmp(arg->name, "user-file") == 0) {
+ mctx->service_flags |= MAIL_STORAGE_SERVICE_FLAG_USERDB_LOOKUP;
+ wildcard_user = "*";
+ mctx->users_list_input = arg->value.v_istream;
+ fieldstr = "-F";
+ array_push_back(&full_args, &fieldstr);
+ fieldstr = ""; /* value doesn't really matter */
+ array_push_back(&full_args, &fieldstr);
+ i_stream_ref(mctx->users_list_input);
+ } else if (strcmp(arg->name, "field") == 0 ||
+ strcmp(arg->name, "flag") == 0) {
+ /* mailbox status, fetch, flags: convert an array into a
+ single space-separated parameter (alternative to
+ fieldstr) */
+ fieldstr = p_array_const_string_join(mctx->pool,
+ &arg->value.v_array, " ");
+ array_push_back(&pargv, &fieldstr);
+ } else if (strcmp(arg->name, "file") == 0) {
+ /* input for doveadm_mail_get_input(),
+ used by e.g. save */
+ if (mctx->cmd_input != NULL) {
+ i_error("Only one file input allowed: %s", arg->name);
+ doveadm_mail_cmd_free(mctx);
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+ mctx->cmd_input = arg->value.v_istream;
+ i_stream_ref(mctx->cmd_input);
+
+ } else if (strcmp(arg->name, "trans-flags") == 0) {
+ /* This parameter allows to set additional
+ * mailbox transaction flags. */
+ mctx->transaction_flags = arg->value.v_int64;
+
+ /* Keep all named special parameters above this line */
+
+ } else if (mctx->v.parse_arg != NULL && arg->short_opt != '\0') {
+ if (doveadm_cmd_parse_arg(mctx, arg, &full_args) < 0) {
+ i_error("Invalid parameter %c", arg->short_opt);
+ doveadm_mail_cmd_free(mctx);
+ doveadm_exit_code = EX_USAGE;
+ }
+ } else if ((arg->flags & CMD_PARAM_FLAG_POSITIONAL) != 0) {
+ /* feed this into pargv */
+ if (arg->type == CMD_PARAM_ARRAY)
+ array_append_array(&pargv, &arg->value.v_array);
+ else if (arg->type == CMD_PARAM_STR)
+ array_push_back(&pargv, &arg->value.v_string);
+ } else {
+ doveadm_exit_code = EX_USAGE;
+ i_error("invalid parameter: %s", arg->name);
+ doveadm_mail_cmd_free(mctx);
+ return;
+ }
+ }
+
+ const char *dashdash = "--";
+ array_push_back(&full_args, &dashdash);
+
+ array_append_zero(&pargv);
+ /* All the -parameters need to be included in full_args so that
+ they're sent to doveadm-server. */
+ unsigned int args_pos = array_count(&full_args);
+ array_append_array(&full_args, &pargv);
+
+ mctx->args = array_idx(&full_args, args_pos);
+ mctx->full_args = array_front(&full_args);
+
+ doveadm_mail_cmd_exec(mctx, wildcard_user);
+ doveadm_mail_cmd_free(mctx);
+}
diff --git a/src/doveadm/doveadm-mail.h b/src/doveadm/doveadm-mail.h
new file mode 100644
index 0000000..af5d0fa
--- /dev/null
+++ b/src/doveadm/doveadm-mail.h
@@ -0,0 +1,206 @@
+#ifndef DOVEADM_MAIL_H
+#define DOVEADM_MAIL_H
+
+#include <stdio.h>
+#include "doveadm.h"
+#include "doveadm-util.h"
+#include "module-context.h"
+#include "mail-error.h"
+#include "mail-storage.h"
+#include "mail-storage-service.h"
+
+struct mailbox;
+struct mailbox_list;
+struct mail_storage;
+struct mail_user;
+struct doveadm_mail_cmd_context;
+
+struct doveadm_mail_cmd_vfuncs {
+ /* Parse one getopt() parameter. This is called for each parameter. */
+ bool (*parse_arg)(struct doveadm_mail_cmd_context *ctx, int c);
+ /* Usually not needed. The preinit() is called just after parsing all
+ parameters, but before any userdb lookups are done. This allows the
+ preinit() to alter the userdb lookup behavior (especially
+ service_flags). */
+ void (*preinit)(struct doveadm_mail_cmd_context *ctx);
+ /* Initialize the command. Most importantly if the function prints
+ anything, this should initialize the headers. It shouldn't however
+ do any actual work. The init() is called also when doveadm is
+ performing the work via doveadm-server, which could be running
+ remotely with completely different Dovecot configuration. */
+ void (*init)(struct doveadm_mail_cmd_context *ctx,
+ const char *const args[]);
+ /* Usually not needed. When iterating through multiple users, use this
+ function to get the next username. Overriding this is usually done
+ only when there's a known username filter, such as the expire
+ plugin. */
+ int (*get_next_user)(struct doveadm_mail_cmd_context *ctx,
+ const char **username_r);
+ /* Usually not needed. This is called between
+ mail_storage_service_lookup() and mail_storage_service_next() for
+ each user. */
+ int (*prerun)(struct doveadm_mail_cmd_context *ctx,
+ struct mail_storage_service_user *service_user,
+ const char **error_r);
+ /* This is the main function which performs all the work for the
+ command. This is called once per each user. */
+ int (*run)(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *mail_user);
+ /* Deinitialize the command. Called once at the end - even if
+ preinit() or init() was never called. */
+ void (*deinit)(struct doveadm_mail_cmd_context *ctx);
+};
+
+struct doveadm_mail_cmd_module_register {
+ unsigned int id;
+};
+
+union doveadm_mail_cmd_module_context {
+ struct doveadm_mail_cmd_vfuncs super;
+ struct doveadm_mail_cmd_module_register *reg;
+};
+
+struct doveadm_mail_cmd_context {
+ pool_t pool;
+ struct doveadm_cmd_context *cctx;
+ const struct doveadm_mail_cmd *cmd;
+ const char *const *args;
+ /* args including -options */
+ const char *const *full_args;
+
+ const char *getopt_args;
+ const struct doveadm_settings *set;
+ enum mail_storage_service_flags service_flags;
+ enum mailbox_transaction_flags transaction_flags;
+ struct mail_storage_service_ctx *storage_service;
+ struct mail_storage_service_input storage_service_input;
+ /* search args aren't set for all mail commands */
+ struct mail_search_args *search_args;
+ struct istream *users_list_input;
+
+ struct mail_storage_service_user *cur_service_user;
+ struct mail_user *cur_mail_user;
+ struct doveadm_mail_cmd_vfuncs v;
+
+ struct istream *cmd_input;
+ int cmd_input_fd;
+
+ ARRAY(union doveadm_mail_cmd_module_context *) module_contexts;
+
+ /* if non-zero, exit with this code */
+ int exit_code;
+
+ /* This command is being called by a remote doveadm client. */
+ bool proxying:1;
+ /* We're handling only a single user */
+ bool iterate_single_user:1;
+ /* We're going through all users (not set for wildcard usernames) */
+ bool iterate_all_users:1;
+ /* Add username header to all replies */
+ bool add_username_header:1;
+};
+
+struct doveadm_mail_cmd {
+ struct doveadm_mail_cmd_context *(*alloc)(void);
+ const char *name;
+ const char *usage_args;
+};
+
+extern void (*hook_doveadm_mail_init)(struct doveadm_mail_cmd_context *ctx);
+extern struct doveadm_mail_cmd_module_register doveadm_mail_cmd_module_register;
+extern char doveadm_mail_cmd_hide;
+
+bool doveadm_is_killed(void);
+int doveadm_killed_signo(void);
+
+void doveadm_mail_help(const struct doveadm_mail_cmd *cmd) ATTR_NORETURN;
+void doveadm_mail_help_name(const char *cmd_name) ATTR_NORETURN;
+void doveadm_mail_try_help_name(const char *cmd_name);
+
+void doveadm_mail_init(void);
+void doveadm_mail_init_finish(void);
+void doveadm_mail_deinit(void);
+
+struct doveadm_mail_cmd_context *
+doveadm_mail_cmd_init(const struct doveadm_mail_cmd *cmd,
+ const struct doveadm_settings *set);
+void doveadm_mail_cmd_deinit(struct doveadm_mail_cmd_context *ctx);
+void doveadm_mail_cmd_free(struct doveadm_mail_cmd_context *ctx);
+int doveadm_mail_single_user(struct doveadm_mail_cmd_context *ctx,
+ const char **error_r);
+int doveadm_mail_server_user(struct doveadm_mail_cmd_context *ctx,
+ const struct mail_storage_service_input *input,
+ const char **error_r);
+void doveadm_mail_server_flush(void);
+
+/* Request input stream to be read (from stdin). This must be called from
+ the command's init() function. */
+void doveadm_mail_get_input(struct doveadm_mail_cmd_context *ctx);
+
+struct mailbox *
+doveadm_mailbox_find(struct mail_user *user, const char *mailbox);
+struct mail_search_args *
+doveadm_mail_build_search_args(const char *const args[]);
+void doveadm_mailbox_args_check(const char *const args[]);
+struct mail_search_args *
+doveadm_mail_mailbox_search_args_build(const char *const args[]);
+
+void expunge_search_args_check(struct mail_search_args *args, const char *cmd);
+
+struct doveadm_mail_cmd_context *
+doveadm_mail_cmd_alloc_size(size_t size);
+#define doveadm_mail_cmd_alloc(type) \
+ (type *)doveadm_mail_cmd_alloc_size(sizeof(type))
+
+void doveadm_mail_failed_error(struct doveadm_mail_cmd_context *ctx,
+ enum mail_error error);
+void doveadm_mail_failed_storage(struct doveadm_mail_cmd_context *ctx,
+ struct mail_storage *storage);
+void doveadm_mail_failed_mailbox(struct doveadm_mail_cmd_context *ctx,
+ struct mailbox *box);
+void doveadm_mail_failed_list(struct doveadm_mail_cmd_context *ctx,
+ struct mailbox_list *list);
+
+extern struct doveadm_cmd_ver2 doveadm_cmd_batch;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_set_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_unset_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_get_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_metadata_list_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_status_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_list_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_create_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_delete_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_rename_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_subscribe_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_unsubscribe_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_fetch_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_save_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_index_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_altmove_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_deduplicate_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_expunge_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_flags_add_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_flags_remove_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_flags_replace_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_import_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_search_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_copy_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_move_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_update_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_path_ver2;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_decision;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_remove;
+extern struct doveadm_cmd_ver2 doveadm_cmd_mailbox_cache_purge;
+extern struct doveadm_cmd_ver2 doveadm_cmd_rebuild_attachments;
+
+#define DOVEADM_CMD_MAIL_COMMON \
+DOVEADM_CMD_PARAM('A', "all-users", CMD_PARAM_BOOL, 0) \
+DOVEADM_CMD_PARAM('S', "socket-path", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0) \
+DOVEADM_CMD_PARAM('\0', "trans-flags", CMD_PARAM_INT64, 0) \
+DOVEADM_CMD_PARAM('F', "user-file", CMD_PARAM_ISTREAM, 0)
+
+#define DOVEADM_CMD_MAIL_USAGE_PREFIX \
+ "[-u <user>|-A] [-S <socket_path>] "
+
+#endif
diff --git a/src/doveadm/doveadm-mailbox-list-iter.c b/src/doveadm/doveadm-mailbox-list-iter.c
new file mode 100644
index 0000000..c77398c
--- /dev/null
+++ b/src/doveadm/doveadm-mailbox-list-iter.c
@@ -0,0 +1,195 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "mail-search.h"
+#include "mail-namespace.h"
+#include "mailbox-list.h"
+#include "doveadm-mail.h"
+#include "doveadm-mailbox-list-iter.h"
+
+struct doveadm_mailbox_list_iter {
+ struct mail_user *user;
+ struct doveadm_mail_cmd_context *ctx;
+ struct mail_search_args *search_args;
+ enum mailbox_list_iter_flags iter_flags;
+
+ struct mailbox_list_iterate_context *iter;
+
+ struct mailbox_info info;
+ ARRAY_TYPE(const_string) patterns;
+ unsigned int pattern_idx;
+
+ bool only_selectable;
+};
+
+static bool
+search_args_get_mailbox_patterns(const struct mail_search_arg *args,
+ ARRAY_TYPE(const_string) *patterns,
+ bool *have_guid, bool *have_wildcards)
+{
+ const struct mail_search_arg *subargs;
+
+ for (; args != NULL; args = args->next) {
+ switch (args->type) {
+ case SEARCH_OR:
+ /* we don't currently try to optimize OR. */
+ break;
+ case SEARCH_SUB:
+ case SEARCH_INTHREAD:
+ subargs = args->value.subargs;
+ for (; subargs != NULL; subargs = subargs->next) {
+ if (!search_args_get_mailbox_patterns(subargs,
+ patterns, have_guid,
+ have_wildcards))
+ return FALSE;
+ }
+ break;
+ case SEARCH_MAILBOX_GLOB:
+ *have_wildcards = TRUE;
+ /* fall through */
+ case SEARCH_MAILBOX:
+ if (args->match_not) {
+ array_clear(patterns);
+ return FALSE;
+ }
+ array_push_back(patterns, &args->value.str);
+ break;
+ case SEARCH_MAILBOX_GUID:
+ *have_guid = TRUE;
+ break;
+ default:
+ break;
+ }
+ }
+ return TRUE;
+}
+
+static struct doveadm_mailbox_list_iter *
+doveadm_mailbox_list_iter_init_nsmask(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user,
+ struct mail_search_args *search_args,
+ enum mailbox_list_iter_flags iter_flags,
+ enum mail_namespace_type ns_mask)
+{
+ static const char *all_pattern = "*";
+ struct doveadm_mailbox_list_iter *iter;
+ bool have_guid = FALSE, have_wildcards = FALSE;
+
+ iter = i_new(struct doveadm_mailbox_list_iter, 1);
+ iter->ctx = ctx;
+ iter->search_args = search_args;
+ iter->user = user;
+ i_array_init(&iter->patterns, 16);
+ (void)search_args_get_mailbox_patterns(search_args->args,
+ &iter->patterns,
+ &have_guid, &have_wildcards);
+
+ if (array_count(&iter->patterns) == 0) {
+ iter_flags |= MAILBOX_LIST_ITER_SKIP_ALIASES;
+ if (have_guid) {
+ ns_mask |= MAIL_NAMESPACE_TYPE_SHARED |
+ MAIL_NAMESPACE_TYPE_PUBLIC;
+ }
+ array_push_back(&iter->patterns, &all_pattern);
+ } else if (have_wildcards) {
+ iter_flags |= MAILBOX_LIST_ITER_STAR_WITHIN_NS;
+ ns_mask |= MAIL_NAMESPACE_TYPE_SHARED |
+ MAIL_NAMESPACE_TYPE_PUBLIC;
+ } else {
+ /* just return the listed mailboxes without actually
+ iterating through. this also allows accessing mailboxes
+ without lookup ACL right */
+ return iter;
+ }
+ array_append_zero(&iter->patterns);
+
+ iter->only_selectable = TRUE;
+ iter->iter_flags = iter_flags;
+ iter->iter = mailbox_list_iter_init_namespaces(user->namespaces,
+ array_front(&iter->patterns),
+ ns_mask, iter_flags);
+ return iter;
+}
+
+struct doveadm_mailbox_list_iter *
+doveadm_mailbox_list_iter_init(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user,
+ struct mail_search_args *search_args,
+ enum mailbox_list_iter_flags iter_flags)
+{
+ enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_PRIVATE;
+
+ return doveadm_mailbox_list_iter_init_nsmask(ctx, user, search_args,
+ iter_flags, ns_mask);
+}
+
+struct doveadm_mailbox_list_iter *
+doveadm_mailbox_list_iter_full_init(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user,
+ struct mail_search_args *search_args,
+ enum mailbox_list_iter_flags iter_flags)
+{
+ enum mail_namespace_type ns_mask = MAIL_NAMESPACE_TYPE_MASK_ALL;
+ struct doveadm_mailbox_list_iter *iter;
+
+ iter = doveadm_mailbox_list_iter_init_nsmask(ctx, user, search_args,
+ iter_flags, ns_mask);
+ iter->only_selectable = FALSE;
+ return iter;
+}
+
+int doveadm_mailbox_list_iter_deinit(struct doveadm_mailbox_list_iter **_iter)
+{
+ struct doveadm_mailbox_list_iter *iter = *_iter;
+ enum mail_error error;
+ int ret;
+
+ *_iter = NULL;
+
+ if (iter->iter == NULL)
+ ret = 0;
+ else if ((ret = mailbox_list_iter_deinit(&iter->iter)) < 0) {
+ i_error("Listing mailboxes failed: %s",
+ mailbox_list_get_last_internal_error(iter->user->namespaces->list, &error));
+ doveadm_mail_failed_error(iter->ctx, error);
+ }
+ array_free(&iter->patterns);
+ i_free(iter);
+ return ret;
+}
+
+const struct mailbox_info *
+doveadm_mailbox_list_iter_next(struct doveadm_mailbox_list_iter *iter)
+{
+ const struct mailbox_info *info;
+ const char *const *patterns;
+ unsigned int count;
+
+ while (iter->iter == NULL) {
+ patterns = array_get(&iter->patterns, &count);
+ if (iter->pattern_idx == count)
+ return NULL;
+
+ iter->info.vname = patterns[iter->pattern_idx++];
+ iter->info.ns = mail_namespace_find(iter->user->namespaces,
+ iter->info.vname);
+ return &iter->info;
+ }
+
+ while ((info = mailbox_list_iter_next(iter->iter)) != NULL) {
+ char sep = mail_namespace_get_sep(info->ns);
+
+ if ((info->flags & (MAILBOX_NOSELECT |
+ MAILBOX_NONEXISTENT)) != 0) {
+ if (iter->only_selectable)
+ continue;
+ }
+
+ if (mail_search_args_match_mailbox(iter->search_args,
+ info->vname, sep))
+ break;
+ }
+ return info;
+}
diff --git a/src/doveadm/doveadm-mailbox-list-iter.h b/src/doveadm/doveadm-mailbox-list-iter.h
new file mode 100644
index 0000000..9b205e2
--- /dev/null
+++ b/src/doveadm/doveadm-mailbox-list-iter.h
@@ -0,0 +1,25 @@
+#ifndef DOVEADM_MAILBOX_LIST_ITER_H
+#define DOVEADM_MAILBOX_LIST_ITER_H
+
+#include "mailbox-list-iter.h"
+
+struct doveadm_mail_cmd_context;
+
+/* List only selectable mailboxes */
+struct doveadm_mailbox_list_iter *
+doveadm_mailbox_list_iter_init(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user,
+ struct mail_search_args *search_args,
+ enum mailbox_list_iter_flags iter_flags);
+/* List all mailboxes */
+struct doveadm_mailbox_list_iter *
+doveadm_mailbox_list_iter_full_init(struct doveadm_mail_cmd_context *ctx,
+ struct mail_user *user,
+ struct mail_search_args *search_args,
+ enum mailbox_list_iter_flags iter_flags);
+int doveadm_mailbox_list_iter_deinit(struct doveadm_mailbox_list_iter **iter);
+
+const struct mailbox_info *
+doveadm_mailbox_list_iter_next(struct doveadm_mailbox_list_iter *iter);
+
+#endif
diff --git a/src/doveadm/doveadm-master.c b/src/doveadm/doveadm-master.c
new file mode 100644
index 0000000..2f161d7
--- /dev/null
+++ b/src/doveadm/doveadm-master.c
@@ -0,0 +1,297 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "istream.h"
+#include "write-full.h"
+#include "master-service.h"
+#include "sleep.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <unistd.h>
+#include <fcntl.h>
+#include <signal.h>
+
+#define MASTER_PID_FILE_NAME "master.pid"
+
+static bool pid_file_read(const char *path, pid_t *pid_r)
+{
+ char buf[32];
+ int fd;
+ ssize_t ret;
+ bool found;
+
+ fd = open(path, O_RDONLY);
+ if (fd == -1) {
+ if (errno == ENOENT)
+ return FALSE;
+ i_fatal("open(%s) failed: %m", path);
+ }
+
+ ret = read(fd, buf, sizeof(buf)-1);
+ if (ret <= 0) {
+ if (ret == 0)
+ i_error("Empty PID file in %s", path);
+ else
+ i_fatal("read(%s) failed: %m", path);
+ found = FALSE;
+ } else {
+ if (buf[ret-1] == '\n')
+ ret--;
+ buf[ret] = '\0';
+ if (str_to_pid(buf, pid_r) < 0)
+ found = FALSE;
+ else {
+ found = !(*pid_r == getpid() ||
+ (kill(*pid_r, 0) < 0 && errno == ESRCH));
+ }
+ }
+ i_close_fd(&fd);
+ return found;
+}
+
+void doveadm_master_send_signal(int signo)
+{
+ const char *pidfile_path;
+ unsigned int i;
+ pid_t pid;
+
+ pidfile_path = t_strconcat(doveadm_settings->base_dir,
+ "/"MASTER_PID_FILE_NAME, NULL);
+
+ if (!pid_file_read(pidfile_path, &pid))
+ i_fatal("Dovecot is not running (read from %s)", pidfile_path);
+
+ if (kill(pid, signo) < 0)
+ i_fatal("kill(%s, %d) failed: %m", dec2str(pid), signo);
+
+ if (signo == SIGTERM) {
+ /* wait for a while for the process to die */
+ i_sleep_msecs(1);
+ for (i = 0; i < 30; i++) {
+ if (kill(pid, 0) < 0) {
+ if (errno != ESRCH)
+ i_error("kill() failed: %m");
+ break;
+ }
+ i_sleep_msecs(100);
+ }
+ }
+}
+
+static void cmd_stop(struct doveadm_cmd_context *cctx ATTR_UNUSED)
+{
+ doveadm_master_send_signal(SIGTERM);
+}
+
+static void cmd_reload(struct doveadm_cmd_context *cctx ATTR_UNUSED)
+{
+ doveadm_master_send_signal(SIGHUP);
+}
+
+static struct istream *master_service_send_cmd(const char *cmd)
+{
+ struct istream *input;
+ const char *path, *line;
+
+ path = t_strconcat(doveadm_settings->base_dir, "/master", NULL);
+ int fd = net_connect_unix(path);
+ if (fd == -1)
+ i_fatal("net_connect_unix(%s) failed: %m", path);
+ net_set_nonblock(fd, FALSE);
+
+ const char *str =
+ t_strdup_printf("VERSION\tmaster-client\t1\t0\n%s\n", cmd);
+ if (write_full(fd, str, strlen(str)) < 0)
+ i_fatal("write(%s) failed: %m", path);
+
+ input = i_stream_create_fd_autoclose(&fd, IO_BLOCK_SIZE);
+ alarm(5);
+ if ((line = i_stream_read_next_line(input)) == NULL)
+ i_fatal("read(%s) failed: %m", path);
+ if (!version_string_verify(line, "master-server", 1)) {
+ i_fatal_status(EX_PROTOCOL,
+ "%s is not a compatible master socket", path);
+ }
+ alarm(0);
+ return input;
+}
+
+static struct istream *
+master_service_send_cmd_with_args(const char *cmd, const char *const *args)
+{
+ string_t *str = t_str_new(128);
+
+ str_append(str, cmd);
+ if (args != NULL) {
+ for (unsigned int i = 0; args[i] != NULL; i++) {
+ str_append_c(str, '\t');
+ str_append_tabescaped(str, args[i]);
+ }
+ }
+ return master_service_send_cmd(str_c(str));
+}
+
+static void cmd_service_stop(struct doveadm_cmd_context *cctx)
+{
+ const char *line, *const *services;
+
+ if (!doveadm_cmd_param_array(cctx, "service", &services))
+ i_fatal("service parameter missing");
+
+ struct istream *input =
+ master_service_send_cmd_with_args("STOP", services);
+
+ alarm(5);
+ if ((line = i_stream_read_next_line(input)) == NULL) {
+ i_error("read(%s) failed: %s", i_stream_get_name(input),
+ i_stream_get_error(input));
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (line[0] == '-') {
+ doveadm_exit_code = DOVEADM_EX_NOTFOUND;
+ i_error("%s", line+1);
+ } else if (line[0] != '+') {
+ i_error("Unexpected input from %s: %s",
+ i_stream_get_name(input), line);
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ alarm(0);
+ i_stream_destroy(&input);
+}
+
+static void cmd_service_status(struct doveadm_cmd_context *cctx)
+{
+ const char *line, *const *services;
+ unsigned int fields_count;
+
+ if (!doveadm_cmd_param_array(cctx, "service", &services))
+ services = NULL;
+
+ struct istream *input = master_service_send_cmd("SERVICE-STATUS");
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_PAGER);
+ doveadm_print_header_simple("name");
+ doveadm_print_header_simple("process_count");
+ doveadm_print_header_simple("process_avail");
+ doveadm_print_header_simple("process_limit");
+ doveadm_print_header_simple("client_limit");
+ doveadm_print_header_simple("throttle_secs");
+ doveadm_print_header_simple("exit_failure_last");
+ doveadm_print_header_simple("exit_failures_in_sec");
+ doveadm_print_header_simple("last_drop_warning");
+ doveadm_print_header_simple("listen_pending");
+ doveadm_print_header_simple("listening");
+ doveadm_print_header_simple("doveadm_stop");
+ doveadm_print_header_simple("process_total");
+ fields_count = doveadm_print_get_headers_count();
+
+ alarm(5);
+ while ((line = i_stream_read_next_line(input)) != NULL) {
+ if (line[0] == '\0')
+ break;
+ T_BEGIN {
+ const char *const *args = t_strsplit_tabescaped(line);
+ if (args[0] != NULL &&
+ (services == NULL ||
+ str_array_find(services, args[0]))) {
+ unsigned int i;
+ for (i = 0; i < fields_count && args[i] != NULL; i++)
+ doveadm_print(args[i]);
+ for (; i < fields_count; i++)
+ doveadm_print("");
+ }
+ } T_END;
+ }
+ if (line == NULL) {
+ i_error("read(%s) failed: %s", i_stream_get_name(input),
+ i_stream_get_error(input));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ alarm(0);
+ i_stream_destroy(&input);
+}
+
+static void cmd_process_status(struct doveadm_cmd_context *cctx)
+{
+ const char *line, *const *services;
+
+ if (!doveadm_cmd_param_array(cctx, "service", &services))
+ services = NULL;
+
+ struct istream *input =
+ master_service_send_cmd_with_args("PROCESS-STATUS", services);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header_simple("name");
+ doveadm_print_header_simple("pid");
+ doveadm_print_header_simple("available_count");
+ doveadm_print_header_simple("total_count");
+ doveadm_print_header_simple("idle_start");
+ doveadm_print_header_simple("last_status_update");
+ doveadm_print_header_simple("last_kill_sent");
+
+ alarm(5);
+ while ((line = i_stream_read_next_line(input)) != NULL) {
+ if (line[0] == '\0')
+ break;
+ T_BEGIN {
+ const char *const *args = t_strsplit_tabescaped(line);
+ if (str_array_length(args) >= 7) {
+ for (unsigned int i = 0; i < 7; i++)
+ doveadm_print(args[i]);
+ }
+ } T_END;
+ }
+ if (line == NULL) {
+ i_error("read(%s) failed: %s", i_stream_get_name(input),
+ i_stream_get_error(input));
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ alarm(0);
+ i_stream_destroy(&input);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_stop_ver2 = {
+ .cmd = cmd_stop,
+ .name = "stop",
+ .usage = "",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_reload_ver2 = {
+ .cmd = cmd_reload,
+ .name = "reload",
+ .usage = "",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_service_stop_ver2 = {
+ .cmd = cmd_service_stop,
+ .name = "service stop",
+ .usage = "<service> [<service> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_service_status_ver2 = {
+ .cmd = cmd_service_status,
+ .name = "service status",
+ .usage = "[<service> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_process_status_ver2 = {
+ .cmd = cmd_process_status,
+ .name = "process status",
+ .usage = "[<service> [...]]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "service", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-mutf7.c b/src/doveadm/doveadm-mutf7.c
new file mode 100644
index 0000000..3f788f5
--- /dev/null
+++ b/src/doveadm/doveadm-mutf7.c
@@ -0,0 +1,56 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "imap-utf7.h"
+#include "doveadm.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+static void cmd_mailbox_mutf7(struct doveadm_cmd_context *cctx)
+{
+ string_t *str;
+ const char *const *names;
+ bool from_utf8, to_utf8;
+ unsigned int i;
+
+ if (!doveadm_cmd_param_array(cctx, "name", &names))
+ help_ver2(&doveadm_cmd_mailbox_mutf7);
+ if (!doveadm_cmd_param_bool(cctx, "from-utf8", &from_utf8)) {
+ if (!doveadm_cmd_param_bool(cctx, "to-utf8", &to_utf8))
+ from_utf8 = TRUE;
+ else
+ from_utf8 = !to_utf8;
+ }
+
+ str = t_str_new(128);
+ for (i = 0; names[i] != NULL; i++) {
+ str_truncate(str, 0);
+ if (from_utf8) {
+ if (imap_utf8_to_utf7(names[i], str) < 0) {
+ i_error("Mailbox name not valid UTF-8: %s",
+ names[i]);
+ doveadm_exit_code = EX_DATAERR;
+ }
+ } else {
+ if (imap_utf7_to_utf8(names[i], str) < 0) {
+ i_error("Mailbox name not valid mUTF-7: %s",
+ names[i]);
+ doveadm_exit_code = EX_DATAERR;
+ }
+ }
+ printf("%s\n", str_c(str));
+ }
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_mailbox_mutf7 = {
+ .name = "mailbox mutf7",
+ .cmd = cmd_mailbox_mutf7,
+ .usage = "[-7|-8] <name> [...]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('7', "to-utf8", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('8', "from-utf8", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-oldstats.c b/src/doveadm/doveadm-oldstats.c
new file mode 100644
index 0000000..4be575e
--- /dev/null
+++ b/src/doveadm/doveadm-oldstats.c
@@ -0,0 +1,604 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "net.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "hash.h"
+#include "str.h"
+#include "strescape.h"
+#include "strescape.h"
+#include "write-full.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <termios.h>
+
+struct top_line {
+ char *id;
+ /* [headers_count] */
+ const char **prev_values, **cur_values;
+
+ bool flip:1;
+};
+
+struct top_context {
+ const char *path;
+ int fd;
+ struct istream *input;
+ const char *sort_type;
+
+ char **headers;
+ unsigned int headers_count;
+
+ pool_t prev_pool, cur_pool;
+ /* id => struct top_line. */
+ HASH_TABLE(char *, struct top_line *) sessions;
+ ARRAY(struct top_line *) lines;
+ int (*lines_sort)(struct top_line *const *, struct top_line *const *);
+
+ unsigned int last_update_idx, user_idx;
+ unsigned int sort_idx1, sort_idx2;
+
+ bool flip:1;
+};
+
+static struct top_context *sort_ctx = NULL;
+static const char *disk_input_field = "disk_input";
+static const char *disk_output_field = "disk_output";
+
+static char **
+p_read_next_line(pool_t pool, struct istream *input)
+{
+ const char *line;
+
+ line = i_stream_read_next_line(input);
+ if (line == NULL)
+ return NULL;
+
+ return p_strsplit_tabescaped(pool, line);
+}
+
+static const char *const*
+read_next_line(struct istream *input)
+{
+ return (const void *)p_read_next_line(pool_datastack_create(), input);
+}
+
+static void stats_dump(const char *path, const char *cmd)
+{
+ struct istream *input;
+ const char *const *args;
+ unsigned int i;
+ int fd;
+
+ fd = doveadm_connect(path);
+ net_set_nonblock(fd, FALSE);
+ if (write_full(fd, cmd, strlen(cmd)) < 0)
+ i_fatal("write(%s) failed: %m", path);
+
+ input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+
+ /* read header */
+ args = read_next_line(input);
+ if (args == NULL)
+ i_fatal("read(%s) unexpectedly disconnected", path);
+ if (*args == NULL)
+ i_info("no statistics available");
+ else {
+ for (; *args != NULL; args++)
+ doveadm_print_header_simple(*args);
+
+ /* read lines */
+ do {
+ T_BEGIN {
+ args = read_next_line(input);
+ if (args != NULL && args[0] == NULL)
+ args = NULL;
+ if (args != NULL) {
+ for (i = 0; args[i] != NULL; i++)
+ doveadm_print(args[i]);
+ }
+ } T_END;
+ } while (args != NULL);
+ }
+ if (input->stream_errno != 0)
+ i_fatal("read(%s) failed: %s", path, i_stream_get_error(input));
+ i_stream_destroy(&input);
+}
+
+static void
+doveadm_cmd_stats_dump(struct doveadm_cmd_context* cctx)
+{
+ const char *path, *cmd;
+ const char *args[3] = {0};
+
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &path))
+ path = t_strconcat(doveadm_settings->base_dir, "/old-stats", NULL);
+
+ if (!doveadm_cmd_param_str(cctx, "type", &args[0])) {
+ i_error("Missing type parameter");
+ doveadm_exit_code = EX_USAGE;
+ return;
+ }
+
+ /* purely optional */
+ if (!doveadm_cmd_param_str(cctx, "filter", &args[1]))
+ args[1] = NULL;
+
+ cmd = t_strdup_printf("EXPORT\t%s\n", t_strarray_join(args, "\t"));
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TAB);
+ stats_dump(path, cmd);
+ return;
+}
+
+static void
+stats_line_set_prev_values(struct top_context *ctx,
+ const struct top_line *old_line,
+ struct top_line *line)
+{
+ const char **values;
+ unsigned int i;
+
+ if (old_line->prev_values == NULL ||
+ strcmp(old_line->cur_values[ctx->last_update_idx],
+ line->cur_values[ctx->last_update_idx]) != 0) {
+ values = old_line->cur_values;
+ } else {
+ values = old_line->prev_values;
+ }
+
+ /* line hasn't been updated, keep old values */
+ line->prev_values =
+ p_new(ctx->cur_pool, const char *, ctx->headers_count);
+ for (i = 0; i < ctx->headers_count; i++)
+ line->prev_values[i] = p_strdup(ctx->cur_pool, values[i]);
+}
+
+static void stats_read(struct top_context *ctx)
+{
+ struct top_line *old_line, *line;
+ unsigned int i;
+ char **args;
+
+ /* read lines */
+ while ((args = p_read_next_line(ctx->cur_pool, ctx->input)) != NULL) {
+ if (args[0] == NULL) {
+ /* end of stats */
+ return;
+ }
+ if (str_array_length((void *)args) != ctx->headers_count)
+ i_fatal("read(%s): invalid stats line", ctx->path);
+
+ line = p_new(ctx->cur_pool, struct top_line, 1);
+ line->id = args[0];
+ line->flip = ctx->flip;
+ line->cur_values = p_new(ctx->cur_pool, const char *, ctx->headers_count);
+ for (i = 0; i < ctx->headers_count; i++)
+ line->cur_values[i] = args[i];
+
+ old_line = hash_table_lookup(ctx->sessions, line->id);
+ if (old_line != NULL) {
+ stats_line_set_prev_values(ctx, old_line, line);
+ array_push_back(&ctx->lines, &line);
+ }
+ hash_table_update(ctx->sessions, line->id, line);
+ }
+
+ if (ctx->input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->path,
+ i_stream_get_error(ctx->input));
+ }
+ i_fatal("read(%s): unexpected EOF", ctx->path);
+}
+
+static void stats_drop_stale(struct top_context *ctx)
+{
+ struct hash_iterate_context *iter;
+ char *id;
+ struct top_line *line;
+
+ iter = hash_table_iterate_init(ctx->sessions);
+ while (hash_table_iterate(iter, ctx->sessions, &id, &line)) {
+ if (line->flip != ctx->flip)
+ hash_table_remove(ctx->sessions, id);
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+static int get_double(const char *str, double *num_r)
+{
+ char *p;
+
+ *num_r = strtod(str, &p);
+ return *p == '\0' ? 0 : -1;
+}
+
+static double sort_cpu_diff(const struct top_line *line)
+{
+ double prev, cur, diff, prev_time, cur_time, time_multiplier;
+
+ if (get_double(line->prev_values[sort_ctx->last_update_idx], &prev_time) < 0 ||
+ get_double(line->cur_values[sort_ctx->last_update_idx], &cur_time) < 0)
+ i_fatal("sorting: invalid last_update value");
+ time_multiplier = (cur_time - prev_time) * 100;
+
+ if (get_double(line->prev_values[sort_ctx->sort_idx1], &prev) < 0 ||
+ get_double(line->cur_values[sort_ctx->sort_idx1], &cur) < 0)
+ i_fatal("sorting: not a double");
+
+ diff = cur - prev;
+
+ if (sort_ctx->sort_idx2 != 0) {
+ if (get_double(line->prev_values[sort_ctx->sort_idx2], &prev) < 0 ||
+ get_double(line->cur_values[sort_ctx->sort_idx2], &cur) < 0)
+ i_fatal("sorting: not a double");
+ diff += cur - prev;
+ }
+ return diff * time_multiplier;
+}
+
+static int sort_cpu(struct top_line *const *l1, struct top_line *const *l2)
+{
+ double d1, d2;
+
+ d1 = sort_cpu_diff(*l1);
+ d2 = sort_cpu_diff(*l2);
+ if (d1 < d2)
+ return -1;
+ if (d1 > d2)
+ return 1;
+ return strcmp((*l1)->cur_values[sort_ctx->user_idx],
+ (*l2)->cur_values[sort_ctx->user_idx]);
+}
+
+static double sort_num_diff(const struct top_line *line)
+{
+ uint64_t prev, cur, diff;
+
+ if (str_to_uint64(line->prev_values[sort_ctx->sort_idx1], &prev) < 0 ||
+ str_to_uint64(line->cur_values[sort_ctx->sort_idx1], &cur) < 0)
+ i_fatal("sorting: not a number");
+ diff = cur - prev;
+
+ if (sort_ctx->sort_idx2 != 0) {
+ if (str_to_uint64(line->prev_values[sort_ctx->sort_idx2], &prev) < 0 ||
+ str_to_uint64(line->cur_values[sort_ctx->sort_idx2], &cur) < 0)
+ i_fatal("sorting: not a number");
+ diff += cur - prev;
+ }
+ return diff;
+}
+
+static int sort_num(struct top_line *const *l1, struct top_line *const *l2)
+{
+ uint64_t n1, n2;
+
+ n1 = sort_num_diff(*l1);
+ n2 = sort_num_diff(*l2);
+ if (n1 < n2)
+ return -1;
+ if (n1 > n2)
+ return 1;
+ return strcmp((*l1)->cur_values[sort_ctx->user_idx],
+ (*l2)->cur_values[sort_ctx->user_idx]);
+}
+
+static bool
+stats_header_find(struct top_context *ctx, const char *name,
+ unsigned int *idx_r)
+{
+ unsigned int i;
+
+ for (i = 0; ctx->headers[i] != NULL; i++) {
+ if (strcmp(ctx->headers[i], name) == 0) {
+ *idx_r = i;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static void stats_top_get_sorting(struct top_context *ctx)
+{
+ if (stats_header_find(ctx, ctx->sort_type, &ctx->sort_idx1))
+ return;
+
+ if (strcmp(ctx->sort_type, "cpu") == 0) {
+ if (!stats_header_find(ctx, "user_cpu", &ctx->sort_idx1) ||
+ !stats_header_find(ctx, "sys_cpu", &ctx->sort_idx2))
+ i_fatal("cpu sort type is missing fields");
+ return;
+ }
+ if (strcmp(ctx->sort_type, "disk") == 0) {
+ if (!stats_header_find(ctx, disk_input_field, &ctx->sort_idx1) ||
+ !stats_header_find(ctx, disk_output_field, &ctx->sort_idx2))
+ i_fatal("disk sort type is missing fields");
+ return;
+ }
+ i_fatal("unknown sort type: %s", ctx->sort_type);
+}
+
+static bool stats_top_round(struct top_context *ctx)
+{
+#define TOP_CMD "EXPORT\tsession\tconnected\n"
+ const char *const *args;
+ pool_t tmp_pool;
+
+ if (write_full(ctx->fd, TOP_CMD, strlen(TOP_CMD)) < 0)
+ i_fatal("write(%s) failed: %m", ctx->path);
+
+ /* read header */
+ if (ctx->headers != NULL) {
+ args = read_next_line(ctx->input);
+ if (args == NULL)
+ i_fatal("read(%s) unexpectedly disconnected", ctx->path);
+ if (*args == NULL)
+ return TRUE;
+ if (str_array_length(args) != ctx->headers_count)
+ i_fatal("headers changed");
+ } else {
+ ctx->headers = p_read_next_line(default_pool, ctx->input);
+ if (ctx->headers == NULL)
+ i_fatal("read(%s) unexpectedly disconnected", ctx->path);
+ if (*ctx->headers == NULL) {
+ i_free_and_null(ctx->headers);
+ return FALSE;
+ }
+ ctx->headers_count = str_array_length((void *)ctx->headers);
+ if (!stats_header_find(ctx, "last_update", &ctx->last_update_idx))
+ i_fatal("last_update header missing");
+ if (!stats_header_find(ctx, "user", &ctx->user_idx))
+ i_fatal("user header missing");
+ stats_top_get_sorting(ctx);
+ }
+
+ array_clear(&ctx->lines);
+ p_clear(ctx->prev_pool);
+ tmp_pool = ctx->prev_pool;
+ ctx->prev_pool = ctx->cur_pool;
+ ctx->cur_pool = tmp_pool;
+
+ ctx->flip = !ctx->flip;
+ stats_read(ctx);
+ stats_drop_stale(ctx);
+
+ sort_ctx = ctx;
+ array_sort(&ctx->lines, *ctx->lines_sort);
+ sort_ctx = NULL;
+ return TRUE;
+}
+
+static void
+stats_top_output_diff(struct top_context *ctx,
+ const struct top_line *line, unsigned int i)
+{
+ uint64_t prev_num, cur_num;
+ double prev_double, cur_double, prev_time, cur_time;
+ char numstr[MAX_INT_STRLEN];
+
+ if (str_to_uint64(line->prev_values[i], &prev_num) == 0 &&
+ str_to_uint64(line->cur_values[i], &cur_num) == 0) {
+ if (i_snprintf(numstr, sizeof(numstr), "%"PRIu64,
+ (cur_num - prev_num)) < 0)
+ i_unreached();
+ doveadm_print(numstr);
+ } else if (get_double(line->prev_values[i], &prev_double) == 0 &&
+ get_double(line->cur_values[i], &cur_double) == 0 &&
+ get_double(line->prev_values[ctx->last_update_idx], &prev_time) == 0 &&
+ get_double(line->cur_values[ctx->last_update_idx], &cur_time) == 0) {
+ /* %CPU */
+ if (i_snprintf(numstr, sizeof(numstr), "%d",
+ (int)((cur_double - prev_double) *
+ (cur_time - prev_time) * 100)) < 0)
+ i_unreached();
+ doveadm_print(numstr);
+ } else {
+ doveadm_print(line->cur_values[i]);
+ }
+}
+
+static void stats_top_output(struct top_context *ctx)
+{
+ static const char *names[] = {
+ "user", "service", "user_cpu", "sys_cpu",
+ "", ""
+ };
+ struct winsize ws;
+ struct top_line *const *lines;
+ unsigned int i, j, row, maxrow, count, indexes[N_ELEMENTS(names)];
+
+ names[4] = disk_input_field;
+ names[5] = disk_output_field;
+
+ /* ANSI clear screen and move cursor to top of screen */
+ printf("\x1b[2J\x1b[1;1H"); fflush(stdout);
+ doveadm_print_deinit();
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+
+ doveadm_print_header("USER", "USER", DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ doveadm_print_header_simple("SERVICE");
+ doveadm_print_header_simple("%CPU");
+ doveadm_print_header_simple("%SYS");
+ doveadm_print_header_simple("DISKIN");
+ doveadm_print_header_simple("DISKOUT");
+
+ if (!stats_top_round(ctx)) {
+ /* no connections yet */
+ return;
+ }
+
+ for (i = 0; i < N_ELEMENTS(names); i++) {
+ if (!stats_header_find(ctx, names[i], &indexes[i]))
+ indexes[i] = UINT_MAX;
+ }
+
+ if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) < 0)
+ ws.ws_row = 24;
+ maxrow = ws.ws_row-1;
+
+ lines = array_get(&ctx->lines, &count);
+ for (i = 0, row = 1; row < maxrow && i < count; i++, row++) {
+ for (j = 0; j < N_ELEMENTS(names); j++) {
+ if (indexes[j] == UINT_MAX)
+ doveadm_print("?");
+ else
+ stats_top_output_diff(ctx, lines[i], indexes[j]);
+ }
+ }
+}
+
+static void stats_top_start(struct top_context *ctx)
+{
+ struct timeout *to;
+
+ stats_top_output(ctx);
+ to = timeout_add(1000, stats_top_output, ctx);
+ io_loop_run(current_ioloop);
+ timeout_remove(&to);
+}
+
+static void stats_top(const char *path, const char *sort_type)
+{
+ struct top_context ctx;
+
+ i_zero(&ctx);
+ ctx.path = path;
+ ctx.fd = doveadm_connect(path);
+ ctx.prev_pool = pool_alloconly_create("stats top", 1024*16);
+ ctx.cur_pool = pool_alloconly_create("stats top", 1024*16);
+ i_array_init(&ctx.lines, 128);
+ hash_table_create(&ctx.sessions, default_pool, 0, str_hash, strcmp);
+ net_set_nonblock(ctx.fd, FALSE);
+
+ ctx.input = i_stream_create_fd(ctx.fd, SIZE_MAX);
+
+ if (strstr(sort_type, "cpu") != NULL)
+ ctx.lines_sort = sort_cpu;
+ else
+ ctx.lines_sort = sort_num;
+ ctx.sort_type = sort_type;
+
+ stats_top_start(&ctx);
+ i_stream_destroy(&ctx.input);
+ hash_table_destroy(&ctx.sessions);
+ array_free(&ctx.lines);
+ pool_unref(&ctx.prev_pool);
+ pool_unref(&ctx.cur_pool);
+ i_close_fd(&ctx.fd);
+}
+
+static void stats_reset(const char *path)
+{
+ const char **ptr ATTR_UNUSED;
+ int fd,ret;
+ string_t *cmd;
+ struct istream *input;
+ const char *line;
+
+ fd = doveadm_connect(path);
+ net_set_nonblock(fd, FALSE);
+ input = i_stream_create_fd(fd, SIZE_MAX);
+
+ cmd = t_str_new(10);
+ str_append(cmd, "RESET");
+/* XXX: Not supported yet.
+ for(ptr = items; *ptr; ptr++)
+ {
+ str_append_c(cmd, '\t');
+ str_append(cmd, *ptr);
+ }
+*/
+ str_append_c(cmd, '\n');
+
+ /* send command */
+ ret = write_full(fd, str_c(cmd), str_len(cmd));
+
+ if (ret < 0) {
+ i_close_fd(&fd);
+ i_error("write(%s) failed: %m", path);
+ return;
+ }
+
+ line = i_stream_read_next_line(input);
+
+ if (line == NULL) {
+ i_error("read(%s) failed: %s", path, i_stream_get_error(input));
+ } else if (!str_begins(line, "OK")) {
+ i_error("%s",line);
+ } else {
+ i_info("Stats reset");
+ }
+
+ i_stream_destroy(&input);
+ i_close_fd(&fd);
+}
+
+static void cmd_stats_top(struct doveadm_cmd_context *cctx)
+{
+ const char *path, *sort_type;
+ bool b;
+
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &path)) {
+ path = t_strconcat(doveadm_settings->base_dir,
+ "/old-stats", NULL);
+ }
+ if (!doveadm_cmd_param_bool(cctx, "show-disk-io", &b) && b) {
+ disk_input_field = "read_bytes";
+ disk_output_field = "write_bytes";
+ }
+ if (!doveadm_cmd_param_str(cctx, "sort-field", &sort_type))
+ sort_type = "disk";
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ stats_top(path, sort_type);
+}
+
+static void cmd_stats_reset(struct doveadm_cmd_context *cctx)
+{
+ const char *path;
+
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &path)) {
+ path = t_strconcat(doveadm_settings->base_dir,
+ "/old-stats", NULL);
+ }
+
+ stats_reset(path);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_oldstats_dump_ver2 = {
+ .cmd = doveadm_cmd_stats_dump,
+ .name = "oldstats dump",
+ .usage = "[-s <stats socket path>] <type> [<filter>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "type", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "filter", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_oldstats_top_ver2 = {
+ .cmd = cmd_stats_top,
+ .name = "oldstats top",
+ .usage = "[-s <stats socket path>] [-b] [<sort field>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('b', "show-disk-io", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0', "sort-field", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+
+struct doveadm_cmd_ver2 doveadm_cmd_oldstats_reset_ver2 = {
+ .cmd = cmd_stats_reset,
+ .name = "oldstats reset",
+ .usage = "[-s <stats socket path>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-penalty.c b/src/doveadm/doveadm-penalty.c
new file mode 100644
index 0000000..2f6d2d5
--- /dev/null
+++ b/src/doveadm/doveadm-penalty.c
@@ -0,0 +1,126 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "net.h"
+#include "istream.h"
+#include "hash.h"
+#include "strescape.h"
+#include "time-util.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <unistd.h>
+
+struct penalty_line {
+ struct ip_addr ip;
+ unsigned int penalty;
+ time_t last_penalty, last_update;
+};
+
+struct penalty_context {
+ const char *anvil_path;
+
+ struct ip_addr net_ip;
+ unsigned int net_bits;
+};
+
+static void penalty_parse_line(const char *line, struct penalty_line *line_r)
+{
+ const char *const *args = t_strsplit_tabescaped(line);
+ const char *ident = args[0];
+ const char *penalty_str = args[1];
+ const char *last_penalty_str = args[2];
+ const char *last_update_str = args[3];
+
+ i_zero(line_r);
+
+ (void)net_addr2ip(ident, &line_r->ip);
+ if (str_to_uint(penalty_str, &line_r->penalty) < 0 ||
+ str_to_time(last_penalty_str, &line_r->last_penalty) < 0 ||
+ str_to_time(last_update_str, &line_r->last_update) < 0)
+ i_fatal("Read invalid penalty line: %s", line);
+}
+
+static void
+penalty_print_line(struct penalty_context *ctx,
+ const struct penalty_line *line)
+{
+ if (ctx->net_bits > 0) {
+ if (!net_is_in_network(&line->ip, &ctx->net_ip, ctx->net_bits))
+ return;
+ }
+
+ doveadm_print(net_ip2addr(&line->ip));
+ doveadm_print(dec2str(line->penalty));
+ doveadm_print(unixdate2str(line->last_penalty));
+ doveadm_print(t_strflocaltime("%H:%M:%S", line->last_update));
+}
+
+static void penalty_lookup(struct penalty_context *ctx)
+{
+#define ANVIL_HANDSHAKE "VERSION\tanvil\t1\t0\n"
+#define ANVIL_CMD ANVIL_HANDSHAKE"PENALTY-DUMP\n"
+ struct istream *input;
+ const char *line;
+ int fd;
+
+ fd = doveadm_connect(ctx->anvil_path);
+ net_set_nonblock(fd, FALSE);
+ if (write(fd, ANVIL_CMD, strlen(ANVIL_CMD)) < 0)
+ i_fatal("write(%s) failed: %m", ctx->anvil_path);
+
+ input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+ while ((line = i_stream_read_next_line(input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ struct penalty_line penalty_line;
+
+ penalty_parse_line(line, &penalty_line);
+ penalty_print_line(ctx, &penalty_line);
+ } T_END;
+ }
+ if (input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->anvil_path,
+ i_stream_get_error(input));
+ }
+
+ i_stream_destroy(&input);
+}
+
+static void cmd_penalty(struct doveadm_cmd_context *cctx)
+{
+ struct penalty_context ctx;
+ const char *netmask;
+
+ i_zero(&ctx);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx.anvil_path)))
+ ctx.anvil_path = t_strconcat(doveadm_settings->base_dir, "/anvil", NULL);
+
+ if (doveadm_cmd_param_str(cctx, "netmask", &netmask)) {
+ if (net_parse_range(netmask, &ctx.net_ip, &ctx.net_bits) != 0) {
+ doveadm_exit_code = EX_USAGE;
+ i_error("Invalid netmask '%s' given", netmask);
+ return;
+ }
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header_simple("IP");
+ doveadm_print_header_simple("penalty");
+ doveadm_print_header_simple("last_penalty");
+ doveadm_print_header_simple("last_update");
+
+ penalty_lookup(&ctx);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_penalty_ver2 = {
+ .name = "penalty",
+ .cmd = cmd_penalty,
+ .usage = "[-a <anvil socket path>] [<ip/bits>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a',"socket-path", CMD_PARAM_STR,0)
+DOVEADM_CMD_PARAM('\0',"netmask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-print-flow.c b/src/doveadm/doveadm-print-flow.c
new file mode 100644
index 0000000..7f4d01e
--- /dev/null
+++ b/src/doveadm/doveadm-print-flow.c
@@ -0,0 +1,111 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ostream.h"
+#include "doveadm-print-private.h"
+
+struct doveadm_print_flow_header {
+ const char *title;
+ enum doveadm_print_header_flags flags;
+};
+
+struct doveadm_print_flow_context {
+ pool_t pool;
+ ARRAY(struct doveadm_print_flow_header) headers;
+ unsigned int header_idx;
+
+ bool streaming:1;
+};
+
+static struct doveadm_print_flow_context *ctx;
+
+static void
+doveadm_print_flow_header(const struct doveadm_print_header *hdr)
+{
+ struct doveadm_print_flow_header *fhdr;
+
+ fhdr = array_append_space(&ctx->headers);
+ fhdr->title = p_strdup(ctx->pool, hdr->title);
+ fhdr->flags = hdr->flags;
+}
+
+static void flow_next_hdr(void)
+{
+ if (++ctx->header_idx < array_count(&ctx->headers))
+ o_stream_nsend(doveadm_print_ostream, " ", 1);
+ else {
+ ctx->header_idx = 0;
+ o_stream_nsend(doveadm_print_ostream, "\n", 1);
+ }
+}
+
+static void doveadm_print_flow_print_header(const struct doveadm_print_flow_header *hdr)
+{
+ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) {
+ o_stream_nsend_str(doveadm_print_ostream, hdr->title);
+ o_stream_nsend(doveadm_print_ostream, "=", 1);
+ }
+}
+
+static void doveadm_print_flow_print(const char *value)
+{
+ const struct doveadm_print_flow_header *hdr =
+ array_idx(&ctx->headers, ctx->header_idx);
+
+ doveadm_print_flow_print_header(hdr);
+ o_stream_nsend_str(doveadm_print_ostream, value);
+ flow_next_hdr();
+}
+
+static void
+doveadm_print_flow_print_stream(const unsigned char *value, size_t size)
+{
+ const struct doveadm_print_flow_header *hdr =
+ array_idx(&ctx->headers, ctx->header_idx);
+
+ if (!ctx->streaming) {
+ ctx->streaming = TRUE;
+ doveadm_print_flow_print_header(hdr);
+ }
+ o_stream_nsend(doveadm_print_ostream, value, size);
+ if (size == 0) {
+ flow_next_hdr();
+ ctx->streaming = FALSE;
+ }
+}
+
+static void doveadm_print_flow_init(void)
+{
+ pool_t pool;
+
+ pool = pool_alloconly_create("doveadm print flow", 1024);
+ ctx = p_new(pool, struct doveadm_print_flow_context, 1);
+ ctx->pool = pool;
+ p_array_init(&ctx->headers, pool, 16);
+}
+
+static void doveadm_print_flow_flush(void)
+{
+ if (ctx->header_idx != 0) {
+ o_stream_nsend(doveadm_print_ostream, "\n", 1);
+ ctx->header_idx = 0;
+ }
+}
+
+static void doveadm_print_flow_deinit(void)
+{
+ pool_unref(&ctx->pool);
+ ctx = NULL;
+}
+
+struct doveadm_print_vfuncs doveadm_print_flow_vfuncs = {
+ "flow",
+
+ doveadm_print_flow_init,
+ doveadm_print_flow_deinit,
+ doveadm_print_flow_header,
+ doveadm_print_flow_print,
+ doveadm_print_flow_print_stream,
+ doveadm_print_flow_flush
+};
diff --git a/src/doveadm/doveadm-print-formatted.c b/src/doveadm/doveadm-print-formatted.c
new file mode 100644
index 0000000..d0518b0
--- /dev/null
+++ b/src/doveadm/doveadm-print-formatted.c
@@ -0,0 +1,92 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "ostream.h"
+#include "doveadm.h"
+#include "doveadm-server.h"
+#include "doveadm-print.h"
+#include "doveadm-print-private.h"
+#include "client-connection.h"
+#include "var-expand.h"
+
+struct doveadm_print_formatted_context {
+ pool_t pool;
+ const char *format;
+ ARRAY(struct var_expand_table) headers;
+ string_t *buf;
+ string_t *vbuf;
+ unsigned int idx;
+};
+
+static struct doveadm_print_formatted_context ctx;
+
+void doveadm_print_formatted_set_format(const char *format)
+{
+ ctx.format = format;
+}
+
+static void doveadm_print_formatted_init(void)
+{
+ i_zero(&ctx);
+ ctx.pool = pool_alloconly_create("doveadm formatted print", 1024);
+ ctx.buf = str_new(ctx.pool, 256);
+ p_array_init(&ctx.headers, ctx.pool, 8);
+ ctx.idx = 0;
+}
+
+static void
+doveadm_print_formatted_header(const struct doveadm_print_header *hdr)
+{
+ struct var_expand_table entry;
+ i_zero(&entry);
+ entry.key = '\0';
+ entry.long_key = p_strdup(ctx.pool, hdr->key);
+ entry.value = NULL;
+ array_push_back(&ctx.headers, &entry);
+}
+
+
+static void doveadm_print_formatted_flush(void)
+{
+ o_stream_nsend(doveadm_print_ostream, str_data(ctx.buf), str_len(ctx.buf));
+ str_truncate(ctx.buf, 0);
+}
+
+static void doveadm_print_formatted_print(const char *value)
+{
+ if (ctx.format == NULL) {
+ i_fatal("formatted formatter cannot be used without a format.");
+ }
+ const char *error;
+ struct var_expand_table *entry = array_idx_modifiable(&ctx.headers, ctx.idx++);
+ entry->value = value;
+
+ if (ctx.idx >= array_count(&ctx.headers)) {
+ if (var_expand(ctx.buf, ctx.format, array_front(&ctx.headers), &error) <= 0) {
+ i_error("Failed to expand print format '%s': %s",
+ ctx.format, error);
+ }
+ doveadm_print_formatted_flush();
+ ctx.idx = 0;
+ }
+
+}
+
+static void doveadm_print_formatted_deinit(void)
+{
+ pool_unref(&ctx.pool);
+}
+
+struct doveadm_print_vfuncs doveadm_print_formatted_vfuncs = {
+ "formatted",
+
+ doveadm_print_formatted_init,
+ doveadm_print_formatted_deinit,
+ doveadm_print_formatted_header,
+ doveadm_print_formatted_print,
+ NULL,
+ doveadm_print_formatted_flush
+};
+
diff --git a/src/doveadm/doveadm-print-json.c b/src/doveadm/doveadm-print-json.c
new file mode 100644
index 0000000..540ecbb
--- /dev/null
+++ b/src/doveadm/doveadm-print-json.c
@@ -0,0 +1,161 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strescape.h"
+#include "ostream.h"
+#include "json-parser.h"
+#include "doveadm.h"
+#include "doveadm-server.h"
+#include "doveadm-print.h"
+#include "doveadm-print-private.h"
+#include "client-connection.h"
+
+struct doveadm_print_json_context {
+ unsigned int header_idx, header_count;
+ bool first_row;
+ bool in_stream;
+ bool flushed;
+ ARRAY(struct doveadm_print_header) headers;
+ pool_t pool;
+ string_t *str;
+};
+
+static struct doveadm_print_json_context ctx;
+
+static void doveadm_print_json_flush_internal(void);
+
+static void doveadm_print_json_init(void)
+{
+ i_zero(&ctx);
+ ctx.pool = pool_alloconly_create("doveadm json print", 1024);
+ ctx.str = str_new(ctx.pool, 256);
+ p_array_init(&ctx.headers, ctx.pool, 1);
+ ctx.first_row = TRUE;
+ ctx.in_stream = FALSE;
+}
+
+static void
+doveadm_print_json_header(const struct doveadm_print_header *hdr)
+{
+ struct doveadm_print_header *lhdr;
+ lhdr = array_append_space(&ctx.headers);
+ lhdr->key = p_strdup(ctx.pool, hdr->key);
+ lhdr->flags = hdr->flags;
+ ctx.header_count++;
+}
+
+static void
+doveadm_print_json_value_header(const struct doveadm_print_header *hdr)
+{
+ // get header name
+ if (ctx.header_idx == 0) {
+ if (ctx.first_row == TRUE) {
+ ctx.first_row = FALSE;
+ str_append_c(ctx.str, '[');
+ } else {
+ str_append_c(ctx.str, ',');
+ }
+ str_append_c(ctx.str, '{');
+ } else {
+ str_append_c(ctx.str, ',');
+ }
+
+ str_append_c(ctx.str, '"');
+ json_append_escaped(ctx.str, hdr->key);
+ str_append_c(ctx.str, '"');
+ str_append_c(ctx.str, ':');
+}
+
+static void
+doveadm_print_json_value_footer(void) {
+ if (++ctx.header_idx == ctx.header_count) {
+ ctx.header_idx = 0;
+ str_append_c(ctx.str, '}');
+ doveadm_print_json_flush_internal();
+ }
+}
+
+static void doveadm_print_json_print(const char *value)
+{
+ const struct doveadm_print_header *hdr = array_idx(&ctx.headers, ctx.header_idx);
+
+ doveadm_print_json_value_header(hdr);
+
+ if (value == NULL) {
+ str_append(ctx.str, "null");
+ } else if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_NUMBER) != 0) {
+ i_assert(str_is_float(value, '\0'));
+ str_append(ctx.str, value);
+ } else {
+ str_append_c(ctx.str, '"');
+ json_append_escaped(ctx.str, value);
+ str_append_c(ctx.str, '"');
+ }
+
+ doveadm_print_json_value_footer();
+}
+
+static void
+doveadm_print_json_print_stream(const unsigned char *value, size_t size)
+{
+ if (!ctx.in_stream) {
+ const struct doveadm_print_header *hdr =
+ array_idx(&ctx.headers, ctx.header_idx);
+ doveadm_print_json_value_header(hdr);
+ i_assert((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_NUMBER) == 0);
+ str_append_c(ctx.str, '"');
+ ctx.in_stream = TRUE;
+ }
+
+ if (size == 0) {
+ str_append_c(ctx.str, '"');
+ doveadm_print_json_value_footer();
+ ctx.in_stream = FALSE;
+ return;
+ }
+
+ json_append_escaped_data(ctx.str, value, size);
+
+ if (str_len(ctx.str) >= IO_BLOCK_SIZE)
+ doveadm_print_json_flush_internal();
+}
+
+static void doveadm_print_json_flush_internal(void)
+{
+ o_stream_nsend(doveadm_print_ostream, str_data(ctx.str), str_len(ctx.str));
+ str_truncate(ctx.str, 0);
+}
+
+static void doveadm_print_json_flush(void)
+{
+ if (ctx.flushed)
+ return;
+ ctx.flushed = TRUE;
+
+ if (ctx.first_row == FALSE)
+ str_append_c(ctx.str,']');
+ else {
+ str_append_c(ctx.str,'[');
+ str_append_c(ctx.str,']');
+ }
+ doveadm_print_json_flush_internal();
+}
+
+static void doveadm_print_json_deinit(void)
+{
+ pool_unref(&ctx.pool);
+}
+
+struct doveadm_print_vfuncs doveadm_print_json_vfuncs = {
+ "json",
+
+ doveadm_print_json_init,
+ doveadm_print_json_deinit,
+ doveadm_print_json_header,
+ doveadm_print_json_print,
+ doveadm_print_json_print_stream,
+ doveadm_print_json_flush
+};
+
diff --git a/src/doveadm/doveadm-print-pager.c b/src/doveadm/doveadm-print-pager.c
new file mode 100644
index 0000000..3e14383
--- /dev/null
+++ b/src/doveadm/doveadm-print-pager.c
@@ -0,0 +1,113 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ostream.h"
+#include "doveadm-print-private.h"
+
+struct doveadm_print_pager_header {
+ const char *title;
+ enum doveadm_print_header_flags flags;
+};
+
+struct doveadm_print_pager_context {
+ pool_t pool;
+ ARRAY(struct doveadm_print_pager_header) headers;
+ unsigned int header_idx;
+
+ bool streaming:1;
+ bool first_page:1;
+};
+
+static struct doveadm_print_pager_context *ctx;
+
+static void
+doveadm_print_pager_header(const struct doveadm_print_header *hdr)
+{
+ struct doveadm_print_pager_header *fhdr;
+
+ fhdr = array_append_space(&ctx->headers);
+ fhdr->flags = hdr->flags;
+ fhdr->title = p_strdup(ctx->pool, hdr->title);
+}
+
+static void pager_next_hdr(void)
+{
+ if (++ctx->header_idx == array_count(&ctx->headers)) {
+ ctx->header_idx = 0;
+ }
+}
+
+static void doveadm_print_pager_print(const char *value)
+{
+ const struct doveadm_print_pager_header *hdr =
+ array_idx(&ctx->headers, ctx->header_idx);
+
+ if (ctx->header_idx == 0 && !ctx->first_page) {
+ o_stream_nsend(doveadm_print_ostream, "\f\n", 2);
+ }
+ ctx->first_page = FALSE;
+ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) {
+ o_stream_nsend_str(doveadm_print_ostream, hdr->title);
+ o_stream_nsend(doveadm_print_ostream, ": ", 2);
+ }
+ o_stream_nsend_str(doveadm_print_ostream, value);
+ o_stream_nsend(doveadm_print_ostream, "\n", 1);
+ pager_next_hdr();
+}
+
+static void
+doveadm_print_pager_print_stream(const unsigned char *value, size_t size)
+{
+ const struct doveadm_print_pager_header *hdr =
+ array_idx(&ctx->headers, ctx->header_idx);
+
+ if (!ctx->streaming) {
+ ctx->streaming = TRUE;
+ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0) {
+ o_stream_nsend_str(doveadm_print_ostream, hdr->title);
+ o_stream_nsend(doveadm_print_ostream, ":\n", 2);
+ }
+ }
+ o_stream_nsend(doveadm_print_ostream, value, size);
+ if (size == 0) {
+ pager_next_hdr();
+ ctx->streaming = FALSE;
+ }
+}
+
+static void doveadm_print_pager_init(void)
+{
+ pool_t pool;
+
+ pool = pool_alloconly_create("doveadm print pager", 1024);
+ ctx = p_new(pool, struct doveadm_print_pager_context, 1);
+ ctx->pool = pool;
+ ctx->first_page = TRUE;
+ p_array_init(&ctx->headers, pool, 16);
+}
+
+static void doveadm_print_pager_flush(void)
+{
+ if (ctx->header_idx != 0) {
+ o_stream_nsend(doveadm_print_ostream, "\n", 1);
+ ctx->header_idx = 0;
+ }
+}
+
+static void doveadm_print_pager_deinit(void)
+{
+ pool_unref(&ctx->pool);
+ ctx = NULL;
+}
+
+struct doveadm_print_vfuncs doveadm_print_pager_vfuncs = {
+ DOVEADM_PRINT_TYPE_PAGER,
+
+ doveadm_print_pager_init,
+ doveadm_print_pager_deinit,
+ doveadm_print_pager_header,
+ doveadm_print_pager_print,
+ doveadm_print_pager_print_stream,
+ doveadm_print_pager_flush
+};
diff --git a/src/doveadm/doveadm-print-private.h b/src/doveadm/doveadm-print-private.h
new file mode 100644
index 0000000..fe85560
--- /dev/null
+++ b/src/doveadm/doveadm-print-private.h
@@ -0,0 +1,31 @@
+#ifndef DOVEADM_PRINT_PRIVATE_H
+#define DOVEADM_PRINT_PRIVATE_H
+
+#include "doveadm-print.h"
+
+struct doveadm_print_header {
+ const char *key;
+ const char *title;
+ enum doveadm_print_header_flags flags;
+};
+
+struct doveadm_print_vfuncs {
+ const char *name;
+
+ void (*init)(void);
+ void (*deinit)(void);
+
+ void (*header)(const struct doveadm_print_header *hdr);
+ void (*print)(const char *value);
+ void (*print_stream)(const unsigned char *value, size_t size);
+ void (*flush)(void);
+};
+
+extern struct doveadm_print_vfuncs doveadm_print_flow_vfuncs;
+extern struct doveadm_print_vfuncs doveadm_print_tab_vfuncs;
+extern struct doveadm_print_vfuncs doveadm_print_table_vfuncs;
+extern struct doveadm_print_vfuncs doveadm_print_pager_vfuncs;
+extern struct doveadm_print_vfuncs doveadm_print_json_vfuncs;
+extern struct doveadm_print_vfuncs doveadm_print_formatted_vfuncs;
+
+#endif
diff --git a/src/doveadm/doveadm-print-server.c b/src/doveadm/doveadm-print-server.c
new file mode 100644
index 0000000..93c6ed9
--- /dev/null
+++ b/src/doveadm/doveadm-print-server.c
@@ -0,0 +1,119 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strescape.h"
+#include "ostream.h"
+#include "doveadm.h"
+#include "doveadm-print-private.h"
+#include "client-connection.h"
+
+#define DOVEADM_PRINT_FLUSH_TIMEOUT_SECS 60
+
+struct doveadm_print_server_context {
+ unsigned int header_idx, header_count;
+
+ string_t *str;
+};
+
+static struct doveadm_print_server_context ctx;
+
+static void doveadm_print_server_flush(void);
+
+static void doveadm_print_server_init(void)
+{
+ ctx.str = str_new(default_pool, 256);
+}
+
+static void doveadm_print_server_deinit(void)
+{
+ str_free(&ctx.str);
+}
+
+static void
+doveadm_print_server_header(const struct doveadm_print_header *hdr ATTR_UNUSED)
+{
+ /* no need to transfer these. the client should already know what
+ it's getting */
+ ctx.header_count++;
+}
+
+static void doveadm_print_server_print(const char *value)
+{
+ str_append_tabescaped(ctx.str, value);
+ str_append_c(ctx.str, '\t');
+
+ if (++ctx.header_idx == ctx.header_count) {
+ ctx.header_idx = 0;
+ doveadm_print_server_flush();
+ }
+}
+
+static void
+doveadm_print_server_print_stream(const unsigned char *value, size_t size)
+{
+ if (size == 0) {
+ doveadm_print_server_print("");
+ return;
+ }
+ str_append_tabescaped_n(ctx.str, value, size);
+
+ if (str_len(ctx.str) >= IO_BLOCK_SIZE)
+ doveadm_print_server_flush();
+}
+
+static int flush_callback(struct doveadm_print_server_context *ctx ATTR_UNUSED)
+{
+
+ int ret;
+ /* Keep flushing until everything is sent */
+ if ((ret = o_stream_flush(doveadm_print_ostream)) != 0)
+ io_loop_stop(current_ioloop);
+ return ret;
+}
+
+static void handle_flush_timeout(struct doveadm_print_server_context *ctx ATTR_UNUSED)
+{
+ io_loop_stop(current_ioloop);
+ o_stream_close(doveadm_print_ostream);
+ i_error("write(%s) failed: Timed out after %u seconds",
+ o_stream_get_name(doveadm_print_ostream),
+ DOVEADM_PRINT_FLUSH_TIMEOUT_SECS);
+}
+
+static void doveadm_print_server_flush(void)
+{
+ o_stream_nsend(doveadm_print_ostream,
+ str_data(ctx.str), str_len(ctx.str));
+ str_truncate(ctx.str, 0);
+ o_stream_uncork(doveadm_print_ostream);
+
+ if (o_stream_get_buffer_used_size(doveadm_print_ostream) < IO_BLOCK_SIZE ||
+ doveadm_print_ostream->stream_errno != 0)
+ return;
+ /* Wait until buffer is flushed to avoid it growing too large */
+ struct ioloop *prev_loop = current_ioloop;
+ struct ioloop *loop = io_loop_create();
+ /* Ensure we don't get stuck here forever */
+ struct timeout *to =
+ timeout_add(DOVEADM_PRINT_FLUSH_TIMEOUT_SECS*1000, handle_flush_timeout, &ctx);
+ o_stream_switch_ioloop_to(doveadm_print_ostream, loop);
+ o_stream_set_flush_callback(doveadm_print_ostream, flush_callback, &ctx);
+ io_loop_run(loop);
+ timeout_remove(&to);
+ o_stream_unset_flush_callback(doveadm_print_ostream);
+ o_stream_switch_ioloop_to(doveadm_print_ostream, prev_loop);
+ io_loop_destroy(&loop);
+}
+
+struct doveadm_print_vfuncs doveadm_print_server_vfuncs = {
+ DOVEADM_PRINT_TYPE_SERVER,
+
+ doveadm_print_server_init,
+ doveadm_print_server_deinit,
+ doveadm_print_server_header,
+ doveadm_print_server_print,
+ doveadm_print_server_print_stream,
+ doveadm_print_server_flush
+};
diff --git a/src/doveadm/doveadm-print-tab.c b/src/doveadm/doveadm-print-tab.c
new file mode 100644
index 0000000..c5871ec
--- /dev/null
+++ b/src/doveadm/doveadm-print-tab.c
@@ -0,0 +1,76 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "ostream.h"
+#include "doveadm-print-private.h"
+
+struct doveadm_print_tab_context {
+ unsigned int header_idx, header_count;
+
+ bool header_written:1;
+};
+
+static struct doveadm_print_tab_context ctx;
+
+static void doveadm_print_tab_flush_header(void)
+{
+ if (!ctx.header_written) {
+ if (!doveadm_print_hide_titles)
+ o_stream_nsend(doveadm_print_ostream, "\n", 1);
+ ctx.header_written = TRUE;
+ }
+}
+
+static void
+doveadm_print_tab_header(const struct doveadm_print_header *hdr)
+{
+ ctx.header_count++;
+ if (!doveadm_print_hide_titles) {
+ if (ctx.header_count > 1)
+ o_stream_nsend(doveadm_print_ostream, "\t", 1);
+ o_stream_nsend_str(doveadm_print_ostream, hdr->title);
+ }
+}
+
+static void doveadm_print_tab_print(const char *value)
+{
+ doveadm_print_tab_flush_header();
+ if (ctx.header_idx > 0)
+ o_stream_nsend(doveadm_print_ostream, "\t", 1);
+ o_stream_nsend_str(doveadm_print_ostream, value);
+
+ if (++ctx.header_idx == ctx.header_count) {
+ ctx.header_idx = 0;
+ o_stream_nsend(doveadm_print_ostream, "\n", 1);
+ }
+}
+
+static void
+doveadm_print_tab_print_stream(const unsigned char *value, size_t size)
+{
+ if (size == 0) {
+ doveadm_print_tab_print("");
+ return;
+ }
+ doveadm_print_tab_flush_header();
+ if (ctx.header_idx > 0)
+ o_stream_nsend(doveadm_print_ostream, "\t", 1);
+ o_stream_nsend(doveadm_print_ostream, value, size);
+}
+
+static void doveadm_print_tab_flush(void)
+{
+ doveadm_print_tab_flush_header();
+}
+
+struct doveadm_print_vfuncs doveadm_print_tab_vfuncs = {
+ "tab",
+
+ NULL,
+ NULL,
+ doveadm_print_tab_header,
+ doveadm_print_tab_print,
+ doveadm_print_tab_print_stream,
+ doveadm_print_tab_flush
+};
diff --git a/src/doveadm/doveadm-print-table.c b/src/doveadm/doveadm-print-table.c
new file mode 100644
index 0000000..25a4441
--- /dev/null
+++ b/src/doveadm/doveadm-print-table.c
@@ -0,0 +1,268 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "unichar.h"
+#include "doveadm-print-private.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/ioctl.h>
+#include <termios.h>
+
+#define DEFAULT_COLUMNS 80
+#define MIN_COLUMNS 30
+#define MAX_BUFFER_LINES 100
+
+struct doveadm_print_table_header {
+ const char *key;
+ const char *title;
+ enum doveadm_print_header_flags flags;
+ size_t min_length, max_length, length;
+};
+
+struct doveadm_print_table_context {
+ pool_t pool;
+ ARRAY(struct doveadm_print_table_header) headers;
+ ARRAY_TYPE(const_string) buffered_values;
+ string_t *stream;
+ unsigned int hdr_idx;
+ unsigned int columns;
+
+ bool lengths_set:1;
+};
+
+static struct doveadm_print_table_context *ctx;
+
+static void
+doveadm_print_table_header(const struct doveadm_print_header *hdr)
+{
+ struct doveadm_print_table_header *thdr;
+
+ thdr = array_append_space(&ctx->headers);
+ thdr->key = p_strdup(ctx->pool, hdr->key);
+ thdr->title = p_strdup(ctx->pool, hdr->title);
+ thdr->length = thdr->max_length = thdr->min_length = strlen(hdr->title);
+ thdr->flags = hdr->flags;
+}
+
+static void doveadm_calc_header_length(void)
+{
+ struct doveadm_print_table_header *headers;
+ const char *value, *const *values;
+ unsigned int i, line, hdr_count, value_count, line_count;
+ size_t len, max_length, orig_length, diff;
+
+ ctx->lengths_set = TRUE;
+
+ headers = array_get_modifiable(&ctx->headers, &hdr_count);
+ values = array_get(&ctx->buffered_values, &value_count);
+ i_assert((value_count % hdr_count) == 0);
+ line_count = value_count / hdr_count;
+
+ /* find min and max lengths of fields */
+ for (line = 0; line < line_count; line++) {
+ for (i = 0; i < hdr_count; i++) {
+ value = values[line*hdr_count + i];
+ len = value == NULL ? 0 : uni_utf8_strlen(value);
+ if (headers[i].min_length > len)
+ headers[i].min_length = len;
+ if (headers[i].max_length < len) {
+ headers[i].max_length = len;
+ headers[i].length = len;
+ }
+ }
+ }
+
+ /* +1 for space between fields */
+ max_length = 0;
+ for (i = 0; i < hdr_count; i++)
+ max_length += headers[i].max_length + 1;
+ max_length--;
+
+ while (max_length > ctx->columns) {
+ /* shrink something so we'll fit */
+ orig_length = max_length;
+ for (i = hdr_count - 1;; i--) {
+ diff = headers[i].length - headers[i].min_length;
+ if (max_length - diff <= ctx->columns) {
+ /* we can finish with this */
+ diff = max_length - ctx->columns;
+ headers[i].length -= diff;
+ max_length -= diff;
+ break;
+ }
+ if (diff > 0) {
+ /* take a bit off from it */
+ headers[i].length -= diff == 1 ? 1 : diff/2;
+ }
+
+ if (i == 0)
+ break;
+ }
+ if (max_length == orig_length) {
+ /* can't shrink it any more */
+ break;
+ }
+ }
+ if (max_length < ctx->columns) {
+ for (i = 0; i < hdr_count; i++) {
+ if ((headers[i].flags & DOVEADM_PRINT_HEADER_FLAG_EXPAND) != 0) {
+ i++;
+ break;
+ }
+ }
+ headers[i-1].length += (ctx->columns - max_length) / 2;
+ }
+}
+
+static size_t utf8_correction(const char *str)
+{
+ size_t i, len = 0;
+
+ for (i = 0; str[i] != '\0'; i++) {
+ if ((str[i] & 0xc0) == 0x80)
+ len++;
+ }
+ return len;
+}
+
+static void doveadm_print_next(const char *value)
+{
+ const struct doveadm_print_table_header *hdr;
+ int value_padded_len;
+
+ hdr = array_idx(&ctx->headers, ctx->hdr_idx);
+
+ value_padded_len = hdr->length + utf8_correction(value);
+ if ((hdr->flags & DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY) == 0)
+ printf("%-*s", value_padded_len, value);
+ else
+ printf("%*s", value_padded_len, value);
+
+ if (++ctx->hdr_idx == array_count(&ctx->headers)) {
+ ctx->hdr_idx = 0;
+ printf("\n");
+ } else {
+ printf(" ");
+ }
+}
+
+static void doveadm_print_headers(void)
+{
+ const struct doveadm_print_table_header *headers;
+ unsigned int i, count;
+
+ if (doveadm_print_hide_titles)
+ return;
+
+ headers = array_get(&ctx->headers, &count);
+ /* if all headers are hidden, don't print any of them */
+ for (i = 0; i < count; i++) {
+ if ((headers[i].flags & DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE) == 0)
+ break;
+ }
+ if (i == count)
+ return;
+ for (i = 0; i < count; i++) {
+ if (i > 0) printf(" ");
+
+ if ((headers[i].flags &
+ DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY) == 0) {
+ printf("%-*s", (int)headers[i].length,
+ headers[i].title);
+ } else {
+ printf("%*s", (int)headers[i].length,
+ headers[i].title);
+ }
+ }
+ printf("\n");
+}
+
+static void doveadm_buffer_flush(void)
+{
+ const char *value;
+
+ doveadm_calc_header_length();
+ doveadm_print_headers();
+
+ array_foreach_elem(&ctx->buffered_values, value)
+ doveadm_print_next(value);
+ array_clear(&ctx->buffered_values);
+}
+
+static void doveadm_print_table_print(const char *value)
+{
+ unsigned int line_count;
+
+ if (!ctx->lengths_set) {
+ line_count = array_count(&ctx->buffered_values) /
+ array_count(&ctx->headers);
+ if (line_count < MAX_BUFFER_LINES) {
+ value = p_strdup(ctx->pool, value);
+ array_push_back(&ctx->buffered_values, &value);
+ return;
+ }
+ doveadm_buffer_flush();
+ }
+ doveadm_print_next(value);
+}
+
+static void
+doveadm_print_table_print_stream(const unsigned char *value, size_t size)
+{
+ if (memchr(value, '\n', size) != NULL)
+ i_fatal("table formatter doesn't support multi-line values");
+
+ if (size != 0)
+ str_append_data(ctx->stream, value, size);
+ else {
+ doveadm_print_table_print(str_c(ctx->stream));
+ str_truncate(ctx->stream, 0);
+ }
+}
+
+static void doveadm_print_table_flush(void)
+{
+ if (!ctx->lengths_set && array_count(&ctx->headers) > 0)
+ doveadm_buffer_flush();
+}
+
+static void doveadm_print_table_init(void)
+{
+ pool_t pool;
+ struct winsize ws;
+
+ pool = pool_alloconly_create("doveadm print table", 2048);
+ ctx = p_new(pool, struct doveadm_print_table_context, 1);
+ ctx->pool = pool;
+ ctx->stream = str_new(default_pool, 128);
+ p_array_init(&ctx->headers, pool, 16);
+ i_array_init(&ctx->buffered_values, 64);
+ ctx->columns = DEFAULT_COLUMNS;
+
+ if (ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0) {
+ ctx->columns = ws.ws_col < MIN_COLUMNS ?
+ MIN_COLUMNS : ws.ws_col;
+ }
+}
+
+static void doveadm_print_table_deinit(void)
+{
+ str_free(&ctx->stream);
+ array_free(&ctx->buffered_values);
+ pool_unref(&ctx->pool);
+ ctx = NULL;
+}
+
+struct doveadm_print_vfuncs doveadm_print_table_vfuncs = {
+ "table",
+
+ doveadm_print_table_init,
+ doveadm_print_table_deinit,
+ doveadm_print_table_header,
+ doveadm_print_table_print,
+ doveadm_print_table_print_stream,
+ doveadm_print_table_flush
+};
diff --git a/src/doveadm/doveadm-print.c b/src/doveadm/doveadm-print.c
new file mode 100644
index 0000000..c1c5fac
--- /dev/null
+++ b/src/doveadm/doveadm-print.c
@@ -0,0 +1,211 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "ostream.h"
+#include "doveadm-print-private.h"
+
+struct doveadm_print_header_context {
+ const char *key;
+ char *sticky_value;
+ bool sticky;
+};
+
+struct doveadm_print_context {
+ pool_t pool;
+ ARRAY(struct doveadm_print_header_context) headers;
+ const struct doveadm_print_vfuncs *v;
+
+ unsigned int header_idx;
+ bool print_stream_open;
+};
+
+bool doveadm_print_hide_titles = FALSE;
+struct ostream *doveadm_print_ostream = NULL;
+
+static struct doveadm_print_context *ctx;
+
+bool doveadm_print_is_initialized(void)
+{
+ return ctx != NULL;
+}
+
+void doveadm_print_header(const char *key, const char *title,
+ enum doveadm_print_header_flags flags)
+{
+ struct doveadm_print_header hdr;
+ struct doveadm_print_header_context *hdr_ctx;
+
+ i_assert(title != NULL);
+
+ i_zero(&hdr);
+ hdr.key = key;
+ hdr.title = title;
+ hdr.flags = flags;
+ ctx->v->header(&hdr);
+
+ hdr_ctx = array_append_space(&ctx->headers);
+ hdr_ctx->key = p_strdup(ctx->pool, key);
+ hdr_ctx->sticky = (flags & DOVEADM_PRINT_HEADER_FLAG_STICKY) != 0;
+}
+
+void doveadm_print_header_simple(const char *key_title)
+{
+ doveadm_print_header(key_title, key_title, 0);
+}
+
+unsigned int doveadm_print_get_headers_count(void)
+{
+ return array_count(&ctx->headers);
+}
+
+static void doveadm_print_sticky_headers(void)
+{
+ const struct doveadm_print_header_context *headers;
+ unsigned int count;
+
+ headers = array_get(&ctx->headers, &count);
+ i_assert(count > 0);
+ for (;;) {
+ if (ctx->header_idx == count)
+ ctx->header_idx = 0;
+ else if (headers[ctx->header_idx].sticky) {
+ ctx->v->print(headers[ctx->header_idx].sticky_value);
+ ctx->header_idx++;
+ } else {
+ break;
+ }
+ }
+}
+
+void doveadm_print(const char *value)
+{
+ i_assert(!ctx->print_stream_open);
+
+ doveadm_print_sticky_headers();
+ ctx->v->print(value);
+ ctx->header_idx++;
+}
+
+void doveadm_print_num(uintmax_t value)
+{
+ T_BEGIN {
+ doveadm_print(dec2str(value));
+ } T_END;
+}
+
+void doveadm_print_stream(const void *value, size_t size)
+{
+ if (!ctx->print_stream_open) {
+ doveadm_print_sticky_headers();
+ ctx->print_stream_open = TRUE;
+ }
+ ctx->v->print_stream(value, size);
+ if (size == 0) {
+ ctx->header_idx++;
+ ctx->print_stream_open = FALSE;
+ }
+}
+
+int doveadm_print_istream(struct istream *input)
+{
+ const unsigned char *data;
+ size_t size;
+ ssize_t ret;
+
+ while ((ret = i_stream_read_more(input, &data, &size)) > 0) {
+ doveadm_print_stream(data, size);
+ i_stream_skip(input, size);
+ }
+ i_assert(ret == -1);
+ doveadm_print_stream("", 0);
+ if (input->stream_errno != 0) {
+ /* caller will log the error */
+ return -1;
+ }
+ return 0;
+}
+
+void doveadm_print_sticky(const char *key, const char *value)
+{
+ struct doveadm_print_header_context *hdr;
+
+ if (ctx == NULL) {
+ /* command doesn't really print anything */
+ return;
+ }
+
+ array_foreach_modifiable(&ctx->headers, hdr) {
+ if (strcmp(hdr->key, key) == 0) {
+ i_free(hdr->sticky_value);
+ hdr->sticky_value = i_strdup(value);
+ return;
+ }
+ }
+ i_unreached();
+}
+
+void doveadm_print_flush(void)
+{
+ if (ctx != NULL && ctx->v->flush != NULL)
+ ctx->v->flush();
+ o_stream_uncork(doveadm_print_ostream);
+ o_stream_cork(doveadm_print_ostream);
+}
+
+void doveadm_print_unstick_headers(void)
+{
+ struct doveadm_print_header_context *hdr;
+
+ if (ctx != NULL) {
+ array_foreach_modifiable(&ctx->headers, hdr)
+ hdr->sticky = FALSE;
+ }
+}
+
+void doveadm_print_init(const char *name)
+{
+ pool_t pool;
+ unsigned int i;
+
+ if (ctx != NULL) {
+ /* already forced the type */
+ return;
+ }
+
+ pool = pool_alloconly_create("doveadm print", 1024);
+ ctx = p_new(pool, struct doveadm_print_context, 1);
+ ctx->pool = pool;
+ p_array_init(&ctx->headers, pool, 16);
+
+ for (i = 0; doveadm_print_vfuncs_all[i] != NULL; i++) {
+ if (strcmp(doveadm_print_vfuncs_all[i]->name, name) == 0) {
+ ctx->v = doveadm_print_vfuncs_all[i];
+ break;
+ }
+ }
+ if (ctx->v == NULL)
+ i_fatal("Unknown print formatter: %s", name);
+ if (ctx->v->init != NULL)
+ ctx->v->init();
+}
+
+void doveadm_print_deinit(void)
+{
+ struct doveadm_print_header_context *hdr;
+
+ if (ctx == NULL)
+ return;
+
+ if (ctx->v->flush != NULL && doveadm_print_ostream != NULL) {
+ ctx->v->flush();
+ o_stream_uncork(doveadm_print_ostream);
+ }
+ if (ctx->v->deinit != NULL)
+ ctx->v->deinit();
+ array_foreach_modifiable(&ctx->headers, hdr)
+ i_free(hdr->sticky_value);
+ pool_unref(&ctx->pool);
+ ctx = NULL;
+}
diff --git a/src/doveadm/doveadm-print.h b/src/doveadm/doveadm-print.h
new file mode 100644
index 0000000..2d610e8
--- /dev/null
+++ b/src/doveadm/doveadm-print.h
@@ -0,0 +1,48 @@
+#ifndef DOVEADM_PRINT_H
+#define DOVEADM_PRINT_H
+
+#define DOVEADM_PRINT_TYPE_TAB "tab"
+#define DOVEADM_PRINT_TYPE_FLOW "flow"
+#define DOVEADM_PRINT_TYPE_PAGER "pager"
+#define DOVEADM_PRINT_TYPE_TABLE "table"
+#define DOVEADM_PRINT_TYPE_SERVER "server"
+#define DOVEADM_PRINT_TYPE_JSON "json"
+#define DOVEADM_PRINT_TYPE_FORMATTED "formatted"
+
+enum doveadm_print_header_flags {
+ DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY = 0x01,
+ DOVEADM_PRINT_HEADER_FLAG_STICKY = 0x02,
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE = 0x04,
+ DOVEADM_PRINT_HEADER_FLAG_EXPAND = 0x08,
+ DOVEADM_PRINT_HEADER_FLAG_NUMBER = 0x10
+};
+
+extern const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[];
+extern bool doveadm_print_hide_titles;
+/* points to either stdout or to doveadm-server's TCP connection */
+extern struct ostream *doveadm_print_ostream;
+
+bool doveadm_print_is_initialized(void);
+
+void doveadm_print_header(const char *key, const char *title,
+ enum doveadm_print_header_flags flags);
+void doveadm_print_header_simple(const char *key_title);
+unsigned int doveadm_print_get_headers_count(void);
+
+void doveadm_print(const char *value);
+void doveadm_print_num(uintmax_t value);
+/* Stream for same field continues until len=0 */
+void doveadm_print_stream(const void *value, size_t size);
+/* Print the whole input stream. Returns 0 if ok, -1 if stream read() failed.
+ The caller must log the error. */
+int doveadm_print_istream(struct istream *input);
+void doveadm_print_sticky(const char *key, const char *value);
+void doveadm_print_flush(void);
+void doveadm_print_unstick_headers(void);
+
+void doveadm_print_init(const char *name);
+void doveadm_print_deinit(void);
+
+void doveadm_print_formatted_set_format(const char *format);
+
+#endif
diff --git a/src/doveadm/doveadm-proxy.c b/src/doveadm/doveadm-proxy.c
new file mode 100644
index 0000000..04fde52
--- /dev/null
+++ b/src/doveadm/doveadm-proxy.c
@@ -0,0 +1,214 @@
+/* Copyright (c) 2011-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "str.h"
+#include "strescape.h"
+#include "ipc-client.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+struct proxy_context {
+ struct ipc_client *ipc;
+ const char *username_field;
+ const char *kick_hosts;
+};
+
+extern struct doveadm_cmd_ver2 doveadm_cmd_proxy[];
+
+static void proxy_cmd_help(struct doveadm_cmd_context *cctx) ATTR_NORETURN;
+
+static struct proxy_context *
+cmd_proxy_init(struct doveadm_cmd_context *cctx)
+{
+ struct proxy_context *ctx;
+ const char *socket_path;
+
+ ctx = t_new(struct proxy_context, 1);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &socket_path)) {
+ socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/ipc", NULL);
+ }
+ (void)doveadm_cmd_param_str(cctx, "passdb-field", &ctx->username_field);
+ (void)doveadm_cmd_param_str(cctx, "host", &ctx->kick_hosts);
+ ctx->ipc = ipc_client_init(socket_path);
+ return ctx;
+}
+
+static void cmd_proxy_list_header(const char *const *args)
+{
+ struct {
+ const char *key;
+ const char *title;
+ } header_map[] = {
+ { "service", "proto" },
+ { "src-ip", "src ip" },
+ { "dest-ip", "dest ip" },
+ { "dest-port", "port" },
+ };
+ for (unsigned int i = 0; args[i] != NULL; i++) {
+ const char *arg = args[i];
+
+ if (strcmp(arg, "username") == 0 ||
+ str_begins(arg, "user_")) {
+ doveadm_print_header(arg, arg,
+ DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ continue;
+ }
+ const char *title = arg;
+ for (unsigned int j = 0; j < N_ELEMENTS(header_map); j++) {
+ if (strcmp(header_map[j].key, arg) == 0) {
+ title = header_map[j].title;
+ break;
+ }
+ }
+ doveadm_print_header(arg, title, 0);
+ }
+}
+
+static void cmd_proxy_list_callback(enum ipc_client_cmd_state state,
+ const char *data, void *context)
+{
+ bool *seen_header = context;
+
+ switch (state) {
+ case IPC_CLIENT_CMD_STATE_REPLY: {
+ const char *const *args = t_strsplit_tabescaped(data);
+
+ if (!*seen_header) {
+ cmd_proxy_list_header(args);
+ *seen_header = TRUE;
+ } else {
+ for (; *args != NULL; args++)
+ doveadm_print(*args);
+ }
+ return;
+ }
+ case IPC_CLIENT_CMD_STATE_OK:
+ break;
+ case IPC_CLIENT_CMD_STATE_ERROR:
+ i_error("LIST-FULL failed: %s", data);
+ break;
+ }
+ io_loop_stop(current_ioloop);
+}
+
+static void cmd_proxy_list(struct doveadm_cmd_context *cctx)
+{
+ struct proxy_context *ctx;
+ bool seen_header = FALSE;
+
+ ctx = cmd_proxy_init(cctx);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+
+ io_loop_set_running(current_ioloop);
+ ipc_client_cmd(ctx->ipc, "proxy\t*\tLIST-FULL",
+ cmd_proxy_list_callback, &seen_header);
+ if (io_loop_is_running(current_ioloop))
+ io_loop_run(current_ioloop);
+ ipc_client_deinit(&ctx->ipc);
+}
+
+static void cmd_proxy_kick_callback(enum ipc_client_cmd_state state,
+ const char *data, void *context ATTR_UNUSED)
+{
+ switch (state) {
+ case IPC_CLIENT_CMD_STATE_REPLY:
+ return;
+ case IPC_CLIENT_CMD_STATE_OK:
+ if (data[0] == '\0')
+ data = "0";
+ doveadm_print(data);
+ break;
+ case IPC_CLIENT_CMD_STATE_ERROR:
+ i_error("KICK failed: %s", data);
+ doveadm_exit_code = EX_TEMPFAIL;
+ break;
+ }
+ io_loop_stop(current_ioloop);
+}
+
+static void cmd_proxy_kick(struct doveadm_cmd_context *cctx)
+{
+ struct proxy_context *ctx;
+ const char *const *users = NULL;
+ string_t *cmd;
+
+ ctx = cmd_proxy_init(cctx);
+ (void)doveadm_cmd_param_array(cctx, "user", &users);
+ if (users == NULL && ctx->kick_hosts == NULL) {
+ proxy_cmd_help(cctx);
+ return;
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FORMATTED);
+ doveadm_print_formatted_set_format("%{count} connections kicked\n");
+ doveadm_print_header_simple("count");
+
+ cmd = t_str_new(128);
+ str_append(cmd, "proxy\t*\t");
+ if (ctx->kick_hosts != NULL) {
+ str_append(cmd, "KICK-HOST\t");
+ str_append(cmd, ctx->kick_hosts);
+ }
+ else if (ctx->username_field == NULL)
+ str_append(cmd, "KICK");
+ else {
+ str_append(cmd, "KICK-ALT\t");
+ str_append_tabescaped(cmd, ctx->username_field);
+ }
+ if (users != NULL) {
+ for (unsigned int i = 0; users[i] != NULL; i++) {
+ str_append_c(cmd, '\t');
+ str_append_tabescaped(cmd, users[i]);
+ }
+ }
+ ipc_client_cmd(ctx->ipc, str_c(cmd), cmd_proxy_kick_callback, NULL);
+ io_loop_run(current_ioloop);
+ ipc_client_deinit(&ctx->ipc);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_proxy[] = {
+{
+ .name = "proxy list",
+ .usage = "[-a <ipc socket path>]",
+ .cmd = cmd_proxy_list,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "proxy kick",
+ .usage = "[-a <ipc socket path>] [-f <passdb field>] [-h <host> [...] | <user> [...]]",
+ .cmd = cmd_proxy_kick,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('f', "passdb-field", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('h', "host", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+}
+};
+
+static void proxy_cmd_help(struct doveadm_cmd_context *cctx)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_proxy); i++) {
+ if (doveadm_cmd_proxy[i].cmd == cctx->cmd->cmd)
+ help_ver2(&doveadm_cmd_proxy[i]);
+ }
+ i_unreached();
+}
+
+void doveadm_register_proxy_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_proxy); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_proxy[i]);
+}
diff --git a/src/doveadm/doveadm-pw.c b/src/doveadm/doveadm-pw.c
new file mode 100644
index 0000000..ed85006
--- /dev/null
+++ b/src/doveadm/doveadm-pw.c
@@ -0,0 +1,135 @@
+/* Copyright (c) 2004-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "password-scheme.h"
+#include "randgen.h"
+#include "doveadm.h"
+#include "askpass.h"
+#include "module-dir.h"
+
+#include <ctype.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+
+#define DEFAULT_SCHEME "CRYPT"
+
+static struct module *modules = NULL;
+
+static void cmd_pw(struct doveadm_cmd_context *cctx)
+{
+ const char *hash = NULL;
+ const char *scheme = NULL;
+ const char *plaintext = NULL;
+ const char *test_hash = NULL;
+ bool list_schemes = FALSE, reverse_verify = FALSE;
+ int64_t rounds_int64;
+ struct module_dir_load_settings mod_set;
+ struct password_generate_params gen_params;
+ i_zero(&gen_params);
+
+ password_schemes_init();
+
+ i_zero(&mod_set);
+ mod_set.abi_version = DOVECOT_ABI_VERSION;
+ mod_set.require_init_funcs = TRUE;
+ mod_set.ignore_dlopen_errors = TRUE;
+ mod_set.debug = doveadm_debug;
+
+ modules = module_dir_load_missing(modules, AUTH_MODULE_DIR, NULL, &mod_set);
+ module_dir_init(modules);
+
+ (void)doveadm_cmd_param_bool(cctx, "list", &list_schemes);
+ (void)doveadm_cmd_param_str(cctx, "plaintext", &plaintext);
+ if (doveadm_cmd_param_int64(cctx, "rounds", &rounds_int64)) {
+ if (rounds_int64 > UINT_MAX)
+ i_fatal("Invalid number of rounds: %"PRId64, rounds_int64);
+ gen_params.rounds = rounds_int64;
+ }
+ (void)doveadm_cmd_param_str(cctx, "scheme", &scheme);
+ if (doveadm_cmd_param_str(cctx, "test-hash", &test_hash))
+ reverse_verify = TRUE;
+ (void)doveadm_cmd_param_str(cctx, "user", &gen_params.user);
+ (void)doveadm_cmd_param_bool(cctx, "reverse-verify", &reverse_verify);
+
+ if (list_schemes) {
+ ARRAY_TYPE(password_scheme_p) arr;
+ const struct password_scheme *const *schemes;
+ unsigned int i, count;
+ t_array_init(&arr, 30);
+ password_schemes_get(&arr);
+ schemes = array_get(&arr, &count);
+ for (i = 0; i < count; i++)
+ printf("%s ", schemes[i]->name);
+ printf("\n");
+ module_dir_unload(&modules);
+ password_schemes_deinit();
+ return;
+ }
+
+ scheme = scheme == NULL ? DEFAULT_SCHEME : t_str_ucase(scheme);
+
+ if (test_hash != NULL && plaintext == NULL)
+ plaintext = t_askpass("Enter password to verify: ");
+ while (plaintext == NULL) {
+ const char *check;
+ static int lives = 3;
+
+ plaintext = t_askpass("Enter new password: ");
+ check = t_askpass("Retype new password: ");
+ if (strcmp(plaintext, check) != 0) {
+ i_error("Passwords don't match!");
+ if (--lives == 0)
+ lib_exit(1);
+ plaintext = NULL;
+ }
+ }
+
+ if (!password_generate_encoded(plaintext, &gen_params, scheme, &hash))
+ i_fatal("Unknown scheme: %s", scheme);
+ if (reverse_verify) {
+ const unsigned char *raw_password;
+ size_t size;
+ const char *error;
+
+ if (test_hash != NULL) {
+ scheme = password_get_scheme(&test_hash);
+ if (scheme == NULL)
+ i_fatal("Missing {scheme} prefix from hash");
+ hash = test_hash;
+ }
+
+ if (password_decode(hash, scheme, &raw_password, &size,
+ &error) <= 0)
+ i_fatal("reverse decode check failed: %s", error);
+
+ if (password_verify(plaintext, &gen_params, scheme,
+ raw_password, size, &error) <= 0) {
+ i_fatal("reverse password verification check failed: %s",
+ error);
+ }
+
+ printf("{%s}%s (verified)\n", scheme, hash);
+ } else {
+ printf("{%s}%s\n", scheme, hash);
+ }
+
+ module_dir_unload(&modules);
+ password_schemes_deinit();
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_pw = {
+ .name = "pw",
+ .cmd = cmd_pw,
+ .usage = "[-l] [-p plaintext] [-r rounds] [-s scheme] [-t hash] [-u user] [-V]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('l', "list", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('p', "plaintext", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('r', "rounds", CMD_PARAM_INT64, 0)
+DOVEADM_CMD_PARAM('s', "scheme", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('t', "test-hash", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('u', "user", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('V', "reverse-verify", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-replicator.c b/src/doveadm/doveadm-replicator.c
new file mode 100644
index 0000000..17af8b9
--- /dev/null
+++ b/src/doveadm/doveadm-replicator.c
@@ -0,0 +1,372 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "ioloop.h"
+#include "str.h"
+#include "strescape.h"
+#include "istream.h"
+#include "write-full.h"
+#include "master-service.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+struct replicator_context {
+ const char *socket_path;
+ const char *priority;
+ const char *user_mask, *username;
+ struct istream *input;
+ bool full_sync;
+};
+
+extern struct doveadm_cmd_ver2 doveadm_cmd_replicator[];
+
+static void replicator_cmd_help(const struct doveadm_cmd_ver2 *cmd) ATTR_NORETURN;
+
+static void
+replicator_send(struct replicator_context *ctx, const char *data)
+{
+ if (write_full(i_stream_get_fd(ctx->input), data, strlen(data)) < 0)
+ i_fatal("write(%s) failed: %m", ctx->socket_path);
+}
+
+static void replicator_connect(struct replicator_context *ctx)
+{
+#define REPLICATOR_HANDSHAKE "VERSION\treplicator-doveadm-client\t1\t0\n"
+ const char *line;
+ int fd;
+
+ fd = doveadm_connect(ctx->socket_path);
+ net_set_nonblock(fd, FALSE);
+
+ ctx->input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+ replicator_send(ctx, REPLICATOR_HANDSHAKE);
+
+ alarm(5);
+ line = i_stream_read_next_line(ctx->input);
+ alarm(0);
+ if (line == NULL) {
+ if (ctx->input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->socket_path,
+ i_stream_get_error(ctx->input));
+ } else if (ctx->input->eof)
+ i_fatal("%s disconnected", ctx->socket_path);
+ else
+ i_fatal("read(%s) timed out", ctx->socket_path);
+ }
+ if (!version_string_verify(line, "replicator-doveadm-server", 1)) {
+ i_fatal_status(EX_PROTOCOL,
+ "%s not a compatible replicator-doveadm socket",
+ ctx->socket_path);
+ }
+}
+
+static void replicator_disconnect(struct replicator_context *ctx)
+{
+ if (ctx->input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->socket_path,
+ i_stream_get_error(ctx->input));
+ }
+ i_stream_destroy(&ctx->input);
+}
+
+static struct replicator_context *
+cmd_replicator_init(struct doveadm_cmd_context *cctx)
+{
+ struct replicator_context *ctx;
+
+ ctx = t_new(struct replicator_context, 1);
+ ctx->socket_path = t_strconcat(doveadm_settings->base_dir,
+ "/replicator-doveadm", NULL);
+
+ (void)doveadm_cmd_param_str(cctx, "socket-path", &ctx->socket_path);
+ (void)doveadm_cmd_param_bool(cctx, "full-sync", &ctx->full_sync);
+ (void)doveadm_cmd_param_str(cctx, "priority", &ctx->priority);
+ (void)doveadm_cmd_param_str(cctx, "user-mask", &ctx->user_mask);
+ (void)doveadm_cmd_param_str(cctx, "user", &ctx->username);
+
+ replicator_connect(ctx);
+ return ctx;
+}
+
+static const char *time_ago(time_t t)
+{
+ int diff = ioloop_time - t;
+
+ if (t == 0)
+ return "-";
+ return t_strdup_printf("%02d:%02d:%02d", diff/3600,
+ (diff/60)%60, diff%60);
+}
+
+static void cmd_replicator_status_overview(struct replicator_context *ctx)
+{
+ char *line, *value;
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("field", "field",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ doveadm_print_header("value", "value",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+
+ replicator_send(ctx, "STATUS\n");
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ value = strchr(line, '\t');
+ if (value != NULL)
+ *value++ = '\0';
+ else
+ value = "";
+ doveadm_print(line);
+ doveadm_print(value);
+ }
+ replicator_disconnect(ctx);
+}
+
+static void cmd_replicator_status(struct doveadm_cmd_context *cctx)
+{
+ struct replicator_context *ctx;
+ const char *line, *const *args;
+ time_t last_fast, last_full, last_success;
+
+ ctx = cmd_replicator_init(cctx);
+ if (ctx->user_mask == NULL) {
+ cmd_replicator_status_overview(ctx);
+ return;
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("username", "username",
+ DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ doveadm_print_header_simple("priority");
+ doveadm_print_header_simple("fast sync");
+ doveadm_print_header_simple("full sync");
+ doveadm_print_header_simple("success sync");
+ doveadm_print_header_simple("failed");
+
+ replicator_send(ctx, t_strdup_printf("STATUS\t%s\n",
+ str_tabescape(ctx->user_mask)));
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ args = t_strsplit_tabescaped(line);
+ if (str_array_length(args) >= 5 &&
+ str_to_time(args[2], &last_fast) == 0 &&
+ str_to_time(args[3], &last_full) == 0 &&
+ str_to_time(args[5], &last_success) == 0) {
+ doveadm_print(args[0]);
+ doveadm_print(args[1]);
+ doveadm_print(time_ago(last_fast));
+ doveadm_print(time_ago(last_full));
+ doveadm_print(time_ago(last_success));
+ doveadm_print(args[4][0] == '0' ? "-" : "y");
+ }
+ } T_END;
+ }
+ if (line == NULL) {
+ i_error("Replicator disconnected unexpectedly");
+ doveadm_exit_code = EX_TEMPFAIL;
+ }
+ replicator_disconnect(ctx);
+}
+
+static void cmd_replicator_dsync_status(struct doveadm_cmd_context *cctx)
+{
+ struct replicator_context *ctx;
+ const char *line;
+ unsigned int i;
+
+ ctx = cmd_replicator_init(cctx);
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ doveadm_print_header("username", "username",
+ DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ doveadm_print_header_simple("type");
+ doveadm_print_header_simple("status");
+
+ replicator_send(ctx, "STATUS-DSYNC\n");
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ const char *const *args = t_strsplit_tabescaped(line);
+
+ for (i = 0; i < 3; i++) {
+ if (args[i] == NULL)
+ break;
+ doveadm_print(args[i]);
+ }
+ for (; i < 3; i++)
+ doveadm_print("");
+ } T_END;
+ }
+ replicator_disconnect(ctx);
+}
+
+static void cmd_replicator_replicate(struct doveadm_cmd_context *cctx)
+{
+ struct replicator_context *ctx;
+ string_t *str;
+ const char *line;
+
+ ctx = cmd_replicator_init(cctx);
+ if (ctx->user_mask == NULL)
+ replicator_cmd_help(cctx->cmd);
+
+ str = t_str_new(128);
+ str_append(str, "REPLICATE\t");
+ if (ctx->priority == NULL)
+ str_append_tabescaped(str, "low");
+ else
+ str_append_tabescaped(str, ctx->priority);
+ str_append_c(str, '\t');
+ if (ctx->full_sync)
+ str_append_c(str, 'f');
+ str_append_c(str, '\t');
+ str_append_tabescaped(str, ctx->user_mask);
+ str_append_c(str, '\n');
+ replicator_send(ctx, str_c(str));
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ doveadm_print_header("result", "result",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ i_error("Replicator disconnected unexpectedly");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (line[0] != '+') {
+ i_error("Replicator failed: %s", line+1);
+ doveadm_exit_code = EX_USAGE;
+ } else {
+ doveadm_print(t_strdup_printf("%s users updated", line+1));
+ }
+ replicator_disconnect(ctx);
+}
+
+static void cmd_replicator_add(struct doveadm_cmd_context *cctx)
+{
+ struct replicator_context *ctx;
+ string_t *str;
+ const char *line;
+
+ ctx = cmd_replicator_init(cctx);
+ if (ctx->user_mask == NULL)
+ replicator_cmd_help(cctx->cmd);
+
+ str = t_str_new(128);
+ str_append(str, "ADD\t");
+ str_append_tabescaped(str, ctx->user_mask);
+ str_append_c(str, '\n');
+ replicator_send(ctx, str_c(str));
+
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ i_error("Replicator disconnected unexpectedly");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (line[0] != '+') {
+ i_error("Replicator failed: %s", line+1);
+ doveadm_exit_code = EX_USAGE;
+ }
+ replicator_disconnect(ctx);
+}
+
+static void cmd_replicator_remove(struct doveadm_cmd_context *cctx)
+{
+ struct replicator_context *ctx;
+ string_t *str;
+ const char *line;
+
+ ctx = cmd_replicator_init(cctx);
+ if (ctx->username == NULL)
+ replicator_cmd_help(cctx->cmd);
+
+ str = t_str_new(128);
+ str_append(str, "REMOVE\t");
+ str_append_tabescaped(str, ctx->username);
+ str_append_c(str, '\n');
+ replicator_send(ctx, str_c(str));
+
+ line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ i_error("Replicator disconnected unexpectedly");
+ doveadm_exit_code = EX_TEMPFAIL;
+ } else if (line[0] != '+') {
+ i_error("Replicator failed: %s", line+1);
+ doveadm_exit_code = EX_USAGE;
+ }
+ replicator_disconnect(ctx);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_replicator[] = {
+{
+ .name = "replicator status",
+ .cmd = cmd_replicator_status,
+ .usage = "[-a <replicator socket path>] [<user mask>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "replicator dsync-status",
+ .cmd = cmd_replicator_dsync_status,
+ .usage = "[-a <replicator socket path>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "replicator replicate",
+ .cmd = cmd_replicator_replicate,
+ .usage = "[-a <replicator socket path>] [-f] [-p <priority>] <user mask>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('f', "full-sync", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('p', "priority", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "replicator add",
+ .cmd = cmd_replicator_add,
+ .usage = "[-a <replicator socket path>] <user mask>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user-mask", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+{
+ .name = "replicator remove",
+ .cmd = cmd_replicator_remove,
+ .usage = "[-a <replicator socket path>] <username>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', "user", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+},
+};
+
+static void replicator_cmd_help(const struct doveadm_cmd_ver2 *cmd)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_replicator); i++) {
+ if (doveadm_cmd_replicator[i].cmd == cmd->cmd)
+ help_ver2(&doveadm_cmd_replicator[i]);
+ }
+ i_unreached();
+}
+
+void doveadm_register_replicator_commands(void)
+{
+ unsigned int i;
+
+ for (i = 0; i < N_ELEMENTS(doveadm_cmd_replicator); i++)
+ doveadm_cmd_register_ver2(&doveadm_cmd_replicator[i]);
+}
diff --git a/src/doveadm/doveadm-server.h b/src/doveadm/doveadm-server.h
new file mode 100644
index 0000000..e111b4a
--- /dev/null
+++ b/src/doveadm/doveadm-server.h
@@ -0,0 +1,34 @@
+#ifndef DOVEADM_SERVER_H
+#define DOVEADM_SERVER_H
+
+extern struct client_connection *doveadm_client;
+extern struct doveadm_print_vfuncs doveadm_print_server_vfuncs;
+
+enum doveadm_proxy_ssl_flags {
+ /* Use SSL/TLS enabled */
+ PROXY_SSL_FLAG_YES = 0x01,
+ /* Don't do SSL handshake immediately after connected */
+ PROXY_SSL_FLAG_STARTTLS = 0x02,
+ /* Don't require that the received certificate is valid */
+ PROXY_SSL_FLAG_ANY_CERT = 0x04
+};
+
+struct doveadm_server {
+ /* hostname:port or socket name for logging */
+ const char *name;
+ /* hostname without port */
+ const char *hostname;
+ /* host ip to use */
+ struct ip_addr ip;
+ /* port to use */
+ in_port_t port;
+
+ /* ssl related settings */
+ enum doveadm_proxy_ssl_flags ssl_flags;
+ struct ssl_iostream_context *ssl_ctx;
+
+ ARRAY(struct server_connection *) connections;
+ ARRAY_TYPE(string) queue;
+};
+
+#endif
diff --git a/src/doveadm/doveadm-settings.c b/src/doveadm/doveadm-settings.c
new file mode 100644
index 0000000..54d6631
--- /dev/null
+++ b/src/doveadm/doveadm-settings.c
@@ -0,0 +1,319 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "var-expand.h"
+#include "settings-parser.h"
+#include "service-settings.h"
+#include "mail-storage-settings.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "master-service-ssl-settings.h"
+#include "iostream-ssl.h"
+#include "doveadm-settings.h"
+
+ARRAY_TYPE(doveadm_setting_root) doveadm_setting_roots;
+bool doveadm_verbose_proctitle;
+
+static pool_t doveadm_settings_pool = NULL;
+
+static bool doveadm_settings_check(void *_set, pool_t pool, const char **error_r);
+
+/* <settings checks> */
+static struct file_listener_settings doveadm_unix_listeners_array[] = {
+ { "doveadm-server", 0600, "", "" }
+};
+static struct file_listener_settings *doveadm_unix_listeners[] = {
+ &doveadm_unix_listeners_array[0]
+};
+static buffer_t doveadm_unix_listeners_buf = {
+ { { doveadm_unix_listeners, sizeof(doveadm_unix_listeners) } }
+};
+/* </settings checks> */
+
+struct service_settings doveadm_service_settings = {
+ .name = "doveadm",
+ .protocol = "",
+ .type = "",
+ .executable = "doveadm-server",
+ .user = "",
+ .group = "",
+ .privileged_group = "",
+ .extra_groups = "$default_internal_group",
+ .chroot = "",
+
+ .drop_priv_before_exec = FALSE,
+
+ .process_min_avail = 0,
+ .process_limit = 0,
+ .client_limit = 1,
+ .service_count = 1,
+ .idle_kill = 0,
+ .vsz_limit = UOFF_T_MAX,
+
+ .unix_listeners = { { &doveadm_unix_listeners_buf,
+ sizeof(doveadm_unix_listeners[0]) } },
+ .fifo_listeners = ARRAY_INIT,
+ .inet_listeners = ARRAY_INIT
+};
+
+#undef DEF
+#define DEF(type, name) \
+ SETTING_DEFINE_STRUCT_##type(#name, name, struct doveadm_settings)
+
+static const struct setting_define doveadm_setting_defines[] = {
+ DEF(STR, base_dir),
+ DEF(STR, libexec_dir),
+ DEF(STR, mail_plugins),
+ DEF(STR, mail_plugin_dir),
+ DEF(STR_VARS, mail_temp_dir),
+ DEF(BOOL, auth_debug),
+ DEF(STR, auth_socket_path),
+ DEF(STR, doveadm_socket_path),
+ DEF(UINT, doveadm_worker_count),
+ DEF(IN_PORT, doveadm_port),
+ { .type = SET_ALIAS, .key = "doveadm_proxy_port" },
+ DEF(ENUM, doveadm_ssl),
+ DEF(STR, doveadm_username),
+ DEF(STR, doveadm_password),
+ DEF(STR, doveadm_allowed_commands),
+ DEF(STR, dsync_alt_char),
+ DEF(STR, dsync_remote_cmd),
+ DEF(STR, director_username_hash),
+ DEF(STR, doveadm_api_key),
+ DEF(STR, dsync_features),
+ DEF(UINT, dsync_commit_msgs_interval),
+ DEF(STR, doveadm_http_rawlog_dir),
+ DEF(STR, dsync_hashed_headers),
+
+ { .type = SET_STRLIST, .key = "plugin",
+ .offset = offsetof(struct doveadm_settings, plugin_envs) },
+
+ SETTING_DEFINE_LIST_END
+};
+
+const struct doveadm_settings doveadm_default_settings = {
+ .base_dir = PKG_RUNDIR,
+ .libexec_dir = PKG_LIBEXECDIR,
+ .mail_plugins = "",
+ .mail_plugin_dir = MODULEDIR,
+ .mail_temp_dir = "/tmp",
+ .auth_debug = FALSE,
+ .auth_socket_path = "auth-userdb",
+ .doveadm_socket_path = "doveadm-server",
+ .doveadm_worker_count = 0,
+ .doveadm_port = 0,
+ .doveadm_ssl = "no:ssl:starttls",
+ .doveadm_username = "doveadm",
+ .doveadm_password = "",
+ .doveadm_allowed_commands = "",
+ .dsync_alt_char = "_",
+ .dsync_remote_cmd = "ssh -l%{login} %{host} doveadm dsync-server -u%u -U",
+ .dsync_features = "",
+ .dsync_hashed_headers = "Date Message-ID",
+ .dsync_commit_msgs_interval = 100,
+ .director_username_hash = "%Lu",
+ .doveadm_api_key = "",
+ .doveadm_http_rawlog_dir = "",
+
+ .plugin_envs = ARRAY_INIT
+};
+
+static const struct setting_parser_info *doveadm_setting_dependencies[] = {
+ &mail_user_setting_parser_info,
+ NULL
+};
+
+const struct setting_parser_info doveadm_setting_parser_info = {
+ .module_name = "doveadm",
+ .defines = doveadm_setting_defines,
+ .defaults = &doveadm_default_settings,
+
+ .type_offset = SIZE_MAX,
+ .struct_size = sizeof(struct doveadm_settings),
+
+ .parent_offset = SIZE_MAX,
+ .check_func = doveadm_settings_check,
+ .dependencies = doveadm_setting_dependencies
+};
+
+struct doveadm_settings *doveadm_settings;
+const struct master_service_settings *service_set;
+
+static void
+fix_base_path(struct doveadm_settings *set, pool_t pool, const char **str)
+{
+ if (*str != NULL && **str != '\0' && **str != '/')
+ *str = p_strconcat(pool, set->base_dir, "/", *str, NULL);
+}
+
+/* <settings checks> */
+struct dsync_feature_list {
+ const char *name;
+ enum dsync_features num;
+};
+
+static const struct dsync_feature_list dsync_feature_list[] = {
+ { "empty-header-workaround", DSYNC_FEATURE_EMPTY_HDR_WORKAROUND },
+ { NULL, 0 }
+};
+
+static int
+dsync_settings_parse_features(struct doveadm_settings *set,
+ const char **error_r)
+{
+ enum dsync_features features = 0;
+ const struct dsync_feature_list *list;
+ const char *const *str;
+
+ str = t_strsplit_spaces(set->dsync_features, " ,");
+ for (; *str != NULL; str++) {
+ list = dsync_feature_list;
+ for (; list->name != NULL; list++) {
+ if (strcasecmp(*str, list->name) == 0) {
+ features |= list->num;
+ break;
+ }
+ }
+ if (list->name == NULL) {
+ *error_r = t_strdup_printf("dsync_features: "
+ "Unknown feature: %s", *str);
+ return -1;
+ }
+ }
+ set->parsed_features = features;
+ return 0;
+}
+
+static bool doveadm_settings_check(void *_set, pool_t pool ATTR_UNUSED,
+ const char **error_r)
+{
+ struct doveadm_settings *set = _set;
+
+#ifndef CONFIG_BINARY
+ fix_base_path(set, pool, &set->auth_socket_path);
+ fix_base_path(set, pool, &set->doveadm_socket_path);
+#endif
+ if (*set->dsync_hashed_headers == '\0') {
+ *error_r = "dsync_hashed_headers must not be empty";
+ return FALSE;
+ }
+ if (*set->dsync_alt_char == '\0') {
+ *error_r = "dsync_alt_char must not be empty";
+ return FALSE;
+ }
+ if (dsync_settings_parse_features(set, error_r) != 0)
+ return FALSE;
+ return TRUE;
+}
+/* </settings checks> */
+
+const struct master_service_ssl_settings *doveadm_ssl_set = NULL;
+
+void doveadm_get_ssl_settings(struct ssl_iostream_settings *set_r, pool_t pool)
+{
+ master_service_ssl_client_settings_to_iostream_set(doveadm_ssl_set,
+ pool, set_r);
+}
+
+void doveadm_settings_expand(struct doveadm_settings *set, pool_t pool)
+{
+ struct var_expand_table tab[] = { { '\0', NULL, NULL } };
+ const char *error;
+
+ if (settings_var_expand(&doveadm_setting_parser_info, set,
+ pool, tab, &error) <= 0)
+ i_fatal("Failed to expand settings: %s", error);
+}
+
+void doveadm_read_settings(void)
+{
+ static const struct setting_parser_info *default_set_roots[] = {
+ &master_service_ssl_setting_parser_info,
+ &doveadm_setting_parser_info,
+ };
+ struct master_service_settings_input input;
+ struct master_service_settings_output output;
+ const struct doveadm_settings *set;
+ struct doveadm_setting_root *root;
+ ARRAY(const struct setting_parser_info *) set_roots;
+ ARRAY_TYPE(const_string) module_names;
+ void **sets;
+ const char *error;
+
+ t_array_init(&set_roots, N_ELEMENTS(default_set_roots) +
+ array_count(&doveadm_setting_roots) + 1);
+ array_append(&set_roots, default_set_roots,
+ N_ELEMENTS(default_set_roots));
+ t_array_init(&module_names, 4);
+ array_foreach_modifiable(&doveadm_setting_roots, root) {
+ array_push_back(&module_names, &root->info->module_name);
+ array_push_back(&set_roots, &root->info);
+ }
+ array_append_zero(&module_names);
+ array_append_zero(&set_roots);
+
+ i_zero(&input);
+ input.roots = array_front(&set_roots);
+ input.module = "doveadm";
+ input.extra_modules = array_front(&module_names);
+ input.service = "doveadm";
+ input.preserve_user = TRUE;
+ input.preserve_home = TRUE;
+ if (master_service_settings_read(master_service, &input,
+ &output, &error) < 0)
+ i_fatal("Error reading configuration: %s", error);
+
+ doveadm_settings_pool = pool_alloconly_create("doveadm settings", 1024);
+ service_set = master_service_settings_get(master_service);
+ service_set = settings_dup(&master_service_setting_parser_info,
+ service_set, doveadm_settings_pool);
+ doveadm_verbose_proctitle = service_set->verbose_proctitle;
+
+ sets = master_service_settings_get_others(master_service);
+ set = sets[1];
+ doveadm_settings = settings_dup(&doveadm_setting_parser_info, set,
+ doveadm_settings_pool);
+ doveadm_ssl_set = settings_dup(&master_service_ssl_setting_parser_info,
+ master_service_ssl_settings_get(master_service),
+ doveadm_settings_pool);
+ doveadm_settings_expand(doveadm_settings, doveadm_settings_pool);
+ doveadm_settings->parsed_features = set->parsed_features; /* copy this value by hand */
+
+ array_foreach_modifiable(&doveadm_setting_roots, root) {
+ unsigned int idx =
+ array_foreach_idx(&doveadm_setting_roots, root);
+ root->settings = settings_dup(root->info, sets[2+idx],
+ doveadm_settings_pool);
+ }
+}
+
+void doveadm_setting_roots_add(const struct setting_parser_info *info)
+{
+ struct doveadm_setting_root *root;
+
+ root = array_append_space(&doveadm_setting_roots);
+ root->info = info;
+}
+
+void *doveadm_setting_roots_get_settings(const struct setting_parser_info *info)
+{
+ const struct doveadm_setting_root *root;
+
+ array_foreach(&doveadm_setting_roots, root) {
+ if (root->info == info)
+ return root->settings;
+ }
+ i_panic("Failed to find settings for module %s", info->module_name);
+}
+
+void doveadm_settings_init(void)
+{
+ i_array_init(&doveadm_setting_roots, 8);
+}
+
+void doveadm_settings_deinit(void)
+{
+ array_free(&doveadm_setting_roots);
+ pool_unref(&doveadm_settings_pool);
+}
diff --git a/src/doveadm/doveadm-settings.h b/src/doveadm/doveadm-settings.h
new file mode 100644
index 0000000..b3c2184
--- /dev/null
+++ b/src/doveadm/doveadm-settings.h
@@ -0,0 +1,65 @@
+#ifndef DOVEADM_SETTINGS_H
+#define DOVEADM_SETTINGS_H
+
+#include "net.h"
+
+struct ssl_iostream_settings;
+
+/* <settings checks> */
+enum dsync_features {
+ DSYNC_FEATURE_EMPTY_HDR_WORKAROUND = 0x1,
+};
+/* </settings checks> */
+
+struct doveadm_settings {
+ const char *base_dir;
+ const char *libexec_dir;
+ const char *mail_plugins;
+ const char *mail_plugin_dir;
+ const char *mail_temp_dir;
+ bool auth_debug;
+ const char *auth_socket_path;
+ const char *doveadm_socket_path;
+ unsigned int doveadm_worker_count;
+ in_port_t doveadm_port;
+ const char *doveadm_ssl;
+ const char *doveadm_username;
+ const char *doveadm_password;
+ const char *doveadm_allowed_commands;
+ const char *dsync_alt_char;
+ const char *dsync_remote_cmd;
+ const char *director_username_hash;
+ const char *doveadm_api_key;
+ const char *dsync_features;
+ const char *dsync_hashed_headers;
+ unsigned int dsync_commit_msgs_interval;
+ const char *doveadm_http_rawlog_dir;
+ enum dsync_features parsed_features;
+ ARRAY(const char *) plugin_envs;
+};
+
+struct doveadm_setting_root {
+ const struct setting_parser_info *info;
+ void *settings;
+};
+ARRAY_DEFINE_TYPE(doveadm_setting_root, struct doveadm_setting_root);
+
+extern const struct setting_parser_info doveadm_setting_parser_info;
+extern struct doveadm_settings *doveadm_settings;
+extern const struct master_service_settings *service_set;
+extern const struct master_service_ssl_settings *doveadm_ssl_set;
+extern ARRAY_TYPE(doveadm_setting_root) doveadm_setting_roots;
+extern bool doveadm_verbose_proctitle;
+
+void doveadm_get_ssl_settings(struct ssl_iostream_settings *set_r, pool_t pool);
+void doveadm_settings_expand(struct doveadm_settings *set, pool_t pool);
+
+void doveadm_setting_roots_add(const struct setting_parser_info *info);
+void *doveadm_setting_roots_get_settings(const struct setting_parser_info *info);
+
+void doveadm_read_settings(void);
+
+void doveadm_settings_init(void);
+void doveadm_settings_deinit(void);
+
+#endif
diff --git a/src/doveadm/doveadm-sis.c b/src/doveadm/doveadm-sis.c
new file mode 100644
index 0000000..85e82e9
--- /dev/null
+++ b/src/doveadm/doveadm-sis.c
@@ -0,0 +1,330 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "hostpid.h"
+#include "randgen.h"
+#include "read-full.h"
+#include "fs-sis-common.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <dirent.h>
+#include <sys/stat.h>
+
+/* Files are in <rootdir>/ha/sh/<hash>-<guid>
+ They may be hard linked to hashes/<hash>
+*/
+
+static const char *sis_get_dir(const char *rootdir, const char *hash)
+{
+ if (strlen(hash) < 4 || strchr(hash, '/') != NULL)
+ i_fatal("Invalid hash in filename: %s", hash);
+ return t_strdup_printf("%s/%c%c/%c%c", rootdir,
+ hash[0], hash[1], hash[2], hash[3]);
+}
+
+static int
+file_contents_equal(const char *path1, const char *path2, ino_t *path2_inode_r)
+{
+ struct stat st1, st2;
+ int fd1, fd2, ret = -1;
+
+ *path2_inode_r = 0;
+
+ /* do a byte-by-byte comparison for the files to find out if they're
+ the same or if this is a hash collision */
+ fd1 = open(path1, O_RDONLY);
+ if (fd1 == -1) {
+ if (errno != ENOENT)
+ i_error("open(%s) failed: %m", path1);
+ return -1;
+ }
+ fd2 = open(path2, O_RDONLY);
+ if (fd2 == -1) {
+ if (errno != ENOENT)
+ i_error("open(%s) failed: %m", path2);
+ i_close_fd(&fd1);
+ return -1;
+ }
+
+ if (fstat(fd1, &st1) < 0)
+ i_error("fstat(%s) failed: %m", path1);
+ else if (fstat(fd2, &st2) < 0)
+ i_error("fstat(%s) failed: %m", path1);
+ else if (st1.st_size != st2.st_size)
+ ret = 0;
+ else {
+ /* @UNSAFE: sizes match. compare. */
+ unsigned char buf1[IO_BLOCK_SIZE], buf2[IO_BLOCK_SIZE];
+ ssize_t ret1;
+ int ret2;
+
+ while ((ret1 = read(fd1, buf1, sizeof(buf1))) > 0) {
+ i_assert((size_t)ret1 <= sizeof(buf2));
+ if ((ret2 = read_full(fd2, buf2, ret1)) <= 0) {
+ if (ret2 < 0)
+ i_error("read(%s) failed: %m", path2);
+ else
+ ret = 0;
+ break;
+ }
+ if (memcmp(buf1, buf2, ret1) != 0) {
+ ret = 0;
+ break;
+ }
+ }
+ if (ret1 < 0)
+ i_error("read(%s) failed: %m", path1);
+ else if (ret1 == 0)
+ ret = 1;
+ *path2_inode_r = st2.st_ino;
+ }
+
+ if (close(fd1) < 0)
+ i_error("close(%s) failed: %m", path1);
+ if (close(fd2) < 0)
+ i_error("close(%s) failed: %m", path2);
+
+ return ret;
+}
+
+static int
+hardlink_replace(const char *src, const char *dest, ino_t src_inode)
+{
+ const char *p, *destdir, *tmppath;
+ unsigned char randbuf[8];
+ struct stat st;
+
+ p = strrchr(dest, '/');
+ i_assert(p != NULL);
+ destdir = t_strdup_until(dest, p);
+
+ random_fill(randbuf, sizeof(randbuf));
+ tmppath = t_strdup_printf("%s/temp.%s.%s.%s",
+ destdir, my_hostname, my_pid,
+ binary_to_hex(randbuf, sizeof(randbuf)));
+ if (link(src, tmppath) < 0) {
+ if (errno == EMLINK)
+ return 0;
+ i_error("link(%s, %s) failed: %m", src, tmppath);
+ return -1;
+ }
+ if (stat(tmppath, &st) < 0) {
+ i_error("stat(%s) failed: %m", tmppath);
+ return -1;
+ }
+ if (st.st_ino != src_inode) {
+ i_unlink(tmppath);
+ return 0;
+ }
+ if (rename(tmppath, dest) < 0) {
+ i_error("rename(%s, %s) failed: %m", src, tmppath);
+ i_unlink(tmppath);
+ return -1;
+ }
+ return 1;
+}
+
+static int sis_try_deduplicate(const char *rootdir, const char *fname)
+{
+ const char *p, *hash, *hashdir, *path, *hashes_dir, *hashes_path;
+ struct stat st;
+ ino_t inode;
+ int ret;
+
+ /* fname should be in <hash>-<guid> format */
+ p = strchr(fname, '-');
+ i_assert(p != NULL);
+
+ hash = t_strdup_until(fname, p);
+ hashdir = sis_get_dir(rootdir, hash);
+ path = t_strdup_printf("%s/%s", hashdir, fname);
+
+ hashes_dir = t_strconcat(hashdir, "/", HASH_DIR_NAME, NULL);
+ hashes_path = t_strconcat(hashes_dir, "/", hash, NULL);
+ if (link(path, hashes_path) == 0) {
+ /* first file with this hash. we're done */
+ return 0;
+ }
+ if (errno == ENOENT) {
+ /* either path was already deleted or hashes dir
+ doesn't exist */
+ if (mkdir(hashes_dir, 0700) < 0) {
+ if (errno == EEXIST)
+ return 0;
+ i_error("mkdir(%s) failed: %m", hashes_dir);
+ return -1;
+ }
+ /* try again */
+ if (link(path, hashes_path) == 0 || errno == ENOENT)
+ return 0;
+ }
+ if (errno != EEXIST) {
+ i_error("link(%s, %s) failed: %m", path, hashes_path);
+ return -1;
+ }
+
+ /* need to do a byte-by-byte comparison. but check first if someone
+ else already had deduplicated the file. */
+ if (stat(path, &st) < 0) {
+ if (errno == ENOENT) {
+ /* just got deleted */
+ return 0;
+ }
+ i_error("stat(%s) failed: %m", path);
+ return -1;
+ }
+ if (st.st_nlink > 1) {
+ /* already deduplicated */
+ return 0;
+ }
+
+ ret = file_contents_equal(path, hashes_path, &inode);
+ if (ret < 0) {
+ if (errno == ENOENT) {
+ /* either path or hashes_path was deleted. */
+ return sis_try_deduplicate(rootdir, fname);
+ }
+ return -1;
+ }
+ if (ret > 0) {
+ /* equal, replace with hard link */
+ ret = hardlink_replace(hashes_path, path, inode);
+ if (ret > 0)
+ return 0;
+ else if (ret < 0)
+ return -1;
+ /* too many hard links or inode changed */
+ }
+
+ /* replace hashes link with this */
+ return hardlink_replace(path, hashes_path, st.st_ino) < 0 ? -1 : 0;
+}
+
+static void cmd_sis_deduplicate(struct doveadm_cmd_context *cctx)
+{
+ const char *rootdir, *queuedir;
+ DIR *dir;
+ struct dirent *d;
+ struct stat st, first_st;
+ string_t *path;
+ size_t dir_len;
+ int ret;
+
+ if (!doveadm_cmd_param_str(cctx, "root-dir", &rootdir) ||
+ !doveadm_cmd_param_str(cctx, "queue-dir", &queuedir))
+ help_ver2(&doveadm_cmd_sis_deduplicate);
+
+ /* go through the filenames in the queue dir and see if
+ we can deduplicate them. */
+ if (stat(rootdir, &st) < 0)
+ i_fatal("stat(%s) failed: %m", rootdir);
+
+ path = t_str_new(256);
+ str_append(path, queuedir);
+ str_append_c(path, '/');
+ dir_len = str_len(path);
+
+ dir = opendir(queuedir);
+ if (dir == NULL)
+ i_fatal("opendir(%s) failed: %m", queuedir);
+
+ first_st.st_size = -1;
+ while ((d = readdir(dir)) != NULL) {
+ if (d->d_name[0] == '.')
+ continue;
+
+ str_truncate(path, dir_len);
+ str_append(path, d->d_name);
+
+ if (first_st.st_size < 0) {
+ if (stat(str_c(path), &first_st) < 0)
+ i_fatal("stat(%s) failed: %m", str_c(path));
+ }
+ if (strchr(d->d_name, '-') == NULL || first_st.st_size != 0) {
+ i_fatal("%s is not a valid sis-queue file, "
+ "is the queue directory correct?",
+ str_c(path));
+ }
+
+ T_BEGIN {
+ ret = sis_try_deduplicate(rootdir, d->d_name);
+ } T_END;
+ if (ret == 0)
+ i_unlink(str_c(path));
+ }
+ if (closedir(dir) < 0)
+ i_error("closedir(%s) failed: %m", queuedir);
+}
+
+static void cmd_sis_find(struct doveadm_cmd_context *cctx)
+{
+ const char *rootdir, *path, *hash;
+ DIR *dir;
+ struct dirent *d;
+ struct stat st;
+ string_t *str;
+ size_t dir_len, hash_len;
+
+ if (!doveadm_cmd_param_str(cctx, "root-dir", &rootdir) ||
+ !doveadm_cmd_param_str(cctx, "hash", &hash) ||
+ strlen(hash) < 4)
+ help_ver2(&doveadm_cmd_sis_find);
+
+ if (stat(rootdir, &st) < 0) {
+ if (errno == ENOENT)
+ i_fatal("Attachment dir doesn't exist: %s", rootdir);
+ i_fatal("stat(%s) failed: %m", rootdir);
+ }
+ hash_len = strlen(hash);
+
+ path = sis_get_dir(rootdir, hash);
+ str = t_str_new(256);
+ str_append(str, path);
+ str_append_c(str, '/');
+ dir_len = str_len(str);
+
+ dir = opendir(path);
+ if (dir == NULL) {
+ if (errno == ENOENT)
+ return;
+ i_fatal("opendir(%s) failed: %m", path);
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_FLOW);
+ doveadm_print_header("path", "path",
+ DOVEADM_PRINT_HEADER_FLAG_HIDE_TITLE);
+ while ((d = readdir(dir)) != NULL) {
+ if (strncmp(d->d_name, hash, hash_len) == 0) {
+ str_truncate(str, dir_len);
+ str_append(str, d->d_name);
+ doveadm_print(str_c(str));
+ }
+ }
+ if (closedir(dir) < 0)
+ i_error("closedir(%s) failed: %m", path);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_sis_deduplicate = {
+ .name = "sis deduplicate",
+ .cmd = cmd_sis_deduplicate,
+ .usage = "<root dir> <queue dir>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "root-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "queue-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+struct doveadm_cmd_ver2 doveadm_cmd_sis_find = {
+ .name = "sis find",
+ .cmd = cmd_sis_find,
+ .usage = "<root dir> <hash>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "root-dir", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "hash", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-stats.c b/src/doveadm/doveadm-stats.c
new file mode 100644
index 0000000..ddb198b
--- /dev/null
+++ b/src/doveadm/doveadm-stats.c
@@ -0,0 +1,343 @@
+/* Copyright (c) 2017-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "istream.h"
+#include "str.h"
+#include "strescape.h"
+#include "write-full.h"
+#include "master-service.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+#include "stats-settings.h"
+
+#include <math.h>
+
+#define DOVEADM_DUMP_DEFAULT_FIELDS \
+ "count sum min max avg median stddev %95"
+
+#define ADD_NAME_PARAM "name"
+#define ADD_DESCR_PARAM "description"
+#define ADD_FIELDS_PARAM "fields"
+#define ADD_GROUPBY_PARAM "group-by"
+#define ADD_FILTER_PARAM "filter"
+#define ADD_EXPORTER_PARAM "exporter"
+#define ADD_EXPORTERINCL_PARAM "exporter-include"
+
+enum doveadm_dump_field_type {
+ DOVEADM_DUMP_FIELD_TYPE_PASSTHROUGH = 0,
+ DOVEADM_DUMP_FIELD_TYPE_STDDEV,
+};
+
+struct stats_cmd_context {
+ string_t *cmd;
+ struct doveadm_cmd_context *cctx;
+ struct istream *input;
+ const char *path;
+ void *data;
+};
+
+struct dump_data {
+ const char **fields;
+ unsigned int field_count;
+ enum doveadm_dump_field_type *field_types;
+};
+
+struct stats_cmd_vfuncs {
+ int (*build_cmd)(struct stats_cmd_context *ctx, const char **error_r);
+ void (*process_response)(struct stats_cmd_context *ctx);
+};
+
+static int build_stats_dump_cmd(struct stats_cmd_context *ctx, const char **error_r);
+static int build_stats_add_cmd(struct stats_cmd_context *ctx, const char **error_r);
+static int build_stats_remove_cmd(struct stats_cmd_context *ctx, const char **error_r);
+
+static void stats_dump_process_response(struct stats_cmd_context *ctx);
+static void stats_modify_process_response(struct stats_cmd_context *ctx);
+
+static void stats_send_cmd(struct stats_cmd_context *ctx);
+
+static struct stats_cmd_vfuncs dump_vfuncs = {
+ .build_cmd = build_stats_dump_cmd,
+ .process_response = stats_dump_process_response
+};
+
+static struct stats_cmd_vfuncs add_vfuncs = {
+ .build_cmd = build_stats_add_cmd,
+ .process_response = stats_modify_process_response
+};
+
+static struct stats_cmd_vfuncs remove_vfuncs = {
+ .build_cmd = build_stats_remove_cmd,
+ .process_response = stats_modify_process_response
+};
+
+static string_t *init_stats_cmd(void)
+{
+ string_t *cmd = t_str_new(128);
+ str_append(cmd, "VERSION\tstats-reader-client\t2\t0\n");
+ return cmd;
+}
+
+static void stats_exec_cmd(struct doveadm_cmd_context *cctx,
+ struct stats_cmd_vfuncs *vfuncs)
+{
+ struct stats_cmd_context ctx;
+ const char *build_cmd_error;
+ ctx.cctx = cctx;
+ if (vfuncs->build_cmd(&ctx, &build_cmd_error) < 0) {
+ i_error("%s", build_cmd_error);
+ return;
+ }
+ stats_send_cmd(&ctx);
+ vfuncs->process_response(&ctx);
+ i_stream_destroy(&ctx.input);
+}
+
+static void handle_disconnection(struct stats_cmd_context *ctx)
+{
+ i_error("read(%s) failed: %s", ctx->path,
+ i_stream_get_disconnect_reason(ctx->input));
+}
+
+static void stats_send_cmd(struct stats_cmd_context *ctx)
+{
+ int fd;
+ const char *line;
+ if (!doveadm_cmd_param_str(ctx->cctx, "socket-path", &ctx->path))
+ ctx->path = t_strconcat(doveadm_settings->base_dir,
+ "/stats-reader", NULL);
+
+ fd = doveadm_connect(ctx->path);
+ net_set_nonblock(fd, FALSE);
+ if (write_full(fd, str_data(ctx->cmd), str_len(ctx->cmd)) < 0)
+ i_fatal("write(%s) failed %m", ctx->path);
+ ctx->input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+
+ if ((line = i_stream_read_next_line(ctx->input)) == NULL)
+ i_fatal("%s: Failed to read VERSION line", ctx->path);
+ else if (!version_string_verify(line, "stats-reader-server", 2)) {
+ i_fatal_status(EX_PROTOCOL,
+ "%s is not a compatible stats-reader socket", ctx->path);
+ }
+}
+
+static void dump_timing(const char *const **args,
+ const enum doveadm_dump_field_type field_types[],
+ unsigned int fields_count)
+{
+ unsigned int i, args_count = str_array_length(*args);
+
+ if (args_count > fields_count)
+ args_count = fields_count;
+ for (i = 0; i < args_count; i++) {
+ const char *value = (*args)[i];
+
+ switch (field_types[i]) {
+ case DOVEADM_DUMP_FIELD_TYPE_PASSTHROUGH:
+ break;
+ case DOVEADM_DUMP_FIELD_TYPE_STDDEV: {
+ double variance = strtod(value, NULL);
+ value = t_strdup_printf("%.02f", sqrt(variance));
+ break;
+ }
+ }
+ doveadm_print(value);
+ }
+ *args += args_count;
+}
+
+static int build_stats_dump_cmd(struct stats_cmd_context *ctx,
+ const char **error_r ATTR_UNUSED)
+{
+ bool reset;
+ struct dump_data *data = t_new(struct dump_data, 1);
+ const char *fields_raw;
+ const char **fields;
+ if (!doveadm_cmd_param_bool(ctx->cctx, "reset", &reset))
+ reset = FALSE;
+ if (!doveadm_cmd_param_str(ctx->cctx, "fields", &fields_raw))
+ fields_raw = DOVEADM_DUMP_DEFAULT_FIELDS;
+
+ fields = t_strsplit_spaces(fields_raw, ", ");
+ data->fields = fields;
+ data->field_count = str_array_length(fields);
+ data->field_types =
+ t_malloc0(sizeof(enum doveadm_dump_field_type) * data->field_count);
+ ctx->data = data;
+ ctx->cmd = init_stats_cmd();
+ str_append(ctx->cmd, reset ? "DUMP-RESET" : "DUMP");
+ unsigned int i;
+ for (i = 0; i < data->field_count; i++) {
+ str_append_c(ctx->cmd, '\t');
+ if (strcmp(fields[i], "stddev") == 0) {
+ data->field_types[i] = DOVEADM_DUMP_FIELD_TYPE_STDDEV;
+ str_append(ctx->cmd, "variance");
+ } else {
+ str_append_tabescaped(ctx->cmd, fields[i]);
+ }
+ }
+ str_append_c(ctx->cmd, '\n');
+ return 0;
+}
+
+static void stats_dump_process_response(struct stats_cmd_context *ctx)
+{
+ unsigned int i;
+ char *line;
+ struct dump_data *data = ctx->data;
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TAB);
+ doveadm_print_header_simple("metric_name");
+ doveadm_print_header_simple("field");
+ for (i = 0; i < data->field_count; i++)
+ doveadm_print_header(data->fields[i], data->fields[i],
+ DOVEADM_PRINT_HEADER_FLAG_NUMBER);
+
+ while ((line = i_stream_read_next_line(ctx->input)) != NULL) {
+ if (line[0] == '\0')
+ break;
+ T_BEGIN {
+ const char *const *args =
+ t_strsplit_tabescaped_inplace(line);
+
+ const char *metric_name = args[0];
+ doveadm_print(metric_name); args++;
+ doveadm_print("duration");
+ dump_timing(&args, data->field_types, data->field_count);
+ while (*args != NULL) {
+ doveadm_print(metric_name);
+ doveadm_print(*args); args++;
+ dump_timing(&args, data->field_types, data->field_count);
+ }
+ } T_END;
+ }
+ if (line == NULL)
+ handle_disconnection(ctx);
+}
+
+static int build_stats_add_cmd(struct stats_cmd_context *ctx,
+ const char **error_r)
+{
+ unsigned int i;
+ const char *parameter;
+ struct {
+ const char *name;
+ const char *default_val;
+ } params[] = {
+ { ADD_NAME_PARAM, "" },
+ { ADD_DESCR_PARAM, "" },
+ { ADD_FIELDS_PARAM, "" },
+ { ADD_GROUPBY_PARAM, "" },
+ { ADD_FILTER_PARAM, "" },
+ { ADD_EXPORTER_PARAM, "" },
+ /* Default exporter-include is to be modified
+ together with stats-settings */
+ { ADD_EXPORTERINCL_PARAM,
+ STATS_METRIC_SETTINGS_DEFAULT_EXPORTER_INCLUDE },
+ };
+
+ ctx->cmd = init_stats_cmd();
+ str_append(ctx->cmd, "METRICS-ADD");
+
+ for (i = 0; i < N_ELEMENTS(params); i++) {
+ if (!doveadm_cmd_param_str(ctx->cctx, params[i].name, &parameter))
+ parameter = params[i].default_val;
+ if (parameter[0] == '\0' &&
+ (strcmp(params[i].name, "name") == 0 ||
+ strcmp(params[i].name, "filter") == 0)) {
+ *error_r =
+ t_strdup_printf("stats add: missing %s parameter",
+ params[i].name);
+ return -1;
+ }
+ str_append_c(ctx->cmd, '\t');
+ str_append_tabescaped(ctx->cmd, parameter);
+ }
+
+ str_append_c(ctx->cmd, '\n');
+ return 0;
+}
+
+static void stats_modify_process_response(struct stats_cmd_context *ctx)
+{
+ const char *line = i_stream_read_next_line(ctx->input);
+ if (line == NULL) {
+ handle_disconnection(ctx);
+ return;
+ }
+ if (line[0] == '-')
+ i_error("%s", ++line);
+ else if (line[0] != '+')
+ i_error("Invalid response: %s", line);
+}
+
+static int build_stats_remove_cmd(struct stats_cmd_context *ctx,
+ const char **error_r)
+{
+ const char *name;
+
+ ctx->cmd = init_stats_cmd();
+ str_append(ctx->cmd, "METRICS-REMOVE\t");
+
+ if (!doveadm_cmd_param_str(ctx->cctx, "name", &name)) {
+ *error_r = "stats remove: missing name parameter";
+ return -1;
+ }
+ str_append_tabescaped(ctx->cmd, name);
+ str_append_c(ctx->cmd, '\n');
+ return 0;
+}
+
+static void doveadm_cmd_stats_dump(struct doveadm_cmd_context *cctx)
+{
+ stats_exec_cmd(cctx, &dump_vfuncs);
+}
+
+static void doveadm_cmd_stats_add(struct doveadm_cmd_context *cctx)
+{
+ stats_exec_cmd(cctx, &add_vfuncs);
+}
+
+static void doveadm_cmd_stats_remove(struct doveadm_cmd_context *cctx)
+{
+ stats_exec_cmd(cctx, &remove_vfuncs);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_stats_dump_ver2 = {
+ .cmd = doveadm_cmd_stats_dump,
+ .name = "stats dump",
+ .usage = "[-s <stats socket path>] [-r] [-f <fields>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('s', "socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('r', "reset", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('f', "fields", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_stats_add_ver2 = {
+ .cmd = doveadm_cmd_stats_add,
+ .name = "stats add",
+ .usage = "[--"ADD_DESCR_PARAM" <string>] "
+ "[--"ADD_EXPORTER_PARAM" <name> [--"ADD_EXPORTERINCL_PARAM" <fields>]] "
+ "[--"ADD_FIELDS_PARAM" <fields>] "
+ "[--"ADD_GROUPBY_PARAM" <fields>] <name> <filter>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', ADD_NAME_PARAM, CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', ADD_FILTER_PARAM, CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', ADD_EXPORTER_PARAM, CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', ADD_EXPORTERINCL_PARAM, CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', ADD_DESCR_PARAM, CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', ADD_FIELDS_PARAM, CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('\0', ADD_GROUPBY_PARAM, CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAMS_END
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_stats_remove_ver2 = {
+ .cmd = doveadm_cmd_stats_remove,
+ .name = "stats remove",
+ .usage = "<name>",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "name", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-util.c b/src/doveadm/doveadm-util.c
new file mode 100644
index 0000000..a65ef7f
--- /dev/null
+++ b/src/doveadm/doveadm-util.c
@@ -0,0 +1,221 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "net.h"
+#include "time-util.h"
+#include "master-service.h"
+#include "module-dir.h"
+#include "doveadm-settings.h"
+#include "doveadm-mail.h"
+#include "doveadm-util.h"
+
+#include <time.h>
+#include <dirent.h>
+#include <sys/stat.h>
+#include <ctype.h>
+
+#define DOVEADM_TCP_CONNECT_TIMEOUT_SECS 30
+
+bool doveadm_verbose = FALSE, doveadm_debug = FALSE, doveadm_server = FALSE;
+static struct module *modules = NULL;
+
+void doveadm_load_modules(void)
+{
+ struct module_dir_load_settings mod_set;
+
+ /* some doveadm plugins have dependencies to mail plugins. we can load
+ only those whose dependencies have been loaded earlier, the rest are
+ ignored. */
+ i_zero(&mod_set);
+ mod_set.abi_version = DOVECOT_ABI_VERSION;
+ mod_set.require_init_funcs = TRUE;
+ mod_set.debug = doveadm_debug;
+ mod_set.ignore_dlopen_errors = TRUE;
+
+ modules = module_dir_load_missing(modules, DOVEADM_MODULEDIR,
+ NULL, &mod_set);
+ module_dir_init(modules);
+}
+
+void doveadm_unload_modules(void)
+{
+ module_dir_unload(&modules);
+}
+
+bool doveadm_has_unloaded_plugin(const char *name)
+{
+ struct module *module;
+ DIR *dir;
+ struct dirent *d;
+ const char *plugin_name;
+ size_t name_len = strlen(name);
+ bool found = FALSE;
+
+ /* first check that it's not actually loaded */
+ for (module = modules; module != NULL; module = module->next) {
+ if (strcmp(module_get_plugin_name(module), name) == 0)
+ return FALSE;
+ }
+
+ dir = opendir(DOVEADM_MODULEDIR);
+ if (dir == NULL)
+ return FALSE;
+
+ while ((d = readdir(dir)) != NULL) {
+ plugin_name = module_file_get_name(d->d_name);
+ if (str_begins(plugin_name, "doveadm_"))
+ plugin_name += 8;
+
+ if (strncmp(plugin_name, name, name_len) == 0 &&
+ (plugin_name[name_len] == '\0' ||
+ strcmp(plugin_name + name_len, "_plugin") == 0)) {
+ found = TRUE;
+ break;
+ }
+ }
+ (void)closedir(dir);
+ return found;
+}
+
+const char *unixdate2str(time_t timestamp)
+{
+ return t_strflocaltime("%Y-%m-%d %H:%M:%S", timestamp);
+}
+
+const char *doveadm_plugin_getenv(const char *name)
+{
+ const char *const *envs;
+ unsigned int i, count;
+
+ if (!array_is_created(&doveadm_settings->plugin_envs))
+ return NULL;
+
+ envs = array_get(&doveadm_settings->plugin_envs, &count);
+ for (i = 0; i < count; i += 2) {
+ if (strcmp(envs[i], name) == 0)
+ return envs[i+1];
+ }
+ return NULL;
+}
+
+static int
+doveadm_tcp_connect_port(const char *host, in_port_t port)
+{
+ struct ip_addr *ips;
+ unsigned int ips_count;
+ int ret, fd;
+
+ alarm(DOVEADM_TCP_CONNECT_TIMEOUT_SECS);
+ ret = net_gethostbyname(host, &ips, &ips_count);
+ if (ret != 0) {
+ i_fatal("Lookup of host %s failed: %s",
+ host, net_gethosterror(ret));
+ }
+ fd = net_connect_ip_blocking(&ips[0], port, NULL);
+ if (fd == -1) {
+ i_fatal("connect(%s:%u) failed: %m",
+ net_ip2addr(&ips[0]), port);
+ }
+ alarm(0);
+ return fd;
+}
+
+int doveadm_tcp_connect(const char *target, in_port_t default_port)
+{
+ const char *host;
+ in_port_t port;
+
+ if (net_str2hostport(target, default_port, &host, &port) < 0) {
+ i_fatal("Port not known for %s. Either set proxy_port "
+ "or use %s:port", target, target);
+ }
+ return doveadm_tcp_connect_port(host, port);
+}
+
+int doveadm_connect_with_default_port(const char *path,
+ in_port_t default_port)
+{
+ int fd;
+
+ /* we'll assume UNIX sockets typically have an absolute path,
+ or at the very least '/' somewhere. */
+ if (strchr(path, '/') == NULL)
+ fd = doveadm_tcp_connect(path, default_port);
+ else {
+ fd = net_connect_unix(path);
+ if (fd == -1)
+ i_fatal("net_connect_unix(%s) failed: %m", path);
+ }
+ return fd;
+}
+
+int doveadm_connect(const char *path)
+{
+ return doveadm_connect_with_default_port(path, 0);
+}
+
+int i_strccdascmp(const char *a, const char *b)
+{
+ while(*a != '\0' && *b != '\0') {
+ if ((*a == ' ' || *a == '-') && *a != *b && *b != ' ' && *b != '-') {
+ if (i_toupper(*(a+1)) == *(b)) a++;
+ else break;
+ } else if ((*b == ' ' || *b == '-') && *a != *b && *a != ' ' && *a != '-') {
+ if (*a == i_toupper(*(b+1))) b++;
+ else break;
+ } else if (!((*a == ' ' || *a == '-') &&
+ (*b == ' ' || *b == '-')) &&
+ (*a != *b)) break;
+ a++; b++;
+ }
+ return *a-*b;
+}
+
+char doveadm_log_type_to_char(enum log_type type)
+{
+ switch(type) {
+ case LOG_TYPE_DEBUG:
+ return '\x01';
+ case LOG_TYPE_INFO:
+ return '\x02';
+ case LOG_TYPE_WARNING:
+ return '\x03';
+ case LOG_TYPE_ERROR:
+ return '\x04';
+ case LOG_TYPE_FATAL:
+ return '\x05';
+ case LOG_TYPE_PANIC:
+ return '\x06';
+ default:
+ i_unreached();
+ }
+}
+
+bool doveadm_log_type_from_char(char c, enum log_type *type_r)
+{
+ switch(c) {
+ case '\x01':
+ *type_r = LOG_TYPE_DEBUG;
+ break;
+ case '\x02':
+ *type_r = LOG_TYPE_INFO;
+ break;
+ case '\x03':
+ *type_r = LOG_TYPE_WARNING;
+ break;
+ case '\x04':
+ *type_r = LOG_TYPE_ERROR;
+ break;
+ case '\x05':
+ *type_r = LOG_TYPE_FATAL;
+ break;
+ case '\x06':
+ *type_r = LOG_TYPE_PANIC;
+ break;
+ default:
+ *type_r = LOG_TYPE_WARNING;
+ return FALSE;
+ }
+ return TRUE;
+}
diff --git a/src/doveadm/doveadm-util.h b/src/doveadm/doveadm-util.h
new file mode 100644
index 0000000..8aefd92
--- /dev/null
+++ b/src/doveadm/doveadm-util.h
@@ -0,0 +1,31 @@
+#ifndef DOVEADM_UTIL_H
+#define DOVEADM_UTIL_H
+
+#include "net.h"
+
+#define DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR 1
+#define DOVEADM_SERVER_PROTOCOL_VERSION_MINOR 2
+#define DOVEADM_SERVER_PROTOCOL_VERSION_LINE "VERSION\tdoveadm-server\t1\t2"
+#define DOVEADM_CLIENT_PROTOCOL_VERSION_LINE "VERSION\tdoveadm-client\t1\t2"
+
+extern bool doveadm_verbose, doveadm_debug, doveadm_server;
+
+const char *unixdate2str(time_t timestamp);
+const char *doveadm_plugin_getenv(const char *name);
+int doveadm_connect(const char *path);
+int doveadm_tcp_connect(const char *target, in_port_t default_port);
+int doveadm_connect_with_default_port(const char *path,
+ in_port_t default_port);
+
+void doveadm_load_modules(void);
+void doveadm_unload_modules(void);
+bool doveadm_has_unloaded_plugin(const char *name);
+
+char doveadm_log_type_to_char(enum log_type type) ATTR_PURE;
+bool doveadm_log_type_from_char(char c, enum log_type *type_r);
+
+/* Similar to strcmp(), except "camel case" == "camel-case" == "camelCase".
+ Otherwise the comparison is case-sensitive. */
+int i_strccdascmp(const char *a, const char *b) ATTR_PURE;
+
+#endif
diff --git a/src/doveadm/doveadm-who.c b/src/doveadm/doveadm-who.c
new file mode 100644
index 0000000..b60e515
--- /dev/null
+++ b/src/doveadm/doveadm-who.c
@@ -0,0 +1,364 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "net.h"
+#include "istream.h"
+#include "wildcard-match.h"
+#include "hash.h"
+#include "str.h"
+#include "strescape.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+#include "doveadm-who.h"
+
+#include <stdio.h>
+#include <unistd.h>
+
+struct who_user {
+ const char *username;
+ const char *service;
+ ARRAY(struct ip_addr) ips;
+ ARRAY(pid_t) pids;
+ unsigned int connection_count;
+};
+
+static void who_user_ip(const struct who_user *user, struct ip_addr *ip_r)
+{
+ if (array_count(&user->ips) == 0)
+ i_zero(ip_r);
+ else {
+ const struct ip_addr *ip = array_front(&user->ips);
+ *ip_r = *ip;
+ }
+}
+
+static unsigned int who_user_hash(const struct who_user *user)
+{
+ struct ip_addr ip;
+ unsigned int hash = str_hash(user->service);
+
+ if (user->username[0] != '\0')
+ hash += str_hash(user->username);
+ else {
+ who_user_ip(user, &ip);
+ hash += net_ip_hash(&ip);
+ }
+ return hash;
+}
+
+static int who_user_cmp(const struct who_user *user1,
+ const struct who_user *user2)
+{
+ if (strcmp(user1->username, user2->username) != 0)
+ return 1;
+ if (strcmp(user1->service, user2->service) != 0)
+ return 1;
+
+ if (user1->username[0] == '\0') {
+ /* tracking only IP addresses, not usernames */
+ struct ip_addr ip1, ip2;
+
+ who_user_ip(user1, &ip1);
+ who_user_ip(user2, &ip2);
+ return net_ip_cmp(&ip1, &ip2);
+ }
+ return 0;
+}
+
+static bool
+who_user_has_ip(const struct who_user *user, const struct ip_addr *ip)
+{
+ const struct ip_addr *ex_ip;
+
+ array_foreach(&user->ips, ex_ip) {
+ if (net_ip_compare(ex_ip, ip))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static int who_parse_line(const char *line, struct who_line *line_r)
+{
+ const char *const *args = t_strsplit_tabescaped(line);
+ const char *ident = args[0];
+ const char *pid_str = args[1];
+ const char *refcount_str = args[2];
+ const char *p, *ip_str;
+
+ i_zero(line_r);
+
+ /* ident = service/ip/username (imap, pop3)
+ or service/username (lmtp) */
+ p = strchr(ident, '/');
+ if (p == NULL)
+ return -1;
+ if (str_to_pid(pid_str, &line_r->pid) < 0)
+ return -1;
+ line_r->service = t_strdup_until(ident, p++);
+ line_r->username = strchr(p, '/');
+ if (line_r->username == NULL) {
+ /* no IP */
+ line_r->username = p;
+ } else {
+ ip_str = t_strdup_until(p, line_r->username++);
+ (void)net_addr2ip(ip_str, &line_r->ip);
+ }
+ if (str_to_uint(refcount_str, &line_r->refcount) < 0)
+ return -1;
+ return 0;
+}
+
+static bool who_user_has_pid(struct who_user *user, pid_t pid)
+{
+ pid_t ex_pid;
+
+ array_foreach_elem(&user->pids, ex_pid) {
+ if (ex_pid == pid)
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void who_aggregate_line(struct who_context *ctx,
+ const struct who_line *line)
+{
+ struct who_user *user, lookup_user;
+
+ lookup_user.username = line->username;
+ lookup_user.service = line->service;
+
+ user = hash_table_lookup(ctx->users, &lookup_user);
+ if (user == NULL) {
+ user = p_new(ctx->pool, struct who_user, 1);
+ user->username = p_strdup(ctx->pool, line->username);
+ user->service = p_strdup(ctx->pool, line->service);
+ p_array_init(&user->ips, ctx->pool, 3);
+ p_array_init(&user->pids, ctx->pool, 8);
+ hash_table_insert(ctx->users, user, user);
+ }
+ user->connection_count += line->refcount;
+
+ if (line->ip.family != 0 && !who_user_has_ip(user, &line->ip))
+ array_push_back(&user->ips, &line->ip);
+
+ if (!who_user_has_pid(user, line->pid))
+ array_push_back(&user->pids, &line->pid);
+}
+
+int who_parse_args(struct who_context *ctx, const char *const *masks)
+{
+ struct ip_addr net_ip;
+ unsigned int i, net_bits;
+
+ for (i = 0; masks[i] != NULL; i++) {
+ if (!str_is_numeric(masks[i], '\0') &&
+ net_parse_range(masks[i], &net_ip, &net_bits) == 0) {
+ if (ctx->filter.net_bits != 0) {
+ i_error("Multiple network masks not supported");
+ doveadm_exit_code = EX_USAGE;
+ return -1;
+ }
+ ctx->filter.net_ip = net_ip;
+ ctx->filter.net_bits = net_bits;
+ } else {
+ if (ctx->filter.username != NULL) {
+ i_error("Multiple username masks not supported");
+ doveadm_exit_code = EX_USAGE;
+ return -1;
+ }
+ ctx->filter.username = masks[i];
+ }
+ }
+ return 0;
+}
+
+void who_lookup(struct who_context *ctx, who_callback_t *callback)
+{
+#define ANVIL_HANDSHAKE "VERSION\tanvil\t1\t0\n"
+#define ANVIL_CMD ANVIL_HANDSHAKE"CONNECT-DUMP\n"
+ struct istream *input;
+ const char *line;
+ int fd;
+
+ fd = doveadm_connect(ctx->anvil_path);
+ net_set_nonblock(fd, FALSE);
+ if (write(fd, ANVIL_CMD, strlen(ANVIL_CMD)) < 0)
+ i_fatal("write(%s) failed: %m", ctx->anvil_path);
+
+ input = i_stream_create_fd_autoclose(&fd, SIZE_MAX);
+ while ((line = i_stream_read_next_line(input)) != NULL) {
+ if (*line == '\0')
+ break;
+ T_BEGIN {
+ struct who_line who_line;
+
+ if (who_parse_line(line, &who_line) < 0)
+ i_error("Invalid input: %s", line);
+ else
+ callback(ctx, &who_line);
+ } T_END;
+ }
+ if (input->stream_errno != 0) {
+ i_fatal("read(%s) failed: %s", ctx->anvil_path,
+ i_stream_get_error(input));
+ }
+
+ i_stream_destroy(&input);
+}
+
+static bool who_user_filter_match(const struct who_user *user,
+ const struct who_filter *filter)
+{
+ if (filter->username != NULL) {
+ if (!wildcard_match_icase(user->username, filter->username))
+ return FALSE;
+ }
+ if (filter->net_bits > 0) {
+ const struct ip_addr *ip;
+ bool ret = FALSE;
+
+ array_foreach(&user->ips, ip) {
+ if (net_is_in_network(ip, &filter->net_ip,
+ filter->net_bits)) {
+ ret = TRUE;
+ break;
+ }
+ }
+ if (!ret)
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static void who_print_user(const struct who_user *user)
+{
+ const struct ip_addr *ip;
+ pid_t pid;
+ string_t *str = t_str_new(256);
+
+ doveadm_print(user->username);
+ doveadm_print(dec2str(user->connection_count));
+ doveadm_print(user->service);
+
+ str_append_c(str, '(');
+ array_foreach_elem(&user->pids, pid)
+ str_printfa(str, "%ld ", (long)pid);
+ if (str_len(str) > 1)
+ str_truncate(str, str_len(str)-1);
+ str_append_c(str, ')');
+ doveadm_print(str_c(str));
+
+ str_truncate(str, 0);
+ str_append_c(str, '(');
+ array_foreach(&user->ips, ip)
+ str_printfa(str, "%s ", net_ip2addr(ip));
+ if (str_len(str) > 1)
+ str_truncate(str, str_len(str)-1);
+ str_append_c(str, ')');
+ doveadm_print(str_c(str));
+
+ doveadm_print_flush();
+}
+
+static void who_print(struct who_context *ctx)
+{
+ struct hash_iterate_context *iter;
+ struct who_user *user;
+
+ doveadm_print_header("username", "username", 0);
+ doveadm_print_header("connections", "#",
+ DOVEADM_PRINT_HEADER_FLAG_RIGHT_JUSTIFY);
+ doveadm_print_header("service", "proto", 0);
+ doveadm_print_header("pids", "(pids)", 0);
+ doveadm_print_header("ips", "(ips)", 0);
+
+ iter = hash_table_iterate_init(ctx->users);
+ while (hash_table_iterate(iter, ctx->users, &user, &user)) {
+ if (who_user_filter_match(user, &ctx->filter)) T_BEGIN {
+ who_print_user(user);
+ } T_END;
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+bool who_line_filter_match(const struct who_line *line,
+ const struct who_filter *filter)
+{
+ if (filter->username != NULL) {
+ if (!wildcard_match_icase(line->username, filter->username))
+ return FALSE;
+ }
+ if (filter->net_bits > 0) {
+ if (!net_is_in_network(&line->ip, &filter->net_ip,
+ filter->net_bits))
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static void who_print_line(struct who_context *ctx,
+ const struct who_line *line)
+{
+ unsigned int i;
+
+ if (!who_line_filter_match(line, &ctx->filter))
+ return;
+
+ for (i = 0; i < line->refcount; i++) T_BEGIN {
+ doveadm_print(line->username);
+ doveadm_print(line->service);
+ doveadm_print(dec2str(line->pid));
+ doveadm_print(net_ip2addr(&line->ip));
+ } T_END;
+}
+
+static void cmd_who(struct doveadm_cmd_context *cctx)
+{
+ const char *const *masks;
+ struct who_context ctx;
+ bool separate_connections = FALSE;
+
+ i_zero(&ctx);
+ if (!doveadm_cmd_param_str(cctx, "socket-path", &(ctx.anvil_path)))
+ ctx.anvil_path = t_strconcat(doveadm_settings->base_dir, "/anvil", NULL);
+ (void)doveadm_cmd_param_bool(cctx, "separate-connections", &separate_connections);
+
+ ctx.pool = pool_alloconly_create("who users", 10240);
+ hash_table_create(&ctx.users, ctx.pool, 0, who_user_hash, who_user_cmp);
+
+ if (doveadm_cmd_param_array(cctx, "mask", &masks)) {
+ if (who_parse_args(&ctx, masks) != 0) {
+ hash_table_destroy(&ctx.users);
+ pool_unref(&ctx.pool);
+ return;
+ }
+ }
+
+ doveadm_print_init(DOVEADM_PRINT_TYPE_TABLE);
+ if (!separate_connections) {
+ who_lookup(&ctx, who_aggregate_line);
+ who_print(&ctx);
+ } else {
+ doveadm_print_header("username", "username",
+ DOVEADM_PRINT_HEADER_FLAG_EXPAND);
+ doveadm_print_header("service", "proto", 0);
+ doveadm_print_header_simple("pid");
+ doveadm_print_header_simple("ip");
+ who_lookup(&ctx, who_print_line);
+ }
+
+ hash_table_destroy(&ctx.users);
+ pool_unref(&ctx.pool);
+}
+
+struct doveadm_cmd_ver2 doveadm_cmd_who_ver2 = {
+ .name = "who",
+ .cmd = cmd_who,
+ .usage = "[-a <anvil socket path>] [-1] [<user mask>] [<ip/bits>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('a',"socket-path", CMD_PARAM_STR, 0)
+DOVEADM_CMD_PARAM('1',"separate-connections", CMD_PARAM_BOOL, 0)
+DOVEADM_CMD_PARAM('\0',"mask", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm-who.h b/src/doveadm/doveadm-who.h
new file mode 100644
index 0000000..57c663a
--- /dev/null
+++ b/src/doveadm/doveadm-who.h
@@ -0,0 +1,37 @@
+#ifndef DOVEADM_WHO_H
+#define DOVEADM_WHO_H
+
+struct who_line {
+ const char *username;
+ const char *service;
+ struct ip_addr ip;
+ pid_t pid;
+ unsigned int refcount;
+};
+
+
+struct who_filter {
+ const char *username;
+ struct ip_addr net_ip;
+ unsigned int net_bits;
+};
+
+struct who_context {
+ const char *anvil_path;
+ struct who_filter filter;
+
+ pool_t pool;
+ HASH_TABLE(struct who_user *, struct who_user *) users;
+};
+
+typedef void who_callback_t(struct who_context *ctx,
+ const struct who_line *line);
+
+int who_parse_args(struct who_context *ctx, const char *const *masks);
+
+void who_lookup(struct who_context *ctx, who_callback_t *callback);
+
+bool who_line_filter_match(const struct who_line *line,
+ const struct who_filter *filter);
+
+#endif /* DOVEADM_WHO_H */
diff --git a/src/doveadm/doveadm-zlib.c b/src/doveadm/doveadm-zlib.c
new file mode 100644
index 0000000..f9bb233
--- /dev/null
+++ b/src/doveadm/doveadm-zlib.c
@@ -0,0 +1,297 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "net.h"
+#include "istream.h"
+#include "ostream.h"
+#include "istream-zlib.h"
+#include "ostream-zlib.h"
+#include "module-dir.h"
+#include "master-service.h"
+#include "compression.h"
+#include "doveadm-dump.h"
+
+#include <stdio.h>
+#include <fcntl.h>
+#include <unistd.h>
+
+static bool test_dump_imapzlib(const char *path)
+{
+ const char *p;
+ char buf[4096];
+ int fd, ret;
+ bool match = FALSE;
+
+ p = strrchr(path, '.');
+ if (p == NULL || (strcmp(p, ".in") != 0 && strcmp(p, ".out") != 0))
+ return FALSE;
+
+ fd = open(path, O_RDONLY);
+ if (fd == -1)
+ return FALSE;
+
+ ret = read(fd, buf, sizeof(buf)-1);
+ if (ret > 0) {
+ buf[ret] = '\0';
+ (void)str_lcase(buf);
+ match = strstr(buf, " ok begin compression.") != NULL ||
+ strstr(buf, " compress deflate") != NULL;
+ }
+ i_close_fd(&fd);
+ return match;
+}
+
+#ifdef HAVE_ZLIB
+static void
+cmd_dump_imapzlib(const char *path, const char *const *args ATTR_UNUSED)
+{
+ struct istream *input, *input2;
+ const unsigned char *data;
+ size_t size;
+ const char *line;
+ int fd;
+
+ fd = open(path, O_RDONLY);
+ if (fd < 0)
+ i_fatal("open(%s) failed: %m", path);
+ input = i_stream_create_fd_autoclose(&fd, 1024*32);
+ while ((line = i_stream_read_next_line(input)) != NULL) {
+ /* skip tag */
+ printf("%s\r\n", line);
+ while (*line != ' ' && *line != '\0') line++;
+ if (*line == '\0')
+ continue;
+ line++;
+
+ if (str_begins(line, "OK Begin compression") ||
+ strcasecmp(line, "COMPRESS DEFLATE") == 0)
+ break;
+ }
+
+ input2 = i_stream_create_deflate(input);
+ i_stream_unref(&input);
+
+ while (i_stream_read_more(input2, &data, &size) != -1) {
+ if (fwrite(data, 1, size, stdout) != size)
+ break;
+ i_stream_skip(input2, size);
+ }
+ if (input2->stream_errno != 0)
+ i_error("read(%s) failed: %s", path, i_stream_get_error(input));
+ i_stream_unref(&input2);
+ fflush(stdout);
+}
+
+struct client {
+ int fd;
+ struct io *io_client, *io_server;
+ struct istream *input, *stdin_input;
+ struct ostream *output;
+ const struct compression_handler *handler;
+ char *algorithm;
+ bool compressed;
+ bool compress_waiting;
+};
+
+static bool
+client_input_get_compress_algorithm(struct client *client, const char *line)
+{
+ /* skip tag */
+ while (*line != ' ' && *line != '\0')
+ line++;
+ if (strncasecmp(line, " COMPRESS ", 10) != 0)
+ return FALSE;
+
+ if (compression_lookup_handler(t_str_lcase(line+10),
+ &client->handler) <= 0)
+ i_fatal("Unsupported compression mechanism: %s", line+10);
+ return TRUE;
+}
+
+static bool client_input_uncompressed(struct client *client)
+{
+ const char *line;
+
+ if (client->compress_waiting) {
+ /* just read all the pipelined input for now */
+ (void)i_stream_read(client->stdin_input);
+ return TRUE;
+ }
+
+ while ((line = i_stream_read_next_line(client->stdin_input)) != NULL) {
+ o_stream_nsend_str(client->output, line);
+ o_stream_nsend(client->output, "\n", 1);
+ if (client_input_get_compress_algorithm(client, line))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void client_input(struct client *client)
+{
+ const unsigned char *data;
+ size_t size;
+
+ if (!client->compressed &&
+ client_input_uncompressed(client)) {
+ /* stop until server has sent reply to COMPRESS command. */
+ client->compress_waiting = TRUE;
+ return;
+ }
+ if (client->compressed) {
+ if (i_stream_read_more(client->stdin_input, &data, &size) > 0) {
+ o_stream_nsend(client->output, data, size);
+ i_stream_skip(client->stdin_input, size);
+ }
+ if (o_stream_flush(client->output) < 0) {
+ i_fatal("write() failed: %s",
+ o_stream_get_error(client->output));
+ }
+ }
+ if (client->stdin_input->eof) {
+ if (client->stdin_input->stream_errno != 0) {
+ i_fatal("read(stdin) failed: %s",
+ i_stream_get_error(client->stdin_input));
+ }
+ master_service_stop(master_service);
+ }
+}
+
+static bool server_input_is_compress_reply(const char *line)
+{
+ /* skip tag */
+ while (*line != ' ' && *line != '\0')
+ line++;
+ return str_begins(line, " OK Begin compression");
+}
+
+static bool server_input_uncompressed(struct client *client)
+{
+ const char *line;
+
+ while ((line = i_stream_read_next_line(client->input)) != NULL) {
+ if (write(STDOUT_FILENO, line, strlen(line)) < 0)
+ i_fatal("write(stdout) failed: %m");
+ if (write(STDOUT_FILENO, "\n", 1) < 0)
+ i_fatal("write(stdout) failed: %m");
+ if (server_input_is_compress_reply(line))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void server_input(struct client *client)
+{
+ const unsigned char *data;
+ size_t size;
+
+ if (i_stream_read(client->input) == -1) {
+ if (client->input->stream_errno != 0) {
+ i_fatal("read(server) failed: %s",
+ i_stream_get_error(client->input));
+ }
+
+ i_info("Server disconnected");
+ master_service_stop(master_service);
+ return;
+ }
+
+ if (!client->compressed && server_input_uncompressed(client)) {
+ /* start compression */
+ struct istream *input;
+ struct ostream *output;
+
+ i_info("<Compression started>");
+ input = client->handler->create_istream(client->input);
+ output = client->handler->create_ostream(client->output, 6);
+ i_stream_unref(&client->input);
+ o_stream_unref(&client->output);
+ client->input = input;
+ client->output = output;
+ client->compressed = TRUE;
+ client->compress_waiting = FALSE;
+ i_stream_set_input_pending(client->stdin_input, TRUE);
+ }
+
+ data = i_stream_get_data(client->input, &size);
+ if (write(STDOUT_FILENO, data, size) < 0)
+ i_fatal("write(stdout) failed: %m");
+ i_stream_skip(client->input, size);
+}
+
+static void cmd_zlibconnect(struct doveadm_cmd_context *cctx)
+{
+ struct client client;
+ const char *host;
+ struct ip_addr *ips;
+ unsigned int ips_count;
+ int64_t port_int64;
+ in_port_t port = 143;
+ int fd, ret;
+
+ if (!doveadm_cmd_param_str(cctx, "host", &host))
+ help_ver2(&doveadm_cmd_zlibconnect);
+ if (doveadm_cmd_param_int64(cctx, "port", &port_int64)) {
+ if (port_int64 == 0 || port_int64 > 65535)
+ i_fatal("Invalid port: %"PRId64, port_int64);
+ port = (in_port_t)port_int64;
+ }
+
+ ret = net_gethostbyname(host, &ips, &ips_count);
+ if (ret != 0) {
+ i_fatal("Host %s lookup failed: %s", host,
+ net_gethosterror(ret));
+ }
+
+ if ((fd = net_connect_ip(&ips[0], port, NULL)) == -1)
+ i_fatal("connect(%s, %u) failed: %m", host, port);
+
+ i_info("Connected to %s port %u.", net_ip2addr(&ips[0]), port);
+
+ i_zero(&client);
+ client.fd = fd;
+ fd_set_nonblock(STDIN_FILENO, TRUE);
+ client.stdin_input = i_stream_create_fd(STDIN_FILENO, SIZE_MAX);
+ client.input = i_stream_create_fd(fd, SIZE_MAX);
+ client.output = o_stream_create_fd(fd, 0);
+ o_stream_set_no_error_handling(client.output, TRUE);
+ client.io_client = io_add_istream(client.stdin_input, client_input, &client);
+ client.io_server = io_add_istream(client.input, server_input, &client);
+ master_service_run(master_service, NULL);
+ io_remove(&client.io_client);
+ io_remove(&client.io_server);
+ i_stream_unref(&client.stdin_input);
+ i_stream_unref(&client.input);
+ o_stream_unref(&client.output);
+ if (close(fd) < 0)
+ i_fatal("close() failed: %m");
+}
+#else
+static void
+cmd_dump_imapzlib(const char *path ATTR_UNUSED,
+ const char *const *args ATTR_UNUSED)
+{
+ i_fatal("Dovecot compiled without zlib support");
+}
+
+static void cmd_zlibconnect(struct doveadm_cmd_context *cctx ATTR_UNUSED)
+{
+ i_fatal("Dovecot compiled without zlib support");
+}
+#endif
+
+struct doveadm_cmd_dump doveadm_cmd_dump_zlib = {
+ "imapzlib",
+ test_dump_imapzlib,
+ cmd_dump_imapzlib
+};
+
+struct doveadm_cmd_ver2 doveadm_cmd_zlibconnect = {
+ .name = "zlibconnect",
+ .cmd = cmd_zlibconnect,
+ .usage = "<host> [<port>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "host", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "port", CMD_PARAM_INT64, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
diff --git a/src/doveadm/doveadm.c b/src/doveadm/doveadm.c
new file mode 100644
index 0000000..f175f2c
--- /dev/null
+++ b/src/doveadm/doveadm.c
@@ -0,0 +1,384 @@
+/* Copyright (c) 2009-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "sort.h"
+#include "ostream.h"
+#include "env-util.h"
+#include "execv-const.h"
+#include "dict.h"
+#include "master-service-private.h"
+#include "master-service-settings.h"
+#include "settings-parser.h"
+#include "doveadm-print-private.h"
+#include "doveadm-dump.h"
+#include "doveadm-mail.h"
+#include "doveadm-settings.h"
+#include "doveadm-dsync.h"
+#include "doveadm.h"
+
+#include <unistd.h>
+
+const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[] = {
+ &doveadm_print_flow_vfuncs,
+ &doveadm_print_tab_vfuncs,
+ &doveadm_print_table_vfuncs,
+ &doveadm_print_pager_vfuncs,
+ &doveadm_print_json_vfuncs,
+ &doveadm_print_formatted_vfuncs,
+ NULL
+};
+
+int doveadm_exit_code = 0;
+
+static void failure_exit_callback(int *status)
+{
+ enum fatal_exit_status fatal_status = *status;
+
+ switch (fatal_status) {
+ case FATAL_LOGWRITE:
+ case FATAL_LOGERROR:
+ case FATAL_LOGOPEN:
+ case FATAL_OUTOFMEM:
+ case FATAL_EXEC:
+ case FATAL_DEFAULT:
+ *status = EX_TEMPFAIL;
+ break;
+ }
+}
+
+static void
+doveadm_usage_compress_lines(FILE *out, const char *str, const char *prefix)
+{
+ const char *cmd, *args, *p, *short_name, *sub_name;
+ const char *prev_name = "", *prev_sub_name = "";
+ const char **lines;
+ unsigned int i, count;
+ size_t prefix_len = strlen(prefix);
+
+ /* split lines */
+ lines = (void *)p_strsplit(pool_datastack_create(), str, "\n");
+ for (count = 0; lines[count] != NULL; count++) ;
+
+ /* sort lines */
+ i_qsort(lines, count, sizeof(*lines), i_strcmp_p);
+
+ /* print lines, compress subcommands into a single line */
+ for (i = 0; i < count; i++) {
+ args = strchr(lines[i], '\t');
+ if (args == NULL) {
+ cmd = lines[i];
+ args = "";
+ } else {
+ cmd = t_strdup_until(lines[i], args);
+ args++;
+ }
+ if (*prefix != '\0') {
+ if (strncmp(cmd, prefix, prefix_len) != 0 ||
+ cmd[prefix_len] != ' ')
+ continue;
+ cmd += prefix_len + 1;
+ }
+
+ p = strchr(cmd, ' ');
+ if (p == NULL) {
+ if (*prev_name != '\0') {
+ fprintf(out, "\n");
+ prev_name = "";
+ }
+ fprintf(out, USAGE_CMDNAME_FMT" %s\n", cmd, args);
+ } else {
+ short_name = t_strdup_until(cmd, p);
+ if (strcmp(prev_name, short_name) != 0) {
+ if (*prev_name != '\0')
+ fprintf(out, "\n");
+ fprintf(out, USAGE_CMDNAME_FMT" %s",
+ short_name, t_strcut(p + 1, ' '));
+ prev_name = short_name;
+ prev_sub_name = "";
+ } else {
+ sub_name = t_strcut(p + 1, ' ');
+ if (strcmp(prev_sub_name, sub_name) != 0) {
+ fprintf(out, "|%s", sub_name);
+ prev_sub_name = sub_name;
+ }
+ }
+ }
+ }
+ if (*prev_name != '\0')
+ fprintf(out, "\n");
+}
+
+static void ATTR_NORETURN
+usage_prefix(const char *prefix)
+{
+ const struct doveadm_cmd_ver2 *cmd2;
+ string_t *str = t_str_new(1024);
+
+ fprintf(stderr, "usage: doveadm [-Dv] [-f <formatter>] ");
+ if (*prefix != '\0')
+ fprintf(stderr, "%s ", prefix);
+ fprintf(stderr, "<command> [<args>]\n");
+
+ array_foreach(&doveadm_cmds_ver2, cmd2)
+ str_printfa(str, "%s\t%s\n", cmd2->name, cmd2->usage);
+
+ doveadm_usage_compress_lines(stderr, str_c(str), prefix);
+
+ lib_exit(EX_USAGE);
+}
+
+void usage(void)
+{
+ usage_prefix("");
+}
+
+void help_ver2(const struct doveadm_cmd_ver2 *cmd)
+{
+ fprintf(stderr, "doveadm %s %s\n", cmd->name, cmd->usage);
+ lib_exit(EX_USAGE);
+}
+
+static void cmd_help(struct doveadm_cmd_context *cctx)
+{
+ const char *cmd, *man_argv[3];
+
+ if (!doveadm_cmd_param_str(cctx, "cmd", &cmd))
+ usage_prefix("");
+
+ env_put("MANPATH", MANDIR);
+ man_argv[0] = "man";
+ man_argv[1] = t_strconcat("doveadm-", cmd, NULL);
+ man_argv[2] = NULL;
+ execvp_const(man_argv[0], man_argv);
+}
+
+static struct doveadm_cmd_ver2 doveadm_cmd_help = {
+ .name = "help",
+ .cmd = cmd_help,
+ .usage = "[<cmd>]",
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "cmd", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+static void cmd_config(struct doveadm_cmd_context *cctx)
+{
+ const char *const *args, **argv;
+
+ if (!doveadm_cmd_param_array(cctx, "args", &args))
+ args = NULL;
+
+ env_put(MASTER_CONFIG_FILE_ENV,
+ master_service_get_config_path(master_service));
+
+ unsigned int len = str_array_length(args);
+ argv = t_new(const char *, len + 2);
+ argv[0] = BINDIR"/doveconf";
+ if (len > 0) {
+ i_assert(args != NULL);
+ memcpy(argv+1, args, len * sizeof(args[0]));
+ }
+ execv_const(argv[0], argv);
+}
+
+static struct doveadm_cmd_ver2 doveadm_cmd_config = {
+ .name = "config",
+ .cmd = cmd_config,
+ .usage = "[doveconf parameters]",
+ .flags = CMD_FLAG_NO_OPTIONS,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+static void cmd_exec(struct doveadm_cmd_context *cctx);
+static struct doveadm_cmd_ver2 doveadm_cmd_exec = {
+ .name = "exec",
+ .cmd = cmd_exec,
+ .usage = "<binary> [binary parameters]",
+ .flags = CMD_FLAG_NO_OPTIONS,
+DOVEADM_CMD_PARAMS_START
+DOVEADM_CMD_PARAM('\0', "binary", CMD_PARAM_STR, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAM('\0', "args", CMD_PARAM_ARRAY, CMD_PARAM_FLAG_POSITIONAL)
+DOVEADM_CMD_PARAMS_END
+};
+
+static void cmd_exec(struct doveadm_cmd_context *cctx)
+{
+ const char *path, *binary, *const *args, **argv;
+
+ if (!doveadm_cmd_param_str(cctx, "binary", &binary))
+ help_ver2(&doveadm_cmd_exec);
+ if (!doveadm_cmd_param_array(cctx, "args", &args))
+ args = NULL;
+
+ path = t_strdup_printf("%s/%s", doveadm_settings->libexec_dir, binary);
+
+ unsigned int len = str_array_length(args);
+ argv = t_new(const char *, len + 2);
+ argv[0] = path;
+ if (len > 0) {
+ i_assert(args != NULL);
+ memcpy(argv+1, args, len * sizeof(args[0]));
+ }
+ execv_const(argv[0], argv);
+}
+
+static bool doveadm_has_subcommands(const char *cmd_name)
+{
+ const struct doveadm_cmd_ver2 *cmd2;
+ size_t len = strlen(cmd_name);
+
+ array_foreach(&doveadm_cmds_ver2, cmd2) {
+ if (strncmp(cmd2->name, cmd_name, len) == 0 &&
+ cmd2->name[len] == ' ')
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static struct doveadm_cmd_ver2 *doveadm_cmdline_commands_ver2[] = {
+ &doveadm_cmd_config,
+ &doveadm_cmd_dump,
+ &doveadm_cmd_exec,
+ &doveadm_cmd_help,
+ &doveadm_cmd_oldstats_top_ver2,
+ &doveadm_cmd_pw,
+ &doveadm_cmd_zlibconnect,
+};
+
+int main(int argc, char *argv[])
+{
+ enum master_service_flags service_flags =
+ MASTER_SERVICE_FLAG_STANDALONE |
+ MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN |
+ MASTER_SERVICE_FLAG_NO_SSL_INIT |
+ MASTER_SERVICE_FLAG_NO_INIT_DATASTACK_FRAME;
+ struct doveadm_cmd_context cctx;
+ const char *cmd_name;
+ unsigned int i;
+ bool quick_init = FALSE;
+ int c;
+
+ i_zero(&cctx);
+ cctx.conn_type = DOVEADM_CONNECTION_TYPE_CLI;
+
+ i_set_failure_exit_callback(failure_exit_callback);
+ doveadm_dsync_main(&argc, &argv);
+
+ /* "+" is GNU extension to stop at the first non-option.
+ others just accept -+ option. */
+ master_service = master_service_init("doveadm", service_flags,
+ &argc, &argv, "+Df:hv");
+ while ((c = master_getopt(master_service)) > 0) {
+ switch (c) {
+ case 'D':
+ doveadm_debug = TRUE;
+ doveadm_verbose = TRUE;
+ break;
+ case 'f':
+ doveadm_print_init(optarg);
+ break;
+ case 'h':
+ doveadm_print_hide_titles = TRUE;
+ break;
+ case 'v':
+ doveadm_verbose = TRUE;
+ break;
+ default:
+ return FATAL_DEFAULT;
+ }
+ }
+ cmd_name = argv[optind];
+
+ if (cmd_name != NULL && strcmp(cmd_name, "help") == 0 &&
+ argv[optind+1] != NULL) {
+ /* "help cmd" doesn't need any configuration */
+ quick_init = TRUE;
+ }
+ master_service_init_log(master_service);
+
+ doveadm_settings_init();
+ doveadm_cmds_init();
+ for (i = 0; i < N_ELEMENTS(doveadm_cmdline_commands_ver2); i++)
+ doveadm_cmd_register_ver2(doveadm_cmdline_commands_ver2[i]);
+ doveadm_register_auth_commands();
+
+ if (cmd_name != NULL && (quick_init ||
+ strcmp(cmd_name, "config") == 0 ||
+ strcmp(cmd_name, "stop") == 0 ||
+ strcmp(cmd_name, "reload") == 0)) {
+ /* special case commands: even if there is something wrong
+ with the config (e.g. mail_plugins), don't fail these
+ commands */
+ if (strcmp(cmd_name, "help") != 0)
+ doveadm_read_settings();
+ quick_init = TRUE;
+ } else {
+ quick_init = FALSE;
+ doveadm_print_ostream = o_stream_create_fd(STDOUT_FILENO, 0);
+ o_stream_set_no_error_handling(doveadm_print_ostream, TRUE);
+ o_stream_cork(doveadm_print_ostream);
+ doveadm_dump_init();
+ doveadm_mail_init();
+ dict_drivers_register_builtin();
+ doveadm_load_modules();
+
+ /* read settings only after loading doveadm plugins, which
+ may modify what settings are read */
+ doveadm_read_settings();
+ if (doveadm_debug && getenv("LOG_STDERR_TIMESTAMP") == NULL)
+ i_set_failure_timestamp_format(master_service->set->log_timestamp);
+ master_service_init_stats_client(master_service, TRUE);
+ /* Load mail_plugins */
+ doveadm_mail_init_finish();
+ /* kludgy: Load the rest of the doveadm plugins after
+ mail_plugins have been loaded. */
+ doveadm_load_modules();
+
+ if (cmd_name == NULL) {
+ /* show usage after registering all plugins */
+ usage_prefix("");
+ }
+ }
+
+ argc -= optind;
+ argv += optind;
+ i_getopt_reset();
+
+ master_service_init_finish(master_service);
+ if (!doveadm_debug) {
+ /* disable debugging unless -D is given */
+ i_set_debug_file("/dev/null");
+ }
+
+ /* this has to be done here because proctitle hack can break
+ the env pointer */
+ cctx.username = getenv("USER");
+
+ if (!doveadm_cmd_try_run_ver2(cmd_name, argc, (const char**)argv, &cctx)) {
+ if (doveadm_has_subcommands(cmd_name))
+ usage_prefix(cmd_name);
+ if (doveadm_has_unloaded_plugin(cmd_name)) {
+ i_fatal("Unknown command '%s', but plugin %s exists. "
+ "Try to set mail_plugins=%s",
+ cmd_name, cmd_name, cmd_name);
+ }
+ usage();
+ }
+
+ if (!quick_init) {
+ doveadm_mail_deinit();
+ doveadm_dump_deinit();
+ doveadm_unload_modules();
+ dict_drivers_unregister_builtin();
+ doveadm_print_deinit();
+ o_stream_unref(&doveadm_print_ostream);
+ }
+ doveadm_cmds_deinit();
+ doveadm_settings_deinit();
+ master_service_deinit(&master_service);
+ return doveadm_exit_code;
+}
diff --git a/src/doveadm/doveadm.h b/src/doveadm/doveadm.h
new file mode 100644
index 0000000..d4f74e5
--- /dev/null
+++ b/src/doveadm/doveadm.h
@@ -0,0 +1,34 @@
+#ifndef DOVEADM_H
+#define DOVEADM_H
+
+#include <sysexits.h>
+#include "doveadm-util.h"
+#include "doveadm-settings.h"
+
+#define USAGE_CMDNAME_FMT " %-12s"
+
+#define DOVEADM_EX_NOTFOUND EX_NOHOST
+#define DOVEADM_EX_NOTPOSSIBLE EX_DATAERR
+#define DOVEADM_EX_UNKNOWN -1
+
+#define DOVEADM_EX_NOREPLICATE 1001
+
+enum doveadm_client_type {
+ DOVEADM_CONNECTION_TYPE_CLI = 0,
+ DOVEADM_CONNECTION_TYPE_TCP,
+ DOVEADM_CONNECTION_TYPE_HTTP,
+};
+
+#include "doveadm-cmd.h"
+
+extern bool doveadm_verbose_proctitle;
+extern int doveadm_exit_code;
+
+void usage(void) ATTR_NORETURN;
+void help_ver2(const struct doveadm_cmd_ver2 *cmd) ATTR_NORETURN;
+void doveadm_master_send_signal(int signo);
+
+const char *doveadm_exit_code_to_str(int code);
+int doveadm_str_to_exit_code(const char *reason);
+
+#endif
diff --git a/src/doveadm/dsync/Makefile.am b/src/doveadm/dsync/Makefile.am
new file mode 100644
index 0000000..20cbfc7
--- /dev/null
+++ b/src/doveadm/dsync/Makefile.am
@@ -0,0 +1,76 @@
+pkglib_LTLIBRARIES = libdovecot-dsync.la
+noinst_LTLIBRARIES = libdsync.la
+
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-ssl-iostream \
+ -I$(top_srcdir)/src/lib-master \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-imap \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage
+
+libdsync_la_SOURCES = \
+ dsync-brain.c \
+ dsync-brain-mailbox.c \
+ dsync-brain-mailbox-tree.c \
+ dsync-brain-mailbox-tree-sync.c \
+ dsync-brain-mails.c \
+ dsync-deserializer.c \
+ dsync-mail.c \
+ dsync-mailbox.c \
+ dsync-mailbox-import.c \
+ dsync-mailbox-export.c \
+ dsync-mailbox-state.c \
+ dsync-mailbox-tree.c \
+ dsync-mailbox-tree-fill.c \
+ dsync-mailbox-tree-sync.c \
+ dsync-serializer.c \
+ dsync-ibc.c \
+ dsync-ibc-stream.c \
+ dsync-ibc-pipe.c \
+ dsync-transaction-log-scan.c
+
+libdovecot_dsync_la_SOURCES =
+libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la
+libdovecot_dsync_la_DEPENDENCIES = libdsync.la
+libdovecot_dsync_la_LDFLAGS = -export-dynamic
+
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+ dsync-brain.h \
+ dsync-ibc.h
+
+noinst_HEADERS = \
+ dsync-brain-private.h \
+ dsync-mail.h \
+ dsync-mailbox.h \
+ dsync-mailbox-import.h \
+ dsync-mailbox-export.h \
+ dsync-mailbox-state.h \
+ dsync-mailbox-tree.h \
+ dsync-mailbox-tree-private.h \
+ dsync-serializer.h \
+ dsync-deserializer.h \
+ dsync-ibc-private.h \
+ dsync-transaction-log-scan.h
+
+test_programs = \
+ test-dsync-mailbox-tree-sync
+
+noinst_PROGRAMS = $(test_programs)
+
+test_libs = \
+ ../../lib-test/libtest.la \
+ ../../lib/liblib.la
+
+test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c
+test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs)
+test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs)
+
+check-local:
+ for bin in $(test_programs); do \
+ if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+ done
diff --git a/src/doveadm/dsync/Makefile.in b/src/doveadm/dsync/Makefile.in
new file mode 100644
index 0000000..ba39895
--- /dev/null
+++ b/src/doveadm/dsync/Makefile.in
@@ -0,0 +1,1019 @@
+# Makefile.in generated by automake 1.16.1 from Makefile.am.
+# @configure_input@
+
+# Copyright (C) 1994-2018 Free Software Foundation, Inc.
+
+# This Makefile.in is free software; the Free Software Foundation
+# gives unlimited permission to copy and/or distribute it,
+# with or without modifications, as long as this notice is preserved.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY, to the extent permitted by law; without
+# even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+# PARTICULAR PURPOSE.
+
+@SET_MAKE@
+
+
+
+VPATH = @srcdir@
+am__is_gnu_make = { \
+ if test -z '$(MAKELEVEL)'; then \
+ false; \
+ elif test -n '$(MAKE_HOST)'; then \
+ true; \
+ elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \
+ true; \
+ else \
+ false; \
+ fi; \
+}
+am__make_running_with_option = \
+ case $${target_option-} in \
+ ?) ;; \
+ *) echo "am__make_running_with_option: internal error: invalid" \
+ "target option '$${target_option-}' specified" >&2; \
+ exit 1;; \
+ esac; \
+ has_opt=no; \
+ sane_makeflags=$$MAKEFLAGS; \
+ if $(am__is_gnu_make); then \
+ sane_makeflags=$$MFLAGS; \
+ else \
+ case $$MAKEFLAGS in \
+ *\\[\ \ ]*) \
+ bs=\\; \
+ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \
+ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \
+ esac; \
+ fi; \
+ skip_next=no; \
+ strip_trailopt () \
+ { \
+ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \
+ }; \
+ for flg in $$sane_makeflags; do \
+ test $$skip_next = yes && { skip_next=no; continue; }; \
+ case $$flg in \
+ *=*|--*) continue;; \
+ -*I) strip_trailopt 'I'; skip_next=yes;; \
+ -*I?*) strip_trailopt 'I';; \
+ -*O) strip_trailopt 'O'; skip_next=yes;; \
+ -*O?*) strip_trailopt 'O';; \
+ -*l) strip_trailopt 'l'; skip_next=yes;; \
+ -*l?*) strip_trailopt 'l';; \
+ -[dEDm]) skip_next=yes;; \
+ -[JT]) skip_next=yes;; \
+ esac; \
+ case $$flg in \
+ *$$target_option*) has_opt=yes; break;; \
+ esac; \
+ done; \
+ test $$has_opt = yes
+am__make_dryrun = (target_option=n; $(am__make_running_with_option))
+am__make_keepgoing = (target_option=k; $(am__make_running_with_option))
+pkgdatadir = $(datadir)/@PACKAGE@
+pkgincludedir = $(includedir)/@PACKAGE@
+pkglibdir = $(libdir)/@PACKAGE@
+pkglibexecdir = $(libexecdir)/@PACKAGE@
+am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd
+install_sh_DATA = $(install_sh) -c -m 644
+install_sh_PROGRAM = $(install_sh) -c
+install_sh_SCRIPT = $(install_sh) -c
+INSTALL_HEADER = $(INSTALL_DATA)
+transform = $(program_transform_name)
+NORMAL_INSTALL = :
+PRE_INSTALL = :
+POST_INSTALL = :
+NORMAL_UNINSTALL = :
+PRE_UNINSTALL = :
+POST_UNINSTALL = :
+build_triplet = @build@
+host_triplet = @host@
+noinst_PROGRAMS = $(am__EXEEXT_1)
+subdir = src/doveadm/dsync
+ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
+am__aclocal_m4_deps = $(top_srcdir)/m4/ac_checktype2.m4 \
+ $(top_srcdir)/m4/ac_typeof.m4 $(top_srcdir)/m4/arc4random.m4 \
+ $(top_srcdir)/m4/blockdev.m4 $(top_srcdir)/m4/c99_vsnprintf.m4 \
+ $(top_srcdir)/m4/clock_gettime.m4 $(top_srcdir)/m4/crypt.m4 \
+ $(top_srcdir)/m4/crypt_xpg6.m4 $(top_srcdir)/m4/dbqlk.m4 \
+ $(top_srcdir)/m4/dirent_dtype.m4 $(top_srcdir)/m4/dovecot.m4 \
+ $(top_srcdir)/m4/fd_passing.m4 $(top_srcdir)/m4/fdatasync.m4 \
+ $(top_srcdir)/m4/flexible_array_member.m4 \
+ $(top_srcdir)/m4/glibc.m4 $(top_srcdir)/m4/gmtime_max.m4 \
+ $(top_srcdir)/m4/gmtime_tm_gmtoff.m4 \
+ $(top_srcdir)/m4/ioloop.m4 $(top_srcdir)/m4/iovec.m4 \
+ $(top_srcdir)/m4/ipv6.m4 $(top_srcdir)/m4/libcap.m4 \
+ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/libwrap.m4 \
+ $(top_srcdir)/m4/linux_mremap.m4 $(top_srcdir)/m4/ltoptions.m4 \
+ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \
+ $(top_srcdir)/m4/lt~obsolete.m4 $(top_srcdir)/m4/mmap_write.m4 \
+ $(top_srcdir)/m4/mntctl.m4 $(top_srcdir)/m4/modules.m4 \
+ $(top_srcdir)/m4/notify.m4 $(top_srcdir)/m4/nsl.m4 \
+ $(top_srcdir)/m4/off_t_max.m4 $(top_srcdir)/m4/pkg.m4 \
+ $(top_srcdir)/m4/pr_set_dumpable.m4 \
+ $(top_srcdir)/m4/q_quotactl.m4 $(top_srcdir)/m4/quota.m4 \
+ $(top_srcdir)/m4/random.m4 $(top_srcdir)/m4/rlimit.m4 \
+ $(top_srcdir)/m4/sendfile.m4 $(top_srcdir)/m4/size_t_signed.m4 \
+ $(top_srcdir)/m4/sockpeercred.m4 $(top_srcdir)/m4/sql.m4 \
+ $(top_srcdir)/m4/ssl.m4 $(top_srcdir)/m4/st_tim.m4 \
+ $(top_srcdir)/m4/static_array.m4 $(top_srcdir)/m4/test_with.m4 \
+ $(top_srcdir)/m4/time_t.m4 $(top_srcdir)/m4/typeof.m4 \
+ $(top_srcdir)/m4/typeof_dev_t.m4 \
+ $(top_srcdir)/m4/uoff_t_max.m4 $(top_srcdir)/m4/vararg.m4 \
+ $(top_srcdir)/m4/want_apparmor.m4 \
+ $(top_srcdir)/m4/want_bsdauth.m4 \
+ $(top_srcdir)/m4/want_bzlib.m4 \
+ $(top_srcdir)/m4/want_cassandra.m4 \
+ $(top_srcdir)/m4/want_cdb.m4 \
+ $(top_srcdir)/m4/want_checkpassword.m4 \
+ $(top_srcdir)/m4/want_clucene.m4 $(top_srcdir)/m4/want_db.m4 \
+ $(top_srcdir)/m4/want_gssapi.m4 $(top_srcdir)/m4/want_icu.m4 \
+ $(top_srcdir)/m4/want_ldap.m4 $(top_srcdir)/m4/want_lua.m4 \
+ $(top_srcdir)/m4/want_lz4.m4 $(top_srcdir)/m4/want_lzma.m4 \
+ $(top_srcdir)/m4/want_mysql.m4 $(top_srcdir)/m4/want_pam.m4 \
+ $(top_srcdir)/m4/want_passwd.m4 $(top_srcdir)/m4/want_pgsql.m4 \
+ $(top_srcdir)/m4/want_prefetch.m4 \
+ $(top_srcdir)/m4/want_shadow.m4 \
+ $(top_srcdir)/m4/want_sodium.m4 $(top_srcdir)/m4/want_solr.m4 \
+ $(top_srcdir)/m4/want_sqlite.m4 \
+ $(top_srcdir)/m4/want_stemmer.m4 \
+ $(top_srcdir)/m4/want_systemd.m4 \
+ $(top_srcdir)/m4/want_textcat.m4 \
+ $(top_srcdir)/m4/want_unwind.m4 $(top_srcdir)/m4/want_zlib.m4 \
+ $(top_srcdir)/m4/want_zstd.m4 $(top_srcdir)/configure.ac
+am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \
+ $(ACLOCAL_M4)
+DIST_COMMON = $(srcdir)/Makefile.am $(noinst_HEADERS) \
+ $(pkginc_lib_HEADERS) $(am__DIST_COMMON)
+mkinstalldirs = $(install_sh) -d
+CONFIG_HEADER = $(top_builddir)/config.h
+CONFIG_CLEAN_FILES =
+CONFIG_CLEAN_VPATH_FILES =
+am__EXEEXT_1 = test-dsync-mailbox-tree-sync$(EXEEXT)
+PROGRAMS = $(noinst_PROGRAMS)
+am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`;
+am__vpath_adj = case $$p in \
+ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \
+ *) f=$$p;; \
+ esac;
+am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`;
+am__install_max = 40
+am__nobase_strip_setup = \
+ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'`
+am__nobase_strip = \
+ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||"
+am__nobase_list = $(am__nobase_strip_setup); \
+ for p in $$list; do echo "$$p $$p"; done | \
+ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \
+ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \
+ if (++n[$$2] == $(am__install_max)) \
+ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \
+ END { for (dir in files) print dir, files[dir] }'
+am__base_list = \
+ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \
+ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g'
+am__uninstall_files_from_dir = { \
+ test -z "$$files" \
+ || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \
+ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \
+ $(am__cd) "$$dir" && rm -f $$files; }; \
+ }
+am__installdirs = "$(DESTDIR)$(pkglibdir)" \
+ "$(DESTDIR)$(pkginc_libdir)"
+LTLIBRARIES = $(noinst_LTLIBRARIES) $(pkglib_LTLIBRARIES)
+am_libdovecot_dsync_la_OBJECTS =
+libdovecot_dsync_la_OBJECTS = $(am_libdovecot_dsync_la_OBJECTS)
+AM_V_lt = $(am__v_lt_@AM_V@)
+am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
+am__v_lt_0 = --silent
+am__v_lt_1 =
+libdovecot_dsync_la_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC \
+ $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=link $(CCLD) \
+ $(AM_CFLAGS) $(CFLAGS) $(libdovecot_dsync_la_LDFLAGS) \
+ $(LDFLAGS) -o $@
+libdsync_la_LIBADD =
+am_libdsync_la_OBJECTS = dsync-brain.lo dsync-brain-mailbox.lo \
+ dsync-brain-mailbox-tree.lo dsync-brain-mailbox-tree-sync.lo \
+ dsync-brain-mails.lo dsync-deserializer.lo dsync-mail.lo \
+ dsync-mailbox.lo dsync-mailbox-import.lo \
+ dsync-mailbox-export.lo dsync-mailbox-state.lo \
+ dsync-mailbox-tree.lo dsync-mailbox-tree-fill.lo \
+ dsync-mailbox-tree-sync.lo dsync-serializer.lo dsync-ibc.lo \
+ dsync-ibc-stream.lo dsync-ibc-pipe.lo \
+ dsync-transaction-log-scan.lo
+libdsync_la_OBJECTS = $(am_libdsync_la_OBJECTS)
+am_test_dsync_mailbox_tree_sync_OBJECTS = \
+ test-dsync-mailbox-tree-sync.$(OBJEXT)
+test_dsync_mailbox_tree_sync_OBJECTS = \
+ $(am_test_dsync_mailbox_tree_sync_OBJECTS)
+AM_V_P = $(am__v_P_@AM_V@)
+am__v_P_ = $(am__v_P_@AM_DEFAULT_V@)
+am__v_P_0 = false
+am__v_P_1 = :
+AM_V_GEN = $(am__v_GEN_@AM_V@)
+am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@)
+am__v_GEN_0 = @echo " GEN " $@;
+am__v_GEN_1 =
+AM_V_at = $(am__v_at_@AM_V@)
+am__v_at_ = $(am__v_at_@AM_DEFAULT_V@)
+am__v_at_0 = @
+am__v_at_1 =
+DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir)
+depcomp = $(SHELL) $(top_srcdir)/depcomp
+am__maybe_remake_depfiles = depfiles
+am__depfiles_remade = ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo \
+ ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo \
+ ./$(DEPDIR)/dsync-brain-mailbox.Plo \
+ ./$(DEPDIR)/dsync-brain-mails.Plo ./$(DEPDIR)/dsync-brain.Plo \
+ ./$(DEPDIR)/dsync-deserializer.Plo \
+ ./$(DEPDIR)/dsync-ibc-pipe.Plo \
+ ./$(DEPDIR)/dsync-ibc-stream.Plo ./$(DEPDIR)/dsync-ibc.Plo \
+ ./$(DEPDIR)/dsync-mail.Plo \
+ ./$(DEPDIR)/dsync-mailbox-export.Plo \
+ ./$(DEPDIR)/dsync-mailbox-import.Plo \
+ ./$(DEPDIR)/dsync-mailbox-state.Plo \
+ ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo \
+ ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo \
+ ./$(DEPDIR)/dsync-mailbox-tree.Plo \
+ ./$(DEPDIR)/dsync-mailbox.Plo ./$(DEPDIR)/dsync-serializer.Plo \
+ ./$(DEPDIR)/dsync-transaction-log-scan.Plo \
+ ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po
+am__mv = mv -f
+COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
+ $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
+LTCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) \
+ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \
+ $(AM_CFLAGS) $(CFLAGS)
+AM_V_CC = $(am__v_CC_@AM_V@)
+am__v_CC_ = $(am__v_CC_@AM_DEFAULT_V@)
+am__v_CC_0 = @echo " CC " $@;
+am__v_CC_1 =
+CCLD = $(CC)
+LINK = $(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+ $(LIBTOOLFLAGS) --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) \
+ $(AM_LDFLAGS) $(LDFLAGS) -o $@
+AM_V_CCLD = $(am__v_CCLD_@AM_V@)
+am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
+am__v_CCLD_0 = @echo " CCLD " $@;
+am__v_CCLD_1 =
+SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \
+ $(test_dsync_mailbox_tree_sync_SOURCES)
+DIST_SOURCES = $(libdovecot_dsync_la_SOURCES) $(libdsync_la_SOURCES) \
+ $(test_dsync_mailbox_tree_sync_SOURCES)
+am__can_run_installinfo = \
+ case $$AM_UPDATE_INFO_DIR in \
+ n|no|NO) false;; \
+ *) (install-info --version) >/dev/null 2>&1;; \
+ esac
+HEADERS = $(noinst_HEADERS) $(pkginc_lib_HEADERS)
+am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP)
+# Read a list of newline-separated strings from the standard input,
+# and print each of them once, without duplicates. Input order is
+# *not* preserved.
+am__uniquify_input = $(AWK) '\
+ BEGIN { nonempty = 0; } \
+ { items[$$0] = 1; nonempty = 1; } \
+ END { if (nonempty) { for (i in items) print i; }; } \
+'
+# Make sure the list of sources is unique. This is necessary because,
+# e.g., the same source file might be shared among _SOURCES variables
+# for different programs/libraries.
+am__define_uniq_tagged_files = \
+ list='$(am__tagged_files)'; \
+ unique=`for i in $$list; do \
+ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \
+ done | $(am__uniquify_input)`
+ETAGS = etags
+CTAGS = ctags
+am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp
+DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
+ACLOCAL = @ACLOCAL@
+ACLOCAL_AMFLAGS = @ACLOCAL_AMFLAGS@
+AMTAR = @AMTAR@
+AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@
+APPARMOR_LIBS = @APPARMOR_LIBS@
+AR = @AR@
+AUTH_CFLAGS = @AUTH_CFLAGS@
+AUTH_LIBS = @AUTH_LIBS@
+AUTOCONF = @AUTOCONF@
+AUTOHEADER = @AUTOHEADER@
+AUTOMAKE = @AUTOMAKE@
+AWK = @AWK@
+BINARY_CFLAGS = @BINARY_CFLAGS@
+BINARY_LDFLAGS = @BINARY_LDFLAGS@
+BISON = @BISON@
+CASSANDRA_CFLAGS = @CASSANDRA_CFLAGS@
+CASSANDRA_LIBS = @CASSANDRA_LIBS@
+CC = @CC@
+CCDEPMODE = @CCDEPMODE@
+CDB_LIBS = @CDB_LIBS@
+CFLAGS = @CFLAGS@
+CLUCENE_CFLAGS = @CLUCENE_CFLAGS@
+CLUCENE_LIBS = @CLUCENE_LIBS@
+COMPRESS_LIBS = @COMPRESS_LIBS@
+CPP = @CPP@
+CPPFLAGS = @CPPFLAGS@
+CRYPT_LIBS = @CRYPT_LIBS@
+CXX = @CXX@
+CXXCPP = @CXXCPP@
+CXXDEPMODE = @CXXDEPMODE@
+CXXFLAGS = @CXXFLAGS@
+CYGPATH_W = @CYGPATH_W@
+DEFS = @DEFS@
+DEPDIR = @DEPDIR@
+DICT_LIBS = @DICT_LIBS@
+DLLIB = @DLLIB@
+DLLTOOL = @DLLTOOL@
+DSYMUTIL = @DSYMUTIL@
+DUMPBIN = @DUMPBIN@
+ECHO_C = @ECHO_C@
+ECHO_N = @ECHO_N@
+ECHO_T = @ECHO_T@
+EGREP = @EGREP@
+EXEEXT = @EXEEXT@
+FGREP = @FGREP@
+FLEX = @FLEX@
+FUZZER_CPPFLAGS = @FUZZER_CPPFLAGS@
+FUZZER_LDFLAGS = @FUZZER_LDFLAGS@
+GREP = @GREP@
+INSTALL = @INSTALL@
+INSTALL_DATA = @INSTALL_DATA@
+INSTALL_PROGRAM = @INSTALL_PROGRAM@
+INSTALL_SCRIPT = @INSTALL_SCRIPT@
+INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@
+KRB5CONFIG = @KRB5CONFIG@
+KRB5_CFLAGS = @KRB5_CFLAGS@
+KRB5_LIBS = @KRB5_LIBS@
+LD = @LD@
+LDAP_LIBS = @LDAP_LIBS@
+LDFLAGS = @LDFLAGS@
+LD_NO_WHOLE_ARCHIVE = @LD_NO_WHOLE_ARCHIVE@
+LD_WHOLE_ARCHIVE = @LD_WHOLE_ARCHIVE@
+LIBCAP = @LIBCAP@
+LIBDOVECOT = @LIBDOVECOT@
+LIBDOVECOT_COMPRESS = @LIBDOVECOT_COMPRESS@
+LIBDOVECOT_DEPS = @LIBDOVECOT_DEPS@
+LIBDOVECOT_DSYNC = @LIBDOVECOT_DSYNC@
+LIBDOVECOT_LA_LIBS = @LIBDOVECOT_LA_LIBS@
+LIBDOVECOT_LDA = @LIBDOVECOT_LDA@
+LIBDOVECOT_LDAP = @LIBDOVECOT_LDAP@
+LIBDOVECOT_LIBFTS = @LIBDOVECOT_LIBFTS@
+LIBDOVECOT_LIBFTS_DEPS = @LIBDOVECOT_LIBFTS_DEPS@
+LIBDOVECOT_LOGIN = @LIBDOVECOT_LOGIN@
+LIBDOVECOT_LUA = @LIBDOVECOT_LUA@
+LIBDOVECOT_LUA_DEPS = @LIBDOVECOT_LUA_DEPS@
+LIBDOVECOT_SQL = @LIBDOVECOT_SQL@
+LIBDOVECOT_STORAGE = @LIBDOVECOT_STORAGE@
+LIBDOVECOT_STORAGE_DEPS = @LIBDOVECOT_STORAGE_DEPS@
+LIBEXTTEXTCAT_CFLAGS = @LIBEXTTEXTCAT_CFLAGS@
+LIBEXTTEXTCAT_LIBS = @LIBEXTTEXTCAT_LIBS@
+LIBICONV = @LIBICONV@
+LIBICU_CFLAGS = @LIBICU_CFLAGS@
+LIBICU_LIBS = @LIBICU_LIBS@
+LIBOBJS = @LIBOBJS@
+LIBS = @LIBS@
+LIBSODIUM_CFLAGS = @LIBSODIUM_CFLAGS@
+LIBSODIUM_LIBS = @LIBSODIUM_LIBS@
+LIBTIRPC_CFLAGS = @LIBTIRPC_CFLAGS@
+LIBTIRPC_LIBS = @LIBTIRPC_LIBS@
+LIBTOOL = @LIBTOOL@
+LIBUNWIND_CFLAGS = @LIBUNWIND_CFLAGS@
+LIBUNWIND_LIBS = @LIBUNWIND_LIBS@
+LIBWRAP_LIBS = @LIBWRAP_LIBS@
+LINKED_STORAGE_LDADD = @LINKED_STORAGE_LDADD@
+LIPO = @LIPO@
+LN_S = @LN_S@
+LTLIBICONV = @LTLIBICONV@
+LTLIBOBJS = @LTLIBOBJS@
+LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@
+LUA_CFLAGS = @LUA_CFLAGS@
+LUA_LIBS = @LUA_LIBS@
+MAINT = @MAINT@
+MAKEINFO = @MAKEINFO@
+MANIFEST_TOOL = @MANIFEST_TOOL@
+MKDIR_P = @MKDIR_P@
+MODULE_LIBS = @MODULE_LIBS@
+MODULE_SUFFIX = @MODULE_SUFFIX@
+MYSQL_CFLAGS = @MYSQL_CFLAGS@
+MYSQL_CONFIG = @MYSQL_CONFIG@
+MYSQL_LIBS = @MYSQL_LIBS@
+NM = @NM@
+NMEDIT = @NMEDIT@
+NOPLUGIN_LDFLAGS = @NOPLUGIN_LDFLAGS@
+OBJDUMP = @OBJDUMP@
+OBJEXT = @OBJEXT@
+OTOOL = @OTOOL@
+OTOOL64 = @OTOOL64@
+PACKAGE = @PACKAGE@
+PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@
+PACKAGE_NAME = @PACKAGE_NAME@
+PACKAGE_STRING = @PACKAGE_STRING@
+PACKAGE_TARNAME = @PACKAGE_TARNAME@
+PACKAGE_URL = @PACKAGE_URL@
+PACKAGE_VERSION = @PACKAGE_VERSION@
+PANDOC = @PANDOC@
+PATH_SEPARATOR = @PATH_SEPARATOR@
+PGSQL_CFLAGS = @PGSQL_CFLAGS@
+PGSQL_LIBS = @PGSQL_LIBS@
+PG_CONFIG = @PG_CONFIG@
+PIE_CFLAGS = @PIE_CFLAGS@
+PIE_LDFLAGS = @PIE_LDFLAGS@
+PKG_CONFIG = @PKG_CONFIG@
+PKG_CONFIG_LIBDIR = @PKG_CONFIG_LIBDIR@
+PKG_CONFIG_PATH = @PKG_CONFIG_PATH@
+QUOTA_LIBS = @QUOTA_LIBS@
+RANLIB = @RANLIB@
+RELRO_LDFLAGS = @RELRO_LDFLAGS@
+RPCGEN = @RPCGEN@
+RUN_TEST = @RUN_TEST@
+SED = @SED@
+SETTING_FILES = @SETTING_FILES@
+SET_MAKE = @SET_MAKE@
+SHELL = @SHELL@
+SQLITE_CFLAGS = @SQLITE_CFLAGS@
+SQLITE_LIBS = @SQLITE_LIBS@
+SQL_CFLAGS = @SQL_CFLAGS@
+SQL_LIBS = @SQL_LIBS@
+SSL_CFLAGS = @SSL_CFLAGS@
+SSL_LIBS = @SSL_LIBS@
+STRIP = @STRIP@
+SYSTEMD_CFLAGS = @SYSTEMD_CFLAGS@
+SYSTEMD_LIBS = @SYSTEMD_LIBS@
+VALGRIND = @VALGRIND@
+VERSION = @VERSION@
+ZSTD_CFLAGS = @ZSTD_CFLAGS@
+ZSTD_LIBS = @ZSTD_LIBS@
+abs_builddir = @abs_builddir@
+abs_srcdir = @abs_srcdir@
+abs_top_builddir = @abs_top_builddir@
+abs_top_srcdir = @abs_top_srcdir@
+ac_ct_AR = @ac_ct_AR@
+ac_ct_CC = @ac_ct_CC@
+ac_ct_CXX = @ac_ct_CXX@
+ac_ct_DUMPBIN = @ac_ct_DUMPBIN@
+am__include = @am__include@
+am__leading_dot = @am__leading_dot@
+am__quote = @am__quote@
+am__tar = @am__tar@
+am__untar = @am__untar@
+bindir = @bindir@
+build = @build@
+build_alias = @build_alias@
+build_cpu = @build_cpu@
+build_os = @build_os@
+build_vendor = @build_vendor@
+builddir = @builddir@
+datadir = @datadir@
+datarootdir = @datarootdir@
+dict_drivers = @dict_drivers@
+docdir = @docdir@
+dvidir = @dvidir@
+exec_prefix = @exec_prefix@
+host = @host@
+host_alias = @host_alias@
+host_cpu = @host_cpu@
+host_os = @host_os@
+host_vendor = @host_vendor@
+htmldir = @htmldir@
+includedir = @includedir@
+infodir = @infodir@
+install_sh = @install_sh@
+libdir = @libdir@
+libexecdir = @libexecdir@
+localedir = @localedir@
+localstatedir = @localstatedir@
+mandir = @mandir@
+mkdir_p = @mkdir_p@
+moduledir = @moduledir@
+oldincludedir = @oldincludedir@
+pdfdir = @pdfdir@
+prefix = @prefix@
+program_transform_name = @program_transform_name@
+psdir = @psdir@
+rundir = @rundir@
+runstatedir = @runstatedir@
+sbindir = @sbindir@
+sharedstatedir = @sharedstatedir@
+sql_drivers = @sql_drivers@
+srcdir = @srcdir@
+ssldir = @ssldir@
+statedir = @statedir@
+sysconfdir = @sysconfdir@
+systemdservicetype = @systemdservicetype@
+systemdsystemunitdir = @systemdsystemunitdir@
+target_alias = @target_alias@
+top_build_prefix = @top_build_prefix@
+top_builddir = @top_builddir@
+top_srcdir = @top_srcdir@
+pkglib_LTLIBRARIES = libdovecot-dsync.la
+noinst_LTLIBRARIES = libdsync.la
+AM_CPPFLAGS = \
+ -I$(top_srcdir)/src/lib \
+ -I$(top_srcdir)/src/lib-test \
+ -I$(top_srcdir)/src/lib-settings \
+ -I$(top_srcdir)/src/lib-ssl-iostream \
+ -I$(top_srcdir)/src/lib-master \
+ -I$(top_srcdir)/src/lib-mail \
+ -I$(top_srcdir)/src/lib-imap \
+ -I$(top_srcdir)/src/lib-index \
+ -I$(top_srcdir)/src/lib-storage
+
+libdsync_la_SOURCES = \
+ dsync-brain.c \
+ dsync-brain-mailbox.c \
+ dsync-brain-mailbox-tree.c \
+ dsync-brain-mailbox-tree-sync.c \
+ dsync-brain-mails.c \
+ dsync-deserializer.c \
+ dsync-mail.c \
+ dsync-mailbox.c \
+ dsync-mailbox-import.c \
+ dsync-mailbox-export.c \
+ dsync-mailbox-state.c \
+ dsync-mailbox-tree.c \
+ dsync-mailbox-tree-fill.c \
+ dsync-mailbox-tree-sync.c \
+ dsync-serializer.c \
+ dsync-ibc.c \
+ dsync-ibc-stream.c \
+ dsync-ibc-pipe.c \
+ dsync-transaction-log-scan.c
+
+libdovecot_dsync_la_SOURCES =
+libdovecot_dsync_la_LIBADD = libdsync.la ../../lib-storage/libdovecot-storage.la ../../lib-dovecot/libdovecot.la
+libdovecot_dsync_la_DEPENDENCIES = libdsync.la
+libdovecot_dsync_la_LDFLAGS = -export-dynamic
+pkginc_libdir = $(pkgincludedir)
+pkginc_lib_HEADERS = \
+ dsync-brain.h \
+ dsync-ibc.h
+
+noinst_HEADERS = \
+ dsync-brain-private.h \
+ dsync-mail.h \
+ dsync-mailbox.h \
+ dsync-mailbox-import.h \
+ dsync-mailbox-export.h \
+ dsync-mailbox-state.h \
+ dsync-mailbox-tree.h \
+ dsync-mailbox-tree-private.h \
+ dsync-serializer.h \
+ dsync-deserializer.h \
+ dsync-ibc-private.h \
+ dsync-transaction-log-scan.h
+
+test_programs = \
+ test-dsync-mailbox-tree-sync
+
+test_libs = \
+ ../../lib-test/libtest.la \
+ ../../lib/liblib.la
+
+test_dsync_mailbox_tree_sync_SOURCES = test-dsync-mailbox-tree-sync.c
+test_dsync_mailbox_tree_sync_LDADD = dsync-mailbox-tree-sync.lo dsync-mailbox-tree.lo $(test_libs)
+test_dsync_mailbox_tree_sync_DEPENDENCIES = $(pkglib_LTLIBRARIES) $(test_libs)
+all: all-am
+
+.SUFFIXES:
+.SUFFIXES: .c .lo .o .obj
+$(srcdir)/Makefile.in: @MAINTAINER_MODE_TRUE@ $(srcdir)/Makefile.am $(am__configure_deps)
+ @for dep in $?; do \
+ case '$(am__configure_deps)' in \
+ *$$dep*) \
+ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \
+ && { if test -f $@; then exit 0; else break; fi; }; \
+ exit 1;; \
+ esac; \
+ done; \
+ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile'; \
+ $(am__cd) $(top_srcdir) && \
+ $(AUTOMAKE) --foreign src/doveadm/dsync/Makefile
+Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status
+ @case '$?' in \
+ *config.status*) \
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \
+ *) \
+ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \
+ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \
+ esac;
+
+$(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+
+$(top_srcdir)/configure: @MAINTAINER_MODE_TRUE@ $(am__configure_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(ACLOCAL_M4): @MAINTAINER_MODE_TRUE@ $(am__aclocal_m4_deps)
+ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh
+$(am__aclocal_m4_deps):
+
+clean-noinstPROGRAMS:
+ @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \
+ echo " rm -f" $$list; \
+ rm -f $$list || exit $$?; \
+ test -n "$(EXEEXT)" || exit 0; \
+ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \
+ echo " rm -f" $$list; \
+ rm -f $$list
+
+clean-noinstLTLIBRARIES:
+ -test -z "$(noinst_LTLIBRARIES)" || rm -f $(noinst_LTLIBRARIES)
+ @list='$(noinst_LTLIBRARIES)'; \
+ locs=`for p in $$list; do echo $$p; done | \
+ sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+ sort -u`; \
+ test -z "$$locs" || { \
+ echo rm -f $${locs}; \
+ rm -f $${locs}; \
+ }
+
+install-pkglibLTLIBRARIES: $(pkglib_LTLIBRARIES)
+ @$(NORMAL_INSTALL)
+ @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+ list2=; for p in $$list; do \
+ if test -f $$p; then \
+ list2="$$list2 $$p"; \
+ else :; fi; \
+ done; \
+ test -z "$$list2" || { \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pkglibdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pkglibdir)" || exit 1; \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 '$(DESTDIR)$(pkglibdir)'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL) $(INSTALL_STRIP_FLAG) $$list2 "$(DESTDIR)$(pkglibdir)"; \
+ }
+
+uninstall-pkglibLTLIBRARIES:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkglib_LTLIBRARIES)'; test -n "$(pkglibdir)" || list=; \
+ for p in $$list; do \
+ $(am__strip_dir) \
+ echo " $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f '$(DESTDIR)$(pkglibdir)/$$f'"; \
+ $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=uninstall rm -f "$(DESTDIR)$(pkglibdir)/$$f"; \
+ done
+
+clean-pkglibLTLIBRARIES:
+ -test -z "$(pkglib_LTLIBRARIES)" || rm -f $(pkglib_LTLIBRARIES)
+ @list='$(pkglib_LTLIBRARIES)'; \
+ locs=`for p in $$list; do echo $$p; done | \
+ sed 's|^[^/]*$$|.|; s|/[^/]*$$||; s|$$|/so_locations|' | \
+ sort -u`; \
+ test -z "$$locs" || { \
+ echo rm -f $${locs}; \
+ rm -f $${locs}; \
+ }
+
+libdovecot-dsync.la: $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_DEPENDENCIES) $(EXTRA_libdovecot_dsync_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(libdovecot_dsync_la_LINK) -rpath $(pkglibdir) $(libdovecot_dsync_la_OBJECTS) $(libdovecot_dsync_la_LIBADD) $(LIBS)
+
+libdsync.la: $(libdsync_la_OBJECTS) $(libdsync_la_DEPENDENCIES) $(EXTRA_libdsync_la_DEPENDENCIES)
+ $(AM_V_CCLD)$(LINK) $(libdsync_la_OBJECTS) $(libdsync_la_LIBADD) $(LIBS)
+
+test-dsync-mailbox-tree-sync$(EXEEXT): $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_DEPENDENCIES) $(EXTRA_test_dsync_mailbox_tree_sync_DEPENDENCIES)
+ @rm -f test-dsync-mailbox-tree-sync$(EXEEXT)
+ $(AM_V_CCLD)$(LINK) $(test_dsync_mailbox_tree_sync_OBJECTS) $(test_dsync_mailbox_tree_sync_LDADD) $(LIBS)
+
+mostlyclean-compile:
+ -rm -f *.$(OBJEXT)
+
+distclean-compile:
+ -rm -f *.tab.c
+
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox-tree.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mailbox.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain-mails.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-brain.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-deserializer.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-pipe.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc-stream.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-ibc.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mail.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-export.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-import.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-state.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-fill.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree-sync.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox-tree.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-mailbox.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-serializer.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dsync-transaction-log-scan.Plo@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po@am__quote@ # am--include-marker
+
+$(am__depfiles_remade):
+ @$(MKDIR_P) $(@D)
+ @echo '# dummy' >$@-t && $(am__mv) $@-t $@
+
+am--depfiles: $(am__depfiles_remade)
+
+.c.o:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ $<
+
+.c.obj:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(COMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ `$(CYGPATH_W) '$<'`
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(COMPILE) -c -o $@ `$(CYGPATH_W) '$<'`
+
+.c.lo:
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(LTCOMPILE) -MT $@ -MD -MP -MF $(DEPDIR)/$*.Tpo -c -o $@ $<
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/$*.Tpo $(DEPDIR)/$*.Plo
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LTCOMPILE) -c -o $@ $<
+
+mostlyclean-libtool:
+ -rm -f *.lo
+
+clean-libtool:
+ -rm -rf .libs _libs
+install-pkginc_libHEADERS: $(pkginc_lib_HEADERS)
+ @$(NORMAL_INSTALL)
+ @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+ if test -n "$$list"; then \
+ echo " $(MKDIR_P) '$(DESTDIR)$(pkginc_libdir)'"; \
+ $(MKDIR_P) "$(DESTDIR)$(pkginc_libdir)" || exit 1; \
+ fi; \
+ for p in $$list; do \
+ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \
+ echo "$$d$$p"; \
+ done | $(am__base_list) | \
+ while read files; do \
+ echo " $(INSTALL_HEADER) $$files '$(DESTDIR)$(pkginc_libdir)'"; \
+ $(INSTALL_HEADER) $$files "$(DESTDIR)$(pkginc_libdir)" || exit $$?; \
+ done
+
+uninstall-pkginc_libHEADERS:
+ @$(NORMAL_UNINSTALL)
+ @list='$(pkginc_lib_HEADERS)'; test -n "$(pkginc_libdir)" || list=; \
+ files=`for p in $$list; do echo $$p; done | sed -e 's|^.*/||'`; \
+ dir='$(DESTDIR)$(pkginc_libdir)'; $(am__uninstall_files_from_dir)
+
+ID: $(am__tagged_files)
+ $(am__define_uniq_tagged_files); mkid -fID $$unique
+tags: tags-am
+TAGS: tags
+
+tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ set x; \
+ here=`pwd`; \
+ $(am__define_uniq_tagged_files); \
+ shift; \
+ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \
+ test -n "$$unique" || unique=$$empty_fix; \
+ if test $$# -gt 0; then \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ "$$@" $$unique; \
+ else \
+ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \
+ $$unique; \
+ fi; \
+ fi
+ctags: ctags-am
+
+CTAGS: ctags
+ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files)
+ $(am__define_uniq_tagged_files); \
+ test -z "$(CTAGS_ARGS)$$unique" \
+ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \
+ $$unique
+
+GTAGS:
+ here=`$(am__cd) $(top_builddir) && pwd` \
+ && $(am__cd) $(top_srcdir) \
+ && gtags -i $(GTAGS_ARGS) "$$here"
+cscopelist: cscopelist-am
+
+cscopelist-am: $(am__tagged_files)
+ list='$(am__tagged_files)'; \
+ case "$(srcdir)" in \
+ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \
+ *) sdir=$(subdir)/$(srcdir) ;; \
+ esac; \
+ for i in $$list; do \
+ if test -f "$$i"; then \
+ echo "$(subdir)/$$i"; \
+ else \
+ echo "$$sdir/$$i"; \
+ fi; \
+ done >> $(top_builddir)/cscope.files
+
+distclean-tags:
+ -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
+
+distdir: $(BUILT_SOURCES)
+ $(MAKE) $(AM_MAKEFLAGS) distdir-am
+
+distdir-am: $(DISTFILES)
+ @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \
+ list='$(DISTFILES)'; \
+ dist_files=`for file in $$list; do echo $$file; done | \
+ sed -e "s|^$$srcdirstrip/||;t" \
+ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \
+ case $$dist_files in \
+ */*) $(MKDIR_P) `echo "$$dist_files" | \
+ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \
+ sort -u` ;; \
+ esac; \
+ for file in $$dist_files; do \
+ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \
+ if test -d $$d/$$file; then \
+ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \
+ if test -d "$(distdir)/$$file"; then \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \
+ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \
+ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \
+ fi; \
+ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \
+ else \
+ test -f "$(distdir)/$$file" \
+ || cp -p $$d/$$file "$(distdir)/$$file" \
+ || exit 1; \
+ fi; \
+ done
+check-am: all-am
+ $(MAKE) $(AM_MAKEFLAGS) check-local
+check: check-am
+all-am: Makefile $(PROGRAMS) $(LTLIBRARIES) $(HEADERS)
+installdirs:
+ for dir in "$(DESTDIR)$(pkglibdir)" "$(DESTDIR)$(pkginc_libdir)"; do \
+ test -z "$$dir" || $(MKDIR_P) "$$dir"; \
+ done
+install: install-am
+install-exec: install-exec-am
+install-data: install-data-am
+uninstall: uninstall-am
+
+install-am: all-am
+ @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am
+
+installcheck: installcheck-am
+install-strip:
+ if test -z '$(STRIP)'; then \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ install; \
+ else \
+ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \
+ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \
+ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \
+ fi
+mostlyclean-generic:
+
+clean-generic:
+
+distclean-generic:
+ -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES)
+ -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES)
+
+maintainer-clean-generic:
+ @echo "This command is intended for maintainers to use"
+ @echo "it deletes files that may require special tools to rebuild."
+clean: clean-am
+
+clean-am: clean-generic clean-libtool clean-noinstLTLIBRARIES \
+ clean-noinstPROGRAMS clean-pkglibLTLIBRARIES mostlyclean-am
+
+distclean: distclean-am
+ -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain-mails.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain.Plo
+ -rm -f ./$(DEPDIR)/dsync-deserializer.Plo
+ -rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo
+ -rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo
+ -rm -f ./$(DEPDIR)/dsync-ibc.Plo
+ -rm -f ./$(DEPDIR)/dsync-mail.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox.Plo
+ -rm -f ./$(DEPDIR)/dsync-serializer.Plo
+ -rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo
+ -rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po
+ -rm -f Makefile
+distclean-am: clean-am distclean-compile distclean-generic \
+ distclean-tags
+
+dvi: dvi-am
+
+dvi-am:
+
+html: html-am
+
+html-am:
+
+info: info-am
+
+info-am:
+
+install-data-am: install-pkginc_libHEADERS
+
+install-dvi: install-dvi-am
+
+install-dvi-am:
+
+install-exec-am: install-pkglibLTLIBRARIES
+
+install-html: install-html-am
+
+install-html-am:
+
+install-info: install-info-am
+
+install-info-am:
+
+install-man:
+
+install-pdf: install-pdf-am
+
+install-pdf-am:
+
+install-ps: install-ps-am
+
+install-ps-am:
+
+installcheck-am:
+
+maintainer-clean: maintainer-clean-am
+ -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree-sync.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain-mailbox-tree.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain-mailbox.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain-mails.Plo
+ -rm -f ./$(DEPDIR)/dsync-brain.Plo
+ -rm -f ./$(DEPDIR)/dsync-deserializer.Plo
+ -rm -f ./$(DEPDIR)/dsync-ibc-pipe.Plo
+ -rm -f ./$(DEPDIR)/dsync-ibc-stream.Plo
+ -rm -f ./$(DEPDIR)/dsync-ibc.Plo
+ -rm -f ./$(DEPDIR)/dsync-mail.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-export.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-import.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-state.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-tree-fill.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-tree-sync.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox-tree.Plo
+ -rm -f ./$(DEPDIR)/dsync-mailbox.Plo
+ -rm -f ./$(DEPDIR)/dsync-serializer.Plo
+ -rm -f ./$(DEPDIR)/dsync-transaction-log-scan.Plo
+ -rm -f ./$(DEPDIR)/test-dsync-mailbox-tree-sync.Po
+ -rm -f Makefile
+maintainer-clean-am: distclean-am maintainer-clean-generic
+
+mostlyclean: mostlyclean-am
+
+mostlyclean-am: mostlyclean-compile mostlyclean-generic \
+ mostlyclean-libtool
+
+pdf: pdf-am
+
+pdf-am:
+
+ps: ps-am
+
+ps-am:
+
+uninstall-am: uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES
+
+.MAKE: check-am install-am install-strip
+
+.PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am \
+ check-local clean clean-generic clean-libtool \
+ clean-noinstLTLIBRARIES clean-noinstPROGRAMS \
+ clean-pkglibLTLIBRARIES cscopelist-am ctags ctags-am distclean \
+ distclean-compile distclean-generic distclean-libtool \
+ distclean-tags distdir dvi dvi-am html html-am info info-am \
+ install install-am install-data install-data-am install-dvi \
+ install-dvi-am install-exec install-exec-am install-html \
+ install-html-am install-info install-info-am install-man \
+ install-pdf install-pdf-am install-pkginc_libHEADERS \
+ install-pkglibLTLIBRARIES install-ps install-ps-am \
+ install-strip installcheck installcheck-am installdirs \
+ maintainer-clean maintainer-clean-generic mostlyclean \
+ mostlyclean-compile mostlyclean-generic mostlyclean-libtool \
+ pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \
+ uninstall-pkginc_libHEADERS uninstall-pkglibLTLIBRARIES
+
+.PRECIOUS: Makefile
+
+
+check-local:
+ for bin in $(test_programs); do \
+ if ! $(RUN_TEST) ./$$bin; then exit 1; fi; \
+ done
+
+# Tell versions [3.59,3.63) of GNU make to not export all variables.
+# Otherwise a system limit (for SysV at least) may be exceeded.
+.NOEXPORT:
diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c
new file mode 100644
index 0000000..2030e04
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mailbox-tree-sync.c
@@ -0,0 +1,229 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-brain-private.h"
+
+static int
+sync_create_box(struct dsync_brain *brain, struct mailbox *box,
+ const guid_128_t mailbox_guid, uint32_t uid_validity,
+ enum mail_error *error_r)
+{
+ struct mailbox_metadata metadata;
+ struct mailbox_update update;
+ enum mail_error error;
+ const char *errstr;
+ int ret;
+
+ i_zero(&update);
+ memcpy(update.mailbox_guid, mailbox_guid, sizeof(update.mailbox_guid));
+ update.uid_validity = uid_validity;
+
+ if (mailbox_create(box, &update, FALSE) < 0) {
+ errstr = mailbox_get_last_internal_error(box, &error);
+ if (error != MAIL_ERROR_EXISTS) {
+ i_error("Can't create mailbox %s: %s",
+ mailbox_get_vname(box), errstr);
+ *error_r = error;
+ return -1;
+ }
+ }
+ if (brain->no_mail_sync) {
+ /* trust that create worked, we can't actually open it
+ and verify. */
+ return 0;
+ }
+ /* sync the mailbox so we can look up its latest status */
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ i_error("Can't sync mailbox %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, error_r));
+ return -1;
+ }
+
+ /* verify that the GUID is what we wanted. if it's not, it probably
+ means that the mailbox had already been created. then we'll use the
+ GUID that is higher.
+
+ mismatching UIDVALIDITY is handled later, because we choose it by
+ checking which mailbox has more messages */
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, &metadata) < 0) {
+ i_error("Can't get mailbox GUID %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, error_r));
+ return -1;
+ }
+
+ ret = memcmp(mailbox_guid, metadata.guid, sizeof(metadata.guid));
+
+ /* if THEIR guid is bigger than OUR guid, and we are not doing
+ backup in either direction, OR GUID did not match and we are
+ receiving backup, try change the mailbox GUID.
+ */
+
+ if ((ret > 0 && !brain->backup_recv &&
+ !brain->backup_send) || (ret != 0 && brain->backup_recv)) {
+ if (brain->debug) {
+ i_debug("brain %c: Changing mailbox %s GUID %s -> %s",
+ brain->master_brain ? 'M' : 'S',
+ mailbox_get_vname(box),
+ guid_128_to_string(metadata.guid),
+ guid_128_to_string(mailbox_guid));
+ }
+ i_zero(&update);
+ memcpy(update.mailbox_guid, mailbox_guid,
+ sizeof(update.mailbox_guid));
+ if (mailbox_update(box, &update) < 0) {
+ i_error("Can't update mailbox GUID %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, error_r));
+ return -1;
+ }
+ /* verify that the update worked */
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID,
+ &metadata) < 0) {
+ i_error("Can't get mailbox GUID %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, error_r));
+ return -1;
+ }
+ if (memcmp(mailbox_guid, metadata.guid,
+ sizeof(metadata.guid)) != 0) {
+ i_error("Backend didn't update mailbox %s GUID",
+ mailbox_get_vname(box));
+ *error_r = MAIL_ERROR_TEMP;
+ return -1;
+ }
+ } else if (ret < 0) {
+ if (brain->debug) {
+ i_debug("brain %c: Other brain should change mailbox "
+ "%s GUID %s -> %s",
+ brain->master_brain ? 'M' : 'S',
+ mailbox_get_vname(box),
+ guid_128_to_string(mailbox_guid),
+ guid_128_to_string(metadata.guid));
+ }
+ }
+ return 0;
+}
+
+int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain,
+ const struct dsync_mailbox_tree_sync_change *change,
+ enum mail_error *error_r)
+{
+ struct mailbox *box = NULL, *destbox;
+ const char *errstr, *func_name = NULL, *storage_name;
+ enum mail_error error;
+ int ret = -1;
+
+ if (brain->backup_send) {
+ i_assert(brain->no_backup_overwrite);
+ return 0;
+ }
+
+ switch (change->type) {
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX:
+ /* make sure we're deleting the correct mailbox */
+ ret = dsync_brain_mailbox_alloc(brain, change->mailbox_guid,
+ &box, &errstr, error_r);
+ if (ret < 0) {
+ i_error("Mailbox sync: Couldn't allocate mailbox %s GUID %s: %s",
+ change->full_name,
+ guid_128_to_string(change->mailbox_guid), errstr);
+ return -1;
+ }
+ if (ret == 0) {
+ dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+ "Mailbox %s GUID %s deletion conflict: %s",
+ change->full_name,
+ guid_128_to_string(change->mailbox_guid), errstr));
+ return 0;
+ }
+ break;
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR:
+ storage_name = mailbox_list_get_storage_name(change->ns->list,
+ change->full_name);
+ if (mailbox_list_delete_dir(change->ns->list, storage_name) == 0)
+ return 0;
+
+ errstr = mailbox_list_get_last_internal_error(change->ns->list, &error);
+ if (error == MAIL_ERROR_NOTFOUND ||
+ error == MAIL_ERROR_EXISTS) {
+ dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+ "Mailbox %s mailbox_list_delete_dir conflict: %s",
+ change->full_name, errstr));
+ return 0;
+ } else {
+ i_error("Mailbox sync: mailbox_list_delete_dir failed: %s",
+ errstr);
+ *error_r = error;
+ return -1;
+ }
+ default:
+ box = mailbox_alloc(change->ns->list, change->full_name, 0);
+ break;
+ }
+ mailbox_skip_create_name_restrictions(box, TRUE);
+ switch (change->type) {
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX:
+ ret = sync_create_box(brain, box, change->mailbox_guid,
+ change->uid_validity, error_r);
+ mailbox_free(&box);
+ return ret;
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR:
+ ret = mailbox_create(box, NULL, TRUE);
+ if (ret < 0 &&
+ mailbox_get_last_mail_error(box) == MAIL_ERROR_EXISTS) {
+ /* it doesn't matter if somebody else created this
+ directory or we automatically did while creating its
+ child mailbox. it's there now anyway and we don't
+ gain anything by treating this failure any
+ differently from success. */
+ ret = 0;
+ }
+ func_name = "mailbox_create";
+ break;
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX:
+ ret = mailbox_delete(box);
+ func_name = "mailbox_delete";
+ break;
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR:
+ i_unreached();
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME:
+ destbox = mailbox_alloc(change->ns->list,
+ change->rename_dest_name, 0);
+ mailbox_skip_create_name_restrictions(destbox, TRUE);
+ ret = mailbox_rename(box, destbox);
+ func_name = "mailbox_rename";
+ mailbox_free(&destbox);
+ break;
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE:
+ ret = mailbox_set_subscribed(box, TRUE);
+ func_name = "mailbox_set_subscribed";
+ break;
+ case DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE:
+ ret = mailbox_set_subscribed(box, FALSE);
+ func_name = "mailbox_set_subscribed";
+ break;
+ }
+ if (ret < 0) {
+ errstr = mailbox_get_last_internal_error(box, &error);
+ if (error == MAIL_ERROR_EXISTS ||
+ error == MAIL_ERROR_NOTFOUND) {
+ /* mailbox was already created or was already deleted.
+ let the next sync figure out what to do */
+ dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+ "Mailbox %s %s conflict: %s",
+ mailbox_get_vname(box), func_name, errstr));
+ ret = 0;
+ } else {
+ i_error("Mailbox %s sync: %s failed: %s",
+ mailbox_get_vname(box), func_name, errstr);
+ *error_r = error;
+ }
+ }
+ mailbox_free(&box);
+ return ret;
+}
diff --git a/src/doveadm/dsync/dsync-brain-mailbox-tree.c b/src/doveadm/dsync/dsync-brain-mailbox-tree.c
new file mode 100644
index 0000000..44b76bf
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mailbox-tree.c
@@ -0,0 +1,624 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "mail-namespace.h"
+#include "mailbox-list-private.h"
+#include "dsync-ibc.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-brain-private.h"
+
+#include <ctype.h>
+
+static void dsync_brain_check_namespaces(struct dsync_brain *brain)
+{
+ struct mail_namespace *ns, *first_ns = NULL;
+ char sep, escape_char;
+
+ i_assert(brain->hierarchy_sep == '\0');
+ i_assert(brain->escape_char == '\0');
+
+ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+ if (!dsync_brain_want_namespace(brain, ns))
+ continue;
+
+ sep = mail_namespace_get_sep(ns);
+ escape_char = mailbox_list_get_settings(ns->list)->vname_escape_char;
+ if (first_ns == NULL) {
+ brain->hierarchy_sep = sep;
+ brain->escape_char = escape_char;
+ first_ns = ns;
+ } else if (brain->hierarchy_sep != sep) {
+ i_fatal("Synced namespaces have conflicting separators "
+ "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")",
+ brain->hierarchy_sep, first_ns->prefix,
+ sep, ns->prefix);
+ } else if (brain->escape_char != escape_char) {
+ i_fatal("Synced namespaces have conflicting escape chars "
+ "('%c' for prefix=\"%s\", '%c' for prefix=\"%s\")",
+ brain->escape_char, first_ns->prefix,
+ escape_char, ns->prefix);
+ }
+ }
+ if (brain->hierarchy_sep != '\0')
+ return;
+
+ i_fatal("All your namespaces have a location setting. "
+ "Only namespaces with empty location settings are converted. "
+ "(One namespace should default to mail_location setting)");
+}
+
+void dsync_brain_mailbox_trees_init(struct dsync_brain *brain)
+{
+ struct mail_namespace *ns;
+
+ dsync_brain_check_namespaces(brain);
+
+ brain->local_mailbox_tree =
+ dsync_mailbox_tree_init(brain->hierarchy_sep,
+ brain->escape_char, brain->alt_char);
+ /* we'll convert remote mailbox names to use our own separator */
+ brain->remote_mailbox_tree =
+ dsync_mailbox_tree_init(brain->hierarchy_sep,
+ brain->escape_char, brain->alt_char);
+
+ /* fill the local mailbox tree */
+ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+ if (!dsync_brain_want_namespace(brain, ns))
+ continue;
+ if (brain->debug)
+ i_debug("brain %c: Namespace %s has location %s",
+ brain->master_brain ? 'M' : 'S',
+ ns->prefix, ns->set->location);
+ if (dsync_mailbox_tree_fill(brain->local_mailbox_tree, ns,
+ brain->sync_box,
+ brain->sync_box_guid,
+ brain->exclude_mailboxes,
+ &brain->mail_error) < 0) {
+ brain->failed = TRUE;
+ break;
+ }
+ }
+
+ brain->local_tree_iter =
+ dsync_mailbox_tree_iter_init(brain->local_mailbox_tree);
+}
+
+static const char *const *
+dsync_brain_mailbox_to_parts(struct dsync_brain *brain, const char *name)
+{
+ char sep[] = { brain->hierarchy_sep, '\0' };
+ char **parts = p_strsplit(unsafe_data_stack_pool, name, sep);
+ for (unsigned int i = 0; parts[i] != NULL; i++) {
+ mailbox_list_name_unescape((const char **)&parts[i],
+ brain->escape_char);
+ }
+ return (const char *const *)parts;
+}
+
+void dsync_brain_send_mailbox_tree(struct dsync_brain *brain)
+{
+ struct dsync_mailbox_node *node;
+ enum dsync_ibc_send_ret ret;
+ const char *full_name;
+
+ while (dsync_mailbox_tree_iter_next(brain->local_tree_iter,
+ &full_name, &node)) {
+ if (node->ns == NULL) {
+ /* This node was created when adding a namespace prefix
+ to the tree that has multiple hierarchical names,
+ but the parent names don't belong to any synced
+ namespace. For example when syncing "-n Shared/user/"
+ so "Shared/" is skipped. Or if there is e.g.
+ "Public/files/" namespace prefix, but no "Public/"
+ namespace at all. */
+ continue;
+ }
+
+ T_BEGIN {
+ const char *const *parts;
+
+ if (brain->debug) {
+ i_debug("brain %c: Local mailbox tree: %s %s",
+ brain->master_brain ? 'M' : 'S', full_name,
+ dsync_mailbox_node_to_string(node));
+ }
+
+ parts = dsync_brain_mailbox_to_parts(brain, full_name);
+ ret = dsync_ibc_send_mailbox_tree_node(brain->ibc,
+ parts, node);
+ } T_END;
+ if (ret == DSYNC_IBC_SEND_RET_FULL)
+ return;
+ }
+ dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
+ dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_TREE);
+
+ brain->state = DSYNC_STATE_SEND_MAILBOX_TREE_DELETES;
+}
+
+void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox_delete *deletes;
+ unsigned int count;
+
+ deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree,
+ &count);
+ dsync_ibc_send_mailbox_deletes(brain->ibc, deletes, count,
+ brain->hierarchy_sep,
+ brain->escape_char);
+
+ brain->state = DSYNC_STATE_RECV_MAILBOX_TREE;
+}
+
+static bool
+dsync_namespace_match_parts(struct mail_namespace *ns,
+ const char *const *name_parts)
+{
+ const char *part, *prefix = ns->prefix;
+ size_t part_len;
+ char ns_sep = mail_namespace_get_sep(ns);
+
+ if ((ns->flags & NAMESPACE_FLAG_INBOX_USER) != 0 &&
+ strcmp(name_parts[0], "INBOX") == 0 && name_parts[1] == NULL)
+ return TRUE;
+
+ for (; *name_parts != NULL && *prefix != '\0'; name_parts++) {
+ part = *name_parts;
+ part_len = strlen(part);
+
+ if (!str_begins(prefix, part))
+ return FALSE;
+ if (prefix[part_len] != ns_sep)
+ return FALSE;
+ prefix += part_len + 1;
+ }
+ if (*name_parts != NULL) {
+ /* namespace prefix found with a mailbox */
+ return TRUE;
+ }
+ if (*prefix == '\0') {
+ /* namespace prefix itself matched */
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static struct mail_namespace *
+dsync_find_namespace(struct dsync_brain *brain, const char *const *name_parts)
+{
+ struct mail_namespace *ns, *best_ns = NULL;
+
+ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+ if (!dsync_brain_want_namespace(brain, ns))
+ continue;
+
+ if (ns->prefix_len == 0) {
+ /* prefix="" is the fallback namespace */
+ if (best_ns == NULL)
+ best_ns = ns;
+ } else if (dsync_namespace_match_parts(ns, name_parts)) {
+ if (best_ns == NULL ||
+ best_ns->prefix_len < ns->prefix_len)
+ best_ns = ns;
+ }
+ }
+ return best_ns;
+}
+
+static bool
+dsync_is_valid_name(struct mail_namespace *ns, const char *vname)
+{
+ struct mailbox *box;
+ bool ret;
+
+ box = mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
+ ret = mailbox_verify_create_name(box) == 0;
+ mailbox_free(&box);
+ return ret;
+}
+
+static bool
+dsync_is_valid_name_until(struct mail_namespace *ns, string_t *vname_full,
+ unsigned int end_pos)
+{
+ const char *vname;
+ if (end_pos == str_len(vname_full))
+ vname = str_c(vname_full);
+ else
+ vname = t_strndup(str_c(vname_full), end_pos);
+ return dsync_is_valid_name(ns, vname);
+}
+
+static bool
+dsync_fix_mailbox_name_until(struct mail_namespace *ns, string_t *vname_full,
+ char alt_char, unsigned int start_pos,
+ unsigned int *_end_pos)
+{
+ unsigned int end_pos = *_end_pos;
+ unsigned int i;
+
+ if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+ return TRUE;
+
+ /* 1) change any real separators to alt separators (this
+ wouldn't be necessary with listescape, but don't bother
+ detecting it) */
+ char list_sep = mailbox_list_get_hierarchy_sep(ns->list);
+ char ns_sep = mail_namespace_get_sep(ns);
+ if (list_sep != ns_sep) {
+ char *v = str_c_modifiable(vname_full);
+ for (i = start_pos; i < end_pos; i++) {
+ if (v[i] == list_sep)
+ v[i] = alt_char;
+ }
+ if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+ return TRUE;
+ }
+
+ /* 2) '/' characters aren't valid without listescape */
+ if (ns_sep != '/' && list_sep != '/') {
+ char *v = str_c_modifiable(vname_full);
+ for (i = start_pos; i < end_pos; i++) {
+ if (v[i] == '/')
+ v[i] = alt_char;
+ }
+ if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+ return TRUE;
+ }
+
+ /* 3) probably some reserved name (e.g. dbox-Mails or ..) */
+ str_insert(vname_full, start_pos, "_"); end_pos++; *_end_pos += 1;
+ if (dsync_is_valid_name_until(ns, vname_full, end_pos))
+ return TRUE;
+
+ return FALSE;
+}
+
+static void
+dsync_fix_mailbox_name(struct mail_namespace *ns, string_t *vname_str,
+ char alt_char)
+{
+ const char *old_vname;
+ char *vname;
+ char ns_sep = mail_namespace_get_sep(ns);
+ guid_128_t guid;
+ unsigned int i, start_pos;
+
+ vname = str_c_modifiable(vname_str);
+ if (strncmp(vname, ns->prefix, ns->prefix_len) == 0)
+ start_pos = ns->prefix_len;
+ else
+ start_pos = 0;
+
+ /* replace control chars */
+ for (i = start_pos; vname[i] != '\0'; i++) {
+ if ((unsigned char)vname[i] < ' ')
+ vname[i] = alt_char;
+ }
+ /* make it valid UTF8 */
+ if (!uni_utf8_str_is_valid(vname)) {
+ old_vname = t_strdup(vname + start_pos);
+ str_truncate(vname_str, start_pos);
+ if (uni_utf8_get_valid_data((const void *)old_vname,
+ strlen(old_vname), vname_str))
+ i_unreached();
+ vname = str_c_modifiable(vname_str);
+ }
+ if (dsync_is_valid_name(ns, vname))
+ return;
+
+ /* Check/fix each hierarchical name separately */
+ const char *p;
+ do {
+ i_assert(start_pos <= str_len(vname_str));
+ p = strchr(str_c(vname_str) + start_pos, ns_sep);
+ unsigned int end_pos;
+ if (p == NULL)
+ end_pos = str_len(vname_str);
+ else
+ end_pos = p - str_c(vname_str);
+
+ if (!dsync_fix_mailbox_name_until(ns, vname_str, alt_char,
+ start_pos, &end_pos)) {
+ /* Couldn't fix it. Name is too long? Just give up and
+ generate a unique name. */
+ guid_128_generate(guid);
+ str_truncate(vname_str, 0);
+ str_append(vname_str, ns->prefix);
+ str_append(vname_str, guid_128_to_string(guid));
+ i_assert(dsync_is_valid_name(ns, str_c(vname_str)));
+ break;
+ }
+ start_pos = end_pos + 1;
+ } while (p != NULL);
+}
+
+static int
+dsync_get_mailbox_name(struct dsync_brain *brain, const char *const *name_parts,
+ const char **name_r, struct mail_namespace **ns_r)
+{
+ struct mail_namespace *ns;
+ const char *p;
+ string_t *vname;
+ char ns_sep;
+
+ i_assert(*name_parts != NULL);
+
+ ns = dsync_find_namespace(brain, name_parts);
+ if (ns == NULL)
+ return -1;
+ ns_sep = mail_namespace_get_sep(ns);
+
+ /* build the mailbox name */
+ char escape_chars[] = {
+ brain->escape_char,
+ ns_sep,
+ '\0'
+ };
+ struct dsync_mailbox_list *dlist = DSYNC_LIST_CONTEXT(ns->list);
+ if (dlist != NULL && !dlist->have_orig_escape_char) {
+ /* The escape character was added only for dsync internally.
+ Normally there is no escape character configured. Change
+ the mailbox names so that it doesn't rely on it. */
+ escape_chars[0] = '\0';
+ }
+ vname = t_str_new(128);
+ for (; *name_parts != NULL; name_parts++) {
+ if (escape_chars[0] != '\0') {
+ mailbox_list_name_escape(*name_parts, escape_chars,
+ vname);
+ } else {
+ for (p = *name_parts; *p != '\0'; p++) {
+ if (*p != ns_sep)
+ str_append_c(vname, *p);
+ else
+ str_append_c(vname, brain->alt_char);
+ }
+ }
+ str_append_c(vname, ns_sep);
+ }
+ str_truncate(vname, str_len(vname)-1);
+
+ dsync_fix_mailbox_name(ns, vname, brain->alt_char);
+ *name_r = str_c(vname);
+ *ns_r = ns;
+ return 0;
+}
+
+static void dsync_brain_mailbox_trees_sync(struct dsync_brain *brain)
+{
+ struct dsync_mailbox_tree_sync_ctx *ctx;
+ const struct dsync_mailbox_tree_sync_change *change;
+ enum dsync_mailbox_trees_sync_type sync_type;
+ enum dsync_mailbox_trees_sync_flags sync_flags =
+ (brain->debug ? DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG : 0) |
+ (brain->master_brain ? DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN : 0);
+ int ret;
+
+ if (brain->no_backup_overwrite)
+ sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY;
+ else if (brain->backup_send)
+ sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL;
+ else if (brain->backup_recv)
+ sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE;
+ else
+ sync_type = DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY;
+
+ ctx = dsync_mailbox_trees_sync_init(brain->local_mailbox_tree,
+ brain->remote_mailbox_tree,
+ sync_type, sync_flags);
+ while ((change = dsync_mailbox_trees_sync_next(ctx)) != NULL) {
+ T_BEGIN {
+ ret = dsync_brain_mailbox_tree_sync_change(
+ brain, change, &brain->mail_error);
+ } T_END;
+ if (ret < 0) {
+ brain->failed = TRUE;
+ break;
+ }
+ }
+ if (dsync_mailbox_trees_sync_deinit(&ctx) < 0)
+ brain->failed = TRUE;
+}
+
+static int
+dsync_brain_recv_mailbox_tree_add(struct dsync_brain *brain,
+ const char *const *parts,
+ const struct dsync_mailbox_node *remote_node,
+ const char *sep)
+{
+ struct dsync_mailbox_node *node;
+ struct mail_namespace *ns;
+ const char *name;
+
+ if (dsync_get_mailbox_name(brain, parts, &name, &ns) < 0)
+ return -1;
+ if (brain->debug) {
+ i_debug("brain %c: Remote mailbox tree: %s %s",
+ brain->master_brain ? 'M' : 'S',
+ t_strarray_join(parts, sep),
+ dsync_mailbox_node_to_string(remote_node));
+ }
+ node = dsync_mailbox_tree_get(brain->remote_mailbox_tree, name);
+ node->ns = ns;
+ dsync_mailbox_node_copy_data(node, remote_node);
+ return 0;
+}
+
+bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox_node *remote_node;
+ struct dsync_mailbox_node *dup_node1, *dup_node2;
+ const char *const *parts;
+ enum dsync_ibc_recv_ret ret;
+ int ret2;
+ char sep[2];
+ bool changed = FALSE;
+
+ sep[0] = brain->hierarchy_sep; sep[1] = '\0';
+ while ((ret = dsync_ibc_recv_mailbox_tree_node(brain->ibc, &parts,
+ &remote_node)) > 0) {
+ T_BEGIN {
+ ret2 = dsync_brain_recv_mailbox_tree_add(
+ brain, parts, remote_node, sep);
+ } T_END;
+ if (ret2 < 0) {
+ i_error("Couldn't find namespace for mailbox %s",
+ t_strarray_join(parts, sep));
+ brain->failed = TRUE;
+ return TRUE;
+ }
+ }
+ if (ret != DSYNC_IBC_RECV_RET_FINISHED)
+ return changed;
+
+ if (dsync_mailbox_tree_build_guid_hash(brain->remote_mailbox_tree,
+ &dup_node1, &dup_node2) < 0) {
+ i_error("Remote sent duplicate mailbox GUID %s for mailboxes %s and %s",
+ guid_128_to_string(dup_node1->mailbox_guid),
+ dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree,
+ dup_node1),
+ dsync_mailbox_node_get_full_name(brain->remote_mailbox_tree,
+ dup_node2));
+ brain->failed = TRUE;
+ }
+
+ brain->state = DSYNC_STATE_RECV_MAILBOX_TREE_DELETES;
+ return TRUE;
+}
+
+static void
+dsync_brain_mailbox_tree_add_delete(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_tree *other_tree,
+ const struct dsync_mailbox_delete *other_del,
+ const struct dsync_mailbox_node **node_r,
+ const char **status_r)
+{
+ const struct dsync_mailbox_node *node;
+ struct dsync_mailbox_node *other_node, *old_node;
+ const char *name;
+
+ /* see if we can find the deletion based on mailbox tree that should
+ still have the mailbox */
+ node = *node_r = dsync_mailbox_tree_find_delete(tree, other_del);
+ if (node == NULL) {
+ *status_r = "not found";
+ return;
+ }
+
+ switch (other_del->type) {
+ case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX:
+ /* mailbox is always deleted */
+ break;
+ case DSYNC_MAILBOX_DELETE_TYPE_DIR:
+ if (other_del->timestamp <= node->last_renamed_or_created) {
+ /* we don't want to delete this directory, we already
+ have a newer timestamp for it */
+ *status_r = "keep directory, we have a newer timestamp";
+ return;
+ }
+ break;
+ case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE:
+ if (other_del->timestamp <= node->last_subscription_change) {
+ /* we don't want to unsubscribe, since we already have
+ a newer subscription timestamp */
+ *status_r = "keep subscription, we have a newer timestamp";
+ return;
+ }
+ break;
+ }
+
+ /* make a node for it in the other mailbox tree */
+ name = dsync_mailbox_node_get_full_name(tree, node);
+ other_node = dsync_mailbox_tree_get(other_tree, name);
+
+ if (other_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ (!guid_128_is_empty(other_node->mailbox_guid) ||
+ other_del->type != DSYNC_MAILBOX_DELETE_TYPE_MAILBOX)) {
+ /* other side has already created a new mailbox or
+ directory with this name, we can't delete it */
+ *status_r = "name has already been recreated";
+ return;
+ }
+
+ /* ok, mark the other node deleted */
+ if (other_del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) {
+ memcpy(other_node->mailbox_guid, node->mailbox_guid,
+ sizeof(other_node->mailbox_guid));
+ }
+ if (other_node->ns != node->ns && other_node->ns != NULL) {
+ /* namespace mismatch for this node. this shouldn't happen
+ normally, but especially during some misconfigurations it's
+ possible that one side has created mailboxes that conflict
+ with another namespace's prefix. since we're here because
+ one of the mailboxes was deleted, we'll just ignore this. */
+ *status_r = "namespace mismatch";
+ return;
+ }
+ other_node->ns = node->ns;
+ if (other_del->type != DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) {
+ other_node->existence = DSYNC_MAILBOX_NODE_DELETED;
+ *status_r = "marked as deleted";
+ } else {
+ other_node->last_subscription_change = other_del->timestamp;
+ other_node->subscribed = FALSE;
+ *status_r = "marked as unsubscribed";
+ }
+
+ if (dsync_mailbox_tree_guid_hash_add(other_tree, other_node,
+ &old_node) < 0)
+ i_unreached();
+}
+
+bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox_node *node;
+ const char *status;
+ const struct dsync_mailbox_delete *deletes;
+ unsigned int i, count;
+ char sep, escape_char;
+
+ if (dsync_ibc_recv_mailbox_deletes(brain->ibc, &deletes, &count,
+ &sep, &escape_char) == 0)
+ return FALSE;
+
+ /* apply remote's mailbox deletions based on our local tree */
+ dsync_mailbox_tree_set_remote_chars(brain->local_mailbox_tree, sep,
+ escape_char);
+ for (i = 0; i < count; i++) {
+ dsync_brain_mailbox_tree_add_delete(brain->local_mailbox_tree,
+ brain->remote_mailbox_tree,
+ &deletes[i], &node, &status);
+ if (brain->debug) {
+ const char *node_name = node == NULL ? "" :
+ dsync_mailbox_node_get_full_name(brain->local_mailbox_tree, node);
+ i_debug("brain %c: Remote mailbox tree deletion: guid=%s type=%s timestamp=%ld name=%s local update=%s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(deletes[i].guid),
+ dsync_mailbox_delete_type_to_string(deletes[i].type),
+ deletes[i].timestamp, node_name, status);
+ }
+ }
+
+ /* apply local mailbox deletions based on remote tree */
+ deletes = dsync_mailbox_tree_get_deletes(brain->local_mailbox_tree,
+ &count);
+ dsync_mailbox_tree_set_remote_chars(brain->remote_mailbox_tree,
+ brain->hierarchy_sep,
+ brain->escape_char);
+ for (i = 0; i < count; i++) {
+ dsync_brain_mailbox_tree_add_delete(brain->remote_mailbox_tree,
+ brain->local_mailbox_tree,
+ &deletes[i], &node, &status);
+ }
+
+ dsync_brain_mailbox_trees_sync(brain);
+ brain->state = brain->master_brain ?
+ DSYNC_STATE_MASTER_SEND_MAILBOX :
+ DSYNC_STATE_SLAVE_RECV_MAILBOX;
+ i_assert(brain->local_tree_iter == NULL);
+ brain->local_tree_iter =
+ dsync_mailbox_tree_iter_init(brain->local_mailbox_tree);
+ return TRUE;
+}
diff --git a/src/doveadm/dsync/dsync-brain-mailbox.c b/src/doveadm/dsync/dsync-brain-mailbox.c
new file mode 100644
index 0000000..759018f
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mailbox.c
@@ -0,0 +1,923 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "mail-cache-private.h"
+#include "mail-namespace.h"
+#include "mail-storage-private.h"
+#include "dsync-ibc.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-mailbox-import.h"
+#include "dsync-mailbox-export.h"
+#include "dsync-transaction-log-scan.h"
+#include "dsync-brain-private.h"
+
+static int
+ns_mailbox_try_alloc(struct dsync_brain *brain, struct mail_namespace *ns,
+ const guid_128_t guid, struct mailbox **box_r,
+ const char **errstr_r, enum mail_error *error_r)
+{
+ enum mailbox_flags flags = 0;
+ struct mailbox *box;
+ enum mailbox_existence existence;
+ int ret;
+
+ if (brain->backup_send) {
+ /* make sure mailbox isn't modified */
+ flags |= MAILBOX_FLAG_READONLY;
+ }
+
+ box = mailbox_alloc_guid(ns->list, guid, flags);
+ ret = mailbox_exists(box, FALSE, &existence);
+ if (ret < 0) {
+ *errstr_r = mailbox_get_last_internal_error(box, error_r);
+ mailbox_free(&box);
+ return -1;
+ }
+ if (existence != MAILBOX_EXISTENCE_SELECT) {
+ mailbox_free(&box);
+ *errstr_r = existence == MAILBOX_EXISTENCE_NONE ?
+ "Mailbox was already deleted" :
+ "Mailbox is no longer selectable";
+ return 0;
+ }
+ *box_r = box;
+ return 1;
+}
+
+int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
+ struct mailbox **box_r, const char **errstr_r,
+ enum mail_error *error_r)
+{
+ struct mail_namespace *ns;
+ int ret;
+
+ *box_r = NULL;
+
+ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+ if (!dsync_brain_want_namespace(brain, ns))
+ continue;
+ if ((ret = ns_mailbox_try_alloc(brain, ns, guid, box_r,
+ errstr_r, error_r)) != 0)
+ return ret;
+ }
+ return 0;
+}
+
+static void
+dsync_mailbox_cache_field_dup(ARRAY_TYPE(mailbox_cache_field) *dest,
+ const ARRAY_TYPE(mailbox_cache_field) *src,
+ pool_t pool)
+{
+ const struct mailbox_cache_field *src_field;
+ struct mailbox_cache_field *dest_field;
+
+ p_array_init(dest, pool, array_count(src));
+ array_foreach(src, src_field) {
+ dest_field = array_append_space(dest);
+ dest_field->name = p_strdup(pool, src_field->name);
+ dest_field->decision = src_field->decision;
+ dest_field->last_used = src_field->last_used;
+ }
+}
+
+
+static const struct dsync_mailbox_state *
+dsync_mailbox_state_find(struct dsync_brain *brain,
+ const guid_128_t mailbox_guid)
+{
+ const uint8_t *guid_p;
+
+ guid_p = mailbox_guid;
+ return hash_table_lookup(brain->mailbox_states, guid_p);
+}
+
+static void
+dsync_mailbox_state_remove(struct dsync_brain *brain,
+ const guid_128_t mailbox_guid)
+{
+ const uint8_t *guid_p;
+
+ guid_p = mailbox_guid;
+ if (hash_table_lookup(brain->mailbox_states, guid_p) != NULL)
+ hash_table_remove(brain->mailbox_states, guid_p);
+}
+
+void dsync_brain_sync_init_box_states(struct dsync_brain *brain)
+{
+ if (brain->backup_send) {
+ /* we have an exporter, but no importer. */
+ brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES;
+ brain->box_recv_state = brain->mail_requests ?
+ DSYNC_BOX_STATE_MAIL_REQUESTS :
+ DSYNC_BOX_STATE_RECV_LAST_COMMON;
+ } else if (brain->backup_recv) {
+ /* we have an importer, but no exporter */
+ brain->box_send_state = brain->mail_requests ?
+ DSYNC_BOX_STATE_MAIL_REQUESTS :
+ DSYNC_BOX_STATE_DONE;
+ brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES;
+ } else {
+ brain->box_send_state = DSYNC_BOX_STATE_ATTRIBUTES;
+ brain->box_recv_state = DSYNC_BOX_STATE_ATTRIBUTES;
+ }
+}
+
+static void
+dsync_brain_sync_mailbox_init(struct dsync_brain *brain,
+ struct mailbox *box,
+ struct file_lock *lock,
+ const struct dsync_mailbox *local_dsync_box,
+ bool wait_for_remote_box)
+{
+ const struct dsync_mailbox_state *state;
+
+ i_assert(brain->box_importer == NULL);
+ i_assert(brain->box_exporter == NULL);
+ i_assert(box->synced);
+
+ brain->box = box;
+ brain->box_lock = lock;
+ brain->pre_box_state = brain->state;
+ if (wait_for_remote_box) {
+ brain->box_send_state = DSYNC_BOX_STATE_MAILBOX;
+ brain->box_recv_state = DSYNC_BOX_STATE_MAILBOX;
+ } else {
+ dsync_brain_sync_init_box_states(brain);
+ }
+ brain->local_dsync_box = *local_dsync_box;
+ if (brain->dsync_box_pool != NULL)
+ p_clear(brain->dsync_box_pool);
+ else {
+ brain->dsync_box_pool =
+ pool_alloconly_create(MEMPOOL_GROWING"dsync brain box pool", 2048);
+ }
+ dsync_mailbox_cache_field_dup(&brain->local_dsync_box.cache_fields,
+ &local_dsync_box->cache_fields,
+ brain->dsync_box_pool);
+ i_zero(&brain->remote_dsync_box);
+
+ state = dsync_mailbox_state_find(brain, local_dsync_box->mailbox_guid);
+ if (state != NULL)
+ brain->mailbox_state = *state;
+ else {
+ i_zero(&brain->mailbox_state);
+ memcpy(brain->mailbox_state.mailbox_guid,
+ local_dsync_box->mailbox_guid,
+ sizeof(brain->mailbox_state.mailbox_guid));
+ brain->mailbox_state.last_uidvalidity =
+ local_dsync_box->uid_validity;
+ }
+}
+
+static void
+dsync_brain_sync_mailbox_init_remote(struct dsync_brain *brain,
+ const struct dsync_mailbox *remote_dsync_box)
+{
+ enum dsync_mailbox_import_flags import_flags = 0;
+ const struct dsync_mailbox_state *state;
+ uint32_t last_common_uid;
+ uint64_t last_common_modseq, last_common_pvt_modseq;
+
+ i_assert(brain->box_importer == NULL);
+ i_assert(brain->log_scan != NULL);
+
+ i_assert(memcmp(brain->local_dsync_box.mailbox_guid,
+ remote_dsync_box->mailbox_guid,
+ sizeof(remote_dsync_box->mailbox_guid)) == 0);
+
+ brain->remote_dsync_box = *remote_dsync_box;
+ dsync_mailbox_cache_field_dup(&brain->remote_dsync_box.cache_fields,
+ &remote_dsync_box->cache_fields,
+ brain->dsync_box_pool);
+
+ state = dsync_mailbox_state_find(brain, remote_dsync_box->mailbox_guid);
+ if (state != NULL) {
+ last_common_uid = state->last_common_uid;
+ last_common_modseq = state->last_common_modseq;
+ last_common_pvt_modseq = state->last_common_pvt_modseq;
+ } else {
+ last_common_uid = 0;
+ last_common_modseq = 0;
+ last_common_pvt_modseq = 0;
+ }
+
+ if (brain->mail_requests)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS;
+ if (brain->master_brain)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN;
+ if (brain->backup_recv && !brain->no_backup_overwrite)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES;
+ if (brain->debug)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_DEBUG;
+ if (brain->local_dsync_box.have_save_guids &&
+ (remote_dsync_box->have_save_guids ||
+ (brain->backup_recv && remote_dsync_box->have_guids)))
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS;
+ if (brain->local_dsync_box.have_only_guid128 ||
+ remote_dsync_box->have_only_guid128)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128;
+ if (brain->no_notify)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY;
+ if (brain->empty_hdr_workaround)
+ import_flags |= DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND;
+
+ brain->box_importer = brain->backup_send ? NULL :
+ dsync_mailbox_import_init(brain->box, brain->virtual_all_box,
+ brain->log_scan,
+ last_common_uid, last_common_modseq,
+ last_common_pvt_modseq,
+ remote_dsync_box->uid_next,
+ remote_dsync_box->first_recent_uid,
+ remote_dsync_box->highest_modseq,
+ remote_dsync_box->highest_pvt_modseq,
+ brain->sync_since_timestamp,
+ brain->sync_until_timestamp,
+ brain->sync_max_size,
+ brain->sync_flag,
+ brain->import_commit_msgs_interval,
+ import_flags, brain->hdr_hash_version,
+ brain->hashed_headers);
+}
+
+int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
+ const struct dsync_mailbox *remote_dsync_box)
+{
+ struct mailbox_status status;
+ enum dsync_mailbox_exporter_flags exporter_flags = 0;
+ uint32_t last_common_uid, highest_wanted_uid;
+ uint64_t last_common_modseq, last_common_pvt_modseq;
+ const char *desync_reason = "";
+ bool pvt_too_old;
+ int ret;
+
+ i_assert(brain->log_scan == NULL);
+ i_assert(brain->box_exporter == NULL);
+
+ last_common_uid = brain->mailbox_state.last_common_uid;
+ last_common_modseq = brain->mailbox_state.last_common_modseq;
+ last_common_pvt_modseq = brain->mailbox_state.last_common_pvt_modseq;
+ highest_wanted_uid = last_common_uid == 0 ?
+ (uint32_t)-1 : last_common_uid;
+ ret = dsync_transaction_log_scan_init(brain->box->view,
+ brain->box->view_pvt,
+ highest_wanted_uid,
+ last_common_modseq,
+ last_common_pvt_modseq,
+ &brain->log_scan, &pvt_too_old);
+ if (ret < 0) {
+ i_error("Failed to read transaction log for mailbox %s",
+ mailbox_get_vname(brain->box));
+ brain->failed = TRUE;
+ return -1;
+ }
+
+ mailbox_get_open_status(brain->box, STATUS_UIDNEXT |
+ STATUS_HIGHESTMODSEQ |
+ STATUS_HIGHESTPVTMODSEQ, &status);
+ if (status.nonpermanent_modseqs)
+ status.highest_modseq = 0;
+ if (ret == 0) {
+ if (pvt_too_old) {
+ desync_reason = t_strdup_printf(
+ "Private modseq %"PRIu64" no longer in transaction log "
+ "(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)",
+ last_common_pvt_modseq,
+ status.highest_pvt_modseq, last_common_uid,
+ status.uidnext);
+ } else {
+ desync_reason = t_strdup_printf(
+ "Modseq %"PRIu64" no longer in transaction log "
+ "(highest=%"PRIu64", last_common_uid=%u, nextuid=%u)",
+ last_common_modseq,
+ status.highest_modseq, last_common_uid,
+ status.uidnext);
+ }
+ }
+
+ if (last_common_uid != 0) {
+ /* if last_common_* is higher than our current ones it means
+ that the incremental sync state is stale and we need to do
+ a full resync */
+ if (status.uidnext < last_common_uid) {
+ desync_reason = t_strdup_printf("uidnext %u < %u",
+ status.uidnext, last_common_uid);
+ ret = 0;
+ } else if (status.highest_modseq < last_common_modseq) {
+ desync_reason = t_strdup_printf("highest_modseq %"PRIu64" < %"PRIu64,
+ status.highest_modseq, last_common_modseq);
+ ret = 0;
+ } else if (status.highest_pvt_modseq < last_common_pvt_modseq) {
+ desync_reason = t_strdup_printf("highest_pvt_modseq %"PRIu64" < %"PRIu64,
+ status.highest_pvt_modseq, last_common_pvt_modseq);
+ ret = 0;
+ }
+ }
+ if (ret == 0) {
+ i_warning("Failed to do incremental sync for mailbox %s, "
+ "retry with a full sync (%s)",
+ mailbox_get_vname(brain->box), desync_reason);
+ dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+ "Incremental sync failed: %s", desync_reason));
+ brain->require_full_resync = TRUE;
+ return 0;
+ }
+
+ if (!brain->mail_requests)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS;
+ if (remote_dsync_box->have_save_guids &&
+ (brain->local_dsync_box.have_save_guids ||
+ (brain->backup_send && brain->local_dsync_box.have_guids)))
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS;
+ if (brain->no_mail_prefetch)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL;
+ if (brain->sync_since_timestamp > 0 ||
+ brain->sync_until_timestamp > 0)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS;
+ if (brain->sync_max_size > 0)
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES;
+ if (remote_dsync_box->messages_count == 0) {
+ /* remote mailbox is empty - we don't really need to export
+ header hashes since they're not going to match anything
+ anyway. */
+ exporter_flags |= DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES;
+ }
+
+ brain->box_exporter = brain->backup_recv ? NULL :
+ dsync_mailbox_export_init(brain->box, brain->log_scan,
+ last_common_uid,
+ exporter_flags,
+ brain->hdr_hash_version,
+ brain->hashed_headers);
+ dsync_brain_sync_mailbox_init_remote(brain, remote_dsync_box);
+ return 1;
+}
+
+void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain)
+{
+ enum mail_error error;
+
+ i_assert(brain->box != NULL);
+
+ array_push_back(&brain->remote_mailbox_states, &brain->mailbox_state);
+ if (brain->box_exporter != NULL) {
+ const char *errstr;
+
+ i_assert(brain->failed || brain->require_full_resync ||
+ brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_CHANGED);
+ if (dsync_mailbox_export_deinit(&brain->box_exporter,
+ &errstr, &error) < 0)
+ i_error("Mailbox export failed: %s", errstr);
+ }
+ if (brain->box_importer != NULL) {
+ uint32_t last_common_uid, last_messages_count;
+ uint64_t last_common_modseq, last_common_pvt_modseq;
+ const char *changes_during_sync;
+ bool require_full_resync;
+
+ i_assert(brain->failed);
+ (void)dsync_mailbox_import_deinit(&brain->box_importer,
+ FALSE,
+ &last_common_uid,
+ &last_common_modseq,
+ &last_common_pvt_modseq,
+ &last_messages_count,
+ &changes_during_sync,
+ &require_full_resync,
+ &brain->mail_error);
+ if (require_full_resync)
+ brain->require_full_resync = TRUE;
+ }
+ if (brain->log_scan != NULL)
+ dsync_transaction_log_scan_deinit(&brain->log_scan);
+ file_lock_free(&brain->box_lock);
+ mailbox_free(&brain->box);
+
+ brain->state = brain->pre_box_state;
+}
+
+static int dsync_box_get(struct mailbox *box, struct dsync_mailbox *dsync_box_r,
+ enum mail_error *error_r)
+{
+ const enum mailbox_status_items status_items =
+ STATUS_UIDVALIDITY | STATUS_UIDNEXT | STATUS_MESSAGES |
+ STATUS_FIRST_RECENT_UID | STATUS_HIGHESTMODSEQ |
+ STATUS_HIGHESTPVTMODSEQ;
+ const enum mailbox_metadata_items metadata_items =
+ MAILBOX_METADATA_CACHE_FIELDS | MAILBOX_METADATA_GUID;
+ struct mailbox_status status;
+ struct mailbox_metadata metadata;
+ const char *errstr;
+ enum mail_error error;
+
+ /* get metadata first, since it may autocreate the mailbox */
+ if (mailbox_get_metadata(box, metadata_items, &metadata) < 0 ||
+ mailbox_get_status(box, status_items, &status) < 0) {
+ errstr = mailbox_get_last_internal_error(box, &error);
+ if (error == MAIL_ERROR_NOTFOUND ||
+ error == MAIL_ERROR_NOTPOSSIBLE) {
+ /* Mailbox isn't selectable, try the next one. We
+ should have already caught \Noselect mailboxes, but
+ check them anyway here. The NOTPOSSIBLE check is
+ mainly for invalid mbox files. */
+ return 0;
+ }
+ i_error("Failed to access mailbox %s: %s",
+ mailbox_get_vname(box), errstr);
+ *error_r = error;
+ return -1;
+ }
+ if (status.nonpermanent_modseqs)
+ status.highest_modseq = 0;
+
+ i_assert(status.uidvalidity != 0 || status.messages == 0);
+
+ i_zero(dsync_box_r);
+ memcpy(dsync_box_r->mailbox_guid, metadata.guid,
+ sizeof(dsync_box_r->mailbox_guid));
+ dsync_box_r->uid_validity = status.uidvalidity;
+ dsync_box_r->uid_next = status.uidnext;
+ dsync_box_r->messages_count = status.messages;
+ dsync_box_r->first_recent_uid = status.first_recent_uid;
+ dsync_box_r->highest_modseq = status.highest_modseq;
+ dsync_box_r->highest_pvt_modseq = status.highest_pvt_modseq;
+ dsync_mailbox_cache_field_dup(&dsync_box_r->cache_fields,
+ metadata.cache_fields,
+ pool_datastack_create());
+ dsync_box_r->have_guids = status.have_guids;
+ dsync_box_r->have_save_guids = status.have_save_guids;
+ dsync_box_r->have_only_guid128 = status.have_only_guid128;
+ return 1;
+}
+
+static bool
+dsync_brain_has_mailbox_state_changed(struct dsync_brain *brain,
+ const struct dsync_mailbox *dsync_box)
+{
+ const struct dsync_mailbox_state *state;
+
+ if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE)
+ return TRUE;
+
+ state = dsync_mailbox_state_find(brain, dsync_box->mailbox_guid);
+ return state == NULL ||
+ state->last_uidvalidity != dsync_box->uid_validity ||
+ state->last_common_uid+1 != dsync_box->uid_next ||
+ state->last_common_modseq != dsync_box->highest_modseq ||
+ state->last_common_pvt_modseq != dsync_box->highest_pvt_modseq ||
+ state->last_messages_count != dsync_box->messages_count;
+}
+
+static int
+dsync_brain_try_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r,
+ struct file_lock **lock_r,
+ struct dsync_mailbox *dsync_box_r)
+{
+ enum mailbox_flags flags = 0;
+ struct dsync_mailbox dsync_box;
+ struct mailbox *box;
+ struct file_lock *lock = NULL;
+ struct dsync_mailbox_node *node;
+ const char *vname = NULL;
+ enum mail_error error;
+ bool synced = FALSE;
+ int ret;
+
+ *box_r = NULL;
+
+ while (dsync_mailbox_tree_iter_next(brain->local_tree_iter, &vname, &node)) {
+ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ !guid_128_is_empty(node->mailbox_guid))
+ break;
+ vname = NULL;
+ }
+ if (vname == NULL) {
+ /* no more mailboxes */
+ dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
+ return -1;
+ }
+
+ if (brain->backup_send) {
+ /* make sure mailbox isn't modified */
+ flags |= MAILBOX_FLAG_READONLY;
+ }
+ box = mailbox_alloc(node->ns->list, vname, flags);
+ for (;;) {
+ if ((ret = dsync_box_get(box, &dsync_box, &error)) <= 0) {
+ if (ret < 0) {
+ brain->mail_error = error;
+ brain->failed = TRUE;
+ }
+ mailbox_free(&box);
+ file_lock_free(&lock);
+ return ret;
+ }
+
+ /* if mailbox's last_common_* state equals the current state,
+ we can skip the mailbox */
+ if (!dsync_brain_has_mailbox_state_changed(brain, &dsync_box)) {
+ if (brain->debug) {
+ i_debug("brain %c: Skipping mailbox %s with unchanged state "
+ "uidvalidity=%u uidnext=%u highestmodseq=%"PRIu64" highestpvtmodseq=%"PRIu64" messages=%u",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box.mailbox_guid),
+ dsync_box.uid_validity,
+ dsync_box.uid_next,
+ dsync_box.highest_modseq,
+ dsync_box.highest_pvt_modseq,
+ dsync_box.messages_count);
+ }
+ mailbox_free(&box);
+ file_lock_free(&lock);
+ return 0;
+ }
+ if (synced) {
+ /* ok, the mailbox really changed */
+ break;
+ }
+
+ /* mailbox appears to have changed. do a full sync here and get the
+ state again. Lock before syncing. */
+ if (dsync_mailbox_lock(brain, box, &lock) < 0) {
+ brain->failed = TRUE;
+ mailbox_free(&box);
+ return -1;
+ }
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ i_error("Can't sync mailbox %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &brain->mail_error));
+ brain->failed = TRUE;
+ mailbox_free(&box);
+ file_lock_free(&lock);
+ return -1;
+ }
+ synced = TRUE;
+ }
+
+ *box_r = box;
+ *lock_r = lock;
+ *dsync_box_r = dsync_box;
+ return 1;
+}
+
+static bool
+dsync_brain_next_mailbox(struct dsync_brain *brain, struct mailbox **box_r,
+ struct file_lock **lock_r,
+ struct dsync_mailbox *dsync_box_r)
+{
+ int ret;
+
+ if (brain->no_mail_sync)
+ return FALSE;
+
+ while ((ret = dsync_brain_try_next_mailbox(brain, box_r, lock_r, dsync_box_r)) == 0)
+ ;
+ return ret > 0;
+}
+
+void dsync_brain_master_send_mailbox(struct dsync_brain *brain)
+{
+ struct dsync_mailbox dsync_box;
+ struct mailbox *box;
+ struct file_lock *lock;
+
+ i_assert(brain->master_brain);
+ i_assert(brain->box == NULL);
+
+ if (!dsync_brain_next_mailbox(brain, &box, &lock, &dsync_box)) {
+ brain->state = DSYNC_STATE_FINISH;
+ dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX);
+ return;
+ }
+
+ /* start exporting this mailbox (wait for remote to start importing) */
+ dsync_ibc_send_mailbox(brain->ibc, &dsync_box);
+ dsync_brain_sync_mailbox_init(brain, box, lock, &dsync_box, TRUE);
+ brain->state = DSYNC_STATE_SYNC_MAILS;
+}
+
+bool dsync_boxes_need_sync(struct dsync_brain *brain,
+ const struct dsync_mailbox *box1,
+ const struct dsync_mailbox *box2,
+ const char **reason_r)
+{
+ if (brain->no_mail_sync)
+ return FALSE;
+ if (brain->sync_type != DSYNC_BRAIN_SYNC_TYPE_CHANGED) {
+ *reason_r = "Full sync";
+ return TRUE;
+ }
+ if (box1->uid_validity != box2->uid_validity)
+ *reason_r = t_strdup_printf("UIDVALIDITY changed: %u -> %u",
+ box1->uid_validity, box2->uid_validity);
+ else if (box1->uid_next != box2->uid_next)
+ *reason_r = t_strdup_printf("UIDNEXT changed: %u -> %u",
+ box1->uid_next, box2->uid_next);
+ else if (box1->messages_count != box2->messages_count)
+ *reason_r = t_strdup_printf("Message count changed: %u -> %u",
+ box1->messages_count, box2->messages_count);
+ else if (box1->highest_modseq != box2->highest_modseq) {
+ *reason_r = t_strdup_printf("HIGHESTMODSEQ changed %"
+ PRIu64" -> %"PRIu64,
+ box1->highest_modseq,
+ box2->highest_modseq);
+ if (box1->highest_modseq == 0 ||
+ box2->highest_modseq == 0) {
+ *reason_r = t_strdup_printf(
+ "%s (Permanent MODSEQs aren't supported)",
+ *reason_r);
+ }
+ } else if (box1->highest_pvt_modseq != box2->highest_pvt_modseq)
+ *reason_r = t_strdup_printf("Private HIGHESTMODSEQ changed %"
+ PRIu64" -> %"PRIu64,
+ box1->highest_pvt_modseq,
+ box2->highest_pvt_modseq);
+ else if (box1->first_recent_uid != box2->first_recent_uid)
+ *reason_r = t_strdup_printf("First RECENT UID changed: %u -> %u",
+ box1->first_recent_uid, box2->first_recent_uid);
+ else
+ return FALSE;
+ return TRUE;
+}
+
+static int
+mailbox_cache_field_name_cmp(const struct mailbox_cache_field *f1,
+ const struct mailbox_cache_field *f2)
+{
+ return strcmp(f1->name, f2->name);
+}
+
+static void
+dsync_cache_fields_update(const struct dsync_mailbox *local_box,
+ const struct dsync_mailbox *remote_box,
+ struct mailbox *box,
+ struct mailbox_update *update)
+{
+ ARRAY_TYPE(mailbox_cache_field) local_sorted, remote_sorted, changes;
+ const struct mailbox_cache_field *local_fields, *remote_fields;
+ unsigned int li, ri, local_count, remote_count;
+ time_t drop_older_timestamp;
+ int ret;
+
+ if (array_count(&remote_box->cache_fields) == 0) {
+ /* remote has no cached fields. there's nothing to update. */
+ return;
+ }
+
+ t_array_init(&local_sorted, array_count(&local_box->cache_fields));
+ t_array_init(&remote_sorted, array_count(&remote_box->cache_fields));
+ array_append_array(&local_sorted, &local_box->cache_fields);
+ array_append_array(&remote_sorted, &remote_box->cache_fields);
+ array_sort(&local_sorted, mailbox_cache_field_name_cmp);
+ array_sort(&remote_sorted, mailbox_cache_field_name_cmp);
+
+ if (array_count(&local_sorted) == 0) {
+ /* local has no cached fields. set them to same as remote. */
+ array_append_zero(&remote_sorted);
+ update->cache_updates = array_front(&remote_sorted);
+ return;
+ }
+
+ /* figure out what to change */
+ local_fields = array_get(&local_sorted, &local_count);
+ remote_fields = array_get(&remote_sorted, &remote_count);
+ t_array_init(&changes, local_count + remote_count);
+ drop_older_timestamp = ioloop_time -
+ box->index->optimization_set.cache.unaccessed_field_drop_secs;
+
+ for (li = ri = 0; li < local_count || ri < remote_count; ) {
+ ret = li == local_count ? 1 :
+ ri == remote_count ? -1 :
+ strcmp(local_fields[li].name, remote_fields[ri].name);
+ if (ret == 0) {
+ /* field exists in both local and remote */
+ const struct mailbox_cache_field *lf = &local_fields[li];
+ const struct mailbox_cache_field *rf = &remote_fields[ri];
+
+ if (lf->last_used > rf->last_used ||
+ (lf->last_used == rf->last_used &&
+ lf->decision > rf->decision)) {
+ /* use local decision and timestamp */
+ } else {
+ array_push_back(&changes, rf);
+ }
+ li++; ri++;
+ } else if (ret < 0) {
+ /* remote field doesn't exist */
+ li++;
+ } else {
+ /* local field doesn't exist */
+ if (remote_fields[ri].last_used < drop_older_timestamp) {
+ /* field hasn't be used for a long time, remote
+ will probably drop this soon as well */
+ } else {
+ array_push_back(&changes, &remote_fields[ri]);
+ }
+ ri++;
+ }
+ }
+ i_assert(li == local_count && ri == remote_count);
+ if (array_count(&changes) > 0) {
+ array_append_zero(&changes);
+ update->cache_updates = array_front(&changes);
+ }
+}
+
+bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
+ struct mailbox *box,
+ const struct dsync_mailbox *local_box,
+ const struct dsync_mailbox *remote_box,
+ const char **reason_r)
+{
+ struct mailbox_update update;
+ const struct dsync_mailbox_state *state;
+ bool ret = TRUE;
+
+ *reason_r = NULL;
+ i_zero(&update);
+
+ if (local_box->uid_validity != remote_box->uid_validity) {
+ /* Keep the UIDVALIDITY for the mailbox that has more
+ messages. If they equal, use the higher UIDVALIDITY. */
+ if (remote_box->messages_count > local_box->messages_count ||
+ (remote_box->messages_count == local_box->messages_count &&
+ remote_box->uid_validity > local_box->uid_validity))
+ update.uid_validity = remote_box->uid_validity;
+
+ state = dsync_mailbox_state_find(brain, local_box->mailbox_guid);
+ if (state != NULL && state->last_common_uid > 0) {
+ /* we can't continue syncing this mailbox in this
+ session, because the other side already started
+ sending mailbox changes, but not for all mails. */
+ dsync_mailbox_state_remove(brain, local_box->mailbox_guid);
+ *reason_r = "UIDVALIDITY changed during a stateful sync, need to restart";
+ brain->failed = TRUE;
+ ret = FALSE;
+ }
+ }
+
+ dsync_cache_fields_update(local_box, remote_box, box, &update);
+
+ if (update.uid_validity == 0 &&
+ update.cache_updates == NULL) {
+ /* no changes */
+ return ret;
+ }
+
+ if (mailbox_update(box, &update) < 0) {
+ i_error("Couldn't update mailbox %s metadata: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &brain->mail_error));
+ brain->failed = TRUE;
+ }
+ return ret;
+}
+
+static void
+dsync_brain_slave_send_mailbox_lost(struct dsync_brain *brain,
+ const struct dsync_mailbox *dsync_box,
+ bool ignore)
+{
+ struct dsync_mailbox delete_box;
+
+ if (brain->debug) {
+ i_debug("brain %c: We don't have mailbox %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid));
+ }
+ i_zero(&delete_box);
+ memcpy(delete_box.mailbox_guid, dsync_box->mailbox_guid,
+ sizeof(delete_box.mailbox_guid));
+ t_array_init(&delete_box.cache_fields, 0);
+ if (ignore)
+ delete_box.mailbox_ignore = TRUE;
+ else
+ delete_box.mailbox_lost = TRUE;
+ dsync_ibc_send_mailbox(brain->ibc, &delete_box);
+}
+
+bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox *dsync_box;
+ struct dsync_mailbox local_dsync_box;
+ struct mailbox *box;
+ struct file_lock *lock;
+ const char *errstr, *resync_reason, *reason;
+ enum mail_error error;
+ int ret;
+ bool resync;
+
+ i_assert(!brain->master_brain);
+ i_assert(brain->box == NULL);
+
+ if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0)
+ return FALSE;
+ if (ret < 0) {
+ brain->state = DSYNC_STATE_FINISH;
+ return TRUE;
+ }
+
+ if (dsync_brain_mailbox_alloc(brain, dsync_box->mailbox_guid,
+ &box, &errstr, &error) < 0) {
+ i_error("Couldn't allocate mailbox GUID %s: %s",
+ guid_128_to_string(dsync_box->mailbox_guid), errstr);
+ brain->mail_error = error;
+ brain->failed = TRUE;
+ return TRUE;
+ }
+ if (box == NULL) {
+ /* mailbox was probably deleted/renamed during sync */
+ if (brain->backup_send && brain->no_backup_overwrite) {
+ if (brain->debug) {
+ i_debug("brain %c: Ignore nonexistent "
+ "mailbox GUID %s with -1 sync",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid));
+ }
+ dsync_brain_slave_send_mailbox_lost(brain, dsync_box, TRUE);
+ return TRUE;
+ }
+ //FIXME: verify this from log, and if not log an error.
+ dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+ "Mailbox GUID %s was lost",
+ guid_128_to_string(dsync_box->mailbox_guid)));
+ dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE);
+ return TRUE;
+ }
+ /* Lock before syncing */
+ if (dsync_mailbox_lock(brain, box, &lock) < 0) {
+ mailbox_free(&box);
+ brain->failed = TRUE;
+ return TRUE;
+ }
+ if (mailbox_sync(box, MAILBOX_SYNC_FLAG_FULL_READ) < 0) {
+ i_error("Can't sync mailbox %s: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &brain->mail_error));
+ file_lock_free(&lock);
+ mailbox_free(&box);
+ brain->failed = TRUE;
+ return TRUE;
+ }
+
+ if ((ret = dsync_box_get(box, &local_dsync_box, &error)) <= 0) {
+ file_lock_free(&lock);
+ mailbox_free(&box);
+ if (ret < 0) {
+ brain->mail_error = error;
+ brain->failed = TRUE;
+ return TRUE;
+ }
+ /* another process just deleted this mailbox? */
+ if (brain->debug) {
+ i_debug("brain %c: Skipping lost mailbox %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid));
+ }
+ dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE);
+ return TRUE;
+ }
+ i_assert(local_dsync_box.uid_validity != 0);
+ i_assert(memcmp(dsync_box->mailbox_guid, local_dsync_box.mailbox_guid,
+ sizeof(dsync_box->mailbox_guid)) == 0);
+
+ resync = !dsync_brain_mailbox_update_pre(brain, box, &local_dsync_box,
+ dsync_box, &resync_reason);
+
+ if (!dsync_boxes_need_sync(brain, &local_dsync_box, dsync_box, &reason)) {
+ /* no fields appear to have changed, skip this mailbox */
+ if (brain->debug) {
+ i_debug("brain %c: Skipping unchanged mailbox %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid));
+ }
+ dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
+ file_lock_free(&lock);
+ mailbox_free(&box);
+ return TRUE;
+ }
+ if (brain->debug) {
+ i_debug("brain %c: Syncing mailbox %s: %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid), reason);
+ }
+
+ /* start export/import */
+ dsync_brain_sync_mailbox_init(brain, box, lock, &local_dsync_box, FALSE);
+ if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0)
+ return TRUE;
+ if (resync)
+ dsync_brain_set_changes_during_sync(brain, resync_reason);
+ if (ret == 0 || resync) {
+ brain->require_full_resync = TRUE;
+ dsync_brain_sync_mailbox_deinit(brain);
+ dsync_brain_slave_send_mailbox_lost(brain, dsync_box, FALSE);
+ return TRUE;
+ }
+
+ dsync_ibc_send_mailbox(brain->ibc, &local_dsync_box);
+ brain->state = DSYNC_STATE_SYNC_MAILS;
+ return TRUE;
+}
diff --git a/src/doveadm/dsync/dsync-brain-mails.c b/src/doveadm/dsync/dsync-brain-mails.c
new file mode 100644
index 0000000..3feea20
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-mails.c
@@ -0,0 +1,438 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "dsync-ibc.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox-import.h"
+#include "dsync-mailbox-export.h"
+#include "dsync-brain-private.h"
+
+const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1] = {
+ "mailbox",
+ "changes",
+ "attributes",
+ "mail_requests",
+ "mails",
+ "recv_last_common",
+ "done"
+};
+
+static bool dsync_brain_master_sync_recv_mailbox(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox *dsync_box;
+ const char *resync_reason, *reason;
+ enum dsync_ibc_recv_ret ret;
+ bool resync;
+
+ i_assert(brain->master_brain);
+
+ if ((ret = dsync_ibc_recv_mailbox(brain->ibc, &dsync_box)) == 0)
+ return FALSE;
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ i_error("Remote sent end-of-list instead of a mailbox");
+ brain->failed = TRUE;
+ return TRUE;
+ }
+ if (memcmp(dsync_box->mailbox_guid, brain->local_dsync_box.mailbox_guid,
+ sizeof(dsync_box->mailbox_guid)) != 0) {
+ i_error("Remote sent mailbox with a wrong GUID");
+ brain->failed = TRUE;
+ return TRUE;
+ }
+
+ if (dsync_box->mailbox_ignore) {
+ /* ignore this box */
+ if (brain->debug)
+ i_debug("brain %c: Ignoring missing remote box GUID %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid));
+ dsync_brain_sync_mailbox_deinit(brain);
+ return TRUE;
+ }
+ if (dsync_box->mailbox_lost) {
+ /* remote lost the mailbox. it's probably already deleted, but
+ verify it on next sync just to be sure */
+ dsync_brain_set_changes_during_sync(brain, t_strdup_printf(
+ "Remote lost mailbox GUID %s (maybe it was just deleted?)",
+ guid_128_to_string(dsync_box->mailbox_guid)));
+ brain->require_full_resync = TRUE;
+ dsync_brain_sync_mailbox_deinit(brain);
+ return TRUE;
+ }
+ resync = !dsync_brain_mailbox_update_pre(brain, brain->box,
+ &brain->local_dsync_box,
+ dsync_box, &resync_reason);
+
+ if (!dsync_boxes_need_sync(brain, &brain->local_dsync_box, dsync_box,
+ &reason)) {
+ /* no fields appear to have changed, skip this mailbox */
+ dsync_brain_sync_mailbox_deinit(brain);
+ return TRUE;
+ }
+ if (brain->debug) {
+ i_debug("brain %c: Syncing mailbox %s: %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(dsync_box->mailbox_guid), reason);
+ }
+ if ((ret = dsync_brain_sync_mailbox_open(brain, dsync_box)) < 0)
+ return TRUE;
+ if (resync)
+ dsync_brain_set_changes_during_sync(brain, resync_reason);
+ if (ret == 0 || resync) {
+ brain->require_full_resync = TRUE;
+ brain->failed = TRUE;
+ dsync_brain_sync_mailbox_deinit(brain);
+ return TRUE;
+ }
+ dsync_brain_sync_init_box_states(brain);
+ return TRUE;
+}
+
+static bool dsync_brain_recv_mailbox_attribute(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox_attribute *attr;
+ struct istream *input;
+ enum dsync_ibc_recv_ret ret;
+
+ if ((ret = dsync_ibc_recv_mailbox_attribute(brain->ibc, &attr)) == 0)
+ return FALSE;
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ brain->box_recv_state = DSYNC_BOX_STATE_CHANGES;
+ return TRUE;
+ }
+ if (dsync_mailbox_import_attribute(brain->box_importer, attr) < 0)
+ brain->failed = TRUE;
+ input = attr->value_stream;
+ i_stream_unref(&input);
+ return TRUE;
+}
+
+static void dsync_brain_send_end_of_list(struct dsync_brain *brain,
+ enum dsync_ibc_eol_type type)
+{
+ i_assert(!brain->failed);
+ dsync_ibc_send_end_of_list(brain->ibc, type);
+}
+
+static int dsync_brain_export_deinit(struct dsync_brain *brain)
+{
+ const char *errstr;
+ enum mail_error error;
+
+ if (dsync_mailbox_export_deinit(&brain->box_exporter,
+ &errstr, &error) < 0) {
+ i_error("Exporting mailbox %s failed: %s",
+ mailbox_get_vname(brain->box), errstr);
+ brain->mail_error = error;
+ brain->failed = TRUE;
+ return -1;
+ }
+ return 0;
+}
+
+static void dsync_brain_send_mailbox_attribute(struct dsync_brain *brain)
+{
+ const struct dsync_mailbox_attribute *attr;
+ int ret;
+
+ while ((ret = dsync_mailbox_export_next_attr(brain->box_exporter, &attr)) > 0) {
+ if (dsync_ibc_send_mailbox_attribute(brain->ibc, attr) == 0)
+ return;
+ }
+ if (ret < 0) {
+ if (dsync_brain_export_deinit(brain) == 0)
+ i_unreached();
+ return;
+ }
+ dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE);
+ brain->box_send_state = DSYNC_BOX_STATE_CHANGES;
+}
+
+static bool dsync_brain_recv_mail_change(struct dsync_brain *brain)
+{
+ const struct dsync_mail_change *change;
+ enum dsync_ibc_recv_ret ret;
+
+ if ((ret = dsync_ibc_recv_change(brain->ibc, &change)) == 0)
+ return FALSE;
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ if (dsync_mailbox_import_changes_finish(brain->box_importer) < 0)
+ brain->failed = TRUE;
+ if (brain->mail_requests && brain->box_exporter != NULL)
+ brain->box_recv_state = DSYNC_BOX_STATE_MAIL_REQUESTS;
+ else
+ brain->box_recv_state = DSYNC_BOX_STATE_MAILS;
+ return TRUE;
+ }
+ if (dsync_mailbox_import_change(brain->box_importer, change) < 0)
+ brain->failed = TRUE;
+ return TRUE;
+}
+
+static void dsync_brain_send_mail_change(struct dsync_brain *brain)
+{
+ const struct dsync_mail_change *change;
+ int ret;
+
+ while ((ret = dsync_mailbox_export_next(brain->box_exporter, &change)) > 0) {
+ if (dsync_ibc_send_change(brain->ibc, change) == 0)
+ return;
+ }
+ if (ret < 0) {
+ if (dsync_brain_export_deinit(brain) == 0)
+ i_unreached();
+ return;
+ }
+ dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_CHANGES);
+ if (brain->mail_requests && brain->box_importer != NULL)
+ brain->box_send_state = DSYNC_BOX_STATE_MAIL_REQUESTS;
+ else
+ brain->box_send_state = DSYNC_BOX_STATE_MAILS;
+}
+
+static bool dsync_brain_recv_mail_request(struct dsync_brain *brain)
+{
+ const struct dsync_mail_request *request;
+ enum dsync_ibc_recv_ret ret;
+
+ i_assert(brain->mail_requests);
+ i_assert(brain->box_exporter != NULL);
+
+ if ((ret = dsync_ibc_recv_mail_request(brain->ibc, &request)) == 0)
+ return FALSE;
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ brain->box_recv_state = brain->box_importer != NULL ?
+ DSYNC_BOX_STATE_MAILS :
+ DSYNC_BOX_STATE_RECV_LAST_COMMON;
+ return TRUE;
+ }
+ dsync_mailbox_export_want_mail(brain->box_exporter, request);
+ return TRUE;
+}
+
+static bool dsync_brain_send_mail_request(struct dsync_brain *brain)
+{
+ const struct dsync_mail_request *request;
+
+ i_assert(brain->mail_requests);
+
+ while ((request = dsync_mailbox_import_next_request(brain->box_importer)) != NULL) {
+ if (dsync_ibc_send_mail_request(brain->ibc, request) == 0)
+ return TRUE;
+ }
+ if (brain->box_recv_state < DSYNC_BOX_STATE_MAIL_REQUESTS)
+ return FALSE;
+
+ dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAIL_REQUESTS);
+ if (brain->box_exporter != NULL)
+ brain->box_send_state = DSYNC_BOX_STATE_MAILS;
+ else {
+ i_assert(brain->box_recv_state != DSYNC_BOX_STATE_DONE);
+ brain->box_send_state = DSYNC_BOX_STATE_DONE;
+ }
+ return TRUE;
+}
+
+static void dsync_brain_sync_half_finished(struct dsync_brain *brain)
+{
+ struct dsync_mailbox_state state;
+ const char *changes_during_sync;
+ bool require_full_resync;
+
+ if (brain->box_recv_state < DSYNC_BOX_STATE_RECV_LAST_COMMON ||
+ brain->box_send_state < DSYNC_BOX_STATE_RECV_LAST_COMMON)
+ return;
+
+ /* finished with this mailbox */
+ i_zero(&state);
+ memcpy(state.mailbox_guid, brain->local_dsync_box.mailbox_guid,
+ sizeof(state.mailbox_guid));
+ state.last_uidvalidity = brain->local_dsync_box.uid_validity;
+ if (brain->box_importer == NULL) {
+ /* this mailbox didn't exist on remote */
+ state.last_common_uid = brain->local_dsync_box.uid_next-1;
+ state.last_common_modseq =
+ brain->local_dsync_box.highest_modseq;
+ state.last_common_pvt_modseq =
+ brain->local_dsync_box.highest_pvt_modseq;
+ state.last_messages_count =
+ brain->local_dsync_box.messages_count;
+ } else {
+ if (dsync_mailbox_import_deinit(&brain->box_importer,
+ !brain->failed,
+ &state.last_common_uid,
+ &state.last_common_modseq,
+ &state.last_common_pvt_modseq,
+ &state.last_messages_count,
+ &changes_during_sync,
+ &require_full_resync,
+ &brain->mail_error) < 0) {
+ if (require_full_resync) {
+ /* don't treat this as brain failure or the
+ state won't be sent to the other brain.
+ this also means we'll continue syncing the
+ following mailboxes. */
+ brain->require_full_resync = TRUE;
+ } else {
+ brain->failed = TRUE;
+ }
+ }
+ if (changes_during_sync != NULL) {
+ state.changes_during_sync = TRUE;
+ dsync_brain_set_changes_during_sync(brain, changes_during_sync);
+ }
+ }
+ if (brain->require_full_resync) {
+ state.last_uidvalidity = 0;
+ state.changes_during_sync = TRUE;
+ }
+ brain->mailbox_state = state;
+ dsync_ibc_send_mailbox_state(brain->ibc, &state);
+}
+
+static bool dsync_brain_recv_mail(struct dsync_brain *brain)
+{
+ struct dsync_mail *mail;
+ enum dsync_ibc_recv_ret ret;
+
+ if ((ret = dsync_ibc_recv_mail(brain->ibc, &mail)) == 0)
+ return FALSE;
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ brain->box_recv_state = DSYNC_BOX_STATE_RECV_LAST_COMMON;
+ if (brain->box_exporter != NULL &&
+ brain->box_send_state >= DSYNC_BOX_STATE_RECV_LAST_COMMON) {
+ if (dsync_brain_export_deinit(brain) < 0)
+ return TRUE;
+ }
+ dsync_brain_sync_half_finished(brain);
+ return TRUE;
+ }
+ if (brain->debug) {
+ i_debug("brain %c: import mail uid %u guid %s",
+ brain->master_brain ? 'M' : 'S', mail->uid, mail->guid);
+ }
+ if (dsync_mailbox_import_mail(brain->box_importer, mail) < 0)
+ brain->failed = TRUE;
+ i_stream_unref(&mail->input);
+ return TRUE;
+}
+
+static bool dsync_brain_send_mail(struct dsync_brain *brain)
+{
+ const struct dsync_mail *mail;
+
+ if (brain->mail_requests &&
+ brain->box_recv_state < DSYNC_BOX_STATE_MAILS) {
+ /* wait for mail requests to finish. we could already start
+ exporting, but then we're going to do quite a lot of
+ separate searches. especially with pipe backend we'd do
+ a separate search for each mail. */
+ return FALSE;
+ }
+
+ while (dsync_mailbox_export_next_mail(brain->box_exporter, &mail) > 0) {
+ if (dsync_ibc_send_mail(brain->ibc, mail) == 0)
+ return TRUE;
+ }
+
+ if (dsync_brain_export_deinit(brain) < 0)
+ return TRUE;
+
+ brain->box_send_state = DSYNC_BOX_STATE_DONE;
+ dsync_brain_send_end_of_list(brain, DSYNC_IBC_EOL_MAILS);
+
+ dsync_brain_sync_half_finished(brain);
+ return TRUE;
+}
+
+static bool dsync_brain_recv_last_common(struct dsync_brain *brain)
+{
+ enum dsync_ibc_recv_ret ret;
+ struct dsync_mailbox_state state;
+
+ if ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) == 0)
+ return FALSE;
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ i_error("Remote sent end-of-list instead of a mailbox state");
+ brain->failed = TRUE;
+ return TRUE;
+ }
+ i_assert(brain->box_send_state == DSYNC_BOX_STATE_DONE);
+ i_assert(memcmp(state.mailbox_guid, brain->local_dsync_box.mailbox_guid,
+ sizeof(state.mailbox_guid)) == 0);
+
+ /* normally the last_common_* values should be the same in local and
+ remote, but during unexpected changes they may differ. use the
+ values that are lower as the final state. */
+ if (brain->mailbox_state.last_common_uid > state.last_common_uid)
+ brain->mailbox_state.last_common_uid = state.last_common_uid;
+ if (brain->mailbox_state.last_common_modseq > state.last_common_modseq)
+ brain->mailbox_state.last_common_modseq = state.last_common_modseq;
+ if (brain->mailbox_state.last_common_pvt_modseq > state.last_common_pvt_modseq)
+ brain->mailbox_state.last_common_pvt_modseq = state.last_common_pvt_modseq;
+ if (state.changes_during_sync)
+ brain->changes_during_remote_sync = TRUE;
+
+ dsync_brain_sync_mailbox_deinit(brain);
+ return TRUE;
+}
+
+bool dsync_brain_sync_mails(struct dsync_brain *brain)
+{
+ bool changed = FALSE;
+
+ i_assert(brain->box != NULL);
+
+ switch (brain->box_recv_state) {
+ case DSYNC_BOX_STATE_MAILBOX:
+ changed = dsync_brain_master_sync_recv_mailbox(brain);
+ break;
+ case DSYNC_BOX_STATE_ATTRIBUTES:
+ changed = dsync_brain_recv_mailbox_attribute(brain);
+ break;
+ case DSYNC_BOX_STATE_CHANGES:
+ changed = dsync_brain_recv_mail_change(brain);
+ break;
+ case DSYNC_BOX_STATE_MAIL_REQUESTS:
+ changed = dsync_brain_recv_mail_request(brain);
+ break;
+ case DSYNC_BOX_STATE_MAILS:
+ changed = dsync_brain_recv_mail(brain);
+ break;
+ case DSYNC_BOX_STATE_RECV_LAST_COMMON:
+ changed = dsync_brain_recv_last_common(brain);
+ break;
+ case DSYNC_BOX_STATE_DONE:
+ break;
+ }
+
+ if (!dsync_ibc_is_send_queue_full(brain->ibc) && !brain->failed) {
+ switch (brain->box_send_state) {
+ case DSYNC_BOX_STATE_MAILBOX:
+ /* wait for mailbox to be received first */
+ break;
+ case DSYNC_BOX_STATE_ATTRIBUTES:
+ dsync_brain_send_mailbox_attribute(brain);
+ changed = TRUE;
+ break;
+ case DSYNC_BOX_STATE_CHANGES:
+ dsync_brain_send_mail_change(brain);
+ changed = TRUE;
+ break;
+ case DSYNC_BOX_STATE_MAIL_REQUESTS:
+ if (dsync_brain_send_mail_request(brain))
+ changed = TRUE;
+ break;
+ case DSYNC_BOX_STATE_MAILS:
+ if (dsync_brain_send_mail(brain))
+ changed = TRUE;
+ break;
+ case DSYNC_BOX_STATE_RECV_LAST_COMMON:
+ i_unreached();
+ case DSYNC_BOX_STATE_DONE:
+ break;
+ }
+ }
+ return changed;
+}
diff --git a/src/doveadm/dsync/dsync-brain-private.h b/src/doveadm/dsync/dsync-brain-private.h
new file mode 100644
index 0000000..9849703
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain-private.h
@@ -0,0 +1,163 @@
+#ifndef DSYNC_BRAIN_PRIVATE_H
+#define DSYNC_BRAIN_PRIVATE_H
+
+#include "hash.h"
+#include "dsync-brain.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-state.h"
+
+#define DSYNC_LOCK_FILENAME ".dovecot-sync.lock"
+#define DSYNC_MAILBOX_LOCK_FILENAME ".dovecot-box-sync.lock"
+#define DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS 30
+
+struct dsync_mailbox_tree_sync_change;
+
+enum dsync_state {
+ DSYNC_STATE_MASTER_RECV_HANDSHAKE,
+ DSYNC_STATE_SLAVE_RECV_HANDSHAKE,
+ /* if sync_type=STATE, the master brain knows the saved "last common
+ mailbox state". this state is sent to the slave. */
+ DSYNC_STATE_MASTER_SEND_LAST_COMMON,
+ DSYNC_STATE_SLAVE_RECV_LAST_COMMON,
+
+ /* both sides send their mailbox trees */
+ DSYNC_STATE_SEND_MAILBOX_TREE,
+ DSYNC_STATE_SEND_MAILBOX_TREE_DELETES,
+ DSYNC_STATE_RECV_MAILBOX_TREE,
+ DSYNC_STATE_RECV_MAILBOX_TREE_DELETES,
+
+ /* master decides in which order mailboxes are synced (it knows the
+ slave's mailboxes by looking at the received mailbox tree) */
+ DSYNC_STATE_MASTER_SEND_MAILBOX,
+ DSYNC_STATE_SLAVE_RECV_MAILBOX,
+ /* once mailbox is selected, the mails inside it are synced.
+ after the mails are synced, another mailbox is synced. */
+ DSYNC_STATE_SYNC_MAILS,
+
+ DSYNC_STATE_FINISH,
+ DSYNC_STATE_DONE
+};
+
+enum dsync_box_state {
+ DSYNC_BOX_STATE_MAILBOX,
+ DSYNC_BOX_STATE_CHANGES,
+ DSYNC_BOX_STATE_ATTRIBUTES,
+ DSYNC_BOX_STATE_MAIL_REQUESTS,
+ DSYNC_BOX_STATE_MAILS,
+ DSYNC_BOX_STATE_RECV_LAST_COMMON,
+ DSYNC_BOX_STATE_DONE
+};
+
+struct dsync_brain {
+ pool_t pool;
+ struct mail_user *user;
+ struct dsync_ibc *ibc;
+ const char *process_title_prefix;
+ ARRAY(struct mail_namespace *) sync_namespaces;
+ const char *sync_box;
+ struct mailbox *virtual_all_box;
+ guid_128_t sync_box_guid;
+ const char *const *exclude_mailboxes;
+ enum dsync_brain_sync_type sync_type;
+ time_t sync_since_timestamp;
+ time_t sync_until_timestamp;
+ uoff_t sync_max_size;
+ const char *sync_flag;
+ char alt_char;
+ unsigned int import_commit_msgs_interval;
+ unsigned int hdr_hash_version;
+
+ unsigned int lock_timeout;
+ int lock_fd;
+ const char *lock_path;
+ struct file_lock *lock;
+
+ char hierarchy_sep, escape_char;
+ struct dsync_mailbox_tree *local_mailbox_tree;
+ struct dsync_mailbox_tree *remote_mailbox_tree;
+ struct dsync_mailbox_tree_iter *local_tree_iter;
+
+ enum dsync_state state, pre_box_state;
+ enum dsync_box_state box_recv_state;
+ enum dsync_box_state box_send_state;
+ unsigned int proctitle_update_counter;
+
+ struct dsync_transaction_log_scan *log_scan;
+ struct dsync_mailbox_importer *box_importer;
+ struct dsync_mailbox_exporter *box_exporter;
+
+ struct mailbox *box;
+ struct file_lock *box_lock;
+ unsigned int mailbox_lock_timeout_secs;
+ struct dsync_mailbox local_dsync_box, remote_dsync_box;
+ pool_t dsync_box_pool;
+ /* list of mailbox states
+ for master brain: given to brain at init and
+ for slave brain: received from DSYNC_STATE_SLAVE_RECV_LAST_COMMON */
+ HASH_TABLE_TYPE(dsync_mailbox_state) mailbox_states;
+ /* DSYNC_STATE_MASTER_SEND_LAST_COMMON: current send position */
+ struct hash_iterate_context *mailbox_states_iter;
+ /* state of the mailbox we're currently syncing, changed at
+ init and deinit */
+ struct dsync_mailbox_state mailbox_state;
+ /* new states for synced mailboxes */
+ ARRAY_TYPE(dsync_mailbox_state) remote_mailbox_states;
+
+ const char *changes_during_sync;
+ enum mail_error mail_error;
+
+ const char *const *hashed_headers;
+
+ bool master_brain:1;
+ bool mail_requests:1;
+ bool backup_send:1;
+ bool backup_recv:1;
+ bool purge:1;
+ bool debug:1;
+ bool sync_visible_namespaces:1;
+ bool no_mail_sync:1;
+ bool no_backup_overwrite:1;
+ bool no_mail_prefetch:1;
+ bool changes_during_remote_sync:1;
+ bool require_full_resync:1;
+ bool verbose_proctitle:1;
+ bool no_notify:1;
+ bool failed:1;
+ bool empty_hdr_workaround:1;
+};
+
+extern const char *dsync_box_state_names[DSYNC_BOX_STATE_DONE+1];
+
+void dsync_brain_mailbox_trees_init(struct dsync_brain *brain);
+void dsync_brain_send_mailbox_tree(struct dsync_brain *brain);
+void dsync_brain_send_mailbox_tree_deletes(struct dsync_brain *brain);
+bool dsync_brain_recv_mailbox_tree(struct dsync_brain *brain);
+bool dsync_brain_recv_mailbox_tree_deletes(struct dsync_brain *brain);
+int dsync_brain_mailbox_tree_sync_change(struct dsync_brain *brain,
+ const struct dsync_mailbox_tree_sync_change *change,
+ enum mail_error *error_r);
+
+void dsync_brain_sync_mailbox_deinit(struct dsync_brain *brain);
+int dsync_brain_mailbox_alloc(struct dsync_brain *brain, const guid_128_t guid,
+ struct mailbox **box_r, const char **errstr_r,
+ enum mail_error *error_r);
+bool dsync_brain_mailbox_update_pre(struct dsync_brain *brain,
+ struct mailbox *box,
+ const struct dsync_mailbox *local_box,
+ const struct dsync_mailbox *remote_box,
+ const char **reason_r);
+bool dsync_boxes_need_sync(struct dsync_brain *brain,
+ const struct dsync_mailbox *box1,
+ const struct dsync_mailbox *box2,
+ const char **reason_r);
+void dsync_brain_sync_init_box_states(struct dsync_brain *brain);
+void dsync_brain_set_changes_during_sync(struct dsync_brain *brain,
+ const char *reason);
+
+void dsync_brain_master_send_mailbox(struct dsync_brain *brain);
+bool dsync_brain_slave_recv_mailbox(struct dsync_brain *brain);
+int dsync_brain_sync_mailbox_open(struct dsync_brain *brain,
+ const struct dsync_mailbox *remote_dsync_box);
+bool dsync_brain_sync_mails(struct dsync_brain *brain);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-brain.c b/src/doveadm/dsync/dsync-brain.c
new file mode 100644
index 0000000..8ff7247
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain.c
@@ -0,0 +1,901 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "hostpid.h"
+#include "str.h"
+#include "file-create-locked.h"
+#include "process-title.h"
+#include "settings-parser.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "mail-namespace.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-ibc.h"
+#include "dsync-brain-private.h"
+#include "dsync-mailbox-import.h"
+#include "dsync-mailbox-export.h"
+
+#include <sys/stat.h>
+
+enum dsync_brain_title {
+ DSYNC_BRAIN_TITLE_NONE = 0,
+ DSYNC_BRAIN_TITLE_LOCKING,
+};
+
+static const char *dsync_state_names[] = {
+ "master_recv_handshake",
+ "slave_recv_handshake",
+ "master_send_last_common",
+ "slave_recv_last_common",
+ "send_mailbox_tree",
+ "send_mailbox_tree_deletes",
+ "recv_mailbox_tree",
+ "recv_mailbox_tree_deletes",
+ "master_send_mailbox",
+ "slave_recv_mailbox",
+ "sync_mails",
+ "finish",
+ "done"
+};
+
+struct dsync_mailbox_list_module dsync_mailbox_list_module =
+ MODULE_CONTEXT_INIT(&mailbox_list_module_register);
+
+static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain);
+
+static const char *
+dsync_brain_get_proctitle_full(struct dsync_brain *brain,
+ enum dsync_brain_title title)
+{
+ string_t *str = t_str_new(128);
+ const char *import_title, *export_title;
+
+ str_append_c(str, '[');
+ if (brain->process_title_prefix != NULL)
+ str_append(str, brain->process_title_prefix);
+ str_append(str, brain->user->username);
+ if (brain->box == NULL) {
+ str_append_c(str, ' ');
+ str_append(str, dsync_state_names[brain->state]);
+ } else {
+ str_append_c(str, ' ');
+ str_append(str, mailbox_get_vname(brain->box));
+ import_title = brain->box_importer == NULL ? "" :
+ dsync_mailbox_import_get_proctitle(brain->box_importer);
+ export_title = brain->box_exporter == NULL ? "" :
+ dsync_mailbox_export_get_proctitle(brain->box_exporter);
+ if (import_title[0] == '\0' && export_title[0] == '\0') {
+ str_printfa(str, " send:%s recv:%s",
+ dsync_box_state_names[brain->box_send_state],
+ dsync_box_state_names[brain->box_recv_state]);
+ } else {
+ if (import_title[0] != '\0') {
+ str_append(str, " import:");
+ str_append(str, import_title);
+ }
+ if (export_title[0] != '\0') {
+ str_append(str, " export:");
+ str_append(str, export_title);
+ }
+ }
+ }
+ switch (title) {
+ case DSYNC_BRAIN_TITLE_NONE:
+ break;
+ case DSYNC_BRAIN_TITLE_LOCKING:
+ str_append(str, " locking "DSYNC_LOCK_FILENAME);
+ break;
+ }
+ str_append_c(str, ']');
+ return str_c(str);
+}
+
+static const char *dsync_brain_get_proctitle(struct dsync_brain *brain)
+{
+ return dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_NONE);
+}
+
+static void dsync_brain_run_io(void *context)
+{
+ struct dsync_brain *brain = context;
+ bool changed, try_pending;
+
+ if (dsync_ibc_has_failed(brain->ibc)) {
+ io_loop_stop(current_ioloop);
+ brain->failed = TRUE;
+ return;
+ }
+
+ try_pending = TRUE;
+ do {
+ if (!dsync_brain_run(brain, &changed)) {
+ io_loop_stop(current_ioloop);
+ break;
+ }
+ if (changed)
+ try_pending = TRUE;
+ else if (try_pending) {
+ if (dsync_ibc_has_pending_data(brain->ibc))
+ changed = TRUE;
+ try_pending = FALSE;
+ }
+ } while (changed);
+}
+
+static struct dsync_brain *
+dsync_brain_common_init(struct mail_user *user, struct dsync_ibc *ibc)
+{
+ struct dsync_brain *brain;
+ const struct master_service_settings *service_set;
+ pool_t pool;
+
+ service_set = master_service_settings_get(master_service);
+ mail_user_ref(user);
+
+ pool = pool_alloconly_create("dsync brain", 10240);
+ brain = p_new(pool, struct dsync_brain, 1);
+ brain->pool = pool;
+ brain->user = user;
+ brain->ibc = ibc;
+ brain->sync_type = DSYNC_BRAIN_SYNC_TYPE_UNKNOWN;
+ brain->lock_fd = -1;
+ brain->verbose_proctitle = service_set->verbose_proctitle;
+ hash_table_create(&brain->mailbox_states, pool, 0,
+ guid_128_hash, guid_128_cmp);
+ p_array_init(&brain->remote_mailbox_states, pool, 64);
+ return brain;
+}
+
+static void
+dsync_brain_set_flags(struct dsync_brain *brain, enum dsync_brain_flags flags)
+{
+ brain->mail_requests =
+ (flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0;
+ brain->backup_send = (flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0;
+ brain->backup_recv = (flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0;
+ brain->debug = (flags & DSYNC_BRAIN_FLAG_DEBUG) != 0;
+ brain->sync_visible_namespaces =
+ (flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0;
+ brain->no_mail_sync = (flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0;
+ brain->no_backup_overwrite =
+ (flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0;
+ brain->no_mail_prefetch =
+ (flags & DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH) != 0;
+ brain->no_notify = (flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0;
+ brain->empty_hdr_workaround = (flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0;
+}
+
+static void
+dsync_brain_open_virtual_all_box(struct dsync_brain *brain,
+ const char *vname)
+{
+ struct mail_namespace *ns;
+
+ ns = mail_namespace_find(brain->user->namespaces, vname);
+ brain->virtual_all_box =
+ mailbox_alloc(ns->list, vname, MAILBOX_FLAG_READONLY);
+}
+
+struct dsync_brain *
+dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
+ enum dsync_brain_sync_type sync_type,
+ enum dsync_brain_flags flags,
+ const struct dsync_brain_settings *set)
+{
+ struct dsync_ibc_settings ibc_set;
+ struct dsync_brain *brain;
+ struct mail_namespace *ns;
+ string_t *sync_ns_str = NULL;
+ const char *error;
+
+ i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
+ i_assert(sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE ||
+ (set->state != NULL && *set->state != '\0'));
+ i_assert(N_ELEMENTS(dsync_state_names) == DSYNC_STATE_DONE+1);
+
+ brain = dsync_brain_common_init(user, ibc);
+ brain->process_title_prefix =
+ p_strdup(brain->pool, set->process_title_prefix);
+ brain->sync_type = sync_type;
+ if (array_count(&set->sync_namespaces) > 0) {
+ sync_ns_str = t_str_new(128);
+ p_array_init(&brain->sync_namespaces, brain->pool,
+ array_count(&set->sync_namespaces));
+ array_foreach_elem(&set->sync_namespaces, ns) {
+ str_append(sync_ns_str, ns->prefix);
+ str_append_c(sync_ns_str, '\n');
+ array_push_back(&brain->sync_namespaces, &ns);
+ }
+ str_delete(sync_ns_str, str_len(sync_ns_str)-1, 1);
+ }
+ brain->alt_char = set->mailbox_alt_char == '\0' ? '_' :
+ set->mailbox_alt_char;
+ brain->sync_since_timestamp = set->sync_since_timestamp;
+ brain->sync_until_timestamp = set->sync_until_timestamp;
+ brain->sync_max_size = set->sync_max_size;
+ brain->sync_flag = p_strdup(brain->pool, set->sync_flag);
+ brain->sync_box = p_strdup(brain->pool, set->sync_box);
+ brain->exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
+ p_strarray_dup(brain->pool, set->exclude_mailboxes);
+ memcpy(brain->sync_box_guid, set->sync_box_guid,
+ sizeof(brain->sync_box_guid));
+ brain->lock_timeout = set->lock_timeout_secs;
+ if (brain->lock_timeout != 0)
+ brain->mailbox_lock_timeout_secs = brain->lock_timeout;
+ else
+ brain->mailbox_lock_timeout_secs =
+ DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS;
+ brain->import_commit_msgs_interval = set->import_commit_msgs_interval;
+ brain->master_brain = TRUE;
+ brain->hashed_headers =
+ (const char*const*)p_strarray_dup(brain->pool, set->hashed_headers);
+ dsync_brain_set_flags(brain, flags);
+
+ if (set->virtual_all_box != NULL)
+ dsync_brain_open_virtual_all_box(brain, set->virtual_all_box);
+
+ if (sync_type != DSYNC_BRAIN_SYNC_TYPE_STATE)
+ ;
+ else if (dsync_mailbox_states_import(brain->mailbox_states, brain->pool,
+ set->state, &error) < 0) {
+ hash_table_clear(brain->mailbox_states, FALSE);
+ i_error("Saved sync state is invalid, "
+ "falling back to full sync: %s", error);
+ brain->sync_type = sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
+ } else {
+ if (brain->debug) {
+ i_debug("brain %c: Imported mailbox states:",
+ brain->master_brain ? 'M' : 'S');
+ dsync_brain_mailbox_states_dump(brain);
+ }
+ }
+ dsync_brain_mailbox_trees_init(brain);
+
+ i_zero(&ibc_set);
+ ibc_set.hostname = my_hostdomain();
+ ibc_set.sync_ns_prefixes = sync_ns_str == NULL ?
+ NULL : str_c(sync_ns_str);
+ ibc_set.sync_box = set->sync_box;
+ ibc_set.virtual_all_box = set->virtual_all_box;
+ ibc_set.exclude_mailboxes = set->exclude_mailboxes;
+ ibc_set.sync_since_timestamp = set->sync_since_timestamp;
+ ibc_set.sync_until_timestamp = set->sync_until_timestamp;
+ ibc_set.sync_max_size = set->sync_max_size;
+ ibc_set.sync_flags = set->sync_flag;
+ memcpy(ibc_set.sync_box_guid, set->sync_box_guid,
+ sizeof(ibc_set.sync_box_guid));
+ ibc_set.alt_char = brain->alt_char;
+ ibc_set.sync_type = sync_type;
+ ibc_set.hdr_hash_v2 = TRUE;
+ ibc_set.lock_timeout = set->lock_timeout_secs;
+ ibc_set.import_commit_msgs_interval = set->import_commit_msgs_interval;
+ ibc_set.hashed_headers = set->hashed_headers;
+ /* reverse the backup direction for the slave */
+ ibc_set.brain_flags = flags & ENUM_NEGATE(DSYNC_BRAIN_FLAG_BACKUP_SEND |
+ DSYNC_BRAIN_FLAG_BACKUP_RECV);
+ if ((flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
+ ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV;
+ else if ((flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0)
+ ibc_set.brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
+ dsync_ibc_send_handshake(ibc, &ibc_set);
+
+ dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain);
+ brain->state = DSYNC_STATE_MASTER_RECV_HANDSHAKE;
+
+ if (brain->verbose_proctitle)
+ process_title_set(dsync_brain_get_proctitle(brain));
+ return brain;
+}
+
+struct dsync_brain *
+dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc,
+ bool local, const char *process_title_prefix,
+ char default_alt_char)
+{
+ struct dsync_ibc_settings ibc_set;
+ struct dsync_brain *brain;
+
+ i_assert(default_alt_char != '\0');
+
+ brain = dsync_brain_common_init(user, ibc);
+ brain->alt_char = default_alt_char;
+ brain->process_title_prefix =
+ p_strdup(brain->pool, process_title_prefix);
+ brain->state = DSYNC_STATE_SLAVE_RECV_HANDSHAKE;
+
+ if (local) {
+ /* both master and slave are running within the same process,
+ update the proctitle only for master. */
+ brain->verbose_proctitle = FALSE;
+ }
+
+ i_zero(&ibc_set);
+ ibc_set.hdr_hash_v2 = TRUE;
+ ibc_set.hostname = my_hostdomain();
+ dsync_ibc_send_handshake(ibc, &ibc_set);
+
+ if (brain->verbose_proctitle)
+ process_title_set(dsync_brain_get_proctitle(brain));
+ dsync_ibc_set_io_callback(ibc, dsync_brain_run_io, brain);
+ return brain;
+}
+
+static void dsync_brain_purge(struct dsync_brain *brain)
+{
+ struct mail_namespace *ns;
+ struct mail_storage *storage;
+
+ for (ns = brain->user->namespaces; ns != NULL; ns = ns->next) {
+ if (!dsync_brain_want_namespace(brain, ns))
+ continue;
+
+ storage = mail_namespace_get_default_storage(ns);
+ if (mail_storage_purge(storage) < 0) {
+ i_error("Purging namespace '%s' failed: %s", ns->prefix,
+ mail_storage_get_last_internal_error(storage, NULL));
+ }
+ }
+}
+
+int dsync_brain_deinit(struct dsync_brain **_brain, enum mail_error *error_r)
+{
+ struct dsync_brain *brain = *_brain;
+ int ret;
+
+ *_brain = NULL;
+
+ if (dsync_ibc_has_timed_out(brain->ibc)) {
+ i_error("Timeout during state=%s%s",
+ dsync_state_names[brain->state],
+ brain->state != DSYNC_STATE_SYNC_MAILS ? "" :
+ t_strdup_printf(" (send=%s recv=%s)",
+ dsync_box_state_names[brain->box_send_state],
+ dsync_box_state_names[brain->box_recv_state]));
+ }
+ if (dsync_ibc_has_failed(brain->ibc) ||
+ brain->state != DSYNC_STATE_DONE)
+ brain->failed = TRUE;
+ dsync_ibc_close_mail_streams(brain->ibc);
+
+ if (brain->purge && !brain->failed)
+ dsync_brain_purge(brain);
+
+ if (brain->box != NULL)
+ dsync_brain_sync_mailbox_deinit(brain);
+ if (brain->virtual_all_box != NULL)
+ mailbox_free(&brain->virtual_all_box);
+ if (brain->local_tree_iter != NULL)
+ dsync_mailbox_tree_iter_deinit(&brain->local_tree_iter);
+ if (brain->local_mailbox_tree != NULL)
+ dsync_mailbox_tree_deinit(&brain->local_mailbox_tree);
+ if (brain->remote_mailbox_tree != NULL)
+ dsync_mailbox_tree_deinit(&brain->remote_mailbox_tree);
+ hash_table_iterate_deinit(&brain->mailbox_states_iter);
+ hash_table_destroy(&brain->mailbox_states);
+
+ pool_unref(&brain->dsync_box_pool);
+
+ if (brain->lock_fd != -1) {
+ /* unlink the lock file before it gets unlocked */
+ i_unlink(brain->lock_path);
+ if (brain->debug) {
+ i_debug("brain %c: Unlocked %s",
+ brain->master_brain ? 'M' : 'S',
+ brain->lock_path);
+ }
+ file_lock_free(&brain->lock);
+ i_close_fd(&brain->lock_fd);
+ }
+
+ ret = brain->failed ? -1 : 0;
+ mail_user_unref(&brain->user);
+
+ *error_r = !brain->failed ? 0 :
+ (brain->mail_error == 0 ? MAIL_ERROR_TEMP : brain->mail_error);
+ pool_unref(&brain->pool);
+ return ret;
+}
+
+static int
+dsync_brain_lock(struct dsync_brain *brain, const char *remote_hostname)
+{
+ const struct file_create_settings lock_set = {
+ .lock_timeout_secs = brain->lock_timeout,
+ .lock_settings = {
+ .lock_method = FILE_LOCK_METHOD_FCNTL,
+ },
+ };
+ const char *home, *error, *local_hostname = my_hostdomain();
+ bool created;
+ int ret;
+
+ if ((ret = strcmp(remote_hostname, local_hostname)) < 0) {
+ /* locking done by remote */
+ if (brain->debug) {
+ i_debug("brain %c: Locking done by remote "
+ "(local hostname=%s, remote hostname=%s)",
+ brain->master_brain ? 'M' : 'S',
+ local_hostname, remote_hostname);
+ }
+ return 0;
+ }
+ if (ret == 0 && !brain->master_brain) {
+ /* running dsync within the same server.
+ locking done by master brain. */
+ if (brain->debug) {
+ i_debug("brain %c: Locking done by local master-brain "
+ "(local hostname=%s, remote hostname=%s)",
+ brain->master_brain ? 'M' : 'S',
+ local_hostname, remote_hostname);
+ }
+ return 0;
+ }
+
+ if ((ret = mail_user_get_home(brain->user, &home)) < 0) {
+ i_error("Couldn't look up user's home dir");
+ return -1;
+ }
+ if (ret == 0) {
+ i_error("User has no home directory");
+ return -1;
+ }
+
+ if (brain->verbose_proctitle)
+ process_title_set(dsync_brain_get_proctitle_full(brain, DSYNC_BRAIN_TITLE_LOCKING));
+ brain->lock_path = p_strconcat(brain->pool, home,
+ "/"DSYNC_LOCK_FILENAME, NULL);
+ brain->lock_fd = file_create_locked(brain->lock_path, &lock_set,
+ &brain->lock, &created, &error);
+ if (brain->lock_fd == -1 && errno == ENOENT) {
+ /* home directory not created */
+ if (mail_user_home_mkdir(brain->user) < 0)
+ return -1;
+ brain->lock_fd = file_create_locked(brain->lock_path, &lock_set,
+ &brain->lock, &created, &error);
+ }
+ if (brain->lock_fd == -1)
+ i_error("Couldn't lock %s: %s", brain->lock_path, error);
+ else if (brain->debug) {
+ i_debug("brain %c: Locking done locally in %s "
+ "(local hostname=%s, remote hostname=%s)",
+ brain->master_brain ? 'M' : 'S',
+ brain->lock_path, local_hostname, remote_hostname);
+ }
+ if (brain->verbose_proctitle)
+ process_title_set(dsync_brain_get_proctitle(brain));
+ return brain->lock_fd == -1 ? -1 : 0;
+}
+
+static void
+dsync_brain_set_hdr_hash_version(struct dsync_brain *brain,
+ const struct dsync_ibc_settings *ibc_set)
+{
+ if (ibc_set->hdr_hash_v3)
+ brain->hdr_hash_version = 3;
+ else if (ibc_set->hdr_hash_v2)
+ brain->hdr_hash_version = 3;
+ else
+ brain->hdr_hash_version = 1;
+}
+
+static bool dsync_brain_master_recv_handshake(struct dsync_brain *brain)
+{
+ const struct dsync_ibc_settings *ibc_set;
+
+ i_assert(brain->master_brain);
+
+ if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0)
+ return FALSE;
+
+ if (brain->lock_timeout > 0) {
+ if (dsync_brain_lock(brain, ibc_set->hostname) < 0) {
+ brain->failed = TRUE;
+ return FALSE;
+ }
+ }
+ dsync_brain_set_hdr_hash_version(brain, ibc_set);
+
+ brain->state = brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE ?
+ DSYNC_STATE_MASTER_SEND_LAST_COMMON :
+ DSYNC_STATE_SEND_MAILBOX_TREE;
+ return TRUE;
+}
+
+static bool dsync_brain_slave_recv_handshake(struct dsync_brain *brain)
+{
+ const struct dsync_ibc_settings *ibc_set;
+ struct mail_namespace *ns;
+ const char *const *prefixes;
+
+ i_assert(!brain->master_brain);
+
+ if (dsync_ibc_recv_handshake(brain->ibc, &ibc_set) == 0)
+ return FALSE;
+ dsync_brain_set_hdr_hash_version(brain, ibc_set);
+
+ if (ibc_set->lock_timeout > 0) {
+ brain->lock_timeout = ibc_set->lock_timeout;
+ brain->mailbox_lock_timeout_secs = brain->lock_timeout;
+ if (dsync_brain_lock(brain, ibc_set->hostname) < 0) {
+ brain->failed = TRUE;
+ return FALSE;
+ }
+ } else {
+ brain->mailbox_lock_timeout_secs =
+ DSYNC_MAILBOX_DEFAULT_LOCK_TIMEOUT_SECS;
+ }
+
+ if (ibc_set->sync_ns_prefixes != NULL) {
+ p_array_init(&brain->sync_namespaces, brain->pool, 4);
+ prefixes = t_strsplit(ibc_set->sync_ns_prefixes, "\n");
+ if (prefixes[0] == NULL) {
+ /* ugly workaround for strsplit API: there was one
+ prefix="" entry */
+ static const char *empty_prefix[] = { "", NULL };
+ prefixes = empty_prefix;
+ }
+ for (; *prefixes != NULL; prefixes++) {
+ ns = mail_namespace_find(brain->user->namespaces,
+ *prefixes);
+ if (ns == NULL) {
+ i_error("Namespace not found: '%s'", *prefixes);
+ brain->failed = TRUE;
+ return FALSE;
+ }
+ array_push_back(&brain->sync_namespaces, &ns);
+ }
+ }
+ brain->sync_box = p_strdup(brain->pool, ibc_set->sync_box);
+ brain->exclude_mailboxes = ibc_set->exclude_mailboxes == NULL ? NULL :
+ p_strarray_dup(brain->pool, ibc_set->exclude_mailboxes);
+ brain->sync_since_timestamp = ibc_set->sync_since_timestamp;
+ brain->sync_until_timestamp = ibc_set->sync_until_timestamp;
+ brain->sync_max_size = ibc_set->sync_max_size;
+ brain->sync_flag = p_strdup(brain->pool, ibc_set->sync_flags);
+ memcpy(brain->sync_box_guid, ibc_set->sync_box_guid,
+ sizeof(brain->sync_box_guid));
+ if (ibc_set->alt_char != '\0')
+ brain->alt_char = ibc_set->alt_char;
+ i_assert(brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_UNKNOWN);
+ brain->sync_type = ibc_set->sync_type;
+
+ dsync_brain_set_flags(brain, ibc_set->brain_flags);
+ if (ibc_set->hashed_headers != NULL)
+ brain->hashed_headers =
+ p_strarray_dup(brain->pool, (const char*const*)ibc_set->hashed_headers);
+ /* this flag is only set on the remote slave brain */
+ brain->purge = (ibc_set->brain_flags &
+ DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0;
+
+ if (ibc_set->virtual_all_box != NULL)
+ dsync_brain_open_virtual_all_box(brain, ibc_set->virtual_all_box);
+ dsync_brain_mailbox_trees_init(brain);
+
+ if (brain->sync_type == DSYNC_BRAIN_SYNC_TYPE_STATE)
+ brain->state = DSYNC_STATE_SLAVE_RECV_LAST_COMMON;
+ else
+ brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
+ return TRUE;
+}
+
+static void dsync_brain_master_send_last_common(struct dsync_brain *brain)
+{
+ struct dsync_mailbox_state *state;
+ uint8_t *guid;
+ enum dsync_ibc_send_ret ret = DSYNC_IBC_SEND_RET_OK;
+
+ i_assert(brain->master_brain);
+
+ if (brain->mailbox_states_iter == NULL) {
+ brain->mailbox_states_iter =
+ hash_table_iterate_init(brain->mailbox_states);
+ }
+
+ for (;;) {
+ if (ret == DSYNC_IBC_SEND_RET_FULL)
+ return;
+ if (!hash_table_iterate(brain->mailbox_states_iter,
+ brain->mailbox_states, &guid, &state))
+ break;
+ ret = dsync_ibc_send_mailbox_state(brain->ibc, state);
+ }
+ hash_table_iterate_deinit(&brain->mailbox_states_iter);
+
+ dsync_ibc_send_end_of_list(brain->ibc, DSYNC_IBC_EOL_MAILBOX_STATE);
+ brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
+}
+
+static void dsync_mailbox_state_add(struct dsync_brain *brain,
+ const struct dsync_mailbox_state *state)
+{
+ struct dsync_mailbox_state *dupstate;
+ uint8_t *guid_p;
+
+ dupstate = p_new(brain->pool, struct dsync_mailbox_state, 1);
+ *dupstate = *state;
+ guid_p = dupstate->mailbox_guid;
+ hash_table_insert(brain->mailbox_states, guid_p, dupstate);
+}
+
+static bool dsync_brain_slave_recv_last_common(struct dsync_brain *brain)
+{
+ struct dsync_mailbox_state state;
+ enum dsync_ibc_recv_ret ret;
+ bool changed = FALSE;
+
+ i_assert(!brain->master_brain);
+
+ while ((ret = dsync_ibc_recv_mailbox_state(brain->ibc, &state)) > 0) {
+ dsync_mailbox_state_add(brain, &state);
+ changed = TRUE;
+ }
+ if (ret == DSYNC_IBC_RECV_RET_FINISHED) {
+ brain->state = DSYNC_STATE_SEND_MAILBOX_TREE;
+ changed = TRUE;
+ }
+ return changed;
+}
+
+static bool dsync_brain_finish(struct dsync_brain *brain)
+{
+ const char *error;
+ enum mail_error mail_error;
+ bool require_full_resync;
+ enum dsync_ibc_recv_ret ret;
+
+ if (!brain->master_brain) {
+ dsync_ibc_send_finish(brain->ibc,
+ brain->failed ? "dsync failed" : NULL,
+ brain->mail_error,
+ brain->require_full_resync);
+ brain->state = DSYNC_STATE_DONE;
+ return TRUE;
+ }
+ ret = dsync_ibc_recv_finish(brain->ibc, &error, &mail_error,
+ &require_full_resync);
+ if (ret == DSYNC_IBC_RECV_RET_TRYAGAIN)
+ return FALSE;
+ if (error != NULL) {
+ i_error("Remote dsync failed: %s", error);
+ brain->failed = TRUE;
+ if (mail_error != 0 &&
+ (brain->mail_error == 0 || brain->mail_error == MAIL_ERROR_TEMP))
+ brain->mail_error = mail_error;
+ }
+ if (require_full_resync)
+ brain->require_full_resync = TRUE;
+ brain->state = DSYNC_STATE_DONE;
+ return TRUE;
+}
+
+static bool dsync_brain_run_real(struct dsync_brain *brain, bool *changed_r)
+{
+ enum dsync_state orig_state = brain->state;
+ enum dsync_box_state orig_box_recv_state = brain->box_recv_state;
+ enum dsync_box_state orig_box_send_state = brain->box_send_state;
+ bool changed = FALSE, ret = TRUE;
+
+ if (brain->failed)
+ return FALSE;
+
+ switch (brain->state) {
+ case DSYNC_STATE_MASTER_RECV_HANDSHAKE:
+ changed = dsync_brain_master_recv_handshake(brain);
+ break;
+ case DSYNC_STATE_SLAVE_RECV_HANDSHAKE:
+ changed = dsync_brain_slave_recv_handshake(brain);
+ break;
+ case DSYNC_STATE_MASTER_SEND_LAST_COMMON:
+ dsync_brain_master_send_last_common(brain);
+ changed = TRUE;
+ break;
+ case DSYNC_STATE_SLAVE_RECV_LAST_COMMON:
+ changed = dsync_brain_slave_recv_last_common(brain);
+ break;
+ case DSYNC_STATE_SEND_MAILBOX_TREE:
+ dsync_brain_send_mailbox_tree(brain);
+ changed = TRUE;
+ break;
+ case DSYNC_STATE_RECV_MAILBOX_TREE:
+ changed = dsync_brain_recv_mailbox_tree(brain);
+ break;
+ case DSYNC_STATE_SEND_MAILBOX_TREE_DELETES:
+ dsync_brain_send_mailbox_tree_deletes(brain);
+ changed = TRUE;
+ break;
+ case DSYNC_STATE_RECV_MAILBOX_TREE_DELETES:
+ changed = dsync_brain_recv_mailbox_tree_deletes(brain);
+ break;
+ case DSYNC_STATE_MASTER_SEND_MAILBOX:
+ dsync_brain_master_send_mailbox(brain);
+ changed = TRUE;
+ break;
+ case DSYNC_STATE_SLAVE_RECV_MAILBOX:
+ changed = dsync_brain_slave_recv_mailbox(brain);
+ break;
+ case DSYNC_STATE_SYNC_MAILS:
+ changed = dsync_brain_sync_mails(brain);
+ break;
+ case DSYNC_STATE_FINISH:
+ changed = dsync_brain_finish(brain);
+ break;
+ case DSYNC_STATE_DONE:
+ changed = TRUE;
+ ret = FALSE;
+ break;
+ }
+ if (brain->verbose_proctitle) {
+ if (orig_state != brain->state ||
+ orig_box_recv_state != brain->box_recv_state ||
+ orig_box_send_state != brain->box_send_state ||
+ ++brain->proctitle_update_counter % 100 == 0)
+ process_title_set(dsync_brain_get_proctitle(brain));
+ }
+ *changed_r = changed;
+ return brain->failed ? FALSE : ret;
+}
+
+bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r)
+{
+ bool ret;
+
+ *changed_r = FALSE;
+
+ if (dsync_ibc_has_failed(brain->ibc)) {
+ brain->failed = TRUE;
+ return FALSE;
+ }
+
+ T_BEGIN {
+ ret = dsync_brain_run_real(brain, changed_r);
+ } T_END;
+ return ret;
+}
+
+static void dsync_brain_mailbox_states_dump(struct dsync_brain *brain)
+{
+ struct hash_iterate_context *iter;
+ struct dsync_mailbox_state *state;
+ uint8_t *guid;
+
+ iter = hash_table_iterate_init(brain->mailbox_states);
+ while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
+ i_debug("brain %c: Mailbox %s state: uidvalidity=%u uid=%u modseq=%"PRIu64" pvt_modseq=%"PRIu64" messages=%u changes_during_sync=%d",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(guid),
+ state->last_uidvalidity,
+ state->last_common_uid,
+ state->last_common_modseq,
+ state->last_common_pvt_modseq,
+ state->last_messages_count,
+ state->changes_during_sync ? 1 : 0);
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+void dsync_brain_get_state(struct dsync_brain *brain, string_t *output)
+{
+ struct hash_iterate_context *iter;
+ struct dsync_mailbox_node *node;
+ const struct dsync_mailbox_state *new_state;
+ struct dsync_mailbox_state *state;
+ const uint8_t *guid_p;
+ uint8_t *guid;
+
+ if (brain->require_full_resync)
+ return;
+
+ /* update mailbox states */
+ array_foreach(&brain->remote_mailbox_states, new_state) {
+ guid_p = new_state->mailbox_guid;
+ state = hash_table_lookup(brain->mailbox_states, guid_p);
+ if (state != NULL)
+ *state = *new_state;
+ else
+ dsync_mailbox_state_add(brain, new_state);
+ }
+
+ /* remove nonexistent mailboxes */
+ iter = hash_table_iterate_init(brain->mailbox_states);
+ while (hash_table_iterate(iter, brain->mailbox_states, &guid, &state)) {
+ node = dsync_mailbox_tree_lookup_guid(brain->local_mailbox_tree,
+ guid);
+ if (node == NULL ||
+ node->existence != DSYNC_MAILBOX_NODE_EXISTS) {
+ if (brain->debug) {
+ i_debug("brain %c: Removed state for deleted mailbox %s",
+ brain->master_brain ? 'M' : 'S',
+ guid_128_to_string(guid));
+ }
+ hash_table_remove(brain->mailbox_states, guid);
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+
+ if (brain->debug) {
+ i_debug("brain %c: Exported mailbox states:",
+ brain->master_brain ? 'M' : 'S');
+ dsync_brain_mailbox_states_dump(brain);
+ }
+ dsync_mailbox_states_export(brain->mailbox_states, output);
+}
+
+enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain)
+{
+ return brain->sync_type;
+}
+
+bool dsync_brain_has_failed(struct dsync_brain *brain)
+{
+ return brain->failed;
+}
+
+const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain,
+ bool *remote_only_r)
+{
+ if (brain->changes_during_sync == NULL &&
+ brain->changes_during_remote_sync) {
+ *remote_only_r = TRUE;
+ return "Remote notified that changes happened during sync";
+ }
+ *remote_only_r = FALSE;
+ return brain->changes_during_sync;
+}
+
+static bool dsync_brain_want_shared_namespace(const struct mail_namespace *ns,
+ const struct mail_namespace *sync_ns)
+{
+ /* Include shared namespaces and all its
+ children in the sync (e.g. "Shared/example.com"
+ will be synced to "Shared/").
+ This also allows "dsync -n Shared/example.com/"
+ with "Shared/example.com/username/" style
+ shared namespace config. */
+ return (ns->type == MAIL_NAMESPACE_TYPE_SHARED) &&
+ (sync_ns->type == MAIL_NAMESPACE_TYPE_SHARED) &&
+ str_begins(ns->prefix, sync_ns->prefix);
+}
+
+bool dsync_brain_want_namespace(struct dsync_brain *brain,
+ struct mail_namespace *ns)
+{
+ struct mail_namespace *sync_ns;
+
+ if (array_is_created(&brain->sync_namespaces)) {
+ array_foreach_elem(&brain->sync_namespaces, sync_ns) {
+ if (ns == sync_ns)
+ return TRUE;
+ if (dsync_brain_want_shared_namespace(ns, sync_ns))
+ return TRUE;
+ }
+ return FALSE;
+ }
+ if (ns->alias_for != NULL) {
+ /* always skip aliases */
+ return FALSE;
+ }
+ if (brain->sync_visible_namespaces) {
+ if ((ns->flags & NAMESPACE_FLAG_HIDDEN) == 0)
+ return TRUE;
+ if ((ns->flags & (NAMESPACE_FLAG_LIST_PREFIX |
+ NAMESPACE_FLAG_LIST_CHILDREN)) != 0)
+ return TRUE;
+ return FALSE;
+ } else {
+ return strcmp(ns->unexpanded_set->location,
+ SETTING_STRVAR_UNEXPANDED) == 0;
+ }
+}
+
+void dsync_brain_set_changes_during_sync(struct dsync_brain *brain,
+ const char *reason)
+{
+ if (brain->debug) {
+ i_debug("brain %c: Change during sync: %s",
+ brain->master_brain ? 'M' : 'S', reason);
+ }
+ if (brain->changes_during_sync == NULL)
+ brain->changes_during_sync = p_strdup(brain->pool, reason);
+}
diff --git a/src/doveadm/dsync/dsync-brain.h b/src/doveadm/dsync/dsync-brain.h
new file mode 100644
index 0000000..5813148
--- /dev/null
+++ b/src/doveadm/dsync/dsync-brain.h
@@ -0,0 +1,130 @@
+#ifndef DSYNC_BRAIN_H
+#define DSYNC_BRAIN_H
+
+#include "module-context.h"
+#include "guid.h"
+#include "mail-error.h"
+#include "mailbox-list-private.h"
+
+struct mail_namespace;
+struct mail_user;
+struct dsync_ibc;
+
+enum dsync_brain_flags {
+ DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS = 0x01,
+ DSYNC_BRAIN_FLAG_BACKUP_SEND = 0x02,
+ DSYNC_BRAIN_FLAG_BACKUP_RECV = 0x04,
+ DSYNC_BRAIN_FLAG_DEBUG = 0x08,
+ DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES= 0x10,
+ /* Sync everything but the actual mails (e.g. mailbox creates,
+ deletes) */
+ DSYNC_BRAIN_FLAG_NO_MAIL_SYNC = 0x20,
+ /* Used with BACKUP_SEND/RECV: Don't force the
+ Use the two-way syncing algorithm, but don't actually modify
+ anything locally. (Useful during migration.) */
+ DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE = 0x40,
+ /* Run storage purge on the remote after syncing.
+ Useful with e.g. a nightly doveadm backup. */
+ DSYNC_BRAIN_FLAG_PURGE_REMOTE = 0x80,
+ /* Don't prefetch mail bodies until they're actually needed. This works
+ only with pipe ibc. It's useful if most of the mails can be copied
+ directly within filesystem without having to read them. */
+ DSYNC_BRAIN_FLAG_NO_MAIL_PREFETCH = 0x100,
+ /* Add MAILBOX_TRANSACTION_FLAG_NO_NOTIFY to transactions. */
+ DSYNC_BRAIN_FLAG_NO_NOTIFY = 0x400,
+ /* Workaround missing Date/Message-ID headers */
+ DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND = 0x800,
+};
+
+enum dsync_brain_sync_type {
+ DSYNC_BRAIN_SYNC_TYPE_UNKNOWN,
+ /* Go through all mailboxes to make sure everything is synced */
+ DSYNC_BRAIN_SYNC_TYPE_FULL,
+ /* Go through all mailboxes that have changed (based on UIDVALIDITY,
+ UIDNEXT, HIGHESTMODSEQ). If both sides have had equal amount of
+ changes in some mailbox, it may get incorrectly skipped. */
+ DSYNC_BRAIN_SYNC_TYPE_CHANGED,
+ /* Use saved state to find out what has changed. */
+ DSYNC_BRAIN_SYNC_TYPE_STATE
+};
+
+struct dsync_brain_settings {
+ const char *process_title_prefix;
+ /* Sync only these namespaces */
+ ARRAY(struct mail_namespace *) sync_namespaces;
+ /* Sync only this mailbox name */
+ const char *sync_box;
+ /* Use this virtual \All mailbox to be able to copy mails with the same
+ GUID instead of saving them twice. With most storages this results
+ in less disk space usage. */
+ const char *virtual_all_box;
+ /* Sync only this mailbox GUID */
+ guid_128_t sync_box_guid;
+ /* Exclude these mailboxes from the sync. They can contain '*'
+ wildcards and be \special-use flags. */
+ const char *const *exclude_mailboxes;
+ /* Alternative character to use in mailbox names where the original
+ character cannot be used. */
+ char mailbox_alt_char;
+ /* Sync only mails with received timestamp at least this high. */
+ time_t sync_since_timestamp;
+ /* Sync only mails with received timestamp less or equal than this */
+ time_t sync_until_timestamp;
+ /* Don't sync mails larger than this. */
+ uoff_t sync_max_size;
+ /* Sync only mails which contains / doesn't contain this flag.
+ '-' at the beginning means this flag must not exist. */
+ const char *sync_flag;
+ /* Headers to hash (defaults to Date, Message-ID) */
+ const char *const *hashed_headers;
+
+ /* If non-zero, use dsync lock file for this user */
+ unsigned int lock_timeout_secs;
+ /* If non-zero, importing will attempt to commit transaction after
+ saving this many messages. */
+ unsigned int import_commit_msgs_interval;
+ /* Input state for DSYNC_BRAIN_SYNC_TYPE_STATE */
+ const char *state;
+};
+
+#define DSYNC_LIST_CONTEXT(obj) \
+ MODULE_CONTEXT(obj, dsync_mailbox_list_module)
+struct dsync_mailbox_list {
+ union mailbox_list_module_context module_ctx;
+ bool have_orig_escape_char;
+};
+extern MODULE_CONTEXT_DEFINE(dsync_mailbox_list_module,
+ &mailbox_list_module_register);
+
+struct dsync_brain *
+dsync_brain_master_init(struct mail_user *user, struct dsync_ibc *ibc,
+ enum dsync_brain_sync_type sync_type,
+ enum dsync_brain_flags flags,
+ const struct dsync_brain_settings *set);
+struct dsync_brain *
+dsync_brain_slave_init(struct mail_user *user, struct dsync_ibc *ibc,
+ bool local, const char *process_title_prefix,
+ char default_alt_char);
+/* Returns 0 if everything was successful, -1 if syncing failed in some way */
+int dsync_brain_deinit(struct dsync_brain **brain, enum mail_error *error_r);
+
+/* Returns TRUE if brain needs to run more, FALSE if it's finished.
+ changed_r is TRUE if anything happened during this run. */
+bool dsync_brain_run(struct dsync_brain *brain, bool *changed_r);
+/* Returns TRUE if brain has failed, and there's no point in continuing. */
+bool dsync_brain_has_failed(struct dsync_brain *brain);
+/* Returns the current sync state string, which can be given as parameter to
+ dsync_brain_master_init() to quickly sync only the new changes. */
+void dsync_brain_get_state(struct dsync_brain *brain, string_t *output);
+/* Returns the sync type that was used. Mainly useful with slave brain. */
+enum dsync_brain_sync_type dsync_brain_get_sync_type(struct dsync_brain *brain);
+/* If there were any unexpected changes during the sync, return the reason
+ for them. Otherwise return NULL. If remote_only_r=TRUE, this brain itself
+ didn't see any changes, but the remote brain did. */
+const char *dsync_brain_get_unexpected_changes_reason(struct dsync_brain *brain,
+ bool *remote_only_r);
+/* Returns TRUE if we want to sync this namespace. */
+bool dsync_brain_want_namespace(struct dsync_brain *brain,
+ struct mail_namespace *ns);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-deserializer.c b/src/doveadm/dsync/dsync-deserializer.c
new file mode 100644
index 0000000..2a90c47
--- /dev/null
+++ b/src/doveadm/dsync/dsync-deserializer.c
@@ -0,0 +1,193 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "str.h"
+#include "strescape.h"
+#include "dsync-serializer.h"
+#include "dsync-deserializer.h"
+
+struct dsync_deserializer {
+ pool_t pool;
+ const char *name;
+ const char *const *required_fields;
+ const char *const *keys;
+ unsigned int *required_field_indexes;
+ unsigned int required_field_count;
+};
+
+struct dsync_deserializer_decoder {
+ pool_t pool;
+ struct dsync_deserializer *deserializer;
+ const char *const *values;
+ unsigned int values_count;
+};
+
+static bool field_find(const char *const *names, const char *name,
+ unsigned int *idx_r)
+{
+ unsigned int i;
+
+ for (i = 0; names[i] != NULL; i++) {
+ if (strcmp(names[i], name) == 0) {
+ *idx_r = i;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+int dsync_deserializer_init(const char *name, const char *const *required_fields,
+ const char *header_line,
+ struct dsync_deserializer **deserializer_r,
+ const char **error_r)
+{
+ struct dsync_deserializer *deserializer;
+ const char **dup_required_fields;
+ unsigned int i, required_count;
+ pool_t pool;
+
+ *deserializer_r = NULL;
+
+ pool = pool_alloconly_create("dsync deserializer", 1024);
+ deserializer = p_new(pool, struct dsync_deserializer, 1);
+ deserializer->pool = pool;
+ deserializer->name = p_strdup(pool, name);
+ deserializer->keys = (void *)p_strsplit_tabescaped(pool, header_line);
+
+ deserializer->required_field_count = required_count =
+ required_fields == NULL ? 0 :
+ str_array_length(required_fields);
+ dup_required_fields = p_new(pool, const char *, required_count + 1);
+ deserializer->required_field_indexes =
+ p_new(pool, unsigned int, required_count + 1);
+ for (i = 0; i < required_count; i++) {
+ dup_required_fields[i] =
+ p_strdup(pool, required_fields[i]);
+ if (!field_find(deserializer->keys, required_fields[i],
+ &deserializer->required_field_indexes[i])) {
+ *error_r = t_strdup_printf(
+ "Header missing required field %s",
+ required_fields[i]);
+ pool_unref(&pool);
+ return -1;
+ }
+ }
+ deserializer->required_fields = dup_required_fields;
+
+ *deserializer_r = deserializer;
+ return 0;
+}
+
+void dsync_deserializer_deinit(struct dsync_deserializer **_deserializer)
+{
+ struct dsync_deserializer *deserializer = *_deserializer;
+
+ *_deserializer = NULL;
+
+ pool_unref(&deserializer->pool);
+}
+
+int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer,
+ const char *input,
+ struct dsync_deserializer_decoder **decoder_r,
+ const char **error_r)
+{
+ struct dsync_deserializer_decoder *decoder;
+ unsigned int i;
+ char **values;
+ pool_t pool;
+
+ *decoder_r = NULL;
+
+ pool = pool_alloconly_create("dsync deserializer decode", 1024);
+ decoder = p_new(pool, struct dsync_deserializer_decoder, 1);
+ decoder->pool = pool;
+ decoder->deserializer = deserializer;
+ values = p_strsplit_tabescaped(pool, input);
+
+ /* fix NULLs */
+ for (i = 0; values[i] != NULL; i++) {
+ if (values[i][0] == NULL_CHR) {
+ /* NULL? */
+ if (values[i][1] == '\0')
+ values[i] = NULL;
+ else
+ values[i] += 1;
+ }
+ }
+ decoder->values_count = i;
+
+ /* see if all required fields exist */
+ for (i = 0; i < deserializer->required_field_count; i++) {
+ unsigned int ridx = deserializer->required_field_indexes[i];
+
+ if (ridx >= decoder->values_count || values[ridx] == NULL) {
+ *error_r = t_strdup_printf("Missing required field %s",
+ deserializer->required_fields[i]);
+ pool_unref(&pool);
+ return -1;
+ }
+ }
+ decoder->values = (void *)values;
+
+ *decoder_r = decoder;
+ return 0;
+}
+
+static bool
+dsync_deserializer_find_field(struct dsync_deserializer *deserializer,
+ const char *key, unsigned int *idx_r)
+{
+ unsigned int i;
+
+ for (i = 0; deserializer->keys[i] != NULL; i++) {
+ if (strcmp(deserializer->keys[i], key) == 0) {
+ *idx_r = i;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder,
+ const char *key, const char **value_r)
+{
+ unsigned int idx;
+
+ if (!dsync_deserializer_find_field(decoder->deserializer, key, &idx) ||
+ idx >= decoder->values_count) {
+ *value_r = NULL;
+ return FALSE;
+ } else {
+ *value_r = decoder->values[idx];
+ return *value_r != NULL;
+ }
+}
+
+const char *
+dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder,
+ const char *key)
+{
+ const char *value;
+
+ if (!dsync_deserializer_decode_try(decoder, key, &value)) {
+ i_panic("dsync_deserializer_decode_get() "
+ "used for non-required key %s", key);
+ }
+ return value;
+}
+
+const char *
+dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder)
+{
+ return decoder->deserializer->name;
+}
+
+void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **_decoder)
+{
+ struct dsync_deserializer_decoder *decoder = *_decoder;
+
+ *_decoder = NULL;
+
+ pool_unref(&decoder->pool);
+}
diff --git a/src/doveadm/dsync/dsync-deserializer.h b/src/doveadm/dsync/dsync-deserializer.h
new file mode 100644
index 0000000..45b19cc
--- /dev/null
+++ b/src/doveadm/dsync/dsync-deserializer.h
@@ -0,0 +1,27 @@
+#ifndef DSYNC_DESERIALIZER_H
+#define DSYNC_DESERIALIZER_H
+
+struct dsync_deserializer;
+struct dsync_deserializer_decoder;
+
+int dsync_deserializer_init(const char *name, const char *const *required_fields,
+ const char *header_line,
+ struct dsync_deserializer **deserializer_r,
+ const char **error_r);
+void dsync_deserializer_deinit(struct dsync_deserializer **deserializer);
+
+int dsync_deserializer_decode_begin(struct dsync_deserializer *deserializer,
+ const char *input,
+ struct dsync_deserializer_decoder **decoder_r,
+ const char **error_r);
+bool dsync_deserializer_decode_try(struct dsync_deserializer_decoder *decoder,
+ const char *key, const char **value_r);
+/* key must be in required fields. The return value is never NULL. */
+const char *
+dsync_deserializer_decode_get(struct dsync_deserializer_decoder *decoder,
+ const char *key);
+const char *
+dsync_deserializer_decoder_get_name(struct dsync_deserializer_decoder *decoder);
+void dsync_deserializer_decode_finish(struct dsync_deserializer_decoder **decoder);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-ibc-pipe.c b/src/doveadm/dsync/dsync-ibc-pipe.c
new file mode 100644
index 0000000..1b8886e
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc-pipe.c
@@ -0,0 +1,599 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "istream.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-state.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-ibc-private.h"
+
+enum item_type {
+ ITEM_END_OF_LIST,
+ ITEM_HANDSHAKE,
+ ITEM_MAILBOX_STATE,
+ ITEM_MAILBOX_TREE_NODE,
+ ITEM_MAILBOX_DELETE,
+ ITEM_MAILBOX,
+ ITEM_MAILBOX_ATTRIBUTE,
+ ITEM_MAIL_CHANGE,
+ ITEM_MAIL_REQUEST,
+ ITEM_MAIL,
+ ITEM_FINISH
+};
+
+struct item {
+ enum item_type type;
+ pool_t pool;
+
+ union {
+ struct dsync_ibc_settings set;
+ struct dsync_mailbox_state state;
+ struct dsync_mailbox_node node;
+ guid_128_t mailbox_guid;
+ struct dsync_mailbox dsync_box;
+ struct dsync_mailbox_attribute attr;
+ struct dsync_mail_change change;
+ struct dsync_mail_request request;
+ struct dsync_mail mail;
+ struct {
+ const struct dsync_mailbox_delete *deletes;
+ unsigned int count;
+ char hierarchy_sep;
+ char escape_char;
+ } mailbox_delete;
+ struct {
+ const char *error;
+ enum mail_error mail_error;
+ bool require_full_resync;
+ } finish;
+ } u;
+};
+
+struct dsync_ibc_pipe {
+ struct dsync_ibc ibc;
+
+ ARRAY(pool_t) pools;
+ ARRAY(struct item) item_queue;
+ struct dsync_ibc_pipe *remote;
+
+ pool_t pop_pool;
+ struct item pop_item;
+};
+
+static pool_t dsync_ibc_pipe_get_pool(struct dsync_ibc_pipe *pipe)
+{
+ pool_t *pools, ret;
+ unsigned int count;
+
+ pools = array_get_modifiable(&pipe->pools, &count);
+ if (count == 0)
+ return pool_alloconly_create(MEMPOOL_GROWING"pipe item pool", 1024);
+
+ ret = pools[count-1];
+ array_delete(&pipe->pools, count-1, 1);
+ p_clear(ret);
+ return ret;
+}
+
+static struct item * ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_pipe_push_item(struct dsync_ibc_pipe *pipe, enum item_type type)
+{
+ struct item *item;
+
+ item = array_append_space(&pipe->item_queue);
+ item->type = type;
+
+ switch (type) {
+ case ITEM_END_OF_LIST:
+ case ITEM_MAILBOX_STATE:
+ case ITEM_MAILBOX_DELETE:
+ break;
+ case ITEM_HANDSHAKE:
+ case ITEM_MAILBOX:
+ case ITEM_MAILBOX_TREE_NODE:
+ case ITEM_MAILBOX_ATTRIBUTE:
+ case ITEM_MAIL_CHANGE:
+ case ITEM_MAIL_REQUEST:
+ case ITEM_MAIL:
+ case ITEM_FINISH:
+ item->pool = dsync_ibc_pipe_get_pool(pipe);
+ break;
+ }
+ return item;
+}
+
+static struct item *
+dsync_ibc_pipe_pop_item(struct dsync_ibc_pipe *pipe, enum item_type type)
+{
+ struct item *item;
+
+ if (array_count(&pipe->item_queue) == 0)
+ return NULL;
+
+ item = array_front_modifiable(&pipe->item_queue);
+ i_assert(item->type == type);
+ pipe->pop_item = *item;
+ array_pop_front(&pipe->item_queue);
+ item = NULL;
+
+ pool_unref(&pipe->pop_pool);
+ pipe->pop_pool = pipe->pop_item.pool;
+ return &pipe->pop_item;
+}
+
+static bool dsync_ibc_pipe_try_pop_eol(struct dsync_ibc_pipe *pipe)
+{
+ const struct item *item;
+
+ if (array_count(&pipe->item_queue) == 0)
+ return FALSE;
+
+ item = array_front(&pipe->item_queue);
+ if (item->type != ITEM_END_OF_LIST)
+ return FALSE;
+
+ array_pop_front(&pipe->item_queue);
+ return TRUE;
+}
+
+static void dsync_ibc_pipe_deinit(struct dsync_ibc *ibc)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+ pool_t pool;
+
+ if (pipe->remote != NULL) {
+ i_assert(pipe->remote->remote == pipe);
+ pipe->remote->remote = NULL;
+ }
+
+ pool_unref(&pipe->pop_pool);
+ array_foreach_modifiable(&pipe->item_queue, item) {
+ pool_unref(&item->pool);
+ }
+ array_foreach_elem(&pipe->pools, pool)
+ pool_unref(&pool);
+ array_free(&pipe->pools);
+ array_free(&pipe->item_queue);
+ i_free(pipe);
+}
+
+static void
+dsync_ibc_pipe_send_handshake(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings *set)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_HANDSHAKE);
+ item->u.set = *set;
+ item->u.set.sync_ns_prefixes =
+ p_strdup(item->pool, set->sync_ns_prefixes);
+ item->u.set.sync_box = p_strdup(item->pool, set->sync_box);
+ item->u.set.virtual_all_box = p_strdup(item->pool, set->virtual_all_box);
+ item->u.set.exclude_mailboxes = set->exclude_mailboxes == NULL ? NULL :
+ p_strarray_dup(item->pool, set->exclude_mailboxes);
+ memcpy(item->u.set.sync_box_guid, set->sync_box_guid,
+ sizeof(item->u.set.sync_box_guid));
+ item->u.set.sync_since_timestamp = set->sync_since_timestamp;
+ item->u.set.sync_until_timestamp = set->sync_until_timestamp;
+ item->u.set.sync_max_size = set->sync_max_size;
+ item->u.set.sync_flags = p_strdup(item->pool, set->sync_flags);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_handshake(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings **set_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_HANDSHAKE);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *set_r = &item->u.set;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static bool dsync_ibc_pipe_is_send_queue_full(struct dsync_ibc *ibc)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+ return array_count(&pipe->remote->item_queue) > 0;
+}
+
+static bool dsync_ibc_pipe_has_pending_data(struct dsync_ibc *ibc)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+ return array_count(&pipe->item_queue) > 0;
+}
+
+static void
+dsync_ibc_pipe_send_end_of_list(struct dsync_ibc *ibc,
+ enum dsync_ibc_eol_type type ATTR_UNUSED)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+ dsync_ibc_pipe_push_item(pipe->remote, ITEM_END_OF_LIST);
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_state(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_state *state)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_STATE);
+ item->u.state = *state;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_state(struct dsync_ibc *ibc,
+ struct dsync_mailbox_state *state_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_STATE);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *state_r = item->u.state;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_tree_node(struct dsync_ibc *ibc,
+ const char *const *name,
+ const struct dsync_mailbox_node *node)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_TREE_NODE);
+
+ /* a little bit kludgy way to send it */
+ item->u.node.name = (void *)p_strarray_dup(item->pool, name);
+ dsync_mailbox_node_copy_data(&item->u.node, node);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_tree_node(struct dsync_ibc *ibc,
+ const char *const **name_r,
+ const struct dsync_mailbox_node **node_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_TREE_NODE);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *name_r = (const void *)item->u.node.name;
+ item->u.node.name = NULL;
+
+ *node_r = &item->u.node;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_deletes(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete *deletes,
+ unsigned int count, char hierarchy_sep,
+ char escape_char)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_DELETE);
+
+ /* we'll assume that the deletes are permanent. this works for now.. */
+ /* a little bit kludgy way to send it */
+ item->u.mailbox_delete.deletes = deletes;
+ item->u.mailbox_delete.count = count;
+ item->u.mailbox_delete.hierarchy_sep = hierarchy_sep;
+ item->u.mailbox_delete.escape_char = escape_char;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_deletes(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete **deletes_r,
+ unsigned int *count_r,
+ char *hierarchy_sep_r,
+ char *escape_char_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_DELETE);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *deletes_r = item->u.mailbox_delete.deletes;
+ *count_r = item->u.mailbox_delete.count;
+ *hierarchy_sep_r = item->u.mailbox_delete.hierarchy_sep;
+ *escape_char_r = item->u.mailbox_delete.escape_char;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox(struct dsync_ibc *ibc,
+ const struct dsync_mailbox *dsync_box)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+ const struct mailbox_cache_field *cf;
+ struct mailbox_cache_field *ncf;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX);
+ item->u.dsync_box = *dsync_box;
+ p_array_init(&item->u.dsync_box.cache_fields, item->pool,
+ array_count(&dsync_box->cache_fields));
+ array_foreach(&dsync_box->cache_fields, cf) {
+ ncf = array_append_space(&item->u.dsync_box.cache_fields);
+ ncf->name = p_strdup(item->pool, cf->name);
+ ncf->decision = cf->decision;
+ ncf->last_used = cf->last_used;
+ }
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox(struct dsync_ibc *ibc,
+ const struct dsync_mailbox **dsync_box_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *dsync_box_r = &item->u.dsync_box;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mailbox_attribute(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute *attr)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAILBOX_ATTRIBUTE);
+ dsync_mailbox_attribute_dup(item->pool, attr, &item->u.attr);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mailbox_attribute(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute **attr_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAILBOX_ATTRIBUTE);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *attr_r = &item->u.attr;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_change(struct dsync_ibc *ibc,
+ const struct dsync_mail_change *change)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_CHANGE);
+ dsync_mail_change_dup(item->pool, change, &item->u.change);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_change(struct dsync_ibc *ibc,
+ const struct dsync_mail_change **change_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_CHANGE);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *change_r = &item->u.change;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mail_request(struct dsync_ibc *ibc,
+ const struct dsync_mail_request *request)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL_REQUEST);
+ item->u.request.guid = p_strdup(item->pool, request->guid);
+ item->u.request.uid = request->uid;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mail_request(struct dsync_ibc *ibc,
+ const struct dsync_mail_request **request_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL_REQUEST);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *request_r = &item->u.request;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_MAIL);
+ item->u.mail.guid = p_strdup(item->pool, mail->guid);
+ item->u.mail.uid = mail->uid;
+ item->u.mail.pop3_uidl = p_strdup(item->pool, mail->pop3_uidl);
+ item->u.mail.pop3_order = mail->pop3_order;
+ item->u.mail.received_date = mail->received_date;
+ if (mail->input != NULL) {
+ item->u.mail.input = mail->input;
+ i_stream_ref(mail->input);
+ }
+ item->u.mail.input_mail = mail->input_mail;
+ item->u.mail.input_mail_uid = mail->input_mail_uid;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ if (dsync_ibc_pipe_try_pop_eol(pipe))
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_MAIL);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *mail_r = &item->u.mail;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_pipe_send_finish(struct dsync_ibc *ibc, const char *error,
+ enum mail_error mail_error,
+ bool require_full_resync)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_push_item(pipe->remote, ITEM_FINISH);
+ item->u.finish.error = p_strdup(item->pool, error);
+ item->u.finish.mail_error = mail_error;
+ item->u.finish.require_full_resync = require_full_resync;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_pipe_recv_finish(struct dsync_ibc *ibc, const char **error_r,
+ enum mail_error *mail_error_r,
+ bool *require_full_resync_r)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+ struct item *item;
+
+ item = dsync_ibc_pipe_pop_item(pipe, ITEM_FINISH);
+ if (item == NULL)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ *error_r = item->u.finish.error;
+ *mail_error_r = item->u.finish.mail_error;
+ *require_full_resync_r = item->u.finish.require_full_resync;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void pipe_close_mail_streams(struct dsync_ibc_pipe *pipe)
+{
+ struct item *item;
+
+ if (array_count(&pipe->item_queue) > 0) {
+ item = array_front_modifiable(&pipe->item_queue);
+ if (item->type == ITEM_MAIL &&
+ item->u.mail.input != NULL)
+ i_stream_unref(&item->u.mail.input);
+ }
+}
+
+static void dsync_ibc_pipe_close_mail_streams(struct dsync_ibc *ibc)
+{
+ struct dsync_ibc_pipe *pipe = (struct dsync_ibc_pipe *)ibc;
+
+ pipe_close_mail_streams(pipe);
+ pipe_close_mail_streams(pipe->remote);
+}
+
+static const struct dsync_ibc_vfuncs dsync_ibc_pipe_vfuncs = {
+ dsync_ibc_pipe_deinit,
+ dsync_ibc_pipe_send_handshake,
+ dsync_ibc_pipe_recv_handshake,
+ dsync_ibc_pipe_send_end_of_list,
+ dsync_ibc_pipe_send_mailbox_state,
+ dsync_ibc_pipe_recv_mailbox_state,
+ dsync_ibc_pipe_send_mailbox_tree_node,
+ dsync_ibc_pipe_recv_mailbox_tree_node,
+ dsync_ibc_pipe_send_mailbox_deletes,
+ dsync_ibc_pipe_recv_mailbox_deletes,
+ dsync_ibc_pipe_send_mailbox,
+ dsync_ibc_pipe_recv_mailbox,
+ dsync_ibc_pipe_send_mailbox_attribute,
+ dsync_ibc_pipe_recv_mailbox_attribute,
+ dsync_ibc_pipe_send_change,
+ dsync_ibc_pipe_recv_change,
+ dsync_ibc_pipe_send_mail_request,
+ dsync_ibc_pipe_recv_mail_request,
+ dsync_ibc_pipe_send_mail,
+ dsync_ibc_pipe_recv_mail,
+ dsync_ibc_pipe_send_finish,
+ dsync_ibc_pipe_recv_finish,
+ dsync_ibc_pipe_close_mail_streams,
+ dsync_ibc_pipe_is_send_queue_full,
+ dsync_ibc_pipe_has_pending_data
+};
+
+static struct dsync_ibc_pipe *
+dsync_ibc_pipe_alloc(void)
+{
+ struct dsync_ibc_pipe *pipe;
+
+ pipe = i_new(struct dsync_ibc_pipe, 1);
+ pipe->ibc.v = dsync_ibc_pipe_vfuncs;
+ i_array_init(&pipe->pools, 4);
+ i_array_init(&pipe->item_queue, 4);
+ return pipe;
+}
+
+void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r, struct dsync_ibc **ibc2_r)
+{
+ struct dsync_ibc_pipe *pipe1, *pipe2;
+
+ pipe1 = dsync_ibc_pipe_alloc();
+ pipe2 = dsync_ibc_pipe_alloc();
+ pipe1->remote = pipe2;
+ pipe2->remote = pipe1;
+ *ibc1_r = &pipe1->ibc;
+ *ibc2_r = &pipe2->ibc;
+}
diff --git a/src/doveadm/dsync/dsync-ibc-private.h b/src/doveadm/dsync/dsync-ibc-private.h
new file mode 100644
index 0000000..c055aca
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc-private.h
@@ -0,0 +1,96 @@
+#ifndef DSYNC_IBC_PRIVATE_H
+#define DSYNC_IBC_PRIVATE_H
+
+#include "dsync-ibc.h"
+
+struct dsync_ibc_vfuncs {
+ void (*deinit)(struct dsync_ibc *ibc);
+
+ void (*send_handshake)(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings *set);
+ enum dsync_ibc_recv_ret
+ (*recv_handshake)(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings **set_r);
+
+ void (*send_end_of_list)(struct dsync_ibc *ibc,
+ enum dsync_ibc_eol_type type);
+
+ void (*send_mailbox_state)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_state *state);
+ enum dsync_ibc_recv_ret
+ (*recv_mailbox_state)(struct dsync_ibc *ibc,
+ struct dsync_mailbox_state *state_r);
+
+ void (*send_mailbox_tree_node)(struct dsync_ibc *ibc,
+ const char *const *name,
+ const struct dsync_mailbox_node *node);
+ enum dsync_ibc_recv_ret
+ (*recv_mailbox_tree_node)(struct dsync_ibc *ibc,
+ const char *const **name_r,
+ const struct dsync_mailbox_node **node_r);
+
+ void (*send_mailbox_deletes)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete *deletes,
+ unsigned int count, char hierarchy_sep,
+ char escape_char);
+ enum dsync_ibc_recv_ret
+ (*recv_mailbox_deletes)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete **deletes_r,
+ unsigned int *count_r,
+ char *hierarchy_sep_r,
+ char *escape_char_r);
+
+ void (*send_mailbox)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox *dsync_box);
+ enum dsync_ibc_recv_ret
+ (*recv_mailbox)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox **dsync_box_r);
+
+ void (*send_mailbox_attribute)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute *attr);
+ enum dsync_ibc_recv_ret
+ (*recv_mailbox_attribute)(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute **attr_r);
+
+ void (*send_change)(struct dsync_ibc *ibc,
+ const struct dsync_mail_change *change);
+ enum dsync_ibc_recv_ret
+ (*recv_change)(struct dsync_ibc *ibc,
+ const struct dsync_mail_change **change_r);
+
+ void (*send_mail_request)(struct dsync_ibc *ibc,
+ const struct dsync_mail_request *request);
+ enum dsync_ibc_recv_ret
+ (*recv_mail_request)(struct dsync_ibc *ibc,
+ const struct dsync_mail_request **request_r);
+
+ void (*send_mail)(struct dsync_ibc *ibc,
+ const struct dsync_mail *mail);
+ enum dsync_ibc_recv_ret
+ (*recv_mail)(struct dsync_ibc *ibc,
+ struct dsync_mail **mail_r);
+
+ void (*send_finish)(struct dsync_ibc *ibc, const char *error,
+ enum mail_error mail_error,
+ bool require_full_resync);
+ enum dsync_ibc_recv_ret
+ (*recv_finish)(struct dsync_ibc *ibc, const char **error_r,
+ enum mail_error *mail_error_r,
+ bool *require_full_resync_r);
+
+ void (*close_mail_streams)(struct dsync_ibc *ibc);
+ bool (*is_send_queue_full)(struct dsync_ibc *ibc);
+ bool (*has_pending_data)(struct dsync_ibc *ibc);
+};
+
+struct dsync_ibc {
+ struct dsync_ibc_vfuncs v;
+
+ io_callback_t *io_callback;
+ void *io_context;
+
+ bool failed:1;
+ bool timeout:1;
+};
+
+#endif
diff --git a/src/doveadm/dsync/dsync-ibc-stream.c b/src/doveadm/dsync/dsync-ibc-stream.c
new file mode 100644
index 0000000..17115b0
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc-stream.c
@@ -0,0 +1,2138 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "safe-mkstemp.h"
+#include "ioloop.h"
+#include "istream.h"
+#include "istream-seekable.h"
+#include "istream-dot.h"
+#include "ostream.h"
+#include "str.h"
+#include "strescape.h"
+#include "master-service.h"
+#include "mail-cache.h"
+#include "mail-storage-private.h"
+#include "dsync-serializer.h"
+#include "dsync-deserializer.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-state.h"
+#include "dsync-mailbox-tree.h"
+#include "dsync-ibc-private.h"
+
+
+#define DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE (1024*128)
+
+#define DSYNC_PROTOCOL_VERSION_MAJOR 3
+#define DSYNC_PROTOCOL_VERSION_MINOR 5
+#define DSYNC_HANDSHAKE_VERSION "VERSION\tdsync\t3\t5\n"
+
+#define DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES 1
+#define DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID 2
+#define DSYNC_PROTOCOL_MINOR_HAVE_FINISH 3
+#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2 4
+#define DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3 5
+
+enum item_type {
+ ITEM_NONE,
+ ITEM_DONE,
+
+ ITEM_HANDSHAKE,
+ ITEM_MAILBOX_STATE,
+ ITEM_MAILBOX_TREE_NODE,
+ ITEM_MAILBOX_DELETE,
+ ITEM_MAILBOX,
+
+ ITEM_MAILBOX_ATTRIBUTE,
+ ITEM_MAIL_CHANGE,
+ ITEM_MAIL_REQUEST,
+ ITEM_MAIL,
+ ITEM_FINISH,
+
+ ITEM_MAILBOX_CACHE_FIELD,
+
+ ITEM_END_OF_LIST
+};
+
+#define END_OF_LIST_LINE "."
+static const struct {
+ /* full human readable name of the item */
+ const char *name;
+ /* unique character identifying the item */
+ char chr;
+ const char *required_keys;
+ const char *optional_keys;
+ unsigned int min_minor_version;
+} items[ITEM_END_OF_LIST+1] = {
+ { NULL, '\0', NULL, NULL, 0 },
+ { .name = "done",
+ .chr = 'X',
+ .optional_keys = ""
+ },
+ { .name = "handshake",
+ .chr = 'H',
+ .required_keys = "hostname",
+ .optional_keys = "sync_ns_prefix sync_box sync_box_guid sync_type "
+ "debug sync_visible_namespaces exclude_mailboxes "
+ "send_mail_requests backup_send backup_recv lock_timeout "
+ "no_mail_sync no_backup_overwrite purge_remote "
+ "no_notify sync_since_timestamp sync_max_size sync_flags sync_until_timestamp "
+ "virtual_all_box empty_hdr_workaround import_commit_msgs_interval "
+ "hashed_headers alt_char"
+ },
+ { .name = "mailbox_state",
+ .chr = 'S',
+ .required_keys = "mailbox_guid last_uidvalidity last_common_uid "
+ "last_common_modseq last_common_pvt_modseq",
+ .optional_keys = "last_messages_count changes_during_sync"
+ },
+ { .name = "mailbox_tree_node",
+ .chr = 'N',
+ .required_keys = "name existence",
+ .optional_keys = "mailbox_guid uid_validity uid_next "
+ "last_renamed_or_created subscribed last_subscription_change"
+ },
+ { .name = "mailbox_delete",
+ .chr = 'D',
+ .required_keys = "hierarchy_sep",
+ .optional_keys = "escape_char mailboxes dirs unsubscribes"
+ },
+ { .name = "mailbox",
+ .chr = 'B',
+ .required_keys = "mailbox_guid uid_validity uid_next messages_count "
+ "first_recent_uid highest_modseq highest_pvt_modseq",
+ .optional_keys = "mailbox_lost mailbox_ignore "
+ "cache_fields have_guids have_save_guids have_only_guid128"
+ },
+ { .name = "mailbox_attribute",
+ .chr = 'A',
+ .required_keys = "type key",
+ .optional_keys = "value stream deleted last_change modseq",
+ .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES
+ },
+ { .name = "mail_change",
+ .chr = 'C',
+ .required_keys = "type uid",
+ .optional_keys = "guid hdr_hash modseq pvt_modseq "
+ "add_flags remove_flags final_flags "
+ "keywords_reset keyword_changes received_timestamp virtual_size"
+ },
+ { .name = "mail_request",
+ .chr = 'R',
+ .optional_keys = "guid uid"
+ },
+ { .name = "mail",
+ .chr = 'M',
+ .optional_keys = "guid uid pop3_uidl pop3_order received_date saved_date stream"
+ },
+ { .name = "finish",
+ .chr = 'F',
+ .optional_keys = "error mail_error require_full_resync",
+ .min_minor_version = DSYNC_PROTOCOL_MINOR_HAVE_FINISH
+ },
+ { .name = "mailbox_cache_field",
+ .chr = 'c',
+ .required_keys = "name decision",
+ .optional_keys = "last_used"
+ },
+
+ { "end_of_list", '\0', NULL, NULL, 0 }
+};
+
+struct dsync_ibc_stream {
+ struct dsync_ibc ibc;
+
+ char *name, *temp_path_prefix;
+ unsigned int timeout_secs;
+ struct istream *input;
+ struct ostream *output;
+ struct io *io;
+ struct timeout *to;
+
+ unsigned int minor_version;
+ struct dsync_serializer *serializers[ITEM_END_OF_LIST];
+ struct dsync_deserializer *deserializers[ITEM_END_OF_LIST];
+
+ pool_t ret_pool;
+ struct dsync_deserializer_decoder *cur_decoder;
+
+ struct istream *value_output, *value_input;
+ struct dsync_mail *cur_mail;
+ struct dsync_mailbox_attribute *cur_attr;
+ char value_output_last;
+
+ enum item_type last_recv_item, last_sent_item;
+ bool last_recv_item_eol:1;
+ bool last_sent_item_eol:1;
+
+ bool version_received:1;
+ bool handshake_received:1;
+ bool has_pending_data:1;
+ bool finish_received:1;
+ bool done_received:1;
+ bool stopped:1;
+};
+
+static const char *dsync_ibc_stream_get_state(struct dsync_ibc_stream *ibc)
+{
+ if (!ibc->version_received)
+ return "version not received";
+ else if (!ibc->handshake_received)
+ return "handshake not received";
+
+ return t_strdup_printf("last sent=%s%s, last recv=%s%s",
+ items[ibc->last_sent_item].name,
+ ibc->last_sent_item_eol ? " (EOL)" : "",
+ items[ibc->last_recv_item].name,
+ ibc->last_recv_item_eol ? " (EOL)" : "");
+}
+
+static void dsync_ibc_stream_stop(struct dsync_ibc_stream *ibc)
+{
+ ibc->stopped = TRUE;
+ i_stream_close(ibc->input);
+ o_stream_close(ibc->output);
+ io_loop_stop(current_ioloop);
+}
+
+static int dsync_ibc_stream_read_mail_stream(struct dsync_ibc_stream *ibc)
+{
+ do {
+ i_stream_skip(ibc->value_input,
+ i_stream_get_data_size(ibc->value_input));
+ } while (i_stream_read(ibc->value_input) > 0);
+ if (ibc->value_input->eof) {
+ if (ibc->value_input->stream_errno != 0) {
+ i_error("dsync(%s): read(%s) failed: %s (%s)", ibc->name,
+ i_stream_get_name(ibc->value_input),
+ i_stream_get_error(ibc->value_input),
+ dsync_ibc_stream_get_state(ibc));
+ dsync_ibc_stream_stop(ibc);
+ return -1;
+ }
+ /* finished reading the mail stream */
+ i_assert(ibc->value_input->eof);
+ i_stream_seek(ibc->value_input, 0);
+ ibc->has_pending_data = TRUE;
+ ibc->value_input = NULL;
+ return 1;
+ }
+ return 0;
+}
+
+static void dsync_ibc_stream_input(struct dsync_ibc_stream *ibc)
+{
+ timeout_reset(ibc->to);
+ if (ibc->value_input != NULL) {
+ if (dsync_ibc_stream_read_mail_stream(ibc) == 0)
+ return;
+ }
+ o_stream_cork(ibc->output);
+ ibc->ibc.io_callback(ibc->ibc.io_context);
+ o_stream_uncork(ibc->output);
+}
+
+static int dsync_ibc_stream_send_value_stream(struct dsync_ibc_stream *ibc)
+{
+ const unsigned char *data;
+ unsigned char add;
+ size_t i, size;
+ int ret;
+
+ while ((ret = i_stream_read_more(ibc->value_output, &data, &size)) > 0) {
+ add = '\0';
+ for (i = 0; i < size; i++) {
+ if (data[i] == '.' &&
+ ((i == 0 && ibc->value_output_last == '\n') ||
+ (i > 0 && data[i-1] == '\n'))) {
+ /* escape the dot */
+ add = '.';
+ break;
+ }
+ }
+
+ if (i > 0) {
+ o_stream_nsend(ibc->output, data, i);
+ ibc->value_output_last = data[i-1];
+ i_stream_skip(ibc->value_output, i);
+ }
+
+ if (o_stream_get_buffer_used_size(ibc->output) >= 4096) {
+ if ((ret = o_stream_flush(ibc->output)) < 0) {
+ dsync_ibc_stream_stop(ibc);
+ return -1;
+ }
+ if (ret == 0) {
+ /* continue later */
+ o_stream_set_flush_pending(ibc->output, TRUE);
+ return 0;
+ }
+ }
+
+ if (add != '\0') {
+ o_stream_nsend(ibc->output, &add, 1);
+ ibc->value_output_last = add;
+ }
+ }
+ i_assert(ret == -1);
+
+ if (ibc->value_output->stream_errno != 0) {
+ i_error("dsync(%s): read(%s) failed: %s (%s)",
+ ibc->name, i_stream_get_name(ibc->value_output),
+ i_stream_get_error(ibc->value_output),
+ dsync_ibc_stream_get_state(ibc));
+ dsync_ibc_stream_stop(ibc);
+ return -1;
+ }
+
+ /* finished sending the stream. use "CRLF." instead of "LF." just in
+ case we're sending binary data that ends with CR. */
+ o_stream_nsend_str(ibc->output, "\r\n.\r\n");
+ i_stream_unref(&ibc->value_output);
+ return 1;
+}
+
+static int dsync_ibc_stream_output(struct dsync_ibc_stream *ibc)
+{
+ struct ostream *output = ibc->output;
+ int ret;
+
+ if ((ret = o_stream_flush(output)) < 0)
+ ret = 1;
+ else if (ibc->value_output != NULL) {
+ if (dsync_ibc_stream_send_value_stream(ibc) < 0)
+ ret = 1;
+ }
+ timeout_reset(ibc->to);
+
+ if (!dsync_ibc_is_send_queue_full(&ibc->ibc))
+ ibc->ibc.io_callback(ibc->ibc.io_context);
+ return ret;
+}
+
+static void dsync_ibc_stream_timeout(struct dsync_ibc_stream *ibc)
+{
+ i_error("dsync(%s): I/O has stalled, no activity for %u seconds (%s)",
+ ibc->name, ibc->timeout_secs, dsync_ibc_stream_get_state(ibc));
+ ibc->ibc.timeout = TRUE;
+ dsync_ibc_stream_stop(ibc);
+}
+
+static void dsync_ibc_stream_init(struct dsync_ibc_stream *ibc)
+{
+ unsigned int i;
+
+ ibc->io = io_add_istream(ibc->input, dsync_ibc_stream_input, ibc);
+ io_set_pending(ibc->io);
+ o_stream_set_no_error_handling(ibc->output, TRUE);
+ o_stream_set_flush_callback(ibc->output, dsync_ibc_stream_output, ibc);
+ ibc->to = timeout_add(ibc->timeout_secs * 1000,
+ dsync_ibc_stream_timeout, ibc);
+ o_stream_cork(ibc->output);
+ o_stream_nsend_str(ibc->output, DSYNC_HANDSHAKE_VERSION);
+
+ /* initialize serializers and send their headers to remote */
+ for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) T_BEGIN {
+ const char *keys;
+
+ keys = items[i].required_keys == NULL ? items[i].optional_keys :
+ t_strconcat(items[i].required_keys, " ",
+ items[i].optional_keys, NULL);
+ if (keys != NULL) {
+ i_assert(items[i].chr != '\0');
+
+ ibc->serializers[i] =
+ dsync_serializer_init(t_strsplit_spaces(keys, " "));
+ o_stream_nsend(ibc->output, &items[i].chr, 1);
+ o_stream_nsend_str(ibc->output,
+ dsync_serializer_encode_header_line(ibc->serializers[i]));
+ }
+ } T_END;
+ o_stream_nsend_str(ibc->output, ".\n");
+ o_stream_uncork(ibc->output);
+}
+
+static void dsync_ibc_stream_deinit(struct dsync_ibc *_ibc)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ unsigned int i;
+
+ for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) {
+ if (ibc->serializers[i] != NULL)
+ dsync_serializer_deinit(&ibc->serializers[i]);
+ if (ibc->deserializers[i] != NULL)
+ dsync_deserializer_deinit(&ibc->deserializers[i]);
+ }
+ if (ibc->cur_decoder != NULL)
+ dsync_deserializer_decode_finish(&ibc->cur_decoder);
+ if (ibc->value_output != NULL)
+ i_stream_unref(&ibc->value_output);
+ else {
+ /* If the remote has not told us that they are closing we
+ notify remote that we're closing. this is mainly to avoid
+ "read() failed: EOF" errors on failing dsyncs.
+
+ Avoid a race condition:
+ We do not tell the remote we are closing if they have
+ already told us because they close the
+ connection after sending ITEM_DONE and will
+ not be ever receive anything else from us unless
+ it just happens to get combined into the same packet
+ as a previous response and is already in the buffer.
+ */
+ if (!ibc->done_received && !ibc->finish_received) {
+ o_stream_nsend_str(ibc->output,
+ t_strdup_printf("%c\n", items[ITEM_DONE].chr));
+ }
+ (void)o_stream_finish(ibc->output);
+ }
+
+ timeout_remove(&ibc->to);
+ io_remove(&ibc->io);
+ i_stream_destroy(&ibc->input);
+ o_stream_destroy(&ibc->output);
+ pool_unref(&ibc->ret_pool);
+ i_free(ibc->temp_path_prefix);
+ i_free(ibc->name);
+ i_free(ibc);
+}
+
+static int dsync_ibc_stream_next_line(struct dsync_ibc_stream *ibc,
+ const char **line_r)
+{
+ string_t *error;
+ const char *line;
+ ssize_t ret;
+
+ line = i_stream_next_line(ibc->input);
+ if (line != NULL) {
+ *line_r = line;
+ return 1;
+ }
+ /* try reading some */
+ if ((ret = i_stream_read(ibc->input)) == -1) {
+ if (ibc->stopped)
+ return -1;
+ error = t_str_new(128);
+ if (ibc->input->stream_errno != 0) {
+ str_printfa(error, "read(%s) failed: %s", ibc->name,
+ i_stream_get_error(ibc->input));
+ } else {
+ i_assert(ibc->input->eof);
+ str_printfa(error, "read(%s) failed: EOF", ibc->name);
+ }
+ str_printfa(error, " (%s)", dsync_ibc_stream_get_state(ibc));
+ i_error("%s", str_c(error));
+ dsync_ibc_stream_stop(ibc);
+ return -1;
+ }
+ i_assert(ret >= 0);
+ *line_r = i_stream_next_line(ibc->input);
+ if (*line_r == NULL) {
+ ibc->has_pending_data = FALSE;
+ return 0;
+ }
+ ibc->has_pending_data = TRUE;
+ return 1;
+}
+
+static void ATTR_FORMAT(3, 4) ATTR_NULL(2)
+dsync_ibc_input_error(struct dsync_ibc_stream *ibc,
+ struct dsync_deserializer_decoder *decoder,
+ const char *fmt, ...)
+{
+ va_list args;
+ const char *error;
+
+ va_start(args, fmt);
+ error = t_strdup_vprintf(fmt, args);
+ if (decoder == NULL)
+ i_error("dsync(%s): %s", ibc->name, error);
+ else {
+ i_error("dsync(%s): %s: %s", ibc->name,
+ dsync_deserializer_decoder_get_name(decoder), error);
+ }
+ va_end(args);
+
+ dsync_ibc_stream_stop(ibc);
+}
+
+static void
+dsync_ibc_stream_send_string(struct dsync_ibc_stream *ibc,
+ const string_t *str)
+{
+ i_assert(ibc->value_output == NULL);
+ o_stream_nsend(ibc->output, str_data(str), str_len(str));
+}
+
+static int seekable_fd_callback(const char **path_r, void *context)
+{
+ struct dsync_ibc_stream *ibc = context;
+ string_t *path;
+ int fd;
+
+ path = t_str_new(128);
+ str_append(path, ibc->temp_path_prefix);
+ fd = safe_mkstemp(path, 0600, (uid_t)-1, (gid_t)-1);
+ if (fd == -1) {
+ i_error("safe_mkstemp(%s) failed: %m", str_c(path));
+ return -1;
+ }
+
+ /* we just want the fd, unlink it */
+ if (i_unlink(str_c(path)) < 0) {
+ /* shouldn't happen.. */
+ i_close_fd(&fd);
+ return -1;
+ }
+
+ *path_r = str_c(path);
+ return fd;
+}
+
+static struct istream *
+dsync_ibc_stream_input_stream(struct dsync_ibc_stream *ibc)
+{
+ struct istream *inputs[2];
+
+ inputs[0] = i_stream_create_dot(ibc->input, FALSE);
+ inputs[1] = NULL;
+ ibc->value_input = i_stream_create_seekable(inputs, MAIL_READ_FULL_BLOCK_SIZE,
+ seekable_fd_callback, ibc);
+ i_stream_unref(&inputs[0]);
+
+ return ibc->value_input;
+}
+
+static int
+dsync_ibc_check_missing_deserializers(struct dsync_ibc_stream *ibc)
+{
+ unsigned int i;
+ int ret = 0;
+
+ for (i = ITEM_DONE + 1; i < ITEM_END_OF_LIST; i++) {
+ if (ibc->deserializers[i] == NULL &&
+ ibc->minor_version >= items[i].min_minor_version &&
+ (items[i].required_keys != NULL ||
+ items[i].optional_keys != NULL)) {
+ dsync_ibc_input_error(ibc, NULL,
+ "Remote didn't handshake deserializer for %s",
+ items[i].name);
+ ret = -1;
+ }
+ }
+ return ret;
+}
+
+static bool
+dsync_ibc_stream_handshake(struct dsync_ibc_stream *ibc, const char *line)
+{
+ enum item_type item = ITEM_NONE;
+ const char *const *required_keys, *error;
+ unsigned int i;
+
+ if (ibc->handshake_received)
+ return TRUE;
+
+ if (!ibc->version_received) {
+ if (!version_string_verify_full(line, "dsync",
+ DSYNC_PROTOCOL_VERSION_MAJOR,
+ &ibc->minor_version)) {
+ dsync_ibc_input_error(ibc, NULL,
+ "Remote dsync doesn't use compatible protocol");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ ibc->version_received = TRUE;
+ return FALSE;
+ }
+
+ if (strcmp(line, END_OF_LIST_LINE) == 0) {
+ /* finished handshaking */
+ if (dsync_ibc_check_missing_deserializers(ibc) < 0)
+ return FALSE;
+ ibc->handshake_received = TRUE;
+ ibc->last_recv_item = ITEM_HANDSHAKE;
+ return FALSE;
+ }
+
+ for (i = 1; i < ITEM_END_OF_LIST; i++) {
+ if (items[i].chr == line[0]) {
+ item = i;
+ break;
+ }
+ }
+ if (item == ITEM_NONE) {
+ /* unknown deserializer, ignore */
+ return FALSE;
+ }
+
+ required_keys = items[item].required_keys == NULL ? NULL :
+ t_strsplit(items[item].required_keys, " ");
+ if (dsync_deserializer_init(items[item].name,
+ required_keys, line + 1,
+ &ibc->deserializers[item], &error) < 0) {
+ dsync_ibc_input_error(ibc, NULL,
+ "Remote sent invalid handshake for %s: %s",
+ items[item].name, error);
+ }
+ return FALSE;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_input_next(struct dsync_ibc_stream *ibc, enum item_type item,
+ struct dsync_deserializer_decoder **decoder_r)
+{
+ enum item_type line_item = ITEM_NONE;
+ const char *line, *error;
+ unsigned int i;
+
+ i_assert(ibc->value_input == NULL);
+
+ timeout_reset(ibc->to);
+
+ do {
+ if (dsync_ibc_stream_next_line(ibc, &line) <= 0)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ } while (!dsync_ibc_stream_handshake(ibc, line));
+
+ ibc->last_recv_item = item;
+ ibc->last_recv_item_eol = FALSE;
+
+ if (strcmp(line, END_OF_LIST_LINE) == 0) {
+ /* end of this list */
+ ibc->last_recv_item_eol = TRUE;
+ return DSYNC_IBC_RECV_RET_FINISHED;
+ }
+ if (line[0] == items[ITEM_DONE].chr) {
+ /* remote cleanly closed the connection, possibly because of
+ some failure (which it should have logged). we don't want to
+ log any stream errors anyway after this. */
+ ibc->done_received = TRUE;
+ dsync_ibc_stream_stop(ibc);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+
+ }
+ for (i = 1; i < ITEM_END_OF_LIST; i++) {
+ if (*line == items[i].chr) {
+ line_item = i;
+ break;
+ }
+ }
+ if (line_item != item) {
+ dsync_ibc_input_error(ibc, NULL,
+ "Received unexpected input %c != %c",
+ *line, items[item].chr);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ if (ibc->cur_decoder != NULL)
+ dsync_deserializer_decode_finish(&ibc->cur_decoder);
+ if (dsync_deserializer_decode_begin(ibc->deserializers[item],
+ line+1, &ibc->cur_decoder,
+ &error) < 0) {
+ dsync_ibc_input_error(ibc, NULL, "Invalid input to %s: %s",
+ items[item].name, error);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ *decoder_r = ibc->cur_decoder;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static struct dsync_serializer_encoder *
+dsync_ibc_send_encode_begin(struct dsync_ibc_stream *ibc, enum item_type item)
+{
+ ibc->last_sent_item = item;
+ ibc->last_sent_item_eol = FALSE;
+ return dsync_serializer_encode_begin(ibc->serializers[item]);
+}
+
+static void
+dsync_ibc_stream_send_handshake(struct dsync_ibc *_ibc,
+ const struct dsync_ibc_settings *set)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+ char sync_type[2];
+
+ str_append_c(str, items[ITEM_HANDSHAKE].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_HANDSHAKE);
+ dsync_serializer_encode_add(encoder, "hostname", set->hostname);
+ if (set->sync_ns_prefixes != NULL) {
+ dsync_serializer_encode_add(encoder, "sync_ns_prefix",
+ set->sync_ns_prefixes);
+ }
+ if (set->sync_box != NULL)
+ dsync_serializer_encode_add(encoder, "sync_box", set->sync_box);
+ if (set->virtual_all_box != NULL) {
+ dsync_serializer_encode_add(encoder, "virtual_all_box",
+ set->virtual_all_box);
+ }
+ if (set->exclude_mailboxes != NULL) {
+ string_t *substr = t_str_new(64);
+ unsigned int i;
+
+ for (i = 0; set->exclude_mailboxes[i] != NULL; i++) {
+ if (i != 0)
+ str_append_c(substr, '\t');
+ str_append_tabescaped(substr, set->exclude_mailboxes[i]);
+ }
+ dsync_serializer_encode_add(encoder, "exclude_mailboxes",
+ str_c(substr));
+ }
+ if (!guid_128_is_empty(set->sync_box_guid)) {
+ dsync_serializer_encode_add(encoder, "sync_box_guid",
+ guid_128_to_string(set->sync_box_guid));
+ }
+
+ sync_type[0] = sync_type[1] = '\0';
+ switch (set->sync_type) {
+ case DSYNC_BRAIN_SYNC_TYPE_UNKNOWN:
+ break;
+ case DSYNC_BRAIN_SYNC_TYPE_FULL:
+ sync_type[0] = 'f';
+ break;
+ case DSYNC_BRAIN_SYNC_TYPE_CHANGED:
+ sync_type[0] = 'c';
+ break;
+ case DSYNC_BRAIN_SYNC_TYPE_STATE:
+ sync_type[0] = 's';
+ break;
+ }
+ if (sync_type[0] != '\0')
+ dsync_serializer_encode_add(encoder, "sync_type", sync_type);
+ if (set->lock_timeout > 0) {
+ dsync_serializer_encode_add(encoder, "lock_timeout",
+ t_strdup_printf("%u", set->lock_timeout));
+ }
+ if (set->import_commit_msgs_interval > 0) {
+ dsync_serializer_encode_add(encoder, "import_commit_msgs_interval",
+ t_strdup_printf("%u", set->import_commit_msgs_interval));
+ }
+ if (set->sync_since_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "sync_since_timestamp",
+ t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+ }
+ if (set->sync_until_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "sync_until_timestamp",
+ t_strdup_printf("%ld", (long)set->sync_since_timestamp));
+ }
+ if (set->sync_max_size > 0) {
+ dsync_serializer_encode_add(encoder, "sync_max_size",
+ t_strdup_printf("%"PRIu64, set->sync_max_size));
+ }
+ if (set->sync_flags != NULL) {
+ dsync_serializer_encode_add(encoder, "sync_flags",
+ set->sync_flags);
+ }
+ if (set->alt_char != '\0') {
+ dsync_serializer_encode_add(encoder, "alt_char",
+ t_strdup_printf("%c", set->alt_char));
+ }
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS) != 0)
+ dsync_serializer_encode_add(encoder, "send_mail_requests", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_SEND) != 0)
+ dsync_serializer_encode_add(encoder, "backup_send", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_BACKUP_RECV) != 0)
+ dsync_serializer_encode_add(encoder, "backup_recv", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_DEBUG) != 0)
+ dsync_serializer_encode_add(encoder, "debug", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES) != 0)
+ dsync_serializer_encode_add(encoder, "sync_visible_namespaces", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_MAIL_SYNC) != 0)
+ dsync_serializer_encode_add(encoder, "no_mail_sync", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE) != 0)
+ dsync_serializer_encode_add(encoder, "no_backup_overwrite", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_PURGE_REMOTE) != 0)
+ dsync_serializer_encode_add(encoder, "purge_remote", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_NO_NOTIFY) != 0)
+ dsync_serializer_encode_add(encoder, "no_notify", "");
+ if ((set->brain_flags & DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND) != 0)
+ dsync_serializer_encode_add(encoder, "empty_hdr_workaround", "");
+ /* this can be NULL in slave */
+ string_t *str2 = t_str_new(32);
+ if (set->hashed_headers != NULL) {
+ for(const char *const *ptr = set->hashed_headers; *ptr != NULL; ptr++) {
+ str_append_tabescaped(str2, *ptr);
+ str_append_c(str2, '\t');
+ }
+ }
+ dsync_serializer_encode_add(encoder, "hashed_headers", str_c(str2));
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_handshake(struct dsync_ibc *_ibc,
+ const struct dsync_ibc_settings **set_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_ibc_settings *set;
+ const char *value;
+ pool_t pool = ibc->ret_pool;
+ enum dsync_ibc_recv_ret ret;
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_HANDSHAKE, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK) {
+ if (ret != DSYNC_IBC_RECV_RET_TRYAGAIN) {
+ i_error("dsync(%s): Unexpected input in handshake",
+ ibc->name);
+ dsync_ibc_stream_stop(ibc);
+ }
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ p_clear(pool);
+ set = p_new(pool, struct dsync_ibc_settings, 1);
+
+ value = dsync_deserializer_decode_get(decoder, "hostname");
+ set->hostname = p_strdup(pool, value);
+ /* now that we know the remote's hostname, use it for the
+ stream's name */
+ i_free(ibc->name);
+ ibc->name = i_strdup(set->hostname);
+
+ if (dsync_deserializer_decode_try(decoder, "sync_ns_prefix", &value))
+ set->sync_ns_prefixes = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "sync_box", &value))
+ set->sync_box = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "virtual_all_box", &value))
+ set->virtual_all_box = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "sync_box_guid", &value) &&
+ guid_128_from_string(value, set->sync_box_guid) < 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid sync_box_guid: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "exclude_mailboxes", &value) &&
+ *value != '\0') {
+ char **boxes = p_strsplit_tabescaped(pool, value);
+ set->exclude_mailboxes = (const void *)boxes;
+ }
+ if (dsync_deserializer_decode_try(decoder, "sync_type", &value)) {
+ switch (value[0]) {
+ case 'f':
+ set->sync_type = DSYNC_BRAIN_SYNC_TYPE_FULL;
+ break;
+ case 'c':
+ set->sync_type = DSYNC_BRAIN_SYNC_TYPE_CHANGED;
+ break;
+ case 's':
+ set->sync_type = DSYNC_BRAIN_SYNC_TYPE_STATE;
+ break;
+ default:
+ dsync_ibc_input_error(ibc, decoder,
+ "Unknown sync_type: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "lock_timeout", &value)) {
+ if (str_to_uint(value, &set->lock_timeout) < 0 ||
+ set->lock_timeout == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid lock_timeout: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "import_commit_msgs_interval", &value)) {
+ if (str_to_uint(value, &set->import_commit_msgs_interval) < 0 ||
+ set->import_commit_msgs_interval == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid import_commit_msgs_interval: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "sync_since_timestamp", &value)) {
+ if (str_to_time(value, &set->sync_since_timestamp) < 0 ||
+ set->sync_since_timestamp == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid sync_since_timestamp: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "sync_until_timestamp", &value)) {
+ if (str_to_time(value, &set->sync_until_timestamp) < 0 ||
+ set->sync_until_timestamp == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid sync_until_timestamp: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "sync_max_size", &value)) {
+ if (str_to_uoff(value, &set->sync_max_size) < 0 ||
+ set->sync_max_size == 0) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid sync_max_size: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "sync_flags", &value))
+ set->sync_flags = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "alt_char", &value))
+ set->alt_char = value[0];
+ if (dsync_deserializer_decode_try(decoder, "send_mail_requests", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_SEND_MAIL_REQUESTS;
+ if (dsync_deserializer_decode_try(decoder, "backup_send", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_SEND;
+ if (dsync_deserializer_decode_try(decoder, "backup_recv", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_BACKUP_RECV;
+ if (dsync_deserializer_decode_try(decoder, "debug", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_DEBUG;
+ if (dsync_deserializer_decode_try(decoder, "sync_visible_namespaces", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_SYNC_VISIBLE_NAMESPACES;
+ if (dsync_deserializer_decode_try(decoder, "no_mail_sync", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_NO_MAIL_SYNC;
+ if (dsync_deserializer_decode_try(decoder, "no_backup_overwrite", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_NO_BACKUP_OVERWRITE;
+ if (dsync_deserializer_decode_try(decoder, "purge_remote", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_PURGE_REMOTE;
+ if (dsync_deserializer_decode_try(decoder, "no_notify", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_NO_NOTIFY;
+ if (dsync_deserializer_decode_try(decoder, "empty_hdr_workaround", &value))
+ set->brain_flags |= DSYNC_BRAIN_FLAG_EMPTY_HDR_WORKAROUND;
+ if (dsync_deserializer_decode_try(decoder, "hashed_headers", &value))
+ set->hashed_headers = (const char*const*)p_strsplit_tabescaped(pool, value);
+ set->hdr_hash_v2 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V2;
+ set->hdr_hash_v3 = ibc->minor_version >= DSYNC_PROTOCOL_MINOR_HAVE_HDR_HASH_V3;
+
+ *set_r = set;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_end_of_list(struct dsync_ibc *_ibc,
+ enum dsync_ibc_eol_type type)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+
+ i_assert(ibc->value_output == NULL);
+
+ switch (type) {
+ case DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE:
+ if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
+ return;
+ break;
+ default:
+ break;
+ }
+
+ ibc->last_sent_item_eol = TRUE;
+ o_stream_nsend_str(ibc->output, END_OF_LIST_LINE"\n");
+}
+
+static void
+dsync_ibc_stream_send_mailbox_state(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox_state *state)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+
+ str_append_c(str, items[ITEM_MAILBOX_STATE].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_STATE);
+ dsync_serializer_encode_add(encoder, "mailbox_guid",
+ guid_128_to_string(state->mailbox_guid));
+ dsync_serializer_encode_add(encoder, "last_uidvalidity",
+ dec2str(state->last_uidvalidity));
+ dsync_serializer_encode_add(encoder, "last_common_uid",
+ dec2str(state->last_common_uid));
+ dsync_serializer_encode_add(encoder, "last_common_modseq",
+ dec2str(state->last_common_modseq));
+ dsync_serializer_encode_add(encoder, "last_common_pvt_modseq",
+ dec2str(state->last_common_pvt_modseq));
+ dsync_serializer_encode_add(encoder, "last_messages_count",
+ dec2str(state->last_messages_count));
+ if (state->changes_during_sync)
+ dsync_serializer_encode_add(encoder, "changes_during_sync", "");
+
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_state(struct dsync_ibc *_ibc,
+ struct dsync_mailbox_state *state_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_deserializer_decoder *decoder;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ i_zero(state_r);
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_STATE, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ value = dsync_deserializer_decode_get(decoder, "mailbox_guid");
+ if (guid_128_from_string(value, state_r->mailbox_guid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "last_uidvalidity");
+ if (str_to_uint32(value, &state_r->last_uidvalidity) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_uidvalidity");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "last_common_uid");
+ if (str_to_uint32(value, &state_r->last_common_uid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_common_uid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "last_common_modseq");
+ if (str_to_uint64(value, &state_r->last_common_modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_common_modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "last_common_pvt_modseq");
+ if (str_to_uint64(value, &state_r->last_common_pvt_modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_common_pvt_modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "last_messages_count", &value) &&
+ str_to_uint32(value, &state_r->last_messages_count) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_messages_count");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "changes_during_sync", &value))
+ state_r->changes_during_sync = TRUE;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mailbox_tree_node(struct dsync_ibc *_ibc,
+ const char *const *name,
+ const struct dsync_mailbox_node *node)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str, *namestr;
+
+ i_assert(*name != NULL);
+
+ str = t_str_new(128);
+ str_append_c(str, items[ITEM_MAILBOX_TREE_NODE].chr);
+
+ /* convert all hierarchy separators to tabs. mailbox names really
+ aren't supposed to have any tabs, but escape them anyway if there
+ are. */
+ namestr = t_str_new(128);
+ for (; *name != NULL; name++) {
+ str_append_tabescaped(namestr, *name);
+ str_append_c(namestr, '\t');
+ }
+ str_truncate(namestr, str_len(namestr)-1);
+
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_TREE_NODE);
+ dsync_serializer_encode_add(encoder, "name", str_c(namestr));
+ switch (node->existence) {
+ case DSYNC_MAILBOX_NODE_NONEXISTENT:
+ dsync_serializer_encode_add(encoder, "existence", "n");
+ break;
+ case DSYNC_MAILBOX_NODE_EXISTS:
+ dsync_serializer_encode_add(encoder, "existence", "y");
+ break;
+ case DSYNC_MAILBOX_NODE_DELETED:
+ dsync_serializer_encode_add(encoder, "existence", "d");
+ break;
+ }
+
+ if (!guid_128_is_empty(node->mailbox_guid)) {
+ dsync_serializer_encode_add(encoder, "mailbox_guid",
+ guid_128_to_string(node->mailbox_guid));
+ }
+ if (node->uid_validity != 0) {
+ dsync_serializer_encode_add(encoder, "uid_validity",
+ dec2str(node->uid_validity));
+ }
+ if (node->uid_next != 0) {
+ dsync_serializer_encode_add(encoder, "uid_next",
+ dec2str(node->uid_next));
+ }
+ if (node->last_renamed_or_created != 0) {
+ dsync_serializer_encode_add(encoder, "last_renamed_or_created",
+ dec2str(node->last_renamed_or_created));
+ }
+ if (node->last_subscription_change != 0) {
+ dsync_serializer_encode_add(encoder, "last_subscription_change",
+ dec2str(node->last_subscription_change));
+ }
+ if (node->subscribed)
+ dsync_serializer_encode_add(encoder, "subscribed", "");
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_tree_node(struct dsync_ibc *_ibc,
+ const char *const **name_r,
+ const struct dsync_mailbox_node **node_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_mailbox_node *node;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_TREE_NODE, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ p_clear(ibc->ret_pool);
+ node = p_new(ibc->ret_pool, struct dsync_mailbox_node, 1);
+
+ value = dsync_deserializer_decode_get(decoder, "name");
+ if (*value == '\0') {
+ dsync_ibc_input_error(ibc, decoder, "Empty name");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ *name_r = (void *)p_strsplit_tabescaped(ibc->ret_pool, value);
+
+ value = dsync_deserializer_decode_get(decoder, "existence");
+ switch (*value) {
+ case 'n':
+ node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+ break;
+ case 'y':
+ node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ break;
+ case 'd':
+ node->existence = DSYNC_MAILBOX_NODE_DELETED;
+ break;
+ }
+
+ if (dsync_deserializer_decode_try(decoder, "mailbox_guid", &value) &&
+ guid_128_from_string(value, node->mailbox_guid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "uid_validity", &value) &&
+ str_to_uint32(value, &node->uid_validity) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "uid_next", &value) &&
+ str_to_uint32(value, &node->uid_next) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid_next");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "last_renamed_or_created", &value) &&
+ str_to_time(value, &node->last_renamed_or_created) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_renamed_or_created");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "last_subscription_change", &value) &&
+ str_to_time(value, &node->last_subscription_change) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_subscription_change");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "subscribed", &value))
+ node->subscribed = TRUE;
+
+ *node_r = node;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_encode_delete(string_t *str,
+ struct dsync_serializer_encoder *encoder,
+ const struct dsync_mailbox_delete *deletes,
+ unsigned int count, const char *key,
+ enum dsync_mailbox_delete_type type)
+{
+ unsigned int i;
+
+ str_truncate(str, 0);
+ for (i = 0; i < count; i++) {
+ if (deletes[i].type == type) {
+ str_append(str, guid_128_to_string(deletes[i].guid));
+ str_printfa(str, " %ld ", (long)deletes[i].timestamp);
+ }
+ }
+ if (str_len(str) > 0) {
+ str_truncate(str, str_len(str)-1);
+ dsync_serializer_encode_add(encoder, key, str_c(str));
+ }
+}
+
+static void
+dsync_ibc_stream_send_mailbox_deletes(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox_delete *deletes,
+ unsigned int count, char hierarchy_sep,
+ char escape_char)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str, *substr;
+ char sep[2];
+
+ str = t_str_new(128);
+ str_append_c(str, items[ITEM_MAILBOX_DELETE].chr);
+
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_DELETE);
+ sep[0] = hierarchy_sep; sep[1] = '\0';
+ dsync_serializer_encode_add(encoder, "hierarchy_sep", sep);
+ sep[0] = escape_char; sep[1] = '\0';
+ dsync_serializer_encode_add(encoder, "escape_char", sep);
+
+ substr = t_str_new(128);
+ dsync_ibc_stream_encode_delete(substr, encoder, deletes, count,
+ "mailboxes",
+ DSYNC_MAILBOX_DELETE_TYPE_MAILBOX);
+ dsync_ibc_stream_encode_delete(substr, encoder, deletes, count,
+ "dirs",
+ DSYNC_MAILBOX_DELETE_TYPE_DIR);
+ dsync_ibc_stream_encode_delete(substr, encoder, deletes, count,
+ "unsubscribes",
+ DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE);
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+ARRAY_DEFINE_TYPE(dsync_mailbox_delete, struct dsync_mailbox_delete);
+static int
+decode_mailbox_deletes(ARRAY_TYPE(dsync_mailbox_delete) *deletes,
+ const char *value, enum dsync_mailbox_delete_type type)
+{
+ struct dsync_mailbox_delete *del;
+ const char *const *tmp;
+ unsigned int i;
+
+ tmp = t_strsplit(value, " ");
+ for (i = 0; tmp[i] != NULL; i += 2) {
+ del = array_append_space(deletes);
+ del->type = type;
+ if (guid_128_from_string(tmp[i], del->guid) < 0)
+ return -1;
+ if (tmp[i+1] == NULL ||
+ str_to_time(tmp[i+1], &del->timestamp) < 0)
+ return -1;
+ }
+ return 0;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_deletes(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox_delete **deletes_r,
+ unsigned int *count_r, char *hierarchy_sep_r,
+ char *escape_char_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_deserializer_decoder *decoder;
+ ARRAY_TYPE(dsync_mailbox_delete) deletes;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_DELETE, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ p_clear(ibc->ret_pool);
+ p_array_init(&deletes, ibc->ret_pool, 16);
+
+ value = dsync_deserializer_decode_get(decoder, "hierarchy_sep");
+ if (strlen(value) != 1) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid hierarchy_sep");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ *hierarchy_sep_r = value[0];
+
+ if (!dsync_deserializer_decode_try(decoder, "escape_char", &value))
+ *escape_char_r = '\0';
+ else {
+ if (strlen(value) > 1) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid escape_char '%s'", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ *escape_char_r = value[0];
+ }
+
+ if (dsync_deserializer_decode_try(decoder, "mailboxes", &value) &&
+ decode_mailbox_deletes(&deletes, value,
+ DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid mailboxes");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "dirs", &value) &&
+ decode_mailbox_deletes(&deletes, value,
+ DSYNC_MAILBOX_DELETE_TYPE_DIR) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid dirs");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "unsubscribes", &value) &&
+ decode_mailbox_deletes(&deletes, value,
+ DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid dirs");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ *deletes_r = array_get(&deletes, count_r);
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static const char *
+get_cache_fields(struct dsync_ibc_stream *ibc,
+ const struct dsync_mailbox *dsync_box)
+{
+ struct dsync_serializer_encoder *encoder;
+ string_t *str;
+ const struct mailbox_cache_field *cache_fields;
+ unsigned int i, count;
+ char decision[3];
+
+ cache_fields = array_get(&dsync_box->cache_fields, &count);
+ if (count == 0)
+ return "";
+
+ str = t_str_new(128);
+ for (i = 0; i < count; i++) {
+ const struct mailbox_cache_field *field = &cache_fields[i];
+
+ encoder = dsync_serializer_encode_begin(ibc->serializers[ITEM_MAILBOX_CACHE_FIELD]);
+ dsync_serializer_encode_add(encoder, "name", field->name);
+
+ memset(decision, 0, sizeof(decision));
+ switch (field->decision & ENUM_NEGATE(MAIL_CACHE_DECISION_FORCED)) {
+ case MAIL_CACHE_DECISION_NO:
+ decision[0] = 'n';
+ break;
+ case MAIL_CACHE_DECISION_TEMP:
+ decision[0] = 't';
+ break;
+ case MAIL_CACHE_DECISION_YES:
+ decision[0] = 'y';
+ break;
+ }
+ i_assert(decision[0] != '\0');
+ if ((field->decision & MAIL_CACHE_DECISION_FORCED) != 0)
+ decision[1] = 'F';
+ dsync_serializer_encode_add(encoder, "decision", decision);
+ if (field->last_used != 0) {
+ dsync_serializer_encode_add(encoder, "last_used",
+ dec2str(field->last_used));
+ }
+ dsync_serializer_encode_finish(&encoder, str);
+ }
+ if (i > 0) {
+ /* remove the trailing LF */
+ str_truncate(str, str_len(str)-1);
+ }
+ return str_c(str);
+}
+
+static void
+dsync_ibc_stream_send_mailbox(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox *dsync_box)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+ const char *value;
+
+ str_append_c(str, items[ITEM_MAILBOX].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX);
+ dsync_serializer_encode_add(encoder, "mailbox_guid",
+ guid_128_to_string(dsync_box->mailbox_guid));
+
+ if (dsync_box->mailbox_lost)
+ dsync_serializer_encode_add(encoder, "mailbox_lost", "");
+ if (dsync_box->mailbox_ignore)
+ dsync_serializer_encode_add(encoder, "mailbox_ignore", "");
+ if (dsync_box->have_guids)
+ dsync_serializer_encode_add(encoder, "have_guids", "");
+ if (dsync_box->have_save_guids)
+ dsync_serializer_encode_add(encoder, "have_save_guids", "");
+ if (dsync_box->have_only_guid128)
+ dsync_serializer_encode_add(encoder, "have_only_guid128", "");
+ dsync_serializer_encode_add(encoder, "uid_validity",
+ dec2str(dsync_box->uid_validity));
+ dsync_serializer_encode_add(encoder, "uid_next",
+ dec2str(dsync_box->uid_next));
+ dsync_serializer_encode_add(encoder, "messages_count",
+ dec2str(dsync_box->messages_count));
+ dsync_serializer_encode_add(encoder, "first_recent_uid",
+ dec2str(dsync_box->first_recent_uid));
+ dsync_serializer_encode_add(encoder, "highest_modseq",
+ dec2str(dsync_box->highest_modseq));
+ dsync_serializer_encode_add(encoder, "highest_pvt_modseq",
+ dec2str(dsync_box->highest_pvt_modseq));
+
+ value = get_cache_fields(ibc, dsync_box);
+ if (value != NULL)
+ dsync_serializer_encode_add(encoder, "cache_fields", value);
+
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static int
+parse_cache_field(struct dsync_ibc_stream *ibc, struct dsync_mailbox *box,
+ const char *value)
+{
+ struct dsync_deserializer_decoder *decoder;
+ struct mailbox_cache_field field;
+ const char *error;
+ int ret = 0;
+
+ if (dsync_deserializer_decode_begin(ibc->deserializers[ITEM_MAILBOX_CACHE_FIELD],
+ value, &decoder, &error) < 0) {
+ dsync_ibc_input_error(ibc, NULL,
+ "cache_field: Invalid input: %s", error);
+ return -1;
+ }
+
+ i_zero(&field);
+ value = dsync_deserializer_decode_get(decoder, "name");
+ field.name = p_strdup(ibc->ret_pool, value);
+
+ value = dsync_deserializer_decode_get(decoder, "decision");
+ switch (*value) {
+ case 'n':
+ field.decision = MAIL_CACHE_DECISION_NO;
+ break;
+ case 't':
+ field.decision = MAIL_CACHE_DECISION_TEMP;
+ break;
+ case 'y':
+ field.decision = MAIL_CACHE_DECISION_YES;
+ break;
+ default:
+ dsync_ibc_input_error(ibc, decoder, "Invalid decision: %s",
+ value);
+ ret = -1;
+ break;
+ }
+ if (value[1] == 'F')
+ field.decision |= MAIL_CACHE_DECISION_FORCED;
+
+ if (dsync_deserializer_decode_try(decoder, "last_used", &value) &&
+ str_to_time(value, &field.last_used) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_used");
+ ret = -1;
+ }
+ array_push_back(&box->cache_fields, &field);
+
+ dsync_deserializer_decode_finish(&decoder);
+ return ret;
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox **dsync_box_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ pool_t pool = ibc->ret_pool;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_mailbox *box;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ p_clear(pool);
+ box = p_new(pool, struct dsync_mailbox, 1);
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ value = dsync_deserializer_decode_get(decoder, "mailbox_guid");
+ if (guid_128_from_string(value, box->mailbox_guid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid mailbox_guid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ if (dsync_deserializer_decode_try(decoder, "mailbox_lost", &value))
+ box->mailbox_lost = TRUE;
+ if (dsync_deserializer_decode_try(decoder, "mailbox_ignore", &value))
+ box->mailbox_ignore = TRUE;
+ if (dsync_deserializer_decode_try(decoder, "have_guids", &value))
+ box->have_guids = TRUE;
+ if (dsync_deserializer_decode_try(decoder, "have_save_guids", &value) ||
+ (box->have_guids && ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_SAVE_GUID))
+ box->have_save_guids = TRUE;
+ if (dsync_deserializer_decode_try(decoder, "have_only_guid128", &value))
+ box->have_only_guid128 = TRUE;
+ value = dsync_deserializer_decode_get(decoder, "uid_validity");
+ if (str_to_uint32(value, &box->uid_validity) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid_validity");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "uid_next");
+ if (str_to_uint32(value, &box->uid_next) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid_next");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "messages_count");
+ if (str_to_uint32(value, &box->messages_count) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid messages_count");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "first_recent_uid");
+ if (str_to_uint32(value, &box->first_recent_uid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid first_recent_uid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "highest_modseq");
+ if (str_to_uint64(value, &box->highest_modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid highest_modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ value = dsync_deserializer_decode_get(decoder, "highest_pvt_modseq");
+ if (str_to_uint64(value, &box->highest_pvt_modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid highest_pvt_modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ p_array_init(&box->cache_fields, pool, 32);
+ if (dsync_deserializer_decode_try(decoder, "cache_fields", &value)) {
+ const char *const *fields = t_strsplit(value, "\n");
+ for (; *fields != NULL; fields++) {
+ if (parse_cache_field(ibc, box, *fields) < 0)
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ }
+
+ *dsync_box_r = box;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mailbox_attribute(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox_attribute *attr)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+ char type[2];
+
+ if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
+ return;
+
+ str_append_c(str, items[ITEM_MAILBOX_ATTRIBUTE].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAILBOX_ATTRIBUTE);
+
+ type[0] = type[1] = '\0';
+ switch (attr->type) {
+ case MAIL_ATTRIBUTE_TYPE_PRIVATE:
+ type[0] = 'p';
+ break;
+ case MAIL_ATTRIBUTE_TYPE_SHARED:
+ type[0] = 's';
+ break;
+ }
+ i_assert(type[0] != '\0');
+ dsync_serializer_encode_add(encoder, "type", type);
+ dsync_serializer_encode_add(encoder, "key", attr->key);
+ if (attr->value != NULL)
+ dsync_serializer_encode_add(encoder, "value", attr->value);
+ else if (attr->value_stream != NULL)
+ dsync_serializer_encode_add(encoder, "stream", "");
+
+ if (attr->deleted)
+ dsync_serializer_encode_add(encoder, "deleted", "");
+ if (attr->last_change != 0) {
+ dsync_serializer_encode_add(encoder, "last_change",
+ dec2str(attr->last_change));
+ }
+ if (attr->modseq != 0) {
+ dsync_serializer_encode_add(encoder, "modseq",
+ dec2str(attr->modseq));
+ }
+
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+
+ if (attr->value_stream != NULL) {
+ ibc->value_output_last = '\0';
+ ibc->value_output = attr->value_stream;
+ i_stream_ref(ibc->value_output);
+ (void)dsync_ibc_stream_send_value_stream(ibc);
+ }
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mailbox_attribute(struct dsync_ibc *_ibc,
+ const struct dsync_mailbox_attribute **attr_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ pool_t pool = ibc->ret_pool;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_mailbox_attribute *attr;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_ATTRIBUTES)
+ return DSYNC_IBC_RECV_RET_FINISHED;
+
+ if (ibc->value_input != NULL) {
+ /* wait until the mail's stream has been read */
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ if (ibc->cur_attr != NULL) {
+ /* finished reading the stream, return the mail now */
+ *attr_r = ibc->cur_attr;
+ ibc->cur_attr = NULL;
+ return DSYNC_IBC_RECV_RET_OK;
+ }
+
+ p_clear(pool);
+ attr = p_new(pool, struct dsync_mailbox_attribute, 1);
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAILBOX_ATTRIBUTE, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ value = dsync_deserializer_decode_get(decoder, "type");
+ switch (*value) {
+ case 'p':
+ attr->type = MAIL_ATTRIBUTE_TYPE_PRIVATE;
+ break;
+ case 's':
+ attr->type = MAIL_ATTRIBUTE_TYPE_SHARED;
+ break;
+ default:
+ dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ value = dsync_deserializer_decode_get(decoder, "key");
+ attr->key = p_strdup(pool, value);
+
+ if (dsync_deserializer_decode_try(decoder, "deleted", &value))
+ attr->deleted = TRUE;
+ if (dsync_deserializer_decode_try(decoder, "last_change", &value) &&
+ str_to_time(value, &attr->last_change) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid last_change");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "modseq", &value) &&
+ str_to_uint64(value, &attr->modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ /* NOTE: stream reading must be the last here, because reading a large
+ stream will be finished later by return TRYAGAIN. We need to
+ deserialize all the other fields before that or they'll get lost. */
+ if (dsync_deserializer_decode_try(decoder, "stream", &value)) {
+ attr->value_stream = dsync_ibc_stream_input_stream(ibc);
+ if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) {
+ ibc->cur_attr = attr;
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ /* already finished reading the stream */
+ i_assert(ibc->value_input == NULL);
+ } else if (dsync_deserializer_decode_try(decoder, "value", &value))
+ attr->value = p_strdup(pool, value);
+
+ *attr_r = attr;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_change(struct dsync_ibc *_ibc,
+ const struct dsync_mail_change *change)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+ char type[2];
+
+ str_append_c(str, items[ITEM_MAIL_CHANGE].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_CHANGE);
+
+ type[0] = type[1] = '\0';
+ switch (change->type) {
+ case DSYNC_MAIL_CHANGE_TYPE_SAVE:
+ type[0] = 's';
+ break;
+ case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:
+ type[0] = 'e';
+ break;
+ case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:
+ type[0] = 'f';
+ break;
+ }
+ i_assert(type[0] != '\0');
+ dsync_serializer_encode_add(encoder, "type", type);
+ dsync_serializer_encode_add(encoder, "uid", dec2str(change->uid));
+ if (change->guid != NULL)
+ dsync_serializer_encode_add(encoder, "guid", change->guid);
+ if (change->hdr_hash != NULL) {
+ dsync_serializer_encode_add(encoder, "hdr_hash",
+ change->hdr_hash);
+ }
+ if (change->modseq != 0) {
+ dsync_serializer_encode_add(encoder, "modseq",
+ dec2str(change->modseq));
+ }
+ if (change->pvt_modseq != 0) {
+ dsync_serializer_encode_add(encoder, "pvt_modseq",
+ dec2str(change->pvt_modseq));
+ }
+ if (change->add_flags != 0) {
+ dsync_serializer_encode_add(encoder, "add_flags",
+ t_strdup_printf("%x", change->add_flags));
+ }
+ if (change->remove_flags != 0) {
+ dsync_serializer_encode_add(encoder, "remove_flags",
+ t_strdup_printf("%x", change->remove_flags));
+ }
+ if (change->final_flags != 0) {
+ dsync_serializer_encode_add(encoder, "final_flags",
+ t_strdup_printf("%x", change->final_flags));
+ }
+ if (change->keywords_reset)
+ dsync_serializer_encode_add(encoder, "keywords_reset", "");
+
+ if (array_is_created(&change->keyword_changes) &&
+ array_count(&change->keyword_changes) > 0) {
+ string_t *kw_str = t_str_new(128);
+ const char *const *changes;
+ unsigned int i, count;
+
+ changes = array_get(&change->keyword_changes, &count);
+ str_append_tabescaped(kw_str, changes[0]);
+ for (i = 1; i < count; i++) {
+ str_append_c(kw_str, '\t');
+ str_append_tabescaped(kw_str, changes[i]);
+ }
+ dsync_serializer_encode_add(encoder, "keyword_changes",
+ str_c(kw_str));
+ }
+ if (change->received_timestamp > 0) {
+ dsync_serializer_encode_add(encoder, "received_timestamp",
+ t_strdup_printf("%"PRIxTIME_T, change->received_timestamp));
+ }
+ if (change->virtual_size > 0) {
+ dsync_serializer_encode_add(encoder, "virtual_size",
+ t_strdup_printf("%llx", (unsigned long long)change->virtual_size));
+ }
+
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_change(struct dsync_ibc *_ibc,
+ const struct dsync_mail_change **change_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ pool_t pool = ibc->ret_pool;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_mail_change *change;
+ const char *value;
+ unsigned int uintval;
+ unsigned long long ullongval;
+ enum dsync_ibc_recv_ret ret;
+
+ p_clear(pool);
+ change = p_new(pool, struct dsync_mail_change, 1);
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_CHANGE, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ value = dsync_deserializer_decode_get(decoder, "type");
+ switch (*value) {
+ case 's':
+ change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE;
+ break;
+ case 'e':
+ change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
+ break;
+ case 'f':
+ change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
+ break;
+ default:
+ dsync_ibc_input_error(ibc, decoder, "Invalid type: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ value = dsync_deserializer_decode_get(decoder, "uid");
+ if (str_to_uint32(value, &change->uid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ if (dsync_deserializer_decode_try(decoder, "guid", &value))
+ change->guid = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "hdr_hash", &value))
+ change->hdr_hash = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "modseq", &value) &&
+ str_to_uint64(value, &change->modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "pvt_modseq", &value) &&
+ str_to_uint64(value, &change->pvt_modseq) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid pvt_modseq");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ if (dsync_deserializer_decode_try(decoder, "add_flags", &value)) {
+ if (str_to_uint_hex(value, &uintval) < 0 ||
+ uintval > (uint8_t)-1) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid add_flags: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ change->add_flags = uintval;
+ }
+ if (dsync_deserializer_decode_try(decoder, "remove_flags", &value)) {
+ if (str_to_uint_hex(value, &uintval) < 0 ||
+ uintval > (uint8_t)-1) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid remove_flags: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ change->remove_flags = uintval;
+ }
+ if (dsync_deserializer_decode_try(decoder, "final_flags", &value)) {
+ if (str_to_uint_hex(value, &uintval) < 0 ||
+ uintval > (uint8_t)-1) {
+ dsync_ibc_input_error(ibc, decoder,
+ "Invalid final_flags: %s", value);
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ change->final_flags = uintval;
+ }
+ if (dsync_deserializer_decode_try(decoder, "keywords_reset", &value))
+ change->keywords_reset = TRUE;
+
+ if (dsync_deserializer_decode_try(decoder, "keyword_changes", &value) &&
+ *value != '\0') {
+ const char *const *changes = t_strsplit_tabescaped(value);
+ unsigned int i, count = str_array_length(changes);
+
+ p_array_init(&change->keyword_changes, pool, count);
+ for (i = 0; i < count; i++) {
+ value = p_strdup(pool, changes[i]);
+ array_push_back(&change->keyword_changes, &value);
+ }
+ }
+ if (dsync_deserializer_decode_try(decoder, "received_timestamp", &value)) {
+ if (str_to_ullong_hex(value, &ullongval) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid received_timestamp");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ change->received_timestamp = ullongval;
+ }
+ if (dsync_deserializer_decode_try(decoder, "virtual_size", &value)) {
+ if (str_to_ullong_hex(value, &ullongval) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid virtual_size");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ change->virtual_size = ullongval;
+ }
+
+ *change_r = change;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mail_request(struct dsync_ibc *_ibc,
+ const struct dsync_mail_request *request)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+
+ str_append_c(str, items[ITEM_MAIL_REQUEST].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL_REQUEST);
+ if (request->guid != NULL)
+ dsync_serializer_encode_add(encoder, "guid", request->guid);
+ if (request->uid != 0) {
+ dsync_serializer_encode_add(encoder, "uid",
+ dec2str(request->uid));
+ }
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mail_request(struct dsync_ibc *_ibc,
+ const struct dsync_mail_request **request_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_mail_request *request;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ p_clear(ibc->ret_pool);
+ request = p_new(ibc->ret_pool, struct dsync_mail_request, 1);
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL_REQUEST, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ if (dsync_deserializer_decode_try(decoder, "guid", &value))
+ request->guid = p_strdup(ibc->ret_pool, value);
+ if (dsync_deserializer_decode_try(decoder, "uid", &value) &&
+ str_to_uint32(value, &request->uid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+
+ *request_r = request;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_mail(struct dsync_ibc *_ibc,
+ const struct dsync_mail *mail)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+
+ i_assert(!mail->minimal_fields);
+ i_assert(ibc->value_output == NULL);
+
+ str_append_c(str, items[ITEM_MAIL].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_MAIL);
+ if (mail->guid != NULL)
+ dsync_serializer_encode_add(encoder, "guid", mail->guid);
+ if (mail->uid != 0)
+ dsync_serializer_encode_add(encoder, "uid", dec2str(mail->uid));
+ if (mail->pop3_uidl != NULL) {
+ dsync_serializer_encode_add(encoder, "pop3_uidl",
+ mail->pop3_uidl);
+ }
+ if (mail->pop3_order > 0) {
+ dsync_serializer_encode_add(encoder, "pop3_order",
+ dec2str(mail->pop3_order));
+ }
+ if (mail->received_date > 0) {
+ dsync_serializer_encode_add(encoder, "received_date",
+ dec2str(mail->received_date));
+ }
+ if (mail->saved_date != 0) {
+ dsync_serializer_encode_add(encoder, "saved_date",
+ dec2str(mail->saved_date));
+ }
+ if (mail->input != NULL)
+ dsync_serializer_encode_add(encoder, "stream", "");
+
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+
+ if (mail->input != NULL) {
+ ibc->value_output_last = '\0';
+ ibc->value_output = mail->input;
+ i_stream_ref(ibc->value_output);
+ (void)dsync_ibc_stream_send_value_stream(ibc);
+ }
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_mail(struct dsync_ibc *_ibc, struct dsync_mail **mail_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ pool_t pool = ibc->ret_pool;
+ struct dsync_deserializer_decoder *decoder;
+ struct dsync_mail *mail;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+
+ if (ibc->value_input != NULL) {
+ /* wait until the mail's stream has been read */
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (ibc->cur_mail != NULL) {
+ /* finished reading the stream, return the mail now */
+ *mail_r = ibc->cur_mail;
+ ibc->cur_mail = NULL;
+ return DSYNC_IBC_RECV_RET_OK;
+ }
+
+ p_clear(pool);
+ mail = p_new(pool, struct dsync_mail, 1);
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_MAIL, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ if (dsync_deserializer_decode_try(decoder, "guid", &value))
+ mail->guid = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "uid", &value) &&
+ str_to_uint32(value, &mail->uid) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid uid");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "pop3_uidl", &value))
+ mail->pop3_uidl = p_strdup(pool, value);
+ if (dsync_deserializer_decode_try(decoder, "pop3_order", &value) &&
+ str_to_uint32(value, &mail->pop3_order) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid pop3_order");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "received_date", &value) &&
+ str_to_time(value, &mail->received_date) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid received_date");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "saved_date", &value) &&
+ str_to_time(value, &mail->saved_date) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid saved_date");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "stream", &value)) {
+ mail->input = dsync_ibc_stream_input_stream(ibc);
+ if (dsync_ibc_stream_read_mail_stream(ibc) <= 0) {
+ ibc->cur_mail = mail;
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ /* already finished reading the stream */
+ i_assert(ibc->value_input == NULL);
+ }
+
+ *mail_r = mail;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void
+dsync_ibc_stream_send_finish(struct dsync_ibc *_ibc, const char *error,
+ enum mail_error mail_error,
+ bool require_full_resync)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_serializer_encoder *encoder;
+ string_t *str = t_str_new(128);
+
+ str_append_c(str, items[ITEM_FINISH].chr);
+ encoder = dsync_ibc_send_encode_begin(ibc, ITEM_FINISH);
+ if (error != NULL)
+ dsync_serializer_encode_add(encoder, "error", error);
+ if (mail_error != 0) {
+ dsync_serializer_encode_add(encoder, "mail_error",
+ dec2str(mail_error));
+ }
+ if (require_full_resync)
+ dsync_serializer_encode_add(encoder, "require_full_resync", "");
+ dsync_serializer_encode_finish(&encoder, str);
+ dsync_ibc_stream_send_string(ibc, str);
+}
+
+static enum dsync_ibc_recv_ret
+dsync_ibc_stream_recv_finish(struct dsync_ibc *_ibc, const char **error_r,
+ enum mail_error *mail_error_r,
+ bool *require_full_resync_r)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ struct dsync_deserializer_decoder *decoder;
+ const char *value;
+ enum dsync_ibc_recv_ret ret;
+ int i = 0;
+
+ *error_r = NULL;
+ *mail_error_r = 0;
+ *require_full_resync_r = FALSE;
+
+ p_clear(ibc->ret_pool);
+
+ if (ibc->minor_version < DSYNC_PROTOCOL_MINOR_HAVE_FINISH)
+ return DSYNC_IBC_RECV_RET_OK;
+
+ ret = dsync_ibc_stream_input_next(ibc, ITEM_FINISH, &decoder);
+ if (ret != DSYNC_IBC_RECV_RET_OK)
+ return ret;
+
+ if (dsync_deserializer_decode_try(decoder, "error", &value))
+ *error_r = p_strdup(ibc->ret_pool, value);
+ if (dsync_deserializer_decode_try(decoder, "mail_error", &value) &&
+ str_to_int(value, &i) < 0) {
+ dsync_ibc_input_error(ibc, decoder, "Invalid mail_error");
+ return DSYNC_IBC_RECV_RET_TRYAGAIN;
+ }
+ if (dsync_deserializer_decode_try(decoder, "require_full_resync", &value))
+ *require_full_resync_r = TRUE;
+ *mail_error_r = i;
+
+ ibc->finish_received = TRUE;
+ return DSYNC_IBC_RECV_RET_OK;
+}
+
+static void dsync_ibc_stream_close_mail_streams(struct dsync_ibc *_ibc)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+
+ if (ibc->value_output != NULL) {
+ i_stream_unref(&ibc->value_output);
+ dsync_ibc_stream_stop(ibc);
+ }
+}
+
+static bool dsync_ibc_stream_is_send_queue_full(struct dsync_ibc *_ibc)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+ size_t bytes;
+
+ if (ibc->value_output != NULL)
+ return TRUE;
+
+ bytes = o_stream_get_buffer_used_size(ibc->output);
+ if (bytes < DSYNC_IBC_STREAM_OUTBUF_THROTTLE_SIZE)
+ return FALSE;
+
+ o_stream_set_flush_pending(ibc->output, TRUE);
+ return TRUE;
+}
+
+static bool dsync_ibc_stream_has_pending_data(struct dsync_ibc *_ibc)
+{
+ struct dsync_ibc_stream *ibc = (struct dsync_ibc_stream *)_ibc;
+
+ return ibc->has_pending_data;
+}
+
+static const struct dsync_ibc_vfuncs dsync_ibc_stream_vfuncs = {
+ dsync_ibc_stream_deinit,
+ dsync_ibc_stream_send_handshake,
+ dsync_ibc_stream_recv_handshake,
+ dsync_ibc_stream_send_end_of_list,
+ dsync_ibc_stream_send_mailbox_state,
+ dsync_ibc_stream_recv_mailbox_state,
+ dsync_ibc_stream_send_mailbox_tree_node,
+ dsync_ibc_stream_recv_mailbox_tree_node,
+ dsync_ibc_stream_send_mailbox_deletes,
+ dsync_ibc_stream_recv_mailbox_deletes,
+ dsync_ibc_stream_send_mailbox,
+ dsync_ibc_stream_recv_mailbox,
+ dsync_ibc_stream_send_mailbox_attribute,
+ dsync_ibc_stream_recv_mailbox_attribute,
+ dsync_ibc_stream_send_change,
+ dsync_ibc_stream_recv_change,
+ dsync_ibc_stream_send_mail_request,
+ dsync_ibc_stream_recv_mail_request,
+ dsync_ibc_stream_send_mail,
+ dsync_ibc_stream_recv_mail,
+ dsync_ibc_stream_send_finish,
+ dsync_ibc_stream_recv_finish,
+ dsync_ibc_stream_close_mail_streams,
+ dsync_ibc_stream_is_send_queue_full,
+ dsync_ibc_stream_has_pending_data
+};
+
+struct dsync_ibc *
+dsync_ibc_init_stream(struct istream *input, struct ostream *output,
+ const char *name, const char *temp_path_prefix,
+ unsigned int timeout_secs)
+{
+ struct dsync_ibc_stream *ibc;
+
+ ibc = i_new(struct dsync_ibc_stream, 1);
+ ibc->ibc.v = dsync_ibc_stream_vfuncs;
+ ibc->input = input;
+ ibc->output = output;
+ i_stream_ref(ibc->input);
+ o_stream_ref(ibc->output);
+ ibc->name = i_strdup(name);
+ ibc->temp_path_prefix = i_strdup(temp_path_prefix);
+ ibc->timeout_secs = timeout_secs;
+ ibc->ret_pool = pool_alloconly_create("ibc stream data", 2048);
+ dsync_ibc_stream_init(ibc);
+ return &ibc->ibc;
+}
diff --git a/src/doveadm/dsync/dsync-ibc.c b/src/doveadm/dsync/dsync-ibc.c
new file mode 100644
index 0000000..ede7b83
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc.c
@@ -0,0 +1,239 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "dsync-mail.h"
+#include "dsync-ibc-private.h"
+
+void dsync_ibc_deinit(struct dsync_ibc **_ibc)
+{
+ struct dsync_ibc *ibc = *_ibc;
+
+ *_ibc = NULL;
+ ibc->v.deinit(ibc);
+}
+
+void dsync_ibc_set_io_callback(struct dsync_ibc *ibc,
+ io_callback_t *callback, void *context)
+{
+ ibc->io_callback = callback;
+ ibc->io_context = context;
+}
+
+void dsync_ibc_send_handshake(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings *set)
+{
+ ibc->v.send_handshake(ibc, set);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_handshake(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings **set_r)
+{
+ return ibc->v.recv_handshake(ibc, set_r);
+}
+
+static enum dsync_ibc_send_ret
+dsync_ibc_send_ret(struct dsync_ibc *ibc)
+{
+ return ibc->v.is_send_queue_full(ibc) ?
+ DSYNC_IBC_SEND_RET_FULL :
+ DSYNC_IBC_SEND_RET_OK;
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type)
+{
+ ibc->v.send_end_of_list(ibc, type);
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_state *state)
+{
+ T_BEGIN {
+ ibc->v.send_mailbox_state(ibc, state);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc,
+ struct dsync_mailbox_state *state_r)
+{
+ return ibc->v.recv_mailbox_state(ibc, state_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc,
+ const char *const *name,
+ const struct dsync_mailbox_node *node)
+{
+ i_assert(*name != NULL);
+
+ T_BEGIN {
+ ibc->v.send_mailbox_tree_node(ibc, name, node);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc,
+ const char *const **name_r,
+ const struct dsync_mailbox_node **node_r)
+{
+ return ibc->v.recv_mailbox_tree_node(ibc, name_r, node_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete *deletes,
+ unsigned int count, char hierarchy_sep,
+ char escape_char)
+{
+ T_BEGIN {
+ ibc->v.send_mailbox_deletes(ibc, deletes, count,
+ hierarchy_sep, escape_char);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete **deletes_r,
+ unsigned int *count_r, char *hierarchy_sep_r,
+ char *escape_char_r)
+{
+ return ibc->v.recv_mailbox_deletes(ibc, deletes_r, count_r,
+ hierarchy_sep_r, escape_char_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mailbox(struct dsync_ibc *ibc,
+ const struct dsync_mailbox *dsync_box)
+{
+ T_BEGIN {
+ ibc->v.send_mailbox(ibc, dsync_box);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox(struct dsync_ibc *ibc,
+ const struct dsync_mailbox **dsync_box_r)
+{
+ return ibc->v.recv_mailbox(ibc, dsync_box_r);
+}
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute *attr)
+{
+ T_BEGIN {
+ ibc->v.send_mailbox_attribute(ibc, attr);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute **attr_r)
+{
+ return ibc->v.recv_mailbox_attribute(ibc, attr_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_change(struct dsync_ibc *ibc,
+ const struct dsync_mail_change *change)
+{
+ i_assert(change->uid > 0);
+
+ T_BEGIN {
+ ibc->v.send_change(ibc, change);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_change(struct dsync_ibc *ibc,
+ const struct dsync_mail_change **change_r)
+{
+ return ibc->v.recv_change(ibc, change_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mail_request(struct dsync_ibc *ibc,
+ const struct dsync_mail_request *request)
+{
+ i_assert(request->guid != NULL || request->uid != 0);
+
+ T_BEGIN {
+ ibc->v.send_mail_request(ibc, request);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail_request(struct dsync_ibc *ibc,
+ const struct dsync_mail_request **request_r)
+{
+ return ibc->v.recv_mail_request(ibc, request_r);
+}
+
+enum dsync_ibc_send_ret
+dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail)
+{
+ i_assert(*mail->guid != '\0' || mail->uid != 0);
+
+ T_BEGIN {
+ ibc->v.send_mail(ibc, mail);
+ } T_END;
+ return dsync_ibc_send_ret(ibc);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r)
+{
+ return ibc->v.recv_mail(ibc, mail_r);
+}
+
+void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error,
+ enum mail_error mail_error,
+ bool require_full_resync)
+{
+ ibc->v.send_finish(ibc, error, mail_error, require_full_resync);
+}
+
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r,
+ enum mail_error *mail_error_r,
+ bool *require_full_resync_r)
+{
+ return ibc->v.recv_finish(ibc, error_r, mail_error_r,
+ require_full_resync_r);
+}
+
+void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc)
+{
+ ibc->v.close_mail_streams(ibc);
+}
+
+bool dsync_ibc_has_failed(struct dsync_ibc *ibc)
+{
+ return ibc->failed;
+}
+
+bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc)
+{
+ return ibc->timeout;
+}
+
+bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc)
+{
+ return ibc->v.is_send_queue_full(ibc);
+}
+
+bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc)
+{
+ return ibc->v.has_pending_data(ibc);
+}
diff --git a/src/doveadm/dsync/dsync-ibc.h b/src/doveadm/dsync/dsync-ibc.h
new file mode 100644
index 0000000..dc85560
--- /dev/null
+++ b/src/doveadm/dsync/dsync-ibc.h
@@ -0,0 +1,178 @@
+#ifndef DSYNC_IBC_H
+#define DSYNC_IBC_H
+
+/* dsync inter-brain communicator */
+
+#include "ioloop.h"
+#include "guid.h"
+#include "mail-error.h"
+#include "dsync-brain.h"
+
+struct dsync_mailbox;
+struct dsync_mailbox_state;
+struct dsync_mailbox_node;
+struct dsync_mailbox_delete;
+struct dsync_mailbox_attribute;
+struct dsync_mail;
+struct dsync_mail_change;
+struct dsync_mail_request;
+
+enum dsync_ibc_send_ret {
+ DSYNC_IBC_SEND_RET_OK = 1,
+ /* send queue is full, stop sending more */
+ DSYNC_IBC_SEND_RET_FULL = 0
+};
+
+enum dsync_ibc_recv_ret {
+ DSYNC_IBC_RECV_RET_FINISHED = -1,
+ /* try again / error (the error handling delayed until io callback) */
+ DSYNC_IBC_RECV_RET_TRYAGAIN = 0,
+ DSYNC_IBC_RECV_RET_OK = 1
+};
+
+enum dsync_ibc_eol_type {
+ DSYNC_IBC_EOL_MAILBOX_STATE,
+ DSYNC_IBC_EOL_MAILBOX_TREE,
+ DSYNC_IBC_EOL_MAILBOX_ATTRIBUTE,
+ DSYNC_IBC_EOL_MAILBOX,
+ DSYNC_IBC_EOL_MAIL_CHANGES,
+ DSYNC_IBC_EOL_MAIL_REQUESTS,
+ DSYNC_IBC_EOL_MAILS
+};
+
+struct dsync_ibc_settings {
+ /* Server hostname. Used for determining which server does the
+ locking. */
+ const char *hostname;
+ /* if non-NULL, sync only these namespaces (LF-separated) */
+ const char *sync_ns_prefixes;
+ /* if non-NULL, sync only this mailbox name */
+ const char *sync_box;
+ /* if non-NULL, use this mailbox for finding messages with GUIDs and
+ copying them instead of saving them again. */
+ const char *virtual_all_box;
+ /* if non-empty, sync only this mailbox GUID */
+ guid_128_t sync_box_guid;
+ /* Exclude these mailboxes from the sync. They can contain '*'
+ wildcards and be \special-use flags. */
+ const char *const *exclude_mailboxes;
+ /* Sync only mails with received timestamp at least this high. */
+ time_t sync_since_timestamp;
+ /* Sync only mails with received timestamp less or equal than this */
+ time_t sync_until_timestamp;
+ /* Don't sync mails larger than this. */
+ uoff_t sync_max_size;
+ /* Sync only mails with specified flags. */
+ const char *sync_flags;
+ /* Hashed headers */
+ const char *const *hashed_headers;
+
+ char alt_char;
+ enum dsync_brain_sync_type sync_type;
+ enum dsync_brain_flags brain_flags;
+ bool hdr_hash_v2;
+ bool hdr_hash_v3;
+ unsigned int lock_timeout;
+ unsigned int import_commit_msgs_interval;
+};
+
+void dsync_ibc_init_pipe(struct dsync_ibc **ibc1_r,
+ struct dsync_ibc **ibc2_r);
+struct dsync_ibc *
+dsync_ibc_init_stream(struct istream *input, struct ostream *output,
+ const char *name, const char *temp_path_prefix,
+ unsigned int timeout_secs);
+void dsync_ibc_deinit(struct dsync_ibc **ibc);
+
+/* I/O callback is called whenever new data is available. It's also called on
+ errors, so check first the error status. */
+void dsync_ibc_set_io_callback(struct dsync_ibc *ibc,
+ io_callback_t *callback, void *context);
+
+void dsync_ibc_send_handshake(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings *set);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_handshake(struct dsync_ibc *ibc,
+ const struct dsync_ibc_settings **set_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_end_of_list(struct dsync_ibc *ibc, enum dsync_ibc_eol_type type);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_state(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_state *state);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_state(struct dsync_ibc *ibc,
+ struct dsync_mailbox_state *state_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_tree_node(struct dsync_ibc *ibc,
+ const char *const *name,
+ const struct dsync_mailbox_node *node);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_tree_node(struct dsync_ibc *ibc,
+ const char *const **name_r,
+ const struct dsync_mailbox_node **node_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_deletes(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete *deletes,
+ unsigned int count, char hierarchy_sep,
+ char escape_char);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_deletes(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_delete **deletes_r,
+ unsigned int *count_r, char *hierarchy_sep_r,
+ char *escape_char_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox(struct dsync_ibc *ibc,
+ const struct dsync_mailbox *dsync_box);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox(struct dsync_ibc *ibc,
+ const struct dsync_mailbox **dsync_box_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mailbox_attribute(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute *attr);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mailbox_attribute(struct dsync_ibc *ibc,
+ const struct dsync_mailbox_attribute **attr_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_change(struct dsync_ibc *ibc,
+ const struct dsync_mail_change *change);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_change(struct dsync_ibc *ibc,
+ const struct dsync_mail_change **change_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mail_request(struct dsync_ibc *ibc,
+ const struct dsync_mail_request *request);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail_request(struct dsync_ibc *ibc,
+ const struct dsync_mail_request **request_r);
+
+enum dsync_ibc_send_ret ATTR_NOWARN_UNUSED_RESULT
+dsync_ibc_send_mail(struct dsync_ibc *ibc, const struct dsync_mail *mail);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_mail(struct dsync_ibc *ibc, struct dsync_mail **mail_r);
+
+void dsync_ibc_send_finish(struct dsync_ibc *ibc, const char *error,
+ enum mail_error mail_error,
+ bool require_full_resync);
+enum dsync_ibc_recv_ret
+dsync_ibc_recv_finish(struct dsync_ibc *ibc, const char **error_r,
+ enum mail_error *mail_error_r,
+ bool *require_full_resync_r);
+
+/* Close any mail input streams that are kept open. This needs to be called
+ before the mail is attempted to be freed (usually on error conditions). */
+void dsync_ibc_close_mail_streams(struct dsync_ibc *ibc);
+
+bool dsync_ibc_has_failed(struct dsync_ibc *ibc);
+bool dsync_ibc_has_timed_out(struct dsync_ibc *ibc);
+bool dsync_ibc_is_send_queue_full(struct dsync_ibc *ibc);
+bool dsync_ibc_has_pending_data(struct dsync_ibc *ibc);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mail.c b/src/doveadm/dsync/dsync-mail.c
new file mode 100644
index 0000000..b82818f
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mail.c
@@ -0,0 +1,156 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hex-binary.h"
+#include "md5.h"
+#include "istream.h"
+#include "istream-crlf.h"
+#include "message-header-hash.h"
+#include "message-size.h"
+#include "mail-storage.h"
+#include "dsync-mail.h"
+
+struct mailbox_header_lookup_ctx *
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers)
+{
+ return mailbox_header_lookup_init(box, hashed_headers);
+}
+
+int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
+ const char *const *hashed_headers, const char **hdr_hash_r)
+{
+ struct istream *hdr_input, *input;
+ struct mailbox_header_lookup_ctx *hdr_ctx;
+ struct message_header_hash_context hash_ctx;
+ struct md5_context md5_ctx;
+ unsigned char md5_result[MD5_RESULTLEN];
+ const unsigned char *data;
+ size_t size;
+ ssize_t sret;
+ int ret = 0;
+
+ hdr_ctx = mailbox_header_lookup_init(mail->box, hashed_headers);
+ ret = mail_get_header_stream(mail, hdr_ctx, &hdr_input);
+ mailbox_header_lookup_unref(&hdr_ctx);
+ if (ret < 0)
+ return -1;
+
+ input = i_stream_create_lf(hdr_input);
+
+ md5_init(&md5_ctx);
+ i_zero(&hash_ctx);
+ while ((sret = i_stream_read_more(input, &data, &size)) > 0) {
+ message_header_hash_more(&hash_ctx, &hash_method_md5, &md5_ctx,
+ version, data, size);
+ i_stream_skip(input, size);
+ }
+ i_assert(sret == -1);
+ if (input->stream_errno != 0)
+ ret = -1;
+ i_stream_unref(&input);
+
+ md5_final(&md5_ctx, md5_result);
+ *hdr_hash_r = binary_to_hex(md5_result, sizeof(md5_result));
+ return ret;
+}
+
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+ struct dsync_mail *dmail_r, const char **error_field_r)
+{
+ const char *guid;
+
+ i_zero(dmail_r);
+
+ if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
+ *error_field_r = "GUID";
+ return -1;
+ }
+ dmail_r->guid = guid;
+ dmail_r->uid = mail->uid;
+
+ dmail_r->input_mail = mail;
+ dmail_r->input_mail_uid = mail->uid;
+
+ if (mail_get_save_date(mail, &dmail_r->saved_date) < 0) {
+ *error_field_r = "saved-date";
+ return -1;
+ }
+ if (!minimal_fill)
+ return dsync_mail_fill_nonminimal(mail, dmail_r, error_field_r);
+ dmail_r->minimal_fields = TRUE;
+ return 0;
+}
+
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+ const char **error_field_r)
+{
+ const char *str;
+
+ if (mail_get_stream(mail, NULL, NULL, &dmail_r->input) < 0) {
+ *error_field_r = "body";
+ return -1;
+ }
+
+ if (mail_get_special(mail, MAIL_FETCH_UIDL_BACKEND, &dmail_r->pop3_uidl) < 0) {
+ *error_field_r = "pop3-uidl";
+ return -1;
+ }
+ if (mail_get_special(mail, MAIL_FETCH_POP3_ORDER, &str) < 0) {
+ *error_field_r = "pop3-order";
+ return -1;
+ }
+ if (*str != '\0') {
+ if (str_to_uint32(str, &dmail_r->pop3_order) < 0)
+ i_unreached();
+ }
+ if (mail_get_received_date(mail, &dmail_r->received_date) < 0) {
+ *error_field_r = "received-date";
+ return -1;
+ }
+ return 0;
+}
+
+static void
+const_string_array_dup(pool_t pool, const ARRAY_TYPE(const_string) *src,
+ ARRAY_TYPE(const_string) *dest)
+{
+ const char *const *strings, *str;
+ unsigned int i, count;
+
+ if (!array_is_created(src))
+ return;
+
+ strings = array_get(src, &count);
+ if (count == 0)
+ return;
+
+ p_array_init(dest, pool, count);
+ for (i = 0; i < count; i++) {
+ str = p_strdup(pool, strings[i]);
+ array_push_back(dest, &str);
+ }
+}
+
+void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
+ struct dsync_mail_change *dest_r)
+{
+ dest_r->type = src->type;
+ dest_r->uid = src->uid;
+ if (src->guid != NULL) {
+ dest_r->guid = *src->guid == '\0' ? "" :
+ p_strdup(pool, src->guid);
+ }
+ dest_r->hdr_hash = p_strdup(pool, src->hdr_hash);
+ dest_r->modseq = src->modseq;
+ dest_r->pvt_modseq = src->pvt_modseq;
+
+ dest_r->add_flags = src->add_flags;
+ dest_r->remove_flags = src->remove_flags;
+ dest_r->final_flags = src->final_flags;
+ dest_r->keywords_reset = src->keywords_reset;
+ const_string_array_dup(pool, &src->keyword_changes,
+ &dest_r->keyword_changes);
+ dest_r->received_timestamp = src->received_timestamp;
+ dest_r->virtual_size = src->virtual_size;
+}
diff --git a/src/doveadm/dsync/dsync-mail.h b/src/doveadm/dsync/dsync-mail.h
new file mode 100644
index 0000000..fc00059
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mail.h
@@ -0,0 +1,108 @@
+#ifndef DSYNC_MAIL_H
+#define DSYNC_MAIL_H
+
+#include "mail-types.h"
+
+struct md5_context;
+struct mail;
+struct mailbox;
+
+struct dsync_mail {
+ /* either GUID="" or uid=0 */
+ const char *guid;
+ uint32_t uid;
+ time_t saved_date;
+
+ /* If non-NULL, we're syncing within the dsync process using ibc-pipe.
+ This mail can be used to mailbox_copy() the mail. */
+ struct mail *input_mail;
+ /* Verify that this equals to input_mail->uid */
+ uint32_t input_mail_uid;
+
+ /* TRUE if the following fields aren't set, because minimal_fill=TRUE
+ parameter was used. */
+ bool minimal_fields;
+
+ const char *pop3_uidl;
+ uint32_t pop3_order;
+ time_t received_date;
+ /* Input stream containing the message text, or NULL if all instances
+ of the message were already expunged from this mailbox. */
+ struct istream *input;
+};
+
+struct dsync_mail_request {
+ /* either GUID=NULL or uid=0 */
+ const char *guid;
+ uint32_t uid;
+};
+
+enum dsync_mail_change_type {
+ DSYNC_MAIL_CHANGE_TYPE_SAVE,
+ DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+ DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE
+};
+
+#define KEYWORD_CHANGE_ADD '+'
+#define KEYWORD_CHANGE_REMOVE '-'
+#define KEYWORD_CHANGE_FINAL '='
+#define KEYWORD_CHANGE_ADD_AND_FINAL '&'
+
+struct dsync_mail_change {
+ enum dsync_mail_change_type type;
+
+ uint32_t uid;
+ /* Message's GUID:
+ - for expunges either 128bit hex or NULL if unknown
+ - "" if backend doesn't support GUIDs */
+ const char *guid;
+ /* If GUID is "", this contains hash of the message header,
+ otherwise NULL */
+ const char *hdr_hash;
+
+ /* Message's current modseq (saves, flag changes) */
+ uint64_t modseq;
+ /* Message's current private modseq (for private flags in
+ shared mailboxes, otherwise 0) */
+ uint64_t pvt_modseq;
+
+ /* List of flag/keyword changes: (saves, flag changes) */
+
+ /* Flags added/removed since last sync, and final flags containing
+ flags that exist now but haven't changed */
+ uint8_t add_flags, remove_flags, final_flags;
+ uint8_t add_pvt_flags, remove_pvt_flags;
+ /* Remove all keywords before applying changes. This is used only with
+ old transaction logs, new ones never reset keywords (just explicitly
+ remove unwanted keywords) */
+ bool keywords_reset;
+ /* +add, -remove, =final, &add_and_final. */
+ ARRAY_TYPE(const_string) keyword_changes;
+
+ /* Received timestamp for saves, if brain.sync_since/until_timestamp is set */
+ time_t received_timestamp;
+ /* Mail's size for saves if brain.sync_max_size is set,
+ UOFF_T_MAX otherwise. */
+ uoff_t virtual_size;
+};
+
+struct mailbox_header_lookup_ctx *
+dsync_mail_get_hash_headers(struct mailbox *box, const char *const *hashed_headers);
+
+int dsync_mail_get_hdr_hash(struct mail *mail, unsigned int version,
+ const char *const *hashed_headers, const char **hdr_hash_r);
+static inline bool dsync_mail_hdr_hash_is_empty(const char *hdr_hash)
+{
+ /* md5(\n) */
+ return strcmp(hdr_hash, "68b329da9893e34099c7d8ad5cb9c940") == 0;
+}
+
+int dsync_mail_fill(struct mail *mail, bool minimal_fill,
+ struct dsync_mail *dmail_r, const char **error_field_r);
+int dsync_mail_fill_nonminimal(struct mail *mail, struct dsync_mail *dmail_r,
+ const char **error_field_r);
+
+void dsync_mail_change_dup(pool_t pool, const struct dsync_mail_change *src,
+ struct dsync_mail_change *dest_r);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-export.c b/src/doveadm/dsync/dsync-mailbox-export.c
new file mode 100644
index 0000000..a58fbec
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-export.c
@@ -0,0 +1,961 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "istream.h"
+#include "mail-index-modseq.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "dsync-transaction-log-scan.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-export.h"
+
+struct dsync_mail_guid_instances {
+ ARRAY_TYPE(seq_range) seqs;
+ bool requested;
+ bool searched;
+};
+
+struct dsync_mailbox_exporter {
+ pool_t pool;
+ struct mailbox *box;
+ struct dsync_transaction_log_scan *log_scan;
+ uint32_t last_common_uid;
+
+ struct mailbox_header_lookup_ctx *wanted_headers;
+ struct mailbox_transaction_context *trans;
+ struct mail_search_context *search_ctx;
+ unsigned int search_pos, search_count;
+ unsigned int hdr_hash_version;
+
+ const char *const *hashed_headers;
+
+ /* GUID => instances */
+ HASH_TABLE(char *, struct dsync_mail_guid_instances *) export_guids;
+ ARRAY_TYPE(seq_range) requested_uids;
+ ARRAY_TYPE(seq_range) search_uids;
+
+ ARRAY_TYPE(seq_range) expunged_seqs;
+ ARRAY_TYPE(const_string) expunged_guids;
+ unsigned int expunged_guid_idx;
+
+ /* uint32_t UID => struct dsync_mail_change */
+ HASH_TABLE(void *, struct dsync_mail_change *) changes;
+ /* changes sorted by UID */
+ ARRAY(struct dsync_mail_change *) sorted_changes;
+ unsigned int change_idx;
+ uint32_t highest_changed_uid;
+
+ struct mailbox_attribute_iter *attr_iter;
+ struct hash_iterate_context *attr_change_iter;
+ enum mail_attribute_type attr_type;
+ struct dsync_mailbox_attribute attr;
+
+ struct dsync_mail_change change;
+ struct dsync_mail dsync_mail;
+
+ const char *error;
+ enum mail_error mail_error;
+
+ bool body_search_initialized:1;
+ bool auto_export_mails:1;
+ bool mails_have_guids:1;
+ bool minimal_dmail_fill:1;
+ bool return_all_mails:1;
+ bool export_received_timestamps:1;
+ bool export_virtual_sizes:1;
+ bool no_hdr_hashes:1;
+};
+
+static int dsync_mail_error(struct dsync_mailbox_exporter *exporter,
+ struct mail *mail, const char *field)
+{
+ const char *errstr;
+ enum mail_error error;
+
+ errstr = mailbox_get_last_internal_error(exporter->box, &error);
+ if (error == MAIL_ERROR_EXPUNGED)
+ return 0;
+
+ exporter->mail_error = error;
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Can't lookup %s for UID=%u: %s",
+ field, mail->uid, errstr);
+ return -1;
+}
+
+static bool
+final_keyword_check(struct dsync_mail_change *change, const char *name,
+ char *type_r)
+{
+ const char *const *changes;
+ unsigned int i, count;
+
+ *type_r = KEYWORD_CHANGE_FINAL;
+
+ changes = array_get(&change->keyword_changes, &count);
+ for (i = 0; i < count; i++) {
+ if (strcmp(changes[i]+1, name) != 0)
+ continue;
+
+ switch (changes[i][0]) {
+ case KEYWORD_CHANGE_ADD:
+ /* replace with ADD_AND_FINAL */
+ array_delete(&change->keyword_changes, i, 1);
+ *type_r = KEYWORD_CHANGE_ADD_AND_FINAL;
+ return FALSE;
+ case KEYWORD_CHANGE_REMOVE:
+ /* a final keyword is marked as removed.
+ this shouldn't normally happen. */
+ array_delete(&change->keyword_changes, i, 1);
+ return FALSE;
+ case KEYWORD_CHANGE_ADD_AND_FINAL:
+ case KEYWORD_CHANGE_FINAL:
+ /* no change */
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static void
+search_update_flag_changes(struct dsync_mailbox_exporter *exporter,
+ struct mail *mail, struct dsync_mail_change *change)
+{
+ const char *const *keywords;
+ unsigned int i;
+ char type;
+
+ i_assert((change->add_flags & change->remove_flags) == 0);
+
+ change->modseq = mail_get_modseq(mail);
+ change->pvt_modseq = mail_get_pvt_modseq(mail);
+ change->final_flags = mail_get_flags(mail);
+
+ keywords = mail_get_keywords(mail);
+ if (!array_is_created(&change->keyword_changes) &&
+ keywords[0] != NULL) {
+ p_array_init(&change->keyword_changes, exporter->pool,
+ str_array_length(keywords));
+ }
+ for (i = 0; keywords[i] != NULL; i++) {
+ /* add the final keyword if it's not already there
+ as +keyword */
+ if (!final_keyword_check(change, keywords[i], &type)) {
+ const char *keyword_change =
+ p_strdup_printf(exporter->pool, "%c%s",
+ type, keywords[i]);
+ array_push_back(&change->keyword_changes,
+ &keyword_change);
+ }
+ }
+}
+
+static int
+exporter_get_guids(struct dsync_mailbox_exporter *exporter,
+ struct mail *mail, const char **guid_r,
+ const char **hdr_hash_r)
+{
+ *guid_r = "";
+ *hdr_hash_r = NULL;
+
+ /* always try to get GUID, even if we're also getting header hash */
+ if (mail_get_special(mail, MAIL_FETCH_GUID, guid_r) < 0)
+ return dsync_mail_error(exporter, mail, "GUID");
+
+ if (!exporter->mails_have_guids) {
+ /* get header hash also */
+ if (exporter->no_hdr_hashes) {
+ *hdr_hash_r = "";
+ return 1;
+ }
+ if (dsync_mail_get_hdr_hash(mail, exporter->hdr_hash_version,
+ exporter->hashed_headers, hdr_hash_r) < 0)
+ return dsync_mail_error(exporter, mail, "hdr-stream");
+ return 1;
+ } else if (**guid_r == '\0') {
+ exporter->mail_error = MAIL_ERROR_TEMP;
+ exporter->error = "Backend doesn't support GUIDs, "
+ "sync with header hashes instead";
+ return -1;
+ } else {
+ /* GUIDs are required, we don't need header hash */
+ return 1;
+ }
+}
+
+static int
+search_update_flag_change_guid(struct dsync_mailbox_exporter *exporter,
+ struct mail *mail)
+{
+ struct dsync_mail_change *change, *log_change;
+ const char *guid, *hdr_hash;
+ int ret;
+
+ change = hash_table_lookup(exporter->changes, POINTER_CAST(mail->uid));
+ if (change != NULL) {
+ i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
+ } else {
+ i_assert(exporter->return_all_mails);
+
+ change = p_new(exporter->pool, struct dsync_mail_change, 1);
+ change->uid = mail->uid;
+ change->type = DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE;
+ hash_table_insert(exporter->changes,
+ POINTER_CAST(mail->uid), change);
+ }
+
+ if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) < 0)
+ return -1;
+ if (ret == 0) {
+ /* the message was expunged during export */
+ i_zero(change);
+ change->type = DSYNC_MAIL_CHANGE_TYPE_EXPUNGE;
+ change->uid = mail->uid;
+
+ /* find its GUID from log if possible */
+ log_change = dsync_transaction_log_scan_find_new_expunge(
+ exporter->log_scan, mail->uid);
+ if (log_change != NULL)
+ change->guid = log_change->guid;
+ } else {
+ change->guid = *guid == '\0' ? "" :
+ p_strdup(exporter->pool, guid);
+ change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+ search_update_flag_changes(exporter, mail, change);
+ }
+ return 0;
+}
+
+static struct dsync_mail_change *
+export_save_change_get(struct dsync_mailbox_exporter *exporter, uint32_t uid)
+{
+ struct dsync_mail_change *change;
+
+ change = hash_table_lookup(exporter->changes, POINTER_CAST(uid));
+ if (change == NULL) {
+ change = p_new(exporter->pool, struct dsync_mail_change, 1);
+ change->uid = uid;
+ hash_table_insert(exporter->changes, POINTER_CAST(uid), change);
+ } else {
+ /* move flag changes into a save. this happens only when
+ last_common_uid isn't known */
+ i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE);
+ i_assert(exporter->last_common_uid == 0);
+ }
+
+ change->type = DSYNC_MAIL_CHANGE_TYPE_SAVE;
+ return change;
+}
+
+static void
+export_add_mail_instance(struct dsync_mailbox_exporter *exporter,
+ struct dsync_mail_change *change, uint32_t seq)
+{
+ struct dsync_mail_guid_instances *instances;
+
+ if (exporter->auto_export_mails && !exporter->mails_have_guids) {
+ /* GUIDs not supported, mail is requested by UIDs */
+ seq_range_array_add(&exporter->requested_uids, change->uid);
+ return;
+ }
+ if (*change->guid == '\0') {
+ /* mail UIDs are manually requested */
+ i_assert(!exporter->mails_have_guids);
+ return;
+ }
+
+ instances = hash_table_lookup(exporter->export_guids, change->guid);
+ if (instances == NULL) {
+ instances = p_new(exporter->pool,
+ struct dsync_mail_guid_instances, 1);
+ p_array_init(&instances->seqs, exporter->pool, 2);
+ hash_table_insert(exporter->export_guids,
+ p_strdup(exporter->pool, change->guid),
+ instances);
+ if (exporter->auto_export_mails)
+ instances->requested = TRUE;
+ }
+ seq_range_array_add(&instances->seqs, seq);
+}
+
+static int
+search_add_save(struct dsync_mailbox_exporter *exporter, struct mail *mail)
+{
+ struct dsync_mail_change *change;
+ const char *guid, *hdr_hash;
+ enum mail_fetch_field wanted_fields = MAIL_FETCH_GUID;
+ time_t received_timestamp = 0;
+ uoff_t virtual_size = UOFF_T_MAX;
+ int ret;
+
+ /* update wanted fields in case we didn't already set them for the
+ search */
+ if (exporter->export_received_timestamps)
+ wanted_fields |= MAIL_FETCH_RECEIVED_DATE;
+ if (exporter->export_virtual_sizes)
+ wanted_fields |= MAIL_FETCH_VIRTUAL_SIZE;
+ mail_add_temp_wanted_fields(mail, wanted_fields,
+ exporter->wanted_headers);
+
+ /* If message is already expunged here, just skip it */
+ if ((ret = exporter_get_guids(exporter, mail, &guid, &hdr_hash)) <= 0)
+ return ret;
+
+ if (exporter->export_received_timestamps) {
+ if (mail_get_received_date(mail, &received_timestamp) < 0)
+ return dsync_mail_error(exporter, mail, "received-time");
+ if (received_timestamp == 0) {
+ /* don't allow timestamps to be zero. we want to have
+ asserts verify that the timestamp is set properly. */
+ received_timestamp = 1;
+ }
+ }
+ if (exporter->export_virtual_sizes) {
+ if (mail_get_virtual_size(mail, &virtual_size) < 0)
+ return dsync_mail_error(exporter, mail, "virtual-size");
+ i_assert(virtual_size != UOFF_T_MAX);
+ }
+
+ change = export_save_change_get(exporter, mail->uid);
+ change->guid = *guid == '\0' ? "" :
+ p_strdup(exporter->pool, guid);
+ change->hdr_hash = p_strdup(exporter->pool, hdr_hash);
+ change->received_timestamp = received_timestamp;
+ change->virtual_size = virtual_size;
+ search_update_flag_changes(exporter, mail, change);
+
+ export_add_mail_instance(exporter, change, mail->seq);
+ return 1;
+}
+
+static void
+dsync_mailbox_export_add_flagchange_uids(struct dsync_mailbox_exporter *exporter,
+ ARRAY_TYPE(seq_range) *uids)
+{
+ struct hash_iterate_context *iter;
+ void *key;
+ struct dsync_mail_change *change;
+
+ iter = hash_table_iterate_init(exporter->changes);
+ while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
+ if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE)
+ seq_range_array_add(uids, change->uid);
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+static void
+dsync_mailbox_export_drop_expunged_flag_changes(struct dsync_mailbox_exporter *exporter)
+{
+ struct hash_iterate_context *iter;
+ void *key;
+ struct dsync_mail_change *change;
+
+ /* any flag changes for UIDs above last_common_uid weren't found by
+ mailbox search, which means they were already expunged. for some
+ reason the log scanner found flag changes for the message, but not
+ the expunge. just remove these. */
+ iter = hash_table_iterate_init(exporter->changes);
+ while (hash_table_iterate(iter, exporter->changes, &key, &change)) {
+ if (change->type == DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE &&
+ change->uid > exporter->last_common_uid)
+ hash_table_remove(exporter->changes, key);
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+static void
+dsync_mailbox_export_search(struct dsync_mailbox_exporter *exporter)
+{
+ struct mail_search_context *search_ctx;
+ struct mail_search_args *search_args;
+ struct mail_search_arg *sarg;
+ struct mail *mail;
+ enum mail_fetch_field wanted_fields = 0;
+ struct mailbox_header_lookup_ctx *wanted_headers = NULL;
+ int ret = 0;
+
+ search_args = mail_search_build_init();
+ sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
+ p_array_init(&sarg->value.seqset, search_args->pool, 1);
+
+ if (exporter->return_all_mails || exporter->last_common_uid == 0) {
+ /* we want to know about all mails */
+ seq_range_array_add_range(&sarg->value.seqset, 1, (uint32_t)-1);
+ } else {
+ /* lookup GUIDs for messages with flag changes */
+ dsync_mailbox_export_add_flagchange_uids(exporter,
+ &sarg->value.seqset);
+ /* lookup new messages */
+ seq_range_array_add_range(&sarg->value.seqset,
+ exporter->last_common_uid + 1,
+ (uint32_t)-1);
+ }
+
+ if (exporter->last_common_uid == 0) {
+ /* we're syncing all mails, so we can request the wanted
+ fields for all the mails */
+ wanted_fields = MAIL_FETCH_GUID;
+ wanted_headers = exporter->wanted_headers;
+ }
+
+ exporter->trans = mailbox_transaction_begin(exporter->box,
+ MAILBOX_TRANSACTION_FLAG_SYNC,
+ __func__);
+ search_ctx = mailbox_search_init(exporter->trans, search_args, NULL,
+ wanted_fields, wanted_headers);
+ mail_search_args_unref(&search_args);
+
+ while (mailbox_search_next(search_ctx, &mail)) {
+ T_BEGIN {
+ if (mail->uid <= exporter->last_common_uid)
+ ret = search_update_flag_change_guid(exporter, mail);
+ else
+ ret = search_add_save(exporter, mail);
+ } T_END;
+ if (ret < 0)
+ break;
+ }
+ i_assert(ret >= 0 || exporter->error != NULL);
+
+ dsync_mailbox_export_drop_expunged_flag_changes(exporter);
+
+ if (mailbox_search_deinit(&search_ctx) < 0 &&
+ exporter->error == NULL) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mail search failed: %s",
+ mailbox_get_last_internal_error(exporter->box,
+ &exporter->mail_error));
+ }
+}
+
+static int dsync_mail_change_p_uid_cmp(struct dsync_mail_change *const *c1,
+ struct dsync_mail_change *const *c2)
+{
+ if ((*c1)->uid < (*c2)->uid)
+ return -1;
+ if ((*c1)->uid > (*c2)->uid)
+ return 1;
+ return 0;
+}
+
+static void
+dsync_mailbox_export_sort_changes(struct dsync_mailbox_exporter *exporter)
+{
+ struct hash_iterate_context *iter;
+ void *key;
+ struct dsync_mail_change *change;
+
+ p_array_init(&exporter->sorted_changes, exporter->pool,
+ hash_table_count(exporter->changes));
+
+ iter = hash_table_iterate_init(exporter->changes);
+ while (hash_table_iterate(iter, exporter->changes, &key, &change))
+ array_push_back(&exporter->sorted_changes, &change);
+ hash_table_iterate_deinit(&iter);
+ array_sort(&exporter->sorted_changes, dsync_mail_change_p_uid_cmp);
+}
+
+static void
+dsync_mailbox_export_attr_init(struct dsync_mailbox_exporter *exporter,
+ enum mail_attribute_type type)
+{
+ exporter->attr_iter =
+ mailbox_attribute_iter_init(exporter->box, type, "");
+ exporter->attr_type = type;
+}
+
+static void
+dsync_mailbox_export_log_scan(struct dsync_mailbox_exporter *exporter,
+ struct dsync_transaction_log_scan *log_scan)
+{
+ HASH_TABLE_TYPE(dsync_uid_mail_change) log_changes;
+ struct hash_iterate_context *iter;
+ void *key;
+ struct dsync_mail_change *change, *dup_change;
+
+ log_changes = dsync_transaction_log_scan_get_hash(log_scan);
+ if (dsync_transaction_log_scan_has_all_changes(log_scan)) {
+ /* we tried to access too old/invalid modseqs. to make sure
+ no changes get lost, we need to send all of the messages */
+ exporter->return_all_mails = TRUE;
+ }
+
+ /* clone the hash table, since we're changing it. */
+ hash_table_create_direct(&exporter->changes, exporter->pool,
+ hash_table_count(log_changes));
+ iter = hash_table_iterate_init(log_changes);
+ while (hash_table_iterate(iter, log_changes, &key, &change)) {
+ dup_change = p_new(exporter->pool, struct dsync_mail_change, 1);
+ *dup_change = *change;
+ hash_table_insert(exporter->changes, key, dup_change);
+ if (exporter->highest_changed_uid < change->uid)
+ exporter->highest_changed_uid = change->uid;
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+struct dsync_mailbox_exporter *
+dsync_mailbox_export_init(struct mailbox *box,
+ struct dsync_transaction_log_scan *log_scan,
+ uint32_t last_common_uid,
+ enum dsync_mailbox_exporter_flags flags,
+ unsigned int hdr_hash_version,
+ const char *const *hashed_headers)
+{
+ struct dsync_mailbox_exporter *exporter;
+ pool_t pool;
+
+ pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox export",
+ 4096);
+ exporter = p_new(pool, struct dsync_mailbox_exporter, 1);
+ exporter->pool = pool;
+ exporter->box = box;
+ exporter->log_scan = log_scan;
+ exporter->last_common_uid = last_common_uid;
+ exporter->auto_export_mails =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS) != 0;
+ exporter->mails_have_guids =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS) != 0;
+ exporter->minimal_dmail_fill =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL) != 0;
+ exporter->export_received_timestamps =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS) != 0;
+ exporter->export_virtual_sizes =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES) != 0;
+ exporter->hdr_hash_version = hdr_hash_version;
+ exporter->no_hdr_hashes =
+ (flags & DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES) != 0;
+ exporter->hashed_headers = hashed_headers;
+
+ p_array_init(&exporter->requested_uids, pool, 16);
+ p_array_init(&exporter->search_uids, pool, 16);
+ hash_table_create(&exporter->export_guids, pool, 0, str_hash, strcmp);
+ p_array_init(&exporter->expunged_seqs, pool, 16);
+ p_array_init(&exporter->expunged_guids, pool, 16);
+
+ if (!exporter->mails_have_guids && !exporter->no_hdr_hashes)
+ exporter->wanted_headers =
+ dsync_mail_get_hash_headers(box, exporter->hashed_headers);
+
+ /* first scan transaction log and save any expunges and flag changes */
+ dsync_mailbox_export_log_scan(exporter, log_scan);
+ /* get saves and also find GUIDs for flag changes */
+ dsync_mailbox_export_search(exporter);
+ /* get the changes sorted by UID */
+ dsync_mailbox_export_sort_changes(exporter);
+
+ dsync_mailbox_export_attr_init(exporter, MAIL_ATTRIBUTE_TYPE_PRIVATE);
+ return exporter;
+}
+
+static int
+dsync_mailbox_export_iter_next_nonexistent_attr(struct dsync_mailbox_exporter *exporter)
+{
+ struct dsync_mailbox_attribute *attr;
+ struct mail_attribute_value value;
+
+ while (hash_table_iterate(exporter->attr_change_iter,
+ dsync_transaction_log_scan_get_attr_hash(exporter->log_scan),
+ &attr, &attr)) {
+ if (attr->exported || !attr->deleted)
+ continue;
+
+ /* lookup the value mainly to get its last_change value. */
+ if (mailbox_attribute_get_stream(exporter->box, attr->type,
+ attr->key, &value) < 0) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mailbox attribute %s lookup failed: %s", attr->key,
+ mailbox_get_last_internal_error(exporter->box,
+ &exporter->mail_error));
+ break;
+ }
+ if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
+ i_stream_unref(&value.value_stream);
+ continue;
+ }
+
+ attr->last_change = value.last_change;
+ if (value.value != NULL || value.value_stream != NULL) {
+ attr->value = p_strdup(exporter->pool, value.value);
+ attr->value_stream = value.value_stream;
+ attr->deleted = FALSE;
+ }
+ attr->exported = TRUE;
+ exporter->attr = *attr;
+ return 1;
+ }
+ hash_table_iterate_deinit(&exporter->attr_change_iter);
+ return 0;
+}
+
+static int
+dsync_mailbox_export_iter_next_attr(struct dsync_mailbox_exporter *exporter)
+{
+ HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
+ struct dsync_mailbox_attribute lookup_attr, *attr;
+ struct dsync_mailbox_attribute *attr_change;
+ const char *key;
+ struct mail_attribute_value value;
+ bool export_all_attrs;
+
+ export_all_attrs = exporter->return_all_mails ||
+ exporter->last_common_uid == 0;
+ attr_changes = dsync_transaction_log_scan_get_attr_hash(exporter->log_scan);
+ lookup_attr.type = exporter->attr_type;
+
+ /* note that the order of processing may be important for some
+ attributes. for example sieve can't set a script active until it's
+ first been created */
+ while ((key = mailbox_attribute_iter_next(exporter->attr_iter)) != NULL) {
+ lookup_attr.key = key;
+ attr_change = hash_table_lookup(attr_changes, &lookup_attr);
+ if (attr_change == NULL && !export_all_attrs)
+ continue;
+
+ if (mailbox_attribute_get_stream(exporter->box,
+ exporter->attr_type, key,
+ &value) < 0) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mailbox attribute %s lookup failed: %s", key,
+ mailbox_get_last_internal_error(exporter->box,
+ &exporter->mail_error));
+ return -1;
+ }
+ if ((value.flags & MAIL_ATTRIBUTE_VALUE_FLAG_READONLY) != 0) {
+ /* readonly attributes can't be changed,
+ no point in exporting them */
+ if (value.value_stream != NULL)
+ i_stream_unref(&value.value_stream);
+ continue;
+ }
+ if (value.value == NULL && value.value_stream == NULL &&
+ (attr_change == NULL || !attr_change->deleted)) {
+ /* the attribute was just deleted?
+ skip for this sync. */
+ continue;
+ }
+ if (attr_change != NULL && attr_change->exported) {
+ /* duplicate attribute returned.
+ shouldn't normally happen, but don't crash. */
+ i_warning("Ignoring duplicate attributes '%s'", key);
+ continue;
+ }
+
+ attr = &exporter->attr;
+ i_zero(attr);
+ attr->type = exporter->attr_type;
+ attr->value = p_strdup(exporter->pool, value.value);
+ attr->value_stream = value.value_stream;
+ attr->last_change = value.last_change;
+ if (attr_change != NULL) {
+ attr_change->exported = TRUE;
+ attr->key = attr_change->key;
+ attr->deleted = attr_change->deleted &&
+ !DSYNC_ATTR_HAS_VALUE(attr);
+ attr->modseq = attr_change->modseq;
+ } else {
+ attr->key = p_strdup(exporter->pool, key);
+ }
+ return 1;
+ }
+ if (mailbox_attribute_iter_deinit(&exporter->attr_iter) < 0) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mailbox attribute iteration failed: %s",
+ mailbox_get_last_internal_error(exporter->box,
+ &exporter->mail_error));
+ return -1;
+ }
+ if (exporter->attr_type == MAIL_ATTRIBUTE_TYPE_PRIVATE) {
+ /* export shared attributes */
+ dsync_mailbox_export_attr_init(exporter,
+ MAIL_ATTRIBUTE_TYPE_SHARED);
+ return dsync_mailbox_export_iter_next_attr(exporter);
+ }
+ exporter->attr_change_iter = hash_table_iterate_init(attr_changes);
+ return dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
+}
+
+int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mailbox_attribute **attr_r)
+{
+ int ret;
+
+ if (exporter->error != NULL)
+ return -1;
+
+ i_stream_unref(&exporter->attr.value_stream);
+
+ if (exporter->attr_iter != NULL) {
+ ret = dsync_mailbox_export_iter_next_attr(exporter);
+ } else {
+ ret = dsync_mailbox_export_iter_next_nonexistent_attr(exporter);
+ }
+ if (ret > 0)
+ *attr_r = &exporter->attr;
+ return ret;
+}
+
+int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mail_change **change_r)
+{
+ struct dsync_mail_change *const *changes;
+ unsigned int count;
+
+ if (exporter->error != NULL)
+ return -1;
+
+ changes = array_get(&exporter->sorted_changes, &count);
+ if (exporter->change_idx == count)
+ return 0;
+ *change_r = changes[exporter->change_idx++];
+ return 1;
+}
+
+static int
+dsync_mailbox_export_body_search_init(struct dsync_mailbox_exporter *exporter)
+{
+ struct mail_search_args *search_args;
+ struct mail_search_arg *sarg;
+ struct hash_iterate_context *iter;
+ const struct seq_range *uids;
+ char *guid;
+ const char *const_guid;
+ enum mail_fetch_field wanted_fields;
+ struct dsync_mail_guid_instances *instances;
+ const struct seq_range *range;
+ unsigned int i, count;
+ uint32_t seq, seq1, seq2;
+
+ i_assert(exporter->search_ctx == NULL);
+
+ search_args = mail_search_build_init();
+ sarg = mail_search_build_add(search_args, SEARCH_SEQSET);
+ p_array_init(&sarg->value.seqset, search_args->pool, 128);
+
+ /* get a list of messages we want to fetch. if there are more than one
+ instance for a GUID, use the first one. */
+ iter = hash_table_iterate_init(exporter->export_guids);
+ while (hash_table_iterate(iter, exporter->export_guids,
+ &guid, &instances)) {
+ if (!instances->requested ||
+ array_count(&instances->seqs) == 0)
+ continue;
+
+ uids = array_front(&instances->seqs);
+ seq = uids[0].seq1;
+ if (!instances->searched) {
+ instances->searched = TRUE;
+ seq_range_array_add(&sarg->value.seqset, seq);
+ } else if (seq_range_exists(&exporter->expunged_seqs, seq)) {
+ /* we're on a second round, refetching expunged
+ messages */
+ seq_range_array_remove(&instances->seqs, seq);
+ seq_range_array_remove(&exporter->expunged_seqs, seq);
+ if (array_count(&instances->seqs) == 0) {
+ /* no instances left */
+ const_guid = guid;
+ array_push_back(&exporter->expunged_guids,
+ &const_guid);
+ continue;
+ }
+ uids = array_front(&instances->seqs);
+ seq = uids[0].seq1;
+ seq_range_array_add(&sarg->value.seqset, seq);
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+
+ /* add requested UIDs */
+ range = array_get(&exporter->requested_uids, &count);
+ for (i = 0; i < count; i++) {
+ mailbox_get_seq_range(exporter->box,
+ range[i].seq1, range[i].seq2,
+ &seq1, &seq2);
+ seq_range_array_add_range(&sarg->value.seqset,
+ seq1, seq2);
+ }
+ array_clear(&exporter->search_uids);
+ array_append_array(&exporter->search_uids, &exporter->requested_uids);
+ array_clear(&exporter->requested_uids);
+
+ wanted_fields = MAIL_FETCH_GUID | MAIL_FETCH_SAVE_DATE;
+ if (!exporter->minimal_dmail_fill) {
+ wanted_fields |= MAIL_FETCH_RECEIVED_DATE |
+ MAIL_FETCH_UIDL_BACKEND | MAIL_FETCH_POP3_ORDER |
+ MAIL_FETCH_STREAM_HEADER | MAIL_FETCH_STREAM_BODY;
+ }
+ exporter->search_count += seq_range_count(&sarg->value.seqset);
+ exporter->search_ctx =
+ mailbox_search_init(exporter->trans, search_args, NULL,
+ wanted_fields, NULL);
+ mail_search_args_unref(&search_args);
+ return array_count(&sarg->value.seqset) > 0 ? 1 : 0;
+}
+
+static void
+dsync_mailbox_export_body_search_deinit(struct dsync_mailbox_exporter *exporter)
+{
+ if (exporter->search_ctx == NULL)
+ return;
+
+ if (mailbox_search_deinit(&exporter->search_ctx) < 0 &&
+ exporter->error == NULL) {
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Mail search failed: %s",
+ mailbox_get_last_internal_error(exporter->box,
+ &exporter->mail_error));
+ }
+}
+
+static int dsync_mailbox_export_mail(struct dsync_mailbox_exporter *exporter,
+ struct mail *mail)
+{
+ struct dsync_mail_guid_instances *instances;
+ const char *error_field;
+
+ if (dsync_mail_fill(mail, exporter->minimal_dmail_fill,
+ &exporter->dsync_mail, &error_field) < 0)
+ return dsync_mail_error(exporter, mail, error_field);
+
+ instances = *exporter->dsync_mail.guid == '\0' ? NULL :
+ hash_table_lookup(exporter->export_guids,
+ exporter->dsync_mail.guid);
+ if (instances != NULL) {
+ /* GUID found */
+ } else if (exporter->dsync_mail.uid != 0) {
+ /* mail requested by UID */
+ } else {
+ exporter->mail_error = MAIL_ERROR_TEMP;
+ exporter->error = p_strdup_printf(exporter->pool,
+ "GUID unexpectedly changed for UID=%u GUID=%s",
+ mail->uid, exporter->dsync_mail.guid);
+ return -1;
+ }
+
+ if (!seq_range_exists(&exporter->search_uids, mail->uid))
+ exporter->dsync_mail.uid = 0;
+ else
+ exporter->dsync_mail.guid = "";
+
+ /* this message was successfully returned, don't try retrying it */
+ if (instances != NULL)
+ array_clear(&instances->seqs);
+ return 1;
+}
+
+void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mail_request *request)
+{
+ struct dsync_mail_guid_instances *instances;
+
+ i_assert(!exporter->auto_export_mails);
+
+ if (request->guid == NULL) {
+ i_assert(request->uid > 0);
+ seq_range_array_add(&exporter->requested_uids, request->uid);
+ return;
+ }
+
+ instances = hash_table_lookup(exporter->export_guids, request->guid);
+ if (instances == NULL) {
+ exporter->mail_error = MAIL_ERROR_TEMP;
+ exporter->error = p_strdup_printf(exporter->pool,
+ "Remote requested unexpected GUID %s", request->guid);
+ return;
+ }
+ instances->requested = TRUE;
+}
+
+int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mail **mail_r)
+{
+ struct mail *mail;
+ const char *const *guids;
+ unsigned int count;
+ int ret;
+
+ if (exporter->error != NULL)
+ return -1;
+ if (!exporter->body_search_initialized) {
+ exporter->body_search_initialized = TRUE;
+ if (dsync_mailbox_export_body_search_init(exporter) < 0) {
+ i_assert(exporter->error != NULL);
+ return -1;
+ }
+ }
+
+ while (mailbox_search_next(exporter->search_ctx, &mail)) {
+ exporter->search_pos++;
+ if ((ret = dsync_mailbox_export_mail(exporter, mail)) > 0) {
+ *mail_r = &exporter->dsync_mail;
+ return 1;
+ }
+ if (ret < 0) {
+ i_assert(exporter->error != NULL);
+ return -1;
+ }
+ /* the message was expunged. if the GUID has another instance,
+ try sending it later. */
+ seq_range_array_add(&exporter->expunged_seqs, mail->seq);
+ }
+ /* if some instances of messages were expunged, retry fetching them
+ with other instances */
+ dsync_mailbox_export_body_search_deinit(exporter);
+ if ((ret = dsync_mailbox_export_body_search_init(exporter)) < 0) {
+ i_assert(exporter->error != NULL);
+ return -1;
+ }
+ if (ret > 0) {
+ /* not finished yet */
+ return dsync_mailbox_export_next_mail(exporter, mail_r);
+ }
+
+ /* finished with messages. if there are any expunged messages,
+ return them */
+ guids = array_get(&exporter->expunged_guids, &count);
+ if (exporter->expunged_guid_idx < count) {
+ i_zero(&exporter->dsync_mail);
+ exporter->dsync_mail.guid =
+ guids[exporter->expunged_guid_idx++];
+ *mail_r = &exporter->dsync_mail;
+ return 1;
+ }
+ return 0;
+}
+
+int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **_exporter,
+ const char **errstr_r, enum mail_error *error_r)
+{
+ struct dsync_mailbox_exporter *exporter = *_exporter;
+
+ *_exporter = NULL;
+
+ if (exporter->attr_iter != NULL)
+ (void)mailbox_attribute_iter_deinit(&exporter->attr_iter);
+ dsync_mailbox_export_body_search_deinit(exporter);
+ (void)mailbox_transaction_commit(&exporter->trans);
+ mailbox_header_lookup_unref(&exporter->wanted_headers);
+
+ i_stream_unref(&exporter->attr.value_stream);
+ hash_table_destroy(&exporter->export_guids);
+ hash_table_destroy(&exporter->changes);
+
+ i_assert((exporter->error != NULL) == (exporter->mail_error != 0));
+
+ *error_r = exporter->mail_error;
+ *errstr_r = t_strdup(exporter->error);
+ pool_unref(&exporter->pool);
+ return *errstr_r != NULL ? -1 : 0;
+}
+
+const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter)
+{
+ if (exporter->search_ctx == NULL)
+ return "";
+ return t_strdup_printf("%u/%u", exporter->search_pos,
+ exporter->search_count);
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-export.h b/src/doveadm/dsync/dsync-mailbox-export.h
new file mode 100644
index 0000000..67e3216
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-export.h
@@ -0,0 +1,37 @@
+#ifndef DSYNC_MAILBOX_EXPORT_H
+#define DSYNC_MAILBOX_EXPORT_H
+
+enum dsync_mailbox_exporter_flags {
+ DSYNC_MAILBOX_EXPORTER_FLAG_AUTO_EXPORT_MAILS = 0x01,
+ DSYNC_MAILBOX_EXPORTER_FLAG_MAILS_HAVE_GUIDS = 0x02,
+ DSYNC_MAILBOX_EXPORTER_FLAG_MINIMAL_DMAIL_FILL = 0x04,
+ DSYNC_MAILBOX_EXPORTER_FLAG_TIMESTAMPS = 0x08,
+ DSYNC_MAILBOX_EXPORTER_FLAG_NO_HDR_HASHES = 0x20,
+ DSYNC_MAILBOX_EXPORTER_FLAG_VSIZES = 0x40,
+};
+
+struct dsync_mailbox_exporter *
+dsync_mailbox_export_init(struct mailbox *box,
+ struct dsync_transaction_log_scan *log_scan,
+ uint32_t last_common_uid,
+ enum dsync_mailbox_exporter_flags flags,
+ unsigned int hdr_hash_version,
+ const char *const *hashed_headers);
+/* Returns 1 if attribute was returned, 0 if no more attributes, -1 on error */
+int dsync_mailbox_export_next_attr(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mailbox_attribute **attr_r);
+/* Returns 1 if change was returned, 0 if no more changes, -1 on error */
+int dsync_mailbox_export_next(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mail_change **change_r);
+
+void dsync_mailbox_export_want_mail(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mail_request *request);
+/* Returns 1 if mail was returned, 0 if no more mails, -1 on error */
+int dsync_mailbox_export_next_mail(struct dsync_mailbox_exporter *exporter,
+ const struct dsync_mail **mail_r);
+int dsync_mailbox_export_deinit(struct dsync_mailbox_exporter **exporter,
+ const char **errstr_r, enum mail_error *error_r);
+
+const char *dsync_mailbox_export_get_proctitle(struct dsync_mailbox_exporter *exporter);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-import.c b/src/doveadm/dsync/dsync-mailbox-import.c
new file mode 100644
index 0000000..8a8fbe4
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-import.c
@@ -0,0 +1,2997 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "hex-binary.h"
+#include "istream.h"
+#include "seq-range-array.h"
+#include "imap-util.h"
+#include "mail-storage-private.h"
+#include "mail-search-build.h"
+#include "dsync-transaction-log-scan.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-mailbox-import.h"
+
+struct importer_mail {
+ const char *guid;
+ uint32_t uid;
+};
+
+struct importer_new_mail {
+ /* linked list of mails for this GUID */
+ struct importer_new_mail *next;
+ /* if non-NULL, this mail exists in both local and remote. this link
+ points to the other side. */
+ struct importer_new_mail *link;
+
+ const char *guid;
+ struct dsync_mail_change *change;
+
+ /* the final UID for the message */
+ uint32_t final_uid;
+ /* the original local UID, or 0 if exists only remotely */
+ uint32_t local_uid;
+ /* the original remote UID, or 0 if exists only remotely */
+ uint32_t remote_uid;
+ /* UID for the mail in the virtual \All mailbox */
+ uint32_t virtual_all_uid;
+
+ bool uid_in_local:1;
+ bool uid_is_usable:1;
+ bool skip:1;
+ bool expunged:1;
+ bool copy_failed:1;
+ bool saved:1;
+};
+
+/* for quickly testing that two-way sync doesn't actually do any unexpected
+ modifications. */
+#define IMPORTER_DEBUG_CHANGE(importer) /*i_assert(!importer->master_brain)*/
+
+HASH_TABLE_DEFINE_TYPE(guid_new_mail, const char *, struct importer_new_mail *);
+HASH_TABLE_DEFINE_TYPE(uid_new_mail, void *, struct importer_new_mail *);
+
+struct dsync_mailbox_importer {
+ pool_t pool;
+ struct mailbox *box;
+ uint32_t last_common_uid;
+ uint64_t last_common_modseq, last_common_pvt_modseq;
+ uint32_t remote_uid_next;
+ uint32_t remote_first_recent_uid;
+ uint64_t remote_highest_modseq, remote_highest_pvt_modseq;
+ time_t sync_since_timestamp;
+ time_t sync_until_timestamp;
+ uoff_t sync_max_size;
+ enum mailbox_transaction_flags transaction_flags;
+ unsigned int hdr_hash_version;
+ unsigned int commit_msgs_interval;
+
+ const char *const *hashed_headers;
+
+ enum mail_flags sync_flag;
+ const char *sync_keyword;
+ bool sync_flag_dontwant;
+
+ struct mailbox_transaction_context *trans, *ext_trans;
+ struct mail_search_context *search_ctx;
+ struct mail *mail, *ext_mail;
+
+ struct mailbox *virtual_all_box;
+ struct mailbox_transaction_context *virtual_trans;
+ struct mail *virtual_mail;
+
+ struct mail *cur_mail;
+ const char *cur_guid;
+ const char *cur_hdr_hash;
+
+ /* UID => struct dsync_mail_change */
+ HASH_TABLE_TYPE(dsync_uid_mail_change) local_changes;
+ HASH_TABLE_TYPE(dsync_attr_change) local_attr_changes;
+
+ ARRAY_TYPE(seq_range) maybe_expunge_uids;
+ ARRAY(struct dsync_mail_change *) maybe_saves;
+
+ /* GUID => struct importer_new_mail */
+ HASH_TABLE_TYPE(guid_new_mail) import_guids;
+ /* UID => struct importer_new_mail */
+ HASH_TABLE_TYPE(uid_new_mail) import_uids;
+
+ ARRAY(struct importer_new_mail *) newmails;
+ ARRAY_TYPE(uint32_t) wanted_uids;
+ ARRAY_TYPE(uint32_t) saved_uids;
+ uint32_t highest_wanted_uid;
+
+ ARRAY(struct dsync_mail_request) mail_requests;
+ unsigned int mail_request_idx;
+
+ uint32_t prev_uid, next_local_seq, local_uid_next;
+ uint64_t local_initial_highestmodseq, local_initial_highestpvtmodseq;
+ unsigned int import_pos, import_count;
+ unsigned int first_unsaved_idx, saves_since_commit;
+
+ enum mail_error mail_error;
+
+ bool failed:1;
+ bool require_full_resync:1;
+ bool debug:1;
+ bool stateful_import:1;
+ bool last_common_uid_found:1;
+ bool cur_uid_has_change:1;
+ bool cur_mail_skip:1;
+ bool local_expunged_guids_set:1;
+ bool new_uids_assigned:1;
+ bool want_mail_requests:1;
+ bool master_brain:1;
+ bool revert_local_changes:1;
+ bool mails_have_guids:1;
+ bool mails_use_guid128:1;
+ bool delete_mailbox:1;
+ bool empty_hdr_workaround:1;
+};
+
+static const char *dsync_mail_change_type_names[] = {
+ "save", "expunge", "flag-change"
+};
+
+static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail,
+ struct importer_new_mail *all_newmails,
+ bool remote_mail);
+static int dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer,
+ bool final);
+
+static void ATTR_FORMAT(2, 3)
+imp_debug(struct dsync_mailbox_importer *importer, const char *fmt, ...)
+{
+ va_list args;
+
+ if (importer->debug) T_BEGIN {
+ va_start(args, fmt);
+ i_debug("brain %c: Import %s: %s",
+ importer->master_brain ? 'M' : 'S',
+ mailbox_get_vname(importer->box),
+ t_strdup_vprintf(fmt, args));
+ va_end(args);
+ } T_END;
+}
+
+static void
+dsync_import_unexpected_state(struct dsync_mailbox_importer *importer,
+ const char *error)
+{
+ if (!importer->stateful_import) {
+ i_error("Mailbox %s: %s", mailbox_get_vname(importer->box),
+ error);
+ } else {
+ i_warning("Mailbox %s doesn't match previous state: %s "
+ "(dsync must be run again without the state)",
+ mailbox_get_vname(importer->box), error);
+ }
+ importer->require_full_resync = TRUE;
+}
+
+static void
+dsync_mailbox_import_search_init(struct dsync_mailbox_importer *importer)
+{
+ struct mail_search_args *search_args;
+ struct mail_search_arg *sarg;
+
+ search_args = mail_search_build_init();
+ sarg = mail_search_build_add(search_args, SEARCH_UIDSET);
+ p_array_init(&sarg->value.seqset, search_args->pool, 128);
+ seq_range_array_add_range(&sarg->value.seqset,
+ importer->last_common_uid+1, (uint32_t)-1);
+
+ importer->search_ctx =
+ mailbox_search_init(importer->trans, search_args, NULL,
+ 0, NULL);
+ mail_search_args_unref(&search_args);
+
+ if (mailbox_search_next(importer->search_ctx, &importer->cur_mail))
+ importer->next_local_seq = importer->cur_mail->seq;
+ /* this flag causes cur_guid to be looked up later */
+ importer->cur_mail_skip = TRUE;
+}
+
+static void
+dsync_mailbox_import_transaction_begin(struct dsync_mailbox_importer *importer)
+{
+ const enum mailbox_transaction_flags ext_trans_flags =
+ importer->transaction_flags |
+ MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+ MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
+
+ importer->trans = mailbox_transaction_begin(importer->box,
+ importer->transaction_flags,
+ "dsync import");
+ importer->ext_trans = mailbox_transaction_begin(importer->box,
+ ext_trans_flags,
+ "dsync ext import");
+ importer->mail = mail_alloc(importer->trans, 0, NULL);
+ importer->ext_mail = mail_alloc(importer->ext_trans, 0, NULL);
+}
+
+struct dsync_mailbox_importer *
+dsync_mailbox_import_init(struct mailbox *box,
+ struct mailbox *virtual_all_box,
+ struct dsync_transaction_log_scan *log_scan,
+ uint32_t last_common_uid,
+ uint64_t last_common_modseq,
+ uint64_t last_common_pvt_modseq,
+ uint32_t remote_uid_next,
+ uint32_t remote_first_recent_uid,
+ uint64_t remote_highest_modseq,
+ uint64_t remote_highest_pvt_modseq,
+ time_t sync_since_timestamp,
+ time_t sync_until_timestamp,
+ uoff_t sync_max_size,
+ const char *sync_flag,
+ unsigned int commit_msgs_interval,
+ enum dsync_mailbox_import_flags flags,
+ unsigned int hdr_hash_version,
+ const char *const *hashed_headers)
+{
+ struct dsync_mailbox_importer *importer;
+ struct mailbox_status status;
+ pool_t pool;
+
+ pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox importer",
+ 10240);
+ importer = p_new(pool, struct dsync_mailbox_importer, 1);
+ importer->pool = pool;
+ importer->box = box;
+ importer->virtual_all_box = virtual_all_box;
+ importer->last_common_uid = last_common_uid;
+ importer->last_common_modseq = last_common_modseq;
+ importer->last_common_pvt_modseq = last_common_pvt_modseq;
+ importer->last_common_uid_found =
+ last_common_uid != 0 || last_common_modseq != 0;
+ importer->remote_uid_next = remote_uid_next;
+ importer->remote_first_recent_uid = remote_first_recent_uid;
+ importer->remote_highest_modseq = remote_highest_modseq;
+ importer->remote_highest_pvt_modseq = remote_highest_pvt_modseq;
+ importer->sync_since_timestamp = sync_since_timestamp;
+ importer->sync_until_timestamp = sync_until_timestamp;
+ importer->sync_max_size = sync_max_size;
+ importer->stateful_import = importer->last_common_uid_found;
+ importer->hashed_headers = hashed_headers;
+
+ if (sync_flag != NULL) {
+ if (sync_flag[0] == '-') {
+ importer->sync_flag_dontwant = TRUE;
+ sync_flag++;
+ }
+ if (sync_flag[0] == '\\')
+ importer->sync_flag = imap_parse_system_flag(sync_flag);
+ else
+ importer->sync_keyword = p_strdup(pool, sync_flag);
+ }
+ importer->commit_msgs_interval = commit_msgs_interval;
+ importer->transaction_flags = MAILBOX_TRANSACTION_FLAG_SYNC;
+ if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY) != 0)
+ importer->transaction_flags |= MAILBOX_TRANSACTION_FLAG_NO_NOTIFY;
+
+ hash_table_create(&importer->import_guids, pool, 0, str_hash, strcmp);
+ hash_table_create_direct(&importer->import_uids, pool, 0);
+ i_array_init(&importer->maybe_expunge_uids, 16);
+ i_array_init(&importer->maybe_saves, 128);
+ i_array_init(&importer->newmails, 128);
+ i_array_init(&importer->wanted_uids, 128);
+ i_array_init(&importer->saved_uids, 128);
+
+ dsync_mailbox_import_transaction_begin(importer);
+
+ if ((flags & DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS) != 0) {
+ i_array_init(&importer->mail_requests, 128);
+ importer->want_mail_requests = TRUE;
+ }
+ importer->master_brain =
+ (flags & DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN) != 0;
+ importer->revert_local_changes =
+ (flags & DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES) != 0;
+ importer->debug = (flags & DSYNC_MAILBOX_IMPORT_FLAG_DEBUG) != 0;
+ importer->mails_have_guids =
+ (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS) != 0;
+ importer->mails_use_guid128 =
+ (flags & DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128) != 0;
+ importer->hdr_hash_version = hdr_hash_version;
+ importer->empty_hdr_workaround =
+ (flags & DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND) != 0;
+
+ mailbox_get_open_status(importer->box, STATUS_UIDNEXT |
+ STATUS_HIGHESTMODSEQ | STATUS_HIGHESTPVTMODSEQ,
+ &status);
+ if (status.nonpermanent_modseqs)
+ status.highest_modseq = 0;
+ importer->local_uid_next = status.uidnext;
+ importer->local_initial_highestmodseq = status.highest_modseq;
+ importer->local_initial_highestpvtmodseq = status.highest_pvt_modseq;
+ dsync_mailbox_import_search_init(importer);
+
+ if (!importer->stateful_import)
+ ;
+ else if (importer->local_uid_next <= last_common_uid) {
+ dsync_import_unexpected_state(importer, t_strdup_printf(
+ "local UIDNEXT %u <= last common UID %u",
+ importer->local_uid_next, last_common_uid));
+ } else if (importer->local_initial_highestmodseq < last_common_modseq) {
+ dsync_import_unexpected_state(importer, t_strdup_printf(
+ "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
+ importer->local_initial_highestmodseq,
+ last_common_modseq));
+ } else if (importer->local_initial_highestpvtmodseq < last_common_pvt_modseq) {
+ dsync_import_unexpected_state(importer, t_strdup_printf(
+ "local HIGHESTMODSEQ %"PRIu64" < last common HIGHESTMODSEQ %"PRIu64,
+ importer->local_initial_highestpvtmodseq,
+ last_common_pvt_modseq));
+ }
+
+ importer->local_changes = dsync_transaction_log_scan_get_hash(log_scan);
+ importer->local_attr_changes = dsync_transaction_log_scan_get_attr_hash(log_scan);
+ return importer;
+}
+
+static int
+dsync_mailbox_import_lookup_attr(struct dsync_mailbox_importer *importer,
+ enum mail_attribute_type type, const char *key,
+ struct dsync_mailbox_attribute **attr_r)
+{
+ struct dsync_mailbox_attribute lookup_attr, *attr;
+ const struct dsync_mailbox_attribute *attr_change;
+ struct mail_attribute_value value;
+
+ *attr_r = NULL;
+
+ if (mailbox_attribute_get_stream(importer->box, type, key, &value) < 0) {
+ i_error("Mailbox %s: Failed to get attribute %s: %s",
+ mailbox_get_vname(importer->box), key,
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ return -1;
+ }
+
+ lookup_attr.type = type;
+ lookup_attr.key = key;
+
+ attr_change = hash_table_lookup(importer->local_attr_changes,
+ &lookup_attr);
+ if (attr_change == NULL &&
+ value.value == NULL && value.value_stream == NULL) {
+ /* we have no knowledge of this attribute */
+ return 0;
+ }
+ attr = t_new(struct dsync_mailbox_attribute, 1);
+ attr->type = type;
+ attr->key = key;
+ attr->value = value.value;
+ attr->value_stream = value.value_stream;
+ attr->last_change = value.last_change;
+ if (attr_change != NULL) {
+ attr->deleted = attr_change->deleted &&
+ !DSYNC_ATTR_HAS_VALUE(attr);
+ attr->modseq = attr_change->modseq;
+ }
+ *attr_r = attr;
+ return 0;
+}
+
+static int
+dsync_istreams_cmp(struct istream *input1, struct istream *input2, int *cmp_r)
+{
+ const unsigned char *data1, *data2;
+ size_t size1, size2, size;
+
+ *cmp_r = -1; /* quiet gcc */
+
+ for (;;) {
+ (void)i_stream_read_more(input1, &data1, &size1);
+ (void)i_stream_read_more(input2, &data2, &size2);
+
+ if (size1 == 0 || size2 == 0)
+ break;
+ size = I_MIN(size1, size2);
+ *cmp_r = memcmp(data1, data2, size);
+ if (*cmp_r != 0)
+ return 0;
+ i_stream_skip(input1, size);
+ i_stream_skip(input2, size);
+ }
+ if (input1->stream_errno != 0) {
+ i_error("read(%s) failed: %s", i_stream_get_name(input1),
+ i_stream_get_error(input1));
+ return -1;
+ }
+ if (input2->stream_errno != 0) {
+ i_error("read(%s) failed: %s", i_stream_get_name(input2),
+ i_stream_get_error(input2));
+ return -1;
+ }
+ if (size1 == 0 && size2 == 0)
+ *cmp_r = 0;
+ else
+ *cmp_r = size1 == 0 ? -1 : 1;
+ return 0;
+}
+
+static int
+dsync_attributes_cmp_values(const struct dsync_mailbox_attribute *attr1,
+ const struct dsync_mailbox_attribute *attr2,
+ int *cmp_r)
+{
+ struct istream *input1, *input2;
+ int ret;
+
+ i_assert(attr1->value_stream != NULL || attr1->value != NULL);
+ i_assert(attr2->value_stream != NULL || attr2->value != NULL);
+
+ if (attr1->value != NULL && attr2->value != NULL) {
+ *cmp_r = strcmp(attr1->value, attr2->value);
+ return 0;
+ }
+ /* at least one of them is a stream. make both of them streams. */
+ input1 = attr1->value_stream != NULL ? attr1->value_stream :
+ i_stream_create_from_data(attr1->value, strlen(attr1->value));
+ input2 = attr2->value_stream != NULL ? attr2->value_stream :
+ i_stream_create_from_data(attr2->value, strlen(attr2->value));
+ i_stream_seek(input1, 0);
+ i_stream_seek(input2, 0);
+ ret = dsync_istreams_cmp(input1, input2, cmp_r);
+ if (attr1->value_stream == NULL)
+ i_stream_unref(&input1);
+ if (attr2->value_stream == NULL)
+ i_stream_unref(&input2);
+ return ret;
+}
+
+static int
+dsync_attributes_cmp(const struct dsync_mailbox_attribute *attr,
+ const struct dsync_mailbox_attribute *local_attr,
+ int *cmp_r)
+{
+ if (DSYNC_ATTR_HAS_VALUE(attr) &&
+ !DSYNC_ATTR_HAS_VALUE(local_attr)) {
+ /* remote has a value and local doesn't -> use it */
+ *cmp_r = 1;
+ return 0;
+ } else if (!DSYNC_ATTR_HAS_VALUE(attr) &&
+ DSYNC_ATTR_HAS_VALUE(local_attr)) {
+ /* remote doesn't have a value, bt local does -> skip */
+ *cmp_r = -1;
+ return 0;
+ }
+
+ return dsync_attributes_cmp_values(attr, local_attr, cmp_r);
+}
+
+static int
+dsync_mailbox_import_attribute_real(struct dsync_mailbox_importer *importer,
+ const struct dsync_mailbox_attribute *attr,
+ const struct dsync_mailbox_attribute *local_attr,
+ const char **result_r)
+{
+ struct mail_attribute_value value;
+ int cmp;
+ bool ignore = FALSE;
+
+ i_assert(DSYNC_ATTR_HAS_VALUE(attr) || attr->deleted);
+
+ if (attr->deleted &&
+ (local_attr == NULL || !DSYNC_ATTR_HAS_VALUE(local_attr))) {
+ /* attribute doesn't exist on either side -> ignore */
+ *result_r = "Nonexistent in both sides";
+ return 0;
+ }
+ if (local_attr == NULL) {
+ /* we haven't seen this locally -> use whatever remote has */
+ *result_r = "Nonexistent locally";
+ } else if (local_attr->modseq <= importer->last_common_modseq &&
+ attr->modseq > importer->last_common_modseq &&
+ importer->last_common_modseq > 0) {
+ /* we're doing incremental syncing, and we can see that the
+ attribute was changed remotely, but not locally -> use it */
+ *result_r = "Changed remotely";
+ } else if (local_attr->modseq > importer->last_common_modseq &&
+ attr->modseq <= importer->last_common_modseq &&
+ importer->last_common_modseq > 0) {
+ /* we're doing incremental syncing, and we can see that the
+ attribute was changed locally, but not remotely -> ignore */
+ *result_r = "Changed locally";
+ ignore = TRUE;
+ } else if (attr->last_change > local_attr->last_change) {
+ /* remote has a newer timestamp -> use it */
+ *result_r = "Remote has newer timestamp";
+ } else if (attr->last_change < local_attr->last_change) {
+ /* remote has an older timestamp -> ignore */
+ *result_r = "Local has newer timestamp";
+ ignore = TRUE;
+ } else {
+ /* the timestamps are the same. now we're down to guessing
+ the right answer, unless the values are actually equal,
+ so check that first. next try to use modseqs, but if even
+ they are the same, fallback to just picking one based on the
+ value. */
+ if (dsync_attributes_cmp(attr, local_attr, &cmp) < 0) {
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+ return -1;
+ }
+ if (cmp == 0) {
+ /* identical scripts */
+ *result_r = "Unchanged value";
+ return 0;
+ }
+
+ if (attr->modseq > local_attr->modseq) {
+ /* remote has a higher modseq -> use it */
+ *result_r = "Remote has newer modseq";
+ } else if (attr->modseq < local_attr->modseq) {
+ /* remote has an older modseq -> ignore */
+ *result_r = "Local has newer modseq";
+ ignore = TRUE;
+ } else if (cmp < 0) {
+ ignore = TRUE;
+ *result_r = "Value changed, but unknown which is newer - picking local";
+ } else {
+ *result_r = "Value changed, but unknown which is newer - picking remote";
+ }
+ }
+ if (ignore)
+ return 0;
+
+ i_zero(&value);
+ value.value = attr->value;
+ value.value_stream = attr->value_stream;
+ value.last_change = attr->last_change;
+ if (mailbox_attribute_set(importer->trans, attr->type,
+ attr->key, &value) < 0) {
+ i_error("Mailbox %s: Failed to set attribute %s: %s",
+ mailbox_get_vname(importer->box), attr->key,
+ mailbox_get_last_internal_error(importer->box, NULL));
+ /* the attributes aren't vital, don't fail everything just
+ because of them. */
+ }
+ return 0;
+}
+
+int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
+ const struct dsync_mailbox_attribute *attr)
+{
+ struct dsync_mailbox_attribute *local_attr;
+ const char *result = "";
+ int ret;
+
+ if (dsync_mailbox_import_lookup_attr(importer, attr->type,
+ attr->key, &local_attr) < 0)
+ ret = -1;
+ else {
+ ret = dsync_mailbox_import_attribute_real(importer, attr,
+ local_attr, &result);
+ if (local_attr != NULL && local_attr->value_stream != NULL)
+ i_stream_unref(&local_attr->value_stream);
+ }
+ imp_debug(importer, "Import attribute %s: %s", attr->key,
+ ret < 0 ? "failed" : result);
+ return ret;
+}
+
+static void dsync_mail_error(struct dsync_mailbox_importer *importer,
+ struct mail *mail, const char *field)
+{
+ const char *errstr;
+ enum mail_error error;
+
+ errstr = mailbox_get_last_internal_error(importer->box, &error);
+ if (error == MAIL_ERROR_EXPUNGED)
+ return;
+
+ i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
+ mailbox_get_vname(mail->box), field, mail->uid, errstr);
+ importer->mail_error = error;
+ importer->failed = TRUE;
+}
+
+static bool
+dsync_mail_change_guid_equals(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change,
+ const char *guid, const char **cmp_guid_r)
+{
+ guid_128_t guid_128, change_guid_128;
+
+ if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ if (guid_128_from_string(change->guid, change_guid_128) < 0)
+ i_unreached();
+ } else if (importer->mails_use_guid128) {
+ mail_generate_guid_128_hash(change->guid, change_guid_128);
+ } else {
+ if (cmp_guid_r != NULL)
+ *cmp_guid_r = change->guid;
+ return strcmp(change->guid, guid) == 0;
+ }
+
+ mail_generate_guid_128_hash(guid, guid_128);
+ if (memcmp(change_guid_128, guid_128, GUID_128_SIZE) != 0) {
+ if (cmp_guid_r != NULL) {
+ *cmp_guid_r = t_strdup_printf("%s(guid128, orig=%s)",
+ binary_to_hex(change_guid_128, sizeof(change_guid_128)),
+ change->guid);
+ }
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static int
+importer_try_next_mail(struct dsync_mailbox_importer *importer,
+ uint32_t wanted_uid)
+{
+ struct mail_private *pmail;
+ const char *hdr_hash;
+
+ if (importer->cur_mail == NULL) {
+ /* end of search */
+ return -1;
+ }
+ while (importer->cur_mail->seq < importer->next_local_seq ||
+ importer->cur_mail->uid < wanted_uid) {
+ if (!importer->cur_uid_has_change &&
+ !importer->last_common_uid_found) {
+ /* this message exists locally, but remote didn't send
+ expunge-change for it. if the message's
+ uid <= last-common-uid, it should be deleted */
+ seq_range_array_add(&importer->maybe_expunge_uids,
+ importer->cur_mail->uid);
+ }
+
+ importer->cur_mail_skip = FALSE;
+ if (!mailbox_search_next(importer->search_ctx,
+ &importer->cur_mail)) {
+ importer->cur_mail = NULL;
+ importer->cur_guid = NULL;
+ importer->cur_hdr_hash = NULL;
+ return -1;
+ }
+ importer->cur_uid_has_change = FALSE;
+ }
+ importer->cur_uid_has_change = importer->cur_mail->uid == wanted_uid;
+ if (importer->mails_have_guids) {
+ if (mail_get_special(importer->cur_mail, MAIL_FETCH_GUID,
+ &importer->cur_guid) < 0) {
+ dsync_mail_error(importer, importer->cur_mail, "GUID");
+ return 0;
+ }
+ } else {
+ if (dsync_mail_get_hdr_hash(importer->cur_mail,
+ importer->hdr_hash_version,
+ importer->hashed_headers,
+ &hdr_hash) < 0) {
+ dsync_mail_error(importer, importer->cur_mail,
+ "header hash");
+ return 0;
+ }
+ pmail = (struct mail_private *)importer->cur_mail;
+ importer->cur_hdr_hash = p_strdup(pmail->pool, hdr_hash);
+ importer->cur_guid = "";
+ }
+ /* make sure next_local_seq gets updated in case we came here
+ because of min_uid */
+ importer->next_local_seq = importer->cur_mail->seq;
+ return 1;
+}
+
+static bool
+importer_next_mail(struct dsync_mailbox_importer *importer, uint32_t wanted_uid)
+{
+ int ret;
+
+ for (;;) {
+ T_BEGIN {
+ ret = importer_try_next_mail(importer, wanted_uid);
+ } T_END;
+ if (ret != 0 || importer->failed)
+ break;
+ importer->next_local_seq = importer->cur_mail->seq + 1;
+ }
+ return ret > 0;
+}
+
+static int
+importer_mail_cmp(const struct importer_mail *m1,
+ const struct importer_mail *m2)
+{
+ int ret;
+
+ if (m1->guid == NULL)
+ return 1;
+ if (m2->guid == NULL)
+ return -1;
+
+ ret = strcmp(m1->guid, m2->guid);
+ if (ret != 0)
+ return ret;
+
+ if (m1->uid < m2->uid)
+ return -1;
+ if (m1->uid > m2->uid)
+ return 1;
+ return 0;
+}
+
+static void newmail_link(struct dsync_mailbox_importer *importer,
+ struct importer_new_mail *newmail, uint32_t remote_uid)
+{
+ struct importer_new_mail *first_mail, **last, *mail, *link = NULL;
+
+ if (*newmail->guid != '\0') {
+ first_mail = hash_table_lookup(importer->import_guids,
+ newmail->guid);
+ if (first_mail == NULL) {
+ /* first mail for this GUID */
+ hash_table_insert(importer->import_guids,
+ newmail->guid, newmail);
+ return;
+ }
+ } else {
+ if (remote_uid == 0) {
+ /* mail exists locally. we don't want to request
+ it, and we'll assume it has no duplicate
+ instances. */
+ return;
+ }
+ first_mail = hash_table_lookup(importer->import_uids,
+ POINTER_CAST(remote_uid));
+ if (first_mail == NULL) {
+ /* first mail for this UID */
+ hash_table_insert(importer->import_uids,
+ POINTER_CAST(remote_uid), newmail);
+ return;
+ }
+ }
+ /* 1) add the newmail to the end of the linked list
+ 2) find our link
+
+ FIXME: this loop is slow if the same GUID has a ton of instances.
+ Could it be improved in some way? */
+ last = &first_mail->next;
+ for (mail = first_mail; mail != NULL; mail = mail->next) {
+ if (mail->final_uid == newmail->final_uid)
+ mail->uid_is_usable = TRUE;
+ if (link == NULL && mail->link == NULL &&
+ mail->uid_in_local != newmail->uid_in_local)
+ link = mail;
+ last = &mail->next;
+ }
+ *last = newmail;
+ if (link != NULL && newmail->link == NULL) {
+ link->link = newmail;
+ newmail->link = link;
+ }
+}
+
+static void
+dsync_mailbox_revert_existing_uid(struct dsync_mailbox_importer *importer,
+ uint32_t uid, const char *reason)
+{
+ i_assert(importer->revert_local_changes);
+
+ /* UID either already exists or UIDNEXT is too high. we can't set the
+ wanted UID, so we'll need to delete the whole mailbox and resync */
+ i_warning("Deleting mailbox '%s': UID=%u already exists locally for a different mail: %s",
+ mailbox_get_vname(importer->box), uid, reason);
+ importer->delete_mailbox = TRUE;
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+}
+
+static bool dsync_mailbox_try_save_cur(struct dsync_mailbox_importer *importer,
+ struct dsync_mail_change *save_change)
+{
+ struct importer_mail m1, m2;
+ struct importer_new_mail *newmail;
+ int diff;
+ bool remote_saved;
+
+ i_zero(&m1);
+ if (importer->cur_mail != NULL) {
+ m1.guid = importer->mails_have_guids ?
+ importer->cur_guid : importer->cur_hdr_hash;
+ m1.uid = importer->cur_mail->uid;
+ }
+ i_zero(&m2);
+ if (save_change != NULL) {
+ m2.guid = importer->mails_have_guids ?
+ save_change->guid : save_change->hdr_hash;
+ m2.uid = save_change->uid;
+ i_assert(save_change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+ }
+
+ if (importer->empty_hdr_workaround && !importer->mails_have_guids &&
+ importer->cur_mail != NULL && save_change != NULL &&
+ (dsync_mail_hdr_hash_is_empty(m1.guid) ||
+ dsync_mail_hdr_hash_is_empty(m2.guid))) {
+ /* one of the headers is empty. assume it's broken and that
+ the header matches what we have currently. */
+ diff = 0;
+ } else {
+ diff = importer_mail_cmp(&m1, &m2);
+ }
+ if (diff < 0) {
+ /* add a record for local mail */
+ i_assert(importer->cur_mail != NULL);
+ if (importer->revert_local_changes) {
+ if (save_change == NULL &&
+ importer->cur_mail->uid >= importer->remote_uid_next) {
+ dsync_mailbox_revert_existing_uid(importer, importer->cur_mail->uid,
+ t_strdup_printf("higher than remote's UIDs (remote UIDNEXT=%u)", importer->remote_uid_next));
+ return TRUE;
+ }
+ mail_expunge(importer->cur_mail);
+ importer->cur_mail_skip = TRUE;
+ importer->next_local_seq++;
+ return FALSE;
+ }
+ newmail = p_new(importer->pool, struct importer_new_mail, 1);
+ newmail->guid = p_strdup(importer->pool, importer->cur_guid);
+ newmail->final_uid = importer->cur_mail->uid;
+ newmail->local_uid = importer->cur_mail->uid;
+ newmail->uid_in_local = TRUE;
+ newmail->uid_is_usable =
+ newmail->final_uid >= importer->remote_uid_next;
+ remote_saved = FALSE;
+ } else if (diff > 0) {
+ i_assert(save_change != NULL);
+ newmail = p_new(importer->pool, struct importer_new_mail, 1);
+ newmail->guid = save_change->guid;
+ newmail->final_uid = save_change->uid;
+ newmail->remote_uid = save_change->uid;
+ newmail->uid_in_local = FALSE;
+ newmail->uid_is_usable =
+ newmail->final_uid >= importer->local_uid_next;
+ if (!newmail->uid_is_usable && importer->revert_local_changes) {
+ dsync_mailbox_revert_existing_uid(importer, newmail->final_uid,
+ t_strdup_printf("UID >= local UIDNEXT=%u", importer->local_uid_next));
+ return TRUE;
+ }
+ remote_saved = TRUE;
+ } else {
+ /* identical */
+ i_assert(importer->cur_mail != NULL);
+ i_assert(save_change != NULL);
+ newmail = p_new(importer->pool, struct importer_new_mail, 1);
+ newmail->guid = save_change->guid;
+ newmail->final_uid = importer->cur_mail->uid;
+ newmail->local_uid = importer->cur_mail->uid;
+ newmail->remote_uid = save_change->uid;
+ newmail->uid_in_local = TRUE;
+ newmail->uid_is_usable = TRUE;
+ newmail->link = newmail;
+ remote_saved = TRUE;
+ }
+
+ if (newmail->uid_in_local) {
+ importer->cur_mail_skip = TRUE;
+ importer->next_local_seq++;
+ }
+ /* NOTE: assumes save_change is allocated from importer pool */
+ newmail->change = save_change;
+
+ array_push_back(&importer->newmails, &newmail);
+ if (newmail->uid_in_local)
+ newmail_link(importer, newmail, 0);
+ else {
+ i_assert(save_change != NULL);
+ newmail_link(importer, newmail, save_change->uid);
+ }
+ return remote_saved;
+}
+
+static bool ATTR_NULL(2)
+dsync_mailbox_try_save(struct dsync_mailbox_importer *importer,
+ struct dsync_mail_change *save_change)
+{
+ if (importer->cur_mail_skip) {
+ if (!importer_next_mail(importer, 0) && save_change == NULL)
+ return FALSE;
+ }
+ return dsync_mailbox_try_save_cur(importer, save_change);
+}
+
+static void dsync_mailbox_save(struct dsync_mailbox_importer *importer,
+ struct dsync_mail_change *save_change)
+{
+ while (!dsync_mailbox_try_save(importer, save_change)) ;
+}
+
+static bool
+dsync_import_set_mail(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+ const char *guid, *cmp_guid;
+
+ if (!mail_set_uid(importer->mail, change->uid))
+ return FALSE;
+ if (change->guid == NULL) {
+ /* GUID is unknown */
+ return TRUE;
+ }
+ if (*change->guid == '\0') {
+ /* backend doesn't support GUIDs. if hdr_hash is set, we could
+ verify it, but since this message really is supposed to
+ match, it's probably too much trouble. */
+ return TRUE;
+ }
+
+ /* verify that GUID matches, just in case */
+ if (mail_get_special(importer->mail, MAIL_FETCH_GUID, &guid) < 0) {
+ dsync_mail_error(importer, importer->mail, "GUID");
+ return FALSE;
+ }
+ if (!dsync_mail_change_guid_equals(importer, change, guid, &cmp_guid)) {
+ dsync_import_unexpected_state(importer, t_strdup_printf(
+ "Unexpected GUID mismatch for UID=%u: %s != %s",
+ change->uid, guid, cmp_guid));
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static bool dsync_check_cur_guid(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+ const char *cmp_guid;
+
+ if (change->guid == NULL || change->guid[0] == '\0' ||
+ importer->cur_guid[0] == '\0')
+ return TRUE;
+ if (!dsync_mail_change_guid_equals(importer, change,
+ importer->cur_guid, &cmp_guid)) {
+ dsync_import_unexpected_state(importer, t_strdup_printf(
+ "Unexpected GUID mismatch (2) for UID=%u: %s != %s",
+ change->uid, importer->cur_guid, cmp_guid));
+ return FALSE;
+ }
+ return TRUE;
+}
+
+static void
+merge_flags(uint32_t local_final, uint32_t local_add, uint32_t local_remove,
+ uint32_t remote_final, uint32_t remote_add, uint32_t remote_remove,
+ uint32_t pvt_mask, bool prefer_remote, bool prefer_pvt_remote,
+ uint32_t *change_add_r, uint32_t *change_remove_r,
+ bool *remote_changed, bool *remote_pvt_changed)
+{
+ uint32_t combined_add, combined_remove, conflict_flags;
+ uint32_t local_wanted, remote_wanted, conflict_pvt_flags;
+
+ /* resolve conflicts */
+ conflict_flags = local_add & remote_remove;
+ if (conflict_flags != 0) {
+ conflict_pvt_flags = conflict_flags & pvt_mask;
+ conflict_flags &= ~pvt_mask;
+ if (prefer_remote)
+ local_add &= ~conflict_flags;
+ else
+ remote_remove &= ~conflict_flags;
+ if (prefer_pvt_remote)
+ local_add &= ~conflict_pvt_flags;
+ else
+ remote_remove &= ~conflict_pvt_flags;
+ }
+ conflict_flags = local_remove & remote_add;
+ if (conflict_flags != 0) {
+ conflict_pvt_flags = conflict_flags & pvt_mask;
+ conflict_flags &= ~pvt_mask;
+ if (prefer_remote)
+ local_remove &= ~conflict_flags;
+ else
+ remote_add &= ~conflict_flags;
+ if (prefer_pvt_remote)
+ local_remove &= ~conflict_pvt_flags;
+ else
+ remote_add &= ~conflict_pvt_flags;
+ }
+
+ combined_add = local_add|remote_add;
+ combined_remove = local_remove|remote_remove;
+ i_assert((combined_add & combined_remove) == 0);
+
+ /* don't change flags that are currently identical in both sides */
+ conflict_flags = local_final ^ remote_final;
+ combined_add &= conflict_flags;
+ combined_remove &= conflict_flags;
+
+ /* see if there are conflicting final flags */
+ local_wanted = (local_final|combined_add) & ~combined_remove;
+ remote_wanted = (remote_final|combined_add) & ~combined_remove;
+
+ conflict_flags = local_wanted ^ remote_wanted;
+ if (conflict_flags != 0) {
+ if (prefer_remote && prefer_pvt_remote)
+ local_wanted = remote_wanted;
+ else if (prefer_remote && !prefer_pvt_remote) {
+ local_wanted = (local_wanted & pvt_mask) |
+ (remote_wanted & ~pvt_mask);
+ } else if (!prefer_remote && prefer_pvt_remote) {
+ local_wanted = (local_wanted & ~pvt_mask) |
+ (remote_wanted & pvt_mask);
+ }
+ }
+
+ *change_add_r = local_wanted & ~local_final;
+ *change_remove_r = local_final & ~local_wanted;
+ if ((local_wanted & ~pvt_mask) != (remote_final & ~pvt_mask))
+ *remote_changed = TRUE;
+ if ((local_wanted & pvt_mask) != (remote_final & pvt_mask))
+ *remote_pvt_changed = TRUE;
+}
+
+static bool
+keyword_find(ARRAY_TYPE(const_string) *keywords, const char *name,
+ unsigned int *idx_r)
+{
+ const char *const *names;
+ unsigned int i, count;
+
+ names = array_get(keywords, &count);
+ for (i = 0; i < count; i++) {
+ if (strcmp(names[i], name) == 0) {
+ *idx_r = i;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static void keywords_append(ARRAY_TYPE(const_string) *dest,
+ const ARRAY_TYPE(const_string) *keywords,
+ uint32_t bits, unsigned int start_idx)
+{
+ const char *name;
+ unsigned int i;
+
+ for (i = 0; i < 32; i++) {
+ if ((bits & (1U << i)) == 0)
+ continue;
+
+ name = array_idx_elem(keywords, start_idx+i);
+ array_push_back(dest, &name);
+ }
+}
+
+static void
+merge_keywords(struct mail *mail, const ARRAY_TYPE(const_string) *local_changes,
+ const ARRAY_TYPE(const_string) *remote_changes,
+ bool prefer_remote,
+ bool *remote_changed, bool *remote_pvt_changed)
+{
+ /* local_changes and remote_changes are assumed to have no
+ duplicates names */
+ uint32_t *local_add, *local_remove, *local_final;
+ uint32_t *remote_add, *remote_remove, *remote_final;
+ uint32_t *change_add, *change_remove;
+ ARRAY_TYPE(const_string) all_keywords, add_keywords, remove_keywords;
+ const char *const *changes, *name, *const *local_keywords;
+ struct mail_keywords *kw;
+ unsigned int i, count, name_idx, array_size;
+
+ local_keywords = mail_get_keywords(mail);
+
+ /* we'll assign a common index for each keyword name and place
+ the changes to separate bit arrays. */
+ if (array_is_created(remote_changes))
+ changes = array_get(remote_changes, &count);
+ else {
+ changes = NULL;
+ count = 0;
+ }
+
+ array_size = str_array_length(local_keywords) + count;
+ if (array_is_created(local_changes))
+ array_size += array_count(local_changes);
+ if (array_size == 0) {
+ /* this message has no keywords */
+ return;
+ }
+ t_array_init(&all_keywords, array_size);
+ t_array_init(&add_keywords, array_size);
+ t_array_init(&remove_keywords, array_size);
+
+ /* @UNSAFE: create large enough arrays to fit all keyword indexes. */
+ array_size = (array_size+31)/32;
+ local_add = t_new(uint32_t, array_size);
+ local_remove = t_new(uint32_t, array_size);
+ local_final = t_new(uint32_t, array_size);
+ remote_add = t_new(uint32_t, array_size);
+ remote_remove = t_new(uint32_t, array_size);
+ remote_final = t_new(uint32_t, array_size);
+ change_add = t_new(uint32_t, array_size);
+ change_remove = t_new(uint32_t, array_size);
+
+ /* get remote changes */
+ for (i = 0; i < count; i++) {
+ name = changes[i]+1;
+ name_idx = array_count(&all_keywords);
+ array_push_back(&all_keywords, &name);
+
+ switch (changes[i][0]) {
+ case KEYWORD_CHANGE_ADD:
+ remote_add[name_idx/32] |= 1U << (name_idx%32);
+ break;
+ case KEYWORD_CHANGE_REMOVE:
+ remote_remove[name_idx/32] |= 1U << (name_idx%32);
+ break;
+ case KEYWORD_CHANGE_FINAL:
+ remote_final[name_idx/32] |= 1U << (name_idx%32);
+ break;
+ case KEYWORD_CHANGE_ADD_AND_FINAL:
+ remote_add[name_idx/32] |= 1U << (name_idx%32);
+ remote_final[name_idx/32] |= 1U << (name_idx%32);
+ break;
+ }
+ }
+
+ /* get local changes. use existing indexes for names when they exist. */
+ if (array_is_created(local_changes))
+ changes = array_get(local_changes, &count);
+ else {
+ changes = NULL;
+ count = 0;
+ }
+ for (i = 0; i < count; i++) {
+ name = changes[i]+1;
+ if (!keyword_find(&all_keywords, name, &name_idx)) {
+ name_idx = array_count(&all_keywords);
+ array_push_back(&all_keywords, &name);
+ }
+
+ switch (changes[i][0]) {
+ case KEYWORD_CHANGE_ADD:
+ case KEYWORD_CHANGE_ADD_AND_FINAL:
+ local_add[name_idx/32] |= 1U << (name_idx%32);
+ break;
+ case KEYWORD_CHANGE_REMOVE:
+ local_remove[name_idx/32] |= 1U << (name_idx%32);
+ break;
+ case KEYWORD_CHANGE_FINAL:
+ break;
+ }
+ }
+ for (i = 0; local_keywords[i] != NULL; i++) {
+ name = local_keywords[i];
+ if (!keyword_find(&all_keywords, name, &name_idx)) {
+ name_idx = array_count(&all_keywords);
+ array_push_back(&all_keywords, &name);
+ }
+ local_final[name_idx/32] |= 1U << (name_idx%32);
+ }
+ i_assert(array_count(&all_keywords) <= array_size*32);
+ array_size = (array_count(&all_keywords)+31) / 32;
+
+ /* merge keywords */
+ for (i = 0; i < array_size; i++) {
+ merge_flags(local_final[i], local_add[i], local_remove[i],
+ remote_final[i], remote_add[i], remote_remove[i],
+ 0, prefer_remote, prefer_remote,
+ &change_add[i], &change_remove[i],
+ remote_changed, remote_pvt_changed);
+ if (change_add[i] != 0) {
+ keywords_append(&add_keywords, &all_keywords,
+ change_add[i], i*32);
+ }
+ if (change_remove[i] != 0) {
+ keywords_append(&remove_keywords, &all_keywords,
+ change_remove[i], i*32);
+ }
+ }
+
+ /* apply changes */
+ if (array_count(&add_keywords) > 0) {
+ array_append_zero(&add_keywords);
+ kw = mailbox_keywords_create_valid(mail->box,
+ array_front(&add_keywords));
+ mail_update_keywords(mail, MODIFY_ADD, kw);
+ mailbox_keywords_unref(&kw);
+ }
+ if (array_count(&remove_keywords) > 0) {
+ array_append_zero(&remove_keywords);
+ kw = mailbox_keywords_create_valid(mail->box,
+ array_front(&remove_keywords));
+ mail_update_keywords(mail, MODIFY_REMOVE, kw);
+ mailbox_keywords_unref(&kw);
+ }
+}
+
+static void
+dsync_mailbox_import_replace_flags(struct mail *mail,
+ const struct dsync_mail_change *change)
+{
+ ARRAY_TYPE(const_string) keywords;
+ struct mail_keywords *kw;
+ const char *const *changes, *name;
+ unsigned int i, count;
+
+ if (array_is_created(&change->keyword_changes))
+ changes = array_get(&change->keyword_changes, &count);
+ else {
+ changes = NULL;
+ count = 0;
+ }
+ t_array_init(&keywords, count+1);
+ for (i = 0; i < count; i++) {
+ switch (changes[i][0]) {
+ case KEYWORD_CHANGE_ADD:
+ case KEYWORD_CHANGE_FINAL:
+ case KEYWORD_CHANGE_ADD_AND_FINAL:
+ name = changes[i]+1;
+ array_push_back(&keywords, &name);
+ break;
+ case KEYWORD_CHANGE_REMOVE:
+ break;
+ }
+ }
+ array_append_zero(&keywords);
+
+ kw = mailbox_keywords_create_valid(mail->box, array_front(&keywords));
+ mail_update_keywords(mail, MODIFY_REPLACE, kw);
+ mailbox_keywords_unref(&kw);
+
+ mail_update_flags(mail, MODIFY_REPLACE,
+ change->add_flags | change->final_flags);
+ if (mail_get_modseq(mail) < change->modseq)
+ mail_update_modseq(mail, change->modseq);
+ if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
+ mail_update_pvt_modseq(mail, change->pvt_modseq);
+}
+
+static void
+dsync_mailbox_import_flag_change(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+ const struct dsync_mail_change *local_change;
+ enum mail_flags local_add, local_remove;
+ uint32_t change_add, change_remove;
+ uint64_t new_modseq;
+ ARRAY_TYPE(const_string) local_keyword_changes = ARRAY_INIT;
+ struct mail *mail;
+ bool prefer_remote, prefer_pvt_remote;
+ bool remote_changed = FALSE, remote_pvt_changed = FALSE;
+
+ i_assert((change->add_flags & change->remove_flags) == 0);
+
+ if (importer->cur_mail != NULL &&
+ importer->cur_mail->uid == change->uid) {
+ if (!dsync_check_cur_guid(importer, change))
+ return;
+ mail = importer->cur_mail;
+ } else {
+ if (!dsync_import_set_mail(importer, change))
+ return;
+ mail = importer->mail;
+ }
+
+ if (importer->revert_local_changes) {
+ /* dsync backup: just make the local look like remote. */
+ dsync_mailbox_import_replace_flags(mail, change);
+ return;
+ }
+
+ local_change = hash_table_lookup(importer->local_changes,
+ POINTER_CAST(change->uid));
+ if (local_change == NULL) {
+ local_add = local_remove = 0;
+ } else {
+ local_add = local_change->add_flags;
+ local_remove = local_change->remove_flags;
+ local_keyword_changes = local_change->keyword_changes;
+ }
+
+ if (mail_get_modseq(mail) < change->modseq)
+ prefer_remote = TRUE;
+ else if (mail_get_modseq(mail) > change->modseq)
+ prefer_remote = FALSE;
+ else {
+ /* identical modseq, we'll just have to pick one.
+ Note that both brains need to pick the same one, otherwise
+ they become unsynced. */
+ prefer_remote = !importer->master_brain;
+ }
+ if (mail_get_pvt_modseq(mail) < change->pvt_modseq)
+ prefer_pvt_remote = TRUE;
+ else if (mail_get_pvt_modseq(mail) > change->pvt_modseq)
+ prefer_pvt_remote = FALSE;
+ else
+ prefer_pvt_remote = !importer->master_brain;
+
+ /* merge flags */
+ merge_flags(mail_get_flags(mail), local_add, local_remove,
+ change->final_flags, change->add_flags, change->remove_flags,
+ mailbox_get_private_flags_mask(mail->box),
+ prefer_remote, prefer_pvt_remote,
+ &change_add, &change_remove,
+ &remote_changed, &remote_pvt_changed);
+
+ if (change_add != 0)
+ mail_update_flags(mail, MODIFY_ADD, change_add);
+ if (change_remove != 0)
+ mail_update_flags(mail, MODIFY_REMOVE, change_remove);
+
+ /* merge keywords */
+ merge_keywords(mail, &local_keyword_changes, &change->keyword_changes,
+ prefer_remote, &remote_changed, &remote_pvt_changed);
+
+ /* update modseqs. try to anticipate when we have to increase modseq
+ to get it closer to what remote has (although we can't guess it
+ exactly correctly) */
+ new_modseq = change->modseq;
+ if (remote_changed && new_modseq <= importer->remote_highest_modseq)
+ new_modseq = importer->remote_highest_modseq+1;
+ if (mail_get_modseq(mail) < new_modseq)
+ mail_update_modseq(mail, new_modseq);
+
+ new_modseq = change->pvt_modseq;
+ if (remote_pvt_changed && new_modseq <= importer->remote_highest_pvt_modseq)
+ new_modseq = importer->remote_highest_pvt_modseq+1;
+ if (mail_get_pvt_modseq(mail) < new_modseq)
+ mail_update_pvt_modseq(mail, new_modseq);
+}
+
+static bool
+dsync_mail_change_have_keyword(const struct dsync_mail_change *change,
+ const char *keyword)
+{
+ const char *str;
+
+ if (!array_is_created(&change->keyword_changes))
+ return FALSE;
+
+ array_foreach_elem(&change->keyword_changes, str) {
+ switch (str[0]) {
+ case KEYWORD_CHANGE_FINAL:
+ case KEYWORD_CHANGE_ADD_AND_FINAL:
+ if (strcasecmp(str+1, keyword) == 0)
+ return TRUE;
+ break;
+ default:
+ break;
+ }
+ }
+ return FALSE;
+}
+
+static bool
+dsync_mailbox_import_want_change(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change,
+ const char **result_r)
+{
+ if (importer->sync_since_timestamp > 0) {
+ i_assert(change->received_timestamp > 0);
+ if (change->received_timestamp < importer->sync_since_timestamp) {
+ /* mail has too old timestamp - skip it */
+ *result_r = "Ignoring missing local mail with too old timestamp";
+ return FALSE;
+ }
+ }
+ if (importer->sync_until_timestamp > 0) {
+ i_assert(change->received_timestamp > 0);
+ if (change->received_timestamp > importer->sync_until_timestamp) {
+ /* mail has too new timestamp - skip it */
+ *result_r = "Ignoring missing local mail with too new timestamp";
+ return FALSE;
+ }
+ }
+ if (importer->sync_max_size > 0) {
+ i_assert(change->virtual_size != UOFF_T_MAX);
+ if (change->virtual_size > importer->sync_max_size) {
+ /* mail is too large - skip it */
+ *result_r = "Ignoring missing local mail with too large size";
+ return FALSE;
+ }
+ }
+ if (importer->sync_flag != 0) {
+ bool have_flag = (change->final_flags & importer->sync_flag) != 0;
+
+ if (have_flag && importer->sync_flag_dontwant) {
+ *result_r = "Ignoring missing local mail that doesn't have wanted flags";
+ return FALSE;
+ }
+ if (!have_flag && !importer->sync_flag_dontwant) {
+ *result_r = "Ignoring missing local mail that has unwanted flags";
+ return FALSE;
+ }
+ }
+ if (importer->sync_keyword != NULL) {
+ bool have_kw = dsync_mail_change_have_keyword(change, importer->sync_keyword);
+
+ if (have_kw && importer->sync_flag_dontwant) {
+ *result_r = "Ignoring missing local mail that doesn't have wanted keywords";
+ return FALSE;
+ }
+ if (!have_kw && !importer->sync_flag_dontwant) {
+ *result_r = "Ignoring missing local mail that has unwanted keywords";
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
+static void
+dsync_mailbox_import_save(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+ struct dsync_mail_change *save;
+ const char *result;
+
+ i_assert(change->guid != NULL);
+
+ if (change->uid == importer->last_common_uid) {
+ /* we've already verified that the GUID matches.
+ apply flag changes if there are any. */
+ i_assert(!importer->last_common_uid_found);
+ dsync_mailbox_import_flag_change(importer, change);
+ return;
+ }
+ if (!dsync_mailbox_import_want_change(importer, change, &result))
+ return;
+
+ save = p_new(importer->pool, struct dsync_mail_change, 1);
+ dsync_mail_change_dup(importer->pool, change, save);
+
+ if (importer->last_common_uid_found) {
+ /* this is a new mail. its UID may or may not conflict with
+ an existing local mail, we'll figure it out later. */
+ i_assert(change->uid > importer->last_common_uid);
+ dsync_mailbox_save(importer, save);
+ } else {
+ /* the local mail is expunged. we'll decide later if we want
+ to save this mail locally or expunge it form remote. */
+ i_assert(change->uid > importer->last_common_uid);
+ i_assert(importer->cur_mail == NULL ||
+ change->uid < importer->cur_mail->uid);
+ array_push_back(&importer->maybe_saves, &save);
+ }
+}
+
+static void
+dsync_mailbox_import_expunge(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+
+ if (importer->last_common_uid_found) {
+ /* expunge the message, unless its GUID unexpectedly doesn't
+ match */
+ i_assert(change->uid <= importer->last_common_uid);
+ if (dsync_import_set_mail(importer, change))
+ mail_expunge(importer->mail);
+ } else if (importer->cur_mail == NULL ||
+ change->uid < importer->cur_mail->uid) {
+ /* already expunged locally, we can ignore this.
+ uid=last_common_uid if we managed to verify from
+ transaction log that the GUIDs match */
+ i_assert(change->uid >= importer->last_common_uid);
+ } else if (change->uid == importer->last_common_uid) {
+ /* already verified that the GUID matches */
+ i_assert(importer->cur_mail->uid == change->uid);
+ if (dsync_check_cur_guid(importer, change))
+ mail_expunge(importer->cur_mail);
+ } else {
+ /* we don't know yet if we should expunge this
+ message or not. queue it until we do. */
+ i_assert(change->uid > importer->last_common_uid);
+ seq_range_array_add(&importer->maybe_expunge_uids, change->uid);
+ }
+}
+
+static void
+dsync_mailbox_rewind_search(struct dsync_mailbox_importer *importer)
+{
+ /* If there are local mails after last_common_uid which we skipped
+ while trying to match the next message, we need to now go back */
+ if (importer->cur_mail != NULL &&
+ importer->cur_mail->uid <= importer->last_common_uid+1)
+ return;
+
+ importer->cur_mail = NULL;
+ importer->cur_guid = NULL;
+ importer->cur_hdr_hash = NULL;
+ importer->next_local_seq = 0;
+
+ (void)mailbox_search_deinit(&importer->search_ctx);
+ dsync_mailbox_import_search_init(importer);
+}
+
+static void
+dsync_mailbox_common_uid_found(struct dsync_mailbox_importer *importer)
+{
+ struct dsync_mail_change *const *saves;
+ struct seq_range_iter iter;
+ unsigned int n, i, count;
+ uint32_t uid;
+
+ if (importer->debug) T_BEGIN {
+ string_t *expunges = t_str_new(64);
+
+ imap_write_seq_range(expunges, &importer->maybe_expunge_uids);
+ imp_debug(importer, "Last common UID=%u. Delayed expunges=%s",
+ importer->last_common_uid, str_c(expunges));
+ } T_END;
+
+ importer->last_common_uid_found = TRUE;
+ dsync_mailbox_rewind_search(importer);
+
+ /* expunge the messages whose expunge-decision we delayed previously */
+ seq_range_array_iter_init(&iter, &importer->maybe_expunge_uids); n = 0;
+ while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+ if (uid > importer->last_common_uid) {
+ /* we expunge messages only up to last_common_uid,
+ ignore the rest */
+ break;
+ }
+
+ if (mail_set_uid(importer->mail, uid))
+ mail_expunge(importer->mail);
+ }
+
+ /* handle pending saves */
+ saves = array_get(&importer->maybe_saves, &count);
+ for (i = 0; i < count; i++) {
+ if (saves[i]->uid > importer->last_common_uid) {
+ imp_debug(importer, "Delayed save UID=%u: Save",
+ saves[i]->uid);
+ dsync_mailbox_save(importer, saves[i]);
+ } else {
+ imp_debug(importer, "Delayed save UID=%u: Ignore",
+ saves[i]->uid);
+ }
+ }
+}
+
+static int
+dsync_mailbox_import_match_msg(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change,
+ const char **result_r)
+{
+ const char *hdr_hash, *cmp_guid;
+
+ if (*change->guid != '\0' && *importer->cur_guid != '\0') {
+ /* we have GUIDs, verify them */
+ if (dsync_mail_change_guid_equals(importer, change,
+ importer->cur_guid, &cmp_guid)) {
+ *result_r = "GUIDs match";
+ return 1;
+ } else {
+ *result_r = t_strdup_printf("GUIDs don't match (%s vs %s)",
+ importer->cur_guid, cmp_guid);
+ return 0;
+ }
+ }
+
+ /* verify hdr_hash if it exists */
+ if (change->hdr_hash == NULL) {
+ i_assert(*importer->cur_guid == '\0');
+ if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ /* the message was already expunged, so we don't know
+ its header. return "unknown". */
+ *result_r = "Unknown match for expunge";
+ return -1;
+ }
+ i_error("Mailbox %s: GUIDs not supported, "
+ "sync with header hashes instead",
+ mailbox_get_vname(importer->box));
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+ *result_r = "Error, invalid parameters";
+ return -1;
+ }
+
+ if (dsync_mail_get_hdr_hash(importer->cur_mail,
+ importer->hdr_hash_version,
+ importer->hashed_headers, &hdr_hash) < 0) {
+ dsync_mail_error(importer, importer->cur_mail, "hdr-stream");
+ *result_r = "Error fetching header stream";
+ return -1;
+ }
+ if (importer->empty_hdr_workaround &&
+ (dsync_mail_hdr_hash_is_empty(change->hdr_hash) ||
+ dsync_mail_hdr_hash_is_empty(hdr_hash))) {
+ *result_r = "Empty headers found with workaround enabled - assuming a match";
+ return 1;
+ } else if (strcmp(change->hdr_hash, hdr_hash) == 0) {
+ *result_r = "Headers hashes match";
+ return 1;
+ } else {
+ *result_r = t_strdup_printf("Headers hashes don't match (%s vs %s)",
+ change->hdr_hash, hdr_hash);
+ return 0;
+ }
+}
+
+static bool
+dsync_mailbox_find_common_expunged_uid(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change,
+ const char **result_r)
+{
+ const struct dsync_mail_change *local_change;
+
+ if (*change->guid == '\0') {
+ /* remote doesn't support GUIDs, can't verify expunge */
+ *result_r = "GUIDs not supported, can't verify expunge";
+ return FALSE;
+ }
+
+ /* local message is expunged. see if we can find its GUID from
+ transaction log and check if the GUIDs match. The GUID in
+ log is a 128bit GUID, so we may need to convert the remote's
+ GUID string to 128bit GUID first. */
+ local_change = hash_table_lookup(importer->local_changes,
+ POINTER_CAST(change->uid));
+ if (local_change == NULL || local_change->guid == NULL) {
+ *result_r = "Expunged local mail's GUID not found";
+ return FALSE;
+ }
+
+ i_assert(local_change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+ if (dsync_mail_change_guid_equals(importer, local_change,
+ change->guid, NULL)) {
+ importer->last_common_uid = change->uid;
+ *result_r = "Expunged local mail's GUID matches remote";
+ } else if (change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ dsync_mailbox_common_uid_found(importer);
+ *result_r = "Expunged local mail's GUID doesn't match remote GUID";
+ } else {
+ /* GUID mismatch for two expunged mails. dsync can't update
+ GUIDs for already expunged messages, so we can't immediately
+ determine that the rest of the messages are a mismatch. so
+ for now we'll just skip over this pair. */
+ *result_r = "Expunged mails' GUIDs don't match - delaying decision";
+ /* NOTE: the return value here doesn't matter, because the only
+ caller that checks for it never reaches this code path */
+ }
+ return TRUE;
+}
+
+static void
+dsync_mailbox_revert_missing(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+ i_assert(importer->revert_local_changes);
+
+ /* mail exists on remote, but not locally. we'll need to
+ insert this mail back, which means deleting the whole
+ mailbox and resyncing. */
+ i_warning("Deleting mailbox '%s': UID=%u GUID=%s is missing locally",
+ mailbox_get_vname(importer->box),
+ change->uid, change->guid);
+ importer->delete_mailbox = TRUE;
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+}
+
+static void
+dsync_mailbox_find_common_uid(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change,
+ const char **result_r)
+{
+ int ret;
+
+ i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+ ((change->received_timestamp > 0 ||
+ (importer->sync_since_timestamp == 0 &&
+ importer->sync_until_timestamp == 0)) &&
+ (change->virtual_size != UOFF_T_MAX || importer->sync_max_size == 0)));
+
+ /* try to find the matching local mail */
+ if (!importer_next_mail(importer, change->uid)) {
+ /* no more local mails. we can still try to match
+ expunged mails though. */
+ if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ /* mail doesn't exist remotely either, don't bother
+ looking it up locally. */
+ *result_r = "Expunged mail not found locally";
+ return;
+ }
+ i_assert(change->guid != NULL);
+ if (!dsync_mailbox_import_want_change(importer, change, result_r))
+ ;
+ else if (importer->local_uid_next <= change->uid) {
+ dsync_mailbox_common_uid_found(importer);
+ *result_r = "Mail's UID is above local UIDNEXT";
+ } else if (importer->revert_local_changes) {
+ dsync_mailbox_revert_missing(importer, change);
+ *result_r = "Reverting local change by deleting mailbox";
+ } else if (!dsync_mailbox_find_common_expunged_uid(importer, change, result_r)) {
+ /* it's unknown if this mail existed locally and was
+ expunged. since we don't want to lose any mails,
+ assume that we need to preserve the mail. use the
+ last message with a matching GUID as the last common
+ UID. */
+ dsync_mailbox_common_uid_found(importer);
+ }
+ *result_r = t_strdup_printf("%s - No more local mails found", *result_r);
+ return;
+ }
+
+ if (change->guid == NULL) {
+ /* we can't know if this UID matches */
+ i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE);
+ *result_r = "Expunged mail has no GUID, can't verify it";
+ return;
+ }
+ if (importer->cur_mail->uid == change->uid) {
+ /* we have a matching local UID. check GUID to see if it's
+ really the same mail or not */
+ if ((ret = dsync_mailbox_import_match_msg(importer, change, result_r)) < 0) {
+ /* unknown */
+ return;
+ }
+ if (ret > 0) {
+ importer->last_common_uid = change->uid;
+ } else if (!importer->revert_local_changes) {
+ /* mismatch - found the first non-common UID */
+ dsync_mailbox_common_uid_found(importer);
+ } else {
+ /* mismatch and we want to revert local changes -
+ need to delete the mailbox. */
+ dsync_mailbox_revert_existing_uid(importer, change->uid, *result_r);
+ }
+ return;
+ }
+ /* mail exists remotely, but doesn't exist locally. */
+ if (!dsync_mailbox_import_want_change(importer, change, result_r))
+ return;
+ if (importer->revert_local_changes &&
+ change->type != DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ dsync_mailbox_revert_missing(importer, change);
+ *result_r = "Reverting local change by deleting mailbox";
+ } else {
+ (void)dsync_mailbox_find_common_expunged_uid(importer, change, result_r);
+ }
+ *result_r = t_strdup_printf("%s (next local mail UID=%u)",
+ *result_r, importer->cur_mail == NULL ? 0 : importer->cur_mail->uid);
+}
+
+int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change)
+{
+ const char *result;
+
+ i_assert(!importer->new_uids_assigned);
+ i_assert(importer->prev_uid < change->uid);
+
+ importer->prev_uid = change->uid;
+
+ if (importer->failed)
+ return -1;
+ if (importer->require_full_resync)
+ return 0;
+
+ if (!importer->last_common_uid_found) {
+ result = NULL;
+ dsync_mailbox_find_common_uid(importer, change, &result);
+ i_assert(result != NULL);
+ } else {
+ result = "New mail";
+ }
+
+ imp_debug(importer, "Import change type=%s GUID=%s UID=%u hdr_hash=%s result=%s",
+ dsync_mail_change_type_names[change->type],
+ change->guid != NULL ? change->guid : "<unknown>", change->uid,
+ change->hdr_hash != NULL ? change->hdr_hash : "", result);
+
+ if (importer->failed)
+ return -1;
+ if (importer->require_full_resync)
+ return 0;
+
+ if (importer->last_common_uid_found) {
+ /* a) uid <= last_common_uid for flag changes and expunges.
+ this happens only when last_common_uid was originally given
+ as parameter to importer.
+
+ when we're finding the last_common_uid ourself,
+ uid>last_common_uid always in here, because
+ last_common_uid_found=TRUE only after we find the first
+ mismatch.
+
+ b) uid > last_common_uid for i) new messages, ii) expunges
+ that were sent "just in case" */
+ if (change->uid <= importer->last_common_uid) {
+ i_assert(change->type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
+ } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ /* ignore */
+ return 0;
+ } else {
+ i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_SAVE);
+ }
+ } else {
+ /* a) uid < last_common_uid can never happen */
+ i_assert(change->uid >= importer->last_common_uid);
+ /* b) uid = last_common_uid if we've verified that the
+ messages' GUIDs match so far.
+
+ c) uid > last_common_uid: i) TYPE_EXPUNGE change has
+ GUID=NULL, so we couldn't verify yet if it matches our
+ local message, ii) local message is expunged and we couldn't
+ find its GUID */
+ if (change->uid > importer->last_common_uid) {
+ i_assert(change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE ||
+ importer->cur_mail == NULL ||
+ change->uid < importer->cur_mail->uid);
+ }
+ }
+
+ switch (change->type) {
+ case DSYNC_MAIL_CHANGE_TYPE_SAVE:
+ dsync_mailbox_import_save(importer, change);
+ break;
+ case DSYNC_MAIL_CHANGE_TYPE_EXPUNGE:
+ dsync_mailbox_import_expunge(importer, change);
+ break;
+ case DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE:
+ i_assert(importer->last_common_uid_found);
+ dsync_mailbox_import_flag_change(importer, change);
+ break;
+ }
+ return importer->failed ? -1 : 0;
+}
+
+static int
+importer_new_mail_final_uid_cmp(struct importer_new_mail *const *newmail1,
+ struct importer_new_mail *const *newmail2)
+{
+ if ((*newmail1)->final_uid < (*newmail2)->final_uid)
+ return -1;
+ if ((*newmail1)->final_uid > (*newmail2)->final_uid)
+ return 1;
+ return 0;
+}
+
+static void
+dsync_mailbox_import_assign_new_uids(struct dsync_mailbox_importer *importer)
+{
+ struct importer_new_mail *newmail;
+ uint32_t common_uid_next, new_uid;
+
+ common_uid_next = I_MAX(importer->local_uid_next,
+ importer->remote_uid_next);
+ array_foreach_elem(&importer->newmails, newmail) {
+ if (newmail->skip) {
+ /* already assigned */
+ i_assert(newmail->final_uid != 0);
+ continue;
+ }
+
+ /* figure out what UID to use for the mail */
+ if (newmail->uid_is_usable) {
+ /* keep the UID */
+ new_uid = newmail->final_uid;
+ } else if (newmail->link != NULL &&
+ newmail->link->uid_is_usable) {
+ /* we can use the linked message's UID and expunge
+ this mail */
+ new_uid = newmail->link->final_uid;
+ } else {
+ i_assert(!importer->revert_local_changes);
+ new_uid = common_uid_next++;
+ imp_debug(importer, "UID %u isn't usable, assigning new UID %u",
+ newmail->final_uid, new_uid);
+ }
+
+ newmail->final_uid = new_uid;
+ if (newmail->link != NULL && newmail->link != newmail) {
+ /* skip processing the linked mail */
+ newmail->link->skip = TRUE;
+ }
+ }
+ importer->last_common_uid = common_uid_next-1;
+ importer->new_uids_assigned = TRUE;
+ /* Sort the newmails by their final_uid. This is used for tracking
+ whether an intermediate commit is allowed. */
+ array_sort(&importer->newmails, importer_new_mail_final_uid_cmp);
+}
+
+static int
+dsync_mailbox_import_local_uid(struct dsync_mailbox_importer *importer,
+ struct mail *mail, uint32_t uid, const char *guid,
+ struct dsync_mail *dmail_r)
+{
+ const char *error_field, *errstr;
+ enum mail_error error;
+
+ if (!mail_set_uid(mail, uid))
+ return 0;
+
+ /* NOTE: Errors are logged, but they don't cause the entire import
+ to fail. */
+ if (dsync_mail_fill(mail, TRUE, dmail_r, &error_field) < 0) {
+ errstr = mailbox_get_last_internal_error(mail->box, &error);
+ if (error == MAIL_ERROR_EXPUNGED)
+ return 0;
+
+ i_error("Mailbox %s: Can't lookup %s for UID=%u: %s",
+ mailbox_get_vname(importer->box),
+ error_field, uid, errstr);
+ return -1;
+ }
+ if (*guid != '\0' && strcmp(guid, dmail_r->guid) != 0) {
+ dsync_import_unexpected_state(importer, t_strdup_printf(
+ "Unexpected GUID mismatch (3) for UID=%u: %s != %s",
+ uid, dmail_r->guid, guid));
+ return -1;
+ }
+ return 1;
+}
+
+static void
+dsync_mailbox_import_saved_uid(struct dsync_mailbox_importer *importer,
+ uint32_t uid)
+{
+ i_assert(importer->search_ctx == NULL);
+
+ if (importer->highest_wanted_uid < uid)
+ importer->highest_wanted_uid = uid;
+ array_push_back(&importer->wanted_uids, &uid);
+}
+
+static void
+dsync_mailbox_import_update_first_saved(struct dsync_mailbox_importer *importer)
+{
+ struct importer_new_mail *const *newmails;
+ unsigned int count;
+
+ newmails = array_get(&importer->newmails, &count);
+ while (importer->first_unsaved_idx < count) {
+ if (!newmails[importer->first_unsaved_idx]->saved)
+ break;
+ importer->first_unsaved_idx++;
+ }
+}
+
+static void
+dsync_mailbox_import_saved_newmail(struct dsync_mailbox_importer *importer,
+ struct importer_new_mail *newmail)
+{
+ dsync_mailbox_import_saved_uid(importer, newmail->final_uid);
+ newmail->saved = TRUE;
+
+ dsync_mailbox_import_update_first_saved(importer);
+ importer->saves_since_commit++;
+ /* we can commit only if all the upcoming mails will have UIDs that
+ are larger than we're committing.
+
+ Note that if any existing UIDs have been changed, the new UID is
+ usually higher than anything that is being saved so we can't do
+ an intermediate commit. It's too much extra work to try to handle
+ that situation. So here this never happens, because then
+ array_count(wanted_uids) is always higher than first_unsaved_idx. */
+ if (importer->saves_since_commit >= importer->commit_msgs_interval &&
+ importer->first_unsaved_idx == array_count(&importer->wanted_uids)) {
+ if (dsync_mailbox_import_commit(importer, FALSE) < 0)
+ importer->failed = TRUE;
+ importer->saves_since_commit = 0;
+ }
+}
+
+static bool
+dsync_msg_change_uid(struct dsync_mailbox_importer *importer,
+ uint32_t old_uid, uint32_t new_uid)
+{
+ struct mail_save_context *save_ctx;
+
+ IMPORTER_DEBUG_CHANGE(importer);
+
+ if (!mail_set_uid(importer->mail, old_uid))
+ return FALSE;
+
+ save_ctx = mailbox_save_alloc(importer->ext_trans);
+ mailbox_save_copy_flags(save_ctx, importer->mail);
+ mailbox_save_set_uid(save_ctx, new_uid);
+ if (mailbox_move(&save_ctx, importer->mail) < 0)
+ return FALSE;
+ dsync_mailbox_import_saved_uid(importer, new_uid);
+ return TRUE;
+}
+
+static bool
+dsync_mailbox_import_change_uid(struct dsync_mailbox_importer *importer,
+ ARRAY_TYPE(seq_range) *unwanted_uids,
+ uint32_t wanted_uid)
+{
+ const struct seq_range *range;
+ unsigned int count, n;
+ struct seq_range_iter iter;
+ uint32_t uid;
+
+ /* optimize by first trying to use the latest UID */
+ range = array_get(unwanted_uids, &count);
+ if (count == 0)
+ return FALSE;
+ if (dsync_msg_change_uid(importer, range[count-1].seq2, wanted_uid)) {
+ seq_range_array_remove(unwanted_uids, range[count-1].seq2);
+ return TRUE;
+ }
+ if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
+ seq_range_array_remove(unwanted_uids, range[count-1].seq2);
+
+ /* now try to use any of them by iterating through them. (would be
+ easier&faster to just iterate backwards, but probably too much
+ trouble to add such API) */
+ n = 0; seq_range_array_iter_init(&iter, unwanted_uids);
+ while (seq_range_array_iter_nth(&iter, n++, &uid)) {
+ if (dsync_msg_change_uid(importer, uid, wanted_uid)) {
+ seq_range_array_remove(unwanted_uids, uid);
+ return TRUE;
+ }
+ if (mailbox_get_last_mail_error(importer->box) == MAIL_ERROR_EXPUNGED)
+ seq_range_array_remove(unwanted_uids, uid);
+ }
+ return FALSE;
+}
+
+static bool
+dsync_mailbox_import_try_local(struct dsync_mailbox_importer *importer,
+ struct importer_new_mail *all_newmails,
+ ARRAY_TYPE(seq_range) *local_uids,
+ ARRAY_TYPE(seq_range) *wanted_uids)
+{
+ ARRAY_TYPE(seq_range) assigned_uids, unwanted_uids;
+ struct seq_range_iter local_iter, wanted_iter;
+ unsigned int local_n, wanted_n;
+ uint32_t local_uid, wanted_uid;
+ struct importer_new_mail *mail;
+ struct dsync_mail dmail;
+
+ if (array_count(local_uids) == 0)
+ return FALSE;
+
+ local_n = wanted_n = 0;
+ seq_range_array_iter_init(&local_iter, local_uids);
+ seq_range_array_iter_init(&wanted_iter, wanted_uids);
+
+ /* wanted_uids contains UIDs that need to exist at the end. those that
+ don't already exist in local_uids have a higher UID than any
+ existing local UID */
+ t_array_init(&assigned_uids, array_count(wanted_uids));
+ t_array_init(&unwanted_uids, 8);
+ while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+ if (seq_range_array_iter_nth(&wanted_iter, wanted_n,
+ &wanted_uid)) {
+ if (local_uid == wanted_uid) {
+ /* we have exactly the UID we want. keep it. */
+ seq_range_array_add(&assigned_uids, wanted_uid);
+ wanted_n++;
+ continue;
+ }
+ i_assert(local_uid < wanted_uid);
+ }
+ /* we no longer want this local UID. */
+ seq_range_array_add(&unwanted_uids, local_uid);
+ }
+
+ /* reuse as many existing messages as possible by changing their UIDs */
+ while (seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
+ if (!dsync_mailbox_import_change_uid(importer, &unwanted_uids,
+ wanted_uid))
+ break;
+ seq_range_array_add(&assigned_uids, wanted_uid);
+ wanted_n++;
+ }
+
+ /* expunge all unwanted messages */
+ local_n = 0; seq_range_array_iter_init(&local_iter, &unwanted_uids);
+ while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+ IMPORTER_DEBUG_CHANGE(importer);
+ if (mail_set_uid(importer->mail, local_uid))
+ mail_expunge(importer->mail);
+ }
+
+ /* mark mails whose UIDs we got to be skipped over later */
+ for (mail = all_newmails; mail != NULL; mail = mail->next) {
+ if (!mail->skip &&
+ seq_range_exists(&assigned_uids, mail->final_uid))
+ mail->skip = TRUE;
+ }
+
+ if (!seq_range_array_iter_nth(&wanted_iter, wanted_n, &wanted_uid)) {
+ /* we've assigned all wanted UIDs */
+ return TRUE;
+ }
+
+ /* try to find one existing message that we can use to copy to the
+ other instances */
+ local_n = 0; seq_range_array_iter_init(&local_iter, local_uids);
+ while (seq_range_array_iter_nth(&local_iter, local_n++, &local_uid)) {
+ if (dsync_mailbox_import_local_uid(importer, importer->mail,
+ local_uid, all_newmails->guid,
+ &dmail) > 0) {
+ if (dsync_mailbox_save_newmails(importer, &dmail,
+ all_newmails, FALSE))
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static bool
+dsync_mailbox_import_try_virtual_all(struct dsync_mailbox_importer *importer,
+ struct importer_new_mail *all_newmails)
+{
+ struct dsync_mail dmail;
+
+ if (all_newmails->virtual_all_uid == 0)
+ return FALSE;
+
+ if (dsync_mailbox_import_local_uid(importer, importer->virtual_mail,
+ all_newmails->virtual_all_uid,
+ all_newmails->guid, &dmail) > 0) {
+ if (dsync_mailbox_save_newmails(importer, &dmail,
+ all_newmails, FALSE))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static bool
+dsync_mailbox_import_handle_mail(struct dsync_mailbox_importer *importer,
+ struct importer_new_mail *all_newmails)
+{
+ ARRAY_TYPE(seq_range) local_uids, wanted_uids;
+ struct dsync_mail_request *request;
+ struct importer_new_mail *mail;
+ const char *request_guid = NULL;
+ uint32_t request_uid = 0;
+
+ i_assert(all_newmails != NULL);
+
+ /* get the list of the current local UIDs and the wanted UIDs.
+ find the first remote instance that we can request in case there are
+ no local instances */
+ t_array_init(&local_uids, 8);
+ t_array_init(&wanted_uids, 8);
+ for (mail = all_newmails; mail != NULL; mail = mail->next) {
+ if (mail->uid_in_local)
+ seq_range_array_add(&local_uids, mail->local_uid);
+ else if (request_guid == NULL) {
+ if (*mail->guid != '\0')
+ request_guid = mail->guid;
+ request_uid = mail->remote_uid;
+ i_assert(request_uid != 0);
+ }
+ if (!mail->skip)
+ seq_range_array_add(&wanted_uids, mail->final_uid);
+ }
+ i_assert(array_count(&wanted_uids) > 0);
+
+ if (!dsync_mailbox_import_try_local(importer, all_newmails,
+ &local_uids, &wanted_uids) &&
+ !dsync_mailbox_import_try_virtual_all(importer, all_newmails)) {
+ /* no local instance. request from remote */
+ IMPORTER_DEBUG_CHANGE(importer);
+ if (importer->want_mail_requests) {
+ request = array_append_space(&importer->mail_requests);
+ request->guid = request_guid;
+ request->uid = request_uid;
+ }
+ return FALSE;
+ }
+ /* successfully handled all the mails locally */
+ importer->import_pos++;
+ return TRUE;
+}
+
+static void
+dsync_mailbox_import_find_virtual_uids(struct dsync_mailbox_importer *importer)
+{
+ struct mail_search_context *search_ctx;
+ struct mail_search_args *search_args;
+ struct importer_new_mail *newmail;
+ struct mail *mail;
+ const char *guid;
+
+ if (mailbox_sync(importer->virtual_all_box, 0) < 0) {
+ i_error("Couldn't sync \\All mailbox '%s': %s",
+ mailbox_get_vname(importer->virtual_all_box),
+ mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
+ return;
+ }
+
+ search_args = mail_search_build_init();
+ mail_search_build_add_all(search_args);
+
+ importer->virtual_trans =
+ mailbox_transaction_begin(importer->virtual_all_box,
+ importer->transaction_flags,
+ __func__);
+ search_ctx = mailbox_search_init(importer->virtual_trans, search_args,
+ NULL, MAIL_FETCH_GUID, NULL);
+ mail_search_args_unref(&search_args);
+
+ while (mailbox_search_next(search_ctx, &mail)) {
+ if (mail_get_special(mail, MAIL_FETCH_GUID, &guid) < 0) {
+ /* ignore errors */
+ continue;
+ }
+ newmail = hash_table_lookup(importer->import_guids, guid);
+ if (newmail != NULL && newmail->virtual_all_uid == 0)
+ newmail->virtual_all_uid = mail->uid;
+ }
+ if (mailbox_search_deinit(&search_ctx) < 0) {
+ i_error("Couldn't search \\All mailbox '%s': %s",
+ mailbox_get_vname(importer->virtual_all_box),
+ mailbox_get_last_internal_error(importer->virtual_all_box, NULL));
+ }
+
+ importer->virtual_mail = mail_alloc(importer->virtual_trans, 0, NULL);
+}
+
+static void
+dsync_mailbox_import_handle_local_mails(struct dsync_mailbox_importer *importer)
+{
+ struct hash_iterate_context *iter;
+ const char *key;
+ void *key2;
+ struct importer_new_mail *mail;
+
+ if (importer->virtual_all_box != NULL &&
+ hash_table_count(importer->import_guids) > 0) {
+ /* find UIDs in \All mailbox for all wanted GUIDs. */
+ dsync_mailbox_import_find_virtual_uids(importer);
+ }
+
+ iter = hash_table_iterate_init(importer->import_guids);
+ while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
+ T_BEGIN {
+ if (dsync_mailbox_import_handle_mail(importer, mail))
+ hash_table_remove(importer->import_guids, key);
+ } T_END;
+ }
+ hash_table_iterate_deinit(&iter);
+
+ iter = hash_table_iterate_init(importer->import_uids);
+ while (hash_table_iterate(iter, importer->import_uids, &key2, &mail)) {
+ T_BEGIN {
+ if (dsync_mailbox_import_handle_mail(importer, mail))
+ hash_table_remove(importer->import_uids, key2);
+ } T_END;
+ }
+ hash_table_iterate_deinit(&iter);
+ if (!importer->mails_have_guids) {
+ array_foreach_elem(&importer->newmails, mail) {
+ if (mail->uid_in_local)
+ (void)dsync_mailbox_import_handle_mail(importer, mail);
+ }
+ }
+}
+
+int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer)
+{
+ i_assert(!importer->new_uids_assigned);
+
+ if (!importer->last_common_uid_found) {
+ /* handle pending expunges and flag updates */
+ dsync_mailbox_common_uid_found(importer);
+ }
+ /* skip common local mails */
+ (void)importer_next_mail(importer, importer->last_common_uid+1);
+ /* if there are any local mails left, add them to newmails list */
+ while (importer->cur_mail != NULL && !importer->failed)
+ (void)dsync_mailbox_try_save(importer, NULL);
+
+ if (importer->search_ctx != NULL) {
+ if (mailbox_search_deinit(&importer->search_ctx) < 0) {
+ i_error("Mailbox %s: Search failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ }
+ }
+ importer->import_count = hash_table_count(importer->import_guids) +
+ hash_table_count(importer->import_uids);
+
+ dsync_mailbox_import_assign_new_uids(importer);
+ /* save mails from local sources where possible,
+ request the rest from remote */
+ if (!importer->failed)
+ dsync_mailbox_import_handle_local_mails(importer);
+ return importer->failed ? -1 : 0;
+}
+
+const struct dsync_mail_request *
+dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer)
+{
+ const struct dsync_mail_request *requests;
+ unsigned int count;
+
+ requests = array_get(&importer->mail_requests, &count);
+ if (importer->mail_request_idx == count)
+ return NULL;
+ return &requests[importer->mail_request_idx++];
+}
+
+static const char *const *
+dsync_mailbox_get_final_keywords(const struct dsync_mail_change *change)
+{
+ ARRAY_TYPE(const_string) keywords;
+ const char *const *changes;
+ unsigned int i, count;
+
+ if (!array_is_created(&change->keyword_changes))
+ return NULL;
+
+ changes = array_get(&change->keyword_changes, &count);
+ t_array_init(&keywords, count);
+ for (i = 0; i < count; i++) {
+ if (changes[i][0] == KEYWORD_CHANGE_ADD ||
+ changes[i][0] == KEYWORD_CHANGE_FINAL ||
+ changes[i][0] == KEYWORD_CHANGE_ADD_AND_FINAL) {
+ const char *name = changes[i]+1;
+
+ array_push_back(&keywords, &name);
+ }
+ }
+ if (array_count(&keywords) == 0)
+ return NULL;
+
+ array_append_zero(&keywords);
+ return array_front(&keywords);
+}
+
+static void
+dsync_mailbox_save_set_metadata(struct dsync_mailbox_importer *importer,
+ struct mail_save_context *save_ctx,
+ const struct dsync_mail_change *change)
+{
+ const char *const *keyword_names;
+ struct mail_keywords *keywords;
+
+ keyword_names = dsync_mailbox_get_final_keywords(change);
+ keywords = keyword_names == NULL ? NULL :
+ mailbox_keywords_create_valid(importer->box,
+ keyword_names);
+ mailbox_save_set_flags(save_ctx, change->final_flags, keywords);
+ if (keywords != NULL)
+ mailbox_keywords_unref(&keywords);
+
+ if (change->modseq > 1) {
+ (void)mailbox_enable(importer->box, MAILBOX_FEATURE_CONDSTORE);
+ mailbox_save_set_min_modseq(save_ctx, change->modseq);
+ }
+ /* FIXME: if there already are private flags, they get lost because
+ saving can't handle updating private index. they get added on the
+ next sync though. if this is fixed here, set min_pvt_modseq also. */
+}
+
+static int
+dsync_msg_try_copy(struct dsync_mailbox_importer *importer,
+ struct mail_save_context **save_ctx_p,
+ struct importer_new_mail **all_newmails_forcopy)
+{
+ struct importer_new_mail *inst;
+
+ for (inst = *all_newmails_forcopy; inst != NULL; inst = inst->next) {
+ if (inst->uid_in_local && !inst->copy_failed &&
+ mail_set_uid(importer->mail, inst->local_uid)) {
+ if (mailbox_copy(save_ctx_p, importer->mail) < 0) {
+ enum mail_error error;
+ const char *errstr;
+
+ errstr = mailbox_get_last_internal_error(importer->box, &error);
+ if (error != MAIL_ERROR_EXPUNGED) {
+ i_warning("Failed to copy mail from UID=%u: "
+ "%s - falling back to other means",
+ inst->local_uid, errstr);
+ }
+ inst->copy_failed = TRUE;
+ return -1;
+ }
+ *all_newmails_forcopy = inst;
+ return 1;
+ }
+ }
+ *all_newmails_forcopy = NULL;
+ return 0;
+}
+
+static void
+dsync_mailbox_save_set_nonminimal(struct mail_save_context *save_ctx,
+ const struct dsync_mail *mail)
+{
+ if (mail->pop3_uidl != NULL && *mail->pop3_uidl != '\0')
+ mailbox_save_set_pop3_uidl(save_ctx, mail->pop3_uidl);
+ if (mail->pop3_order > 0)
+ mailbox_save_set_pop3_order(save_ctx, mail->pop3_order);
+ mailbox_save_set_received_date(save_ctx, mail->received_date, 0);
+}
+
+static struct mail_save_context *
+dsync_mailbox_save_init(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail,
+ struct importer_new_mail *newmail)
+{
+ struct mail_save_context *save_ctx;
+
+ save_ctx = mailbox_save_alloc(importer->ext_trans);
+ mailbox_save_set_uid(save_ctx, newmail->final_uid);
+ if (*mail->guid != '\0')
+ mailbox_save_set_guid(save_ctx, mail->guid);
+ if (mail->saved_date != 0)
+ mailbox_save_set_save_date(save_ctx, mail->saved_date);
+ dsync_mailbox_save_set_metadata(importer, save_ctx, newmail->change);
+
+ if (!mail->minimal_fields)
+ dsync_mailbox_save_set_nonminimal(save_ctx, mail);
+ return save_ctx;
+}
+
+static bool
+dsync_mailbox_save_body(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail,
+ struct importer_new_mail *newmail,
+ struct importer_new_mail **all_newmails_forcopy,
+ bool remote_mail)
+{
+ struct mail_save_context *save_ctx;
+ struct istream *input;
+ ssize_t ret;
+ bool save_failed = FALSE;
+
+ /* try to save the mail by copying an existing mail */
+ save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
+ if ((ret = dsync_msg_try_copy(importer, &save_ctx, all_newmails_forcopy)) < 0) {
+ if (save_ctx == NULL)
+ save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
+ }
+ if (ret <= 0 && mail->input_mail != NULL) {
+ /* copy using the source mail */
+ i_assert(mail->input_mail->uid == mail->input_mail_uid);
+ if (mailbox_copy(&save_ctx, mail->input_mail) == 0)
+ ret = 1;
+ else {
+ enum mail_error error;
+ const char *errstr;
+
+ errstr = mailbox_get_last_internal_error(importer->box, &error);
+ if (error != MAIL_ERROR_EXPUNGED) {
+ i_warning("Failed to copy source UID=%u mail: "
+ "%s - falling back to regular saving",
+ mail->input_mail->uid, errstr);
+ }
+ ret = -1;
+ save_ctx = dsync_mailbox_save_init(importer, mail, newmail);
+ }
+
+ }
+ if (ret > 0) {
+ i_assert(save_ctx == NULL);
+ dsync_mailbox_import_saved_newmail(importer, newmail);
+ return TRUE;
+ }
+ /* fallback to saving from remote stream */
+ if (!remote_mail) {
+ /* the mail isn't remote yet. we were just trying to copy a
+ local mail to avoid downloading the remote mail. */
+ mailbox_save_cancel(&save_ctx);
+ return FALSE;
+ }
+ if (mail->minimal_fields) {
+ struct dsync_mail mail2;
+ const char *error_field;
+
+ i_assert(mail->input_mail != NULL);
+
+ if (dsync_mail_fill_nonminimal(mail->input_mail, &mail2,
+ &error_field) < 0) {
+ i_error("Mailbox %s: Failed to read mail %s uid=%u: %s",
+ mailbox_get_vname(importer->box),
+ error_field, mail->uid,
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ mailbox_save_cancel(&save_ctx);
+ return TRUE;
+ }
+ dsync_mailbox_save_set_nonminimal(save_ctx, &mail2);
+ input = mail2.input;
+ } else {
+ input = mail->input;
+ }
+
+ if (input == NULL) {
+ /* it was just expunged in remote, skip it */
+ mailbox_save_cancel(&save_ctx);
+ return TRUE;
+ }
+
+ i_stream_seek(input, 0);
+ if (mailbox_save_begin(&save_ctx, input) < 0) {
+ i_error("Mailbox %s: Saving failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ return TRUE;
+ }
+ while ((ret = i_stream_read(input)) > 0 || ret == -2) {
+ if (mailbox_save_continue(save_ctx) < 0) {
+ save_failed = TRUE;
+ ret = -1;
+ break;
+ }
+ }
+ i_assert(ret == -1);
+
+ if (input->stream_errno != 0) {
+ i_error("Mailbox %s: read(msg input) failed: %s",
+ mailbox_get_vname(importer->box),
+ i_stream_get_error(input));
+ mailbox_save_cancel(&save_ctx);
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+ } else if (save_failed) {
+ i_error("Mailbox %s: Saving failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ mailbox_save_cancel(&save_ctx);
+ importer->failed = TRUE;
+ } else {
+ i_assert(input->eof);
+ if (mailbox_save_finish(&save_ctx) < 0) {
+ i_error("Mailbox %s: Saving failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ } else {
+ dsync_mailbox_import_saved_newmail(importer, newmail);
+ }
+ }
+ return TRUE;
+}
+
+static bool dsync_mailbox_save_newmails(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail,
+ struct importer_new_mail *all_newmails,
+ bool remote_mail)
+{
+ struct importer_new_mail *newmail, *all_newmails_forcopy;
+ bool ret = TRUE;
+
+ /* if all_newmails list is large, avoid scanning through the
+ uninteresting ones for each newmail */
+ all_newmails_forcopy = all_newmails;
+
+ /* save all instances of the message */
+ for (newmail = all_newmails; newmail != NULL && ret; newmail = newmail->next) {
+ if (!newmail->skip) T_BEGIN {
+ if (!dsync_mailbox_save_body(importer, mail, newmail,
+ &all_newmails_forcopy,
+ remote_mail))
+ ret = FALSE;
+ } T_END;
+ }
+ return ret;
+}
+
+int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail)
+{
+ struct importer_new_mail *all_newmails;
+
+ i_assert(mail->input == NULL || mail->input->seekable);
+ i_assert(importer->new_uids_assigned);
+
+ if (importer->failed)
+ return -1;
+ if (importer->require_full_resync)
+ return 0;
+
+ imp_debug(importer, "Import mail body for GUID=%s UID=%u",
+ mail->guid, mail->uid);
+
+ all_newmails = *mail->guid != '\0' ?
+ hash_table_lookup(importer->import_guids, mail->guid) :
+ hash_table_lookup(importer->import_uids, POINTER_CAST(mail->uid));
+ if (all_newmails == NULL) {
+ if (importer->want_mail_requests) {
+ i_error("Mailbox %s: Remote sent unwanted message body for "
+ "GUID=%s UID=%u",
+ mailbox_get_vname(importer->box),
+ mail->guid, mail->uid);
+ } else {
+ imp_debug(importer, "Skip unwanted mail body for "
+ "GUID=%s UID=%u", mail->guid, mail->uid);
+ }
+ return 0;
+ }
+ if (*mail->guid != '\0')
+ hash_table_remove(importer->import_guids, mail->guid);
+ else {
+ hash_table_remove(importer->import_uids,
+ POINTER_CAST(mail->uid));
+ }
+ importer->import_pos++;
+ if (!dsync_mailbox_save_newmails(importer, mail, all_newmails, TRUE))
+ i_unreached();
+ return importer->failed ? -1 : 0;
+}
+
+static int
+reassign_uids_in_seq_range(struct dsync_mailbox_importer *importer,
+ const ARRAY_TYPE(seq_range) *unwanted_uids)
+{
+ struct mailbox *box = importer->box;
+ const enum mailbox_transaction_flags trans_flags =
+ importer->transaction_flags |
+ MAILBOX_TRANSACTION_FLAG_EXTERNAL |
+ MAILBOX_TRANSACTION_FLAG_ASSIGN_UIDS;
+ struct mailbox_transaction_context *trans;
+ struct mail_search_args *search_args;
+ struct mail_search_arg *arg;
+ struct mail_search_context *search_ctx;
+ struct mail_save_context *save_ctx;
+ struct mail *mail;
+ unsigned int renumber_count = 0;
+ int ret = 1;
+
+ if (array_count(unwanted_uids) == 0)
+ return 1;
+
+ if (importer->debug) T_BEGIN {
+ string_t *str = t_str_new(256);
+ imap_write_seq_range(str, unwanted_uids);
+ imp_debug(importer, "Reassign UIDs: %s", str_c(str));
+ } T_END;
+
+ search_args = mail_search_build_init();
+ arg = mail_search_build_add(search_args, SEARCH_UIDSET);
+ p_array_init(&arg->value.seqset, search_args->pool,
+ array_count(unwanted_uids));
+ array_append_array(&arg->value.seqset, unwanted_uids);
+
+ trans = mailbox_transaction_begin(box, trans_flags, __func__);
+ search_ctx = mailbox_search_init(trans, search_args, NULL, 0, NULL);
+ mail_search_args_unref(&search_args);
+
+ while (mailbox_search_next(search_ctx, &mail)) {
+ save_ctx = mailbox_save_alloc(trans);
+ mailbox_save_copy_flags(save_ctx, mail);
+ if (mailbox_move(&save_ctx, mail) < 0) {
+ i_error("Mailbox %s: Couldn't move mail within mailbox: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &importer->mail_error));
+ ret = -1;
+ } else if (ret > 0) {
+ ret = 0;
+ }
+ renumber_count++;
+ }
+ if (mailbox_search_deinit(&search_ctx) < 0) {
+ i_error("Mailbox %s: mail search failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &importer->mail_error));
+ ret = -1;
+ }
+
+ if (mailbox_transaction_commit(&trans) < 0) {
+ i_error("Mailbox %s: UID reassign commit failed: %s",
+ mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &importer->mail_error));
+ ret = -1;
+ }
+ if (ret == 0) {
+ imp_debug(importer, "Mailbox %s: Change during sync: "
+ "Renumbered %u of %u unwanted UIDs",
+ mailbox_get_vname(box),
+ renumber_count, array_count(unwanted_uids));
+ }
+ return ret;
+}
+
+static int
+reassign_unwanted_uids(struct dsync_mailbox_importer *importer,
+ const char **changes_during_sync_r)
+{
+ ARRAY_TYPE(seq_range) unwanted_uids;
+ const uint32_t *wanted_uids, *saved_uids;
+ uint32_t highest_seen_uid;
+ unsigned int i, wanted_count, saved_count;
+ int ret = 0;
+
+ wanted_uids = array_get(&importer->wanted_uids, &wanted_count);
+ saved_uids = array_get(&importer->saved_uids, &saved_count);
+ i_assert(wanted_count == saved_count);
+ if (wanted_count == 0)
+ return 0;
+ /* wanted_uids contains the UIDs we tried to save mails with.
+ if nothing changed during dsync, we should have the expected UIDs
+ (saved_uids) and all is well.
+
+ if any new messages got inserted during dsync, we'll need to fix up
+ the UIDs and let the next dsync fix up the other side. for example:
+
+ remote uids = 5,7,9 = wanted_uids
+ remote uidnext = 12
+ locally added new uid=5 ->
+ saved_uids = 10,7,9
+
+ we'll now need to reassign UIDs 5 and 10. to be fully future-proof
+ we'll reassign all UIDs between [original local uidnext .. highest
+ UID we think we know] that aren't in saved_uids. */
+
+ /* create uidset for the list of UIDs we don't want to exist */
+ t_array_init(&unwanted_uids, 8);
+ highest_seen_uid = I_MAX(importer->remote_uid_next-1,
+ importer->highest_wanted_uid);
+ i_assert(importer->local_uid_next <= highest_seen_uid);
+ seq_range_array_add_range(&unwanted_uids,
+ importer->local_uid_next, highest_seen_uid);
+ for (i = 0; i < wanted_count; i++) {
+ i_assert(i < wanted_count);
+ if (saved_uids[i] == wanted_uids[i])
+ seq_range_array_remove(&unwanted_uids, saved_uids[i]);
+ }
+
+ ret = reassign_uids_in_seq_range(importer, &unwanted_uids);
+ if (ret == 0) {
+ *changes_during_sync_r = t_strdup_printf(
+ "%u UIDs changed due to UID conflicts",
+ seq_range_count(&unwanted_uids));
+ /* conflicting changes during sync, revert our last-common-uid
+ back to a safe value. */
+ importer->last_common_uid = importer->local_uid_next - 1;
+ }
+ return ret < 0 ? -1 : 0;
+}
+
+static int
+dsync_mailbox_import_commit(struct dsync_mailbox_importer *importer, bool final)
+{
+ struct mail_transaction_commit_changes changes;
+ struct seq_range_iter iter;
+ uint32_t uid;
+ unsigned int n;
+ int ret = importer->failed ? -1 : 0;
+
+ mail_free(&importer->mail);
+ mail_free(&importer->ext_mail);
+
+ /* commit saves */
+ if (mailbox_transaction_commit_get_changes(&importer->ext_trans,
+ &changes) < 0) {
+ i_error("Mailbox %s: Save commit failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box, &importer->mail_error));
+ /* removed wanted_uids that weren't actually saved */
+ array_delete(&importer->wanted_uids,
+ array_count(&importer->saved_uids),
+ array_count(&importer->wanted_uids) -
+ array_count(&importer->saved_uids));
+ mailbox_transaction_rollback(&importer->trans);
+ ret = -1;
+ } else {
+ /* remember the UIDs that were successfully saved */
+ if (importer->debug) T_BEGIN {
+ string_t *str = t_str_new(256);
+ imap_write_seq_range(str, &changes.saved_uids);
+ imp_debug(importer, "Saved UIDs: %s", str_c(str));
+ } T_END;
+ seq_range_array_iter_init(&iter, &changes.saved_uids); n = 0;
+ while (seq_range_array_iter_nth(&iter, n++, &uid))
+ array_push_back(&importer->saved_uids, &uid);
+ pool_unref(&changes.pool);
+
+ /* commit flag changes and expunges */
+ if (mailbox_transaction_commit(&importer->trans) < 0) {
+ i_error("Mailbox %s: Commit failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ ret = -1;
+ }
+ }
+
+ if (!final)
+ dsync_mailbox_import_transaction_begin(importer);
+ return ret;
+}
+
+static int dsync_mailbox_import_finish(struct dsync_mailbox_importer *importer,
+ const char **changes_during_sync_r)
+{
+ struct mailbox_update update;
+ int ret;
+
+ ret = dsync_mailbox_import_commit(importer, TRUE);
+
+ if (ret == 0) {
+ /* update mailbox metadata if we successfully saved
+ everything. */
+ i_zero(&update);
+ update.min_next_uid = importer->remote_uid_next;
+ update.min_first_recent_uid =
+ I_MIN(importer->last_common_uid+1,
+ importer->remote_first_recent_uid);
+ update.min_highest_modseq = importer->remote_highest_modseq;
+ update.min_highest_pvt_modseq = importer->remote_highest_pvt_modseq;
+
+ imp_debug(importer, "Finish update: min_next_uid=%u "
+ "min_first_recent_uid=%u min_highest_modseq=%"PRIu64" "
+ "min_highest_pvt_modseq=%"PRIu64,
+ update.min_next_uid, update.min_first_recent_uid,
+ update.min_highest_modseq,
+ update.min_highest_pvt_modseq);
+
+ if (mailbox_update(importer->box, &update) < 0) {
+ i_error("Mailbox %s: Update failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ ret = -1;
+ }
+ }
+
+ /* sync mailbox to finish flag changes and expunges. */
+ if (mailbox_sync(importer->box, 0) < 0) {
+ i_error("Mailbox %s: Sync failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ ret = -1;
+ }
+ if (ret == 0) {
+ /* give new UIDs to messages that got saved with unwanted UIDs.
+ do it only if the whole transaction succeeded. */
+ if (reassign_unwanted_uids(importer, changes_during_sync_r) < 0)
+ ret = -1;
+ }
+ return ret;
+}
+
+static void
+dsync_mailbox_import_check_missing_guid_imports(struct dsync_mailbox_importer *importer)
+{
+ struct hash_iterate_context *iter;
+ const char *key;
+ struct importer_new_mail *mail;
+
+ iter = hash_table_iterate_init(importer->import_guids);
+ while (hash_table_iterate(iter, importer->import_guids, &key, &mail)) {
+ for (; mail != NULL; mail = mail->next) {
+ if (mail->skip)
+ continue;
+
+ i_error("Mailbox %s: Remote didn't send mail GUID=%s (UID=%u)",
+ mailbox_get_vname(importer->box),
+ mail->guid, mail->remote_uid);
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+static void
+dsync_mailbox_import_check_missing_uid_imports(struct dsync_mailbox_importer *importer)
+{
+ struct hash_iterate_context *iter;
+ void *key;
+ struct importer_new_mail *mail;
+
+ iter = hash_table_iterate_init(importer->import_uids);
+ while (hash_table_iterate(iter, importer->import_uids, &key, &mail)) {
+ for (; mail != NULL; mail = mail->next) {
+ if (mail->skip)
+ continue;
+
+ i_error("Mailbox %s: Remote didn't send mail UID=%u",
+ mailbox_get_vname(importer->box),
+ mail->remote_uid);
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+}
+
+int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **_importer,
+ bool success,
+ uint32_t *last_common_uid_r,
+ uint64_t *last_common_modseq_r,
+ uint64_t *last_common_pvt_modseq_r,
+ uint32_t *last_messages_count_r,
+ const char **changes_during_sync_r,
+ bool *require_full_resync_r,
+ enum mail_error *error_r)
+{
+ struct dsync_mailbox_importer *importer = *_importer;
+ struct mailbox_status status;
+ int ret;
+
+ *_importer = NULL;
+ *changes_during_sync_r = NULL;
+ *require_full_resync_r = importer->require_full_resync;
+
+ if ((!success || importer->require_full_resync) && !importer->failed) {
+ importer->mail_error = MAIL_ERROR_TEMP;
+ importer->failed = TRUE;
+ }
+
+ if (!importer->new_uids_assigned && !importer->failed)
+ dsync_mailbox_import_assign_new_uids(importer);
+
+ if (!importer->failed) {
+ dsync_mailbox_import_check_missing_guid_imports(importer);
+ dsync_mailbox_import_check_missing_uid_imports(importer);
+ }
+
+ if (importer->search_ctx != NULL) {
+ if (mailbox_search_deinit(&importer->search_ctx) < 0) {
+ i_error("Mailbox %s: Search failed: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ }
+ }
+ if (dsync_mailbox_import_finish(importer, changes_during_sync_r) < 0)
+ importer->failed = TRUE;
+
+ if (importer->virtual_mail != NULL)
+ mail_free(&importer->virtual_mail);
+ if (importer->virtual_trans != NULL)
+ (void)mailbox_transaction_commit(&importer->virtual_trans);
+
+ hash_table_destroy(&importer->import_guids);
+ hash_table_destroy(&importer->import_uids);
+ array_free(&importer->maybe_expunge_uids);
+ array_free(&importer->maybe_saves);
+ array_free(&importer->wanted_uids);
+ array_free(&importer->saved_uids);
+ array_free(&importer->newmails);
+ if (array_is_created(&importer->mail_requests))
+ array_free(&importer->mail_requests);
+
+ *last_common_uid_r = importer->last_common_uid;
+ if (*changes_during_sync_r == NULL) {
+ *last_common_modseq_r = importer->remote_highest_modseq;
+ *last_common_pvt_modseq_r = importer->remote_highest_pvt_modseq;
+ } else {
+ /* local changes occurred during dsync. we exported changes up
+ to local_initial_highestmodseq, so all of the changes have
+ happened after it. we want the next run to see those changes,
+ so return it as the last common modseq */
+ *last_common_modseq_r = importer->local_initial_highestmodseq;
+ *last_common_pvt_modseq_r = importer->local_initial_highestpvtmodseq;
+ }
+ if (importer->delete_mailbox) {
+ if (mailbox_delete(importer->box) < 0) {
+ i_error("Couldn't delete mailbox %s: %s",
+ mailbox_get_vname(importer->box),
+ mailbox_get_last_internal_error(importer->box,
+ &importer->mail_error));
+ importer->failed = TRUE;
+ }
+ *last_messages_count_r = 0;
+ } else {
+ mailbox_get_open_status(importer->box, STATUS_MESSAGES, &status);
+ *last_messages_count_r = status.messages;
+ }
+
+ i_assert(importer->failed == (importer->mail_error != 0));
+ ret = importer->failed ? -1 : 0;
+ *error_r = importer->mail_error;
+ pool_unref(&importer->pool);
+ return ret;
+}
+
+const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer)
+{
+ if (importer->search_ctx != NULL)
+ return "";
+ return t_strdup_printf("%u/%u", importer->import_pos,
+ importer->import_count);
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-import.h b/src/doveadm/dsync/dsync-mailbox-import.h
new file mode 100644
index 0000000..6a2020e
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-import.h
@@ -0,0 +1,63 @@
+#ifndef DSYNC_MAILBOX_IMPORT_H
+#define DSYNC_MAILBOX_IMPORT_H
+
+#include "mail-error.h"
+
+enum dsync_mailbox_import_flags {
+ DSYNC_MAILBOX_IMPORT_FLAG_MASTER_BRAIN = 0x01,
+ DSYNC_MAILBOX_IMPORT_FLAG_WANT_MAIL_REQUESTS = 0x02,
+ DSYNC_MAILBOX_IMPORT_FLAG_REVERT_LOCAL_CHANGES = 0x04,
+ DSYNC_MAILBOX_IMPORT_FLAG_DEBUG = 0x08,
+ DSYNC_MAILBOX_IMPORT_FLAG_MAILS_HAVE_GUIDS = 0x10,
+ DSYNC_MAILBOX_IMPORT_FLAG_MAILS_USE_GUID128 = 0x20,
+ DSYNC_MAILBOX_IMPORT_FLAG_NO_NOTIFY = 0x40,
+ DSYNC_MAILBOX_IMPORT_FLAG_EMPTY_HDR_WORKAROUND = 0x100
+};
+
+struct mailbox;
+struct dsync_mailbox_attribute;
+struct dsync_mail;
+struct dsync_mail_change;
+struct dsync_transaction_log_scan;
+
+struct dsync_mailbox_importer *
+dsync_mailbox_import_init(struct mailbox *box,
+ struct mailbox *virtual_all_box,
+ struct dsync_transaction_log_scan *log_scan,
+ uint32_t last_common_uid,
+ uint64_t last_common_modseq,
+ uint64_t last_common_pvt_modseq,
+ uint32_t remote_uid_next,
+ uint32_t remote_first_recent_uid,
+ uint64_t remote_highest_modseq,
+ uint64_t remote_highest_pvt_modseq,
+ time_t sync_since_timestamp,
+ time_t sync_until_timestamp,
+ uoff_t sync_max_size,
+ const char *sync_flag,
+ unsigned int commit_msgs_interval,
+ enum dsync_mailbox_import_flags flags,
+ unsigned int hdr_hash_version,
+ const char *const *hashed_headers);
+int dsync_mailbox_import_attribute(struct dsync_mailbox_importer *importer,
+ const struct dsync_mailbox_attribute *attr);
+int dsync_mailbox_import_change(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail_change *change);
+int dsync_mailbox_import_changes_finish(struct dsync_mailbox_importer *importer);
+const struct dsync_mail_request *
+dsync_mailbox_import_next_request(struct dsync_mailbox_importer *importer);
+int dsync_mailbox_import_mail(struct dsync_mailbox_importer *importer,
+ const struct dsync_mail *mail);
+int dsync_mailbox_import_deinit(struct dsync_mailbox_importer **importer,
+ bool success,
+ uint32_t *last_common_uid_r,
+ uint64_t *last_common_modseq_r,
+ uint64_t *last_common_pvt_modseq_r,
+ uint32_t *last_messages_count_r,
+ const char **changes_during_sync_r,
+ bool *require_full_resync_r,
+ enum mail_error *error_r);
+
+const char *dsync_mailbox_import_get_proctitle(struct dsync_mailbox_importer *importer);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-state.c b/src/doveadm/dsync/dsync-mailbox-state.c
new file mode 100644
index 0000000..cca24d4
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-state.c
@@ -0,0 +1,127 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "base64.h"
+#include "crc32.h"
+#include "hash.h"
+#include "dsync-mailbox-state.h"
+
+#define DSYNC_STATE_MAJOR_VERSION 1
+#define DSYNC_STATE_MINOR_VERSION 0
+
+#define V0_MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8)
+#define MAILBOX_SIZE (GUID_128_SIZE + 4 + 4 + 8 + 8 + 4)
+
+static void put_uint32(buffer_t *output, uint32_t num)
+{
+ uint8_t tmp[sizeof(uint32_t)];
+
+ cpu32_to_le_unaligned(num, tmp);
+
+ buffer_append(output, tmp, sizeof(tmp));
+}
+
+void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states,
+ string_t *output)
+{
+ struct hash_iterate_context *iter;
+ struct dsync_mailbox_state *state;
+ uint8_t *guid;
+ buffer_t *buf = t_buffer_create(128);
+ uint32_t crc = 0;
+
+ buffer_append_c(buf, DSYNC_STATE_MAJOR_VERSION);
+ buffer_append_c(buf, DSYNC_STATE_MINOR_VERSION);
+ buffer_append_c(buf, '\0');
+ buffer_append_c(buf, '\0');
+
+ iter = hash_table_iterate_init(states);
+ while (hash_table_iterate(iter, states, &guid, &state)) {
+ buffer_append(buf, state->mailbox_guid,
+ sizeof(state->mailbox_guid));
+ put_uint32(buf, state->last_uidvalidity);
+ put_uint32(buf, state->last_common_uid);
+ put_uint32(buf, state->last_common_modseq & 0xffffffffU);
+ put_uint32(buf, state->last_common_modseq >> 32);
+ put_uint32(buf, state->last_common_pvt_modseq & 0xffffffffU);
+ put_uint32(buf, state->last_common_pvt_modseq >> 32);
+ put_uint32(buf, state->last_messages_count); /* v1 */
+ if (buf->used % 3 == 0) {
+ crc = crc32_data_more(crc, buf->data, buf->used);
+ base64_encode(buf->data, buf->used, output);
+ buffer_set_used_size(buf, 0);
+ }
+ }
+ hash_table_iterate_deinit(&iter);
+
+ crc = crc32_data_more(crc, buf->data, buf->used);
+ put_uint32(buf, crc);
+ base64_encode(buf->data, buf->used, output);
+}
+
+static int dsync_mailbox_states_retry_import_v0(const buffer_t *buf)
+{
+ const unsigned char *data = buf->data;
+
+ /* v0 had no version header and no last_messages_count */
+
+ if ((buf->used-4) % V0_MAILBOX_SIZE != 0 ||
+ le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4))
+ return -1;
+ /* looks like valid v0 format, silently treat it as empty state */
+ return 0;
+}
+
+int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states,
+ pool_t pool, const char *input,
+ const char **error_r)
+{
+ struct dsync_mailbox_state *state;
+ buffer_t *buf;
+ uint8_t *guid_p;
+ const unsigned char *data;
+ unsigned int i, count;
+
+ buf = t_buffer_create(strlen(input));
+ if (base64_decode(input, strlen(input), NULL, buf) < 0) {
+ *error_r = "Invalid base64 data";
+ return -1;
+ }
+ /* v1: 4 byte header, mailboxes[], CRC32 */
+ data = buf->data;
+
+ if (buf->used == 4 && le32_to_cpu_unaligned(data) == 0) {
+ /* v0: Empty state */
+ return 0;
+ }
+ if (buf->used < 8) {
+ *error_r = "Input too small";
+ return -1;
+ }
+
+ if ((buf->used-8) % MAILBOX_SIZE != 0) {
+ *error_r = "Invalid input size";
+ return dsync_mailbox_states_retry_import_v0(buf);
+ }
+
+ if (le32_to_cpu_unaligned(data + buf->used-4) != crc32_data(data, buf->used-4)) {
+ *error_r = "CRC32 mismatch";
+ return dsync_mailbox_states_retry_import_v0(buf);
+ }
+ data += 4;
+ count = (buf->used-8) / MAILBOX_SIZE;
+
+ for (i = 0; i < count; i++, data += MAILBOX_SIZE) {
+ state = p_new(pool, struct dsync_mailbox_state, 1);
+ memcpy(state->mailbox_guid, data, GUID_128_SIZE);
+ state->last_uidvalidity = le32_to_cpu_unaligned(data + GUID_128_SIZE);
+ state->last_common_uid = le32_to_cpu_unaligned(data + GUID_128_SIZE + 4);
+ state->last_common_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 8);
+ state->last_common_pvt_modseq = le64_to_cpu_unaligned(data + GUID_128_SIZE + 16);
+ state->last_messages_count = le32_to_cpu_unaligned(data + GUID_128_SIZE + 24);
+ guid_p = state->mailbox_guid;
+ hash_table_insert(states, guid_p, state);
+ }
+ return 0;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-state.h b/src/doveadm/dsync/dsync-mailbox-state.h
new file mode 100644
index 0000000..a01f531
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-state.h
@@ -0,0 +1,24 @@
+#ifndef DSYNC_MAILBOX_STATE_H
+#define DSYNC_MAILBOX_STATE_H
+
+#include "guid.h"
+
+struct dsync_mailbox_state {
+ guid_128_t mailbox_guid;
+ uint32_t last_uidvalidity;
+ uint32_t last_common_uid;
+ uint64_t last_common_modseq;
+ uint64_t last_common_pvt_modseq;
+ uint32_t last_messages_count;
+ bool changes_during_sync;
+};
+ARRAY_DEFINE_TYPE(dsync_mailbox_state, struct dsync_mailbox_state);
+HASH_TABLE_DEFINE_TYPE(dsync_mailbox_state, uint8_t *, struct dsync_mailbox_state *);
+
+void dsync_mailbox_states_export(const HASH_TABLE_TYPE(dsync_mailbox_state) states,
+ string_t *output);
+int dsync_mailbox_states_import(HASH_TABLE_TYPE(dsync_mailbox_state) states,
+ pool_t pool, const char *input,
+ const char **error_r);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-fill.c b/src/doveadm/dsync/dsync-mailbox-tree-fill.c
new file mode 100644
index 0000000..c523e6b
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree-fill.c
@@ -0,0 +1,405 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "guid.h"
+#include "str.h"
+#include "wildcard-match.h"
+#include "mailbox-log.h"
+#include "mail-namespace.h"
+#include "mail-storage.h"
+#include "mailbox-list-iter.h"
+#include "dsync-mailbox-tree-private.h"
+
+static int
+dsync_mailbox_tree_add_node(struct dsync_mailbox_tree *tree,
+ const struct mailbox_info *info,
+ struct dsync_mailbox_node **node_r)
+{
+ struct dsync_mailbox_node *node;
+
+ node = dsync_mailbox_tree_get(tree, info->vname);
+ if (node->ns == info->ns)
+ ;
+ else if (node->ns == NULL) {
+ i_assert(tree->root.ns == NULL);
+ node->ns = info->ns;
+ } else {
+ i_error("Mailbox '%s' exists in two namespaces: '%s' and '%s'",
+ info->vname, node->ns->prefix, info->ns->prefix);
+ return -1;
+ }
+ *node_r = node;
+ return 0;
+}
+
+static int
+dsync_mailbox_tree_add_exists_node(struct dsync_mailbox_tree *tree,
+ const struct mailbox_info *info,
+ struct dsync_mailbox_node **node_r,
+ enum mail_error *error_r)
+{
+ if (dsync_mailbox_tree_add_node(tree, info, node_r) < 0) {
+ *error_r = MAIL_ERROR_TEMP;
+ return -1;
+ }
+ (*node_r)->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ return 0;
+}
+
+static int
+dsync_mailbox_tree_get_selectable(struct mailbox *box,
+ struct mailbox_metadata *metadata_r,
+ struct mailbox_status *status_r)
+{
+ /* try the fast path */
+ if (mailbox_get_metadata(box, MAILBOX_METADATA_GUID, metadata_r) < 0)
+ return -1;
+ if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
+ return -1;
+
+ i_assert(!guid_128_is_empty(metadata_r->guid));
+ if (status_r->uidvalidity != 0)
+ return 0;
+
+ /* no UIDVALIDITY assigned yet. syncing a mailbox should add it. */
+ if (mailbox_sync(box, 0) < 0)
+ return -1;
+ if (mailbox_get_status(box, STATUS_UIDVALIDITY | STATUS_UIDNEXT, status_r) < 0)
+ return -1;
+ i_assert(status_r->uidvalidity != 0);
+ return 0;
+}
+
+static int dsync_mailbox_tree_add(struct dsync_mailbox_tree *tree,
+ const struct mailbox_info *info,
+ const guid_128_t box_guid,
+ enum mail_error *error_r)
+{
+ struct dsync_mailbox_node *node;
+ struct mailbox *box;
+ struct mailbox_metadata metadata;
+ struct mailbox_status status;
+ const char *errstr;
+ enum mail_error error;
+ int ret = 0;
+
+ if ((info->flags & MAILBOX_NONEXISTENT) != 0)
+ return 0;
+ if ((info->flags & MAILBOX_NOSELECT) != 0) {
+ return !guid_128_is_empty(box_guid) ? 0 :
+ dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r);
+ }
+
+ /* get GUID and UIDVALIDITY for selectable mailbox */
+ box = mailbox_alloc(info->ns->list, info->vname, MAILBOX_FLAG_READONLY);
+ if (dsync_mailbox_tree_get_selectable(box, &metadata, &status) < 0) {
+ errstr = mailbox_get_last_internal_error(box, &error);
+ switch (error) {
+ case MAIL_ERROR_NOTFOUND:
+ /* mailbox was just deleted? */
+ break;
+ case MAIL_ERROR_NOTPOSSIBLE:
+ /* invalid mbox files? ignore */
+ break;
+ default:
+ i_error("Failed to access mailbox %s: %s",
+ info->vname, errstr);
+ *error_r = error;
+ ret = -1;
+ }
+ mailbox_free(&box);
+ return ret;
+ }
+ mailbox_free(&box);
+
+ if (!guid_128_is_empty(box_guid) &&
+ !guid_128_equals(box_guid, metadata.guid)) {
+ /* unwanted mailbox */
+ return 0;
+ }
+ if (dsync_mailbox_tree_add_exists_node(tree, info, &node, error_r) < 0)
+ return -1;
+ memcpy(node->mailbox_guid, metadata.guid,
+ sizeof(node->mailbox_guid));
+ node->uid_validity = status.uidvalidity;
+ node->uid_next = status.uidnext;
+ return 0;
+}
+
+static struct dsync_mailbox_node *
+dsync_mailbox_tree_find_sha(struct dsync_mailbox_tree *tree,
+ struct mail_namespace *ns, const guid_128_t sha128)
+{
+ struct dsync_mailbox_node *node;
+
+ if (!hash_table_is_created(tree->name128_hash))
+ dsync_mailbox_tree_build_name128_hash(tree);
+
+ node = hash_table_lookup(tree->name128_hash, sha128);
+ return node == NULL || node->ns != ns ? NULL : node;
+}
+
+static int
+dsync_mailbox_tree_add_change_timestamps(struct dsync_mailbox_tree *tree,
+ struct mail_namespace *ns)
+{
+ struct dsync_mailbox_node *node;
+ struct dsync_mailbox_delete *del;
+ struct mailbox_log *log;
+ struct mailbox_log_iter *iter;
+ const struct mailbox_log_record *rec;
+ const uint8_t *guid_p;
+ time_t timestamp;
+
+ log = mailbox_list_get_changelog(ns->list);
+ if (log == NULL)
+ return 0;
+
+ iter = mailbox_log_iter_init(log);
+ while ((rec = mailbox_log_iter_next(iter)) != NULL) {
+ /* For DELETE_MAILBOX the record_guid is the mailbox GUID.
+ Otherwise it's 128bit SHA1 of the mailbox vname. */
+ node = rec->type == MAILBOX_LOG_RECORD_DELETE_MAILBOX ? NULL :
+ dsync_mailbox_tree_find_sha(tree, ns, rec->mailbox_guid);
+
+ timestamp = mailbox_log_record_get_timestamp(rec);
+ switch (rec->type) {
+ case MAILBOX_LOG_RECORD_DELETE_MAILBOX:
+ guid_p = rec->mailbox_guid;
+ if (hash_table_lookup(tree->guid_hash, guid_p) != NULL) {
+ /* mailbox still exists. maybe it was restored
+ from backup or something. */
+ break;
+ }
+ del = array_append_space(&tree->deletes);
+ del->type = DSYNC_MAILBOX_DELETE_TYPE_MAILBOX;
+ del->timestamp = timestamp;
+ memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
+ break;
+ case MAILBOX_LOG_RECORD_DELETE_DIR:
+ if (node != NULL &&
+ node->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+ /* directory exists again, skip it */
+ break;
+ }
+ /* we don't know what directory name was deleted,
+ just its hash. if the name still exists on the other
+ dsync side, it can match this deletion to the
+ name. */
+ del = array_append_space(&tree->deletes);
+ del->type = DSYNC_MAILBOX_DELETE_TYPE_DIR;
+ del->timestamp = timestamp;
+ memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
+ break;
+ case MAILBOX_LOG_RECORD_CREATE_DIR:
+ if (node == NULL) {
+ /* directory has been deleted again, skip it */
+ break;
+ }
+ /* notify the remote that we want to keep this
+ directory created (unless remote has a newer delete
+ timestamp) */
+ node->last_renamed_or_created = timestamp;
+ break;
+ case MAILBOX_LOG_RECORD_RENAME:
+ if (node != NULL)
+ node->last_renamed_or_created = timestamp;
+ break;
+ case MAILBOX_LOG_RECORD_SUBSCRIBE:
+ if (node != NULL)
+ node->last_subscription_change = timestamp;
+ break;
+ case MAILBOX_LOG_RECORD_UNSUBSCRIBE:
+ if (node != NULL) {
+ node->last_subscription_change = timestamp;
+ break;
+ }
+ /* The mailbox is already deleted, but it may still
+ exist on the other side (even the subscription
+ alone). */
+ del = array_append_space(&tree->deletes);
+ del->type = DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE;
+ del->timestamp = timestamp;
+ memcpy(del->guid, rec->mailbox_guid, sizeof(del->guid));
+ break;
+ }
+ }
+ if (mailbox_log_iter_deinit(&iter) < 0) {
+ i_error("Mailbox log iteration for namespace '%s' failed",
+ ns->prefix);
+ return -1;
+ }
+ return 0;
+}
+
+static int
+dsync_mailbox_tree_fix_guid_duplicate(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node1,
+ struct dsync_mailbox_node *node2)
+{
+ struct mailbox *box;
+ struct mailbox_update update;
+ struct dsync_mailbox_node *change_node;
+ const char *change_vname;
+ int ret = 0;
+
+ i_zero(&update);
+ guid_128_generate(update.mailbox_guid);
+
+ /* just in case the duplication exists in both sides,
+ make them choose the same node */
+ if (strcmp(dsync_mailbox_node_get_full_name(tree, node1),
+ dsync_mailbox_node_get_full_name(tree, node2)) <= 0)
+ change_node = node1;
+ else
+ change_node = node2;
+
+ change_vname = dsync_mailbox_node_get_full_name(tree, change_node);
+ i_error("Duplicate mailbox GUID %s for mailboxes %s and %s - "
+ "giving a new GUID %s to %s",
+ guid_128_to_string(node1->mailbox_guid),
+ dsync_mailbox_node_get_full_name(tree, node1),
+ dsync_mailbox_node_get_full_name(tree, node2),
+ guid_128_to_string(update.mailbox_guid), change_vname);
+
+ i_assert(node1->ns != NULL && node2->ns != NULL);
+ box = mailbox_alloc(change_node->ns->list, change_vname, 0);
+ if (mailbox_update(box, &update) < 0) {
+ i_error("Couldn't update mailbox %s GUID: %s",
+ change_vname, mailbox_get_last_internal_error(box, NULL));
+ ret = -1;
+ } else {
+ memcpy(change_node->mailbox_guid, update.mailbox_guid,
+ sizeof(change_node->mailbox_guid));
+ }
+ mailbox_free(&box);
+ return ret;
+}
+
+static bool
+dsync_mailbox_info_is_wanted(const struct mailbox_info *info,
+ const char *box_name,
+ const char *const *exclude_mailboxes)
+{
+ const char *const *info_specialuses;
+ unsigned int i;
+
+ if (exclude_mailboxes == NULL &&
+ (box_name == NULL || box_name[0] != '\\'))
+ return TRUE;
+
+ info_specialuses = info->special_use == NULL ? NULL :
+ t_strsplit(info->special_use, " ");
+ /* include */
+ if (box_name != NULL && box_name[0] == '\\') {
+ if (info_specialuses == NULL ||
+ !str_array_icase_find(info_specialuses, box_name))
+ return FALSE;
+ }
+ /* exclude */
+ if (exclude_mailboxes == NULL)
+ return TRUE;
+ for (i = 0; exclude_mailboxes[i] != NULL; i++) {
+ const char *exclude = exclude_mailboxes[i];
+
+ if (exclude[0] == '\\') {
+ /* special-use */
+ if (info_specialuses != NULL &&
+ str_array_icase_find(info_specialuses, exclude))
+ return FALSE;
+ } else {
+ /* mailbox with wildcards */
+ if (wildcard_match(info->vname, exclude))
+ return FALSE;
+ }
+ }
+ return TRUE;
+}
+
+int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree,
+ struct mail_namespace *ns, const char *box_name,
+ const guid_128_t box_guid,
+ const char *const *exclude_mailboxes,
+ enum mail_error *error_r)
+{
+ const enum mailbox_list_iter_flags list_flags =
+ /* FIXME: we'll skip symlinks, because we can't handle them
+ currently. in future we could detect them and create them
+ by creating the symlink. */
+ MAILBOX_LIST_ITER_SKIP_ALIASES |
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES;
+ const enum mailbox_list_iter_flags subs_list_flags =
+ MAILBOX_LIST_ITER_NO_AUTO_BOXES |
+ MAILBOX_LIST_ITER_SELECT_SUBSCRIBED |
+ MAILBOX_LIST_ITER_RETURN_NO_FLAGS;
+ struct mailbox_list_iterate_context *iter;
+ struct dsync_mailbox_node *node, *dup_node1, *dup_node2;
+ const struct mailbox_info *info;
+ const char *list_pattern =
+ box_name != NULL && box_name[0] != '\\' ? box_name : "*";
+ int ret = 0;
+
+ i_assert(mail_namespace_get_sep(ns) == tree->sep);
+
+ /* assign namespace to its root, so it gets copied to children */
+ if (ns->prefix_len > 0) {
+ const char *vname = t_strndup(ns->prefix, ns->prefix_len-1);
+ node = dsync_mailbox_tree_get(tree, vname);
+ node->ns = ns;
+
+ struct mailbox_info ns_info = {
+ .vname = vname,
+ .ns = ns,
+ };
+ if (dsync_mailbox_tree_add(tree, &ns_info, box_guid, error_r) < 0)
+ return -1;
+ } else {
+ tree->root.ns = ns;
+ }
+
+ /* first add all of the existing mailboxes */
+ iter = mailbox_list_iter_init(ns->list, list_pattern, list_flags);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) T_BEGIN {
+ if (dsync_mailbox_info_is_wanted(info, box_name,
+ exclude_mailboxes)) {
+ if (dsync_mailbox_tree_add(tree, info, box_guid, error_r) < 0)
+ ret = -1;
+ }
+ } T_END;
+ if (mailbox_list_iter_deinit(&iter) < 0) {
+ i_error("Mailbox listing for namespace '%s' failed: %s",
+ ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r));
+ ret = -1;
+ }
+
+ /* add subscriptions */
+ iter = mailbox_list_iter_init(ns->list, list_pattern, subs_list_flags);
+ while ((info = mailbox_list_iter_next(iter)) != NULL) {
+ if (dsync_mailbox_tree_add_node(tree, info, &node) == 0)
+ node->subscribed = TRUE;
+ else {
+ *error_r = MAIL_ERROR_TEMP;
+ ret = -1;
+ }
+ }
+ if (mailbox_list_iter_deinit(&iter) < 0) {
+ i_error("Mailbox listing for namespace '%s' failed: %s",
+ ns->prefix, mailbox_list_get_last_internal_error(ns->list, error_r));
+ ret = -1;
+ }
+ if (ret < 0)
+ return -1;
+
+ while (dsync_mailbox_tree_build_guid_hash(tree, &dup_node1,
+ &dup_node2) < 0) {
+ if (dsync_mailbox_tree_fix_guid_duplicate(tree, dup_node1, dup_node2) < 0)
+ return -1;
+ }
+
+ /* add timestamps */
+ if (dsync_mailbox_tree_add_change_timestamps(tree, ns) < 0)
+ return -1;
+ return 0;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-private.h b/src/doveadm/dsync/dsync-mailbox-tree-private.h
new file mode 100644
index 0000000..0614151
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree-private.h
@@ -0,0 +1,38 @@
+#ifndef DSYNC_MAILBOX_TREE_PRIVATE_H
+#define DSYNC_MAILBOX_TREE_PRIVATE_H
+
+#include "dsync-mailbox-tree.h"
+
+struct dsync_mailbox_tree {
+ pool_t pool;
+ char sep, sep_str[2], remote_sep, alt_char;
+ char escape_char, remote_escape_char;
+ /* root node isn't part of the real mailbox tree. its name is "" and
+ it has no siblings */
+ struct dsync_mailbox_node root;
+
+ unsigned int iter_count;
+
+ ARRAY(struct dsync_mailbox_delete) deletes;
+
+ /* guid_128_t => struct dsync_mailbox_node */
+ HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_hash;
+ HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) name128_remotesep_hash;
+ HASH_TABLE(uint8_t *, struct dsync_mailbox_node *) guid_hash;
+};
+
+void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree);
+
+int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1,
+ struct dsync_mailbox_node *const *n2);
+
+void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node *parent);
+void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node);
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src);
+bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1,
+ struct dsync_mailbox_tree *tree2);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox-tree-sync.c b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
new file mode 100644
index 0000000..e4a50ae
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree-sync.c
@@ -0,0 +1,1473 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "buffer.h"
+#include "str.h"
+#include "md5.h"
+#include "hex-binary.h"
+#include "aqueue.h"
+#include "hash.h"
+#include "dsync-brain-private.h"
+#include "dsync-mailbox-tree-private.h"
+
+#define TEMP_MAX_NAME_LEN 100
+#define TEMP_SUFFIX_MAX_LEN (sizeof("temp-")-1 + 8)
+#define TEMP_SUFFIX_FORMAT "temp-%x"
+
+struct dsync_mailbox_tree_bfs_iter {
+ struct dsync_mailbox_tree *tree;
+
+ ARRAY(struct dsync_mailbox_node *) queue_arr;
+ struct aqueue *queue;
+ struct dsync_mailbox_node *cur;
+};
+
+struct dsync_mailbox_tree_sync_ctx {
+ pool_t pool;
+ struct dsync_mailbox_tree *local_tree, *remote_tree;
+ enum dsync_mailbox_trees_sync_type sync_type;
+ enum dsync_mailbox_trees_sync_flags sync_flags;
+ unsigned int combined_mailboxes_count;
+
+ ARRAY(struct dsync_mailbox_tree_sync_change) changes;
+ unsigned int change_idx;
+ bool failed;
+};
+
+static struct dsync_mailbox_tree_bfs_iter *
+dsync_mailbox_tree_bfs_iter_init(struct dsync_mailbox_tree *tree)
+{
+ struct dsync_mailbox_tree_bfs_iter *iter;
+
+ iter = i_new(struct dsync_mailbox_tree_bfs_iter, 1);
+ iter->tree = tree;
+ i_array_init(&iter->queue_arr, 32);
+ iter->queue = aqueue_init(&iter->queue_arr.arr);
+ iter->cur = tree->root.first_child;
+ return iter;
+}
+
+static bool
+dsync_mailbox_tree_bfs_iter_next(struct dsync_mailbox_tree_bfs_iter *iter,
+ struct dsync_mailbox_node **node_r)
+{
+ if (iter->cur == NULL) {
+ if (aqueue_count(iter->queue) == 0)
+ return FALSE;
+ iter->cur = array_idx_elem(&iter->queue_arr,
+ aqueue_idx(iter->queue, 0));
+ aqueue_delete_tail(iter->queue);
+ }
+ *node_r = iter->cur;
+
+ if (iter->cur->first_child != NULL)
+ aqueue_append(iter->queue, &iter->cur->first_child);
+ iter->cur = iter->cur->next;
+ return TRUE;
+}
+
+static void
+dsync_mailbox_tree_bfs_iter_deinit(struct dsync_mailbox_tree_bfs_iter **_iter)
+{
+ struct dsync_mailbox_tree_bfs_iter *iter = *_iter;
+
+ *_iter = NULL;
+
+ aqueue_deinit(&iter->queue);
+ array_free(&iter->queue_arr);
+ i_free(iter);
+}
+
+static void
+sync_add_dir_change(struct dsync_mailbox_tree_sync_ctx *ctx,
+ const struct dsync_mailbox_node *node,
+ enum dsync_mailbox_tree_sync_type type)
+{
+ struct dsync_mailbox_tree_sync_change *change;
+ const char *name;
+
+ i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+
+ name = dsync_mailbox_node_get_full_name(ctx->local_tree, node);
+
+ change = array_append_space(&ctx->changes);
+ change->type = type;
+ change->ns = node->ns;
+ change->full_name = p_strdup(ctx->pool, name);
+}
+
+static void
+sync_add_create_change(struct dsync_mailbox_tree_sync_ctx *ctx,
+ const struct dsync_mailbox_node *node, const char *name)
+{
+ struct dsync_mailbox_tree_sync_change *change;
+
+ i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+
+ change = array_append_space(&ctx->changes);
+ change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX;
+ change->ns = node->ns;
+ change->full_name = p_strdup(ctx->pool, name);
+ memcpy(change->mailbox_guid, node->mailbox_guid,
+ sizeof(change->mailbox_guid));
+ change->uid_validity = node->uid_validity;
+}
+
+static void sort_siblings(ARRAY_TYPE(dsync_mailbox_node) *siblings)
+{
+ struct dsync_mailbox_node *const *nodes;
+ unsigned int i, count;
+
+ array_sort(siblings, dsync_mailbox_node_name_cmp);
+
+ nodes = array_get(siblings, &count);
+ if (count == 0)
+ return;
+
+ nodes[0]->parent->first_child = nodes[0];
+ for (i = 1; i < count; i++)
+ nodes[i-1]->next = nodes[i];
+ nodes[count-1]->next = NULL;
+}
+
+static void
+sync_set_node_deleted(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node)
+{
+ const uint8_t *guid_p;
+
+ /* for the rest of this sync assume that the mailbox has
+ already been deleted */
+ guid_p = node->mailbox_guid;
+ hash_table_remove(tree->guid_hash, guid_p);
+
+ node->existence = DSYNC_MAILBOX_NODE_DELETED;
+ memset(node->mailbox_guid, 0, sizeof(node->mailbox_guid));
+ node->uid_validity = 0;
+}
+
+static void
+sync_delete_mailbox_node(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node, const char *reason)
+{
+ struct dsync_mailbox_tree_sync_change *change;
+ const char *name;
+
+ if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 &&
+ tree == ctx->local_tree) {
+ i_debug("brain %c: Deleting mailbox '%s' (GUID %s): %s",
+ (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+ dsync_mailbox_node_get_full_name(tree, node),
+ guid_128_to_string(node->mailbox_guid), reason);
+ }
+
+ if (tree == ctx->local_tree &&
+ node->existence != DSYNC_MAILBOX_NODE_DELETED) {
+ /* delete this mailbox locally */
+ i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+ change = array_append_space(&ctx->changes);
+ change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX;
+ change->ns = node->ns;
+ name = dsync_mailbox_node_get_full_name(tree, node);
+ change->full_name = p_strdup(ctx->pool, name);
+ memcpy(change->mailbox_guid, node->mailbox_guid,
+ sizeof(change->mailbox_guid));
+ }
+ sync_set_node_deleted(tree, node);
+}
+
+static void
+sync_delete_mailbox(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node, const char *reason)
+{
+ struct dsync_mailbox_tree *other_tree;
+ struct dsync_mailbox_node *other_node;
+ const uint8_t *guid_p;
+
+ other_tree = tree == ctx->local_tree ?
+ ctx->remote_tree : ctx->local_tree;
+ guid_p = node->mailbox_guid;
+ other_node = hash_table_lookup(other_tree->guid_hash, guid_p);
+ if (other_node == NULL) {
+ /* doesn't exist / already deleted */
+ } else {
+ sync_delete_mailbox_node(ctx, other_tree, other_node, reason);
+ }
+ sync_delete_mailbox_node(ctx, tree, node, reason);
+}
+
+static void
+sync_tree_sort_and_delete_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ bool twoway_sync)
+{
+ struct dsync_mailbox_tree_bfs_iter *iter;
+ struct dsync_mailbox_node *node, *parent = NULL;
+ ARRAY_TYPE(dsync_mailbox_node) siblings;
+
+ t_array_init(&siblings, 64);
+
+ iter = dsync_mailbox_tree_bfs_iter_init(tree);
+ while (dsync_mailbox_tree_bfs_iter_next(iter, &node)) {
+ if (node->parent != parent) {
+ sort_siblings(&siblings);
+ array_clear(&siblings);
+ parent = node->parent;
+ }
+ if (node->existence == DSYNC_MAILBOX_NODE_DELETED &&
+ !dsync_mailbox_node_is_dir(node)) {
+ if (twoway_sync) {
+ /* this mailbox was deleted. delete it from the
+ other side as well */
+ sync_delete_mailbox(ctx, tree, node,
+ "Mailbox has been deleted");
+ } else {
+ /* treat the node as if it didn't exist. it'll
+ get either recreated or deleted later. in
+ any case this function must handle all
+ existence=DELETED mailbox nodes by changing
+ them into directories (setting GUID=0) or
+ we'll assert-crash later */
+ sync_set_node_deleted(tree, node);
+ }
+ }
+ ctx->combined_mailboxes_count++;
+ array_push_back(&siblings, &node);
+ }
+ sort_siblings(&siblings);
+ dsync_mailbox_tree_bfs_iter_deinit(&iter);
+}
+
+static bool node_names_equal(const struct dsync_mailbox_node *n1,
+ const struct dsync_mailbox_node *n2)
+{
+ while (n1 != NULL && n2 != NULL) {
+ if (strcmp(n1->name, n2->name) != 0)
+ return FALSE;
+ n1 = n1->parent;
+ n2 = n2->parent;
+ }
+ return n1 == NULL && n2 == NULL;
+}
+
+static void
+dsync_mailbox_tree_node_attach_sorted(struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node *parent)
+{
+ struct dsync_mailbox_node **p;
+
+ node->parent = parent;
+ for (p = &parent->first_child; *p != NULL; p = &(*p)->next) {
+ if (dsync_mailbox_node_name_cmp(p, &node) > 0)
+ break;
+ }
+ node->next = *p;
+ *p = node;
+}
+
+static void
+dsync_mailbox_tree_node_move_sorted(struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node *parent)
+{
+ /* detach from old parent */
+ dsync_mailbox_tree_node_detach(node);
+ /* attach to new parent */
+ dsync_mailbox_tree_node_attach_sorted(node, parent);
+}
+
+static struct dsync_mailbox_node *
+sorted_tree_get(struct dsync_mailbox_tree *tree, const char *name)
+{
+ struct dsync_mailbox_node *node, *parent, *ret;
+
+ node = ret = dsync_mailbox_tree_get(tree, name);
+ while (node->parent != NULL &&
+ node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT) {
+ parent = node->parent;
+ dsync_mailbox_tree_node_detach(node);
+ dsync_mailbox_tree_node_attach_sorted(node, parent);
+ node = parent;
+ }
+ return ret;
+}
+
+static struct dsync_mailbox_node *
+sync_node_new(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node **pos,
+ struct dsync_mailbox_node *parent,
+ const struct dsync_mailbox_node *src)
+{
+ struct dsync_mailbox_node *node;
+
+ node = p_new(tree->pool, struct dsync_mailbox_node, 1);
+ node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+ node->name = p_strdup(tree->pool, src->name);
+ node->sync_temporary_name = src->sync_temporary_name;
+ node->ns = src->ns;
+ node->parent = parent;
+ node->next = *pos;
+ *pos = node;
+ return node;
+}
+
+static struct dsync_mailbox_node *
+sorted_tree_get_by_node_name(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_tree *other_tree,
+ struct dsync_mailbox_node *other_node)
+{
+ const char *parent_name;
+
+ if (other_node == &other_tree->root)
+ return &tree->root;
+
+ parent_name = dsync_mailbox_node_get_full_name(other_tree, other_node);
+ return sorted_tree_get(tree, parent_name);
+}
+
+static bool node_has_child(struct dsync_mailbox_node *parent, const char *name)
+{
+ struct dsync_mailbox_node *node;
+
+ for (node = parent->first_child; node != NULL; node = node->next) {
+ if (strcmp(node->name, name) == 0)
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static bool
+node_has_existent_children(struct dsync_mailbox_node *node, bool dirs_ok)
+{
+ for (node = node->first_child; node != NULL; node = node->next) {
+ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ (dirs_ok || !dsync_mailbox_node_is_dir(node)))
+ return TRUE;
+ if (node_has_existent_children(node, dirs_ok))
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static bool node_is_existent(struct dsync_mailbox_node *node)
+{
+ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS)
+ return TRUE;
+ return node_has_existent_children(node, TRUE);
+}
+
+static bool sync_node_is_namespace_prefix(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node)
+{
+ const char *full_name;
+ size_t prefix_len = node->ns == NULL ? 0 : node->ns->prefix_len;
+
+ if (strcmp(node->name, "INBOX") == 0 && node->parent == &tree->root)
+ return TRUE;
+
+ if (prefix_len == 0)
+ return FALSE;
+
+ full_name = dsync_mailbox_node_get_full_name(tree, node);
+ if (node->ns->prefix[prefix_len-1] == mail_namespace_get_sep(node->ns))
+ prefix_len--;
+ return strncmp(full_name, node->ns->prefix, prefix_len) == 0 &&
+ full_name[prefix_len] == '\0';
+}
+
+static void
+sync_rename_node_to_temp(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node *new_parent,
+ const char **reason_r)
+{
+ struct dsync_mailbox_tree_sync_change *change;
+ const char *old_name, *new_name, *p;
+ char name[TEMP_MAX_NAME_LEN+1];
+ buffer_t buf;
+ size_t prefix_len, max_prefix_len;
+ unsigned int counter = 1;
+
+ i_assert(!sync_node_is_namespace_prefix(tree, node));
+
+ buffer_create_from_data(&buf, name, sizeof(name));
+ max_prefix_len = TEMP_MAX_NAME_LEN - TEMP_SUFFIX_MAX_LEN - 1;
+ if (node->sync_temporary_name) {
+ /* the source name was also a temporary name. drop the
+ -<suffix> from it */
+ p = strrchr(node->name, '-');
+ i_assert(p != NULL);
+ if (max_prefix_len > (size_t)(p - node->name))
+ max_prefix_len = p - node->name;
+ }
+ str_append_max(&buf, node->name, max_prefix_len);
+ str_append_c(&buf, '-');
+ prefix_len = buf.used;
+
+ do {
+ str_truncate(&buf, prefix_len);
+ str_printfa(&buf, TEMP_SUFFIX_FORMAT, counter++);
+ /* the generated name is quite unlikely to exist,
+ but check anyway.. */
+ } while (node_has_child(node->parent, str_c(&buf)));
+
+ old_name = tree != ctx->local_tree ? NULL :
+ dsync_mailbox_node_get_full_name(tree, node);
+
+ *reason_r = t_strdup_printf("Renamed '%s' to '%s'", node->name, str_c(&buf));
+ node->name = p_strdup(tree->pool, str_c(&buf));
+ node->sync_temporary_name = TRUE;
+ node->last_renamed_or_created = 0;
+ dsync_mailbox_tree_node_move_sorted(node, new_parent);
+
+ if (tree == ctx->local_tree && node_is_existent(node)) {
+ /* we're modifying a local tree. remember this change. */
+ new_name = dsync_mailbox_node_get_full_name(tree, node);
+
+ i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+ i_assert(strcmp(old_name, "INBOX") != 0);
+ change = array_append_space(&ctx->changes);
+ change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME;
+ change->ns = node->ns;
+ change->full_name = p_strdup(ctx->pool, old_name);
+ change->rename_dest_name = p_strdup(ctx->pool, new_name);
+ }
+}
+
+static bool node_has_parent(struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node *parent)
+{
+ for (; node != NULL; node = node->parent) {
+ if (node == parent)
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void
+sync_rename_node(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *temp_node,
+ struct dsync_mailbox_node *node,
+ const struct dsync_mailbox_node *other_node,
+ const char **reason_r)
+{
+ struct dsync_mailbox_tree_sync_change *change;
+ struct dsync_mailbox_tree *other_tree;
+ struct dsync_mailbox_node *parent;
+ const char *name, *other_name;
+
+ i_assert(node != NULL);
+ i_assert(other_node != NULL);
+
+ /* move/rename node in the tree, so that its position/name is identical
+ to other_node (in other_tree). temp_node's name is changed to
+ temporary name (i.e. it assumes that node's name becomes temp_node's
+ original name) */
+ other_tree = tree == ctx->local_tree ?
+ ctx->remote_tree : ctx->local_tree;
+
+ parent = sorted_tree_get_by_node_name(tree, other_tree,
+ other_node->parent);
+ if (node_has_parent(parent, node)) {
+ /* don't introduce a loop. temporarily rename node
+ under root. */
+ sync_rename_node_to_temp(ctx, tree, node, &tree->root, reason_r);
+ *reason_r = t_strconcat(*reason_r, " (Don't introduce loop)", NULL);
+ return;
+ }
+ sync_rename_node_to_temp(ctx, tree, temp_node, temp_node->parent, reason_r);
+
+ /* get the old name before it's modified */
+ name = dsync_mailbox_node_get_full_name(tree, node);
+
+ /* set the new name */
+ *reason_r = t_strdup_printf("%s + Renamed '%s' to '%s'",
+ *reason_r, name, other_node->name);
+ node->name = p_strdup(tree->pool, other_node->name);
+ node->sync_temporary_name = other_node->sync_temporary_name;
+ node->last_renamed_or_created = other_node->last_renamed_or_created;
+ /* change node's parent if necessary. in any case detach+reattach it
+ sorted, because the nodes must be sorted by name, and the node's
+ name (or its parent) changed. */
+ dsync_mailbox_tree_node_move_sorted(node, parent);
+
+ if (tree == ctx->local_tree && node_is_existent(node)) {
+ /* we're modifying a local tree. remember this change. */
+ other_name = dsync_mailbox_node_get_full_name(other_tree, other_node);
+
+ i_assert(ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL);
+ i_assert(strcmp(name, "INBOX") != 0);
+ change = array_append_space(&ctx->changes);
+ change->type = DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME;
+ change->ns = node->ns;
+ change->full_name = p_strdup(ctx->pool, name);
+ change->rename_dest_name = p_strdup(ctx->pool, other_name);
+ }
+}
+
+static int node_mailbox_guids_cmp(struct dsync_mailbox_node *node1,
+ struct dsync_mailbox_node *node2)
+{
+ int ret;
+
+ while (node1 != NULL && node2 != NULL) {
+ if (node1->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ node2->existence != DSYNC_MAILBOX_NODE_EXISTS)
+ return -1;
+ if (node2->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ node1->existence != DSYNC_MAILBOX_NODE_EXISTS)
+ return 1;
+
+ ret = memcmp(node1->mailbox_guid, node2->mailbox_guid,
+ sizeof(node1->mailbox_guid));
+ if (ret != 0)
+ return ret;
+
+ ret = node_mailbox_guids_cmp(node1->first_child,
+ node2->first_child);
+ if (ret != 0)
+ return ret;
+ node1 = node1->next;
+ node2 = node2->next;
+ }
+ if (node1 == NULL && node2 == NULL)
+ return 0;
+ return node1 != NULL ? -1 : 1;
+}
+
+static int node_mailbox_names_cmp(struct dsync_mailbox_node *node1,
+ struct dsync_mailbox_node *node2)
+{
+ int ret;
+
+ while (node1 != NULL && node2 != NULL) {
+ ret = strcmp(node1->name, node2->name);
+ if (ret != 0)
+ return ret;
+
+ ret = node_mailbox_names_cmp(node1->first_child,
+ node2->first_child);
+ if (ret != 0)
+ return ret;
+ node1 = node1->next;
+ node2 = node2->next;
+ }
+ if (node1 == NULL && node2 == NULL)
+ return 0;
+ return node1 != NULL ? -1 : 1;
+}
+
+static int node_mailbox_trees_cmp(struct dsync_mailbox_node *node1,
+ struct dsync_mailbox_node *node2)
+{
+ int ret;
+
+ ret = node_mailbox_guids_cmp(node1, node2);
+ if (ret == 0) {
+ /* only a directory name changed and all the timestamps
+ are equal. just pick the alphabetically smaller. */
+ ret = node_mailbox_names_cmp(node1, node2);
+ }
+ i_assert(ret != 0);
+ return ret;
+}
+
+static time_t nodes_get_timestamp(struct dsync_mailbox_node *node1,
+ struct dsync_mailbox_node *node2)
+{
+ time_t ts = 0;
+
+ /* avoid using temporary names in case all the timestamps are 0 */
+ if (node1 != NULL && !node1->sync_temporary_name)
+ ts = node1->last_renamed_or_created + 1;
+ if (node2 != NULL && !node2->sync_temporary_name &&
+ ts <= node2->last_renamed_or_created)
+ ts = node2->last_renamed_or_created + 1;
+ return ts;
+}
+
+static bool sync_node_is_namespace_root(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node)
+{
+ if (node == NULL)
+ return FALSE;
+ if (node == &tree->root)
+ return TRUE;
+ return sync_node_is_namespace_prefix(tree, node);
+}
+
+static bool ATTR_NULL(3, 4)
+sync_rename_lower_ts(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_node *local_node1,
+ struct dsync_mailbox_node *remote_node1,
+ struct dsync_mailbox_node *local_node2,
+ struct dsync_mailbox_node *remote_node2,
+ const char **reason_r)
+{
+ time_t local_ts, remote_ts;
+
+ /* We're scanning the tree at the position of local_node1
+ and remote_node2. They have identical names. We also know that
+ local_node1&remote_node1 and local_node2&remote_node2 are "the same"
+ either because their GUIDs or (in case of one being a directory)
+ their childrens' GUIDs match. We don't know where local_node2 or
+ remote_node1 are located in the mailbox tree, or if they exist
+ at all. Note that node1 and node2 may be the same node pointers. */
+ i_assert(strcmp(local_node1->name, remote_node2->name) == 0);
+
+ if (sync_node_is_namespace_root(ctx->remote_tree, remote_node1) ||
+ sync_node_is_namespace_root(ctx->remote_tree, remote_node2) ||
+ sync_node_is_namespace_root(ctx->local_tree, local_node1) ||
+ sync_node_is_namespace_root(ctx->local_tree, local_node2)) {
+ local_node1->sync_delayed_guid_change = TRUE;
+ remote_node2->sync_delayed_guid_change = TRUE;
+ *reason_r = "Can't rename namespace prefixes - will be merged later";
+ return FALSE;
+ }
+
+ local_ts = nodes_get_timestamp(local_node1, local_node2);
+ remote_ts = nodes_get_timestamp(remote_node1, remote_node2);
+
+ if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+ local_ts = remote_ts+1;
+ else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+ remote_ts = local_ts+1;
+
+ /* The algorithm must be deterministic regardless of the sync direction,
+ so in case the timestamps are equal we need to resort to looking at
+ the other data. We'll start by looking at the nodes' mailbox GUIDs,
+ but if both of them don't exist continue looking into their
+ children. */
+ if (local_ts > remote_ts ||
+ (local_ts == remote_ts &&
+ node_mailbox_trees_cmp(local_node1, remote_node2) < 0)) {
+ /* local nodes have a higher timestamp. we only want to do
+ renames where the destination parent is the current node's
+ (local_node1/remote_node2) parent. */
+
+ /* Numbers are GUIDs, letters are mailbox names:
+
+ local 1A <-name conflict-> remote 2A
+ local 2B <- potentially -> remote 1[BC]
+
+ Here we want to preserve the local 1A & 2B names: */
+ if (local_node2 == NULL) {
+ /* local : 1A
+ remote: 1B, 2A -> 2A-temp, 1A */
+ sync_rename_node(ctx, ctx->remote_tree, remote_node2,
+ remote_node1, local_node1, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(local: local_node2=NULL)", NULL);
+ return TRUE;
+ } else if (remote_node1 == remote_node2) {
+ /* FIXME: this fixes an infinite loop when in
+ rename2 test, think it through why :) */
+ *reason_r = "local: remote_node1=remote_node2";
+ } else if (remote_node1 != NULL) {
+ /* a) local_node1->parent == local_node2->parent
+
+ local : 1A, 2B
+ remote: 1B, 2A -> 2A-temp, 1A(, 2B)
+ remote: 1C, 2A -> 2B, 1A
+ remote: 1C, 2A, 3B -> 2A-temp, 1A(, 3B-temp, 2B)
+
+ b) local_node1->parent != local_node2->parent
+
+ local : 1X/A, 2Y/B
+ remote: 1Y/B, 2X/A -> 2X/A-temp, 1X/A(, 2Y/B)
+ remote: 1Z/C, 2X/A -> 2X/A-temp, 1X/A
+ remote: 1Z/C, 2X/A, 3Y/B -> 2X/A-temp, 1X/A
+
+ We can handle all of these more easily by simply
+ always renaming 2 to a temporary name and handling
+ it when we reach B handling. */
+ sync_rename_node(ctx, ctx->remote_tree, remote_node2,
+ remote_node1, local_node1, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(local: remote_node1=NULL)", NULL);
+ return TRUE;
+ } else if (node_has_parent(local_node1, local_node2) &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+ /* node2 is a parent of node1, but it should be
+ vice versa */
+ sync_rename_node_to_temp(ctx, ctx->local_tree,
+ local_node1, local_node2->parent, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(local: node2 parent of node1)", NULL);
+ return TRUE;
+ } else if (node_has_parent(local_node2, local_node1) &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+ /* node1 is a parent of node2, but it should be
+ vice versa */
+ sync_rename_node_to_temp(ctx, ctx->local_tree,
+ local_node2, local_node1->parent, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(local: node1 parent of node2)", NULL);
+ return TRUE;
+ } else if (local_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+ sync_rename_node_to_temp(ctx, ctx->remote_tree,
+ remote_node2, remote_node2->parent, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(local: local_node1 exists)", NULL);
+ return TRUE;
+ } else {
+ /* local : 1A, 2B
+ remote: 2A -> (2B)
+ remote: 2A, 3B -> (3B-temp, 2B) */
+ *reason_r = "local: unchanged";
+ }
+ } else {
+ /* remote nodes have a higher timestamp */
+ if (remote_node1 == NULL) {
+ sync_rename_node(ctx, ctx->local_tree, local_node1,
+ local_node2, remote_node2, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(remote: remote_node1=NULL)", NULL);
+ return TRUE;
+ } else if (local_node2 == local_node1) {
+ *reason_r = "remote: remote_node2=remote_node1";
+ } else if (local_node2 != NULL) {
+ sync_rename_node(ctx, ctx->local_tree, local_node1,
+ local_node2, remote_node2, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(remote: local_node2=NULL)", NULL);
+ return TRUE;
+ } else if (node_has_parent(remote_node1, remote_node2) &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+ sync_rename_node_to_temp(ctx, ctx->remote_tree,
+ remote_node1, remote_node2->parent, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(remote: node2 parent of node1)", NULL);
+ return TRUE;
+ } else if (node_has_parent(remote_node2, remote_node1) &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+ sync_rename_node_to_temp(ctx, ctx->remote_tree,
+ remote_node2, remote_node1->parent, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(remote: node1 parent of node2)", NULL);
+ return TRUE;
+ } else if (remote_node2->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+ sync_rename_node_to_temp(ctx, ctx->local_tree,
+ local_node1, local_node1->parent, reason_r);
+ *reason_r = t_strconcat(*reason_r, "(remote: remote_node2 exists)", NULL);
+ return TRUE;
+ } else {
+ *reason_r = "remote: unchanged";
+ }
+ }
+ return FALSE;
+}
+
+static bool sync_rename_conflict(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_node *local_node1,
+ struct dsync_mailbox_node *remote_node2,
+ const char **reason_r)
+{
+ struct dsync_mailbox_node *local_node2, *remote_node1;
+ const uint8_t *guid_p;
+ bool ret;
+
+ guid_p = local_node1->mailbox_guid;
+ remote_node1 = hash_table_lookup(ctx->remote_tree->guid_hash, guid_p);
+ guid_p = remote_node2->mailbox_guid;
+ local_node2 = hash_table_lookup(ctx->local_tree->guid_hash, guid_p);
+
+ if ((remote_node1 != NULL && remote_node1->existence == DSYNC_MAILBOX_NODE_EXISTS) ||
+ (local_node2 != NULL && local_node2->existence == DSYNC_MAILBOX_NODE_EXISTS)) {
+ /* conflicting name, rename the one with lower timestamp */
+ ret = sync_rename_lower_ts(ctx, local_node1, remote_node1,
+ local_node2, remote_node2, reason_r);
+ *reason_r = t_strconcat("conflicting name: ", *reason_r, NULL);
+ return ret;
+ } else if (dsync_mailbox_node_is_dir(local_node1) ||
+ dsync_mailbox_node_is_dir(remote_node2)) {
+ /* one of the nodes is a directory, and the other is a mailbox
+ that doesn't exist on the other side. there is no conflict,
+ we'll just need to create the mailbox later. */
+ *reason_r = "mailbox not selectable yet";
+ return FALSE;
+ } else {
+ /* both nodes are mailboxes that don't exist on the other side.
+ we'll merge these mailboxes together later and change their
+ GUIDs and UIDVALIDITYs to be the same */
+ local_node1->sync_delayed_guid_change = TRUE;
+ remote_node2->sync_delayed_guid_change = TRUE;
+ *reason_r = "GUIDs conflict - will be merged later";
+ return FALSE;
+ }
+}
+
+static struct dsync_mailbox_node *
+sync_find_branch(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_tree *other_tree,
+ struct dsync_mailbox_node *dir_node)
+{
+ struct dsync_mailbox_node *node, *other_node;
+ const uint8_t *guid_p;
+
+ for (node = dir_node->first_child; node != NULL; node = node->next) {
+ if (dsync_mailbox_node_is_dir(node)) {
+ other_node = sync_find_branch(tree, other_tree, node);
+ if (other_node != NULL)
+ return other_node;
+ } else {
+ guid_p = node->mailbox_guid;
+ other_node = hash_table_lookup(other_tree->guid_hash,
+ guid_p);
+ if (other_node != NULL)
+ return other_node->parent;
+ }
+ }
+ return NULL;
+}
+
+static bool sync_rename_directory(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_node *local_node1,
+ struct dsync_mailbox_node *remote_node2,
+ const char **reason_r)
+{
+ struct dsync_mailbox_node *remote_node1, *local_node2;
+
+ /* see if we can find matching mailbox branches based on the nodes'
+ child mailboxes (with GUIDs). we can then rename the entire branch.
+ don't try to do this for namespace prefixes though. */
+ remote_node1 = sync_find_branch(ctx->local_tree,
+ ctx->remote_tree, local_node1);
+ local_node2 = sync_find_branch(ctx->remote_tree, ctx->local_tree,
+ remote_node2);
+ if (remote_node1 == NULL || local_node2 == NULL) {
+ *reason_r = "Directory rename branch not found";
+ return FALSE;
+ }
+ if (node_names_equal(remote_node1, local_node2)) {
+ *reason_r = "Directory name paths are equal";
+ return FALSE;
+ }
+
+ return sync_rename_lower_ts(ctx, local_node1, remote_node1,
+ local_node2, remote_node2, reason_r);
+}
+
+static bool sync_rename_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_node *local_parent,
+ struct dsync_mailbox_node *remote_parent)
+{
+ struct dsync_mailbox_node **local_nodep = &local_parent->first_child;
+ struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child;
+ struct dsync_mailbox_node *local_node, *remote_node;
+ const char *reason;
+ string_t *debug = NULL;
+ bool changed;
+
+ if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0)
+ debug = t_str_new(128);
+
+ /* the nodes are sorted by their names. */
+ while (*local_nodep != NULL || *remote_nodep != NULL) {
+ local_node = *local_nodep;
+ remote_node = *remote_nodep;
+
+ if (local_node == NULL ||
+ (remote_node != NULL &&
+ strcmp(local_node->name, remote_node->name) > 0)) {
+ /* add a missing local node */
+ local_node = sync_node_new(ctx->local_tree, local_nodep,
+ local_parent, remote_node);
+ }
+ if (remote_node == NULL ||
+ strcmp(remote_node->name, local_node->name) > 0) {
+ /* add a missing remote node */
+ remote_node = sync_node_new(ctx->remote_tree, remote_nodep,
+ remote_parent, local_node);
+ }
+ i_assert(strcmp(local_node->name, remote_node->name) == 0);
+ if (debug != NULL) {
+ str_truncate(debug, 0);
+ str_append(debug, "Mailbox ");
+ dsync_mailbox_node_append_full_name(debug, ctx->local_tree, local_node);
+ str_printfa(debug, ": local=%s/%ld/%d, remote=%s/%ld/%d",
+ guid_128_to_string(local_node->mailbox_guid),
+ (long)local_node->last_renamed_or_created,
+ local_node->existence,
+ guid_128_to_string(remote_node->mailbox_guid),
+ (long)remote_node->last_renamed_or_created,
+ remote_node->existence);
+ }
+
+ if (dsync_mailbox_node_is_dir(local_node) &&
+ dsync_mailbox_node_is_dir(remote_node)) {
+ /* both nodes are directories (or other side is
+ nonexistent). see if we can match them by their
+ child mailboxes */
+ changed = sync_rename_directory(ctx, local_node, remote_node, &reason);
+ } else if (dsync_mailbox_node_guids_equal(local_node,
+ remote_node)) {
+ /* mailboxes are equal, no need to rename */
+ reason = "Mailboxes are equal";
+ changed = FALSE;
+ } else {
+ /* mailbox naming conflict */
+ changed = sync_rename_conflict(ctx, local_node,
+ remote_node, &reason);
+ }
+ /* handle children, if there are any */
+ if (debug != NULL) {
+ i_debug("brain %c: %s: %s",
+ (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+ str_c(debug), reason);
+ }
+
+ if (!changed) T_BEGIN {
+ changed = sync_rename_mailboxes(ctx, local_node,
+ remote_node);
+ } T_END;
+ if (changed)
+ return TRUE;
+
+ local_nodep = &local_node->next;
+ remote_nodep = &remote_node->next;
+ }
+ return FALSE;
+}
+
+static bool mailbox_node_hash_first_child(struct dsync_mailbox_node *node,
+ struct md5_context *md5)
+{
+ for (node = node->first_child; node != NULL; node = node->next) {
+ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS) {
+ md5_update(md5, node->mailbox_guid,
+ sizeof(node->mailbox_guid));
+ md5_update(md5, node->name, strlen(node->name));
+ return TRUE;
+ }
+ if (node->first_child != NULL) {
+ if (mailbox_node_hash_first_child(node, md5))
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static const char *
+mailbox_node_generate_suffix(struct dsync_mailbox_node *node)
+{
+ struct md5_context md5;
+ unsigned char digest[MD5_RESULTLEN];
+
+ if (!dsync_mailbox_node_is_dir(node))
+ return guid_128_to_string(node->mailbox_guid);
+
+ md5_init(&md5);
+ if (!mailbox_node_hash_first_child(node, &md5))
+ i_unreached(); /* we would have deleted it */
+ md5_final(&md5, digest);
+ return binary_to_hex(digest, sizeof(digest));
+}
+
+static void suffix_inc(string_t *str)
+{
+ char *data;
+ size_t i;
+
+ data = str_c_modifiable(str) + str_len(str)-1;
+ for (i = str_len(str); i > 0; i--, data--) {
+ if ((*data >= '0' && *data < '9') ||
+ (*data >= 'a' && *data < 'f')) {
+ *data += 1;
+ return;
+ } else if (*data == '9') {
+ *data = 'a';
+ return;
+ } else if (*data != 'f') {
+ i_unreached();
+ }
+ }
+ i_unreached();
+}
+
+static void
+sync_rename_temp_mailbox_node(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node,
+ const char **reason_r)
+{
+ const char *p, *new_suffix;
+ string_t *str = t_str_new(256);
+ size_t max_prefix_len;
+
+ /* The name is currently <oldname>-<temp>. Both sides need to
+ use equivalent names, so we'll replace the <temp> if possible
+ with a) mailbox GUID, b) sha1 of childrens' (GUID|name)s. In the
+ very unlikely case of such name already existing, just increase
+ the last letters until it's not found. */
+ new_suffix = mailbox_node_generate_suffix(node);
+
+ p = strrchr(node->name, '-');
+ i_assert(p != NULL);
+ p++;
+ max_prefix_len = TEMP_MAX_NAME_LEN - strlen(new_suffix) - 1;
+ if (max_prefix_len > (size_t)(p-node->name))
+ max_prefix_len = p-node->name;
+ str_append_max(str, node->name, max_prefix_len);
+ str_append(str, new_suffix);
+ while (node_has_child(node->parent, str_c(str)))
+ suffix_inc(str);
+
+ *reason_r = t_strdup_printf("Renamed '%s' to '%s'",
+ dsync_mailbox_node_get_full_name(tree, node), str_c(str));
+ node->name = p_strdup(tree->pool, str_c(str));
+
+ dsync_mailbox_tree_node_move_sorted(node, node->parent);
+ node->sync_temporary_name = FALSE;
+}
+
+static void
+sync_rename_delete_node_dirs(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node)
+{
+ struct dsync_mailbox_node *child;
+
+ for (child = node->first_child; child != NULL; child = child->next)
+ sync_rename_delete_node_dirs(ctx, tree, child);
+
+ if (tree == ctx->local_tree &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL &&
+ node->existence != DSYNC_MAILBOX_NODE_NONEXISTENT) {
+ sync_add_dir_change(ctx, node,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR);
+ }
+ node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+ node->sync_temporary_name = FALSE;
+}
+
+static bool
+sync_rename_temp_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node, bool *renames_r)
+{
+ const char *reason;
+
+ for (; node != NULL; node = node->next) {
+ while (sync_rename_temp_mailboxes(ctx, tree, node->first_child, renames_r)) ;
+
+ if (!node->sync_temporary_name) {
+ } else if (dsync_mailbox_node_is_dir(node) &&
+ (node->first_child == NULL ||
+ !node_has_existent_children(node, FALSE))) {
+ /* we can just delete this directory and
+ any child directories it may have */
+ if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) {
+ i_debug("brain %c: %s mailbox %s: Delete directory-only tree",
+ (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+ tree == ctx->local_tree ? "local" : "remote",
+ dsync_mailbox_node_get_full_name(tree, node));
+ }
+ sync_rename_delete_node_dirs(ctx, tree, node);
+ } else {
+ T_BEGIN {
+ *renames_r = TRUE;
+ sync_rename_temp_mailbox_node(tree, node, &reason);
+ if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0) {
+ i_debug("brain %c: %s mailbox %s: %s",
+ (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S',
+ tree == ctx->local_tree ? "local" : "remote",
+ dsync_mailbox_node_get_full_name(tree, node), reason);
+ }
+ } T_END;
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static int
+dsync_mailbox_tree_handle_renames(struct dsync_mailbox_tree_sync_ctx *ctx,
+ bool *renames_r)
+{
+ unsigned int max_renames, count = 0;
+ bool changed;
+
+ *renames_r = FALSE;
+
+ max_renames = ctx->combined_mailboxes_count * 3;
+ do {
+ T_BEGIN {
+ changed = sync_rename_mailboxes(ctx, &ctx->local_tree->root,
+ &ctx->remote_tree->root);
+ } T_END;
+ if ((ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG) != 0 &&
+ changed) {
+ i_debug("brain %c: -- Mailbox renamed, restart sync --",
+ (ctx->sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN) != 0 ? 'M' : 'S');
+ }
+ } while (changed && ++count <= max_renames);
+
+ if (changed) {
+ i_error("BUG: Mailbox renaming algorithm got into a potentially infinite loop, aborting");
+ return -1;
+ }
+
+ while (sync_rename_temp_mailboxes(ctx, ctx->local_tree, &ctx->local_tree->root, renames_r)) ;
+ while (sync_rename_temp_mailboxes(ctx, ctx->remote_tree, &ctx->remote_tree->root, renames_r)) ;
+ return 0;
+}
+
+static bool sync_is_wrong_mailbox(struct dsync_mailbox_node *node,
+ const struct dsync_mailbox_node *wanted_node,
+ const char **reason_r)
+{
+ if (wanted_node->existence != DSYNC_MAILBOX_NODE_EXISTS) {
+ *reason_r = wanted_node->existence == DSYNC_MAILBOX_NODE_DELETED ?
+ "Mailbox has been deleted" : "Mailbox doesn't exist";
+ return TRUE;
+ }
+ if (node->uid_validity != wanted_node->uid_validity) {
+ *reason_r = t_strdup_printf("UIDVALIDITY changed (%u -> %u)",
+ wanted_node->uid_validity,
+ node->uid_validity);
+ return TRUE;
+ }
+ if (node->uid_next > wanted_node->uid_next) {
+ /* we can't lower the UIDNEXT */
+ *reason_r = t_strdup_printf("UIDNEXT is too high (%u > %u)",
+ node->uid_next,
+ wanted_node->uid_next);
+ return TRUE;
+ }
+ if (memcmp(node->mailbox_guid, wanted_node->mailbox_guid,
+ sizeof(node->mailbox_guid)) != 0) {
+ *reason_r = "GUID changed";
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void
+sync_delete_wrong_mailboxes_branch(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_tree *wanted_tree,
+ struct dsync_mailbox_node *node,
+ const struct dsync_mailbox_node *wanted_node)
+{
+ const char *reason;
+ int ret;
+
+ while (node != NULL && wanted_node != NULL) {
+ if (node->first_child != NULL) {
+ sync_delete_wrong_mailboxes_branch(ctx,
+ tree, wanted_tree,
+ node->first_child, wanted_node->first_child);
+ }
+ ret = strcmp(node->name, wanted_node->name);
+ if (ret < 0) {
+ /* node shouldn't exist */
+ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ !dsync_mailbox_node_is_dir(node)) {
+ sync_delete_mailbox_node(ctx, tree, node,
+ "Mailbox doesn't exist");
+ }
+ node = node->next;
+ } else if (ret > 0) {
+ /* wanted_node doesn't exist. it's created later. */
+ wanted_node = wanted_node->next;
+ } else T_BEGIN {
+ if (sync_is_wrong_mailbox(node, wanted_node, &reason) &&
+ node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ !dsync_mailbox_node_is_dir(node))
+ sync_delete_mailbox_node(ctx, tree, node, reason);
+ node = node->next;
+ wanted_node = wanted_node->next;
+ } T_END;
+ }
+ for (; node != NULL; node = node->next) {
+ /* node and its children shouldn't exist */
+ if (node->first_child != NULL) {
+ sync_delete_wrong_mailboxes_branch(ctx,
+ tree, wanted_tree,
+ node->first_child, NULL);
+ }
+ if (node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ !dsync_mailbox_node_is_dir(node))
+ sync_delete_mailbox_node(ctx, tree, node, "Mailbox doesn't exist");
+ }
+}
+
+static void
+sync_delete_wrong_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_tree *wanted_tree)
+{
+ sync_delete_wrong_mailboxes_branch(ctx, tree, wanted_tree,
+ tree->root.first_child,
+ wanted_tree->root.first_child);
+}
+
+static void sync_create_mailboxes(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_tree *tree)
+{
+ struct dsync_mailbox_tree *other_tree;
+ struct dsync_mailbox_tree_iter *iter;
+ struct dsync_mailbox_node *node, *other_node;
+ const char *name;
+ const uint8_t *guid_p;
+
+ other_tree = tree == ctx->local_tree ?
+ ctx->remote_tree : ctx->local_tree;
+
+ iter = dsync_mailbox_tree_iter_init(tree);
+ while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+ /* make sure the renaming handled everything */
+ i_assert(!node->sync_temporary_name);
+ if (dsync_mailbox_node_is_dir(node))
+ continue;
+
+ i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS);
+
+ guid_p = node->mailbox_guid;
+ other_node = hash_table_lookup(other_tree->guid_hash, guid_p);
+ if (other_node == NULL)
+ other_node = sorted_tree_get(other_tree, name);
+ if (dsync_mailbox_node_is_dir(other_node)) {
+ /* create a missing mailbox */
+ other_node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ other_node->ns = node->ns;
+ other_node->uid_validity = node->uid_validity;
+ memcpy(other_node->mailbox_guid, node->mailbox_guid,
+ sizeof(other_node->mailbox_guid));
+ if (other_tree == ctx->local_tree)
+ sync_add_create_change(ctx, other_node, name);
+ } else if (!guid_128_equals(node->mailbox_guid,
+ other_node->mailbox_guid)) {
+ /* mailbox with same name exists both locally and
+ remotely, but they have different GUIDs and neither
+ side has the other's GUID. typically this means that
+ both sides had autocreated some mailboxes (e.g.
+ INBOX). we'll just change the GUID for one of
+ them. */
+ i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS);
+ i_assert(node->ns == other_node->ns);
+
+ if (other_tree == ctx->local_tree)
+ sync_add_create_change(ctx, node, name);
+ } else {
+ /* existing mailbox. mismatching UIDVALIDITY is handled
+ later while syncing the mailbox. */
+ i_assert(node->existence == DSYNC_MAILBOX_NODE_EXISTS);
+ i_assert(node->ns == other_node->ns);
+ }
+ }
+ dsync_mailbox_tree_iter_deinit(&iter);
+}
+
+static void
+sync_subscription(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_node *local_node,
+ struct dsync_mailbox_node *remote_node)
+{
+ bool use_local;
+
+ if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+ use_local = TRUE;
+ else if (ctx->sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+ use_local = FALSE;
+ else if (local_node->last_subscription_change > remote_node->last_subscription_change)
+ use_local = TRUE;
+ else if (local_node->last_subscription_change < remote_node->last_subscription_change)
+ use_local = FALSE;
+ else {
+ /* local and remote have equal timestamps. prefer to subscribe
+ rather than unsubscribe. */
+ use_local = local_node->subscribed;
+ }
+ if (use_local) {
+ /* use local subscription state */
+ remote_node->subscribed = local_node->subscribed;
+ } else {
+ /* use remote subscription state */
+ local_node->subscribed = remote_node->subscribed;
+ sync_add_dir_change(ctx, local_node, local_node->subscribed ?
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE :
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE);
+ }
+}
+
+static void sync_mailbox_child_dirs(struct dsync_mailbox_tree_sync_ctx *ctx,
+ struct dsync_mailbox_node *local_parent,
+ struct dsync_mailbox_node *remote_parent)
+{
+ struct dsync_mailbox_node **local_nodep = &local_parent->first_child;
+ struct dsync_mailbox_node **remote_nodep = &remote_parent->first_child;
+ struct dsync_mailbox_node *local_node, *remote_node;
+ int ret;
+
+ /* NOTE: the nodes are always sorted. renaming created all of the
+ interesting nodes, but it may have left some extra nonexistent nodes
+ lying around, which we will delete. */
+ while (*local_nodep != NULL && *remote_nodep != NULL) {
+ local_node = *local_nodep;
+ remote_node = *remote_nodep;
+
+ ret = strcmp(local_node->name, remote_node->name);
+ if (ret < 0) {
+ i_assert(!node_is_existent(local_node));
+ *local_nodep = local_node->next;
+ continue;
+ }
+ if (ret > 0) {
+ i_assert(!node_is_existent(remote_node));
+ *remote_nodep = remote_node->next;
+ continue;
+ }
+
+ if (local_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ remote_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+ /* create to remote */
+ remote_node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ }
+ if (remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ local_node->existence == DSYNC_MAILBOX_NODE_NONEXISTENT &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+ /* create to local */
+ local_node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ sync_add_dir_change(ctx, local_node,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR);
+ }
+
+ /* create/delete child dirs */
+ sync_mailbox_child_dirs(ctx, local_node, remote_node);
+
+ if (local_node->subscribed != remote_node->subscribed)
+ sync_subscription(ctx, local_node, remote_node);
+
+ if (local_node->existence == DSYNC_MAILBOX_NODE_DELETED &&
+ !node_has_existent_children(local_node, TRUE) &&
+ remote_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE) {
+ /* delete from remote */
+ i_assert(!node_has_existent_children(remote_node, TRUE));
+ remote_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+ }
+ if (remote_node->existence == DSYNC_MAILBOX_NODE_DELETED &&
+ !node_has_existent_children(remote_node, TRUE) &&
+ local_node->existence == DSYNC_MAILBOX_NODE_EXISTS &&
+ ctx->sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL) {
+ /* delete from local */
+ i_assert(!node_has_existent_children(local_node, TRUE));
+ local_node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+ sync_add_dir_change(ctx, local_node,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR);
+ }
+
+ local_nodep = &local_node->next;
+ remote_nodep = &remote_node->next;
+ }
+ while (*local_nodep != NULL) {
+ i_assert(!node_is_existent(*local_nodep));
+ *local_nodep = (*local_nodep)->next;
+ }
+ while (*remote_nodep != NULL) {
+ i_assert(!node_is_existent(*remote_nodep));
+ *remote_nodep = (*remote_nodep)->next;
+ }
+}
+
+static void sync_mailbox_dirs(struct dsync_mailbox_tree_sync_ctx *ctx)
+{
+ sync_mailbox_child_dirs(ctx, &ctx->local_tree->root,
+ &ctx->remote_tree->root);
+}
+
+static void
+dsync_mailbox_tree_update_child_timestamps(struct dsync_mailbox_node *node,
+ time_t parent_timestamp)
+{
+ time_t ts;
+
+ if (node->last_renamed_or_created < parent_timestamp)
+ node->last_renamed_or_created = parent_timestamp;
+ ts = node->last_renamed_or_created;
+
+ for (node = node->first_child; node != NULL; node = node->next)
+ dsync_mailbox_tree_update_child_timestamps(node, ts);
+}
+
+struct dsync_mailbox_tree_sync_ctx *
+dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
+ struct dsync_mailbox_tree *remote_tree,
+ enum dsync_mailbox_trees_sync_type sync_type,
+ enum dsync_mailbox_trees_sync_flags sync_flags)
+{
+ struct dsync_mailbox_tree_sync_ctx *ctx;
+ unsigned int rename_counter = 0;
+ bool renames;
+ pool_t pool;
+
+ i_assert(hash_table_is_created(local_tree->guid_hash));
+ i_assert(hash_table_is_created(remote_tree->guid_hash));
+
+ pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox trees sync",
+ 1024*64);
+ ctx = p_new(pool, struct dsync_mailbox_tree_sync_ctx, 1);
+ ctx->pool = pool;
+ ctx->local_tree = local_tree;
+ ctx->remote_tree = remote_tree;
+ ctx->sync_type = sync_type;
+ ctx->sync_flags = sync_flags;
+ i_array_init(&ctx->changes, 128);
+
+again:
+ renames = FALSE;
+ ctx->combined_mailboxes_count = 0;
+ sync_tree_sort_and_delete_mailboxes(ctx, remote_tree,
+ sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY);
+ sync_tree_sort_and_delete_mailboxes(ctx, local_tree,
+ sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY);
+
+ dsync_mailbox_tree_update_child_timestamps(&local_tree->root, 0);
+ dsync_mailbox_tree_update_child_timestamps(&remote_tree->root, 0);
+ if ((sync_flags & DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES) == 0) {
+ if (dsync_mailbox_tree_handle_renames(ctx, &renames) < 0) {
+ ctx->failed = TRUE;
+ return ctx;
+ }
+ }
+
+ /* if we're not doing a two-way sync, delete now any mailboxes, which
+ a) shouldn't exist, b) doesn't have a matching GUID/UIDVALIDITY,
+ c) has a too high UIDNEXT */
+ if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+ sync_delete_wrong_mailboxes(ctx, remote_tree, local_tree);
+ else if (sync_type == DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+ sync_delete_wrong_mailboxes(ctx, local_tree, remote_tree);
+
+ if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL)
+ sync_create_mailboxes(ctx, remote_tree);
+ if (sync_type != DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE)
+ sync_create_mailboxes(ctx, local_tree);
+ if (renames && rename_counter++ <= ctx->combined_mailboxes_count*3) {
+ /* this rename algorithm is just horrible. we're retrying this
+ because the final sync_rename_temp_mailbox_node() calls
+ give different names to local & remote mailbox trees.
+ something's not right here, but this looping is better than
+ a crash in sync_mailbox_dirs() due to trees not matching. */
+ goto again;
+ }
+ sync_mailbox_dirs(ctx);
+ return ctx;
+}
+
+const struct dsync_mailbox_tree_sync_change *
+dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx)
+{
+ if (ctx->change_idx == array_count(&ctx->changes))
+ return NULL;
+ return array_idx(&ctx->changes, ctx->change_idx++);
+}
+
+int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **_ctx)
+{
+ struct dsync_mailbox_tree_sync_ctx *ctx = *_ctx;
+ int ret = ctx->failed ? -1 : 0;
+
+ *_ctx = NULL;
+
+ array_free(&ctx->changes);
+ pool_unref(&ctx->pool);
+ return ret;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-tree.c b/src/doveadm/dsync/dsync-mailbox-tree.c
new file mode 100644
index 0000000..e34f335
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree.c
@@ -0,0 +1,554 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "hash.h"
+#include "str.h"
+#include "sort.h"
+#include "mailbox-list-private.h"
+#include "dsync-mailbox-tree-private.h"
+
+
+struct dsync_mailbox_tree_iter {
+ struct dsync_mailbox_tree *tree;
+
+ struct dsync_mailbox_node *cur;
+ string_t *name;
+};
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_init(char sep, char escape_char, char alt_char)
+{
+ struct dsync_mailbox_tree *tree;
+ pool_t pool;
+
+ i_assert(sep != '\0');
+ i_assert(alt_char != '\0');
+
+ pool = pool_alloconly_create(MEMPOOL_GROWING"dsync mailbox tree", 4096);
+ tree = p_new(pool, struct dsync_mailbox_tree, 1);
+ tree->pool = pool;
+ tree->sep = tree->sep_str[0] = sep;
+ tree->escape_char = escape_char;
+ tree->alt_char = alt_char;
+ tree->root.name = "";
+ i_array_init(&tree->deletes, 32);
+ return tree;
+}
+
+void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **_tree)
+{
+ struct dsync_mailbox_tree *tree = *_tree;
+
+ *_tree = NULL;
+ hash_table_destroy(&tree->name128_hash);
+ hash_table_destroy(&tree->guid_hash);
+ array_free(&tree->deletes);
+ pool_unref(&tree->pool);
+}
+
+static struct dsync_mailbox_node *
+dsync_mailbox_node_find(struct dsync_mailbox_node *nodes, const char *name)
+{
+ for (; nodes != NULL; nodes = nodes->next) {
+ if (strcmp(name, nodes->name) == 0)
+ return nodes;
+ }
+ return NULL;
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree,
+ const char *full_name)
+{
+ struct dsync_mailbox_node *node = &tree->root;
+
+ T_BEGIN {
+ const char *const *path;
+
+ path = t_strsplit(full_name, tree->sep_str);
+ for (; *path != NULL && node != NULL; path++)
+ node = dsync_mailbox_node_find(node->first_child, *path);
+ } T_END;
+ return node;
+}
+
+void dsync_mailbox_tree_node_attach(struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node *parent)
+{
+ node->parent = parent;
+ node->next = parent->first_child;
+ parent->first_child = node;
+}
+
+void dsync_mailbox_tree_node_detach(struct dsync_mailbox_node *node)
+{
+ struct dsync_mailbox_node **p;
+
+ for (p = &node->parent->first_child;; p = &(*p)->next) {
+ if (*p == node) {
+ *p = node->next;
+ break;
+ }
+ }
+ node->parent = NULL;
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name)
+{
+ struct dsync_mailbox_node *parent = NULL, *node = &tree->root;
+
+ i_assert(tree->iter_count == 0);
+
+ T_BEGIN {
+ const char *const *path;
+
+ /* find the existing part */
+ path = t_strsplit(full_name, tree->sep_str);
+ for (; *path != NULL; path++) {
+ parent = node;
+ node = dsync_mailbox_node_find(node->first_child, *path);
+ if (node == NULL)
+ break;
+ }
+ /* create the rest */
+ for (; *path != NULL; path++) {
+ node = p_new(tree->pool, struct dsync_mailbox_node, 1);
+ node->name = p_strdup(tree->pool, *path);
+ node->ns = parent->ns;
+ dsync_mailbox_tree_node_attach(node, parent);
+ parent = node;
+ }
+ } T_END;
+ return node;
+}
+
+static void
+node_get_full_name_recurse(const struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_node *node, string_t *str)
+{
+ if (node->parent != &tree->root)
+ node_get_full_name_recurse(tree, node->parent, str);
+
+ str_append(str, node->name);
+ str_append_c(str, tree->sep);
+}
+
+const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_node *node)
+{
+ string_t *str = t_str_new(128);
+ dsync_mailbox_node_append_full_name(str, tree, node);
+ return str_c(str);
+}
+
+void dsync_mailbox_node_append_full_name(string_t *str,
+ const struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_node *node)
+{
+ i_assert(node->parent != NULL);
+
+ node_get_full_name_recurse(tree, node, str);
+ /* remove the trailing separator */
+ str_truncate(str, str_len(str)-1);
+}
+
+void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest,
+ const struct dsync_mailbox_node *src)
+{
+ memcpy(dest->mailbox_guid, src->mailbox_guid,
+ sizeof(dest->mailbox_guid));
+ dest->uid_validity = src->uid_validity;
+ dest->uid_next = src->uid_next;
+ dest->existence = src->existence;
+ dest->last_renamed_or_created = src->last_renamed_or_created;
+ dest->subscribed = src->subscribed;
+ dest->last_subscription_change = src->last_subscription_change;
+}
+
+struct dsync_mailbox_tree_iter *
+dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree)
+{
+ struct dsync_mailbox_tree_iter *iter;
+
+ iter = i_new(struct dsync_mailbox_tree_iter, 1);
+ iter->tree = tree;
+ iter->name = str_new(default_pool, 128);
+ iter->cur = &tree->root;
+
+ tree->iter_count++;
+ return iter;
+}
+
+static size_t node_get_full_name_length(struct dsync_mailbox_node *node)
+{
+ if (node->parent->parent == NULL)
+ return strlen(node->name);
+ else {
+ return strlen(node->name) + 1 +
+ node_get_full_name_length(node->parent);
+ }
+}
+
+bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter,
+ const char **full_name_r,
+ struct dsync_mailbox_node **node_r)
+{
+ size_t len;
+
+ if (iter->cur->first_child != NULL)
+ iter->cur = iter->cur->first_child;
+ else {
+ while (iter->cur->next == NULL) {
+ if (iter->cur == &iter->tree->root)
+ return FALSE;
+ iter->cur = iter->cur->parent;
+ }
+ iter->cur = iter->cur->next;
+ len = iter->cur->parent == &iter->tree->root ? 0 :
+ node_get_full_name_length(iter->cur->parent);
+ str_truncate(iter->name, len);
+ }
+ if (str_len(iter->name) > 0)
+ str_append_c(iter->name, iter->tree->sep);
+ str_append(iter->name, iter->cur->name);
+ *full_name_r = str_c(iter->name);
+ *node_r = iter->cur;
+ return TRUE;
+}
+
+void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **_iter)
+{
+ struct dsync_mailbox_tree_iter *iter = *_iter;
+
+ *_iter = NULL;
+
+ i_assert(iter->tree->iter_count > 0);
+ iter->tree->iter_count--;
+
+ str_free(&iter->name);
+ i_free(iter);
+}
+
+void dsync_mailbox_tree_build_name128_hash(struct dsync_mailbox_tree *tree)
+{
+ struct dsync_mailbox_tree_iter *iter;
+ struct dsync_mailbox_node *node;
+ const char *name;
+ guid_128_t *sha128;
+ uint8_t *guid_p;
+
+ i_assert(!hash_table_is_created(tree->name128_hash));
+
+ hash_table_create(&tree->name128_hash,
+ tree->pool, 0, guid_128_hash, guid_128_cmp);
+ iter = dsync_mailbox_tree_iter_init(tree);
+ while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+ sha128 = p_new(tree->pool, guid_128_t, 1);
+ mailbox_name_get_sha128(name, *sha128);
+ guid_p = *sha128;
+ hash_table_insert(tree->name128_hash, guid_p, node);
+ }
+ dsync_mailbox_tree_iter_deinit(&iter);
+}
+
+static const char *
+convert_name_to_remote_sep(struct dsync_mailbox_tree *tree, const char *name)
+{
+ string_t *str = t_str_new(128);
+
+ char remote_escape_chars[3] = {
+ tree->remote_escape_char,
+ tree->remote_sep,
+ '\0'
+ };
+
+ for (;;) {
+ const char *end = strchr(name, tree->sep);
+ const char *name_part = end == NULL ? name :
+ t_strdup_until(name, end++);
+
+ if (tree->escape_char != '\0')
+ mailbox_list_name_unescape(&name_part, tree->escape_char);
+ if (remote_escape_chars[0] != '\0') {
+ /* The local name can be fully escaped to remote
+ name and back. */
+ mailbox_list_name_escape(name_part, remote_escape_chars,
+ str);
+ } else {
+ /* There is no remote escape char, so for conflicting
+ separator use the alt_char. */
+ for (; *name_part != '\0'; name_part++) {
+ if (*name_part == tree->remote_sep)
+ str_append_c(str, tree->alt_char);
+ else
+ str_append_c(str, *name_part);
+ }
+ }
+ if (end == NULL)
+ break;
+ str_append_c(str, tree->remote_sep);
+ }
+ return str_c(str);
+}
+
+static void
+dsync_mailbox_tree_build_name128_remotesep_hash(struct dsync_mailbox_tree *tree)
+{
+ struct dsync_mailbox_tree_iter *iter;
+ struct dsync_mailbox_node *node;
+ const char *name;
+ guid_128_t *sha128;
+ uint8_t *guid_p;
+
+ i_assert(tree->sep != tree->remote_sep);
+ i_assert(!hash_table_is_created(tree->name128_remotesep_hash));
+
+ hash_table_create(&tree->name128_remotesep_hash, tree->pool, 0,
+ guid_128_hash, guid_128_cmp);
+ iter = dsync_mailbox_tree_iter_init(tree);
+ while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+ sha128 = p_new(tree->pool, guid_128_t, 1);
+ T_BEGIN {
+ const char *remote_name =
+ convert_name_to_remote_sep(tree, name);
+ mailbox_name_get_sha128(remote_name, *sha128);
+ } T_END;
+ guid_p = *sha128;
+ hash_table_insert(tree->name128_remotesep_hash, guid_p, node);
+ }
+ dsync_mailbox_tree_iter_deinit(&iter);
+}
+
+int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node **old_node_r)
+{
+ struct dsync_mailbox_node *old_node;
+ uint8_t *guid = node->mailbox_guid;
+
+ if (guid_128_is_empty(node->mailbox_guid))
+ return 0;
+
+ *old_node_r = old_node = hash_table_lookup(tree->guid_hash, guid);
+ if (old_node == NULL)
+ hash_table_insert(tree->guid_hash, guid, node);
+ else if (old_node != node)
+ return -1;
+ return 0;
+}
+
+int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node **dup_node1_r,
+ struct dsync_mailbox_node **dup_node2_r)
+{
+ struct dsync_mailbox_tree_iter *iter;
+ struct dsync_mailbox_node *node, *old_node;
+ const char *name;
+ int ret = 0;
+
+ if (!hash_table_is_created(tree->guid_hash)) {
+ hash_table_create(&tree->guid_hash, tree->pool, 0,
+ guid_128_hash, guid_128_cmp);
+ }
+ iter = dsync_mailbox_tree_iter_init(tree);
+ while (dsync_mailbox_tree_iter_next(iter, &name, &node)) {
+ if (dsync_mailbox_tree_guid_hash_add(tree, node, &old_node) < 0) {
+ *dup_node1_r = node;
+ *dup_node2_r = old_node;
+ ret = -1;
+ }
+ }
+ dsync_mailbox_tree_iter_deinit(&iter);
+ return ret;
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree,
+ const guid_128_t guid)
+{
+ const uint8_t *guid_p = guid;
+
+ return hash_table_lookup(tree->guid_hash, guid_p);
+}
+
+const struct dsync_mailbox_delete *
+dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree,
+ unsigned int *count_r)
+{
+ return array_get(&tree->deletes, count_r);
+}
+
+struct dsync_mailbox_node *
+dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_delete *del)
+{
+ const uint8_t *guid_p = del->guid;
+
+ i_assert(hash_table_is_created(tree->guid_hash));
+ i_assert(tree->remote_sep != '\0');
+
+ if (del->type == DSYNC_MAILBOX_DELETE_TYPE_MAILBOX) {
+ /* find node by GUID */
+ return hash_table_lookup(tree->guid_hash, guid_p);
+ }
+
+ /* find node by name. this is a bit tricky, since the hierarchy
+ separator may differ from ours. */
+ if (tree->sep == tree->remote_sep) {
+ if (!hash_table_is_created(tree->name128_hash))
+ dsync_mailbox_tree_build_name128_hash(tree);
+ return hash_table_lookup(tree->name128_hash, guid_p);
+ } else {
+ if (!hash_table_is_created(tree->name128_remotesep_hash))
+ dsync_mailbox_tree_build_name128_remotesep_hash(tree);
+ return hash_table_lookup(tree->name128_remotesep_hash, guid_p);
+ }
+}
+
+void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree,
+ char remote_sep, char escape_char)
+{
+ i_assert(tree->remote_sep == '\0');
+ i_assert(tree->remote_escape_char == '\0');
+
+ tree->remote_sep = remote_sep;
+ tree->remote_escape_char = escape_char;
+}
+
+static void
+dsync_mailbox_tree_dup_nodes(struct dsync_mailbox_tree *dest_tree,
+ const struct dsync_mailbox_node *src,
+ string_t *path)
+{
+ size_t prefix_len = str_len(path);
+ struct dsync_mailbox_node *node;
+
+ if (prefix_len > 0) {
+ str_append_c(path, dest_tree->sep);
+ prefix_len++;
+ }
+ for (; src != NULL; src = src->next) {
+ str_truncate(path, prefix_len);
+ str_append(path, src->name);
+ node = dsync_mailbox_tree_get(dest_tree, str_c(path));
+
+ node->ns = src->ns;
+ memcpy(node->mailbox_guid, src->mailbox_guid,
+ sizeof(node->mailbox_guid));
+ node->uid_validity = src->uid_validity;
+ node->uid_next = src->uid_next;
+ node->existence = src->existence;
+ node->last_renamed_or_created = src->last_renamed_or_created;
+ node->subscribed = src->subscribed;
+ node->last_subscription_change = src->last_subscription_change;
+
+ if (src->first_child != NULL) {
+ dsync_mailbox_tree_dup_nodes(dest_tree,
+ src->first_child, path);
+ }
+ }
+}
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_dup(const struct dsync_mailbox_tree *src)
+{
+ struct dsync_mailbox_tree *dest;
+ string_t *str = t_str_new(128);
+
+ dest = dsync_mailbox_tree_init(src->sep, src->escape_char,
+ src->alt_char);
+ dsync_mailbox_tree_dup_nodes(dest, &src->root, str);
+ return dest;
+}
+
+int dsync_mailbox_node_name_cmp(struct dsync_mailbox_node *const *n1,
+ struct dsync_mailbox_node *const *n2)
+{
+ return strcmp((*n1)->name, (*n2)->name);
+}
+
+static bool
+dsync_mailbox_nodes_equal(const struct dsync_mailbox_node *node1,
+ const struct dsync_mailbox_node *node2)
+{
+ return strcmp(node1->name, node2->name) == 0 &&
+ node1->ns == node2->ns &&
+ memcmp(node1->mailbox_guid, node2->mailbox_guid,
+ sizeof(node1->mailbox_guid)) == 0 &&
+ node1->uid_validity == node2->uid_validity &&
+ node1->existence == node2->existence &&
+ node1->subscribed == node2->subscribed;
+}
+
+static bool
+dsync_mailbox_branches_equal(struct dsync_mailbox_node *node1,
+ struct dsync_mailbox_node *node2)
+{
+ /* this function is used only for unit tests, so performance doesn't
+ really matter */
+ struct dsync_mailbox_node *n, **snodes1, **snodes2;
+ unsigned int i, count;
+
+ for (n = node1, count = 0; n != NULL; n = n->next) count++;
+ for (n = node2, i = 0; n != NULL; n = n->next) i++;
+ if (i != count)
+ return FALSE;
+ if (count == 0)
+ return TRUE;
+
+ /* sort the trees by name */
+ snodes1 = t_new(struct dsync_mailbox_node *, count);
+ snodes2 = t_new(struct dsync_mailbox_node *, count);
+ for (n = node1, i = 0; n != NULL; n = n->next, i++)
+ snodes1[i] = n;
+ for (n = node2, i = 0; n != NULL; n = n->next, i++)
+ snodes2[i] = n;
+ i_qsort(snodes1, count, sizeof(*snodes1), dsync_mailbox_node_name_cmp);
+ i_qsort(snodes2, count, sizeof(*snodes2), dsync_mailbox_node_name_cmp);
+
+ for (i = 0; i < count; i++) {
+ if (!dsync_mailbox_nodes_equal(snodes1[i], snodes2[i]))
+ return FALSE;
+ if (!dsync_mailbox_branches_equal(snodes1[i]->first_child,
+ snodes2[i]->first_child))
+ return FALSE;
+ }
+ return TRUE;
+}
+
+bool dsync_mailbox_trees_equal(struct dsync_mailbox_tree *tree1,
+ struct dsync_mailbox_tree *tree2)
+{
+ bool ret;
+
+ T_BEGIN {
+ ret = dsync_mailbox_branches_equal(&tree1->root, &tree2->root);
+ } T_END;
+ return ret;
+}
+
+const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node)
+{
+ return t_strdup_printf("guid=%s uid_validity=%u uid_next=%u subs=%s last_change=%ld last_subs=%ld",
+ guid_128_to_string(node->mailbox_guid),
+ node->uid_validity, node->uid_next,
+ node->subscribed ? "yes" : "no",
+ (long)node->last_renamed_or_created,
+ (long)node->last_subscription_change);
+}
+
+const char *
+dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type)
+{
+ switch (type) {
+ case DSYNC_MAILBOX_DELETE_TYPE_MAILBOX:
+ return "mailbox";
+ case DSYNC_MAILBOX_DELETE_TYPE_DIR:
+ return "dir";
+ case DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE:
+ return "unsubscribe";
+ }
+ i_unreached();
+}
diff --git a/src/doveadm/dsync/dsync-mailbox-tree.h b/src/doveadm/dsync/dsync-mailbox-tree.h
new file mode 100644
index 0000000..6969813
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox-tree.h
@@ -0,0 +1,205 @@
+#ifndef DSYNC_MAILBOX_TREE_H
+#define DSYNC_MAILBOX_TREE_H
+
+#include "guid.h"
+#include "mail-error.h"
+
+struct mail_namespace;
+struct dsync_brain;
+
+enum dsync_mailbox_trees_sync_type {
+ /* two-way sync for both mailboxes */
+ DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY,
+ /* make remote tree look exactly like the local tree */
+ DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_LOCAL,
+ /* make local tree look exactly like the remote tree */
+ DSYNC_MAILBOX_TREES_SYNC_TYPE_PRESERVE_REMOTE
+};
+
+enum dsync_mailbox_trees_sync_flags {
+ /* Enable debugging */
+ DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG = 0x01,
+ /* Show ourself as "master brain" in the debug output */
+ DSYNC_MAILBOX_TREES_SYNC_FLAG_MASTER_BRAIN = 0x02,
+ /* Disable mailbox renaming logic. This is just a kludge that should
+ be removed once the renaming logic has no more bugs.. */
+ DSYNC_MAILBOX_TREES_SYNC_FLAG_NO_RENAMES = 0x04
+};
+
+enum dsync_mailbox_node_existence {
+ /* this is just a filler node for children or for
+ subscription deletion */
+ DSYNC_MAILBOX_NODE_NONEXISTENT = 0,
+ /* if mailbox GUID is set, the mailbox exists.
+ otherwise the directory exists. */
+ DSYNC_MAILBOX_NODE_EXISTS,
+ /* if mailbox GUID is set, the mailbox has been deleted.
+ otherwise the directory has been deleted. */
+ DSYNC_MAILBOX_NODE_DELETED
+};
+
+struct dsync_mailbox_node {
+ struct dsync_mailbox_node *parent, *next, *first_child;
+
+ /* namespace where this node belongs to */
+ struct mail_namespace *ns;
+ /* this node's name (not including parents) */
+ const char *name;
+ /* mailbox GUID, or full of zeros if this is about a directory name */
+ guid_128_t mailbox_guid;
+ /* mailbox's UIDVALIDITY/UIDNEXT (may be 0 if not assigned yet) */
+ uint32_t uid_validity, uid_next;
+
+ /* existence of this mailbox/directory.
+ doesn't affect subscription state. */
+ enum dsync_mailbox_node_existence existence;
+ /* last time the mailbox/directory was created/renamed,
+ 0 if not known */
+ time_t last_renamed_or_created;
+
+ /* last time the subscription state was changed, 0 if not known */
+ time_t last_subscription_change;
+ /* is this mailbox or directory subscribed? */
+ bool subscribed:1;
+
+ /* Internal syncing flags: */
+ bool sync_delayed_guid_change:1;
+ bool sync_temporary_name:1;
+};
+ARRAY_DEFINE_TYPE(dsync_mailbox_node, struct dsync_mailbox_node *);
+
+#define dsync_mailbox_node_guids_equal(node1, node2) \
+ (memcmp((node1)->mailbox_guid, (node2)->mailbox_guid, \
+ sizeof(guid_128_t)) == 0)
+
+#define dsync_mailbox_node_is_dir(node) \
+ guid_128_is_empty((node)->mailbox_guid)
+
+enum dsync_mailbox_delete_type {
+ /* Delete mailbox by given GUID */
+ DSYNC_MAILBOX_DELETE_TYPE_MAILBOX = 1,
+ /* Delete mailbox directory by given SHA1 name */
+ DSYNC_MAILBOX_DELETE_TYPE_DIR,
+ /* Unsubscribe mailbox by given SHA1 name */
+ DSYNC_MAILBOX_DELETE_TYPE_UNSUBSCRIBE,
+};
+
+struct dsync_mailbox_delete {
+ enum dsync_mailbox_delete_type type;
+ guid_128_t guid;
+ time_t timestamp;
+};
+
+enum dsync_mailbox_tree_sync_type {
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_BOX,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_CREATE_DIR,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_BOX,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_DELETE_DIR,
+ /* Rename given mailbox name and its children */
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_RENAME,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_SUBSCRIBE,
+ DSYNC_MAILBOX_TREE_SYNC_TYPE_UNSUBSCRIBE
+};
+
+struct dsync_mailbox_tree_sync_change {
+ enum dsync_mailbox_tree_sync_type type;
+
+ /* for all types: */
+ struct mail_namespace *ns;
+ const char *full_name;
+
+ /* for create_box and delete_box: */
+ guid_128_t mailbox_guid;
+ /* for create_box: */
+ uint32_t uid_validity;
+ /* for rename: */
+ const char *rename_dest_name;
+};
+
+struct dsync_mailbox_tree *
+dsync_mailbox_tree_init(char sep, char escape_char, char alt_char);
+void dsync_mailbox_tree_deinit(struct dsync_mailbox_tree **tree);
+
+/* Lookup a mailbox node by name. Returns NULL if not known. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup(struct dsync_mailbox_tree *tree,
+ const char *full_name);
+/* Lookup a mailbox node by GUID. Returns NULL if not known.
+ The mailbox GUID hash must have been build before calling this. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_lookup_guid(struct dsync_mailbox_tree *tree,
+ const guid_128_t guid);
+/* Lookup or create a mailbox node by name. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_get(struct dsync_mailbox_tree *tree, const char *full_name);
+
+/* Returns full name for the given mailbox node. */
+const char *dsync_mailbox_node_get_full_name(const struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_node *node);
+void dsync_mailbox_node_append_full_name(string_t *str,
+ const struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_node *node);
+
+/* Copy everything from src to dest, except name and hierarchy pointers */
+void dsync_mailbox_node_copy_data(struct dsync_mailbox_node *dest,
+ const struct dsync_mailbox_node *src);
+
+/* Add nodes to tree from the given namespace. If box_name or box_guid is
+ non-NULL, add only that mailbox to the tree. */
+int dsync_mailbox_tree_fill(struct dsync_mailbox_tree *tree,
+ struct mail_namespace *ns, const char *box_name,
+ const guid_128_t box_guid,
+ const char *const *exclude_mailboxes,
+ enum mail_error *error_r);
+
+/* Return all known deleted mailboxes and directories. */
+const struct dsync_mailbox_delete *
+dsync_mailbox_tree_get_deletes(struct dsync_mailbox_tree *tree,
+ unsigned int *count_r);
+/* Return mailbox node for a given delete record, or NULL if it doesn't exist.
+ The delete record is intended to come from another tree, possibly with
+ a different hierarchy separator. dsync_mailbox_tree_build_guid_hash() must
+ have been called before this. */
+struct dsync_mailbox_node *
+dsync_mailbox_tree_find_delete(struct dsync_mailbox_tree *tree,
+ const struct dsync_mailbox_delete *del);
+/* Build GUID lookup hash, if it's not already built. Returns 0 if ok, -1 if
+ there are duplicate GUIDs. The nodes with the duplicate GUIDs are
+ returned. */
+int dsync_mailbox_tree_build_guid_hash(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node **dup_node1_r,
+ struct dsync_mailbox_node **dup_node2_r);
+/* Manually add a new node to hash. */
+int dsync_mailbox_tree_guid_hash_add(struct dsync_mailbox_tree *tree,
+ struct dsync_mailbox_node *node,
+ struct dsync_mailbox_node **old_node_r);
+/* Set remote separator used for directory deletions in
+ dsync_mailbox_tree_find_delete() */
+void dsync_mailbox_tree_set_remote_chars(struct dsync_mailbox_tree *tree,
+ char remote_sep,
+ char remote_escape_char);
+
+/* Iterate through all nodes in a tree (depth-first) */
+struct dsync_mailbox_tree_iter *
+dsync_mailbox_tree_iter_init(struct dsync_mailbox_tree *tree);
+bool dsync_mailbox_tree_iter_next(struct dsync_mailbox_tree_iter *iter,
+ const char **full_name_r,
+ struct dsync_mailbox_node **node_r);
+void dsync_mailbox_tree_iter_deinit(struct dsync_mailbox_tree_iter **iter);
+
+/* Sync local and remote trees so at the end they're exactly the same.
+ Return changes done to local tree. */
+struct dsync_mailbox_tree_sync_ctx *
+dsync_mailbox_trees_sync_init(struct dsync_mailbox_tree *local_tree,
+ struct dsync_mailbox_tree *remote_tree,
+ enum dsync_mailbox_trees_sync_type sync_type,
+ enum dsync_mailbox_trees_sync_flags sync_flags);
+const struct dsync_mailbox_tree_sync_change *
+dsync_mailbox_trees_sync_next(struct dsync_mailbox_tree_sync_ctx *ctx);
+int dsync_mailbox_trees_sync_deinit(struct dsync_mailbox_tree_sync_ctx **ctx);
+
+const char *dsync_mailbox_node_to_string(const struct dsync_mailbox_node *node);
+const char *
+dsync_mailbox_delete_type_to_string(enum dsync_mailbox_delete_type type);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-mailbox.c b/src/doveadm/dsync/dsync-mailbox.c
new file mode 100644
index 0000000..d6d06fd
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox.c
@@ -0,0 +1,61 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "istream.h"
+#include "mail-storage-private.h"
+#include "dsync-brain-private.h"
+#include "dsync-mailbox.h"
+
+void dsync_mailbox_attribute_dup(pool_t pool,
+ const struct dsync_mailbox_attribute *src,
+ struct dsync_mailbox_attribute *dest_r)
+{
+ dest_r->type = src->type;
+ dest_r->key = p_strdup(pool, src->key);
+ dest_r->value = p_strdup(pool, src->value);
+ if (src->value_stream != NULL) {
+ dest_r->value_stream = src->value_stream;
+ i_stream_ref(dest_r->value_stream);
+ }
+
+ dest_r->deleted = src->deleted;
+ dest_r->last_change = src->last_change;
+ dest_r->modseq = src->modseq;
+}
+
+int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box,
+ struct file_lock **lock_r)
+{
+ const char *path, *error;
+ int ret;
+
+ /* Make sure the mailbox is open - locking requires it */
+ if (mailbox_open(box) < 0) {
+ i_error("Can't open mailbox %s: %s", mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &brain->mail_error));
+ return -1;
+ }
+
+ ret = mailbox_get_path_to(box, MAILBOX_LIST_PATH_TYPE_INDEX, &path);
+ if (ret < 0) {
+ i_error("Can't get mailbox %s path: %s", mailbox_get_vname(box),
+ mailbox_get_last_internal_error(box, &brain->mail_error));
+ return -1;
+ }
+ if (ret == 0) {
+ /* No index files - don't do any locking. In theory we still
+ could, but this lock is mainly meant to prevent replication
+ problems, and replication wouldn't work without indexes. */
+ *lock_r = NULL;
+ return 0;
+ }
+
+ if (mailbox_lock_file_create(box, DSYNC_MAILBOX_LOCK_FILENAME,
+ brain->mailbox_lock_timeout_secs,
+ lock_r, &error) <= 0) {
+ i_error("Failed to lock mailbox %s for dsyncing: %s",
+ box->vname, error);
+ return -1;
+ }
+ return 0;
+}
diff --git a/src/doveadm/dsync/dsync-mailbox.h b/src/doveadm/dsync/dsync-mailbox.h
new file mode 100644
index 0000000..7e81c0c
--- /dev/null
+++ b/src/doveadm/dsync/dsync-mailbox.h
@@ -0,0 +1,44 @@
+#ifndef DSYNC_MAILBOX_H
+#define DSYNC_MAILBOX_H
+
+#include "mail-storage.h"
+
+struct dsync_brain;
+
+/* Mailbox that is going to be synced. Its name was already sent in the
+ mailbox tree. */
+struct dsync_mailbox {
+ guid_128_t mailbox_guid;
+ bool mailbox_lost;
+ bool mailbox_ignore;
+ bool have_guids, have_save_guids, have_only_guid128;
+
+ uint32_t uid_validity, uid_next, messages_count, first_recent_uid;
+ uint64_t highest_modseq, highest_pvt_modseq;
+ ARRAY_TYPE(mailbox_cache_field) cache_fields;
+};
+
+struct dsync_mailbox_attribute {
+ enum mail_attribute_type type;
+ const char *key;
+ /* if both values are NULL = not looked up yet / deleted */
+ const char *value;
+ struct istream *value_stream;
+
+ time_t last_change; /* 0 = unknown */
+ uint64_t modseq; /* 0 = unknown */
+
+ bool deleted; /* attribute is known to have been deleted */
+ bool exported; /* internally used by exporting */
+};
+#define DSYNC_ATTR_HAS_VALUE(attr) \
+ ((attr)->value != NULL || (attr)->value_stream != NULL)
+
+void dsync_mailbox_attribute_dup(pool_t pool,
+ const struct dsync_mailbox_attribute *src,
+ struct dsync_mailbox_attribute *dest_r);
+
+int dsync_mailbox_lock(struct dsync_brain *brain, struct mailbox *box,
+ struct file_lock **lock_r);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-serializer.c b/src/doveadm/dsync/dsync-serializer.c
new file mode 100644
index 0000000..7b369b8
--- /dev/null
+++ b/src/doveadm/dsync/dsync-serializer.c
@@ -0,0 +1,117 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "str.h"
+#include "strescape.h"
+#include "dsync-serializer.h"
+
+struct dsync_serializer {
+ pool_t pool;
+ const char *const *keys;
+ unsigned int keys_count;
+};
+
+struct dsync_serializer_encoder {
+ pool_t pool;
+ struct dsync_serializer *serializer;
+ ARRAY_TYPE(const_string) values;
+};
+
+struct dsync_serializer *dsync_serializer_init(const char *const keys[])
+{
+ struct dsync_serializer *serializer;
+ pool_t pool;
+ const char **dup_keys;
+ unsigned int i, count;
+
+ pool = pool_alloconly_create("dsync serializer", 512);
+ serializer = p_new(pool, struct dsync_serializer, 1);
+ serializer->pool = pool;
+
+ count = str_array_length(keys);
+ dup_keys = p_new(pool, const char *, count + 1);
+ for (i = 0; i < count; i++)
+ dup_keys[i] = p_strdup(pool, keys[i]);
+ serializer->keys = dup_keys;
+ serializer->keys_count = count;
+ return serializer;
+}
+
+void dsync_serializer_deinit(struct dsync_serializer **_serializer)
+{
+ struct dsync_serializer *serializer = *_serializer;
+
+ *_serializer = NULL;
+
+ pool_unref(&serializer->pool);
+}
+
+const char *
+dsync_serializer_encode_header_line(struct dsync_serializer *serializer)
+{
+ string_t *str = t_str_new(128);
+ unsigned int i;
+
+ for (i = 0; serializer->keys[i] != NULL; i++) {
+ if (i > 0)
+ str_append_c(str, '\t');
+ str_append_tabescaped(str, serializer->keys[i]);
+ }
+ str_append_c(str, '\n');
+ return str_c(str);
+}
+
+struct dsync_serializer_encoder *
+dsync_serializer_encode_begin(struct dsync_serializer *serializer)
+{
+ struct dsync_serializer_encoder *encoder;
+ pool_t pool = pool_alloconly_create("dsync serializer encode", 1024);
+
+ encoder = p_new(pool, struct dsync_serializer_encoder, 1);
+ encoder->pool = pool;
+ encoder->serializer = serializer;
+ p_array_init(&encoder->values, pool, serializer->keys_count);
+ return encoder;
+}
+
+void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder,
+ const char *key, const char *value)
+{
+ unsigned int i;
+
+ for (i = 0; encoder->serializer->keys[i] != NULL; i++) {
+ if (strcmp(encoder->serializer->keys[i], key) == 0) {
+ value = p_strdup(encoder->pool, value);
+ array_idx_set(&encoder->values, i, &value);
+ return;
+ }
+ }
+ i_panic("Unknown key: %s", key);
+}
+
+void dsync_serializer_encode_finish(struct dsync_serializer_encoder **_encoder,
+ string_t *output)
+{
+ struct dsync_serializer_encoder *encoder = *_encoder;
+ const char *const *values;
+ unsigned int i, count;
+
+ *_encoder = NULL;
+
+ values = array_get(&encoder->values, &count);
+ for (i = 0; i < count; i++) {
+ if (i > 0)
+ str_append_c(output, '\t');
+ if (values[i] == NULL)
+ str_append_c(output, NULL_CHR);
+ else {
+ if (values[i][0] == NULL_CHR)
+ str_append_c(output, NULL_CHR);
+ str_append_tabescaped(output, values[i]);
+ }
+ }
+ str_append_c(output, '\n');
+
+ pool_unref(&encoder->pool);
+}
diff --git a/src/doveadm/dsync/dsync-serializer.h b/src/doveadm/dsync/dsync-serializer.h
new file mode 100644
index 0000000..1ed6c31
--- /dev/null
+++ b/src/doveadm/dsync/dsync-serializer.h
@@ -0,0 +1,18 @@
+#ifndef DSYNC_SERIALIZER_H
+#define DSYNC_SERIALIZER_H
+
+#define NULL_CHR '\002'
+
+struct dsync_serializer *dsync_serializer_init(const char *const keys[]);
+void dsync_serializer_deinit(struct dsync_serializer **serializer);
+
+const char *
+dsync_serializer_encode_header_line(struct dsync_serializer *serializer);
+struct dsync_serializer_encoder *
+dsync_serializer_encode_begin(struct dsync_serializer *serializer);
+void dsync_serializer_encode_add(struct dsync_serializer_encoder *encoder,
+ const char *key, const char *value);
+void dsync_serializer_encode_finish(struct dsync_serializer_encoder **encoder,
+ string_t *output);
+
+#endif
diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.c b/src/doveadm/dsync/dsync-transaction-log-scan.c
new file mode 100644
index 0000000..dd2834a
--- /dev/null
+++ b/src/doveadm/dsync/dsync-transaction-log-scan.c
@@ -0,0 +1,608 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "hash.h"
+#include "mail-index-modseq.h"
+#include "mail-storage-private.h"
+#include "dsync-mail.h"
+#include "dsync-mailbox.h"
+#include "dsync-transaction-log-scan.h"
+
+struct dsync_transaction_log_scan {
+ pool_t pool;
+ HASH_TABLE_TYPE(dsync_uid_mail_change) changes;
+ HASH_TABLE_TYPE(dsync_attr_change) attr_changes;
+ struct mail_index_view *view;
+ uint32_t highest_wanted_uid;
+
+ uint32_t last_log_seq;
+ uoff_t last_log_offset;
+
+ bool returned_all_changes;
+};
+
+static bool ATTR_NOWARN_UNUSED_RESULT
+export_change_get(struct dsync_transaction_log_scan *ctx, uint32_t uid,
+ enum dsync_mail_change_type type,
+ struct dsync_mail_change **change_r)
+{
+ struct dsync_mail_change *change;
+ const char *orig_guid;
+
+ i_assert(uid > 0);
+ i_assert(type != DSYNC_MAIL_CHANGE_TYPE_SAVE);
+
+ *change_r = NULL;
+
+ if (uid > ctx->highest_wanted_uid)
+ return FALSE;
+
+ change = hash_table_lookup(ctx->changes, POINTER_CAST(uid));
+ if (change == NULL) {
+ /* first change for this UID */
+ change = p_new(ctx->pool, struct dsync_mail_change, 1);
+ change->uid = uid;
+ change->type = type;
+ hash_table_insert(ctx->changes, POINTER_CAST(uid), change);
+ } else if (type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ /* expunge overrides flag changes */
+ orig_guid = change->guid;
+ i_zero(change);
+ change->type = type;
+ change->uid = uid;
+ change->guid = orig_guid;
+ } else if (change->type == DSYNC_MAIL_CHANGE_TYPE_EXPUNGE) {
+ /* already expunged, this change doesn't matter */
+ return FALSE;
+ } else {
+ /* another flag update */
+ }
+ *change_r = change;
+ return TRUE;
+}
+
+static void
+log_add_expunge(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr)
+{
+ const struct mail_transaction_expunge *rec = data, *end;
+ struct dsync_mail_change *change;
+ uint32_t uid;
+
+ if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+ /* this is simply a request for expunge */
+ return;
+ }
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++) {
+ for (uid = rec->uid1; uid <= rec->uid2; uid++) {
+ export_change_get(ctx, uid,
+ DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+ &change);
+ }
+ }
+}
+
+static bool
+log_add_expunge_uid(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr, uint32_t uid)
+{
+ const struct mail_transaction_expunge *rec = data, *end;
+ struct dsync_mail_change *change;
+
+ if ((hdr->type & MAIL_TRANSACTION_EXTERNAL) == 0) {
+ /* this is simply a request for expunge */
+ return FALSE;
+ }
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++) {
+ if (uid >= rec->uid1 && uid <= rec->uid2) {
+ export_change_get(ctx, uid,
+ DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+ &change);
+ return TRUE;
+ }
+ }
+ return FALSE;
+}
+
+static void
+log_add_expunge_guid(struct dsync_transaction_log_scan *ctx,
+ struct mail_index_view *view, const void *data,
+ const struct mail_transaction_header *hdr)
+{
+ const struct mail_transaction_expunge_guid *rec = data, *end;
+ struct dsync_mail_change *change;
+ uint32_t seq;
+ bool external;
+
+ external = (hdr->type & MAIL_TRANSACTION_EXTERNAL) != 0;
+
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++) {
+ if (!external && mail_index_lookup_seq(view, rec->uid, &seq)) {
+ /* expunge request that hasn't been actually done yet.
+ we check non-external ones because they might have
+ the GUID while external ones don't. */
+ continue;
+ }
+ if (export_change_get(ctx, rec->uid,
+ DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+ &change) &&
+ !guid_128_is_empty(rec->guid_128)) T_BEGIN {
+ change->guid = p_strdup(ctx->pool,
+ guid_128_to_string(rec->guid_128));
+ } T_END;
+ }
+}
+
+static bool
+log_add_expunge_guid_uid(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr, uint32_t uid)
+{
+ const struct mail_transaction_expunge_guid *rec = data, *end;
+ struct dsync_mail_change *change;
+
+ /* we're assuming UID is already known to be expunged */
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++) {
+ if (rec->uid != uid)
+ continue;
+
+ if (!export_change_get(ctx, rec->uid,
+ DSYNC_MAIL_CHANGE_TYPE_EXPUNGE,
+ &change))
+ i_unreached();
+ if (!guid_128_is_empty(rec->guid_128)) T_BEGIN {
+ change->guid = p_strdup(ctx->pool,
+ guid_128_to_string(rec->guid_128));
+ } T_END;
+ return TRUE;
+ }
+ return FALSE;
+}
+
+static void
+log_add_flag_update(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr)
+{
+ const struct mail_transaction_flag_update *rec = data, *end;
+ struct dsync_mail_change *change;
+ uint32_t uid;
+
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++) {
+ for (uid = rec->uid1; uid <= rec->uid2; uid++) {
+ if (export_change_get(ctx, uid,
+ DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+ &change)) {
+ change->add_flags |= rec->add_flags;
+ change->remove_flags &= ENUM_NEGATE(rec->add_flags);
+ change->remove_flags |= rec->remove_flags;
+ change->add_flags &= ENUM_NEGATE(rec->remove_flags);
+ }
+ }
+ }
+}
+
+static void
+log_add_keyword_reset(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr)
+{
+ const struct mail_transaction_keyword_reset *rec = data, *end;
+ struct dsync_mail_change *change;
+ uint32_t uid;
+
+ end = CONST_PTR_OFFSET(data, hdr->size);
+ for (; rec != end; rec++) {
+ for (uid = rec->uid1; uid <= rec->uid2; uid++) {
+ if (!export_change_get(ctx, uid,
+ DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+ &change))
+ continue;
+
+ change->keywords_reset = TRUE;
+ if (array_is_created(&change->keyword_changes))
+ array_clear(&change->keyword_changes);
+ }
+ }
+}
+
+static void
+keywords_change_remove(struct dsync_mail_change *change, const char *name)
+{
+ const char *const *changes;
+ unsigned int i, count;
+
+ changes = array_get(&change->keyword_changes, &count);
+ for (i = 0; i < count; i++) {
+ if (strcmp(changes[i]+1, name) == 0) {
+ array_delete(&change->keyword_changes, i, 1);
+ break;
+ }
+ }
+}
+
+static void
+log_add_keyword_update(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr)
+{
+ const struct mail_transaction_keyword_update *rec = data;
+ struct dsync_mail_change *change;
+ const char *kw_name, *change_str;
+ const uint32_t *uids, *end;
+ unsigned int uids_offset;
+ uint32_t uid;
+
+ uids_offset = sizeof(*rec) + rec->name_size;
+ if ((uids_offset % 4) != 0)
+ uids_offset += 4 - (uids_offset % 4);
+
+ kw_name = t_strndup((const void *)(rec+1), rec->name_size);
+ switch (rec->modify_type) {
+ case MODIFY_ADD:
+ change_str = p_strdup_printf(ctx->pool, "%c%s",
+ KEYWORD_CHANGE_ADD, kw_name);
+ break;
+ case MODIFY_REMOVE:
+ change_str = p_strdup_printf(ctx->pool, "%c%s",
+ KEYWORD_CHANGE_REMOVE, kw_name);
+ break;
+ default:
+ i_unreached();
+ }
+
+ uids = CONST_PTR_OFFSET(rec, uids_offset);
+ end = CONST_PTR_OFFSET(rec, hdr->size);
+
+ for (; uids < end; uids += 2) {
+ for (uid = uids[0]; uid <= uids[1]; uid++) {
+ if (!export_change_get(ctx, uid,
+ DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+ &change))
+ continue;
+ if (!array_is_created(&change->keyword_changes)) {
+ p_array_init(&change->keyword_changes,
+ ctx->pool, 4);
+ } else {
+ keywords_change_remove(change, kw_name);
+ }
+ array_push_back(&change->keyword_changes, &change_str);
+ }
+ }
+}
+
+static void
+log_add_modseq_update(struct dsync_transaction_log_scan *ctx, const void *data,
+ const struct mail_transaction_header *hdr, bool pvt_scan)
+{
+ const struct mail_transaction_modseq_update *rec = data, *end;
+ struct dsync_mail_change *change;
+ uint64_t modseq;
+
+ /* update message's modseq, possibly by creating an empty flag change */
+ end = CONST_PTR_OFFSET(rec, hdr->size);
+ for (; rec != end; rec++) {
+ if (rec->uid == 0) {
+ /* highestmodseq update */
+ continue;
+ }
+
+ if (!export_change_get(ctx, rec->uid,
+ DSYNC_MAIL_CHANGE_TYPE_FLAG_CHANGE,
+ &change))
+ continue;
+
+ modseq = rec->modseq_low32 |
+ ((uint64_t)rec->modseq_high32 << 32);
+ if (!pvt_scan) {
+ if (change->modseq < modseq)
+ change->modseq = modseq;
+ } else {
+ if (change->pvt_modseq < modseq)
+ change->pvt_modseq = modseq;
+ }
+ }
+}
+
+static void
+log_add_attribute_update_key(struct dsync_transaction_log_scan *ctx,
+ const char *attr_change, uint64_t modseq)
+{
+ struct dsync_mailbox_attribute lookup_attr, *attr;
+
+ i_assert(strlen(attr_change) > 2); /* checked by lib-index */
+
+ lookup_attr.type = attr_change[1] == 'p' ?
+ MAIL_ATTRIBUTE_TYPE_PRIVATE : MAIL_ATTRIBUTE_TYPE_SHARED;
+ lookup_attr.key = attr_change+2;
+
+ attr = hash_table_lookup(ctx->attr_changes, &lookup_attr);
+ if (attr == NULL) {
+ attr = p_new(ctx->pool, struct dsync_mailbox_attribute, 1);
+ attr->type = lookup_attr.type;
+ attr->key = p_strdup(ctx->pool, lookup_attr.key);
+ hash_table_insert(ctx->attr_changes, attr, attr);
+ }
+ attr->deleted = attr_change[0] == '-';
+ attr->modseq = modseq;
+}
+
+static void
+log_add_attribute_update(struct dsync_transaction_log_scan *ctx,
+ const void *data,
+ const struct mail_transaction_header *hdr,
+ uint64_t modseq)
+{
+ const char *attr_changes = data;
+ unsigned int i;
+
+ for (i = 0; i < hdr->size && attr_changes[i] != '\0'; ) {
+ log_add_attribute_update_key(ctx, attr_changes+i, modseq);
+ i += strlen(attr_changes+i) + 1;
+ }
+}
+
+static int
+dsync_log_set(struct dsync_transaction_log_scan *ctx,
+ struct mail_index_view *view, bool pvt_scan,
+ struct mail_transaction_log_view *log_view, uint64_t modseq)
+{
+ uint32_t log_seq, end_seq;
+ uoff_t log_offset, end_offset;
+ const char *reason;
+ bool reset;
+ int ret;
+
+ end_seq = view->log_file_head_seq;
+ end_offset = view->log_file_head_offset;
+
+ if (modseq != 0 &&
+ mail_index_modseq_get_next_log_offset(view, modseq,
+ &log_seq, &log_offset)) {
+ /* scan the view only up to end of the current view.
+ if there are more changes, we don't care about them until
+ the next sync. */
+ ret = mail_transaction_log_view_set(log_view,
+ log_seq, log_offset,
+ end_seq, end_offset,
+ &reset, &reason);
+ if (ret != 0)
+ return ret;
+ }
+
+ /* return everything we've got (until the end of the view) */
+ if (!pvt_scan)
+ ctx->returned_all_changes = TRUE;
+ if (mail_transaction_log_view_set_all(log_view) < 0)
+ return -1;
+
+ mail_transaction_log_view_get_prev_pos(log_view, &log_seq, &log_offset);
+ if (log_seq > end_seq ||
+ (log_seq == end_seq && log_offset > end_offset)) {
+ end_seq = log_seq;
+ end_offset = log_offset;
+ }
+ ret = mail_transaction_log_view_set(log_view,
+ log_seq, log_offset,
+ end_seq, end_offset,
+ &reset, &reason);
+ if (ret == 0) {
+ /* we shouldn't get here. _view_set_all() already
+ reserved all the log files, the _view_set() only
+ removed unwanted ones. */
+ i_error("%s: Couldn't set transaction log view (seq %u..%u): %s",
+ view->index->filepath, log_seq, end_seq, reason);
+ ret = -1;
+ }
+ if (ret < 0)
+ return -1;
+ if (modseq != 0) {
+ /* we didn't see all the changes that we wanted to */
+ return 0;
+ }
+ return 1;
+}
+
+static int
+dsync_log_scan(struct dsync_transaction_log_scan *ctx,
+ struct mail_index_view *view, uint64_t modseq, bool pvt_scan)
+{
+ struct mail_transaction_log_view *log_view;
+ const struct mail_transaction_header *hdr;
+ const void *data;
+ uint32_t file_seq, max_seq;
+ uoff_t file_offset, max_offset;
+ uint64_t cur_modseq;
+ int ret;
+
+ log_view = mail_transaction_log_view_open(view->index->log);
+ if ((ret = dsync_log_set(ctx, view, pvt_scan, log_view, modseq)) < 0) {
+ mail_transaction_log_view_close(&log_view);
+ return -1;
+ }
+
+ /* read the log only up to current position in view */
+ max_seq = view->log_file_expunge_seq;
+ max_offset = view->log_file_expunge_offset;
+
+ mail_transaction_log_view_get_prev_pos(log_view, &file_seq,
+ &file_offset);
+
+ while (mail_transaction_log_view_next(log_view, &hdr, &data) > 0) {
+ mail_transaction_log_view_get_prev_pos(log_view, &file_seq,
+ &file_offset);
+ if (file_offset >= max_offset && file_seq == max_seq)
+ break;
+
+ if ((hdr->type & MAIL_TRANSACTION_SYNC) != 0) {
+ /* ignore changes done by dsync, unless we can get
+ expunged message's GUID from it */
+ if ((hdr->type & MAIL_TRANSACTION_TYPE_MASK) !=
+ MAIL_TRANSACTION_EXPUNGE_GUID)
+ continue;
+ }
+
+ switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+ case MAIL_TRANSACTION_EXPUNGE:
+ if (!pvt_scan)
+ log_add_expunge(ctx, data, hdr);
+ break;
+ case MAIL_TRANSACTION_EXPUNGE_GUID:
+ if (!pvt_scan)
+ log_add_expunge_guid(ctx, view, data, hdr);
+ break;
+ case MAIL_TRANSACTION_FLAG_UPDATE:
+ log_add_flag_update(ctx, data, hdr);
+ break;
+ case MAIL_TRANSACTION_KEYWORD_RESET:
+ log_add_keyword_reset(ctx, data, hdr);
+ break;
+ case MAIL_TRANSACTION_KEYWORD_UPDATE:
+ T_BEGIN {
+ log_add_keyword_update(ctx, data, hdr);
+ } T_END;
+ break;
+ case MAIL_TRANSACTION_MODSEQ_UPDATE:
+ log_add_modseq_update(ctx, data, hdr, pvt_scan);
+ break;
+ case MAIL_TRANSACTION_ATTRIBUTE_UPDATE:
+ cur_modseq = mail_transaction_log_view_get_prev_modseq(log_view);
+ log_add_attribute_update(ctx, data, hdr, cur_modseq);
+ break;
+ }
+ }
+
+ if (!pvt_scan) {
+ ctx->last_log_seq = file_seq;
+ ctx->last_log_offset = file_offset;
+ }
+ mail_transaction_log_view_close(&log_view);
+ return ret;
+}
+
+static int
+dsync_mailbox_attribute_cmp(const struct dsync_mailbox_attribute *attr1,
+ const struct dsync_mailbox_attribute *attr2)
+{
+ if (attr1->type < attr2->type)
+ return -1;
+ if (attr1->type > attr2->type)
+ return 1;
+ return strcmp(attr1->key, attr2->key);
+}
+
+static unsigned int
+dsync_mailbox_attribute_hash(const struct dsync_mailbox_attribute *attr)
+{
+ return str_hash(attr->key) ^ attr->type;
+}
+
+int dsync_transaction_log_scan_init(struct mail_index_view *view,
+ struct mail_index_view *pvt_view,
+ uint32_t highest_wanted_uid,
+ uint64_t modseq, uint64_t pvt_modseq,
+ struct dsync_transaction_log_scan **scan_r,
+ bool *pvt_too_old_r)
+{
+ struct dsync_transaction_log_scan *ctx;
+ pool_t pool;
+ int ret, ret2;
+
+ *pvt_too_old_r = FALSE;
+
+ pool = pool_alloconly_create(MEMPOOL_GROWING"dsync transaction log scan",
+ 10240);
+ ctx = p_new(pool, struct dsync_transaction_log_scan, 1);
+ ctx->pool = pool;
+ hash_table_create_direct(&ctx->changes, pool, 0);
+ hash_table_create(&ctx->attr_changes, pool, 0,
+ dsync_mailbox_attribute_hash,
+ dsync_mailbox_attribute_cmp);
+ ctx->view = view;
+ ctx->highest_wanted_uid = highest_wanted_uid;
+
+ if ((ret = dsync_log_scan(ctx, view, modseq, FALSE)) < 0)
+ return -1;
+ if (pvt_view != NULL) {
+ if ((ret2 = dsync_log_scan(ctx, pvt_view, pvt_modseq, TRUE)) < 0)
+ return -1;
+ if (ret2 == 0) {
+ ret = 0;
+ *pvt_too_old_r = TRUE;
+ }
+ }
+
+ *scan_r = ctx;
+ return ret;
+}
+
+HASH_TABLE_TYPE(dsync_uid_mail_change)
+dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan)
+{
+ return scan->changes;
+}
+
+HASH_TABLE_TYPE(dsync_attr_change)
+dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan)
+{
+ return scan->attr_changes;
+}
+
+bool
+dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan)
+{
+ return scan->returned_all_changes;
+}
+
+struct dsync_mail_change *
+dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan,
+ uint32_t uid)
+{
+ struct mail_transaction_log_view *log_view;
+ const struct mail_transaction_header *hdr;
+ const void *data;
+ const char *reason;
+ bool reset, found = FALSE;
+
+ i_assert(uid > 0);
+
+ if (scan->highest_wanted_uid < uid)
+ scan->highest_wanted_uid = uid;
+
+ log_view = mail_transaction_log_view_open(scan->view->index->log);
+ if (mail_transaction_log_view_set(log_view,
+ scan->last_log_seq,
+ scan->last_log_offset,
+ (uint32_t)-1, UOFF_T_MAX,
+ &reset, &reason) > 0) {
+ while (!found &&
+ mail_transaction_log_view_next(log_view, &hdr, &data) > 0) {
+ switch (hdr->type & MAIL_TRANSACTION_TYPE_MASK) {
+ case MAIL_TRANSACTION_EXPUNGE:
+ if (log_add_expunge_uid(scan, data, hdr, uid))
+ found = TRUE;
+ break;
+ case MAIL_TRANSACTION_EXPUNGE_GUID:
+ if (log_add_expunge_guid_uid(scan, data, hdr, uid))
+ found = TRUE;
+ break;
+ }
+ }
+ }
+ mail_transaction_log_view_close(&log_view);
+
+ return !found ? NULL :
+ hash_table_lookup(scan->changes, POINTER_CAST(uid));
+}
+
+void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **_scan)
+{
+ struct dsync_transaction_log_scan *scan = *_scan;
+
+ *_scan = NULL;
+
+ hash_table_destroy(&scan->changes);
+ hash_table_destroy(&scan->attr_changes);
+ pool_unref(&scan->pool);
+}
diff --git a/src/doveadm/dsync/dsync-transaction-log-scan.h b/src/doveadm/dsync/dsync-transaction-log-scan.h
new file mode 100644
index 0000000..458b775
--- /dev/null
+++ b/src/doveadm/dsync/dsync-transaction-log-scan.h
@@ -0,0 +1,32 @@
+#ifndef DSYNC_TRANSACTION_LOG_SCAN_H
+#define DSYNC_TRANSACTION_LOG_SCAN_H
+
+HASH_TABLE_DEFINE_TYPE(dsync_uid_mail_change,
+ void *, struct dsync_mail_change *);
+HASH_TABLE_DEFINE_TYPE(dsync_attr_change,
+ struct dsync_mailbox_attribute *,
+ struct dsync_mailbox_attribute *);
+
+struct mail_index_view;
+struct dsync_transaction_log_scan;
+
+int dsync_transaction_log_scan_init(struct mail_index_view *view,
+ struct mail_index_view *pvt_view,
+ uint32_t highest_wanted_uid,
+ uint64_t modseq, uint64_t pvt_modseq,
+ struct dsync_transaction_log_scan **scan_r,
+ bool *pvt_too_old_r);
+HASH_TABLE_TYPE(dsync_uid_mail_change)
+dsync_transaction_log_scan_get_hash(struct dsync_transaction_log_scan *scan);
+HASH_TABLE_TYPE(dsync_attr_change)
+dsync_transaction_log_scan_get_attr_hash(struct dsync_transaction_log_scan *scan);
+/* Returns TRUE if the entire transaction log was scanned */
+bool dsync_transaction_log_scan_has_all_changes(struct dsync_transaction_log_scan *scan);
+/* If the given UID has been expunged after the initial log scan, create/update
+ a change record for it and return it. */
+struct dsync_mail_change *
+dsync_transaction_log_scan_find_new_expunge(struct dsync_transaction_log_scan *scan,
+ uint32_t uid);
+void dsync_transaction_log_scan_deinit(struct dsync_transaction_log_scan **scan);
+
+#endif
diff --git a/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
new file mode 100644
index 0000000..b068137
--- /dev/null
+++ b/src/doveadm/dsync/test-dsync-mailbox-tree-sync.c
@@ -0,0 +1,781 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "sha1.h"
+#include "str.h"
+#include "mailbox-list-private.h"
+#include "dsync-mailbox-tree-private.h"
+#include "test-common.h"
+
+#include <stdio.h>
+
+#define MAX_DEPTH 4
+#define TEST_NAMESPACE_NAME "INBOX"
+
+static struct mail_namespace inbox_namespace = {
+ .prefix = TEST_NAMESPACE_NAME"/",
+ .prefix_len = sizeof(TEST_NAMESPACE_NAME)-1 + 1
+};
+
+char mail_namespace_get_sep(struct mail_namespace *ns ATTR_UNUSED)
+{
+ return '/';
+}
+
+void mailbox_name_get_sha128(const char *name, guid_128_t guid_128_r)
+{
+ unsigned char sha[SHA1_RESULTLEN];
+
+ sha1_get_digest(name, strlen(name), sha);
+ memcpy(guid_128_r, sha, I_MIN(GUID_128_SIZE, sizeof(sha)));
+}
+
+void mailbox_list_name_unescape(const char **name ATTR_UNUSED,
+ char escape_char ATTR_UNUSED)
+{
+}
+
+void mailbox_list_name_escape(const char *name,
+ const char *escape_chars ATTR_UNUSED,
+ string_t *dest)
+{
+ str_append(dest, name);
+}
+
+static struct dsync_mailbox_node *
+node_create(struct dsync_mailbox_tree *tree, unsigned int counter,
+ const char *name, unsigned int last_renamed_or_created)
+{
+ struct dsync_mailbox_node *node;
+
+ node = dsync_mailbox_tree_get(tree, name);
+ memcpy(node->mailbox_guid, &counter, sizeof(counter));
+ node->uid_validity = counter;
+ node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ node->last_renamed_or_created = last_renamed_or_created;
+ return node;
+}
+
+static struct dsync_mailbox_node *
+random_node_create(struct dsync_mailbox_tree *tree, unsigned int counter,
+ const char *name)
+{
+ return node_create(tree, counter, name, i_rand_limit(10));
+}
+
+static void nodes_create(struct dsync_mailbox_tree *tree, unsigned int *counter,
+ const char *const *names)
+{
+ for (; *names != NULL; names++) {
+ *counter += 1;
+ node_create(tree, *counter, *names, 0);
+ }
+}
+
+static void nodes_delete(struct dsync_mailbox_tree *tree, unsigned int *counter,
+ const char *const *names)
+{
+ struct dsync_mailbox_node *node;
+
+ for (; *names != NULL; names++) {
+ *counter += 1;
+ node = node_create(tree, *counter, *names, 0);
+ node->existence = DSYNC_MAILBOX_NODE_DELETED;
+ }
+}
+
+static void
+create_random_nodes(struct dsync_mailbox_tree *tree, const char *parent_name,
+ unsigned int depth, unsigned int *counter)
+{
+ unsigned int parent_len, i, nodes_count = i_rand_minmax(1, 3);
+ string_t *str;
+
+ if (depth == MAX_DEPTH)
+ return;
+
+ str = t_str_new(32);
+ if (*parent_name != '\0')
+ str_printfa(str, "%s/", parent_name);
+ parent_len = str_len(str);
+
+ for (i = 0; i < nodes_count; i++) {
+ *counter += 1;
+ str_truncate(str, parent_len);
+ str_printfa(str, "%u.%u", depth, i);
+ random_node_create(tree, *counter, str_c(str));
+ create_random_nodes(tree, str_c(str), depth+1, counter);
+ }
+}
+
+static struct dsync_mailbox_tree *create_random_tree(void)
+{
+ struct dsync_mailbox_tree *tree;
+ unsigned int counter = 0;
+
+ tree = dsync_mailbox_tree_init('/', '\0', '_');
+ create_random_nodes(tree, "", 0, &counter);
+ return tree;
+}
+
+static void test_tree_nodes_fixup(struct dsync_mailbox_node **pos,
+ unsigned int *newguid_counter)
+{
+ struct dsync_mailbox_node *node;
+
+ for (node = *pos; node != NULL; node = node->next) {
+ if (node->sync_delayed_guid_change) {
+ /* the real code will pick one of the GUIDs.
+ we don't really care which one gets picked, so we'll
+ just change them to the same new one */
+ memcpy(node->mailbox_guid, newguid_counter,
+ sizeof(*newguid_counter));
+ node->uid_validity = *newguid_counter;
+ *newguid_counter += 1;
+ }
+ if (node->existence == DSYNC_MAILBOX_NODE_DELETED)
+ node->existence = DSYNC_MAILBOX_NODE_NONEXISTENT;
+ test_tree_nodes_fixup(&node->first_child, newguid_counter);
+ if (node->existence != DSYNC_MAILBOX_NODE_EXISTS &&
+ node->first_child == NULL) {
+ /* nonexistent node, drop it */
+ *pos = node->next;
+ } else {
+ pos = &node->next;
+ }
+ }
+}
+
+static void test_tree_fixup(struct dsync_mailbox_tree *tree)
+{
+ unsigned int newguid_counter = INT_MAX;
+
+ test_tree_nodes_fixup(&tree->root.first_child, &newguid_counter);
+}
+
+static void nodes_dump(const struct dsync_mailbox_node *node, unsigned int depth)
+{
+ unsigned int i;
+
+ for (; node != NULL; node = node->next) {
+ for (i = 0; i < depth; i++) printf(" ");
+ printf("%-*s guid:%.5s uidv:%u %d%d %ld\n", 40-depth, node->name,
+ guid_128_to_string(node->mailbox_guid), node->uid_validity,
+ node->existence, node->subscribed ? 1 : 0,
+ (long)node->last_renamed_or_created);
+ nodes_dump(node->first_child, depth+1);
+ }
+}
+
+static void trees_dump(struct dsync_mailbox_tree *tree1,
+ struct dsync_mailbox_tree *tree2)
+{
+ printf("tree1:\n");
+ nodes_dump(tree1->root.first_child, 1);
+ printf("tree2:\n");
+ nodes_dump(tree2->root.first_child, 1);
+}
+
+static void test_trees_nofree(struct dsync_mailbox_tree *tree1,
+ struct dsync_mailbox_tree **_tree2)
+{
+ struct dsync_mailbox_tree *tree2 = *_tree2;
+ struct dsync_mailbox_tree *orig_tree1, *orig_tree2;
+ struct dsync_mailbox_tree_sync_ctx *ctx;
+ struct dsync_mailbox_node *dup_node1, *dup_node2;
+
+ orig_tree1 = dsync_mailbox_tree_dup(tree1);
+ orig_tree2 = dsync_mailbox_tree_dup(tree2);
+
+ /* test tree1 -> tree2 */
+ dsync_mailbox_tree_build_guid_hash(tree1, &dup_node1, &dup_node2);
+ dsync_mailbox_tree_build_guid_hash(tree2, &dup_node1, &dup_node2);
+ ctx = dsync_mailbox_trees_sync_init(tree1, tree2,
+ DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY,
+ DSYNC_MAILBOX_TREES_SYNC_FLAG_DEBUG);
+ while (dsync_mailbox_trees_sync_next(ctx) != NULL) {
+ }
+ dsync_mailbox_trees_sync_deinit(&ctx);
+ test_tree_fixup(tree1);
+ test_tree_fixup(tree2);
+ if (!dsync_mailbox_trees_equal(tree1, tree2)) {
+ test_assert(FALSE);
+ trees_dump(tree1, tree2);
+ }
+
+ /* test tree2 -> tree1 */
+ dsync_mailbox_tree_build_guid_hash(orig_tree1, &dup_node1, &dup_node2);
+ dsync_mailbox_tree_build_guid_hash(orig_tree2, &dup_node1, &dup_node2);
+ ctx = dsync_mailbox_trees_sync_init(orig_tree2, orig_tree1,
+ DSYNC_MAILBOX_TREES_SYNC_TYPE_TWOWAY, 0);
+ while (dsync_mailbox_trees_sync_next(ctx) != NULL) {
+ }
+ dsync_mailbox_trees_sync_deinit(&ctx);
+ test_tree_fixup(orig_tree1);
+ test_tree_fixup(orig_tree2);
+ if (!dsync_mailbox_trees_equal(orig_tree1, orig_tree2)) {
+ test_assert(FALSE);
+ trees_dump(orig_tree1, orig_tree2);
+ }
+
+ /* make sure both directions produced equal trees */
+ if (!dsync_mailbox_trees_equal(tree1, orig_tree1)) {
+ test_assert(FALSE);
+ trees_dump(tree1, orig_tree1);
+ }
+
+ dsync_mailbox_tree_deinit(_tree2);
+ dsync_mailbox_tree_deinit(&orig_tree1);
+ dsync_mailbox_tree_deinit(&orig_tree2);
+}
+
+static void
+test_tree_nodes_add_namespace(struct dsync_mailbox_node *node,
+ struct mail_namespace *ns)
+{
+ for (; node != NULL; node = node->next) {
+ node->ns = ns;
+ test_tree_nodes_add_namespace(node->first_child, ns);
+ }
+}
+
+static void
+test_tree_add_namespace(struct dsync_mailbox_tree *tree,
+ struct mail_namespace *ns)
+{
+ struct dsync_mailbox_node *node, *n;
+
+ node = dsync_mailbox_tree_get(tree, TEST_NAMESPACE_NAME);
+ node->existence = DSYNC_MAILBOX_NODE_EXISTS;
+ i_assert(tree->root.first_child == node);
+ i_assert(node->first_child == NULL);
+ node->first_child = node->next;
+ for (n = node->first_child; n != NULL; n = n->next)
+ n->parent = node;
+ node->next = NULL;
+
+ test_tree_nodes_add_namespace(&tree->root, ns);
+}
+
+static void test_trees(struct dsync_mailbox_tree *tree1,
+ struct dsync_mailbox_tree *tree2)
+{
+ struct dsync_mailbox_tree *tree1_dup, *tree2_dup;
+
+ tree1_dup = dsync_mailbox_tree_dup(tree1);
+ tree2_dup = dsync_mailbox_tree_dup(tree2);
+
+ /* test without namespace prefix */
+ test_trees_nofree(tree1, &tree2);
+ dsync_mailbox_tree_deinit(&tree1);
+
+ /* test with namespace prefix */
+ test_tree_add_namespace(tree1_dup, &inbox_namespace);
+ test_tree_add_namespace(tree2_dup, &inbox_namespace);
+ test_trees_nofree(tree1_dup, &tree2_dup);
+ dsync_mailbox_tree_deinit(&tree1_dup);
+}
+
+static void test_dsync_mailbox_tree_sync_creates(void)
+{
+ static const char *common_nodes[] = { "foo", "foo/bar", NULL };
+ static const char *create1_nodes[] = { "bar", "foo/baz", NULL };
+ static const char *create2_nodes[] = { "foo/xyz", "foo/bar/3", NULL };
+ struct dsync_mailbox_tree *tree1, *tree2;
+ unsigned int counter = 0;
+
+ test_begin("dsync mailbox tree sync creates");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ nodes_create(tree1, &counter, common_nodes);
+ tree2 = dsync_mailbox_tree_dup(tree1);
+ nodes_create(tree1, &counter, create1_nodes);
+ nodes_create(tree2, &counter, create2_nodes);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_deletes(void)
+{
+ static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", NULL };
+ static const char *delete1_nodes[] = { "1", "2", NULL };
+ static const char *delete2_nodes[] = { "2/s1", "x/y", NULL };
+ struct dsync_mailbox_tree *tree1, *tree2;
+ unsigned int counter = 0;
+
+ test_begin("dsync mailbox tree sync deletes");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ nodes_create(tree1, &counter, common_nodes);
+ tree2 = dsync_mailbox_tree_dup(tree1);
+ nodes_delete(tree1, &counter, delete1_nodes);
+ nodes_delete(tree2, &counter, delete2_nodes);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames1(void)
+{
+ static const char *common_nodes[] = { "1", "2", "3", "2/s1", "2/s2", "x/y", "3/s3", NULL };
+ struct dsync_mailbox_tree *tree1, *tree2;
+ struct dsync_mailbox_node *node;
+ unsigned int counter = 0;
+
+ test_begin("dsync mailbox tree sync renames 1");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ nodes_create(tree1, &counter, common_nodes);
+ tree2 = dsync_mailbox_tree_dup(tree1);
+
+ node = dsync_mailbox_tree_get(tree1, "1");
+ node->name = "a";
+ node->last_renamed_or_created = 1000;
+ node = dsync_mailbox_tree_get(tree2, "2");
+ node->name = "b";
+ node->last_renamed_or_created = 1000;
+
+ node = dsync_mailbox_tree_get(tree1, "3/s3");
+ node->name = "z";
+ node->last_renamed_or_created = 1000;
+ dsync_mailbox_tree_node_detach(node);
+ dsync_mailbox_tree_node_attach(node, &tree1->root);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames2(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 2");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/1", 1);
+ node_create(tree1, 2, "0/1/2", 3);
+
+ node_create(tree2, 1, "0", 0);
+ node_create(tree2, 2, "0/1/2", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames3(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 3");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/2", 1);
+ node_create(tree1, 2, "0/3", 1);
+
+ node_create(tree2, 1, "0/4/5", 0);
+ node_create(tree2, 2, "1", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames4(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 4");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/b", 0);
+ node_create(tree1, 2, "c", 2);
+
+ node_create(tree2, 2, "0/a", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames5(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 5");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "b", 0);
+ node_create(tree1, 2, "c", 2);
+
+ node_create(tree2, 2, "0/a", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames6(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 6");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/1", 0);
+ node_create(tree1, 2, "0/2", 1);
+
+ node_create(tree2, 1, "0", 1);
+ node_create(tree2, 2, "0/3", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames7(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 7");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/2", 0);
+ node_create(tree2, 1, "1/2", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames8(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 8");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/1", 0);
+ node_create(tree1, 2, "0/2", 1);
+
+ node_create(tree2, 1, "0", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames9(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 9");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/1/2", 0);
+ node_create(tree1, 2, "0/3", 1);
+
+ node_create(tree2, 1, "0", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames10(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 10");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/1", 0);
+ node_create(tree1, 3, "0/2/3", 0);
+
+ node_create(tree2, 1, "0", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames11(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 11");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/1", 2);
+ node_create(tree1, 0, "0/1/2", 0);
+
+ node_create(tree2, 1, "0", 1);
+ node_create(tree2, 0, "0/1/2", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames12(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 12");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/2", 0);
+ node_create(tree1, 2, "1", 0);
+ node_create(tree1, 3, "1/4", 0);
+ node_create(tree1, 4, "1/4/5", 1);
+
+ node_create(tree2, 1, "1", 2);
+ node_create(tree2, 2, "1/4", 3);
+ node_create(tree2, 3, "1/4/6", 4);
+ node_create(tree2, 4, "1/3", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames13(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 13");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 4, "0.0/1.0/2.1", 0);
+ node_create(tree1, 5, "0.1", 2);
+ node_create(tree1, 6, "0.1/1.0", 2);
+ node_create(tree1, 7, "0.1/1.0/2.0", 8);
+
+ node_create(tree2, 5, "0.1/1.0", 5);
+ node_create(tree2, 6, "0.1/1.0/2.0", 8);
+ node_create(tree2, 7, "0.1/1.1", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames14(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 14");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "1", 0);
+ node_create(tree1, 2, "1/2", 0);
+ node_create(tree1, 3, "1/2/4", 1);
+
+ node_create(tree2, 1, "1/2", 3);
+ node_create(tree2, 2, "1/2/5", 4);
+ node_create(tree2, 3, "1/2/4", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames15(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 15");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "1", 0);
+ node_create(tree2, 2, "1", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames16(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 16");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "1/2", 4);
+ node_create(tree1, 2, "1", 2);
+
+ node_create(tree2, 1, "2", 1);
+ node_create(tree2, 2, "1/2", 3);
+ node_create(tree2, 3, "1", 5);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames17(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 17");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "1", 1);
+
+ node_create(tree2, 1, "1/2", 0);
+ node_create(tree2, 2, "1", 2);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames18(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 18");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 2, "a", 5);
+ node_create(tree1, 4, "a/c", 2);
+ node_create(tree1, 5, "b", 6);
+
+ node_create(tree2, 1, "a", 7);
+ node_create(tree2, 2, "b", 3);
+ node_create(tree2, 3, "b/c", 4);
+ node_create(tree2, 4, "d", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames19(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 19");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "0/2/1", 1);
+ node_create(tree1, 2, "0/4", 3);
+ node_create(tree1, 3, "0/2", 2);
+
+ node_create(tree2, 1, "1", 0);
+ node_create(tree2, 2, "1/3", 4);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames20(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 20");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "1", 0);
+ node_create(tree1, 2, "0", 0);
+ node_create(tree1, 3, "0/2", 0);
+ /* rename 0 -> 1/0 */
+ node_create(tree2, 1, "1", 0);
+ node_create(tree2, 2, "1/0", 1);
+ node_create(tree2, 3, "1/0/2", 0);
+
+ test_trees_nofree(tree1, &tree2);
+ test_assert(tree1->root.first_child != NULL &&
+ tree1->root.first_child->next == NULL);
+ dsync_mailbox_tree_deinit(&tree1);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_renames21(void)
+{
+#if 0
+ /* FIXME: we can't currently test this without crashing */
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 21");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 1, "INBOX", 0);
+ node_create(tree1, 2, "foo", 0);
+ /* swap INBOX and foo - the INBOX name is important since it's
+ treated specially */
+ node_create(tree2, 1, "foo", 0);
+ node_create(tree2, 2, "INBOX", 1);
+
+ test_trees(tree1, tree2);
+ test_end();
+#endif
+}
+
+static void test_dsync_mailbox_tree_sync_renames22(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync renames 22");
+ tree1 = dsync_mailbox_tree_init('/', '\0', '_');
+ tree2 = dsync_mailbox_tree_init('/', '\0', '_');
+
+ node_create(tree1, 3, "p/a", 0);
+ node_create(tree1, 0, "p/2", 0);
+ node_create(tree1, 5, "p/2/h", 0);
+
+ node_create(tree2, 4, "p/1/z", 0);
+ node_create(tree2, 1, "p/2", 0);
+ node_create(tree2, 2, "p/2/a", 0);
+ node_create(tree2, 5, "p/2/y", 0);
+ node_create(tree2, 3, "p/3", 0);
+
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+static void test_dsync_mailbox_tree_sync_random(void)
+{
+ struct dsync_mailbox_tree *tree1, *tree2;
+
+ test_begin("dsync mailbox tree sync random");
+ tree1 = create_random_tree();
+ tree2 = create_random_tree();
+ test_trees(tree1, tree2);
+ test_end();
+}
+
+int main(void)
+{
+ static void (*const test_functions[])(void) = {
+ test_dsync_mailbox_tree_sync_creates,
+ test_dsync_mailbox_tree_sync_deletes,
+ test_dsync_mailbox_tree_sync_renames1,
+ test_dsync_mailbox_tree_sync_renames2,
+ test_dsync_mailbox_tree_sync_renames3,
+ test_dsync_mailbox_tree_sync_renames4,
+ test_dsync_mailbox_tree_sync_renames5,
+ test_dsync_mailbox_tree_sync_renames6,
+ test_dsync_mailbox_tree_sync_renames7,
+ test_dsync_mailbox_tree_sync_renames8,
+ test_dsync_mailbox_tree_sync_renames9,
+ test_dsync_mailbox_tree_sync_renames10,
+ test_dsync_mailbox_tree_sync_renames11,
+ test_dsync_mailbox_tree_sync_renames12,
+ test_dsync_mailbox_tree_sync_renames13,
+ test_dsync_mailbox_tree_sync_renames14,
+ test_dsync_mailbox_tree_sync_renames15,
+ test_dsync_mailbox_tree_sync_renames16,
+ test_dsync_mailbox_tree_sync_renames17,
+ test_dsync_mailbox_tree_sync_renames18,
+ test_dsync_mailbox_tree_sync_renames19,
+ test_dsync_mailbox_tree_sync_renames20,
+ test_dsync_mailbox_tree_sync_renames21,
+ test_dsync_mailbox_tree_sync_renames22,
+ test_dsync_mailbox_tree_sync_random,
+ NULL
+ };
+ return test_run(test_functions);
+}
diff --git a/src/doveadm/main.c b/src/doveadm/main.c
new file mode 100644
index 0000000..9635b88
--- /dev/null
+++ b/src/doveadm/main.c
@@ -0,0 +1,129 @@
+/* Copyright (c) 2005-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "restrict-access.h"
+#include "process-title.h"
+#include "master-service.h"
+#include "settings-parser.h"
+#include "dict.h"
+#include "doveadm.h"
+#include "client-connection.h"
+#include "client-connection-private.h"
+#include "doveadm-settings.h"
+#include "doveadm-dump.h"
+#include "doveadm-mail.h"
+#include "doveadm-print-private.h"
+#include "doveadm-server.h"
+#include "ostream.h"
+
+const struct doveadm_print_vfuncs *doveadm_print_vfuncs_all[] = {
+ &doveadm_print_server_vfuncs,
+ &doveadm_print_json_vfuncs,
+ NULL
+};
+
+struct client_connection *doveadm_client;
+int doveadm_exit_code = 0;
+
+static void doveadm_die(void)
+{
+ /* do nothing. doveadm connections should be over soon. */
+}
+
+static void client_connected(struct master_service_connection *conn)
+{
+ if (doveadm_client != NULL) {
+ i_error("doveadm server can handle only a single client");
+ return;
+ }
+
+ master_service_client_connection_accept(conn);
+ if (strcmp(conn->name, "http") == 0) {
+ doveadm_client = client_connection_http_create(conn->fd, conn->ssl);
+ } else {
+ doveadm_client = client_connection_tcp_create(conn->fd, conn->listen_fd,
+ conn->ssl);
+ }
+}
+
+void help_ver2(const struct doveadm_cmd_ver2 *cmd)
+{
+ i_fatal("Client sent invalid command. Usage: %s %s",
+ cmd->name, cmd->usage);
+}
+
+static void main_preinit(void)
+{
+ restrict_access_by_env(RESTRICT_ACCESS_FLAG_ALLOW_ROOT, NULL);
+ restrict_access_allow_coredumps(TRUE);
+}
+
+static void main_init(void)
+{
+ doveadm_server = TRUE;
+
+ doveadm_settings_init();
+ doveadm_cmds_init();
+ doveadm_register_auth_server_commands();
+ doveadm_dump_init();
+ doveadm_mail_init();
+ dict_drivers_register_builtin();
+ doveadm_load_modules();
+ /* read settings only after loading doveadm plugins, which
+ may modify what settings are read */
+ doveadm_read_settings();
+ /* Load mail_plugins */
+ doveadm_mail_init_finish();
+ /* kludgy: Load the rest of the doveadm plugins after
+ mail_plugins have been loaded. */
+ doveadm_load_modules();
+
+ doveadm_server_init();
+ if (doveadm_verbose_proctitle)
+ process_title_set("[idling]");
+}
+
+static void main_deinit(void)
+{
+ doveadm_server_deinit();
+ doveadm_mail_deinit();
+ doveadm_dump_deinit();
+ doveadm_unload_modules();
+ dict_drivers_unregister_builtin();
+ doveadm_print_deinit();
+ doveadm_cmds_deinit();
+ doveadm_settings_deinit();
+}
+
+int main(int argc, char *argv[])
+{
+ enum master_service_flags service_flags =
+ MASTER_SERVICE_FLAG_KEEP_CONFIG_OPEN |
+ MASTER_SERVICE_FLAG_HAVE_STARTTLS;
+ int c;
+
+ master_service = master_service_init("doveadm", service_flags,
+ &argc, &argv, "D");
+ while ((c = master_getopt(master_service)) > 0) {
+ switch (c) {
+ case 'D':
+ doveadm_debug = TRUE;
+ doveadm_verbose = TRUE;
+ break;
+ default:
+ return FATAL_DEFAULT;
+ }
+ }
+
+ master_service_init_log(master_service);
+ main_preinit();
+ master_service_set_die_callback(master_service, doveadm_die);
+
+ main_init();
+ master_service_init_finish(master_service);
+ master_service_run(master_service, client_connected);
+
+ main_deinit();
+ master_service_deinit(&master_service);
+ return 0;
+}
diff --git a/src/doveadm/server-connection.c b/src/doveadm/server-connection.c
new file mode 100644
index 0000000..a0e6500
--- /dev/null
+++ b/src/doveadm/server-connection.c
@@ -0,0 +1,691 @@
+/* Copyright (c) 2010-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "array.h"
+#include "base64.h"
+#include "ioloop.h"
+#include "net.h"
+#include "istream.h"
+#include "istream-multiplex.h"
+#include "ostream.h"
+#include "ostream-dot.h"
+#include "str.h"
+#include "strescape.h"
+#include "iostream-ssl.h"
+#include "master-service.h"
+#include "master-service-settings.h"
+#include "settings-parser.h"
+#include "doveadm.h"
+#include "doveadm-print.h"
+#include "doveadm-util.h"
+#include "doveadm-server.h"
+#include "doveadm-settings.h"
+#include "server-connection.h"
+
+#include <sysexits.h>
+#include <unistd.h>
+
+#define DOVEADM_LOG_CHANNEL_ID 'L'
+
+#define MAX_INBUF_SIZE (1024*32)
+
+enum server_reply_state {
+ SERVER_REPLY_STATE_DONE = 0,
+ SERVER_REPLY_STATE_PRINT,
+ SERVER_REPLY_STATE_RET
+};
+
+struct server_connection {
+ struct doveadm_server *server;
+
+ pool_t pool;
+ struct doveadm_settings *set;
+
+ int fd;
+ unsigned int minor;
+
+ struct io *io;
+ struct io *io_log;
+ struct istream *input;
+ struct istream *log_input;
+ struct ostream *output;
+ struct ssl_iostream *ssl_iostream;
+
+ struct istream *cmd_input;
+ struct ostream *cmd_output;
+ const char *delayed_cmd;
+ server_cmd_callback_t *callback;
+ void *context;
+
+ enum server_reply_state state;
+
+ bool version_received:1;
+ bool authenticate_sent:1;
+ bool authenticated:1;
+ bool streaming:1;
+ bool ssl_done:1;
+};
+
+static struct server_connection *printing_conn = NULL;
+static ARRAY(struct doveadm_server *) print_pending_servers = ARRAY_INIT;
+
+static void server_connection_input(struct server_connection *conn);
+static bool server_connection_input_one(struct server_connection *conn);
+static int server_connection_init_ssl(struct server_connection *conn,
+ const char **error_r);
+
+static void server_set_print_pending(struct doveadm_server *server)
+{
+ struct doveadm_server *pending_server;
+
+ if (!array_is_created(&print_pending_servers))
+ i_array_init(&print_pending_servers, 16);
+ array_foreach_elem(&print_pending_servers, pending_server) {
+ if (pending_server == server)
+ return;
+ }
+ array_push_back(&print_pending_servers, &server);
+}
+
+static void server_print_connection_released(struct doveadm_server *server)
+{
+ struct server_connection *const *conns;
+ unsigned int i, count;
+
+ conns = array_get(&server->connections, &count);
+ for (i = 0; i < count; i++) {
+ if (conns[i]->io != NULL)
+ continue;
+
+ conns[i]->io = io_add(conns[i]->fd, IO_READ,
+ server_connection_input, conns[i]);
+ io_set_pending(conns[i]->io);
+ }
+}
+
+static void print_connection_released(void)
+{
+ struct doveadm_server *server;
+
+ printing_conn = NULL;
+ if (!array_is_created(&print_pending_servers))
+ return;
+
+ array_foreach_elem(&print_pending_servers, server)
+ server_print_connection_released(server);
+ array_free(&print_pending_servers);
+}
+
+static int server_connection_send_cmd_input_more(struct server_connection *conn)
+{
+ enum ostream_send_istream_result res;
+ int ret = -1;
+
+ /* ostream-dot writes only up to max buffer size, so keep it non-zero */
+ o_stream_set_max_buffer_size(conn->cmd_output, IO_BLOCK_SIZE);
+ res = o_stream_send_istream(conn->cmd_output, conn->cmd_input);
+ o_stream_set_max_buffer_size(conn->cmd_output, SIZE_MAX);
+
+ switch (res) {
+ case OSTREAM_SEND_ISTREAM_RESULT_FINISHED:
+ break;
+ case OSTREAM_SEND_ISTREAM_RESULT_WAIT_INPUT:
+ return 1;
+ case OSTREAM_SEND_ISTREAM_RESULT_WAIT_OUTPUT:
+ return 0;
+ case OSTREAM_SEND_ISTREAM_RESULT_ERROR_INPUT:
+ i_error("read(%s) failed: %s",
+ i_stream_get_name(conn->cmd_input),
+ i_stream_get_error(conn->cmd_input));
+ break;
+ case OSTREAM_SEND_ISTREAM_RESULT_ERROR_OUTPUT:
+ i_error("write(%s) failed: %s",
+ o_stream_get_name(conn->cmd_output),
+ o_stream_get_error(conn->cmd_output));
+ break;
+ }
+ if (res == OSTREAM_SEND_ISTREAM_RESULT_FINISHED) {
+ if ((ret = o_stream_finish(conn->cmd_output)) == 0)
+ return 0;
+ else if (ret < 0) {
+ i_error("write(%s) failed: %s",
+ o_stream_get_name(conn->cmd_output),
+ o_stream_get_error(conn->cmd_output));
+ }
+ }
+
+ i_stream_destroy(&conn->cmd_input);
+ o_stream_destroy(&conn->cmd_output);
+ return ret;
+}
+
+static void server_connection_send_cmd_input(struct server_connection *conn)
+{
+ if (conn->cmd_input == NULL)
+ return;
+
+ conn->cmd_output = o_stream_create_dot(conn->output, TRUE);
+ (void)server_connection_send_cmd_input_more(conn);
+}
+
+static int server_connection_output(struct server_connection *conn)
+{
+ int ret;
+
+ ret = o_stream_flush(conn->output);
+ if (ret > 0 && conn->cmd_input != NULL && conn->delayed_cmd == NULL)
+ ret = server_connection_send_cmd_input_more(conn);
+ if (ret < 0)
+ server_connection_destroy(&conn);
+ return ret;
+}
+
+static void
+server_connection_callback(struct server_connection *conn,
+ int exit_code, const char *error)
+{
+ server_cmd_callback_t *callback = conn->callback;
+
+ conn->callback = NULL;
+ callback(exit_code, error, conn->context);
+}
+
+static void stream_data(string_t *str, const unsigned char *data, size_t size)
+{
+ str_truncate(str, 0);
+ str_append_tabunescaped(str, data, size);
+ doveadm_print_stream(str->data, str->used);
+}
+
+static void server_flush_field(struct server_connection *conn, string_t *str,
+ const unsigned char *data, size_t size)
+{
+ if (conn->streaming) {
+ conn->streaming = FALSE;
+ if (size > 0)
+ stream_data(str, data, size);
+ doveadm_print_stream("", 0);
+ } else {
+ str_truncate(str, 0);
+ str_append_tabunescaped(str, data, size);
+ doveadm_print(str_c(str));
+ }
+}
+
+static void
+server_handle_input(struct server_connection *conn,
+ const unsigned char *data, size_t size)
+{
+ string_t *str;
+ size_t i, start;
+
+ if (printing_conn == conn) {
+ /* continue printing */
+ } else if (printing_conn == NULL) {
+ printing_conn = conn;
+ } else {
+ /* someone else is printing. don't continue until it
+ goes away */
+ server_set_print_pending(conn->server);
+ io_remove(&conn->io);
+ return;
+ }
+
+ if (data[size-1] == '\001') {
+ /* last character is an escape */
+ size--;
+ }
+
+ str = t_str_new(128);
+ for (i = start = 0; i < size; i++) {
+ if (data[i] == '\n') {
+ if (i != start) {
+ i_error("doveadm server sent broken print input");
+ server_connection_destroy(&conn);
+ return;
+ }
+ conn->state = SERVER_REPLY_STATE_RET;
+ i_stream_skip(conn->input, i + 1);
+
+ print_connection_released();
+ return;
+ }
+ if (data[i] == '\t') {
+ server_flush_field(conn, str, data + start, i - start);
+ start = i + 1;
+ }
+ }
+ if (start != size) {
+ conn->streaming = TRUE;
+ stream_data(str, data + start, size - start);
+ }
+ i_stream_skip(conn->input, size);
+}
+
+static void server_connection_authenticated(struct server_connection *conn)
+{
+ conn->authenticated = TRUE;
+ if (conn->delayed_cmd != NULL) {
+ o_stream_nsend_str(conn->output, conn->delayed_cmd);
+ conn->delayed_cmd = NULL;
+ server_connection_send_cmd_input(conn);
+ }
+}
+
+static int
+server_connection_authenticate(struct server_connection *conn)
+{
+ string_t *plain = t_str_new(128);
+ string_t *cmd = t_str_new(128);
+
+ if (*conn->set->doveadm_password == '\0') {
+ i_error("doveadm_password not set, "
+ "can't authenticate to remote server");
+ return -1;
+ }
+
+ str_append_c(plain, '\0');
+ str_append(plain, conn->set->doveadm_username);
+ str_append_c(plain, '\0');
+ str_append(plain, conn->set->doveadm_password);
+
+ str_append(cmd, "PLAIN\t");
+ base64_encode(plain->data, plain->used, cmd);
+ str_append_c(cmd, '\n');
+
+ o_stream_nsend(conn->output, cmd->data, cmd->used);
+ conn->authenticate_sent = TRUE;
+ return 0;
+}
+
+static void server_log_disconnect_error(struct server_connection *conn)
+{
+ const char *error;
+
+ error = conn->ssl_iostream == NULL ? NULL :
+ ssl_iostream_get_last_error(conn->ssl_iostream);
+ if (error == NULL) {
+ error = conn->input->stream_errno == 0 ? "EOF" :
+ strerror(conn->input->stream_errno);
+ }
+ i_error("doveadm server disconnected before handshake: %s", error);
+}
+
+static void server_connection_print_log(struct server_connection *conn)
+{
+ const char *line;
+ struct failure_context ctx;
+ i_zero(&ctx);
+
+ while((line = i_stream_read_next_line(conn->log_input))!=NULL) {
+ /* skip empty lines */
+ if (*line == '\0') continue;
+
+ if (!doveadm_log_type_from_char(line[0], &ctx.type))
+ i_warning("Doveadm server sent invalid log type 0x%02x",
+ line[0]);
+ line++;
+ i_log_type(&ctx, "remote(%s): %s", conn->server->name, line);
+ }
+}
+
+static void server_connection_start_multiplex(struct server_connection *conn)
+{
+ struct istream *is = conn->input;
+ conn->input = i_stream_create_multiplex(is, MAX_INBUF_SIZE);
+ i_stream_unref(&is);
+ io_remove(&conn->io);
+ conn->io = io_add_istream(conn->input, server_connection_input, conn);
+ conn->log_input = i_stream_multiplex_add_channel(conn->input, DOVEADM_LOG_CHANNEL_ID);
+ conn->io_log = io_add_istream(conn->log_input, server_connection_print_log, conn);
+ i_stream_set_return_partial_line(conn->log_input, TRUE);
+}
+
+static void server_connection_input(struct server_connection *conn)
+{
+ const char *line;
+ const char *error;
+
+ if (i_stream_read(conn->input) < 0) {
+ /* disconnected */
+ server_log_disconnect_error(conn);
+ server_connection_destroy(&conn);
+ return;
+ }
+
+ while (!conn->authenticated) {
+ if ((line = i_stream_next_line(conn->input)) == NULL) {
+ if (conn->input->eof) {
+ /* we'll also get here if the line is too long */
+ server_log_disconnect_error(conn);
+ server_connection_destroy(&conn);
+ }
+ return;
+ }
+ /* Allow VERSION before or after the "+" or "-" line,
+ because v2.2.33 sent the version after and newer
+ versions send before. */
+ if (!conn->version_received &&
+ str_begins(line, "VERSION\t")) {
+ if (!version_string_verify_full(line, "doveadm-client",
+ DOVEADM_SERVER_PROTOCOL_VERSION_MAJOR,
+ &conn->minor)) {
+ i_error("doveadm server not compatible with this client"
+ "(mixed old and new binaries?)");
+ server_connection_destroy(&conn);
+ return;
+ }
+ conn->version_received = TRUE;
+ } else if (strcmp(line, "+") == 0) {
+ if (conn->minor > 0)
+ server_connection_start_multiplex(conn);
+ server_connection_authenticated(conn);
+ } else if (strcmp(line, "-") == 0) {
+ if (conn->authenticate_sent) {
+ i_error("doveadm authentication failed (%s)",
+ line+1);
+ server_connection_destroy(&conn);
+ return;
+ }
+ if (!conn->ssl_done &&
+ (conn->server->ssl_flags & PROXY_SSL_FLAG_STARTTLS) != 0) {
+ io_remove(&conn->io);
+ if (conn->minor < 2) {
+ i_error("doveadm STARTTLS failed: Server does not support it");
+ server_connection_destroy(&conn);
+ return;
+ }
+ /* send STARTTLS */
+ o_stream_nsend_str(conn->output, "STARTTLS\n");
+ if (server_connection_init_ssl(conn, &error) < 0) {
+ i_error("doveadm STARTTLS failed: %s", error);
+ server_connection_destroy(&conn);
+ return;
+ }
+ conn->ssl_done = TRUE;
+ conn->io = io_add_istream(conn->input, server_connection_input, conn);
+ }
+ if (server_connection_authenticate(conn) < 0) {
+ server_connection_destroy(&conn);
+ return;
+ }
+ } else {
+ i_error("doveadm server sent invalid handshake: %s",
+ line);
+ server_connection_destroy(&conn);
+ return;
+ }
+ }
+
+ while (server_connection_input_one(conn)) ;
+}
+
+static bool server_connection_input_one(struct server_connection *conn)
+{
+ const unsigned char *data;
+ size_t size;
+ const char *line;
+ int exit_code;
+
+ /* check logs - NOTE: must be before i_stream_get_data() since checking
+ for logs may add data to our channel. */
+ if (conn->log_input != NULL)
+ (void)server_connection_print_log(conn);
+
+ data = i_stream_get_data(conn->input, &size);
+ if (size == 0)
+ return FALSE;
+
+ switch (conn->state) {
+ case SERVER_REPLY_STATE_DONE:
+ i_error("doveadm server sent unexpected input");
+ server_connection_destroy(&conn);
+ return FALSE;
+ case SERVER_REPLY_STATE_PRINT:
+ server_handle_input(conn, data, size);
+ if (conn->state != SERVER_REPLY_STATE_RET)
+ return FALSE;
+ /* fall through */
+ case SERVER_REPLY_STATE_RET:
+ line = i_stream_next_line(conn->input);
+ if (line == NULL)
+ return FALSE;
+ if (line[0] == '+')
+ server_connection_callback(conn, 0, "");
+ else if (line[0] == '-') {
+ line++;
+ exit_code = doveadm_str_to_exit_code(line);
+ if (exit_code == DOVEADM_EX_UNKNOWN &&
+ str_to_int(line, &exit_code) < 0) {
+ /* old doveadm-server */
+ exit_code = EX_TEMPFAIL;
+ }
+ server_connection_callback(conn, exit_code, line);
+ } else {
+ i_error("doveadm server sent broken input "
+ "(expected cmd reply): %s", line);
+ server_connection_destroy(&conn);
+ return FALSE;
+ }
+ if (conn->callback == NULL) {
+ /* we're finished, close the connection */
+ server_connection_destroy(&conn);
+ return FALSE;
+ }
+ return TRUE;
+ }
+ i_unreached();
+}
+
+static int server_connection_read_settings(struct server_connection *conn,
+ const char **error_r)
+{
+ const struct setting_parser_info *set_roots[] = {
+ &doveadm_setting_parser_info,
+ NULL
+ };
+ struct master_service_settings_input input;
+ struct master_service_settings_output output;
+ const char *error;
+ in_port_t port;
+ void *set;
+
+ i_zero(&input);
+ input.roots = set_roots;
+ input.service = "doveadm";
+
+ (void)net_getsockname(conn->fd, &input.local_ip, &port);
+ (void)net_getpeername(conn->fd, &input.remote_ip, &port);
+
+ if (master_service_settings_read(master_service, &input,
+ &output, &error) < 0) {
+ *error_r = t_strdup_printf(
+ "Error reading configuration: %s", error);
+ return -1;
+ }
+ set = master_service_settings_get_others(master_service)[0];
+ conn->set = settings_dup(&doveadm_setting_parser_info, set, conn->pool);
+ return 0;
+}
+
+static int server_connection_init_ssl(struct server_connection *conn,
+ const char **error_r)
+{
+ struct ssl_iostream_settings ssl_set;
+ const char *error;
+
+ if (conn->server->ssl_flags == 0)
+ return 0;
+
+ doveadm_get_ssl_settings(&ssl_set, pool_datastack_create());
+
+ if ((conn->server->ssl_flags & PROXY_SSL_FLAG_ANY_CERT) != 0)
+ ssl_set.allow_invalid_cert = TRUE;
+ if (ssl_set.allow_invalid_cert)
+ ssl_set.verbose_invalid_cert = TRUE;
+
+ if (conn->server->ssl_ctx == NULL &&
+ ssl_iostream_client_context_cache_get(&ssl_set,
+ &conn->server->ssl_ctx,
+ &error) < 0) {
+ *error_r = t_strdup_printf(
+ "Couldn't initialize SSL client: %s", error);
+ return -1;
+ }
+
+ if (io_stream_create_ssl_client(conn->server->ssl_ctx,
+ conn->server->hostname, &ssl_set,
+ &conn->input, &conn->output,
+ &conn->ssl_iostream, &error) < 0) {
+ *error_r = t_strdup_printf(
+ "Couldn't initialize SSL client: %s", error);
+ return -1;
+ }
+ if (ssl_iostream_handshake(conn->ssl_iostream) < 0) {
+ *error_r = t_strdup_printf(
+ "SSL handshake failed: %s",
+ ssl_iostream_get_last_error(conn->ssl_iostream));
+ return -1;
+ }
+ return 0;
+}
+
+int server_connection_create(struct doveadm_server *server,
+ struct server_connection **conn_r,
+ const char **error_r)
+{
+ const char *target;
+ struct server_connection *conn;
+ pool_t pool;
+
+ pool = pool_alloconly_create("doveadm server connection", 1024*16);
+ conn = p_new(pool, struct server_connection, 1);
+ conn->pool = pool;
+ conn->server = server;
+ if (server->ip.family != 0) {
+ (void)net_ipport2str(&server->ip, server->port, &target);
+ } else {
+ target = server->name;
+ }
+ conn->fd = doveadm_connect_with_default_port(target,
+ doveadm_settings->doveadm_port);
+ net_set_nonblock(conn->fd, TRUE);
+ conn->input = i_stream_create_fd(conn->fd, MAX_INBUF_SIZE);
+ conn->output = o_stream_create_fd(conn->fd, SIZE_MAX);
+ o_stream_set_flush_callback(conn->output, server_connection_output, conn);
+ o_stream_set_no_error_handling(conn->output, TRUE);
+
+ i_stream_set_name(conn->input, server->name);
+ o_stream_set_name(conn->output, server->name);
+
+ array_push_back(&conn->server->connections, &conn);
+
+ if (server_connection_read_settings(conn, error_r) < 0 ||
+ ((server->ssl_flags & PROXY_SSL_FLAG_STARTTLS) == 0 &&
+ server_connection_init_ssl(conn, error_r) < 0)) {
+ server_connection_destroy(&conn);
+ return -1;
+ }
+ conn->io = io_add_istream(conn->input, server_connection_input, conn);
+
+ conn->state = SERVER_REPLY_STATE_DONE;
+ o_stream_nsend_str(conn->output, DOVEADM_SERVER_PROTOCOL_VERSION_LINE"\n");
+
+ *conn_r = conn;
+ return 0;
+}
+
+void server_connection_destroy(struct server_connection **_conn)
+{
+ struct server_connection *conn = *_conn;
+ struct server_connection *const *conns;
+ const char *error;
+ unsigned int i, count;
+
+ *_conn = NULL;
+
+ conns = array_get(&conn->server->connections, &count);
+ for (i = 0; i < count; i++) {
+ if (conns[i] == conn) {
+ array_delete(&conn->server->connections, i, 1);
+ break;
+ }
+ }
+
+ if (conn->callback != NULL) {
+ error = conn->ssl_iostream == NULL ? NULL :
+ ssl_iostream_get_last_error(conn->ssl_iostream);
+ if (error == NULL) {
+ error = conn->input->stream_errno == 0 ? "EOF" :
+ strerror(conn->input->stream_errno);
+ }
+ server_connection_callback(conn, SERVER_EXIT_CODE_DISCONNECTED,
+ error);
+ }
+ if (printing_conn == conn)
+ print_connection_released();
+
+ i_stream_destroy(&conn->input);
+ o_stream_destroy(&conn->output);
+ i_stream_destroy(&conn->cmd_input);
+ /* close cmd_output after its parent, so the "." isn't sent */
+ o_stream_destroy(&conn->cmd_output);
+ ssl_iostream_destroy(&conn->ssl_iostream);
+ io_remove(&conn->io_log);
+ /* make sure all logs got consumed */
+ if (conn->log_input != NULL)
+ server_connection_print_log(conn);
+ i_stream_unref(&conn->log_input);
+ io_remove(&conn->io);
+ i_close_fd(&conn->fd);
+ pool_unref(&conn->pool);
+}
+
+struct doveadm_server *
+server_connection_get_server(struct server_connection *conn)
+{
+ return conn->server;
+}
+
+void server_connection_cmd(struct server_connection *conn, const char *line,
+ struct istream *cmd_input,
+ server_cmd_callback_t *callback, void *context)
+{
+ i_assert(conn->delayed_cmd == NULL);
+
+ conn->state = SERVER_REPLY_STATE_PRINT;
+ if (cmd_input != NULL) {
+ i_assert(conn->cmd_input == NULL);
+ i_stream_ref(cmd_input);
+ conn->cmd_input = cmd_input;
+ }
+ if (!conn->authenticated)
+ conn->delayed_cmd = p_strdup(conn->pool, line);
+ else {
+ o_stream_nsend_str(conn->output, line);
+ server_connection_send_cmd_input(conn);
+ }
+ conn->callback = callback;
+ conn->context = context;
+}
+
+bool server_connection_is_idle(struct server_connection *conn)
+{
+ return conn->callback == NULL;
+}
+
+void server_connection_extract(struct server_connection *conn,
+ struct istream **istream_r,
+ struct ostream **ostream_r,
+ struct ssl_iostream **ssl_iostream_r)
+{
+ *istream_r = conn->input;
+ *ostream_r = conn->output;
+ *ssl_iostream_r = conn->ssl_iostream;
+
+ conn->input = NULL;
+ conn->output = NULL;
+ conn->ssl_iostream = NULL;
+ io_remove(&conn->io);
+ conn->fd = -1;
+}
diff --git a/src/doveadm/server-connection.h b/src/doveadm/server-connection.h
new file mode 100644
index 0000000..1db30b9
--- /dev/null
+++ b/src/doveadm/server-connection.h
@@ -0,0 +1,35 @@
+#ifndef SERVER_CONNECTION_H
+#define SERVER_CONNECTION_H
+
+#define SERVER_EXIT_CODE_DISCONNECTED 1000
+
+struct doveadm_server;
+struct server_connection;
+struct ssl_iostream;
+
+typedef void server_cmd_callback_t(int exit_code, const char *error,
+ void *context);
+
+int server_connection_create(struct doveadm_server *server,
+ struct server_connection **conn_r,
+ const char **error_r);
+void server_connection_destroy(struct server_connection **conn);
+
+/* Return the server given to create() */
+struct doveadm_server *
+server_connection_get_server(struct server_connection *conn);
+
+void server_connection_cmd(struct server_connection *conn, const char *line,
+ struct istream *cmd_input,
+ server_cmd_callback_t *callback, void *context);
+/* Returns TRUE if no command is being processed */
+bool server_connection_is_idle(struct server_connection *conn);
+
+/* Extract iostreams from connection. Afterwards the server_connection simply
+ waits for itself to be destroyed. */
+void server_connection_extract(struct server_connection *conn,
+ struct istream **istream_r,
+ struct ostream **ostream_r,
+ struct ssl_iostream **ssl_iostream_r);
+
+#endif
diff --git a/src/doveadm/test-doveadm-util.c b/src/doveadm/test-doveadm-util.c
new file mode 100644
index 0000000..aec1b39
--- /dev/null
+++ b/src/doveadm/test-doveadm-util.c
@@ -0,0 +1,48 @@
+/* Copyright (c) 2016-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+#include "test-common.h"
+#include "doveadm-settings.h"
+#include "doveadm-util.h"
+
+struct doveadm_settings *doveadm_settings; /* just to avoid linker error */
+
+static void test_i_strccdascmp(void)
+{
+ test_begin("i_strccdascmp()");
+
+ test_assert(i_strccdascmp("", "")==0);
+ test_assert(i_strccdascmp("", "-")!=0);
+ test_assert(i_strccdascmp("-", "")!=0);
+ test_assert(i_strccdascmp("-", "-")==0);
+ test_assert(i_strccdascmp("-\0baz", "-\0bar")==0);
+ test_assert(i_strccdascmp("", "a")!=0);
+ test_assert(i_strccdascmp("a", "")!=0);
+ test_assert(i_strccdascmp("a", "a")==0);
+ test_assert(i_strccdascmp("a-", "a-")==0);
+ test_assert(i_strccdascmp("a-a", "a-a")==0);
+ test_assert(i_strccdascmp("ca", "ba")!=0);
+
+ test_assert(i_strccdascmp("camel case", "camel case")==0);
+ test_assert(i_strccdascmp("camel case", "camel-case")==0);
+ test_assert(i_strccdascmp("camel case", "camelCase")==0);
+
+ test_assert(i_strccdascmp("camel case", "camel-case")==0);
+ test_assert(i_strccdascmp("camel-case", "camel-case")==0);
+ test_assert(i_strccdascmp("camelCase", "camel-case")==0);
+
+ test_assert(i_strccdascmp("camel case", "camel Case")==-i_strccdascmp("camel Case", "camel case"));
+ test_assert(i_strccdascmp("camel-case", "camel Case")==-i_strccdascmp("camel Case", "camel-case"));
+ test_assert(i_strccdascmp("camel dase", "camel case")==-i_strccdascmp("camel case", "camel dase"));
+
+ test_end();
+}
+
+int main(void)
+{
+ static void (*const test_functions[])(void) = {
+ test_i_strccdascmp,
+ NULL
+ };
+ return test_run(test_functions);
+}